github/nzbToMedia
1
0
Fork 0
mirror of https://github.com/clinton-hall/nzbToMedia.git synced 2025-08-19 12:59:36 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'nightly' into patch-1

Browse source 
# Conflicts:
#	core/__init__.py
This commit is contained in:
BBsan2k 2016-10-22 10:14:05 +02:00
commit a3824f27bb
512 changed files with 86505 additions and 22341 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -5,6 +5,7 @@
*.log
*.pid
*.db
*.dbm
/userscripts/
/logs/
/.idea/

239
TorrentToMedia.py
View file

@ -1,28 +1,26 @@
#!/usr/bin/env python2
# coding=utf-8
import datetime
import os
import time
import shutil
import sys
import core
from subprocess import Popen
from libs.six import text_type
from core import logger, nzbToMediaDB
from core.nzbToMediaUtil import convert_to_ascii, CharReplace, plex_update
from core.nzbToMediaUtil import convert_to_ascii, CharReplace, plex_update, replace_links
from core.nzbToMediaUserScript import external_script
def processTorrent(inputDirectory, inputName, inputCategory, inputHash, inputID, clientAgent):
status = 1 # 1 = failed | 0 = success
root = 0
foundFile = 0
uniquePath = 1
if clientAgent != 'manual' and not core.DOWNLOADINFO:
logger.debug('Adding TORRENT download info for directory %s to database' % (inputDirectory))
logger.debug('Adding TORRENT download info for directory {0} to database'.format(inputDirectory))
myDB = nzbToMediaDB.DBConnection()
encoded = False
inputDirectory1 = inputDirectory
inputName1 = inputName
@ -32,89 +30,82 @@ def processTorrent(inputDirectory, inputName, inputCategory, inputHash, inputID,
except:
pass
controlValueDict = {"input_directory": unicode(inputDirectory1)}
newValueDict = {"input_name": unicode(inputName1),
"input_hash": unicode(inputHash),
"input_id": unicode(inputID),
"client_agent": unicode(clientAgent),
controlValueDict = {"input_directory": text_type(inputDirectory1)}
newValueDict = {"input_name": text_type(inputName1),
"input_hash": text_type(inputHash),
"input_id": text_type(inputID),
"client_agent": text_type(clientAgent),
"status": 0,
"last_update": datetime.date.today().toordinal()
}
myDB.upsert("downloads", newValueDict, controlValueDict)
logger.debug("Received Directory: %s | Name: %s | Category: %s" % (inputDirectory, inputName, inputCategory))
logger.debug("Received Directory: {0} | Name: {1} | Category: {2}".format(inputDirectory, inputName, inputCategory))
inputDirectory, inputName, inputCategory, root = core.category_search(inputDirectory, inputName,
inputCategory, root,
core.CATEGORIES) # Confirm the category by parsing directory structure
# Confirm the category by parsing directory structure
inputDirectory, inputName, inputCategory, root = core.category_search(inputDirectory, inputName, inputCategory,
root, core.CATEGORIES)
if inputCategory == "":
inputCategory = "UNCAT"
usercat = inputCategory
try:
inputName = inputName.encode(core.SYS_ENCODING)
except: pass
except UnicodeError:
pass
try:
inputDirectory = inputDirectory.encode(core.SYS_ENCODING)
except: pass
except UnicodeError:
pass
logger.debug("Determined Directory: %s | Name: %s | Category: %s" % (inputDirectory, inputName, inputCategory))
logger.debug("Determined Directory: {0} | Name: {1} | Category: {2}".format
(inputDirectory, inputName, inputCategory))
# auto-detect section
section = core.CFG.findsection(inputCategory).isenabled()
if section is None:
section = core.CFG.findsection("ALL").isenabled()
if section is None:
logger.error(
'Category:[%s] is not defined or is not enabled. Please rename it or ensure it is enabled for the appropriate section in your autoProcessMedia.cfg and try again.' % (
inputCategory))
logger.error('Category:[{0}] is not defined or is not enabled. '
'Please rename it or ensure it is enabled for the appropriate section '
'in your autoProcessMedia.cfg and try again.'.format
(inputCategory))
return [-1, ""]
else:
usercat = "ALL"
if len(section) > 1:
logger.error(
'Category:[%s] is not unique, %s are using it. Please rename it or disable all other sections using the same category name in your autoProcessMedia.cfg and try again.' % (
usercat, section.keys()))
logger.error('Category:[{0}] is not unique, {1} are using it. '
'Please rename it or disable all other sections using the same category name '
'in your autoProcessMedia.cfg and try again.'.format
(usercat, section.keys()))
return [-1, ""]
if section:
sectionName = section.keys()[0]
logger.info('Auto-detected SECTION:%s' % (sectionName))
logger.info('Auto-detected SECTION:{0}'.format(sectionName))
else:
logger.error("Unable to locate a section with subsection:%s enabled in your autoProcessMedia.cfg, exiting!" % (
inputCategory))
logger.error("Unable to locate a section with subsection:{0} "
"enabled in your autoProcessMedia.cfg, exiting!".format
(inputCategory))
return [-1, ""]
try:
Torrent_NoLink = int(section[usercat]["Torrent_NoLink"])
except:
Torrent_NoLink = 0
section = dict(section[sectionName][usercat]) # Type cast to dict() to allow effective usage of .get()
try:
keep_archive = int(section[usercat]["keep_archive"])
except:
keep_archive = 0
try:
extract = int(section[usercat]['extract'])
except:
extract = 0
try:
uniquePath = int(section[usercat]["unique_path"])
except:
uniquePath = 1
Torrent_NoLink = int(section.get("Torrent_NoLink", 0))
keep_archive = int(section.get("keep_archive", 0))
extract = int(section.get('extract', 0))
uniquePath = int(section.get("unique_path", 1))
if clientAgent != 'manual':
core.pause_torrent(clientAgent, inputHash, inputID, inputName)
# Incase input is not directory, make sure to create one.
# In case input is not directory, make sure to create one.
# This way Processing is isolated.
if not os.path.isdir(os.path.join(inputDirectory, inputName)):
basename = os.path.basename(inputDirectory)
basename = core.sanitizeName(inputName) \
if inputName == basename else os.path.splitext(core.sanitizeName(inputName)[0])
if inputName == basename else os.path.splitext(core.sanitizeName(inputName))[0]
outputDestination = os.path.join(core.OUTPUTDIRECTORY, inputCategory, basename)
elif uniquePath:
outputDestination = os.path.normpath(
@ -124,20 +115,21 @@ def processTorrent(inputDirectory, inputName, inputCategory, inputHash, inputID,
core.os.path.join(core.OUTPUTDIRECTORY, inputCategory))
try:
outputDestination = outputDestination.encode(core.SYS_ENCODING)
except: pass
except UnicodeError:
pass
if outputDestination in inputDirectory:
outputDestination = inputDirectory
logger.info("Output directory set to: %s" % (outputDestination))
logger.info("Output directory set to: {0}".format(outputDestination))
if core.SAFE_MODE and outputDestination == core.TORRENT_DEFAULTDIR:
logger.error(
'The output directory:[%s] is the Download Directory. Edit outputDirectory in autoProcessMedia.cfg. Exiting' % (
inputDirectory))
logger.error('The output directory:[{0}] is the Download Directory. '
'Edit outputDirectory in autoProcessMedia.cfg. Exiting'.format
(inputDirectory))
return [-1, ""]
logger.debug("Scanning files in directory: %s" % (inputDirectory))
logger.debug("Scanning files in directory: {0}".format(inputDirectory))
if sectionName == 'HeadPhones':
core.NOFLATTEN.extend(
@ -149,7 +141,7 @@ def processTorrent(inputDirectory, inputName, inputCategory, inputHash, inputID,
inputFiles = core.listMediaFiles(inputDirectory, archives=False)
else:
inputFiles = core.listMediaFiles(inputDirectory)
logger.debug("Found %s files in %s" % (str(len(inputFiles)), inputDirectory))
logger.debug("Found {0} files in {1}".format(len(inputFiles), inputDirectory))
for inputFile in inputFiles:
filePath = os.path.dirname(inputFile)
fileName, fileExt = os.path.splitext(os.path.basename(inputFile))
@ -160,18 +152,20 @@ def processTorrent(inputDirectory, inputName, inputCategory, inputHash, inputID,
if not os.path.basename(filePath) in outputDestination:
targetFile = core.os.path.join(
core.os.path.join(outputDestination, os.path.basename(filePath)), fullFileName)
logger.debug(
"Setting outputDestination to %s to preserve folder structure" % (os.path.dirname(targetFile)))
logger.debug("Setting outputDestination to {0} to preserve folder structure".format
(os.path.dirname(targetFile)))
try:
targetFile = targetFile.encode(core.SYS_ENCODING)
except: pass
except UnicodeError:
pass
if root == 1:
if not foundFile:
logger.debug("Looking for %s in: %s" % (inputName, inputFile))
if (core.sanitizeName(inputName) in core.sanitizeName(inputFile)) or (
core.sanitizeName(fileName) in core.sanitizeName(inputName)):
logger.debug("Looking for {0} in: {1}".format(inputName, inputFile))
if any([core.sanitizeName(inputName) in core.sanitizeName(inputFile),
core.sanitizeName(fileName) in core.sanitizeName(inputName)]):
foundFile = True
logger.debug("Found file %s that matches Torrent Name %s" % (fullFileName, inputName))
logger.debug("Found file {0} that matches Torrent Name {1}".format
(fullFileName, inputName))
else:
continue
@ -183,7 +177,8 @@ def processTorrent(inputDirectory, inputName, inputCategory, inputHash, inputID,
logger.debug("Looking for files with modified/created dates less than 5 minutes old.")
if (mtime_lapse < datetime.timedelta(minutes=5)) or (ctime_lapse < datetime.timedelta(minutes=5)):
foundFile = True
logger.debug("Found file %s with date modifed/created less than 5 minutes ago." % (fullFileName))
logger.debug("Found file {0} with date modified/created less than 5 minutes ago.".format
(fullFileName))
else:
continue # This file has not been recently moved or created, skip it
@ -192,15 +187,16 @@ def processTorrent(inputDirectory, inputName, inputCategory, inputHash, inputID,
core.copy_link(inputFile, targetFile, core.USELINK)
core.rmReadOnly(targetFile)
except:
logger.error("Failed to link: %s to %s" % (inputFile, targetFile))
logger.error("Failed to link: {0} to {1}".format(inputFile, targetFile))
inputName, outputDestination = convert_to_ascii(inputName, outputDestination)
if extract == 1:
logger.debug('Checking for archives to extract in directory: %s' % (inputDirectory))
logger.debug('Checking for archives to extract in directory: {0}'.format(inputDirectory))
core.extractFiles(inputDirectory, outputDestination, keep_archive)
if not inputCategory in core.NOFLATTEN: #don't flatten hp in case multi cd albums, and we need to copy this back later.
if inputCategory not in core.NOFLATTEN:
# don't flatten hp in case multi cd albums, and we need to copy this back later.
core.flatten(outputDestination)
# Now check if video files exist in destination:
@ -208,51 +204,55 @@ def processTorrent(inputDirectory, inputName, inputCategory, inputHash, inputID,
numVideos = len(
core.listMediaFiles(outputDestination, media=True, audio=False, meta=False, archives=False))
if numVideos > 0:
logger.info("Found %s media files in %s" % (numVideos, outputDestination))
logger.info("Found {0} media files in {1}".format(numVideos, outputDestination))
status = 0
elif extract != 1:
logger.info("Found no media files in %s. Sending to %s to process" % (outputDestination, sectionName))
logger.info("Found no media files in {0}. Sending to {1} to process".format(outputDestination, sectionName))
status = 0
else:
logger.warning("Found no media files in %s" % outputDestination)
logger.warning("Found no media files in {0}".format(outputDestination))
# Only these sections can handling failed downloads so make sure everything else gets through without the check for failed
if not sectionName in ['CouchPotato', 'SickBeard', 'NzbDrone']:
# Only these sections can handling failed downloads
# so make sure everything else gets through without the check for failed
if sectionName not in ['CouchPotato', 'SickBeard', 'NzbDrone']:
status = 0
logger.info("Calling %s:%s to post-process:%s" % (sectionName, usercat, inputName))
logger.info("Calling {0}:{1} to post-process:{2}".format(sectionName, usercat, inputName))
if core.TORRENT_CHMOD_DIRECTORY:
core.rchmod(outputDestination, core.TORRENT_CHMOD_DIRECTORY)
result = [ 0, "" ]
result = [0, ""]
if sectionName == 'UserScript':
result = external_script(outputDestination, inputName, inputCategory, section[usercat])
result = external_script(outputDestination, inputName, inputCategory, section)
elif sectionName == 'CouchPotato':
result = core.autoProcessMovie().process(sectionName,outputDestination, inputName, status, clientAgent, inputHash,
inputCategory)
elif sectionName in ['SickBeard','NzbDrone']:
result = core.autoProcessMovie().process(sectionName, outputDestination, inputName,
status, clientAgent, inputHash, inputCategory)
elif sectionName in ['SickBeard', 'NzbDrone']:
if inputHash:
inputHash = inputHash.upper()
result = core.autoProcessTV().processEpisode(sectionName,outputDestination, inputName, status, clientAgent,
inputHash, inputCategory)
result = core.autoProcessTV().processEpisode(sectionName, outputDestination, inputName,
status, clientAgent, inputHash, inputCategory)
elif sectionName == 'HeadPhones':
result = core.autoProcessMusic().process(sectionName,outputDestination, inputName, status, clientAgent, inputCategory)
result = core.autoProcessMusic().process(sectionName, outputDestination, inputName,
status, clientAgent, inputCategory)
elif sectionName == 'Mylar':
result = core.autoProcessComics().processEpisode(sectionName,outputDestination, inputName, status, clientAgent,
inputCategory)
result = core.autoProcessComics().processEpisode(sectionName, outputDestination, inputName,
status, clientAgent, inputCategory)
elif sectionName == 'Gamez':
result = core.autoProcessGames().process(sectionName,outputDestination, inputName, status, clientAgent, inputCategory)
result = core.autoProcessGames().process(sectionName, outputDestination, inputName,
status, clientAgent, inputCategory)
plex_update(inputCategory)
if result[0] != 0:
if not core.TORRENT_RESUME_ON_FAILURE:
logger.error("A problem was reported in the autoProcess* script. torrent won't resume seeding (settings)")
logger.error("A problem was reported in the autoProcess* script. "
"Torrent won't resume seeding (settings)")
elif clientAgent != 'manual':
logger.error(
"A problem was reported in the autoProcess* script. If torrent was paused we will resume seeding")
logger.error("A problem was reported in the autoProcess* script. "
"If torrent was paused we will resume seeding")
core.resume_torrent(clientAgent, inputHash, inputID, inputName)
else:
@ -262,14 +262,15 @@ def processTorrent(inputDirectory, inputName, inputCategory, inputHash, inputID,
# remove torrent
if core.USELINK == 'move-sym' and not core.DELETE_ORIGINAL == 1:
logger.debug('Checking for sym-links to re-direct in: %s' % (inputDirectory))
logger.debug('Checking for sym-links to re-direct in: {0}'.format(inputDirectory))
for dirpath, dirs, files in os.walk(inputDirectory):
for file in files:
logger.debug('Checking symlink: %s' % (os.path.join(dirpath,file)))
core.replace_links(os.path.join(dirpath,file))
logger.debug('Checking symlink: {0}'.format(os.path.join(dirpath, file)))
replace_links(os.path.join(dirpath, file))
core.remove_torrent(clientAgent, inputHash, inputID, inputName)
if not sectionName == 'UserScript': # for user script, we assume this is cleaned by the script or option USER_SCRIPT_CLEAN
if not sectionName == 'UserScript':
# for user script, we assume this is cleaned by the script or option USER_SCRIPT_CLEAN
# cleanup our processing folders of any misc unwanted files and empty directories
core.cleanDir(outputDestination, sectionName, inputCategory)
@ -284,14 +285,14 @@ def main(args):
clientAgent = core.TORRENT_CLIENTAGENT
logger.info("#########################################################")
logger.info("## ..::[%s]::.. ##" % os.path.basename(__file__))
logger.info("## ..::[{0}]::.. ##".format(os.path.basename(__file__)))
logger.info("#########################################################")
# debug command line options
logger.debug("Options passed into TorrentToMedia: %s" % (args))
logger.debug("Options passed into TorrentToMedia: {0}".format(args))
# Post-Processing Result
result = [ 0, "" ]
result = [0, ""]
try:
inputDirectory, inputName, inputCategory, inputHash, inputID = core.parse_args(clientAgent, args)
@ -310,54 +311,50 @@ def main(args):
if not core.CFG[section][subsection].isenabled():
continue
for dirName in core.getDirs(section, subsection, link='hard'):
logger.info("Starting manual run for %s:%s - Folder:%s" % (section, subsection, dirName))
logger.info("Starting manual run for {0}:{1} - Folder:{2}".format
(section, subsection, dirName))
logger.info("Checking database for download info for %s ..." % (os.path.basename(dirName)))
logger.info("Checking database for download info for {0} ...".format
(os.path.basename(dirName)))
core.DOWNLOADINFO = core.get_downloadInfo(os.path.basename(dirName), 0)
if core.DOWNLOADINFO:
logger.info(
"Found download info for %s, setting variables now ..." % (os.path.basename(dirName)))
clientAgent = text_type(core.DOWNLOADINFO[0].get('client_agent', 'manual'))
inputHash = text_type(core.DOWNLOADINFO[0].get('input_hash', ''))
inputID = text_type(core.DOWNLOADINFO[0].get('input_id', ''))
logger.info("Found download info for {0}, "
"setting variables now ...".format(os.path.basename(dirName)))
else:
logger.info(
'Unable to locate download info for %s, continuing to try and process this release ...' % (
os.path.basename(dirName))
)
try:
clientAgent = str(core.DOWNLOADINFO[0]['client_agent'])
except:
logger.info('Unable to locate download info for {0}, '
'continuing to try and process this release ...'.format
(os.path.basename(dirName)))
clientAgent = 'manual'
try:
inputHash = str(core.DOWNLOADINFO[0]['input_hash'])
except:
inputHash = None
try:
inputID = str(core.DOWNLOADINFO[0]['input_id'])
except:
inputID = None
inputHash = ''
inputID = ''
if clientAgent.lower() not in core.TORRENT_CLIENTS and clientAgent != 'manual':
if clientAgent.lower() not in core.TORRENT_CLIENTS:
continue
try:
dirName = dirName.encode(core.SYS_ENCODING)
except: pass
except UnicodeError:
pass
inputName = os.path.basename(dirName)
try:
inputName = inputName.encode(core.SYS_ENCODING)
except: pass
except UnicodeError:
pass
results = processTorrent(dirName, inputName, subsection, inputHash, inputID,
results = processTorrent(dirName, inputName, subsection, inputHash or None, inputID or None,
clientAgent)
if results[0] != 0:
logger.error("A problem was reported when trying to perform a manual run for %s:%s." % (
section, subsection))
logger.error("A problem was reported when trying to perform a manual run for {0}:{1}.".format
(section, subsection))
result = results
if result[0] == 0:
logger.info("The %s script completed successfully." % (args[0]))
logger.info("The {0} script completed successfully.".format(args[0]))
else:
logger.error("A problem was reported in the %s script." % (args[0]))
logger.error("A problem was reported in the {0} script.".format(args[0]))
del core.MYAPP
return result[0]

8
autoProcessMedia.cfg.spec
View file

@ -64,6 +64,8 @@
remote_path = 0
##### Set to path where download client places completed downloads locally for this category
watch_dir =
##### Set the recursive directory permissions to the following (0 to disable)
chmodDirectory = 0
[SickBeard]
#### autoProcessing for TV Series
@ -85,8 +87,10 @@
process_method =
# force processing of already processed content when running a manual scan.
force = 0
# tell SickRage to delete all source files after processing.
# tell SickRage/Medusa to delete all source files after processing.
delete_on = 0
# tell Medusa to ignore check for associated subtitle check when postponing release
ignore_subs = 0
extract = 1
nzbExtractionBy = Downloader
# Set this to minimum required size to consider a media file valid (in MB)
@ -97,6 +101,8 @@
remote_path = 0
##### Set to path where download client places completed downloads locally for this category
watch_dir =
##### Set the recursive directory permissions to the following (0 to disable)
chmodDirectory = 0
[NzbDrone]
#### autoProcessing for TV Series

275
core/__init__.py
View file

@ -1,3 +1,7 @@
# coding=utf-8
from __future__ import print_function
import locale
import os
import re
@ -6,6 +10,7 @@ import sys
import platform
import time
# init libs
PROGRAM_DIR = os.path.dirname(os.path.normpath(os.path.abspath(os.path.join(__file__, os.pardir))))
LIBS_DIR = os.path.join(PROGRAM_DIR, 'libs')
@ -25,6 +30,8 @@ CONFIG_TV_FILE = os.path.join(PROGRAM_DIR, 'autoProcessTv.cfg')
TEST_FILE = os.path.join(os.path.join(PROGRAM_DIR, 'tests'), 'test.mp4')
MYAPP = None
from six.moves import reload_module
from core.autoProcess.autoProcessComics import autoProcessComics
from core.autoProcess.autoProcessGames import autoProcessGames
from core.autoProcess.autoProcessMovie import autoProcessMovie
@ -32,16 +39,18 @@ from core.autoProcess.autoProcessMusic import autoProcessMusic
from core.autoProcess.autoProcessTV import autoProcessTV
from core import logger, versionCheck, nzbToMediaDB
from core.nzbToMediaConfig import config
from core.nzbToMediaUtil import category_search, sanitizeName, copy_link, parse_args, flatten, getDirs, \
rmReadOnly,rmDir, pause_torrent, resume_torrent, remove_torrent, listMediaFiles, \
extractFiles, cleanDir, update_downloadInfoStatus, get_downloadInfo, WakeUp, makeDir, cleanDir, \
create_torrent_class, listMediaFiles, RunningProcess
from core.nzbToMediaUtil import (
category_search, sanitizeName, copy_link, parse_args, flatten, getDirs,
rmReadOnly, rmDir, pause_torrent, resume_torrent, remove_torrent, listMediaFiles,
extractFiles, cleanDir, update_downloadInfoStatus, get_downloadInfo, WakeUp, makeDir, cleanDir,
create_torrent_class, listMediaFiles, RunningProcess,
)
from core.transcoder import transcoder
from core.databases import mainDB
# Client Agents
NZB_CLIENTS = ['sabnzbd','nzbget']
TORRENT_CLIENTS = ['transmission', 'deluge', 'utorrent', 'rtorrent', 'other']
NZB_CLIENTS = ['sabnzbd', 'nzbget', 'manual']
TORRENT_CLIENTS = ['transmission', 'deluge', 'utorrent', 'rtorrent', 'other', 'manual']
# sabnzbd constants
SABNZB_NO_OF_ARGUMENTS = 8
@ -54,14 +63,17 @@ FORK_FAILED = "failed"
FORK_FAILED_TORRENT = "failed-torrent"
FORK_SICKRAGETV = "sickragetv"
FORK_SICKRAGE = "sickrage"
FORK_MEDUSA = "medusa"
FORK_SICKGEAR = "sickgear"
FORKS[FORK_DEFAULT] = {"dir": None}
FORKS[FORK_FAILED] = {"dirName": None, "failed": None}
FORKS[FORK_FAILED_TORRENT] = {"dir": None, "failed": None, "process_method": None}
FORKS[FORK_SICKRAGETV] = {"proc_dir": None, "failed": None, "process_method": None, "force": None, "delete_on": None}
FORKS[FORK_SICKRAGE] = {"proc_dir": None, "failed": None, "process_method": None, "force": None, "delete_on": None}
FORKS[FORK_MEDUSA] = {"proc_dir": None, "failed": None, "process_method": None, "force": None, "delete_on": None, "ignore_subs":None}
FORKS[FORK_SICKGEAR] = {"dir": None, "failed": None, "process_method": None, "force": None}
ALL_FORKS = {"dir": None, "dirName": None, "proc_dir": None, "failed": None, "process_method": None, "force": None, "delete_on": None}
ALL_FORKS = {"dir": None, "dirName": None, "proc_dir": None, "failed": None, "process_method": None, "force": None,
"delete_on": None, "ignore_subs": None}
# NZBGet Exit Codes
NZBGET_POSTPROCESS_PARCHECK = 92
@ -201,6 +213,7 @@ USER_SCRIPT_RUNONCE = None
__INITIALIZED__ = False
def initialize(section=None):
global NZBGET_POSTPROCESS_ERROR, NZBGET_POSTPROCESS_NONE, NZBGET_POSTPROCESS_PARCHECK, NZBGET_POSTPROCESS_SUCCESS, \
NZBTOMEDIA_TIMEOUT, FORKS, FORK_DEFAULT, FORK_FAILED_TORRENT, FORK_FAILED, \
@ -224,7 +237,7 @@ def initialize(section=None):
if __INITIALIZED__:
return False
if os.environ.has_key('NTM_LOGFILE'):
if 'NTM_LOGFILE' in os.environ:
LOG_FILE = os.environ['NTM_LOGFILE']
LOG_DIR = os.path.split(LOG_FILE)[0]
@ -247,16 +260,17 @@ def initialize(section=None):
SYS_ENCODING = 'UTF-8'
if not hasattr(sys, "setdefaultencoding"):
reload(sys)
reload_module(sys)
try:
# pylint: disable=E1101
# On non-unicode builds this will raise an AttributeError, if encoding type is not valid it throws a LookupError
sys.setdefaultencoding(SYS_ENCODING)
except:
print 'Sorry, you MUST add the nzbToMedia folder to the PYTHONPATH environment variable'
print 'or find another way to force Python to use ' + SYS_ENCODING + ' for string encoding.'
if os.environ.has_key('NZBOP_SCRIPTDIR'):
print('Sorry, you MUST add the nzbToMedia folder to the PYTHONPATH environment variable'
'\nor find another way to force Python to use {codec} for string encoding.'.format
(codec=SYS_ENCODING))
if 'NZBOP_SCRIPTDIR' in os.environ:
sys.exit(NZBGET_POSTPROCESS_ERROR)
else:
sys.exit(1)
@ -266,18 +280,18 @@ def initialize(section=None):
# run migrate to convert old cfg to new style cfg plus fix any cfg missing values/options.
if not config.migrate():
logger.error("Unable to migrate config file %s, exiting ..." % (CONFIG_FILE))
if os.environ.has_key('NZBOP_SCRIPTDIR'):
logger.error("Unable to migrate config file {0}, exiting ...".format(CONFIG_FILE))
if 'NZBOP_SCRIPTDIR' in os.environ:
pass # We will try and read config from Environment.
else:
sys.exit(-1)
# run migrate to convert NzbGet data from old cfg style to new cfg style
if os.environ.has_key('NZBOP_SCRIPTDIR'):
if 'NZBOP_SCRIPTDIR' in os.environ:
CFG = config.addnzbget()
else: # load newly migrated config
logger.info("Loading config from [%s]" % (CONFIG_FILE))
logger.info("Loading config from [{0}]".format(CONFIG_FILE))
CFG = config()
# Enable/Disable DEBUG Logging
@ -288,7 +302,7 @@ def initialize(section=None):
if LOG_ENV:
for item in os.environ:
logger.info("%s: %s" % (item, os.environ[item]), "ENVIRONMENT")
logger.info("{0}: {1}".format(item, os.environ[item]), "ENVIRONMENT")
# initialize the main SB database
nzbToMediaDB.upgradeDatabase(nzbToMediaDB.DBConnection(), mainDB.InitialSchema)
@ -315,14 +329,16 @@ def initialize(section=None):
# restart nzbToMedia
try:
del MYAPP
except: pass
except:
pass
restart()
else:
logger.error("Update wasn't successful, not restarting. Check your log for more information.")
# Set Current Version
logger.info(
'nzbToMedia Version:' + NZBTOMEDIA_VERSION + ' Branch:' + GIT_BRANCH + ' (' + platform.system() + ' ' + platform.release() + ')')
logger.info('nzbToMedia Version:{version} Branch:{branch} ({system} {release})'.format
(version=NZBTOMEDIA_VERSION, branch=GIT_BRANCH,
system=platform.system(), release=platform.release()))
if int(CFG["WakeOnLan"]["wake"]) == 1:
WakeUp()
@ -333,8 +349,10 @@ def initialize(section=None):
SABNZBDAPIKEY = CFG["Nzb"]["sabnzbd_apikey"]
NZB_DEFAULTDIR = CFG["Nzb"]["default_downloadDirectory"]
GROUPS = CFG["Custom"]["remove_group"]
if isinstance(GROUPS, str): GROUPS = GROUPS.split(',')
if GROUPS == ['']: GROUPS = None
if isinstance(GROUPS, str):
GROUPS = GROUPS.split(',')
if GROUPS == ['']:
GROUPS = None
TORRENT_CLIENTAGENT = CFG["Torrent"]["clientAgent"] # utorrent | deluge | transmission | rtorrent | vuze |other
USELINK = CFG["Torrent"]["useLink"] # no | hard | sym
@ -342,8 +360,10 @@ def initialize(section=None):
TORRENT_DEFAULTDIR = CFG["Torrent"]["default_downloadDirectory"]
CATEGORIES = (CFG["Torrent"]["categories"]) # music,music_videos,pictures,software
NOFLATTEN = (CFG["Torrent"]["noFlatten"])
if isinstance(NOFLATTEN, str): NOFLATTEN = NOFLATTEN.split(',')
if isinstance(CATEGORIES, str): CATEGORIES = CATEGORIES.split(',')
if isinstance(NOFLATTEN, str):
NOFLATTEN = NOFLATTEN.split(',')
if isinstance(CATEGORIES, str):
CATEGORIES = CATEGORIES.split(',')
DELETE_ORIGINAL = int(CFG["Torrent"]["deleteOriginal"])
TORRENT_CHMOD_DIRECTORY = int(str(CFG["Torrent"]["chmodDirectory"]), 8)
TORRENT_RESUME_ON_FAILURE = int(CFG["Torrent"]["resumeOnFailure"])
@ -364,9 +384,12 @@ def initialize(section=None):
REMOTEPATHS = CFG["Network"]["mount_points"] or []
if REMOTEPATHS:
if isinstance(REMOTEPATHS, list): REMOTEPATHS = ','.join(REMOTEPATHS) # fix in case this imported as list.
REMOTEPATHS = [ tuple(item.split(',')) for item in REMOTEPATHS.split('|') ] # /volume1/Public/,E:\|/volume2/share/,\\NAS\
REMOTEPATHS = [ (local.strip(), remote.strip()) for local, remote in REMOTEPATHS ] # strip trailing and leading whitespaces
if isinstance(REMOTEPATHS, list):
REMOTEPATHS = ','.join(REMOTEPATHS) # fix in case this imported as list.
REMOTEPATHS = [tuple(item.split(',')) for item in
REMOTEPATHS.split('|')] # /volume1/Public/,E:\|/volume2/share/,\\NAS\
REMOTEPATHS = [(local.strip(), remote.strip()) for local, remote in
REMOTEPATHS] # strip trailing and leading whitespaces
PLEXSSL = int(CFG["Plex"]["plex_ssl"])
PLEXHOST = CFG["Plex"]["plex_host"]
@ -374,62 +397,79 @@ def initialize(section=None):
PLEXTOKEN = CFG["Plex"]["plex_token"]
PLEXSEC = CFG["Plex"]["plex_sections"] or []
if PLEXSEC:
if isinstance(PLEXSEC, list): PLEXSEC = ','.join(PLEXSEC) # fix in case this imported as list.
PLEXSEC = [ tuple(item.split(',')) for item in PLEXSEC.split('|') ]
if isinstance(PLEXSEC, list):
PLEXSEC = ','.join(PLEXSEC) # fix in case this imported as list.
PLEXSEC = [tuple(item.split(',')) for item in PLEXSEC.split('|')]
devnull = open(os.devnull, 'w')
try:
subprocess.Popen(["nice"], stdout=devnull, stderr=devnull).communicate()
NICENESS.extend(['nice', '-n%s' % (int(CFG["Posix"]["niceness"]))])
except: pass
NICENESS.extend(['nice', '-n{0}'.format(int(CFG["Posix"]["niceness"]))])
except:
pass
try:
subprocess.Popen(["ionice"], stdout=devnull, stderr=devnull).communicate()
try:
NICENESS.extend(['ionice', '-c%s' % (int(CFG["Posix"]["ionice_class"]))])
except: pass
NICENESS.extend(['ionice', '-c{0}'.format(int(CFG["Posix"]["ionice_class"]))])
except:
pass
try:
if 'ionice' in NICENESS:
NICENESS.extend(['-n%s' % (int(CFG["Posix"]["ionice_classdata"]))])
NICENESS.extend(['-n{0}'.format(int(CFG["Posix"]["ionice_classdata"]))])
else:
NICENESS.extend(['ionice', '-n%s' % (int(CFG["Posix"]["ionice_classdata"]))])
except: pass
except: pass
NICENESS.extend(['ionice', '-n{0}'.format(int(CFG["Posix"]["ionice_classdata"]))])
except:
pass
except:
pass
devnull.close()
COMPRESSEDCONTAINER = [re.compile('.r\d{2}$', re.I),
re.compile('.part\d+.rar$', re.I),
re.compile('.rar$', re.I)]
COMPRESSEDCONTAINER += [re.compile('%s$' % ext, re.I) for ext in CFG["Extensions"]["compressedExtensions"]]
COMPRESSEDCONTAINER += [re.compile('{0}$'.format(ext), re.I) for ext in CFG["Extensions"]["compressedExtensions"]]
MEDIACONTAINER = CFG["Extensions"]["mediaExtensions"]
AUDIOCONTAINER = CFG["Extensions"]["audioExtensions"]
METACONTAINER = CFG["Extensions"]["metaExtensions"] # .nfo,.sub,.srt
if isinstance(COMPRESSEDCONTAINER, str): COMPRESSEDCONTAINER = COMPRESSEDCONTAINER.split(',')
if isinstance(MEDIACONTAINER, str): MEDIACONTAINER = MEDIACONTAINER.split(',')
if isinstance(AUDIOCONTAINER, str): AUDIOCONTAINER = AUDIOCONTAINER.split(',')
if isinstance(METACONTAINER, str): METACONTAINER = METACONTAINER.split(',')
if isinstance(COMPRESSEDCONTAINER, str):
COMPRESSEDCONTAINER = COMPRESSEDCONTAINER.split(',')
if isinstance(MEDIACONTAINER, str):
MEDIACONTAINER = MEDIACONTAINER.split(',')
if isinstance(AUDIOCONTAINER, str):
AUDIOCONTAINER = AUDIOCONTAINER.split(',')
if isinstance(METACONTAINER, str):
METACONTAINER = METACONTAINER.split(',')
GETSUBS = int(CFG["Transcoder"]["getSubs"])
TRANSCODE = int(CFG["Transcoder"]["transcode"])
DUPLICATE = int(CFG["Transcoder"]["duplicate"])
CONCAT = int(CFG["Transcoder"]["concat"])
IGNOREEXTENSIONS = (CFG["Transcoder"]["ignoreExtensions"])
if isinstance(IGNOREEXTENSIONS, str): IGNOREEXTENSIONS = IGNOREEXTENSIONS.split(',')
if isinstance(IGNOREEXTENSIONS, str):
IGNOREEXTENSIONS = IGNOREEXTENSIONS.split(',')
OUTPUTFASTSTART = int(CFG["Transcoder"]["outputFastStart"])
GENERALOPTS = (CFG["Transcoder"]["generalOptions"])
if isinstance(GENERALOPTS, str): GENERALOPTS = GENERALOPTS.split(',')
if GENERALOPTS == ['']: GENERALOPTS = []
if not '-fflags' in GENERALOPTS: GENERALOPTS.append('-fflags')
if not '+genpts' in GENERALOPTS: GENERALOPTS.append('+genpts')
if isinstance(GENERALOPTS, str):
GENERALOPTS = GENERALOPTS.split(',')
if GENERALOPTS == ['']:
GENERALOPTS = []
if '-fflags' not in GENERALOPTS:
GENERALOPTS.append('-fflags')
if '+genpts' not in GENERALOPTS:
GENERALOPTS.append('+genpts')
try:
OUTPUTQUALITYPERCENT = int(CFG["Transcoder"]["outputQualityPercent"])
except: pass
except:
pass
OUTPUTVIDEOPATH = CFG["Transcoder"]["outputVideoPath"]
PROCESSOUTPUT = int(CFG["Transcoder"]["processOutput"])
ALANGUAGE = CFG["Transcoder"]["audioLanguage"]
AINCLUDE = int(CFG["Transcoder"]["allAudioLanguages"])
SLANGUAGES = CFG["Transcoder"]["subLanguages"]
if isinstance(SLANGUAGES, str): SLANGUAGES = SLANGUAGES.split(',')
if SLANGUAGES == ['']: SLANGUAGES = []
if isinstance(SLANGUAGES, str):
SLANGUAGES = SLANGUAGES.split(',')
if SLANGUAGES == ['']:
SLANGUAGES = []
SINCLUDE = int(CFG["Transcoder"]["allSubLanguages"])
SEXTRACT = int(CFG["Transcoder"]["extractSubs"])
SEMBED = int(CFG["Transcoder"]["embedSubs"])
@ -437,62 +477,80 @@ def initialize(section=None):
VEXTENSION = CFG["Transcoder"]["outputVideoExtension"].strip()
VCODEC = CFG["Transcoder"]["outputVideoCodec"].strip()
VCODEC_ALLOW = CFG["Transcoder"]["VideoCodecAllow"].strip()
if isinstance(VCODEC_ALLOW, str): VCODEC_ALLOW = VCODEC_ALLOW.split(',')
if VCODEC_ALLOW == ['']: VCODEC_ALLOW = []
if isinstance(VCODEC_ALLOW, str):
VCODEC_ALLOW = VCODEC_ALLOW.split(',')
if VCODEC_ALLOW == ['']:
VCODEC_ALLOW = []
VPRESET = CFG["Transcoder"]["outputVideoPreset"].strip()
try:
VFRAMERATE = float(CFG["Transcoder"]["outputVideoFramerate"].strip())
except: pass
except:
pass
try:
VCRF = int(CFG["Transcoder"]["outputVideoCRF"].strip())
except: pass
except:
pass
try:
VLEVEL = CFG["Transcoder"]["outputVideoLevel"].strip()
except: pass
except:
pass
try:
VBITRATE = int((CFG["Transcoder"]["outputVideoBitrate"].strip()).replace('k','000'))
except: pass
VBITRATE = int((CFG["Transcoder"]["outputVideoBitrate"].strip()).replace('k', '000'))
except:
pass
VRESOLUTION = CFG["Transcoder"]["outputVideoResolution"]
ACODEC = CFG["Transcoder"]["outputAudioCodec"].strip()
ACODEC_ALLOW = CFG["Transcoder"]["AudioCodecAllow"].strip()
if isinstance(ACODEC_ALLOW, str): ACODEC_ALLOW = ACODEC_ALLOW.split(',')
if ACODEC_ALLOW == ['']: ACODEC_ALLOW = []
if isinstance(ACODEC_ALLOW, str):
ACODEC_ALLOW = ACODEC_ALLOW.split(',')
if ACODEC_ALLOW == ['']:
ACODEC_ALLOW = []
try:
ACHANNELS = int(CFG["Transcoder"]["outputAudioChannels"].strip())
except: pass
except:
pass
try:
ABITRATE = int((CFG["Transcoder"]["outputAudioBitrate"].strip()).replace('k','000'))
except: pass
ABITRATE = int((CFG["Transcoder"]["outputAudioBitrate"].strip()).replace('k', '000'))
except:
pass
ACODEC2 = CFG["Transcoder"]["outputAudioTrack2Codec"].strip()
ACODEC2_ALLOW = CFG["Transcoder"]["AudioCodec2Allow"].strip()
if isinstance(ACODEC2_ALLOW, str): ACODEC2_ALLOW = ACODEC2_ALLOW.split(',')
if ACODEC2_ALLOW == ['']: ACODEC2_ALLOW = []
if isinstance(ACODEC2_ALLOW, str):
ACODEC2_ALLOW = ACODEC2_ALLOW.split(',')
if ACODEC2_ALLOW == ['']:
ACODEC2_ALLOW = []
try:
ACHANNELS2 = int(CFG["Transcoder"]["outputAudioTrack2Channels"].strip())
except: pass
except:
pass
try:
ABITRATE2 = int((CFG["Transcoder"]["outputAudioTrack2Bitrate"].strip()).replace('k','000'))
except: pass
ABITRATE2 = int((CFG["Transcoder"]["outputAudioTrack2Bitrate"].strip()).replace('k', '000'))
except:
pass
ACODEC3 = CFG["Transcoder"]["outputAudioOtherCodec"].strip()
ACODEC3_ALLOW = CFG["Transcoder"]["AudioOtherCodecAllow"].strip()
if isinstance(ACODEC3_ALLOW, str): ACODEC3_ALLOW = ACODEC3_ALLOW.split(',')
if ACODEC3_ALLOW == ['']: ACODEC3_ALLOW = []
if isinstance(ACODEC3_ALLOW, str):
ACODEC3_ALLOW = ACODEC3_ALLOW.split(',')
if ACODEC3_ALLOW == ['']:
ACODEC3_ALLOW = []
try:
ACHANNELS3 = int(CFG["Transcoder"]["outputAudioOtherChannels"].strip())
except: pass
except:
pass
try:
ABITRATE3 = int((CFG["Transcoder"]["outputAudioOtherBitrate"].strip()).replace('k','000'))
except: pass
ABITRATE3 = int((CFG["Transcoder"]["outputAudioOtherBitrate"].strip()).replace('k', '000'))
except:
pass
SCODEC = CFG["Transcoder"]["outputSubtitleCodec"].strip()
BURN = int(CFG["Transcoder"]["burnInSubtitle"].strip())
DEFAULTS = CFG["Transcoder"]["outputDefault"].strip()
HWACCEL = int(CFG["Transcoder"]["hwAccel"])
allow_subs = ['.mkv','.mp4', '.m4v', 'asf', 'wma', 'wmv']
allow_subs = ['.mkv', '.mp4', '.m4v', 'asf', 'wma', 'wmv']
codec_alias = {
'libx264':['libx264', 'h264', 'h.264', 'AVC', 'MPEG-4'],
'libmp3lame':['libmp3lame', 'mp3'],
'libfaac':['libfaac', 'aac', 'faac']
'libx264': ['libx264', 'h264', 'h.264', 'AVC', 'MPEG-4'],
'libmp3lame': ['libmp3lame', 'mp3'],
'libfaac': ['libfaac', 'aac', 'faac']
}
transcode_defaults = {
'iPad':{
@ -609,6 +667,7 @@ def initialize(section=None):
'SCODEC':'mov_text'
}
}
}
if DEFAULTS and DEFAULTS in transcode_defaults:
VEXTENSION = transcode_defaults[DEFAULTS]['VEXTENSION']
VCODEC = transcode_defaults[DEFAULTS]['VCODEC']
@ -638,25 +697,29 @@ def initialize(section=None):
if VEXTENSION in allow_subs:
ALLOWSUBS = 1
if not VCODEC_ALLOW and VCODEC: VCODEC_ALLOW.extend([VCODEC])
if not VCODEC_ALLOW and VCODEC:
VCODEC_ALLOW.extend([VCODEC])
for codec in VCODEC_ALLOW:
if codec in codec_alias:
extra = [ item for item in codec_alias[codec] if item not in VCODEC_ALLOW ]
extra = [item for item in codec_alias[codec] if item not in VCODEC_ALLOW]
VCODEC_ALLOW.extend(extra)
if not ACODEC_ALLOW and ACODEC: ACODEC_ALLOW.extend([ACODEC])
if not ACODEC_ALLOW and ACODEC:
ACODEC_ALLOW.extend([ACODEC])
for codec in ACODEC_ALLOW:
if codec in codec_alias:
extra = [ item for item in codec_alias[codec] if item not in ACODEC_ALLOW ]
extra = [item for item in codec_alias[codec] if item not in ACODEC_ALLOW]
ACODEC_ALLOW.extend(extra)
if not ACODEC2_ALLOW and ACODEC2: ACODEC2_ALLOW.extend([ACODEC2])
if not ACODEC2_ALLOW and ACODEC2:
ACODEC2_ALLOW.extend([ACODEC2])
for codec in ACODEC2_ALLOW:
if codec in codec_alias:
extra = [ item for item in codec_alias[codec] if item not in ACODEC2_ALLOW ]
extra = [item for item in codec_alias[codec] if item not in ACODEC2_ALLOW]
ACODEC2_ALLOW.extend(extra)
if not ACODEC3_ALLOW and ACODEC3: ACODEC3_ALLOW.extend([ACODEC3])
if not ACODEC3_ALLOW and ACODEC3:
ACODEC3_ALLOW.extend([ACODEC3])
for codec in ACODEC3_ALLOW:
if codec in codec_alias:
extra = [ item for item in codec_alias[codec] if item not in ACODEC3_ALLOW ]
extra = [item for item in codec_alias[codec] if item not in ACODEC3_ALLOW]
ACODEC3_ALLOW.extend(extra)
codec_alias = {} # clear memory
@ -682,47 +745,59 @@ def initialize(section=None):
else:
try:
SEVENZIP = subprocess.Popen(['which', '7z'], stdout=subprocess.PIPE).communicate()[0].strip()
except: pass
except:
pass
if not SEVENZIP:
try:
SEVENZIP = subprocess.Popen(['which', '7zr'], stdout=subprocess.PIPE).communicate()[0].strip()
except: pass
except:
pass
if not SEVENZIP:
try:
SEVENZIP = subprocess.Popen(['which', '7za'], stdout=subprocess.PIPE).communicate()[0].strip()
except: pass
except:
pass
if not SEVENZIP:
SEVENZIP = None
logger.warning("Failed to locate 7zip. Transcosing of disk images and extraction of .7z files will not be possible!")
if os.path.isfile(os.path.join(FFMPEG_PATH, 'ffmpeg')) or os.access(os.path.join(FFMPEG_PATH, 'ffmpeg'), os.X_OK):
logger.warning(
"Failed to locate 7zip. Transcosing of disk images and extraction of .7z files will not be possible!")
if os.path.isfile(os.path.join(FFMPEG_PATH, 'ffmpeg')) or os.access(os.path.join(FFMPEG_PATH, 'ffmpeg'),
os.X_OK):
FFMPEG = os.path.join(FFMPEG_PATH, 'ffmpeg')
elif os.path.isfile(os.path.join(FFMPEG_PATH, 'avconv')) or os.access(os.path.join(FFMPEG_PATH, 'avconv'), os.X_OK):
elif os.path.isfile(os.path.join(FFMPEG_PATH, 'avconv')) or os.access(os.path.join(FFMPEG_PATH, 'avconv'),
os.X_OK):
FFMPEG = os.path.join(FFMPEG_PATH, 'avconv')
else:
try:
FFMPEG = subprocess.Popen(['which', 'ffmpeg'], stdout=subprocess.PIPE).communicate()[0].strip()
except: pass
except:
pass
if not FFMPEG:
try:
FFMPEG = subprocess.Popen(['which', 'avconv'], stdout=subprocess.PIPE).communicate()[0].strip()
except: pass
except:
pass
if not FFMPEG:
FFMPEG = None
logger.warning("Failed to locate ffmpeg. Transcoding disabled!")
logger.warning("Install ffmpeg with x264 support to enable this feature ...")
if os.path.isfile(os.path.join(FFMPEG_PATH, 'ffprobe')) or os.access(os.path.join(FFMPEG_PATH, 'ffprobe'), os.X_OK):
if os.path.isfile(os.path.join(FFMPEG_PATH, 'ffprobe')) or os.access(os.path.join(FFMPEG_PATH, 'ffprobe'),
os.X_OK):
FFPROBE = os.path.join(FFMPEG_PATH, 'ffprobe')
elif os.path.isfile(os.path.join(FFMPEG_PATH, 'avprobe')) or os.access(os.path.join(FFMPEG_PATH, 'avprobe'), os.X_OK):
elif os.path.isfile(os.path.join(FFMPEG_PATH, 'avprobe')) or os.access(os.path.join(FFMPEG_PATH, 'avprobe'),
os.X_OK):
FFPROBE = os.path.join(FFMPEG_PATH, 'avprobe')
else:
try:
FFPROBE = subprocess.Popen(['which', 'ffprobe'], stdout=subprocess.PIPE).communicate()[0].strip()
except: pass
except:
pass
if not FFPROBE:
try:
FFPROBE = subprocess.Popen(['which', 'avprobe'], stdout=subprocess.PIPE).communicate()[0].strip()
except: pass
except:
pass
if not FFPROBE:
FFPROBE = None
if CHECK_MEDIA:
@ -731,7 +806,7 @@ def initialize(section=None):
# check for script-defied section and if None set to allow sections
SECTIONS = CFG[tuple(x for x in CFG if CFG[x].sections and CFG[x].isenabled()) if not section else (section,)]
for section,subsections in SECTIONS.items():
for section, subsections in SECTIONS.items():
CATEGORIES.extend([subsection for subsection in subsections if CFG[section][subsection].isenabled()])
CATEGORIES = list(set(CATEGORIES))
@ -741,6 +816,7 @@ def initialize(section=None):
# finished initalizing
return True
def restart():
install_type = versionCheck.CheckVersion().install_type
@ -752,7 +828,7 @@ def restart():
if popen_list:
popen_list += SYS_ARGV
logger.log(u"Restarting nzbToMedia with " + str(popen_list))
logger.log(u"Restarting nzbToMedia with {args}".format(args=popen_list))
logger.close()
p = subprocess.Popen(popen_list, cwd=os.getcwd())
p.wait()
@ -760,8 +836,9 @@ def restart():
os._exit(status)
def rchmod(path, mod):
logger.log("Changing file mode of %s to %s" % (path, oct(mod)))
logger.log("Changing file mode of {0} to {1}".format(path, oct(mod)))
os.chmod(path, mod)
if not os.path.isdir(path):
return # Skip files

1
core/autoProcess/__init__.py
View file

@ -0,0 +1 @@
# coding=utf-8

81
core/autoProcess/autoProcessComics.py
View file

@ -1,81 +1,70 @@
# coding=utf-8
import os
import time
import core
import requests
import time
from core.nzbToMediaUtil import convert_to_ascii, remoteDir, server_responding
from core.nzbToMediaSceneExceptions import process_all_exceptions
from core import logger
requests.packages.urllib3.disable_warnings()
class autoProcessComics:
class autoProcessComics(object):
def processEpisode(self, section, dirName, inputName=None, status=0, clientAgent='manual', inputCategory=None):
if int(status) != 0:
logger.warning("FAILED DOWNLOAD DETECTED, nothing to process.",section)
return [1, "%s: Failed to post-process. %s does not support failed downloads" % (section, section) ]
logger.warning("FAILED DOWNLOAD DETECTED, nothing to process.", section)
return [1, "{0}: Failed to post-process. {1} does not support failed downloads".format(section, section)]
host = core.CFG[section][inputCategory]["host"]
port = core.CFG[section][inputCategory]["port"]
username = core.CFG[section][inputCategory]["username"]
password = core.CFG[section][inputCategory]["password"]
try:
ssl = int(core.CFG[section][inputCategory]["ssl"])
except:
ssl = 0
try:
web_root = core.CFG[section][inputCategory]["web_root"]
except:
web_root = ""
try:
remote_path = int(core.CFG[section][inputCategory]["remote_path"])
except:
remote_path = 0
cfg = dict(core.CFG[section][inputCategory])
if ssl:
protocol = "https://"
else:
protocol = "http://"
host = cfg["host"]
port = cfg["port"]
username = cfg["username"]
password = cfg["password"]
ssl = int(cfg.get("ssl", 0))
web_root = cfg.get("web_root", "")
remote_path = int(cfg.get("remote_path"), 0)
protocol = "https://" if ssl else "http://"
url = "%s%s:%s%s/post_process" % (protocol, host, port, web_root)
url = "{0}{1}:{2}{3}/post_process".format(protocol, host, port, web_root)
if not server_responding(url):
logger.error("Server did not respond. Exiting", section)
return [1, "%s: Failed to post-process - %s did not respond." % (section, section) ]
return [1, "{0}: Failed to post-process - {1} did not respond.".format(section, section)]
inputName, dirName = convert_to_ascii(inputName, dirName)
clean_name, ext = os.path.splitext(inputName)
if len(ext) == 4: # we assume this was a standrard extension.
if len(ext) == 4: # we assume this was a standard extension.
inputName = clean_name
params = {}
params['nzb_folder'] = dirName
params = {
'nzb_folder': remoteDir(dirName) if remote_path else dirName,
}
if remote_path:
params['nzb_folder'] = remoteDir(dirName)
if inputName != None:
if inputName is not None:
params['nzb_name'] = inputName
success = False
logger.debug("Opening URL: %s" % (url), section)
logger.debug("Opening URL: {0}".format(url), section)
try:
r = requests.get(url, auth=(username, password), params=params, stream=True, verify=False, timeout=(30, 300))
except requests.ConnectionError:
logger.error("Unable to open URL", section)
return [1, "%s: Failed to post-process - Unable to connect to %s" % (section, section) ]
return [1, "{0}: Failed to post-process - Unable to connect to {1}".format(section, section)]
for line in r.iter_lines():
if line: logger.postprocess("%s" % (line), section)
if ("Post Processing SUCCESSFUL!" or "Post Processing SUCCESSFULL!")in line: success = True
if line:
logger.postprocess("{0}".format(line), section)
if "Post Processing SUCCESSFUL" in line:
success = True
if not r.status_code in [requests.codes.ok, requests.codes.created, requests.codes.accepted]:
logger.error("Server returned status %s" % (str(r.status_code)), section)
return [1, "%s: Failed to post-process - Server returned status %s" % (section, str(r.status_code)) ]
if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]:
logger.error("Server returned status {0}".format(r.status_code), section)
return [1, "{0}: Failed to post-process - Server returned status {1}".format(section, r.status_code)]
if success:
logger.postprocess("SUCCESS: This issue has been processed successfully",section)
return [0, "%s: Successfully post-processed %s" % (section, inputName) ]
logger.postprocess("SUCCESS: This issue has been processed successfully", section)
return [0, "{0}: Successfully post-processed {1}".format(section, inputName)]
else:
logger.warning("The issue does not appear to have successfully processed. Please check your Logs",section)
return [1, "%s: Failed to post-process - Returned log from %s was not as expected." % (section, section) ]
logger.warning("The issue does not appear to have successfully processed. Please check your Logs", section)
return [1, "{0}: Failed to post-process - Returned log from {1} was not as expected.".format(section, section)]

83
core/autoProcess/autoProcessGames.py
View file

@ -1,86 +1,77 @@
# coding=utf-8
import os
import core
import requests
import shutil
from core.nzbToMediaUtil import convert_to_ascii, server_responding
from core.nzbToMediaSceneExceptions import process_all_exceptions
from core import logger
requests.packages.urllib3.disable_warnings()
class autoProcessGames:
class autoProcessGames(object):
def process(self, section, dirName, inputName=None, status=0, clientAgent='manual', inputCategory=None):
status = int(status)
host = core.CFG[section][inputCategory]["host"]
port = core.CFG[section][inputCategory]["port"]
apikey = core.CFG[section][inputCategory]["apikey"]
try:
library = core.CFG[section][inputCategory]["library"]
except:
library = None
try:
ssl = int(core.CFG[section][inputCategory]["ssl"])
except:
ssl = 0
try:
web_root = core.CFG[section][inputCategory]["web_root"]
except:
web_root = ""
cfg = dict(core.CFG[section][inputCategory])
if ssl:
protocol = "https://"
else:
protocol = "http://"
host = cfg["host"]
port = cfg["port"]
apikey = cfg["apikey"]
library = cfg.get("library")
ssl = int(cfg.get("ssl", 0))
web_root = cfg.get("web_root", "")
protocol = "https://" if ssl else "http://"
url = "%s%s:%s%s/api" % (protocol, host, port, web_root)
url = "{0}{1}:{2}{3}/api".format(protocol, host, port, web_root)
if not server_responding(url):
logger.error("Server did not respond. Exiting", section)
return [1, "%s: Failed to post-process - %s did not respond." % (section, section) ]
return [1, "{0}: Failed to post-process - {1} did not respond.".format(section, section)]
inputName, dirName = convert_to_ascii(inputName, dirName)
fields = inputName.split("-")
gamezID = fields[0].replace("[","").replace("]","").replace(" ","")
gamezID = fields[0].replace("[", "").replace("]", "").replace(" ", "")
downloadStatus = 'Wanted'
if status == 0:
downloadStatus = 'Downloaded'
downloadStatus = 'Downloaded' if status == 0 else 'Wanted'
params = {}
params['api_key'] = apikey
params['mode'] = 'UPDATEREQUESTEDSTATUS'
params['db_id'] = gamezID
params['status'] = downloadStatus
params = {
'api_key': apikey,
'mode': 'UPDATEREQUESTEDSTATUS',
'db_id': gamezID,
'status': downloadStatus
}
logger.debug("Opening URL: %s" % (url),section)
logger.debug("Opening URL: {0}".format(url), section)
try:
r = requests.get(url, params=params, verify=False, timeout=(30, 300))
except requests.ConnectionError:
logger.error("Unable to open URL")
return [1, "%s: Failed to post-process - Unable to connect to %s" % (section, section) ]
return [1, "{0}: Failed to post-process - Unable to connect to {1}".format(section, section)]
result = r.json()
logger.postprocess("%s" % (result),section)
logger.postprocess("{0}".format(result), section)
if library:
logger.postprocess("moving files to library: %s" % (library),section)
logger.postprocess("moving files to library: {0}".format(library), section)
try:
shutil.move(dirName, os.path.join(library, inputName))
except:
logger.error("Unable to move %s to %s" % (dirName, os.path.join(library, inputName)), section)
return [1, "%s: Failed to post-process - Unable to move files" % (section) ]
logger.error("Unable to move {0} to {1}".format(dirName, os.path.join(library, inputName)), section)
return [1, "{0}: Failed to post-process - Unable to move files".format(section)]
else:
logger.error("No library specified to move files to. Please edit your configuration.", section)
return [1, "%s: Failed to post-process - No library defined in %s" % (section, section) ]
return [1, "{0}: Failed to post-process - No library defined in {1}".format(section, section)]
if not r.status_code in [requests.codes.ok, requests.codes.created, requests.codes.accepted]:
logger.error("Server returned status %s" % (str(r.status_code)), section)
return [1, "%s: Failed to post-process - Server returned status %s" % (section, str(r.status_code)) ]
if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]:
logger.error("Server returned status {0}".format(r.status_code), section)
return [1, "{0}: Failed to post-process - Server returned status {1}".format(section, r.status_code)]
elif result['success']:
logger.postprocess("SUCCESS: Status for %s has been set to %s in Gamez" % (gamezID, downloadStatus),section)
return [0, "%s: Successfully post-processed %s" % (section, inputName) ]
logger.postprocess("SUCCESS: Status for {0} has been set to {1} in Gamez".format(gamezID, downloadStatus), section)
return [0, "{0}: Successfully post-processed {1}".format(section, inputName)]
else:
logger.error("FAILED: Status for %s has NOT been updated in Gamez" % (gamezID),section)
return [1, "%s: Failed to post-process - Returned log from %s was not as expected." % (section, section) ]
logger.error("FAILED: Status for {0} has NOT been updated in Gamez".format(gamezID), section)
return [1, "{0}: Failed to post-process - Returned log from {1} was not as expected.".format(section, section)]

202
core/autoProcess/autoProcessMovie.py
View file

@ -1,3 +1,5 @@
# coding=utf-8
import os
import time
import requests
@ -10,12 +12,13 @@ from core.transcoder import transcoder
requests.packages.urllib3.disable_warnings()
class autoProcessMovie:
class autoProcessMovie(object):
def get_release(self, baseURL, imdbid=None, download_id=None, release_id=None):
results = {}
params = {}
# determin cmd and params to send to CouchPotato to get our results
# determine cmd and params to send to CouchPotato to get our results
section = 'movies'
cmd = "/media.list"
if release_id or imdbid:
@ -24,27 +27,28 @@ class autoProcessMovie:
params['id'] = release_id or imdbid
url = baseURL + cmd
logger.debug("Opening URL: %s with PARAMS: %s" % (url, params))
logger.debug("Opening URL: {0} with PARAMS: {1}".format(url, params))
try:
r = requests.get(url, params=params, verify=False, timeout=(30, 60))
except requests.ConnectionError:
logger.error("Unable to open URL %s" % url)
logger.error("Unable to open URL {0}".format(url))
return results
try:
result = r.json()
except:
except ValueError:
# ValueError catches simplejson's JSONDecodeError and json's ValueError
logger.error("CouchPotato returned the following non-json data")
for line in r.iter_lines():
logger.error("%s" %(line))
logger.error("{0}".format(line))
return results
if not result['success']:
if 'error' in result:
logger.error(str(result['error']))
logger.error('{0}'.format(result['error']))
else:
logger.error("no media found for id %s" % (params['id']))
logger.error("no media found for id {0}".format(params['id']))
return results
# Gather release info and return it back, no need to narrow results
@ -53,7 +57,8 @@ class autoProcessMovie:
id = result[section]['_id']
results[id] = result[section]
return results
except:pass
except:
pass
# Gather release info and proceed with trying to narrow results to one release choice
@ -100,39 +105,24 @@ class autoProcessMovie:
def process(self, section, dirName, inputName=None, status=0, clientAgent="manual", download_id="", inputCategory=None, failureLink=None):
host = core.CFG[section][inputCategory]["host"]
port = core.CFG[section][inputCategory]["port"]
apikey = core.CFG[section][inputCategory]["apikey"]
method = core.CFG[section][inputCategory]["method"]
delete_failed = int(core.CFG[section][inputCategory]["delete_failed"])
wait_for = int(core.CFG[section][inputCategory]["wait_for"])
cfg = dict(core.CFG[section][inputCategory])
try:
ssl = int(core.CFG[section][inputCategory]["ssl"])
except:
ssl = 0
try:
web_root = core.CFG[section][inputCategory]["web_root"]
except:
web_root = ""
try:
remote_path = int(core.CFG[section][inputCategory]["remote_path"])
except:
remote_path = 0
try:
extract = int(section[inputCategory]["extract"])
except:
extract = 0
host = cfg["host"]
port = cfg["port"]
apikey = cfg["apikey"]
method = cfg["method"]
delete_failed = int(cfg["delete_failed"])
wait_for = int(cfg["wait_for"])
ssl = int(cfg.get("ssl", 0))
web_root = cfg.get("web_root", "")
remote_path = int(cfg.get("remote_path", 0))
extract = int(cfg.get("extract", 0))
protocol = "https://" if ssl else "http://"
if ssl:
protocol = "https://"
else:
protocol = "http://"
baseURL = "%s%s:%s%s/api/%s" % (protocol, host, port, web_root, apikey)
baseURL = "{0}{1}:{2}{3}/api/{4}".format(protocol, host, port, web_root, apikey)
if not server_responding(baseURL):
logger.error("Server did not respond. Exiting", section)
return [1, "%s: Failed to post-process - %s did not respond." % (section, section) ]
return [1, "{0}: Failed to post-process - {1} did not respond.".format(section, section)]
imdbid = find_imdbid(dirName, inputName)
release = self.get_release(baseURL, imdbid, download_id)
@ -166,7 +156,7 @@ class autoProcessMovie:
inputName, dirName = convert_to_ascii(inputName, dirName)
if not listMediaFiles(dirName, media=True, audio=False, meta=False, archives=False) and listMediaFiles(dirName, media=False, audio=False, meta=False, archives=True) and extract:
logger.debug('Checking for archives to extract in directory: %s' % (dirName))
logger.debug('Checking for archives to extract in directory: {0}'.format(dirName))
core.extractFiles(dirName)
inputName, dirName = convert_to_ascii(inputName, dirName)
@ -179,41 +169,47 @@ class autoProcessMovie:
if transcoder.isVideoGood(video, status):
import_subs(video)
good_files += 1
if num_files > 0 and good_files == num_files:
if num_files and good_files == num_files:
if status:
logger.info("Status shown as failed from Downloader, but %s valid video files found. Setting as success." % (str(good_files)), section)
logger.info("Status shown as failed from Downloader, but {0} valid video files found. Setting as success.".format(good_files), section)
status = 0
elif num_files > 0 and good_files < num_files:
elif num_files and good_files < num_files:
logger.info("Status shown as success from Downloader, but corrupt video files found. Setting as failed.", section)
if os.environ.has_key('NZBOP_VERSION') and os.environ['NZBOP_VERSION'][0:5] >= '14.0':
if 'NZBOP_VERSION' in os.environ and os.environ['NZBOP_VERSION'][0:5] >= '14.0':
print('[NZB] MARK=BAD')
if failureLink:
failureLink = failureLink + '&corrupt=true'
failureLink += '&corrupt=true'
status = 1
elif clientAgent == "manual":
logger.warning("No media files found in directory %s to manually process." % (dirName), section)
logger.warning("No media files found in directory {0} to manually process.".format(dirName), section)
return [0, ""] # Success (as far as this script is concerned)
else:
logger.warning("No media files found in directory %s. Processing this as a failed download" % (dirName), section)
logger.warning("No media files found in directory {0}. Processing this as a failed download".format(dirName), section)
status = 1
if os.environ.has_key('NZBOP_VERSION') and os.environ['NZBOP_VERSION'][0:5] >= '14.0':
if 'NZBOP_VERSION' in os.environ and os.environ['NZBOP_VERSION'][0:5] >= '14.0':
print('[NZB] MARK=BAD')
if status == 0:
if core.TRANSCODE == 1:
result, newDirName = transcoder.Transcode_directory(dirName)
if result == 0:
logger.debug("Transcoding succeeded for files in %s" % (dirName), section)
logger.debug("Transcoding succeeded for files in {0}".format(dirName), section)
dirName = newDirName
chmod_directory = int(str(cfg.get("chmodDirectory", "0")), 8)
logger.debug("Config setting 'chmodDirectory' currently set to {0}".format(oct(chmod_directory)), section)
if chmod_directory:
logger.info("Attempting to set the octal permission of '{0}' on directory '{1}'".format(oct(chmod_directory), dirName), section)
core.rchmod(dirName, chmod_directory)
else:
logger.error("Transcoding failed for files in %s" % (dirName), section)
return [1, "%s: Failed to post-process - Transcoding failed" % (section) ]
logger.error("Transcoding failed for files in {0}".format(dirName), section)
return [1, "{0}: Failed to post-process - Transcoding failed".format(section)]
for video in listMediaFiles(dirName, media=True, audio=False, meta=False, archives=False):
if not release and not ".cp(tt" in video and imdbid:
if not release and ".cp(tt" not in video and imdbid:
videoName, videoExt = os.path.splitext(video)
video2 = "%s.cp(%s)%s" % (videoName, imdbid, videoExt)
video2 = "{0}.cp({1}){2}".format(videoName, imdbid, videoExt)
if not (clientAgent in [core.TORRENT_CLIENTAGENT, 'manual'] and core.USELINK == 'move-sym'):
logger.debug('Renaming: %s to: %s' % (video, video2))
logger.debug('Renaming: {0} to: {1}'.format(video, video2))
os.rename(video, video2)
params = {}
@ -221,9 +217,7 @@ class autoProcessMovie:
params['downloader'] = downloader or clientAgent
params['download_id'] = download_id
params['media_folder'] = dirName
if remote_path:
params['media_folder'] = remoteDir(dirName)
params['media_folder'] = remoteDir(dirName) if remote_path else dirName
if method == "manage":
command = "/manage.update"
@ -231,129 +225,129 @@ class autoProcessMovie:
else:
command = "/renamer.scan"
url = "%s%s" % (baseURL, command)
url = "{0}{1}".format(baseURL, command)
logger.debug("Opening URL: %s with PARAMS: %s" % (url, params), section)
logger.debug("Opening URL: {0} with PARAMS: {1}".format(url, params), section)
logger.postprocess("Starting %s scan for %s" % (method, inputName), section)
logger.postprocess("Starting {0} scan for {1}".format(method, inputName), section)
try:
r = requests.get(url, params=params, verify=False, timeout=(30, 1800))
except requests.ConnectionError:
logger.error("Unable to open URL", section)
return [1, "%s: Failed to post-process - Unable to connect to %s" % (section, section) ]
return [1, "{0}: Failed to post-process - Unable to connect to {1}".format(section, section)]
result = r.json()
if not r.status_code in [requests.codes.ok, requests.codes.created, requests.codes.accepted]:
logger.error("Server returned status %s" % (str(r.status_code)), section)
return [1, "%s: Failed to post-process - Server returned status %s" % (section, str(r.status_code)) ]
if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]:
logger.error("Server returned status {0}".format(r.status_code), section)
return [1, "{0}: Failed to post-process - Server returned status {1}".format(section, r.status_code)]
elif result['success']:
logger.postprocess("SUCCESS: Finished %s scan for folder %s" % (method, dirName), section)
logger.postprocess("SUCCESS: Finished {0} scan for folder {1}".format(method, dirName), section)
if method == "manage":
return [0, "%s: Successfully post-processed %s" % (section, inputName) ]
return [0, "{0}: Successfully post-processed {1}".format(section, inputName)]
else:
logger.error("FAILED: %s scan was unable to finish for folder %s. exiting!" % (method, dirName),
logger.error("FAILED: {0} scan was unable to finish for folder {1}. exiting!".format(method, dirName),
section)
return [1, "%s: Failed to post-process - Server did not return success" % (section) ]
return [1, "{0}: Failed to post-process - Server did not return success".format(section)]
else:
core.FAILED = True
logger.postprocess("FAILED DOWNLOAD DETECTED FOR %s" % (inputName), section)
logger.postprocess("FAILED DOWNLOAD DETECTED FOR {0}".format(inputName), section)
if failureLink:
reportNzb(failureLink, clientAgent)
if delete_failed and os.path.isdir(dirName) and not os.path.dirname(dirName) == dirName:
logger.postprocess("Deleting failed files and folder %s" % dirName, section)
logger.postprocess("Deleting failed files and folder {0}".format(dirName), section)
rmDir(dirName)
if not release_id and not media_id:
logger.error("Could not find a downloaded movie in the database matching %s, exiting!" % inputName,
logger.error("Could not find a downloaded movie in the database matching {0}, exiting!".format(inputName),
section)
return [1, "%s: Failed to post-process - Failed download not found in %s" % (section, section) ]
return [1, "{0}: Failed to post-process - Failed download not found in {1}".format(section, section)]
if release_id:
logger.postprocess("Setting failed release %s to ignored ..." % (inputName), section)
logger.postprocess("Setting failed release {0} to ignored ...".format(inputName), section)
url = baseURL + "/release.ignore"
url = "{url}/release.ignore".format(url=baseURL)
params = {'id': release_id}
logger.debug("Opening URL: %s with PARAMS: %s" % (url, params), section)
logger.debug("Opening URL: {0} with PARAMS: {1}".format(url, params), section)
try:
r = requests.get(url, params=params, verify=False, timeout=(30, 120))
except requests.ConnectionError:
logger.error("Unable to open URL %s" % (url), section)
return [1, "%s: Failed to post-process - Unable to connect to %s" % (section, section) ]
logger.error("Unable to open URL {0}".format(url), section)
return [1, "{0}: Failed to post-process - Unable to connect to {1}".format(section, section)]
result = r.json()
if not r.status_code in [requests.codes.ok, requests.codes.created, requests.codes.accepted]:
logger.error("Server returned status %s" % (str(r.status_code)), section)
return [1, "%s: Failed to post-process - Server returned status %s" % (section, str(r.status_code)) ]
if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]:
logger.error("Server returned status {0}".format(r.status_code), section)
return [1, "{0}: Failed to post-process - Server returned status {1}".format(section, r.status_code)]
elif result['success']:
logger.postprocess("SUCCESS: %s has been set to ignored ..." % (inputName), section)
logger.postprocess("SUCCESS: {0} has been set to ignored ...".format(inputName), section)
else:
logger.warning("FAILED: Unable to set %s to ignored!" % (inputName), section)
return [1, "%s: Failed to post-process - Unable to set %s to ignored" % (section, inputName) ]
logger.warning("FAILED: Unable to set {0} to ignored!".format(inputName), section)
return [1, "{0}: Failed to post-process - Unable to set {1} to ignored".format(section, inputName)]
logger.postprocess("Trying to snatch the next highest ranked release.", section)
url = "%s/movie.searcher.try_next" % (baseURL)
logger.debug("Opening URL: %s" % (url), section)
url = "{0}/movie.searcher.try_next".format(baseURL)
logger.debug("Opening URL: {0}".format(url), section)
try:
r = requests.get(url, params={'media_id': media_id}, verify=False, timeout=(30, 600))
except requests.ConnectionError:
logger.error("Unable to open URL %s" % (url), section)
return [1, "%s: Failed to post-process - Unable to connect to %s" % (section, section) ]
logger.error("Unable to open URL {0}".format(url), section)
return [1, "{0}: Failed to post-process - Unable to connect to {1}".format(section, section)]
result = r.json()
if not r.status_code in [requests.codes.ok, requests.codes.created, requests.codes.accepted]:
logger.error("Server returned status %s" % (str(r.status_code)), section)
return [1, "%s: Failed to post-process - Server returned status %s" % (section, str(r.status_code)) ]
if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]:
logger.error("Server returned status {0}".format(r.status_code), section)
return [1, "{0}: Failed to post-process - Server returned status {1}".format(section, r.status_code)]
elif result['success']:
logger.postprocess("SUCCESS: Snatched the next highest release ...", section)
return [0, "%s: Successfully snatched next highest release" % (section) ]
return [0, "{0}: Successfully snatched next highest release".format(section)]
else:
logger.postprocess("SUCCESS: Unable to find a new release to snatch now. CP will keep searching!", section)
return [0, "%s: No new release found now. %s will keep searching" % (section, section) ]
return [0, "{0}: No new release found now. {1} will keep searching".format(section, section)]
# Added a releease that was not in the wanted list so confirm rename successful by finding this movie media.list.
# Added a release that was not in the wanted list so confirm rename successful by finding this movie media.list.
if not release:
download_id = None # we don't want to filter new releases based on this.
# we will now check to see if CPS has finished renaming before returning to TorrentToMedia and unpausing.
timeout = time.time() + 60 * wait_for
while (time.time() < timeout): # only wait 2 (default) minutes, then return.
while time.time() < timeout: # only wait 2 (default) minutes, then return.
logger.postprocess("Checking for status change, please stand by ...", section)
release = self.get_release(baseURL, imdbid, download_id, release_id)
if release:
try:
if release_id is None and release_status_old is None: # we didn't have a release before, but now we do.
logger.postprocess("SUCCESS: Movie %s has now been added to CouchPotato" % (imdbid), section)
return [0, "%s: Successfully post-processed %s" % (section, inputName) ]
logger.postprocess("SUCCESS: Movie {0} has now been added to CouchPotato".format(imdbid), section)
return [0, "{0}: Successfully post-processed {1}".format(section, inputName)]
release_status_new = release[release_id]['status']
if release_status_new != release_status_old:
logger.postprocess("SUCCESS: Release %s has now been marked with a status of [%s]" % (
logger.postprocess("SUCCESS: Release {0} has now been marked with a status of [{1}]".format(
inputName, str(release_status_new).upper()), section)
return [0, "%s: Successfully post-processed %s" % (section, inputName) ]
return [0, "{0}: Successfully post-processed {1}".format(section, inputName)]
except:
pass
if not os.path.isdir(dirName):
logger.postprocess("SUCCESS: Input Directory [%s] has been processed and removed" % (
logger.postprocess("SUCCESS: Input Directory [{0}] has been processed and removed".format(
dirName), section)
return [0, "%s: Successfully post-processed %s" % (section, inputName) ]
return [0, "{0}: Successfully post-processed {1}".format(section, inputName)]
elif not listMediaFiles(dirName, media=True, audio=False, meta=False, archives=True):
logger.postprocess("SUCCESS: Input Directory [%s] has no remaining media files. This has been fully processed." % (
logger.postprocess("SUCCESS: Input Directory [{0}] has no remaining media files. This has been fully processed.".format(
dirName), section)
return [0, "%s: Successfully post-processed %s" % (section, inputName) ]
return [0, "{0}: Successfully post-processed {1}".format(section, inputName)]
# pause and let CouchPotatoServer catch its breath
time.sleep(10 * wait_for)
# The status hasn't changed. we have waited 2 minutes which is more than enough. uTorrent can resule seeding now.
# The status hasn't changed. we have waited 2 minutes which is more than enough. uTorrent can resume seeding now.
logger.warning(
"%s does not appear to have changed status after %s minutes, Please check your logs." % (inputName, wait_for),
"{0} does not appear to have changed status after {1} minutes, Please check your logs.".format(inputName, wait_for),
section)
return [1, "%s: Failed to post-process - No change in status" % (section) ]
return [1, "{0}: Failed to post-process - No change in status".format(section)]

118
core/autoProcess/autoProcessMusic.py
View file

@ -1,3 +1,5 @@
# coding=utf-8
import os
import time
import requests
@ -9,65 +11,53 @@ from core import logger
requests.packages.urllib3.disable_warnings()
class autoProcessMusic:
class autoProcessMusic(object):
def get_status(self, url, apikey, dirName):
logger.debug("Attempting to get current status for release:%s" % (os.path.basename(dirName)))
logger.debug("Attempting to get current status for release:{0}".format(os.path.basename(dirName)))
params = {}
params['apikey'] = apikey
params['cmd'] = "getHistory"
params = {
'apikey': apikey,
'cmd': "getHistory"
}
logger.debug("Opening URL: %s with PARAMS: %s" % (url, params))
logger.debug("Opening URL: {0} with PARAMS: {1}".format(url, params))
try:
r = requests.get(url, params=params, verify=False, timeout=(30, 120))
except Exception, e:
except requests.RequestException:
logger.error("Unable to open URL")
return None
try:
result = r.json()
except ValueError:
# ValueError catches simplejson's JSONDecodeError and json's ValueError
return None
for album in result:
if os.path.basename(dirName) == album['FolderName']:
return album["Status"].lower()
except:
return None
def process(self, section, dirName, inputName=None, status=0, clientAgent="manual", inputCategory=None):
status = int(status)
host = core.CFG[section][inputCategory]["host"]
port = core.CFG[section][inputCategory]["port"]
apikey = core.CFG[section][inputCategory]["apikey"]
wait_for = int(core.CFG[section][inputCategory]["wait_for"])
cfg = dict(core.CFG[section][inputCategory])
try:
ssl = int(core.CFG[section][inputCategory]["ssl"])
except:
ssl = 0
try:
web_root = core.CFG[section][inputCategory]["web_root"]
except:
web_root = ""
try:
remote_path = int(core.CFG[section][inputCategory]["remote_path"])
except:
remote_path = 0
try:
extract = int(section[inputCategory]["extract"])
except:
extract = 0
host = cfg["host"]
port = cfg["port"]
apikey = cfg["apikey"]
wait_for = int(cfg["wait_for"])
ssl = int(cfg.get("ssl", 0))
web_root = cfg.get("web_root", "")
remote_path = int(cfg.get("remote_path", 0))
extract = int(cfg.get("extract", 0))
protocol = "https://" if ssl else "http://"
if ssl:
protocol = "https://"
else:
protocol = "http://"
url = "%s%s:%s%s/api" % (protocol,host,port,web_root)
url = "{0}{1}:{2}{3}/api".format(protocol, host, port, web_root)
if not server_responding(url):
logger.error("Server did not respond. Exiting", section)
return [1, "%s: Failed to post-process - %s did not respond." % (section, section) ]
return [1, "{0}: Failed to post-process - {1} did not respond.".format(section, section)]
if not os.path.isdir(dirName) and os.path.isfile(dirName): # If the input directory is a file, assume single file download and split dir/name.
dirName = os.path.split(os.path.normpath(dirName))[0]
@ -83,63 +73,61 @@ class autoProcessMusic:
inputName, dirName = convert_to_ascii(inputName, dirName)
if not listMediaFiles(dirName, media=False, audio=True, meta=False, archives=False) and listMediaFiles(dirName, media=False, audio=False, meta=False, archives=True) and extract:
logger.debug('Checking for archives to extract in directory: %s' % (dirName))
logger.debug('Checking for archives to extract in directory: {0}'.format(dirName))
core.extractFiles(dirName)
inputName, dirName = convert_to_ascii(inputName, dirName)
if listMediaFiles(dirName, media=False, audio=True, meta=False, archives=False) and status:
logger.info("Status shown as failed from Downloader, but %s valid video files found. Setting as successful." % (str(good_files)), section)
logger.info("Status shown as failed from Downloader, but valid video files found. Setting as successful.", section)
status = 0
if status == 0:
params = {}
params['apikey'] = apikey
params['cmd'] = "forceProcess"
params['dir'] = os.path.dirname(dirName)
if remote_path:
params['dir'] = remoteDir(os.path.dirname(dirName))
params = {
'apikey': apikey,
'cmd': "forceProcess",
'dir': remoteDir(os.path.dirname(dirName)) if remote_path else os.path.dirname(dirName)
}
release_status = self.get_status(url, apikey, dirName)
if not release_status:
logger.error("Could not find a status for %s, is it in the wanted list ?" % (inputName),section)
logger.error("Could not find a status for {0}, is it in the wanted list ?".format(inputName), section)
logger.debug("Opening URL: %s with PARAMS: %s" % (url, params), section)
logger.debug("Opening URL: {0} with PARAMS: {1}".format(url, params), section)
try:
r = requests.get(url, params=params, verify=False, timeout=(30, 300))
except requests.ConnectionError:
logger.error("Unable to open URL %s" % (url) ,section)
return [1, "%s: Failed to post-process - Unable to connect to %s" % (section, section) ]
logger.error("Unable to open URL {0}".format(url), section)
return [1, "{0}: Failed to post-process - Unable to connect to {1}".format(section, section)]
logger.debug("Result: %s" % (r.text),section)
logger.debug("Result: {0}".format(r.text), section)
if not r.status_code in [requests.codes.ok, requests.codes.created, requests.codes.accepted]:
logger.error("Server returned status %s" % (str(r.status_code)), section)
return [1, "%s: Failed to post-process - Server returned status %s" % (section, str(r.status_code)) ]
if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]:
logger.error("Server returned status {0}".format(r.status_code), section)
return [1, "{0}: Failed to post-process - Server returned status {1}".format(section, r.status_code)]
elif r.text == "OK":
logger.postprocess("SUCCESS: Post-Processing started for %s in folder %s ..." % (inputName, dirName),section)
logger.postprocess("SUCCESS: Post-Processing started for {0} in folder {1} ...".format(inputName, dirName), section)
else:
logger.error("FAILED: Post-Processing has NOT started for %s in folder %s. exiting!" % (inputName, dirName),section)
return [1, "%s: Failed to post-process - Returned log from %s was not as expected." % (section, section) ]
logger.error("FAILED: Post-Processing has NOT started for {0} in folder {1}. exiting!".format(inputName, dirName), section)
return [1, "{0}: Failed to post-process - Returned log from {1} was not as expected.".format(section, section)]
else:
logger.warning("FAILED DOWNLOAD DETECTED", section)
return [1, "%s: Failed to post-process. %s does not support failed downloads" % (section, section) ]
return [1, "{0}: Failed to post-process. {1} does not support failed downloads".format(section, section)]
# we will now wait for this album to be processed before returning to TorrentToMedia and unpausing.
timeout = time.time() + 60 * wait_for
while (time.time() < timeout):
while time.time() < timeout:
current_status = self.get_status(url, apikey, dirName)
if current_status is not None and current_status != release_status: # Something has changed. CPS must have processed this movie.
logger.postprocess("SUCCESS: This release is now marked as status [%s]" % (current_status),section)
return [0, "%s: Successfully post-processed %s" % (section, inputName) ]
logger.postprocess("SUCCESS: This release is now marked as status [{0}]".format(current_status), section)
return [0, "{0}: Successfully post-processed {1}".format(section, inputName)]
if not os.path.isdir(dirName):
logger.postprocess("SUCCESS: The input directory %s has been removed Processing must have finished." % (dirName),section)
return [0, "%s: Successfully post-processed %s" % (section, inputName) ]
logger.postprocess("SUCCESS: The input directory {0} has been removed Processing must have finished.".format(dirName), section)
return [0, "{0}: Successfully post-processed {1}".format(section, inputName)]
time.sleep(10 * wait_for)
# The status hasn't changed. uTorrent can resume seeding now.
logger.warning("The music album does not appear to have changed status after %s minutes. Please check your Logs" % (wait_for),section)
return [1, "%s: Failed to post-process - No change in wanted status" % (section) ]
logger.warning("The music album does not appear to have changed status after {0} minutes. Please check your Logs".format(wait_for), section)
return [1, "{0}: Failed to post-process - No change in wanted status".format(section)]

229
core/autoProcess/autoProcessTV.py
View file

@ -1,3 +1,5 @@
# coding=utf-8
import copy
import os
import time
@ -14,106 +16,70 @@ from core.transcoder import transcoder
requests.packages.urllib3.disable_warnings()
class autoProcessTV:
class autoProcessTV(object):
def command_complete(self, url, params, headers, section):
r = None
try:
r = requests.get(url, params=params, headers=headers, stream=True, verify=False, timeout=(30, 60))
except requests.ConnectionError:
logger.error("Unable to open URL: %s" % (url1), section)
logger.error("Unable to open URL: {0}".format(url), section)
return None
if not r.status_code in [requests.codes.ok, requests.codes.created, requests.codes.accepted]:
logger.error("Server returned status %s" % (str(r.status_code)), section)
if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]:
logger.error("Server returned status {0}".format(r.status_code), section)
return None
else:
try:
res = json.loads(r.content)
return res['state']
except:
logger.error("%s did not return expected json data." % section, section)
return r.json()['state']
except (ValueError, KeyError):
# ValueError catches simplejson's JSONDecodeError and json's ValueError
logger.error("{0} did not return expected json data.".format(section), section)
return None
def CDH(self, url2, headers):
r = None
def CDH(self, url2, headers, section="MAIN"):
try:
r = requests.get(url2, params={}, headers=headers, stream=True, verify=False, timeout=(30, 60))
except requests.ConnectionError:
logger.error("Unable to open URL: %s" % (url2), section)
logger.error("Unable to open URL: {0}".format(url2), section)
return False
if not r.status_code in [requests.codes.ok, requests.codes.created, requests.codes.accepted]:
logger.error("Server returned status %s" % (str(r.status_code)), section)
if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]:
logger.error("Server returned status {0}".format(r.status_code), section)
return False
else:
try:
res = json.loads(r.content)
return res["enableCompletedDownloadHandling"]
except:
return r.json().get("enableCompletedDownloadHandling", False)
except ValueError:
# ValueError catches simplejson's JSONDecodeError and json's ValueError
return False
def processEpisode(self, section, dirName, inputName=None, failed=False, clientAgent = "manual", download_id=None, inputCategory=None, failureLink=None):
host = core.CFG[section][inputCategory]["host"]
port = core.CFG[section][inputCategory]["port"]
try:
ssl = int(core.CFG[section][inputCategory]["ssl"])
except:
ssl = 0
if ssl:
protocol = "https://"
else:
protocol = "http://"
try:
web_root = core.CFG[section][inputCategory]["web_root"]
except:
web_root = ""
if not server_responding("%s%s:%s%s" % (protocol,host,port,web_root)):
def processEpisode(self, section, dirName, inputName=None, failed=False, clientAgent="manual", download_id=None, inputCategory=None, failureLink=None):
cfg = dict(core.CFG[section][inputCategory])
host = cfg["host"]
port = cfg["port"]
ssl = int(cfg.get("ssl", 0))
web_root = cfg.get("web_root", "")
protocol = "https://" if ssl else "http://"
if not server_responding("{0}{1}:{2}{3}".format(protocol, host, port, web_root)):
logger.error("Server did not respond. Exiting", section)
return [1, "%s: Failed to post-process - %s did not respond." % (section, section) ]
return [1, "{0}: Failed to post-process - {1} did not respond.".format(section, section)]
# auto-detect correct fork
fork, fork_params = autoFork(section, inputCategory)
try:
username = core.CFG[section][inputCategory]["username"]
password = core.CFG[section][inputCategory]["password"]
except:
username = ""
password = ""
try:
apikey = core.CFG[section][inputCategory]["apikey"]
except:
apikey = ""
try:
delete_failed = int(core.CFG[section][inputCategory]["delete_failed"])
except:
delete_failed = 0
try:
nzbExtractionBy = core.CFG[section][inputCategory]["nzbExtractionBy"]
except:
nzbExtractionBy = "Downloader"
try:
process_method = core.CFG[section][inputCategory]["process_method"]
except:
process_method = None
try:
remote_path = int(core.CFG[section][inputCategory]["remote_path"])
except:
remote_path = 0
try:
wait_for = int(core.CFG[section][inputCategory]["wait_for"])
except:
wait_for = 2
try:
force = int(core.CFG[section][inputCategory]["force"])
except:
force = 0
try:
delete_on = int(core.CFG[section][inputCategory]["delete_on"])
except:
delete_on = 0
try:
extract = int(section[inputCategory]["extract"])
except:
extract = 0
username = cfg.get("username", "")
password = cfg.get("password", "")
apikey = cfg.get("apikey", "")
delete_failed = int(cfg.get("delete_failed", 0))
nzbExtractionBy = cfg.get("nzbExtractionBy", "Downloader")
process_method = cfg.get("process_method")
remote_path = int(cfg.get("remote_path", 0))
wait_for = int(cfg.get("wait_for", 2))
force = int(cfg.get("force", 0))
delete_on = int(cfg.get("delete_on", 0))
ignore_subs = int(cfg.get("ignore_subs", 0))
extract = int(cfg.get("extract", 0))
if not os.path.isdir(dirName) and os.path.isfile(dirName): # If the input directory is a file, assume single file download and split dir/name.
dirName = os.path.split(os.path.normpath(dirName))[0]
@ -130,12 +96,12 @@ class autoProcessTV:
# won't process the directory because it doesn't exist.
try:
os.makedirs(dirName) # Attempt to create the directory
except OSError, e:
except OSError as e:
# Re-raise the error if it wasn't about the directory not existing
if e.errno != errno.EEXIST:
raise
if not 'process_method' in fork_params or (clientAgent in ['nzbget','sabnzbd'] and nzbExtractionBy != "Destination"):
if 'process_method' not in fork_params or (clientAgent in ['nzbget', 'sabnzbd'] and nzbExtractionBy != "Destination"):
if inputName:
process_all_exceptions(inputName, dirName)
inputName, dirName = convert_to_ascii(inputName, dirName)
@ -143,7 +109,7 @@ class autoProcessTV:
# Now check if tv files exist in destination.
if not listMediaFiles(dirName, media=True, audio=False, meta=False, archives=False):
if listMediaFiles(dirName, media=False, audio=False, meta=False, archives=True) and extract:
logger.debug('Checking for archives to extract in directory: %s' % (dirName))
logger.debug('Checking for archives to extract in directory: {0}'.format(dirName))
core.extractFiles(dirName)
inputName, dirName = convert_to_ascii(inputName, dirName)
@ -168,12 +134,12 @@ class autoProcessTV:
logger.info('Found corrupt videos. Setting status Failed')
status = 1
failed = 1
if os.environ.has_key('NZBOP_VERSION') and os.environ['NZBOP_VERSION'][0:5] >= '14.0':
if 'NZBOP_VERSION' in os.environ and os.environ['NZBOP_VERSION'][0:5] >= '14.0':
print('[NZB] MARK=BAD')
if failureLink:
failureLink = failureLink + '&corrupt=true'
failureLink += '&corrupt=true'
elif clientAgent == "manual":
logger.warning("No media files found in directory %s to manually process." % (dirName), section)
logger.warning("No media files found in directory {0} to manually process.".format(dirName), section)
return [0, ""] # Success (as far as this script is concerned)
elif nzbExtractionBy == "Destination":
logger.info("Check for media files ignored because nzbExtractionBy is set to Destination.")
@ -186,23 +152,30 @@ class autoProcessTV:
status = 1
failed = 1
else:
logger.warning("No media files found in directory %s. Processing this as a failed download" % (dirName), section)
logger.warning("No media files found in directory {0}. Processing this as a failed download".format(dirName), section)
status = 1
failed = 1
if os.environ.has_key('NZBOP_VERSION') and os.environ['NZBOP_VERSION'][0:5] >= '14.0':
if 'NZBOP_VERSION' in os.environ and os.environ['NZBOP_VERSION'][0:5] >= '14.0':
print('[NZB] MARK=BAD')
if status == 0 and core.TRANSCODE == 1: # only transcode successful downloads
result, newDirName = transcoder.Transcode_directory(dirName)
if result == 0:
logger.debug("SUCCESS: Transcoding succeeded for files in %s" % (dirName), section)
logger.debug("SUCCESS: Transcoding succeeded for files in {0}".format(dirName), section)
dirName = newDirName
chmod_directory = int(str(cfg.get("chmodDirectory", "0")), 8)
logger.debug("Config setting 'chmodDirectory' currently set to {0}".format(oct(chmod_directory)), section)
if chmod_directory:
logger.info("Attempting to set the octal permission of '{0}' on directory '{1}'".format(oct(chmod_directory), dirName), section)
core.rchmod(dirName, chmod_directory)
else:
logger.error("FAILED: Transcoding failed for files in %s" % (dirName), section)
return [1, "%s: Failed to post-process - Transcoding failed" % (section) ]
logger.error("FAILED: Transcoding failed for files in {0}".format(dirName), section)
return [1, "{0}: Failed to post-process - Transcoding failed".format(section)]
# configure SB params to pass
fork_params['quiet'] = 1
fork_params['proc_type'] = 'manual'
if inputName is not None:
fork_params['nzbName'] = inputName
@ -233,8 +206,14 @@ class autoProcessTV:
else:
del fork_params[param]
if param == "ignore_subs":
if ignore_subs:
fork_params[param] = ignore_subs
else:
del fork_params[param]
# delete any unused params so we don't pass them to SB by mistake
[fork_params.pop(k) for k,v in fork_params.items() if v is None]
[fork_params.pop(k) for k, v in fork_params.items() if v is None]
if status == 0:
logger.postprocess("SUCCESS: The download succeeded, sending a post-process request", section)
@ -243,30 +222,30 @@ class autoProcessTV:
if failureLink:
reportNzb(failureLink, clientAgent)
if 'failed' in fork_params:
logger.postprocess("FAILED: The download failed. Sending 'failed' process request to %s branch" % (fork), section)
logger.postprocess("FAILED: The download failed. Sending 'failed' process request to {0} branch".format(fork), section)
elif section == "NzbDrone":
logger.postprocess("FAILED: The download failed. Sending failed download to %s for CDH processing" % (fork), section)
return [1, "%s: Downlaod Failed. Sending back to %s" % (section, section) ] # Return as failed to flag this in the downloader.
logger.postprocess("FAILED: The download failed. Sending failed download to {0} for CDH processing".format(fork), section)
return [1, "{0}: Download Failed. Sending back to {1}".format(section, section)] # Return as failed to flag this in the downloader.
else:
logger.postprocess("FAILED: The download failed. %s branch does not handle failed downloads. Nothing to process" % (fork), section)
logger.postprocess("FAILED: The download failed. {0} branch does not handle failed downloads. Nothing to process".format(fork), section)
if delete_failed and os.path.isdir(dirName) and not os.path.dirname(dirName) == dirName:
logger.postprocess("Deleting failed files and folder %s" % (dirName), section)
logger.postprocess("Deleting failed files and folder {0}".format(dirName), section)
rmDir(dirName)
return [1, "%s: Failed to post-process. %s does not support failed downloads" % (section, section) ] # Return as failed to flag this in the downloader.
return [1, "{0}: Failed to post-process. {1} does not support failed downloads".format(section, section)] # Return as failed to flag this in the downloader.
url = None
if section == "SickBeard":
url = "%s%s:%s%s/home/postprocess/processEpisode" % (protocol,host,port,web_root)
url = "{0}{1}:{2}{3}/home/postprocess/processEpisode".format(protocol, host, port, web_root)
elif section == "NzbDrone":
url = "%s%s:%s%s/api/command" % (protocol, host, port, web_root)
url2 = "%s%s:%s%s/api/config/downloadClient" % (protocol, host, port, web_root)
url = "{0}{1}:{2}{3}/api/command".format(protocol, host, port, web_root)
url2 = "{0}{1}:{2}{3}/api/config/downloadClient".format(protocol, host, port, web_root)
headers = {"X-Api-Key": apikey}
params = {'sortKey': 'series.title', 'page': 1, 'pageSize': 1, 'sortDir': 'asc'}
# params = {'sortKey': 'series.title', 'page': 1, 'pageSize': 1, 'sortDir': 'asc'}
if remote_path:
logger.debug("remote_path: %s" % (remoteDir(dirName)),section)
logger.debug("remote_path: {0}".format(remoteDir(dirName)), section)
data = {"name": "DownloadedEpisodesScan", "path": remoteDir(dirName), "downloadClientId": download_id}
else:
logger.debug("path: %s" % (dirName),section)
logger.debug("path: {0}".format(dirName), section)
data = {"name": "DownloadedEpisodesScan", "path": dirName, "downloadClientId": download_id}
if not download_id:
data.pop("downloadClientId")
@ -274,31 +253,29 @@ class autoProcessTV:
try:
if section == "SickBeard":
logger.debug("Opening URL: %s with params: %s" % (url, str(fork_params)), section)
r = None
logger.debug("Opening URL: {0} with params: {1}".format(url, fork_params), section)
s = requests.Session()
login = "%s%s:%s%s/login" % (protocol,host,port,web_root)
login = "{0}{1}:{2}{3}/login".format(protocol, host, port, web_root)
login_params = {'username': username, 'password': password}
s.post(login, data=login_params, stream=True, verify=False, timeout=(30, 60))
r = s.get(url, auth=(username, password), params=fork_params, stream=True, verify=False, timeout=(30, 1800))
elif section == "NzbDrone":
logger.debug("Opening URL: %s with data: %s" % (url, str(data)), section)
r = None
logger.debug("Opening URL: {0} with data: {1}".format(url, data), section)
r = requests.post(url, data=data, headers=headers, stream=True, verify=False, timeout=(30, 1800))
except requests.ConnectionError:
logger.error("Unable to open URL: %s" % (url), section)
return [1, "%s: Failed to post-process - Unable to connect to %s" % (section, section) ]
logger.error("Unable to open URL: {0}".format(url), section)
return [1, "{0}: Failed to post-process - Unable to connect to {1}".format(section, section)]
if not r.status_code in [requests.codes.ok, requests.codes.created, requests.codes.accepted]:
logger.error("Server returned status %s" % (str(r.status_code)), section)
return [1, "%s: Failed to post-process - Server returned status %s" % (section, str(r.status_code)) ]
if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]:
logger.error("Server returned status {0}".format(r.status_code), section)
return [1, "{0}: Failed to post-process - Server returned status {1}".format(section, r.status_code)]
Success = False
Started = False
if section == "SickBeard":
for line in r.iter_lines():
if line:
logger.postprocess("%s" % (line), section)
logger.postprocess("{0}".format(line), section)
if "Moving file from" in line:
inputName = os.path.split(line)[1]
if "Processing succeeded" in line or "Successfully processed" in line:
@ -307,23 +284,23 @@ class autoProcessTV:
try:
res = json.loads(r.content)
scan_id = int(res['id'])
logger.debug("Scan started with id: %s" % (str(scan_id)), section)
logger.debug("Scan started with id: {0}".format(scan_id), section)
Started = True
except Exception as e:
logger.warning("No scan id was returned due to: %s" % (e), section)
logger.warning("No scan id was returned due to: {0}".format(e), section)
scan_id = None
Started = False
if status != 0 and delete_failed and not os.path.dirname(dirName) == dirName:
logger.postprocess("Deleting failed files and folder %s" % (dirName),section)
logger.postprocess("Deleting failed files and folder {0}".format(dirName), section)
rmDir(dirName)
if Success:
return [0, "%s: Successfully post-processed %s" % (section, inputName) ]
return [0, "{0}: Successfully post-processed {1}".format(section, inputName)]
elif section == "NzbDrone" and Started:
n = 0
params = {}
url = url + "/" + str(scan_id)
url = "{0}/{1}".format(url, scan_id)
while n < 6: # set up wait_for minutes to see if command completes..
time.sleep(10 * wait_for)
command_status = self.command_complete(url, params, headers, section)
@ -331,21 +308,21 @@ class autoProcessTV:
break
n += 1
if command_status:
logger.debug("The Scan command return status: %s" % (command_status), section)
logger.debug("The Scan command return status: {0}".format(command_status), section)
if not os.path.exists(dirName):
logger.debug("The directory %s has been removed. Renaming was successful." % (dirName), section)
return [0, "%s: Successfully post-processed %s" % (section, inputName) ]
logger.debug("The directory {0} has been removed. Renaming was successful.".format(dirName), section)
return [0, "{0}: Successfully post-processed {1}".format(section, inputName)]
elif command_status and command_status in ['completed']:
logger.debug("The Scan command has completed successfully. Renaming was successful.", section)
return [0, "%s: Successfully post-processed %s" % (section, inputName) ]
return [0, "{0}: Successfully post-processed {1}".format(section, inputName)]
elif command_status and command_status in ['failed']:
logger.debug("The Scan command has failed. Renaming was not successful.", section)
#return [1, "%s: Failed to post-process %s" % (section, inputName) ]
if self.CDH(url2, headers):
logger.debug("The Scan command did not return status completed, but complete Download Handling is enabled. Passing back to %s." % (section), section)
return [status, "%s: Complete DownLoad Handling is enabled. Passing back to %s" % (section, section) ]
# return [1, "%s: Failed to post-process %s" % (section, inputName) ]
if self.CDH(url2, headers, section=section):
logger.debug("The Scan command did not return status completed, but complete Download Handling is enabled. Passing back to {0}.".format(section), section)
return [status, "{0}: Complete DownLoad Handling is enabled. Passing back to {1}".format(section, section)]
else:
logger.warning("The Scan command did not return a valid status. Renaming was not successful.", section)
return [1, "%s: Failed to post-process %s" % (section, inputName) ]
return [1, "{0}: Failed to post-process {1}".format(section, inputName)]
else:
return [1, "%s: Failed to post-process - Returned log from %s was not as expected." % (section, section) ] # We did not receive Success confirmation.
return [1, "{0}: Failed to post-process - Returned log from {1} was not as expected.".format(section, section)] # We did not receive Success confirmation.

1
core/databases/__init__.py
View file

@ -1 +1,2 @@
# coding=utf-8
__all__ = ["mainDB"]

24
core/databases/mainDB.py
View file

@ -1,10 +1,12 @@
import core
# coding=utf-8
from core import logger, nzbToMediaDB
from core.nzbToMediaUtil import backupVersionedFile
MIN_DB_VERSION = 1 # oldest db version we support migrating from
MAX_DB_VERSION = 2
def backupDatabase(version):
logger.info("Backing up database before upgrade")
if not backupVersionedFile(nzbToMediaDB.dbFilename(), version):
@ -12,6 +14,7 @@ def backupDatabase(version):
else:
logger.info("Proceeding with upgrade")
# ======================
# = Main DB Migrations =
# ======================
@ -39,18 +42,17 @@ class InitialSchema(nzbToMediaDB.SchemaUpgrade):
cur_db_version = self.checkDBVersion()
if cur_db_version < MIN_DB_VERSION:
logger.log_error_and_exit("Your database version (" + str(
cur_db_version) + ") is too old to migrate from what this version of nzbToMedia supports (" + \
str(MIN_DB_VERSION) + ").\n" + \
"Please remove nzbtomedia.db file to begin fresh."
)
logger.log_error_and_exit(u"Your database version ({current}) is too old to migrate "
u"from what this version of nzbToMedia supports ({min})."
u"\nPlease remove nzbtomedia.db file to begin fresh.".format
(current=cur_db_version, min=MIN_DB_VERSION))
if cur_db_version > MAX_DB_VERSION:
logger.log_error_and_exit("Your database version (" + str(
cur_db_version) + ") has been incremented past what this version of nzbToMedia supports (" + \
str(MAX_DB_VERSION) + ").\n" + \
"If you have used other forks of nzbToMedia, your database may be unusable due to their modifications."
)
logger.log_error_and_exit(u"Your database version ({current}) has been incremented "
u"past what this version of nzbToMedia supports ({max})."
u"\nIf you have used other forks of nzbToMedia, your database "
u"may be unusable due to their modifications.".format
(current=cur_db_version, max=MAX_DB_VERSION))
if cur_db_version < MAX_DB_VERSION: # We need to upgrade.
queries = [
"CREATE TABLE downloads2 (input_directory TEXT, input_name TEXT, input_hash TEXT, input_id TEXT, client_agent TEXT, status INTEGER, last_update NUMERIC, CONSTRAINT pk_downloadID PRIMARY KEY (input_directory, input_name));",

1
core/extractor/__init__.py
View file

@ -0,0 +1 @@
# coding=utf-8

65
core/extractor/extractor.py
View file

@ -1,3 +1,5 @@
# coding=utf-8
import os
import platform
import shutil
@ -7,6 +9,7 @@ import core
from subprocess import call, Popen
import subprocess
def extract(filePath, outputDestination):
success = 0
# Using Windows
@ -21,9 +24,9 @@ def extract(filePath, outputDestination):
# Using unix
else:
required_cmds = ["unrar", "unzip", "tar", "unxz", "unlzma", "7zr", "bunzip2"]
## Possible future suport:
# ## Possible future suport:
# gunzip: gz (cmd will delete original archive)
## the following do not extract to dest dir
# ## the following do not extract to dest dir
# ".xz": ["xz", "-d --keep"],
# ".lzma": ["xz", "-d --format=lzma --keep"],
# ".bz2": ["bzip2", "-d --keep"],
@ -42,15 +45,20 @@ def extract(filePath, outputDestination):
if not os.getenv('TR_TORRENT_DIR'):
devnull = open(os.devnull, 'w')
for cmd in required_cmds:
if call(['which', cmd], stdout=devnull, stderr=devnull): #note, returns 0 if exists, or 1 if doesn't exist.
if cmd == "7zr" and not call(["which", "7z"]): # we do have "7z" command
EXTRACT_COMMANDS[".7z"] = ["7z", "x"]
elif cmd == "7zr" and not call(["which", "7za"]): # we do have "7za" command
EXTRACT_COMMANDS[".7z"] = ["7za", "x"]
else:
if call(['which', cmd], stdout=devnull,
stderr=devnull): # note, returns 0 if exists, or 1 if doesn't exist.
for k, v in EXTRACT_COMMANDS.items():
if cmd in v[0]:
core.logger.error("EXTRACTOR: %s not found, disabling support for %s" % (cmd, k))
if not call(["which", "7zr"], stdout=devnull, stderr=devnull): # we do have "7zr"
EXTRACT_COMMANDS[k] = ["7zr", "x", "-y"]
elif not call(["which", "7z"], stdout=devnull, stderr=devnull): # we do have "7z"
EXTRACT_COMMANDS[k] = ["7z", "x", "-y"]
elif not call(["which", "7za"], stdout=devnull, stderr=devnull): # we do have "7za"
EXTRACT_COMMANDS[k] = ["7za", "x", "-y"]
else:
core.logger.error("EXTRACTOR: {cmd} not found, "
"disabling support for {feature}".format
(cmd=cmd, feature=k))
del EXTRACT_COMMANDS[k]
devnull.close()
else:
@ -64,7 +72,7 @@ def extract(filePath, outputDestination):
if ext[1] in (".gz", ".bz2", ".lzma"):
# Check if this is a tar
if os.path.splitext(ext[0])[1] == ".tar":
cmd = EXTRACT_COMMANDS[".tar" + ext[1]]
cmd = EXTRACT_COMMANDS[".tar{ext}".format(ext=ext[1])]
elif ext[1] in (".1", ".01", ".001") and os.path.splitext(ext[0])[1] in (".rar", ".zip", ".7z"):
cmd = EXTRACT_COMMANDS[os.path.splitext(ext[0])[1]]
elif ext[1] in (".cb7", ".cba", ".cbr", ".cbt", ".cbz"): # don't extract these comic book archives.
@ -73,7 +81,8 @@ def extract(filePath, outputDestination):
if ext[1] in EXTRACT_COMMANDS:
cmd = EXTRACT_COMMANDS[ext[1]]
else:
core.logger.debug("EXTRACTOR: Unknown file type: %s" % ext[1])
core.logger.debug("EXTRACTOR: Unknown file type: {ext}".format
(ext=ext[1]))
return False
# Create outputDestination folder
@ -84,8 +93,10 @@ def extract(filePath, outputDestination):
else:
passwords = []
core.logger.info("Extracting %s to %s" % (filePath, outputDestination))
core.logger.debug("Extracting %s %s %s" % (cmd, filePath, outputDestination))
core.logger.info("Extracting {file} to {destination}".format
(file=filePath, destination=outputDestination))
core.logger.debug("Extracting {cmd} {file} {destination}".format
(cmd=cmd, file=filePath, destination=outputDestination))
origFiles = []
origDirs = []
@ -112,7 +123,8 @@ def extract(filePath, outputDestination):
p = Popen(cmd2, stdout=devnull, stderr=devnull, startupinfo=info) # should extract files fine.
res = p.wait()
if (res >= 0 and os.name == 'nt') or res == 0: # for windows chp returns process id if successful or -1*Error code. Linux returns 0 for successful.
core.logger.info("EXTRACTOR: Extraction was successful for %s to %s" % (filePath, outputDestination))
core.logger.info("EXTRACTOR: Extraction was successful for {file} to {destination}".format
(file=filePath, destination=outputDestination))
success = 1
elif len(passwords) > 0:
core.logger.info("EXTRACTOR: Attempting to extract with passwords")
@ -120,20 +132,23 @@ def extract(filePath, outputDestination):
if password == "": # if edited in windows or otherwise if blank lines.
continue
cmd2 = cmd
#append password here.
passcmd = "-p" + password
# append password here.
passcmd = "-p{pwd}".format(pwd=password)
cmd2.append(passcmd)
p = Popen(cmd2, stdout=devnull, stderr=devnull, startupinfo=info) # should extract files fine.
res = p.wait()
if (res >= 0 and platform == 'Windows') or res == 0:
core.logger.info("EXTRACTOR: Extraction was successful for %s to %s using password: %s" % (
filePath, outputDestination, password))
core.logger.info("EXTRACTOR: Extraction was successful "
"for {file} to {destination} using password: {pwd}".format
(file=filePath, destination=outputDestination, pwd=password))
success = 1
break
else:
continue
except:
core.logger.error("EXTRACTOR: Extraction failed for %s. Could not call command %s" % (filePath, cmd))
core.logger.error("EXTRACTOR: Extraction failed for {file}. "
"Could not call command {cmd}".format
(file=filePath, cmd=cmd))
os.chdir(pwd)
return False
@ -141,20 +156,24 @@ def extract(filePath, outputDestination):
os.chdir(pwd) # Go back to our Original Working Directory
if success:
# sleep to let files finish writing to disk
sleep (3)
sleep(3)
perms = stat.S_IMODE(os.lstat(os.path.split(filePath)[0]).st_mode)
for dir, subdirs, files in os.walk(outputDestination):
for subdir in subdirs:
if not os.path.join(dir, subdir) in origFiles:
try:
os.chmod(os.path.join(dir, subdir), perms)
except: pass
except:
pass
for file in files:
if not os.path.join(dir, file) in origFiles:
try:
shutil.copymode(filePath, os.path.join(dir, file))
except: pass
except:
pass
return True
else:
core.logger.error("EXTRACTOR: Extraction failed for %s. Result was %s" % (filePath, res))
core.logger.error("EXTRACTOR: Extraction failed for {file}. "
"Result was {result}".format
(file=filePath, result=res))
return False

12
core/gh_api.py
View file

@ -1,5 +1,8 @@
import json
# coding=utf-8
import requests
from six import iteritems
class GitHub(object):
"""
@ -17,10 +20,11 @@ class GitHub(object):
Access the API at the path given and with the optional params given.
"""
url = 'https://api.github.com/' + '/'.join(path)
url = 'https://api.github.com/{path}'.format(path='/'.join(path))
if params and type(params) is dict:
url += '?' + '&'.join([str(x) + '=' + str(params[x]) for x in params.keys()])
url += '?{params}'.format(params='&'.join(['{key}={value}'.format(key=k, value=v)
for k, v in iteritems(params)]))
data = requests.get(url, verify=False)
@ -57,6 +61,6 @@ class GitHub(object):
Returns a deserialized json object containing the compare info. See http://developer.github.com/v3/repos/commits/
"""
access_API = self._access_API(
['repos', self.github_repo_user, self.github_repo, 'compare', base + '...' + head],
['repos', self.github_repo_user, self.github_repo, 'compare', '{base}...{head}'.format(base=base, head=head)],
params={'per_page': per_page})
return access_API

1
core/linktastic/__init__.py
View file

@ -0,0 +1 @@
# coding=utf-8

17
core/linktastic/linktastic.py
View file

@ -1,3 +1,4 @@
# coding=utf-8
# Linktastic Module
# - A python2/3 compatible module that can create hardlinks/symlinks on windows-based systems
#
@ -29,16 +30,17 @@ if os.name == 'nt':
info = subprocess.STARTUPINFO()
info.dwFlags |= subprocess.STARTF_USESHOWWINDOW
# Prevent spaces from messing with us!
def _escape_param(param):
return '"%s"' % param
return '"{0}"'.format(param)
# Private function to create link on nt-based systems
def _link_windows(src, dest):
try:
subprocess.check_output(
'cmd /C mklink /H %s %s' % (_escape_param(dest), _escape_param(src)),
'cmd /C mklink /H {0} {1}'.format(_escape_param(dest), _escape_param(src)),
stderr=subprocess.STDOUT, startupinfo=info)
except CalledProcessError as err:
@ -52,7 +54,7 @@ def _link_windows(src, dest):
def _symlink_windows(src, dest):
try:
subprocess.check_output(
'cmd /C mklink %s %s' % (_escape_param(dest), _escape_param(src)),
'cmd /C mklink {0} {1}'.format(_escape_param(dest), _escape_param(src)),
stderr=subprocess.STDOUT, startupinfo=info)
except CalledProcessError as err:
raise IOError(err.output.decode('utf-8'))
@ -61,10 +63,11 @@ def _symlink_windows(src, dest):
# print(stdout)
# assume if they ret-coded 0 we're good
def _dirlink_windows(src, dest):
try:
subprocess.check_output(
'cmd /C mklink /J %s %s' % (_escape_param(dest), _escape_param(src)),
'cmd /C mklink /J {0} {1}'.format(_escape_param(dest), _escape_param(src)),
stderr=subprocess.STDOUT, startupinfo=info)
except CalledProcessError as err:
raise IOError(err.output.decode('utf-8'))
@ -73,10 +76,11 @@ def _dirlink_windows(src, dest):
# print(stdout)
# assume if they ret-coded 0 we're good
def _junctionlink_windows(src, dest):
try:
subprocess.check_output(
'cmd /C mklink /D %s %s' % (_escape_param(dest), _escape_param(src)),
'cmd /C mklink /D {0} {1}'.format(_escape_param(dest), _escape_param(src)),
stderr=subprocess.STDOUT, startupinfo=info)
except CalledProcessError as err:
raise IOError(err.output.decode('utf-8'))
@ -85,6 +89,7 @@ def _junctionlink_windows(src, dest):
# print(stdout)
# assume if they ret-coded 0 we're good
# Create a hard link to src named as dest
# This version of link, unlike os.link, supports nt systems as well
def link(src, dest):
@ -101,6 +106,7 @@ def symlink(src, dest):
else:
os.symlink(src, dest)
# Create a symlink to src named as dest, but don't fail if you're on nt
def dirlink(src, dest):
if os.name == 'nt':
@ -108,6 +114,7 @@ def dirlink(src, dest):
else:
os.symlink(src, dest)
# Create a symlink to src named as dest, but don't fail if you're on nt
def junctionlink(src, dest):
if os.name == 'nt':

27
core/logger.py
View file

@ -1,3 +1,4 @@
# coding=utf-8
from __future__ import with_statement
import os
@ -26,6 +27,7 @@ reverseNames = {u'ERROR': ERROR,
u'POSTPROCESS': POSTPROCESS,
u'DB': DB}
class NTMRotatingLogHandler(object):
def __init__(self, log_file, num_files, num_bytes):
self.num_files = num_files
@ -67,7 +69,7 @@ class NTMRotatingLogHandler(object):
if self.cur_handler:
old_handler = self.cur_handler
else:
#Add a new logging levels
# Add a new logging levels
logging.addLevelName(21, 'POSTPROCESS')
logging.addLevelName(5, 'DB')
@ -134,7 +136,7 @@ class NTMRotatingLogHandler(object):
i: Log number to ues
"""
return self.log_file_path + ('.' + str(i) if i else '')
return self.log_file_path + ('.{0}'.format(i) if i else '')
def _num_logs(self):
"""
@ -191,9 +193,9 @@ class NTMRotatingLogHandler(object):
self.writes_since_check += 1
try:
message = u"%s: %s" % (str(section).upper(), toLog)
except:
message = u"%s: Message contains non-utf-8 string" % (str(section).upper())
message = u"{0}: {1}".format(section.upper(), toLog)
except UnicodeError:
message = u"{0}: Message contains non-utf-8 string".format(section.upper())
out_line = message
@ -226,14 +228,15 @@ class NTMRotatingLogHandler(object):
def log_error_and_exit(self, error_msg):
log(error_msg, ERROR)
if os.environ.has_key('NZBOP_SCRIPTDIR'):
if 'NZBOP_SCRIPTDIR' in os.environ:
sys.exit(core.NZBGET_POSTPROCESS_ERROR)
elif not self.console_logging:
sys.exit(error_msg.encode(core.SYS_ENCODING, 'xmlcharrefreplace'))
else:
sys.exit(1)
class DispatchingFormatter:
class DispatchingFormatter(object):
def __init__(self, formatters, default_formatter):
self._formatters = formatters
self._default_formatter = default_formatter
@ -242,31 +245,41 @@ class DispatchingFormatter:
formatter = self._formatters.get(record.name, self._default_formatter)
return formatter.format(record)
ntm_log_instance = NTMRotatingLogHandler(core.LOG_FILE, NUM_LOGS, LOG_SIZE)
def log(toLog, logLevel=MESSAGE, section='MAIN'):
ntm_log_instance.log(toLog, logLevel, section)
def info(toLog, section='MAIN'):
log(toLog, MESSAGE, section)
def error(toLog, section='MAIN'):
log(toLog, ERROR, section)
def warning(toLog, section='MAIN'):
log(toLog, WARNING, section)
def debug(toLog, section='MAIN'):
log(toLog, DEBUG, section)
def postprocess(toLog, section='POSTPROCESS'):
log(toLog, POSTPROCESS, section)
def db(toLog, section='DB'):
log(toLog, DB, section)
def log_error_and_exit(error_msg):
ntm_log_instance.log_error_and_exit(error_msg)
def close():
ntm_log_instance.close_log()

93
core/nzbToMediaAutoFork.py
View file

@ -1,103 +1,96 @@
import urllib
import core
# coding=utf-8
import requests
from six import iteritems
import core
from core import logger
def autoFork(section, inputCategory):
# auto-detect correct section
# config settings
try:
host = core.CFG[section][inputCategory]["host"]
port = core.CFG[section][inputCategory]["port"]
except:
host = None
port = None
try:
username = core.CFG[section][inputCategory]["username"]
password = core.CFG[section][inputCategory]["password"]
except:
username = None
password = None
cfg = dict(core.CFG[section][inputCategory])
host = cfg.get("host")
port = cfg.get("port")
username = cfg.get("username")
password = cfg.get("password")
apikey = cfg.get("apikey")
ssl = int(cfg.get("ssl", 0))
web_root = cfg.get("web_root", "")
try:
apikey = core.CFG[section][inputCategory]["apikey"]
except:
apikey = None
try:
ssl = int(core.CFG[section][inputCategory]["ssl"])
except:
ssl = 0
try:
web_root = core.CFG[section][inputCategory]["web_root"]
except:
web_root = ""
try:
fork = core.FORKS.items()[core.FORKS.keys().index(core.CFG[section][inputCategory]["fork"])]
fork = core.FORKS.items()[core.FORKS.keys().index(cfg.get("fork", "auto"))]
except:
fork = "auto"
if ssl:
protocol = "https://"
else:
protocol = "http://"
protocol = "https://" if ssl else "http://"
detected = False
if section == "NzbDrone":
logger.info("Attempting to verify %s fork" % inputCategory)
url = "%s%s:%s%s/api/rootfolder" % (protocol,host,port,web_root)
headers={"X-Api-Key": apikey}
logger.info("Attempting to verify {category} fork".format
(category=inputCategory))
url = "{protocol}{host}:{port}{root}/api/rootfolder".format(
protocol=protocol, host=host, port=port, root=web_root)
headers = {"X-Api-Key": apikey}
try:
r = requests.get(url, headers=headers, stream=True, verify=False)
except requests.ConnectionError:
logger.warning("Could not connect to %s:%s to verify fork!" % (section, inputCategory))
logger.warning("Could not connect to {0}:{1} to verify fork!".format(section, inputCategory))
if not r.ok:
logger.warning("Connection to %s:%s failed! Check your configuration" % (section, inputCategory))
logger.warning("Connection to {section}:{category} failed! "
"Check your configuration".format
(section=section, category=inputCategory))
fork = ['default', {}]
elif fork == "auto":
params = core.ALL_FORKS
rem_params = []
logger.info("Attempting to auto-detect %s fork" % inputCategory)
logger.info("Attempting to auto-detect {category} fork".format(category=inputCategory))
# define the order to test. Default must be first since the default fork doesn't reject parameters.
# then in order of most unique parameters.
url = "%s%s:%s%s/home/postprocess/" % (protocol,host,port,web_root)
url = "{protocol}{host}:{port}{root}/home/postprocess/".format(
protocol=protocol, host=host, port=port, root=web_root)
# attempting to auto-detect fork
try:
if username and password:
s = requests.Session()
login = "%s%s:%s%s/login" % (protocol,host,port,web_root)
login = "{protocol}{host}:{port}{root}/login".format(
protocol=protocol, host=host, port=port, root=web_root)
login_params = {'username': username, 'password': password}
s.post(login, data=login_params, stream=True, verify=False)
r = s.get(url, auth=(username, password), verify=False)
else:
r = requests.get(url, verify=False)
except requests.ConnectionError:
logger.info("Could not connect to %s:%s to perform auto-fork detection!" % (section, inputCategory))
logger.info("Could not connect to {section}:{category} to perform auto-fork detection!".format
(section=section, category=inputCategory))
r = []
if r and r.ok:
for param in params:
if not 'name="%s"' %(param) in r.text:
if not 'name="{param}"'.format(param=param) in r.text:
rem_params.append(param)
for param in rem_params:
params.pop(param)
for fork in sorted(core.FORKS.iteritems(), reverse=False):
for fork in sorted(iteritems(core.FORKS), reverse=False):
if params == fork[1]:
detected = True
break
if detected:
logger.info("%s:%s fork auto-detection successful ..." % (section, inputCategory))
logger.info("{section}:{category} fork auto-detection successful ...".format
(section=section, category=inputCategory))
elif rem_params:
logger.info("%s:%s fork auto-detection found custom params %s" % (section, inputCategory, params))
logger.info("{section}:{category} fork auto-detection found custom params {params}".format
(section=section, category=inputCategory, params=params))
fork = ['custom', params]
else:
logger.info("%s:%s fork auto-detection failed" % (section, inputCategory))
logger.info("{section}:{category} fork auto-detection failed".format
(section=section, category=inputCategory))
fork = core.FORKS.items()[core.FORKS.keys().index(core.FORK_DEFAULT)]
logger.info("%s:%s fork set to %s" % (section, inputCategory, fork[0]))
logger.info("{section}:{category} fork set to {fork}".format
(section=section, category=inputCategory, fork=fork[0]))
return fork[0], fork[1]

178
core/nzbToMediaConfig.py
View file

@ -1,3 +1,6 @@
# coding=utf-8
from six import iteritems
import os
import shutil
import copy
@ -7,13 +10,15 @@ from core import logger
from itertools import chain
class Section(configobj.Section):
class Section(configobj.Section, object):
def isenabled(section):
# checks if subsection enabled, returns true/false if subsection specified otherwise returns true/false in {}
if not section.sections:
try:
value = list(ConfigObj.find_key(section, 'enabled'))[0]
except:value = 0
except:
value = 0
if int(value) == 1:
return section
else:
@ -22,7 +27,8 @@ class Section(configobj.Section):
for subsection in subsections:
try:
value = list(ConfigObj.find_key(subsections, 'enabled'))[0]
except:value = 0
except:
value = 0
if int(value) != 1:
del to_return[section_name][subsection]
@ -38,7 +44,8 @@ class Section(configobj.Section):
for subsection in to_return:
try:
value = list(ConfigObj.find_key(to_return[subsection], key))[0]
except:value = None
except:
value = None
if not value:
del to_return[subsection]
@ -79,6 +86,7 @@ class Section(configobj.Section):
return to_return
class ConfigObj(configobj.ConfigObj, Section):
def __init__(self, *args, **kw):
if len(args) == 0:
@ -110,16 +118,16 @@ class ConfigObj(configobj.ConfigObj, Section):
if not os.path.isfile(core.CONFIG_FILE):
shutil.copyfile(core.CONFIG_SPEC_FILE, core.CONFIG_FILE)
CFG_OLD = config(core.CONFIG_FILE)
except Exception, e:
logger.debug("Error %s when copying to .cfg" % (e))
except Exception as error:
logger.debug("Error {msg} when copying to .cfg".format(msg=error))
try:
# check for autoProcessMedia.cfg.spec and create if it does not exist
if not os.path.isfile(core.CONFIG_SPEC_FILE):
shutil.copyfile(core.CONFIG_FILE, core.CONFIG_SPEC_FILE)
CFG_NEW = config(core.CONFIG_SPEC_FILE)
except Exception, e:
logger.debug("Error %s when copying to .spec" % (e))
except Exception as error:
logger.debug("Error {msg} when copying to .spec".format(msg=error))
# check for autoProcessMedia.cfg and autoProcessMedia.cfg.spec and if they don't exist return and fail
if CFG_NEW is None or CFG_OLD is None:
@ -144,7 +152,7 @@ class ConfigObj(configobj.ConfigObj, Section):
continue
def cleanup_values(values, section):
for option, value in values.iteritems():
for option, value in iteritems(values):
if section in ['CouchPotato']:
if option == ['outputDirectory']:
CFG_NEW['Torrent'][option] = os.path.split(os.path.normpath(value))[0]
@ -180,7 +188,7 @@ class ConfigObj(configobj.ConfigObj, Section):
CFG_NEW['Posix'][option] = value
values.pop(option)
if option == "remote_path":
if value and not value in ['0', '1', 0, 1]:
if value and value not in ['0', '1', 0, 1]:
value = 1
elif not value:
value = 0
@ -189,7 +197,8 @@ class ConfigObj(configobj.ConfigObj, Section):
if not list(ConfigObj.find_key(CFG_NEW, option)):
try:
values.pop(option)
except: pass
except:
pass
return values
@ -220,7 +229,7 @@ class ConfigObj(configobj.ConfigObj, Section):
subsection = None
if section in list(chain.from_iterable(subsections.values())):
subsection = section
section = ''.join([k for k,v in subsections.iteritems() if subsection in v])
section = ''.join([k for k, v in iteritems(subsections) if subsection in v])
process_section(section, subsection)
elif section in subsections.keys():
subsection = subsections[section]
@ -229,7 +238,7 @@ class ConfigObj(configobj.ConfigObj, Section):
process_section(section, subsection)
# create a backup of our old config
CFG_OLD.filename = core.CONFIG_FILE + ".old"
CFG_OLD.filename ="{config}.old".format(config=core.CONFIG_FILE)
CFG_OLD.write()
# write our new config to autoProcessMedia.cfg
@ -244,13 +253,15 @@ class ConfigObj(configobj.ConfigObj, Section):
CFG_NEW = config()
try:
if os.environ.has_key('NZBPO_NDCATEGORY') and os.environ.has_key('NZBPO_SBCATEGORY'):
if 'NZBPO_NDCATEGORY' in os.environ and 'NZBPO_SBCATEGORY' in os.environ:
if os.environ['NZBPO_NDCATEGORY'] == os.environ['NZBPO_SBCATEGORY']:
logger.warning("%s category is set for SickBeard and NzbDrone. Please check your config in NZBGet" % (os.environ['NZBPO_NDCATEGORY']))
logger.warning("{x} category is set for SickBeard and NzbDrone. "
"Please check your config in NZBGet".format
(x=os.environ['NZBPO_NDCATEGORY']))
section = "Nzb"
key = 'NZBOP_DESTDIR'
if os.environ.has_key(key):
if key in os.environ:
option = 'default_downloadDirectory'
value = os.environ[key]
CFG_NEW[section][option] = value
@ -259,8 +270,8 @@ class ConfigObj(configobj.ConfigObj, Section):
envKeys = ['AUTO_UPDATE', 'CHECK_MEDIA', 'SAFE_MODE']
cfgKeys = ['auto_update', 'check_media', 'safe_mode']
for index in range(len(envKeys)):
key = 'NZBPO_' + envKeys[index]
if os.environ.has_key(key):
key = 'NZBPO_{index}'.format(index=envKeys[index])
if key in os.environ:
option = cfgKeys[index]
value = os.environ[key]
CFG_NEW[section][option] = value
@ -269,20 +280,22 @@ class ConfigObj(configobj.ConfigObj, Section):
envKeys = ['MOUNTPOINTS']
cfgKeys = ['mount_points']
for index in range(len(envKeys)):
key = 'NZBPO_' + envKeys[index]
if os.environ.has_key(key):
key = 'NZBPO_{index}'.format(index=envKeys[index])
if key in os.environ:
option = cfgKeys[index]
value = os.environ[key]
CFG_NEW[section][option] = value
section = "CouchPotato"
envCatKey = 'NZBPO_CPSCATEGORY'
envKeys = ['ENABLED', 'APIKEY', 'HOST', 'PORT', 'SSL', 'WEB_ROOT', 'METHOD', 'DELETE_FAILED', 'REMOTE_PATH', 'WAIT_FOR', 'WATCH_DIR']
cfgKeys = ['enabled', 'apikey', 'host', 'port', 'ssl', 'web_root', 'method', 'delete_failed', 'remote_path', 'wait_for', 'watch_dir']
if os.environ.has_key(envCatKey):
envKeys = ['ENABLED', 'APIKEY', 'HOST', 'PORT', 'SSL', 'WEB_ROOT', 'METHOD', 'DELETE_FAILED', 'REMOTE_PATH',
'WAIT_FOR', 'WATCH_DIR']
cfgKeys = ['enabled', 'apikey', 'host', 'port', 'ssl', 'web_root', 'method', 'delete_failed', 'remote_path',
'wait_for', 'watch_dir']
if envCatKey in os.environ:
for index in range(len(envKeys)):
key = 'NZBPO_CPS' + envKeys[index]
if os.environ.has_key(key):
key = 'NZBPO_CPS{index}'.format(index=envKeys[index])
if key in os.environ:
option = cfgKeys[index]
value = os.environ[key]
if os.environ[envCatKey] not in CFG_NEW[section].sections:
@ -292,12 +305,14 @@ class ConfigObj(configobj.ConfigObj, Section):
section = "SickBeard"
envCatKey = 'NZBPO_SBCATEGORY'
envKeys = ['ENABLED', 'HOST', 'PORT', 'USERNAME', 'PASSWORD', 'SSL', 'WEB_ROOT', 'WATCH_DIR', 'FORK', 'DELETE_FAILED', 'TORRENT_NOLINK', 'NZBEXTRACTIONBY', 'REMOTE_PATH', 'PROCESS_METHOD']
cfgKeys = ['enabled', 'host', 'port', 'username', 'password', 'ssl', 'web_root', 'watch_dir', 'fork', 'delete_failed', 'Torrent_NoLink', 'nzbExtractionBy', 'remote_path', 'process_method']
if os.environ.has_key(envCatKey):
envKeys = ['ENABLED', 'HOST', 'PORT', 'USERNAME', 'PASSWORD', 'SSL', 'WEB_ROOT', 'WATCH_DIR', 'FORK',
'DELETE_FAILED', 'TORRENT_NOLINK', 'NZBEXTRACTIONBY', 'REMOTE_PATH', 'PROCESS_METHOD']
cfgKeys = ['enabled', 'host', 'port', 'username', 'password', 'ssl', 'web_root', 'watch_dir', 'fork',
'delete_failed', 'Torrent_NoLink', 'nzbExtractionBy', 'remote_path', 'process_method']
if envCatKey in os.environ:
for index in range(len(envKeys)):
key = 'NZBPO_SB' + envKeys[index]
if os.environ.has_key(key):
key = 'NZBPO_SB{index}'.format(index=envKeys[index])
if key in os.environ:
option = cfgKeys[index]
value = os.environ[key]
if os.environ[envCatKey] not in CFG_NEW[section].sections:
@ -311,10 +326,10 @@ class ConfigObj(configobj.ConfigObj, Section):
envCatKey = 'NZBPO_HPCATEGORY'
envKeys = ['ENABLED', 'APIKEY', 'HOST', 'PORT', 'SSL', 'WEB_ROOT', 'WAIT_FOR', 'WATCH_DIR', 'REMOTE_PATH']
cfgKeys = ['enabled', 'apikey', 'host', 'port', 'ssl', 'web_root', 'wait_for', 'watch_dir', 'remote_path']
if os.environ.has_key(envCatKey):
if envCatKey in os.environ:
for index in range(len(envKeys)):
key = 'NZBPO_HP' + envKeys[index]
if os.environ.has_key(key):
key = 'NZBPO_HP{index}'.format(index=envKeys[index])
if key in os.environ:
option = cfgKeys[index]
value = os.environ[key]
if os.environ[envCatKey] not in CFG_NEW[section].sections:
@ -324,12 +339,14 @@ class ConfigObj(configobj.ConfigObj, Section):
section = "Mylar"
envCatKey = 'NZBPO_MYCATEGORY'
envKeys = ['ENABLED', 'HOST', 'PORT', 'USERNAME', 'PASSWORD', 'APIKEY', 'SSL', 'WEB_ROOT', 'WATCH_DIR', 'REMOTE_PATH']
cfgKeys = ['enabled', 'host', 'port', 'username', 'password', 'apikey', 'ssl', 'web_root', 'watch_dir', 'remote_path']
if os.environ.has_key(envCatKey):
envKeys = ['ENABLED', 'HOST', 'PORT', 'USERNAME', 'PASSWORD', 'APIKEY', 'SSL', 'WEB_ROOT', 'WATCH_DIR',
'REMOTE_PATH']
cfgKeys = ['enabled', 'host', 'port', 'username', 'password', 'apikey', 'ssl', 'web_root', 'watch_dir',
'remote_path']
if envCatKey in os.environ:
for index in range(len(envKeys)):
key = 'NZBPO_MY' + envKeys[index]
if os.environ.has_key(key):
key = 'NZBPO_MY{index}'.format(index=envKeys[index])
if key in os.environ:
option = cfgKeys[index]
value = os.environ[key]
if os.environ[envCatKey] not in CFG_NEW[section].sections:
@ -341,10 +358,10 @@ class ConfigObj(configobj.ConfigObj, Section):
envCatKey = 'NZBPO_GZCATEGORY'
envKeys = ['ENABLED', 'APIKEY', 'HOST', 'PORT', 'SSL', 'WEB_ROOT', 'WATCH_DIR', 'LIBRARY', 'REMOTE_PATH']
cfgKeys = ['enabled', 'apikey', 'host', 'port', 'ssl', 'web_root', 'watch_dir', 'library', 'remote_path']
if os.environ.has_key(envCatKey):
if envCatKey in os.environ:
for index in range(len(envKeys)):
key = 'NZBPO_GZ' + envKeys[index]
if os.environ.has_key(key):
key = 'NZBPO_GZ{index}'.format(index=envKeys[index])
if key in os.environ:
option = cfgKeys[index]
value = os.environ[key]
if os.environ[envCatKey] not in CFG_NEW[section].sections:
@ -354,12 +371,14 @@ class ConfigObj(configobj.ConfigObj, Section):
section = "NzbDrone"
envCatKey = 'NZBPO_NDCATEGORY'
envKeys = ['ENABLED', 'HOST', 'APIKEY', 'PORT', 'SSL', 'WEB_ROOT', 'WATCH_DIR', 'FORK', 'DELETE_FAILED', 'TORRENT_NOLINK', 'NZBEXTRACTIONBY', 'WAIT_FOR', 'DELETE_FAILED', 'REMOTE_PATH']
cfgKeys = ['enabled', 'host', 'apikey', 'port', 'ssl', 'web_root', 'watch_dir', 'fork', 'delete_failed', 'Torrent_NoLink', 'nzbExtractionBy', 'wait_for', 'delete_failed', 'remote_path']
if os.environ.has_key(envCatKey):
envKeys = ['ENABLED', 'HOST', 'APIKEY', 'PORT', 'SSL', 'WEB_ROOT', 'WATCH_DIR', 'FORK', 'DELETE_FAILED',
'TORRENT_NOLINK', 'NZBEXTRACTIONBY', 'WAIT_FOR', 'DELETE_FAILED', 'REMOTE_PATH']
cfgKeys = ['enabled', 'host', 'apikey', 'port', 'ssl', 'web_root', 'watch_dir', 'fork', 'delete_failed',
'Torrent_NoLink', 'nzbExtractionBy', 'wait_for', 'delete_failed', 'remote_path']
if envCatKey in os.environ:
for index in range(len(envKeys)):
key = 'NZBPO_ND' + envKeys[index]
if os.environ.has_key(key):
key = 'NZBPO_ND{index}'.format(index=envKeys[index])
if key in os.environ:
option = cfgKeys[index]
value = os.environ[key]
if os.environ[envCatKey] not in CFG_NEW[section].sections:
@ -373,8 +392,8 @@ class ConfigObj(configobj.ConfigObj, Section):
envKeys = ['COMPRESSEDEXTENSIONS', 'MEDIAEXTENSIONS', 'METAEXTENSIONS']
cfgKeys = ['compressedExtensions', 'mediaExtensions', 'metaExtensions']
for index in range(len(envKeys)):
key = 'NZBPO_' + envKeys[index]
if os.environ.has_key(key):
key = 'NZBPO_{index}'.format(index=envKeys[index])
if key in os.environ:
option = cfgKeys[index]
value = os.environ[key]
CFG_NEW[section][option] = value
@ -383,26 +402,36 @@ class ConfigObj(configobj.ConfigObj, Section):
envKeys = ['NICENESS', 'IONICE_CLASS', 'IONICE_CLASSDATA']
cfgKeys = ['niceness', 'ionice_class', 'ionice_classdata']
for index in range(len(envKeys)):
key = 'NZBPO_' + envKeys[index]
if os.environ.has_key(key):
key = 'NZBPO_{index}'.format(index=envKeys[index])
if key in os.environ:
option = cfgKeys[index]
value = os.environ[key]
CFG_NEW[section][option] = value
section = "Transcoder"
envKeys = ['TRANSCODE', 'DUPLICATE', 'IGNOREEXTENSIONS', 'OUTPUTFASTSTART', 'OUTPUTVIDEOPATH', 'PROCESSOUTPUT', 'AUDIOLANGUAGE', 'ALLAUDIOLANGUAGES', 'SUBLANGUAGES',
'ALLSUBLANGUAGES', 'EMBEDSUBS', 'BURNINSUBTITLE', 'EXTRACTSUBS', 'EXTERNALSUBDIR', 'OUTPUTDEFAULT', 'OUTPUTVIDEOEXTENSION', 'OUTPUTVIDEOCODEC', 'VIDEOCODECALLOW',
'OUTPUTVIDEOPRESET', 'OUTPUTVIDEOFRAMERATE', 'OUTPUTVIDEOBITRATE', 'OUTPUTAUDIOCODEC', 'AUDIOCODECALLOW', 'OUTPUTAUDIOBITRATE', 'OUTPUTQUALITYPERCENT', 'GETSUBS',
'OUTPUTAUDIOTRACK2CODEC', 'AUDIOCODEC2ALLOW', 'OUTPUTAUDIOTRACK2BITRATE', 'OUTPUTAUDIOOTHERCODEC', 'AUDIOOTHERCODECALLOW', 'OUTPUTAUDIOOTHERBITRATE',
'OUTPUTSUBTITLECODEC', 'OUTPUTAUDIOCHANNELS', 'OUTPUTAUDIOTRACK2CHANNELS', 'OUTPUTAUDIOOTHERCHANNELS']
cfgKeys = ['transcode', 'duplicate', 'ignoreExtensions', 'outputFastStart', 'outputVideoPath', 'processOutput', 'audioLanguage', 'allAudioLanguages', 'subLanguages',
'allSubLanguages', 'embedSubs', 'burnInSubtitle', 'extractSubs', 'externalSubDir', 'outputDefault', 'outputVideoExtension', 'outputVideoCodec', 'VideoCodecAllow',
'outputVideoPreset', 'outputVideoFramerate', 'outputVideoBitrate', 'outputAudioCodec', 'AudioCodecAllow', 'outputAudioBitrate', 'outputQualityPercent', 'getSubs',
'outputAudioTrack2Codec', 'AudioCodec2Allow', 'outputAudioTrack2Bitrate', 'outputAudioOtherCodec', 'AudioOtherCodecAllow', 'outputAudioOtherBitrate',
'outputSubtitleCodec', 'outputAudioChannels', 'outputAudioTrack2Channels', 'outputAudioOtherChannels']
envKeys = ['TRANSCODE', 'DUPLICATE', 'IGNOREEXTENSIONS', 'OUTPUTFASTSTART', 'OUTPUTVIDEOPATH',
'PROCESSOUTPUT', 'AUDIOLANGUAGE', 'ALLAUDIOLANGUAGES', 'SUBLANGUAGES',
'ALLSUBLANGUAGES', 'EMBEDSUBS', 'BURNINSUBTITLE', 'EXTRACTSUBS', 'EXTERNALSUBDIR',
'OUTPUTDEFAULT', 'OUTPUTVIDEOEXTENSION', 'OUTPUTVIDEOCODEC', 'VIDEOCODECALLOW',
'OUTPUTVIDEOPRESET', 'OUTPUTVIDEOFRAMERATE', 'OUTPUTVIDEOBITRATE', 'OUTPUTAUDIOCODEC',
'AUDIOCODECALLOW', 'OUTPUTAUDIOBITRATE', 'OUTPUTQUALITYPERCENT', 'GETSUBS',
'OUTPUTAUDIOTRACK2CODEC', 'AUDIOCODEC2ALLOW', 'OUTPUTAUDIOTRACK2BITRATE',
'OUTPUTAUDIOOTHERCODEC', 'AUDIOOTHERCODECALLOW', 'OUTPUTAUDIOOTHERBITRATE',
'OUTPUTSUBTITLECODEC', 'OUTPUTAUDIOCHANNELS', 'OUTPUTAUDIOTRACK2CHANNELS',
'OUTPUTAUDIOOTHERCHANNELS']
cfgKeys = ['transcode', 'duplicate', 'ignoreExtensions', 'outputFastStart', 'outputVideoPath',
'processOutput', 'audioLanguage', 'allAudioLanguages', 'subLanguages',
'allSubLanguages', 'embedSubs', 'burnInSubtitle', 'extractSubs', 'externalSubDir',
'outputDefault', 'outputVideoExtension', 'outputVideoCodec', 'VideoCodecAllow',
'outputVideoPreset', 'outputVideoFramerate', 'outputVideoBitrate', 'outputAudioCodec',
'AudioCodecAllow', 'outputAudioBitrate', 'outputQualityPercent', 'getSubs',
'outputAudioTrack2Codec', 'AudioCodec2Allow', 'outputAudioTrack2Bitrate',
'outputAudioOtherCodec', 'AudioOtherCodecAllow', 'outputAudioOtherBitrate',
'outputSubtitleCodec', 'outputAudioChannels', 'outputAudioTrack2Channels',
'outputAudioOtherChannels']
for index in range(len(envKeys)):
key = 'NZBPO_' + envKeys[index]
if os.environ.has_key(key):
key = 'NZBPO_{index}'.format(index=envKeys[index])
if key in os.environ:
option = cfgKeys[index]
value = os.environ[key]
CFG_NEW[section][option] = value
@ -411,20 +440,22 @@ class ConfigObj(configobj.ConfigObj, Section):
envKeys = ['WAKE', 'HOST', 'PORT', 'MAC']
cfgKeys = ['wake', 'host', 'port', 'mac']
for index in range(len(envKeys)):
key = 'NZBPO_WOL' + envKeys[index]
if os.environ.has_key(key):
key = 'NZBPO_WOL{index}'.format(index=envKeys[index])
if key in os.environ:
option = cfgKeys[index]
value = os.environ[key]
CFG_NEW[section][option] = value
section = "UserScript"
envCatKey = 'NZBPO_USCATEGORY'
envKeys = ['USER_SCRIPT_MEDIAEXTENSIONS', 'USER_SCRIPT_PATH', 'USER_SCRIPT_PARAM', 'USER_SCRIPT_RUNONCE', 'USER_SCRIPT_SUCCESSCODES', 'USER_SCRIPT_CLEAN', 'USDELAY', 'USREMOTE_PATH']
cfgKeys = ['user_script_mediaExtensions', 'user_script_path', 'user_script_param', 'user_script_runOnce', 'user_script_successCodes', 'user_script_clean', 'delay', 'remote_path']
if os.environ.has_key(envCatKey):
envKeys = ['USER_SCRIPT_MEDIAEXTENSIONS', 'USER_SCRIPT_PATH', 'USER_SCRIPT_PARAM', 'USER_SCRIPT_RUNONCE',
'USER_SCRIPT_SUCCESSCODES', 'USER_SCRIPT_CLEAN', 'USDELAY', 'USREMOTE_PATH']
cfgKeys = ['user_script_mediaExtensions', 'user_script_path', 'user_script_param', 'user_script_runOnce',
'user_script_successCodes', 'user_script_clean', 'delay', 'remote_path']
if envCatKey in os.environ:
for index in range(len(envKeys)):
key = 'NZBPO_' + envKeys[index]
if os.environ.has_key(key):
key = 'NZBPO_{index}'.format(index=envKeys[index])
if key in os.environ:
option = cfgKeys[index]
value = os.environ[key]
if os.environ[envCatKey] not in CFG_NEW[section].sections:
@ -432,18 +463,19 @@ class ConfigObj(configobj.ConfigObj, Section):
CFG_NEW[section][os.environ[envCatKey]][option] = value
CFG_NEW[section][os.environ[envCatKey]]['enabled'] = 1
except Exception, e:
logger.debug("Error %s when applying NZBGet config" % (e))
except Exception as error:
logger.debug("Error {msg} when applying NZBGet config".format(msg=error))
try:
# write our new config to autoProcessMedia.cfg
CFG_NEW.filename = core.CONFIG_FILE
CFG_NEW.write()
except Exception, e:
logger.debug("Error %s when writing changes to .cfg" % (e))
except Exception as error:
logger.debug("Error {msg} when writing changes to .cfg".format(msg=error))
return CFG_NEW
configobj.Section = Section
configobj.ConfigObj = ConfigObj
config = ConfigObj

128
core/nzbToMediaDB.py
View file

@ -1,4 +1,6 @@
from __future__ import with_statement
# coding=utf-8
from __future__ import print_function, with_statement
import re
import sqlite3
@ -7,6 +9,7 @@ import time
import core
from core import logger
def dbFilename(filename="nzbtomedia.db", suffix=None):
"""
@param filename: The sqlite database filename to use. If not specified,
@ -16,11 +19,11 @@ def dbFilename(filename="nzbtomedia.db", suffix=None):
@return: the correct location of the database file.
"""
if suffix:
filename = "%s.%s" % (filename, suffix)
filename = "{0}.{1}".format(filename, suffix)
return core.os.path.join(core.PROGRAM_DIR, filename)
class DBConnection:
class DBConnection(object):
def __init__(self, filename="nzbtomedia.db", suffix=None, row_type=None):
self.filename = filename
@ -34,7 +37,7 @@ class DBConnection:
result = None
try:
result = self.select("SELECT db_version FROM db_version")
except sqlite3.OperationalError, e:
except sqlite3.OperationalError as e:
if "no such table: db_version" in e.args[0]:
return 0
@ -44,7 +47,7 @@ class DBConnection:
return 0
def fetch(self, query, args=None):
if query == None:
if query is None:
return
sqlResult = None
@ -52,35 +55,36 @@ class DBConnection:
while attempt < 5:
try:
if args == None:
logger.log(self.filename + ": " + query, logger.DB)
if args is None:
logger.log("{name}: {query}".format(name=self.filename, query=query), logger.DB)
cursor = self.connection.cursor()
cursor.execute(query)
sqlResult = cursor.fetchone()[0]
else:
logger.log(self.filename + ": " + query + " with args " + str(args), logger.DB)
logger.log("{name}: {query} with args {args}".format
(name=self.filename, query=query, args=args), logger.DB)
cursor = self.connection.cursor()
cursor.execute(query, args)
sqlResult = cursor.fetchone()[0]
# get out of the connection attempt loop since we were successful
break
except sqlite3.OperationalError, e:
if "unable to open database file" in e.args[0] or "database is locked" in e.args[0]:
logger.log(u"DB error: " + str(e), logger.WARNING)
except sqlite3.OperationalError as error:
if "unable to open database file" in error.args[0] or "database is locked" in error.args[0]:
logger.log(u"DB error: {msg}".format(msg=error), logger.WARNING)
attempt += 1
time.sleep(1)
else:
logger.log(u"DB error: " + str(e), logger.ERROR)
logger.log(u"DB error: {msg}".format(msg=error), logger.ERROR)
raise
except sqlite3.DatabaseError, e:
logger.log(u"Fatal error executing query: " + str(e), logger.ERROR)
except sqlite3.DatabaseError as error:
logger.log(u"Fatal error executing query: {msg}".format(msg=error), logger.ERROR)
raise
return sqlResult
def mass_action(self, querylist, logTransaction=False):
if querylist == None:
if querylist is None:
return
sqlResult = []
@ -95,33 +99,32 @@ class DBConnection:
sqlResult.append(self.connection.execute(qu[0]))
elif len(qu) > 1:
if logTransaction:
logger.log(qu[0] + " with args " + str(qu[1]), logger.DEBUG)
logger.log(u"{query} with args {args}".format(query=qu[0], args=qu[1]), logger.DEBUG)
sqlResult.append(self.connection.execute(qu[0], qu[1]))
self.connection.commit()
logger.log(u"Transaction with " + str(len(querylist)) + u" query's executed", logger.DEBUG)
logger.log(u"Transaction with {x} query's executed".format(x=len(querylist)), logger.DEBUG)
return sqlResult
except sqlite3.OperationalError, e:
except sqlite3.OperationalError as error:
sqlResult = []
if self.connection:
self.connection.rollback()
if "unable to open database file" in e.args[0] or "database is locked" in e.args[0]:
logger.log(u"DB error: " + str(e), logger.WARNING)
if "unable to open database file" in error.args[0] or "database is locked" in error.args[0]:
logger.log(u"DB error: {msg}".format(msg=error), logger.WARNING)
attempt += 1
time.sleep(1)
else:
logger.log(u"DB error: " + str(e), logger.ERROR)
logger.log(u"DB error: {msg}".format(msg=error), logger.ERROR)
raise
except sqlite3.DatabaseError, e:
sqlResult = []
except sqlite3.DatabaseError as error:
if self.connection:
self.connection.rollback()
logger.log(u"Fatal error executing query: " + str(e), logger.ERROR)
logger.log(u"Fatal error executing query: {msg}".format(msg=error), logger.ERROR)
raise
return sqlResult
def action(self, query, args=None):
if query == None:
if query is None:
return
sqlResult = None
@ -129,35 +132,35 @@ class DBConnection:
while attempt < 5:
try:
if args == None:
logger.log(self.filename + ": " + query, logger.DB)
if args is None:
logger.log(u"{name}: {query}".format(name=self.filename, query=query), logger.DB)
sqlResult = self.connection.execute(query)
else:
logger.log(self.filename + ": " + query + " with args " + str(args), logger.DB)
logger.log(u"{name}: {query} with args {args}".format
(name=self.filename, query=query, args=args), logger.DB)
sqlResult = self.connection.execute(query, args)
self.connection.commit()
# get out of the connection attempt loop since we were successful
break
except sqlite3.OperationalError, e:
if "unable to open database file" in e.args[0] or "database is locked" in e.args[0]:
logger.log(u"DB error: " + str(e), logger.WARNING)
except sqlite3.OperationalError as error:
if "unable to open database file" in error.args[0] or "database is locked" in error.args[0]:
logger.log(u"DB error: {msg}".format(msg=error), logger.WARNING)
attempt += 1
time.sleep(1)
else:
logger.log(u"DB error: " + str(e), logger.ERROR)
logger.log(u"DB error: {msg}".format(msg=error), logger.ERROR)
raise
except sqlite3.DatabaseError, e:
logger.log(u"Fatal error executing query: " + str(e), logger.ERROR)
except sqlite3.DatabaseError as error:
logger.log(u"Fatal error executing query: {msg}".format(msg=error), logger.ERROR)
raise
return sqlResult
def select(self, query, args=None):
sqlResults = self.action(query, args).fetchall()
if sqlResults == None:
if sqlResults is None:
return []
return sqlResults
@ -166,21 +169,32 @@ class DBConnection:
changesBefore = self.connection.total_changes
genParams = lambda myDict: [x + " = ?" for x in myDict.keys()]
genParams = lambda myDict: ["{key} = ?".format(key=k) for k in myDict.keys()]
query = "UPDATE " + tableName + " SET " + ", ".join(genParams(valueDict)) + " WHERE " + " AND ".join(
genParams(keyDict))
self.action(query, valueDict.values() + keyDict.values())
self.action(
"UPDATE {table} "
"SET {params} "
"WHERE {conditions}".format(
table=tableName,
params=", ".join(genParams(valueDict)),
conditions=" AND ".join(genParams(keyDict))),
valueDict.values() + keyDict.values()
)
if self.connection.total_changes == changesBefore:
query = "INSERT OR IGNORE INTO " + tableName + " (" + ", ".join(valueDict.keys() + keyDict.keys()) + ")" + \
" VALUES (" + ", ".join(["?"] * len(valueDict.keys() + keyDict.keys())) + ")"
self.action(query, valueDict.values() + keyDict.values())
self.action(
"INSERT OR IGNORE INTO {table} ({columns}) "
"VALUES ({values})".format(
table=tableName,
columns=", ".join(valueDict.keys() + keyDict.keys()),
values=", ".join(["?"] * len(valueDict.keys() + keyDict.keys()))
)
, valueDict.values() + keyDict.values()
)
def tableInfo(self, tableName):
# FIXME ? binding is not supported here, but I cannot find a way to escape a string manually
cursor = self.connection.execute("PRAGMA table_info(%s)" % tableName)
cursor = self.connection.execute("PRAGMA table_info({0})".format(tableName))
columns = {}
for column in cursor:
columns[column['name']] = {'type': column['type']}
@ -221,17 +235,22 @@ def prettyName(class_name):
def _processUpgrade(connection, upgradeClass):
instance = upgradeClass(connection)
logger.log(u"Checking " + prettyName(upgradeClass.__name__) + " database upgrade", logger.DEBUG)
logger.log(u"Checking {name} database upgrade".format
(name=prettyName(upgradeClass.__name__)), logger.DEBUG)
if not instance.test():
logger.log(u"Database upgrade required: " + prettyName(upgradeClass.__name__), logger.MESSAGE)
logger.log(u"Database upgrade required: {name}".format
(name=prettyName(upgradeClass.__name__)), logger.MESSAGE)
try:
instance.execute()
except sqlite3.DatabaseError, e:
print "Error in " + str(upgradeClass.__name__) + ": " + str(e)
except sqlite3.DatabaseError as error:
print(u"Error in {name}: {msg}".format
(name=upgradeClass.__name__, msg=error))
raise
logger.log(upgradeClass.__name__ + " upgrade completed", logger.DEBUG)
logger.log(u"{name} upgrade completed".format
(name=upgradeClass.__name__), logger.DEBUG)
else:
logger.log(upgradeClass.__name__ + " upgrade not required", logger.DEBUG)
logger.log(u"{name} upgrade not required".format
(name=upgradeClass.__name__), logger.DEBUG)
for upgradeSubClass in upgradeClass.__subclasses__():
_processUpgrade(connection, upgradeSubClass)
@ -243,14 +262,14 @@ class SchemaUpgrade(object):
self.connection = connection
def hasTable(self, tableName):
return len(self.connection.action("SELECT 1 FROM sqlite_master WHERE name = ?;", (tableName, )).fetchall()) > 0
return len(self.connection.action("SELECT 1 FROM sqlite_master WHERE name = ?;", (tableName,)).fetchall()) > 0
def hasColumn(self, tableName, column):
return column in self.connection.tableInfo(tableName)
def addColumn(self, table, column, type="NUMERIC", default=0):
self.connection.action("ALTER TABLE %s ADD %s %s" % (table, column, type))
self.connection.action("UPDATE %s SET %s = ?" % (table, column), (default,))
self.connection.action("ALTER TABLE {0} ADD {1} {2}".format(table, column, type))
self.connection.action("UPDATE {0} SET {1} = ?".format(table, column), (default,))
def checkDBVersion(self):
result = self.connection.select("SELECT db_version FROM db_version")
@ -263,4 +282,3 @@ class SchemaUpgrade(object):
new_version = self.checkDBVersion() + 1
self.connection.action("UPDATE db_version SET db_version = ?", [new_version])
return new_version

60
core/nzbToMediaSceneExceptions.py
View file

@ -1,3 +1,4 @@
# coding=utf-8
import os
import re
import core
@ -5,19 +6,24 @@ import shlex
from core import logger
from core.nzbToMediaUtil import listMediaFiles
reverse_list = [r"\.\d{2}e\d{2}s\.", r"\.[pi]0801\.", r"\.p027\.", r"\.[pi]675\.", r"\.[pi]084\.", r"\.p063\.", r"\b[45]62[xh]\.", r"\.yarulb\.", r"\.vtd[hp]\.",
r"\.ld[.-]?bew\.", r"\.pir.?(dov|dvd|bew|db|rb)\.", r"\brdvd\.", r"\.vts\.", r"\.reneercs\.", r"\.dcv\.", r"\b(pir|mac)dh\b", r"\.reporp\.", r"\.kcaper\.",
reverse_list = [r"\.\d{2}e\d{2}s\.", r"\.[pi]0801\.", r"\.p027\.", r"\.[pi]675\.", r"\.[pi]084\.", r"\.p063\.",
r"\b[45]62[xh]\.", r"\.yarulb\.", r"\.vtd[hp]\.",
r"\.ld[.-]?bew\.", r"\.pir.?(dov|dvd|bew|db|rb)\.", r"\brdvd\.", r"\.vts\.", r"\.reneercs\.",
r"\.dcv\.", r"\b(pir|mac)dh\b", r"\.reporp\.", r"\.kcaper\.",
r"\.lanretni\.", r"\b3ca\b", r"\.cstn\."]
reverse_pattern = re.compile('|'.join(reverse_list), flags=re.IGNORECASE)
season_pattern = re.compile(r"(.*\.\d{2}e\d{2}s\.)(.*)", flags=re.IGNORECASE)
word_pattern = re.compile(r"([^A-Z0-9]*[A-Z0-9]+)")
media_list = [r"\.s\d{2}e\d{2}\.", r"\.1080[pi]\.", r"\.720p\.", r"\.576[pi]", r"\.480[pi]\.", r"\.360p\.", r"\.[xh]26[45]\b", r"\.bluray\.", r"\.[hp]dtv\.",
r"\.web[.-]?dl\.", r"\.(vod|dvd|web|bd|br).?rip\.", r"\.dvdr\b", r"\.stv\.", r"\.screener\.", r"\.vcd\.", r"\bhd(cam|rip)\b", r"\.proper\.", r"\.repack\.",
media_list = [r"\.s\d{2}e\d{2}\.", r"\.1080[pi]\.", r"\.720p\.", r"\.576[pi]", r"\.480[pi]\.", r"\.360p\.",
r"\.[xh]26[45]\b", r"\.bluray\.", r"\.[hp]dtv\.",
r"\.web[.-]?dl\.", r"\.(vod|dvd|web|bd|br).?rip\.", r"\.dvdr\b", r"\.stv\.", r"\.screener\.", r"\.vcd\.",
r"\bhd(cam|rip)\b", r"\.proper\.", r"\.repack\.",
r"\.internal\.", r"\bac3\b", r"\.ntsc\.", r"\.pal\.", r"\.secam\.", r"\bdivx\b", r"\bxvid\b"]
media_pattern = re.compile('|'.join(media_list), flags=re.IGNORECASE)
garbage_name = re.compile(r"^[a-zA-Z0-9]*$")
char_replace = [[r"(\w)1\.(\w)",r"\1i\2"]
]
char_replace = [[r"(\w)1\.(\w)", r"\1i\2"]
]
def process_all_exceptions(name, dirname):
rename_script(dirname)
@ -39,6 +45,7 @@ def process_all_exceptions(name, dirname):
if newfilename != filename:
rename_file(filename, newfilename)
def strip_groups(filename):
if not core.GROUPS:
return filename
@ -52,28 +59,33 @@ def strip_groups(filename):
newfilePath = os.path.join(dirname, newfile)
return newfilePath
def rename_file(filename, newfilePath):
logger.debug("Replacing file name %s with download name %s" % (filename, newfilePath), "EXCEPTION")
logger.debug("Replacing file name {old} with download name {new}".format
(old=filename, new=newfilePath), "EXCEPTION")
try:
os.rename(filename, newfilePath)
except Exception,e:
logger.error("Unable to rename file due to: %s" % (str(e)), "EXCEPTION")
except Exception as error:
logger.error("Unable to rename file due to: {error}".format(error=error), "EXCEPTION")
def replace_filename(filename, dirname, name):
head, fileExtension = os.path.splitext(os.path.basename(filename))
if media_pattern.search(os.path.basename(dirname).replace(' ','.')) is not None:
if media_pattern.search(os.path.basename(dirname).replace(' ', '.')) is not None:
newname = os.path.basename(dirname).replace(' ', '.')
logger.debug("Replacing file name %s with directory name %s" % (head, newname), "EXCEPTION")
elif media_pattern.search(name.replace(' ','.').lower()) is not None:
logger.debug("Replacing file name {old} with directory name {new}".format(old=head, new=newname), "EXCEPTION")
elif media_pattern.search(name.replace(' ', '.').lower()) is not None:
newname = name.replace(' ', '.')
logger.debug("Replacing file name %s with download name %s" % (head, newname), "EXCEPTION")
logger.debug("Replacing file name {old} with download name {new}".format
(old=head, new=newname), "EXCEPTION")
else:
logger.warning("No name replacement determined for %s" % (head), "EXCEPTION")
logger.warning("No name replacement determined for {name}".format(name=head), "EXCEPTION")
newname = name
newfile = newname + fileExtension
newfilePath = os.path.join(dirname, newfile)
return newfilePath
def reverse_filename(filename, dirname, name):
head, fileExtension = os.path.splitext(os.path.basename(filename))
na_parts = season_pattern.search(head)
@ -84,25 +96,27 @@ def reverse_filename(filename, dirname, name):
for wp in word_p:
if wp[0] == ".":
new_words += "."
new_words += re.sub(r"\W","",wp)
new_words += re.sub(r"\W", "", wp)
else:
new_words = na_parts.group(2)
for cr in char_replace:
new_words = re.sub(cr[0],cr[1],new_words)
new_words = re.sub(cr[0], cr[1], new_words)
newname = new_words[::-1] + na_parts.group(1)[::-1]
else:
newname = head[::-1].title()
newname = newname.replace(' ', '.')
logger.debug("Reversing filename %s to %s" % (head, newname), "EXCEPTION")
logger.debug("Reversing filename {old} to {new}".format
(old=head, new=newname), "EXCEPTION")
newfile = newname + fileExtension
newfilePath = os.path.join(dirname, newfile)
return newfilePath
def rename_script(dirname):
rename_file = ""
for dir, dirs, files in os.walk(dirname):
for file in files:
if re.search('(rename\S*\.(sh|bat)$)',file,re.IGNORECASE):
if re.search('(rename\S*\.(sh|bat)$)', file, re.IGNORECASE):
rename_file = os.path.join(dir, file)
dirname = dir
break
@ -118,13 +132,13 @@ def rename_script(dirname):
dest = os.path.join(dirname, cmd[1].split('\\')[-1].split('/')[-1])
if os.path.isfile(dest):
continue
logger.debug("Renaming file %s to %s" % (orig, dest), "EXCEPTION")
logger.debug("Renaming file {source} to {destination}".format
(source=orig, destination=dest), "EXCEPTION")
try:
os.rename(orig, dest)
except Exception,e:
logger.error("Unable to rename file due to: %s" % (str(e)), "EXCEPTION")
except Exception as error:
logger.error("Unable to rename file due to: {error}".format(error=error), "EXCEPTION")
# dict for custom groups
# we can add more to this list
#__customgroups__ = {'Q o Q': process_qoq, '-ECI': process_eci}
# _customgroups = {'Q o Q': process_qoq, '-ECI': process_eci}

68
core/nzbToMediaUserScript.py
View file

@ -1,3 +1,4 @@
# coding=utf-8
import os
import core
from subprocess import Popen
@ -5,45 +6,43 @@ from core.transcoder import transcoder
from core.nzbToMediaUtil import import_subs, listMediaFiles, rmDir
from core import logger
def external_script(outputDestination, torrentName, torrentLabel, settings):
final_result = 0 # start at 0.
num_files = 0
try:
core.USER_SCRIPT_MEDIAEXTENSIONS = settings["user_script_mediaExtensions"]
if isinstance(core.USER_SCRIPT_MEDIAEXTENSIONS, str): core.USER_SCRIPT_MEDIAEXTENSIONS = core.USER_SCRIPT_MEDIAEXTENSIONS.split(',')
if isinstance(core.USER_SCRIPT_MEDIAEXTENSIONS, str):
core.USER_SCRIPT_MEDIAEXTENSIONS = core.USER_SCRIPT_MEDIAEXTENSIONS.split(',')
except:
core.USER_SCRIPT_MEDIAEXTENSIONS = []
try:
core.USER_SCRIPT = settings["user_script_path"]
except:
core.USER_SCRIPT = None
if core.USER_SCRIPT is None or core.USER_SCRIPT == "None": # do nothing and return success.
core.USER_SCRIPT = settings.get("user_script_path")
if not core.USER_SCRIPT or core.USER_SCRIPT == "None": # do nothing and return success.
return [0, ""]
try:
core.USER_SCRIPT_PARAM = settings["user_script_param"]
if isinstance(core.USER_SCRIPT_PARAM, str): core.USER_SCRIPT_PARAM = core.USER_SCRIPT_PARAM.split(',')
if isinstance(core.USER_SCRIPT_PARAM, str):
core.USER_SCRIPT_PARAM = core.USER_SCRIPT_PARAM.split(',')
except:
core.USER_SCRIPT_PARAM = []
try:
core.USER_SCRIPT_SUCCESSCODES = settings["user_script_successCodes"]
if isinstance(core.USER_SCRIPT_SUCCESSCODES, str): core.USER_SCRIPT_SUCCESSCODES = core.USER_SCRIPT_SUCCESSCODES.split(',')
if isinstance(core.USER_SCRIPT_SUCCESSCODES, str):
core.USER_SCRIPT_SUCCESSCODES = core.USER_SCRIPT_SUCCESSCODES.split(',')
except:
core.USER_SCRIPT_SUCCESSCODES = 0
try:
core.USER_SCRIPT_CLEAN = int(settings["user_script_clean"])
except:
core.USER_SCRIPT_CLEAN = 1
try:
core.USER_SCRIPT_RUNONCE = int(settings["user_script_runOnce"])
except:
core.USER_SCRIPT_RUNONCE = 1
core.USER_SCRIPT_CLEAN = int(settings.get("user_script_clean", 1))
core.USER_SCRIPT_RUNONCE = int(settings.get("user_script_runOnce", 1))
if core.CHECK_MEDIA:
for video in listMediaFiles(outputDestination, media=True, audio=False, meta=False, archives=False):
if transcoder.isVideoGood(video, 0):
import_subs(video)
else:
logger.info("Corrupt video file found %s. Deleting." % (video), "USERSCRIPT")
logger.info("Corrupt video file found {0}. Deleting.".format(video), "USERSCRIPT")
os.unlink(video)
for dirpath, dirnames, filenames in os.walk(outputDestination):
@ -53,66 +52,65 @@ def external_script(outputDestination, torrentName, torrentLabel, settings):
fileName, fileExtension = os.path.splitext(file)
if fileExtension in core.USER_SCRIPT_MEDIAEXTENSIONS or "ALL" in core.USER_SCRIPT_MEDIAEXTENSIONS:
num_files = num_files + 1
num_files += 1
if core.USER_SCRIPT_RUNONCE == 1 and num_files > 1: # we have already run once, so just continue to get number of files.
continue
command = [core.USER_SCRIPT]
for param in core.USER_SCRIPT_PARAM:
if param == "FN":
command.append('%s' % file)
command.append('{0}'.format(file))
continue
elif param == "FP":
command.append('%s' % filePath)
command.append('{0}'.format(filePath))
continue
elif param == "TN":
command.append('%s' % torrentName)
command.append('{0}'.format(torrentName))
continue
elif param == "TL":
command.append('%s' % torrentLabel)
command.append('{0}'.format(torrentLabel))
continue
elif param == "DN":
if core.USER_SCRIPT_RUNONCE == 1:
command.append('%s' % outputDestination)
command.append('{0}'.format(outputDestination))
else:
command.append('%s' % dirpath)
command.append('{0}'.format(dirpath))
continue
else:
command.append(param)
continue
cmd = ""
for item in command:
cmd = cmd + " " + item
logger.info("Running script %s on file %s." % (cmd, filePath), "USERSCRIPT")
cmd = "{cmd} {item}".format(cmd=cmd, item=item)
logger.info("Running script {cmd} on file {path}.".format(cmd=cmd, path=filePath), "USERSCRIPT")
try:
p = Popen(command)
res = p.wait()
if str(res) in core.USER_SCRIPT_SUCCESSCODES: # Linux returns 0 for successful.
logger.info("UserScript %s was successfull" % (command[0]))
logger.info("UserScript {0} was successfull".format(command[0]))
result = 0
else:
logger.error("UserScript %s has failed with return code: %s" % (command[0], res), "USERSCRIPT")
logger.error("UserScript {0} has failed with return code: {1}".format(command[0], res), "USERSCRIPT")
logger.info(
"If the UserScript completed successfully you should add %s to the user_script_successCodes" % (
"If the UserScript completed successfully you should add {0} to the user_script_successCodes".format(
res), "USERSCRIPT")
result = int(1)
except:
logger.error("UserScript %s has failed" % (command[0]), "USERSCRIPT")
logger.error("UserScript {0} has failed".format(command[0]), "USERSCRIPT")
result = int(1)
final_result = final_result + result
final_result += result
num_files_new = 0
for dirpath, dirnames, filenames in os.walk(outputDestination):
for file in filenames:
filePath = core.os.path.join(dirpath, file)
fileName, fileExtension = os.path.splitext(file)
if fileExtension in core.USER_SCRIPT_MEDIAEXTENSIONS or core.USER_SCRIPT_MEDIAEXTENSIONS == "ALL":
num_files_new = num_files_new + 1
num_files_new += 1
if core.USER_SCRIPT_CLEAN == int(1) and num_files_new == 0 and final_result == 0:
logger.info("All files have been processed. Cleaning outputDirectory %s" % (outputDestination))
logger.info("All files have been processed. Cleaning outputDirectory {0}".format(outputDestination))
rmDir(outputDestination)
elif core.USER_SCRIPT_CLEAN == int(1) and num_files_new != 0:
logger.info("%s files were processed, but %s still remain. outputDirectory will not be cleaned." % (
logger.info("{0} files were processed, but {1} still remain. outputDirectory will not be cleaned.".format(
num_files, num_files_new))
return [final_result, '']

418
core/nzbToMediaUtil.py
View file

File diff suppressed because it is too large Load diff

6
core/synchronousdeluge/__init__.py
View file

@ -1,3 +1,4 @@
# coding=utf-8
"""A synchronous implementation of the Deluge RPC protocol
based on gevent-deluge by Christopher Rosell.
@ -14,10 +15,9 @@ Example usage:
download_location = client.core.get_config_value("download_location").get()
"""
from core.synchronousdeluge.exceptions import DelugeRPCError
__title__ = "synchronous-deluge"
__version__ = "0.1"
__author__ = "Christian Dale"
from core.synchronousdeluge.exceptions import DelugeRPCError

43
core/synchronousdeluge/client.py
View file

@ -1,16 +1,15 @@
# coding=utf-8
import os
import platform
from collections import defaultdict
from itertools import imap
from exceptions import DelugeRPCError
from protocol import DelugeRPCRequest, DelugeRPCResponse
from transfer import DelugeTransfer
from .exceptions import DelugeRPCError
from .protocol import DelugeRPCRequest, DelugeRPCResponse
from .transfer import DelugeTransfer
__all__ = ["DelugeClient"]
RPC_RESPONSE = 1
RPC_ERROR = 2
RPC_EVENT = 3
@ -24,13 +23,13 @@ class DelugeClient(object):
self._request_counter = 0
def _get_local_auth(self):
auth_file = ""
username = password = ""
if platform.system() in ('Windows', 'Microsoft'):
appDataPath = os.environ.get("APPDATA")
if not appDataPath:
import _winreg
hkey = _winreg.OpenKey(_winreg.HKEY_CURRENT_USER, "Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Shell Folders")
hkey = _winreg.OpenKey(_winreg.HKEY_CURRENT_USER,
"Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Shell Folders")
appDataReg = _winreg.QueryValueEx(hkey, "AppData")
appDataPath = appDataReg[0]
_winreg.CloseKey(hkey)
@ -40,10 +39,9 @@ class DelugeClient(object):
from xdg.BaseDirectory import save_config_path
try:
auth_file = os.path.join(save_config_path("deluge"), "auth")
except OSError, e:
except OSError:
return username, password
if os.path.exists(auth_file):
for line in open(auth_file):
if line.startswith("#"):
@ -52,7 +50,7 @@ class DelugeClient(object):
line = line.strip()
try:
lsplit = line.split(":")
except Exception, e:
except Exception:
continue
if len(lsplit) == 2:
@ -63,9 +61,9 @@ class DelugeClient(object):
continue
if username == "localclient":
return (username, password)
return username, password
return ("", "")
return "", ""
def _create_module_method(self, module, method):
fullname = "{0}.{1}".format(module, method)
@ -107,20 +105,20 @@ class DelugeClient(object):
message_type = message[0]
# if message_type == RPC_EVENT:
# event = message[1]
# values = message[2]
#
# if event in self._event_handlers:
# for handler in self._event_handlers[event]:
# gevent.spawn(handler, *values)
#
# elif message_type in (RPC_RESPONSE, RPC_ERROR):
# if message_type == RPC_EVENT:
# event = message[1]
# values = message[2]
#
# if event in self._event_handlers:
# for handler in self._event_handlers[event]:
# gevent.spawn(handler, *values)
#
# elif message_type in (RPC_RESPONSE, RPC_ERROR):
if message_type in (RPC_RESPONSE, RPC_ERROR):
request_id = message[1]
value = message[2]
if request_id == self._request_counter :
if request_id == self._request_counter:
if message_type == RPC_RESPONSE:
response.set(value)
elif message_type == RPC_ERROR:
@ -159,4 +157,3 @@ class DelugeClient(object):
def disconnect(self):
"""Disconnects from the daemon."""
self.transfer.disconnect()

3
core/synchronousdeluge/exceptions.py
View file

@ -1,5 +1,7 @@
# coding=utf-8
__all__ = ["DelugeRPCError"]
class DelugeRPCError(Exception):
def __init__(self, name, msg, traceback):
self.name = name
@ -8,4 +10,3 @@ class DelugeRPCError(Exception):
def __str__(self):
return "{0}: {1}: {2}".format(self.__class__.__name__, self.name, self.msg)

6
core/synchronousdeluge/protocol.py
View file

@ -1,5 +1,7 @@
# coding=utf-8
__all__ = ["DelugeRPCRequest", "DelugeRPCResponse"]
class DelugeRPCRequest(object):
def __init__(self, request_id, method, *args, **kwargs):
self.request_id = request_id
@ -8,7 +10,8 @@ class DelugeRPCRequest(object):
self.kwargs = kwargs
def format(self):
return (self.request_id, self.method, self.args, self.kwargs)
return self.request_id, self.method, self.args, self.kwargs
class DelugeRPCResponse(object):
def __init__(self):
@ -35,4 +38,3 @@ class DelugeRPCResponse(object):
return self.value
else:
raise self._exception

237
core/synchronousdeluge/rencode.py
View file

@ -1,4 +1,4 @@
# coding=utf-8
"""
rencode -- Web safe object pickling/unpickling.
@ -9,9 +9,9 @@ BitTorrent project. For complex, heterogeneous data structures with
many small elements, r-encodings take up significantly less space than
b-encodings:
>>> len(rencode.dumps({'a':0, 'b':[1,2], 'c':99}))
>>> len(rencode.dumps({'a': 0, 'b': [1, 2], 'c': 99}))
13
>>> len(bencode.bencode({'a':0, 'b':[1,2], 'c':99}))
>>> len(bencode.bencode({'a': 0, 'b': [1, 2], 'c': 99}))
26
The rencode format is not standardized, and may change with different
@ -19,6 +19,13 @@ rencode module versions, so you should check that you are using the
same rencode version throughout your project.
"""
import struct
from threading import Lock
from six import PY3
if PY3:
long = int
__version__ = '1.0.1'
__all__ = ['dumps', 'loads']
@ -62,9 +69,6 @@ __all__ = ['dumps', 'loads']
# (The rencode module is licensed under the above license as well).
#
import struct
from threading import Lock
# Default number of bits for serialized floats, either 32 or 64 (also a parameter for dumps()).
DEFAULT_FLOAT_BITS = 32
@ -104,9 +108,10 @@ STR_FIXED_START = 128
STR_FIXED_COUNT = 64
# Lists with length embedded in typecode.
LIST_FIXED_START = STR_FIXED_START+STR_FIXED_COUNT
LIST_FIXED_START = STR_FIXED_START + STR_FIXED_COUNT
LIST_FIXED_COUNT = 64
def decode_int(x, f):
f += 1
newf = x.index(CHR_TERM, f)
@ -119,35 +124,42 @@ def decode_int(x, f):
if x[f] == '-':
if x[f + 1] == '0':
raise ValueError
elif x[f] == '0' and newf != f+1:
elif x[f] == '0' and newf != f + 1:
raise ValueError
return (n, newf+1)
return n, newf + 1
def decode_intb(x, f):
f += 1
return (struct.unpack('!b', x[f:f+1])[0], f+1)
return struct.unpack('!b', x[f:f + 1])[0], f + 1
def decode_inth(x, f):
f += 1
return (struct.unpack('!h', x[f:f+2])[0], f+2)
return struct.unpack('!h', x[f:f + 2])[0], f + 2
def decode_intl(x, f):
f += 1
return (struct.unpack('!l', x[f:f+4])[0], f+4)
return struct.unpack('!l', x[f:f + 4])[0], f + 4
def decode_intq(x, f):
f += 1
return (struct.unpack('!q', x[f:f+8])[0], f+8)
return struct.unpack('!q', x[f:f + 8])[0], f + 8
def decode_float32(x, f):
f += 1
n = struct.unpack('!f', x[f:f+4])[0]
return (n, f+4)
n = struct.unpack('!f', x[f:f + 4])[0]
return n, f + 4
def decode_float64(x, f):
f += 1
n = struct.unpack('!d', x[f:f+8])[0]
return (n, f+8)
n = struct.unpack('!d', x[f:f + 8])[0]
return n, f + 8
def decode_string(x, f):
colon = x.index(':', f)
@ -155,123 +167,147 @@ def decode_string(x, f):
n = int(x[f:colon])
except (OverflowError, ValueError):
n = long(x[f:colon])
if x[f] == '0' and colon != f+1:
if x[f] == '0' and colon != f + 1:
raise ValueError
colon += 1
s = x[colon:colon+n]
s = x[colon:colon + n]
try:
t = s.decode("utf8")
if len(t) != len(s):
s = t
except UnicodeDecodeError:
pass
return (s, colon+n)
return s, colon + n
def decode_list(x, f):
r, f = [], f+1
r, f = [], f + 1
while x[f] != CHR_TERM:
v, f = decode_func[x[f]](x, f)
r.append(v)
return (tuple(r), f + 1)
return tuple(r), f + 1
def decode_dict(x, f):
r, f = {}, f+1
r, f = {}, f + 1
while x[f] != CHR_TERM:
k, f = decode_func[x[f]](x, f)
r[k], f = decode_func[x[f]](x, f)
return (r, f + 1)
return r, f + 1
def decode_true(x, f):
return (True, f+1)
return True, f + 1
def decode_false(x, f):
return (False, f+1)
return False, f + 1
def decode_none(x, f):
return (None, f+1)
return None, f + 1
decode_func = {
'0': decode_string,
'1': decode_string,
'2': decode_string,
'3': decode_string,
'4': decode_string,
'5': decode_string,
'6': decode_string,
'7': decode_string,
'8': decode_string,
'9': decode_string,
CHR_LIST: decode_list,
CHR_DICT: decode_dict,
CHR_INT: decode_int,
CHR_INT1: decode_intb,
CHR_INT2: decode_inth,
CHR_INT4: decode_intl,
CHR_INT8: decode_intq,
CHR_FLOAT32: decode_float32,
CHR_FLOAT64: decode_float64,
CHR_TRUE: decode_true,
CHR_FALSE: decode_false,
CHR_NONE: decode_none,
}
decode_func = {}
decode_func['0'] = decode_string
decode_func['1'] = decode_string
decode_func['2'] = decode_string
decode_func['3'] = decode_string
decode_func['4'] = decode_string
decode_func['5'] = decode_string
decode_func['6'] = decode_string
decode_func['7'] = decode_string
decode_func['8'] = decode_string
decode_func['9'] = decode_string
decode_func[CHR_LIST ] = decode_list
decode_func[CHR_DICT ] = decode_dict
decode_func[CHR_INT ] = decode_int
decode_func[CHR_INT1 ] = decode_intb
decode_func[CHR_INT2 ] = decode_inth
decode_func[CHR_INT4 ] = decode_intl
decode_func[CHR_INT8 ] = decode_intq
decode_func[CHR_FLOAT32] = decode_float32
decode_func[CHR_FLOAT64] = decode_float64
decode_func[CHR_TRUE ] = decode_true
decode_func[CHR_FALSE ] = decode_false
decode_func[CHR_NONE ] = decode_none
def make_fixed_length_string_decoders():
def make_decoder(slen):
def f(x, f):
s = x[f+1:f+1+slen]
s = x[f + 1:f + 1 + slen]
try:
t = s.decode("utf8")
if len(t) != len(s):
s = t
except UnicodeDecodeError:
pass
return (s, f+1+slen)
return s, f + 1 + slen
return f
for i in range(STR_FIXED_COUNT):
decode_func[chr(STR_FIXED_START+i)] = make_decoder(i)
decode_func[chr(STR_FIXED_START + i)] = make_decoder(i)
make_fixed_length_string_decoders()
def make_fixed_length_list_decoders():
def make_decoder(slen):
def f(x, f):
r, f = [], f+1
r, f = [], f + 1
for i in range(slen):
v, f = decode_func[x[f]](x, f)
r.append(v)
return (tuple(r), f)
return tuple(r), f
return f
for i in range(LIST_FIXED_COUNT):
decode_func[chr(LIST_FIXED_START+i)] = make_decoder(i)
decode_func[chr(LIST_FIXED_START + i)] = make_decoder(i)
make_fixed_length_list_decoders()
def make_fixed_length_int_decoders():
def make_decoder(j):
def f(x, f):
return (j, f+1)
return j, f + 1
return f
for i in range(INT_POS_FIXED_COUNT):
decode_func[chr(INT_POS_FIXED_START+i)] = make_decoder(i)
decode_func[chr(INT_POS_FIXED_START + i)] = make_decoder(i)
for i in range(INT_NEG_FIXED_COUNT):
decode_func[chr(INT_NEG_FIXED_START+i)] = make_decoder(-1-i)
decode_func[chr(INT_NEG_FIXED_START + i)] = make_decoder(-1 - i)
make_fixed_length_int_decoders()
def make_fixed_length_dict_decoders():
def make_decoder(slen):
def f(x, f):
r, f = {}, f+1
r, f = {}, f + 1
for j in range(slen):
k, f = decode_func[x[f]](x, f)
r[k], f = decode_func[x[f]](x, f)
return (r, f)
return r, f
return f
for i in range(DICT_FIXED_COUNT):
decode_func[chr(DICT_FIXED_START+i)] = make_decoder(i)
decode_func[chr(DICT_FIXED_START + i)] = make_decoder(i)
make_fixed_length_dict_decoders()
def encode_dict(x,r):
def encode_dict(x, r):
r.append(CHR_DICT)
for k, v in x.items():
encode_func[type(k)](k, r)
@ -288,13 +324,15 @@ def loads(x):
raise ValueError
return r
from types import StringType, IntType, LongType, DictType, ListType, TupleType, FloatType, NoneType, UnicodeType
def encode_int(x, r):
if 0 <= x < INT_POS_FIXED_COUNT:
r.append(chr(INT_POS_FIXED_START+x))
r.append(chr(INT_POS_FIXED_START + x))
elif -INT_NEG_FIXED_COUNT <= x < 0:
r.append(chr(INT_NEG_FIXED_START-1-x))
r.append(chr(INT_NEG_FIXED_START - 1 - x))
elif -128 <= x < 128:
r.extend((CHR_INT1, struct.pack('!b', x)))
elif -32768 <= x < 32768:
@ -309,27 +347,34 @@ def encode_int(x, r):
raise ValueError('overflow')
r.extend((CHR_INT, s, CHR_TERM))
def encode_float32(x, r):
r.extend((CHR_FLOAT32, struct.pack('!f', x)))
def encode_float64(x, r):
r.extend((CHR_FLOAT64, struct.pack('!d', x)))
def encode_bool(x, r):
r.extend({False: CHR_FALSE, True: CHR_TRUE}[bool(x)])
def encode_none(x, r):
r.extend(CHR_NONE)
def encode_string(x, r):
if len(x) < STR_FIXED_COUNT:
r.extend((chr(STR_FIXED_START + len(x)), x))
else:
r.extend((str(len(x)), ':', x))
def encode_unicode(x, r):
encode_string(x.encode("utf8"), r)
def encode_list(x, r):
if len(x) < LIST_FIXED_COUNT:
r.append(chr(LIST_FIXED_START + len(x)))
@ -341,7 +386,8 @@ def encode_list(x, r):
encode_func[type(i)](i, r)
r.append(CHR_TERM)
def encode_dict(x,r):
def encode_dict(x, r):
if len(x) < DICT_FIXED_COUNT:
r.append(chr(DICT_FIXED_START + len(x)))
for k, v in x.items():
@ -354,24 +400,28 @@ def encode_dict(x,r):
encode_func[type(v)](v, r)
r.append(CHR_TERM)
encode_func = {}
encode_func[IntType] = encode_int
encode_func[LongType] = encode_int
encode_func[StringType] = encode_string
encode_func[ListType] = encode_list
encode_func[TupleType] = encode_list
encode_func[DictType] = encode_dict
encode_func[NoneType] = encode_none
encode_func[UnicodeType] = encode_unicode
encode_func = {
IntType: encode_int,
LongType: encode_int,
StringType: encode_string,
ListType: encode_list,
TupleType: encode_list,
DictType: encode_dict,
NoneType: encode_none,
UnicodeType: encode_unicode,
}
lock = Lock()
try:
from types import BooleanType
encode_func[BooleanType] = encode_bool
except ImportError:
pass
def dumps(x, float_bits=DEFAULT_FLOAT_BITS):
"""
Dump data structure to str.
@ -385,48 +435,53 @@ def dumps(x, float_bits=DEFAULT_FLOAT_BITS):
elif float_bits == 64:
encode_func[FloatType] = encode_float64
else:
raise ValueError('Float bits (%d) is not 32 or 64' % float_bits)
raise ValueError('Float bits ({0:d}) is not 32 or 64'.format(float_bits))
r = []
encode_func[type(x)](x, r)
finally:
lock.release()
return ''.join(r)
def test():
f1 = struct.unpack('!f', struct.pack('!f', 25.5))[0]
f2 = struct.unpack('!f', struct.pack('!f', 29.3))[0]
f3 = struct.unpack('!f', struct.pack('!f', -0.6))[0]
L = (({'a':15, 'bb':f1, 'ccc':f2, '':(f3,(),False,True,'')},('a',10**20),tuple(range(-100000,100000)),'b'*31,'b'*62,'b'*64,2**30,2**33,2**62,2**64,2**30,2**33,2**62,2**64,False,False, True, -1, 2, 0),)
L = (({'a': 15, 'bb': f1, 'ccc': f2, '': (f3, (), False, True, '')}, ('a', 10 ** 20), tuple(range(-100000, 100000)),
'b' * 31, 'b' * 62, 'b' * 64, 2 ** 30, 2 ** 33, 2 ** 62, 2 ** 64, 2 ** 30, 2 ** 33, 2 ** 62, 2 ** 64, False,
False, True, -1, 2, 0),)
assert loads(dumps(L)) == L
d = dict(zip(range(-100000,100000),range(-100000,100000)))
d.update({'a':20, 20:40, 40:41, f1:f2, f2:f3, f3:False, False:True, True:False})
L = (d, {}, {5:6}, {7:7,True:8}, {9:10, 22:39, 49:50, 44: ''})
d = dict(zip(range(-100000, 100000), range(-100000, 100000)))
d.update({'a': 20, 20: 40, 40: 41, f1: f2, f2: f3, f3: False, False: True, True: False})
L = (d, {}, {5: 6}, {7: 7, True: 8}, {9: 10, 22: 39, 49: 50, 44: ''})
assert loads(dumps(L)) == L
L = ('', 'a'*10, 'a'*100, 'a'*1000, 'a'*10000, 'a'*100000, 'a'*1000000, 'a'*10000000)
L = ('', 'a' * 10, 'a' * 100, 'a' * 1000, 'a' * 10000, 'a' * 100000, 'a' * 1000000, 'a' * 10000000)
assert loads(dumps(L)) == L
L = tuple([dict(zip(range(n),range(n))) for n in range(100)]) + ('b',)
L = tuple([dict(zip(range(n), range(n))) for n in range(100)]) + ('b',)
assert loads(dumps(L)) == L
L = tuple([dict(zip(range(n),range(-n,0))) for n in range(100)]) + ('b',)
L = tuple([dict(zip(range(n), range(-n, 0))) for n in range(100)]) + ('b',)
assert loads(dumps(L)) == L
L = tuple([tuple(range(n)) for n in range(100)]) + ('b',)
assert loads(dumps(L)) == L
L = tuple(['a'*n for n in range(1000)]) + ('b',)
L = tuple(['a' * n for n in range(1000)]) + ('b',)
assert loads(dumps(L)) == L
L = tuple(['a'*n for n in range(1000)]) + (None,True,None)
L = tuple(['a' * n for n in range(1000)]) + (None, True, None)
assert loads(dumps(L)) == L
assert loads(dumps(None)) == None
assert loads(dumps({None:None})) == {None:None}
assert 1e-10<abs(loads(dumps(1.1))-1.1)<1e-6
assert 1e-10<abs(loads(dumps(1.1,32))-1.1)<1e-6
assert abs(loads(dumps(1.1,64))-1.1)<1e-12
assert loads(dumps(None)) is None
assert loads(dumps({None: None})) == {None: None}
assert 1e-10 < abs(loads(dumps(1.1)) - 1.1) < 1e-6
assert 1e-10 < abs(loads(dumps(1.1, 32)) - 1.1) < 1e-6
assert abs(loads(dumps(1.1, 64)) - 1.1) < 1e-12
assert loads(dumps(u"Hello World!!"))
try:
import psyco
psyco.bind(dumps)
psyco.bind(loads)
except ImportError:
pass
if __name__ == '__main__':
test()

5
core/synchronousdeluge/transfer.py
View file

@ -1,3 +1,4 @@
# coding=utf-8
import zlib
import struct
import socket
@ -5,9 +6,9 @@ import ssl
from core.synchronousdeluge import rencode
__all__ = ["DelugeTransfer"]
class DelugeTransfer(object):
def __init__(self):
self.sock = None
@ -53,5 +54,3 @@ class DelugeTransfer(object):
buf = dobj.unused_data
yield message

1
core/transcoder/__init__.py
View file

@ -1 +1,2 @@
# coding=utf-8
__author__ = 'Justin'

388
core/transcoder/transcoder.py
View file

@ -1,15 +1,18 @@
# coding=utf-8
from six import iteritems
import errno
import os
import platform
import subprocess
import urllib2
import traceback
import core
import json
import shutil
import re
from core import logger
from core.nzbToMediaUtil import makeDir
from babelfish import Language
def isVideoGood(videofile, status):
fileNameExt = os.path.basename(videofile)
@ -19,7 +22,7 @@ def isVideoGood(videofile, status):
disable = True
else:
test_details, res = getVideoDetails(core.TEST_FILE)
if res !=0 or test_details.get("error"):
if res != 0 or test_details.get("error"):
disable = True
logger.info("DISABLED: ffprobe failed to analyse test file. Stopping corruption check.", 'TRANSCODER')
if test_details.get("streams"):
@ -27,41 +30,46 @@ def isVideoGood(videofile, status):
audStreams = [item for item in test_details["streams"] if item["codec_type"] == "audio"]
if not (len(vidStreams) > 0 and len(audStreams) > 0):
disable = True
logger.info("DISABLED: ffprobe failed to analyse streams from test file. Stopping corruption check.", 'TRANSCODER')
logger.info("DISABLED: ffprobe failed to analyse streams from test file. Stopping corruption check.",
'TRANSCODER')
if disable:
if status: # if the download was "failed", assume bad. If it was successful, assume good.
return False
else:
return True
logger.info('Checking [%s] for corruption, please stand by ...' % (fileNameExt), 'TRANSCODER')
logger.info('Checking [{0}] for corruption, please stand by ...'.format(fileNameExt), 'TRANSCODER')
video_details, result = getVideoDetails(videofile)
if result != 0:
logger.error("FAILED: [%s] is corrupted!" % (fileNameExt), 'TRANSCODER')
logger.error("FAILED: [{0}] is corrupted!".format(fileNameExt), 'TRANSCODER')
return False
if video_details.get("error"):
logger.info("FAILED: [%s] returned error [%s]." % (fileNameExt, str(video_details.get("error"))), 'TRANSCODER')
logger.info("FAILED: [{0}] returned error [{1}].".format(fileNameExt, video_details.get("error")), 'TRANSCODER')
return False
if video_details.get("streams"):
videoStreams = [item for item in video_details["streams"] if item["codec_type"] == "video"]
audioStreams = [item for item in video_details["streams"] if item["codec_type"] == "audio"]
if len(videoStreams) > 0 and len(audioStreams) > 0:
logger.info("SUCCESS: [%s] has no corruption." % (fileNameExt), 'TRANSCODER')
logger.info("SUCCESS: [{0}] has no corruption.".format(fileNameExt), 'TRANSCODER')
return True
else:
logger.info("FAILED: [%s] has %s video streams and %s audio streams. Assume corruption." % (fileNameExt, str(len(videoStreams)), str(len(audioStreams))), 'TRANSCODER')
logger.info("FAILED: [{0}] has {1} video streams and {2} audio streams. "
"Assume corruption.".format
(fileNameExt, len(videoStreams), len(audioStreams)), 'TRANSCODER')
return False
def zip_out(file, img, bitbucket):
procin = None
cmd = [core.SEVENZIP, '-so', 'e', img, file]
try:
procin = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=bitbucket)
except:
logger.error("Extracting [%s] has failed" % (file), 'TRANSCODER')
logger.error("Extracting [{0}] has failed".format(file), 'TRANSCODER')
return procin
def getVideoDetails(videofile, img=None, bitbucket=None):
video_details = {}
result = 1
@ -75,7 +83,8 @@ def getVideoDetails(videofile, img=None, bitbucket=None):
try:
if img:
videofile = '-'
command = [core.FFPROBE, '-v', 'quiet', print_format, 'json', '-show_format', '-show_streams', '-show_error', videofile]
command = [core.FFPROBE, '-v', 'quiet', print_format, 'json', '-show_format', '-show_streams', '-show_error',
videofile]
print_cmd(command)
if img:
procin = zip_out(file, img, bitbucket)
@ -86,7 +95,8 @@ def getVideoDetails(videofile, img=None, bitbucket=None):
out, err = proc.communicate()
result = proc.returncode
video_details = json.loads(out)
except: pass
except:
pass
if not video_details:
try:
command = [core.FFPROBE, '-v', 'quiet', print_format, 'json', '-show_format', '-show_streams', videofile]
@ -100,9 +110,10 @@ def getVideoDetails(videofile, img=None, bitbucket=None):
result = proc.returncode
video_details = json.loads(out)
except:
logger.error("Checking [%s] has failed" % (file), 'TRANSCODER')
logger.error("Checking [{0}] has failed".format(file), 'TRANSCODER')
return video_details, result
def buildCommands(file, newDir, movieName, bitbucket):
if isinstance(file, str):
inputFile = file
@ -115,13 +126,13 @@ def buildCommands(file, newDir, movieName, bitbucket):
if check and core.CONCAT:
name = movieName
elif check:
name = ('%s.cd%s' % (movieName, check.groups()[0]))
name = ('{0}.cd{1}'.format(movieName, check.groups()[0]))
elif core.CONCAT and re.match("(.+)[cC][dD][0-9]", name):
name = re.sub("([\ \.\-\_\=\:]+[cC][dD][0-9])", "", name)
if ext == core.VEXTENSION and newDir == dir: # we need to change the name to prevent overwriting itself.
core.VEXTENSION = '-transcoded' + core.VEXTENSION # adds '-transcoded.ext'
core.VEXTENSION = '-transcoded{ext}'.format(ext=core.VEXTENSION) # adds '-transcoded.ext'
else:
img, data = file.iteritems().next()
img, data = iteritems(file).next()
name = data['name']
video_details, result = getVideoDetails(data['files'][0], img, bitbucket)
inputFile = '-'
@ -133,12 +144,12 @@ def buildCommands(file, newDir, movieName, bitbucket):
video_cmd = []
audio_cmd = []
audio_cmd2 = []
audio_cmd3 = []
sub_cmd = []
meta_cmd = []
other_cmd = []
if not video_details or not video_details.get("streams"): # we couldn't read streams with ffprobe. Set defaults to try transcoding.
if not video_details or not video_details.get(
"streams"): # we couldn't read streams with ffprobe. Set defaults to try transcoding.
videoStreams = []
audioStreams = []
subStreams = []
@ -155,7 +166,7 @@ def buildCommands(file, newDir, movieName, bitbucket):
if core.VBITRATE:
video_cmd.extend(['-b:v', str(core.VBITRATE)])
if core.VRESOLUTION:
video_cmd.extend(['-vf', 'scale=' + core.VRESOLUTION])
video_cmd.extend(['-vf', 'scale={vres}'.format(vres=core.VRESOLUTION)])
if core.VPRESET:
video_cmd.extend(['-preset', core.VPRESET])
if core.VCRF:
@ -165,7 +176,8 @@ def buildCommands(file, newDir, movieName, bitbucket):
if core.ACODEC:
audio_cmd.extend(['-c:a', core.ACODEC])
if core.ACODEC in ['aac', 'dts']: # Allow users to use the experimental AAC codec that's built into recent versions of ffmpeg
if core.ACODEC in ['aac',
'dts']: # Allow users to use the experimental AAC codec that's built into recent versions of ffmpeg
audio_cmd.extend(['-strict', '-2'])
else:
audio_cmd.extend(['-c:a', 'copy'])
@ -191,23 +203,16 @@ def buildCommands(file, newDir, movieName, bitbucket):
audioStreams = [item for item in video_details["streams"] if item["codec_type"] == "audio"]
subStreams = [item for item in video_details["streams"] if item["codec_type"] == "subtitle"]
if core.VEXTENSION not in ['.mkv', '.mpegts']:
subStreams = [item for item in video_details["streams"] if item["codec_type"] == "subtitle" and item["codec_name"] != "hdmv_pgs_subtitle" and item["codec_name"] != "pgssub"]
subStreams = [item for item in video_details["streams"] if
item["codec_type"] == "subtitle" and item["codec_name"] != "hdmv_pgs_subtitle" and item[
"codec_name"] != "pgssub"]
for video in videoStreams:
codec = video["codec_name"]
try:
fr = video["avg_frame_rate"]
except: fr = 0
try:
width = video["width"]
except: width = 0
try:
height = video["height"]
except: height = 0
fr = video.get("avg_frame_rate", 0)
width = video.get("width", 0)
height = video.get("height", 0)
scale = core.VRESOLUTION
try:
framerate = float(fr.split('/')[0])/float(fr.split('/')[1])
except: framerate = 0
if codec in core.VCODEC_ALLOW or not core.VCODEC:
video_cmd.extend(['-c:v', 'copy'])
else:
@ -215,16 +220,22 @@ def buildCommands(file, newDir, movieName, bitbucket):
if core.VFRAMERATE and not (core.VFRAMERATE * 0.999 <= fr <= core.VFRAMERATE * 1.001):
video_cmd.extend(['-r', str(core.VFRAMERATE)])
if scale:
w_scale = width/float(scale.split(':')[0])
h_scale = height/float(scale.split(':')[1])
w_scale = width / float(scale.split(':')[0])
h_scale = height / float(scale.split(':')[1])
if w_scale > h_scale: # widescreen, Scale by width only.
scale = scale.split(':')[0] + ":" + str(int((height/w_scale)/2)*2)
scale = "{width}:{height}".format(
width=scale.split(':')[0],
height=int((height / w_scale) / 2) * 2,
)
if w_scale > 1:
video_cmd.extend(['-vf', 'scale=' + scale])
else: # lower or mathcing ratio, scale by height only.
scale = str(int((width/h_scale)/2)*2) + ":" + scale.split(':')[1]
video_cmd.extend(['-vf', 'scale={width}'.format(width=scale)])
else: # lower or matching ratio, scale by height only.
scale = "{width}:{height}".format(
width=int((width / h_scale) / 2) * 2,
height=scale.split(':')[1],
)
if h_scale > 1:
video_cmd.extend(['-vf', 'scale=' + scale])
video_cmd.extend(['-vf', 'scale={height}'.format(height=scale)])
if core.VBITRATE:
video_cmd.extend(['-b:v', str(core.VBITRATE)])
if core.VPRESET:
@ -238,69 +249,54 @@ def buildCommands(file, newDir, movieName, bitbucket):
video_cmd[1] = core.VCODEC
if core.VCODEC == 'copy': # force copy. therefore ignore all other video transcoding.
video_cmd = ['-c:v', 'copy']
map_cmd.extend(['-map', '0:' + str(video["index"])])
map_cmd.extend(['-map', '0:{index}'.format(index=video["index"])])
break # Only one video needed
used_audio = 0
a_mapped = []
if audioStreams:
try:
audio1 = [ item for item in audioStreams if item["tags"]["language"] == core.ALANGUAGE ]
audio1 = [item for item in audioStreams if item["tags"]["language"] == core.ALANGUAGE]
except: # no language tags. Assume only 1 language.
audio1 = audioStreams
audio2 = [ item for item in audio1 if item["codec_name"] in core.ACODEC_ALLOW ]
try:
audio3 = [ item for item in audioStreams if item["tags"]["language"] != core.ALANGUAGE ]
audio2 = [item for item in audio1 if item["codec_name"] in core.ACODEC_ALLOW]
except:
audio2 = []
try:
audio3 = [item for item in audioStreams if item["tags"]["language"] != core.ALANGUAGE]
except:
audio3 = []
if audio2: # right language and codec...
map_cmd.extend(['-map', '0:' + str(audio2[0]["index"])])
map_cmd.extend(['-map', '0:{index}'.format(index=audio2[0]["index"])])
a_mapped.extend([audio2[0]["index"]])
try:
bitrate = int(audio2[0]["bit_rate"])/1000
except: bitrate = 0
try:
channels = int(audio2[0]["channels"])
except: channels = 0
audio_cmd.extend(['-c:a:' + str(used_audio), 'copy'])
bitrate = int(float(audio2[0].get("bit_rate", 0))) / 1000
channels = int(float(audio2[0].get("channels", 0)))
audio_cmd.extend(['-c:a:{0}'.format(used_audio), 'copy'])
elif audio1: # right language wrong codec.
map_cmd.extend(['-map', '0:' + str(audio1[0]["index"])])
map_cmd.extend(['-map', '0:{index}'.format(index=audio1[0]["index"])])
a_mapped.extend([audio1[0]["index"]])
try:
bitrate = int(audio1[0]["bit_rate"])/1000
except: bitrate = 0
try:
channels = int(audio1[0]["channels"])
except: channels = 0
if core.ACODEC:
audio_cmd.extend(['-c:a:' + str(used_audio), core.ACODEC])
else:
audio_cmd.extend(['-c:a:' + str(used_audio), 'copy'])
bitrate = int(float(audio1[0].get("bit_rate", 0))) / 1000
channels = int(float(audio1[0].get("channels", 0)))
audio_cmd.extend(['-c:a:{0}'.format(used_audio), core.ACODEC if core.ACODEC else 'copy'])
elif audio3: # just pick the default audio track
map_cmd.extend(['-map', '0:' + str(audio3[0]["index"])])
map_cmd.extend(['-map', '0:{index}'.format(index=audio3[0]["index"])])
a_mapped.extend([audio3[0]["index"]])
try:
bitrate = int(audio3[0]["bit_rate"])/1000
except: bitrate = 0
try:
channels = int(audio3[0]["channels"])
except: channels = 0
if core.ACODEC:
audio_cmd.extend(['-c:a:' + str(used_audio), core.ACODEC])
else:
audio_cmd.extend(['-c:a:' + str(used_audio), 'copy'])
bitrate = int(float(audio3[0].get("bit_rate", 0))) / 1000
channels = int(float(audio3[0].get("channels", 0)))
audio_cmd.extend(['-c:a:{0}'.format(used_audio), core.ACODEC if core.ACODEC else 'copy'])
if core.ACHANNELS and channels and channels > core.ACHANNELS:
audio_cmd.extend(['-ac:a:' + str(used_audio), str(core.ACHANNELS)])
audio_cmd.extend(['-ac:a:{0}'.format(used_audio), str(core.ACHANNELS)])
if audio_cmd[1] == 'copy':
audio_cmd[1] = core.ACODEC
if core.ABITRATE and not (core.ABITRATE * 0.9 < bitrate < core.ABITRATE * 1.1):
audio_cmd.extend(['-b:a:' + str(used_audio), str(core.ABITRATE)])
audio_cmd.extend(['-b:a:{0}'.format(used_audio), str(core.ABITRATE)])
if audio_cmd[1] == 'copy':
audio_cmd[1] = core.ACODEC
if core.OUTPUTQUALITYPERCENT:
audio_cmd.extend(['-q:a:' + str(used_audio), str(core.OUTPUTQUALITYPERCENT)])
audio_cmd.extend(['-q:a:{0}'.format(used_audio), str(core.OUTPUTQUALITYPERCENT)])
if audio_cmd[1] == 'copy':
audio_cmd[1] = core.ACODEC
if audio_cmd[1] in ['aac', 'dts']:
@ -308,54 +304,45 @@ def buildCommands(file, newDir, movieName, bitbucket):
if core.ACODEC2_ALLOW:
used_audio += 1
audio4 = [ item for item in audio1 if item["codec_name"] in core.ACODEC2_ALLOW ]
try:
audio4 = [item for item in audio1 if item["codec_name"] in core.ACODEC2_ALLOW]
except:
audio4 = []
if audio4: # right language and codec.
map_cmd.extend(['-map', '0:' + str(audio4[0]["index"])])
map_cmd.extend(['-map', '0:{index}'.format(index=audio4[0]["index"])])
a_mapped.extend([audio4[0]["index"]])
try:
bitrate = int(audio4[0]["bit_rate"])/1000
except: bitrate = 0
try:
channels = int(audio4[0]["channels"])
except: channels = 0
audio_cmd2.extend(['-c:a:' + str(used_audio), 'copy'])
bitrate = int(float(audio4[0].get("bit_rate", 0))) / 1000
channels = int(float(audio4[0].get("channels", 0)))
audio_cmd2.extend(['-c:a:{0}'.format(used_audio), 'copy'])
elif audio1: # right language wrong codec.
map_cmd.extend(['-map', '0:' + str(audio1[0]["index"])])
map_cmd.extend(['-map', '0:{index}'.format(index=audio1[0]["index"])])
a_mapped.extend([audio1[0]["index"]])
try:
bitrate = int(audio1[0]["bit_rate"])/1000
except: bitrate = 0
try:
channels = int(audio1[0]["channels"])
except: channels = 0
bitrate = int(float(audio1[0].get("bit_rate", 0))) / 1000
channels = int(float(audio1[0].get("channels", 0)))
if core.ACODEC2:
audio_cmd2.extend(['-c:a:' + str(used_audio), core.ACODEC2])
audio_cmd2.extend(['-c:a:{0}'.format(used_audio), core.ACODEC2])
else:
audio_cmd2.extend(['-c:a:' + str(used_audio), 'copy'])
audio_cmd2.extend(['-c:a:{0}'.format(used_audio), 'copy'])
elif audio3: # just pick the default audio track
map_cmd.extend(['-map', '0:' + str(audio3[0]["index"])])
map_cmd.extend(['-map', '0:{index}'.format(index=audio3[0]["index"])])
a_mapped.extend([audio3[0]["index"]])
try:
bitrate = int(audio3[0]["bit_rate"])/1000
except: bitrate = 0
try:
channels = int(audio3[0]["channels"])
except: channels = 0
bitrate = int(float(audio3[0].get("bit_rate", 0))) / 1000
channels = int(float(audio3[0].get("channels", 0)))
if core.ACODEC2:
audio_cmd2.extend(['-c:a:' + str(used_audio), core.ACODEC2])
audio_cmd2.extend(['-c:a:{0}'.format(used_audio), core.ACODEC2])
else:
audio_cmd2.extend(['-c:a:' + str(used_audio), 'copy'])
audio_cmd2.extend(['-c:a:{0}'.format(used_audio), 'copy'])
if core.ACHANNELS2 and channels and channels > core.ACHANNELS2:
audio_cmd2.extend(['-ac:a:' + str(used_audio), str(core.ACHANNELS2)])
audio_cmd2.extend(['-ac:a:{0}'.format(used_audio), str(core.ACHANNELS2)])
if audio_cmd2[1] == 'copy':
audio_cmd2[1] = core.ACODEC2
if core.ABITRATE2 and not (core.ABITRATE2 * 0.9 < bitrate < core.ABITRATE2 * 1.1):
audio_cmd2.extend(['-b:a:' + str(used_audio), str(core.ABITRATE2)])
audio_cmd2.extend(['-b:a:{0}'.format(used_audio), str(core.ABITRATE2)])
if audio_cmd2[1] == 'copy':
audio_cmd2[1] = core.ACODEC2
if core.OUTPUTQUALITYPERCENT:
audio_cmd2.extend(['-q:a:' + str(used_audio), str(core.OUTPUTQUALITYPERCENT)])
audio_cmd2.extend(['-q:a:{0}'.format(used_audio), str(core.OUTPUTQUALITYPERCENT)])
if audio_cmd2[1] == 'copy':
audio_cmd2[1] = core.ACODEC2
if audio_cmd2[1] in ['aac', 'dts']:
@ -367,32 +354,28 @@ def buildCommands(file, newDir, movieName, bitbucket):
if audio["index"] in a_mapped:
continue
used_audio += 1
map_cmd.extend(['-map', '0:' + str(audio["index"])])
map_cmd.extend(['-map', '0:{index}'.format(index=audio["index"])])
audio_cmd3 = []
try:
bitrate = int(audio["bit_rate"])/1000
except: bitrate = 0
try:
channels = int(audio["channels"])
except: channels = 0
bitrate = int(float(audio.get("bit_rate", 0))) / 1000
channels = int(float(audio.get("channels", 0)))
if audio["codec_name"] in core.ACODEC3_ALLOW:
audio_cmd3.extend(['-c:a:' + str(used_audio), 'copy'])
audio_cmd3.extend(['-c:a:{0}'.format(used_audio), 'copy'])
else:
if core.ACODEC3:
audio_cmd3.extend(['-c:a:' + str(used_audio), core.ACODEC3])
audio_cmd3.extend(['-c:a:{0}'.format(used_audio), core.ACODEC3])
else:
audio_cmd3.extend(['-c:a:' + str(used_audio), 'copy'])
audio_cmd3.extend(['-c:a:{0}'.format(used_audio), 'copy'])
if core.ACHANNELS3 and channels and channels > core.ACHANNELS3:
audio_cmd3.extend(['-ac:a:' + str(used_audio), str(core.ACHANNELS3)])
audio_cmd3.extend(['-ac:a:{0}'.format(used_audio), str(core.ACHANNELS3)])
if audio_cmd3[1] == 'copy':
audio_cmd3[1] = core.ACODEC3
if core.ABITRATE3 and not (core.ABITRATE3 * 0.9 < bitrate < core.ABITRATE3 * 1.1):
audio_cmd3.extend(['-b:a:' + str(used_audio), str(core.ABITRATE3)])
audio_cmd3.extend(['-b:a:{0}'.format(used_audio), str(core.ABITRATE3)])
if audio_cmd3[1] == 'copy':
audio_cmd3[1] = core.ACODEC3
if core.OUTPUTQUALITYPERCENT > 0:
audio_cmd3.extend(['-q:a:' + str(used_audio), str(core.OUTPUTQUALITYPERCENT)])
audio_cmd3.extend(['-q:a:{0}'.format(used_audio), str(core.OUTPUTQUALITYPERCENT)])
if audio_cmd3[1] == 'copy':
audio_cmd3[1] = core.ACODEC3
if audio_cmd3[1] in ['aac', 'dts']:
@ -400,17 +383,17 @@ def buildCommands(file, newDir, movieName, bitbucket):
audio_cmd.extend(audio_cmd3)
s_mapped = []
subs1 = []
burnt = 0
n = 0
for lan in core.SLANGUAGES:
try:
subs1 = [ item for item in subStreams if item["tags"]["language"] == lan ]
except: subs1 = []
subs1 = [item for item in subStreams if item["tags"]["language"] == lan]
except:
subs1 = []
if core.BURN and not subs1 and not burnt and os.path.isfile(file):
for subfile in get_subs(file):
if lan in os.path.split(subfile)[1]:
video_cmd.extend(['-vf', 'subtitles=' + subfile])
video_cmd.extend(['-vf', 'subtitles={subs}'.format(subs=subfile)])
burnt = 1
for sub in subs1:
if core.BURN and not burnt and os.path.isfile(inputFile):
@ -419,11 +402,11 @@ def buildCommands(file, newDir, movieName, bitbucket):
if subStreams[index]["index"] == sub["index"]:
subloc = index
break
video_cmd.extend(['-vf', 'subtitles=' + inputFile + ':si=' + str(subloc)])
video_cmd.extend(['-vf', 'subtitles={sub}:si={loc}'.format(sub=inputFile, loc=subloc)])
burnt = 1
if not core.ALLOWSUBS:
break
map_cmd.extend(['-map', '0:' + str(sub["index"])])
map_cmd.extend(['-map', '0:{index}'.format(index=sub["index"])])
s_mapped.extend([sub["index"]])
if core.SINCLUDE:
@ -432,7 +415,7 @@ def buildCommands(file, newDir, movieName, bitbucket):
break
if sub["index"] in s_mapped:
continue
map_cmd.extend(['-map', '0:' + str(sub["index"])])
map_cmd.extend(['-map', '0:{index}'.format(index=sub["index"])])
s_mapped.extend([sub["index"]])
if core.OUTPUTFASTSTART:
@ -445,18 +428,27 @@ def buildCommands(file, newDir, movieName, bitbucket):
if core.GENERALOPTS:
command.extend(core.GENERALOPTS)
command.extend([ '-i', inputFile])
command.extend(['-i', inputFile])
if core.SEMBED and os.path.isfile(file):
for subfile in get_subs(file):
sub_details, result = getVideoDetails(subfile)
if not sub_details or not sub_details.get("streams"):
continue
lan = os.path.splitext(os.path.splitext(subfile)[0])[1]
command.extend(['-i', subfile])
meta_cmd.extend(['-metadata:s:s:' + str(len(s_mapped) + n), 'language=' + lan[1:]])
lan = os.path.splitext(os.path.splitext(subfile)[0])[1][1:].split('-')[0]
metlan = None
try:
if len(lan) == 3:
metlan = Language(lan)
if len(lan) == 2:
metlan = Language.fromalpha2(lan)
except: pass
if metlan:
meta_cmd.extend(['-metadata:s:s:{x}'.format(x=len(s_mapped) + n),
'language={lang}'.format(lang=metlan.alpha3)])
n += 1
map_cmd.extend(['-map', str(n) + ':0'])
map_cmd.extend(['-map', '{x}:0'.format(x=n)])
if not core.ALLOWSUBS or (not s_mapped and not n):
sub_cmd.extend(['-sn'])
@ -477,6 +469,7 @@ def buildCommands(file, newDir, movieName, bitbucket):
command = core.NICENESS + command
return command
def get_subs(file):
filepaths = []
subExt = ['.srt', '.sub', '.idx']
@ -485,9 +478,10 @@ def get_subs(file):
for dirname, dirs, filenames in os.walk(dir):
for filename in filenames:
filepaths.extend([os.path.join(dirname, filename)])
subfiles = [ item for item in filepaths if os.path.splitext(item)[1] in subExt and name in item ]
subfiles = [item for item in filepaths if os.path.splitext(item)[1] in subExt and name in item]
return subfiles
def extract_subs(file, newfilePath, bitbucket):
video_details, result = getVideoDetails(file)
if not video_details:
@ -500,32 +494,34 @@ def extract_subs(file, newfilePath, bitbucket):
name = os.path.splitext(os.path.split(newfilePath)[1])[0]
try:
subStreams = [item for item in video_details["streams"] if item["codec_type"] == "subtitle" and item["tags"]["language"] in core.SLANGUAGES and item["codec_name"] != "hdmv_pgs_subtitle" and item["codec_name"] != "pgssub"]
subStreams = [item for item in video_details["streams"] if
item["codec_type"] == "subtitle" and item["tags"]["language"] in core.SLANGUAGES and item[
"codec_name"] != "hdmv_pgs_subtitle" and item["codec_name"] != "pgssub"]
except:
subStreams = [item for item in video_details["streams"] if item["codec_type"] == "subtitle" and item["codec_name"] != "hdmv_pgs_subtitle" and item["codec_name"] != "pgssub"]
subStreams = [item for item in video_details["streams"] if
item["codec_type"] == "subtitle" and item["codec_name"] != "hdmv_pgs_subtitle" and item[
"codec_name"] != "pgssub"]
num = len(subStreams)
for n in range(num):
sub = subStreams[n]
idx = sub["index"]
try:
lan = sub["tags"]["language"]
except:
lan = "unk"
lan = sub.get("tags", {}).get("language", "unk")
if num == 1:
outputFile = os.path.join(subdir, "%s.srt" %(name))
outputFile = os.path.join(subdir, "{0}.srt".format(name))
if os.path.isfile(outputFile):
outputFile = os.path.join(subdir, "%s.%s.srt" %(name, n))
outputFile = os.path.join(subdir, "{0}.{1}.srt".format(name, n))
else:
outputFile = os.path.join(subdir, "%s.%s.srt" %(name, lan))
outputFile = os.path.join(subdir, "{0}.{1}.srt".format(name, lan))
if os.path.isfile(outputFile):
outputFile = os.path.join(subdir, "%s.%s.%s.srt" %(name, lan, n))
outputFile = os.path.join(subdir, "{0}.{1}.{2}.srt".format(name, lan, n))
command = [core.FFMPEG, '-loglevel', 'warning', '-i', file, '-vn', '-an', '-codec:' + str(idx), 'srt', outputFile]
command = [core.FFMPEG, '-loglevel', 'warning', '-i', file, '-vn', '-an',
'-codec:{index}'.format(index=idx), 'srt', outputFile]
if platform.system() != 'Windows':
command = core.NICENESS + command
logger.info("Extracting %s subtitle from: %s" % (lan, file))
logger.info("Extracting {0} subtitle from: {1}".format(lan, file))
print_cmd(command)
result = 1 # set result to failed in case call fails.
try:
@ -538,30 +534,30 @@ def extract_subs(file, newfilePath, bitbucket):
if result == 0:
try:
shutil.copymode(file, outputFile)
except: pass
logger.info("Extracting %s subtitle from %s has succeeded" % (lan, file))
except:
pass
logger.info("Extracting {0} subtitle from {1} has succeeded".format(lan, file))
else:
logger.error("Extracting subtitles has failed")
def processList(List, newDir, bitbucket):
remList = []
newList = []
delList = []
combine = []
vtsPath = None
success = True
for item in List:
newfile = None
ext = os.path.splitext(item)[1].lower()
if ext in ['.iso', '.bin', '.img'] and not ext in core.IGNOREEXTENSIONS:
logger.debug("Attempting to rip disk image: %s" % (item), "TRANSCODER")
if ext in ['.iso', '.bin', '.img'] and ext not in core.IGNOREEXTENSIONS:
logger.debug("Attempting to rip disk image: {0}".format(item), "TRANSCODER")
newList.extend(ripISO(item, newDir, bitbucket))
remList.append(item)
elif re.match(".+VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb]", item) and not '.vob' in core.IGNOREEXTENSIONS:
logger.debug("Found VIDEO_TS image file: %s" % (item), "TRANSCODER")
elif re.match(".+VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb]", item) and '.vob' not in core.IGNOREEXTENSIONS:
logger.debug("Found VIDEO_TS image file: {0}".format(item), "TRANSCODER")
if not vtsPath:
try:
vtsPath = re.match("(.+VIDEO_TS)",item).groups()[0]
vtsPath = re.match("(.+VIDEO_TS)", item).groups()[0]
except:
vtsPath = os.path.split(item)[0]
remList.append(item)
@ -570,48 +566,50 @@ def processList(List, newDir, bitbucket):
elif core.CONCAT and re.match(".+[cC][dD][0-9].", item):
remList.append(item)
combine.append(item)
else: continue
else:
continue
if vtsPath:
newList.extend(combineVTS(vtsPath))
if combine:
newList.extend(combineCD(combine))
for file in newList:
if isinstance(file, str) and not 'concat:' in file and not os.path.isfile(file):
if isinstance(file, str) and 'concat:' not in file and not os.path.isfile(file):
success = False
break
if success and newList:
List.extend(newList)
for item in remList:
List.remove(item)
logger.debug("Successfully extracted .vob file %s from disk image" % (newList[0]), "TRANSCODER")
logger.debug("Successfully extracted .vob file {0} from disk image".format(newList[0]), "TRANSCODER")
elif newList and not success:
newList = []
remList = []
logger.error("Failed extracting .vob files from disk image. Stopping transcoding.", "TRANSCODER")
return List, remList, newList, success
def ripISO(item, newDir, bitbucket):
newFiles = []
failure_dir = 'failure'
# Mount the ISO in your OS and call combineVTS.
if not core.SEVENZIP:
logger.error("No 7zip installed. Can't extract image file %s" % (item), "TRANSCODER")
logger.error("No 7zip installed. Can't extract image file {0}".format(item), "TRANSCODER")
newFiles = [failure_dir]
return newFiles
cmd = [core.SEVENZIP, 'l', item]
try:
logger.debug("Attempting to extract .vob from image file %s" % (item), "TRANSCODER")
logger.debug("Attempting to extract .vob from image file {0}".format(item), "TRANSCODER")
print_cmd(cmd)
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=bitbucket)
out, err = proc.communicate()
result = proc.returncode
fileList = [ re.match(".+(VIDEO_TS[\\\/]VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb])", line).groups()[0] for line in out.splitlines() if re.match(".+VIDEO_TS[\\\/]VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb]", line) ]
fileList = [re.match(".+(VIDEO_TS[\\\/]VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb])", line).groups()[0] for line in
out.splitlines() if re.match(".+VIDEO_TS[\\\/]VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb]", line)]
combined = []
for n in range(99):
concat = []
m = 1
while True:
vtsName = 'VIDEO_TS%sVTS_%02d_%d.VOB' % (os.sep, n+1, m)
vtsName = 'VIDEO_TS{0}VTS_{1:02d}_{2:d}.VOB'.format(os.sep, n + 1, m)
if vtsName in fileList:
concat.append(vtsName)
m += 1
@ -622,19 +620,22 @@ def ripISO(item, newDir, bitbucket):
if core.CONCAT:
combined.extend(concat)
continue
name = '%s.cd%s' % (os.path.splitext(os.path.split(item)[1])[0] ,str(n+1))
newFiles.append({item: {'name': name , 'files': concat}})
name = '{name}.cd{x}'.format(
name=os.path.splitext(os.path.split(item)[1])[0], x=n + 1
)
newFiles.append({item: {'name': name, 'files': concat}})
if core.CONCAT:
name = os.path.splitext(os.path.split(item)[1])[0]
newFiles.append({item: {'name': name , 'files': combined}})
newFiles.append({item: {'name': name, 'files': combined}})
if not newFiles:
logger.error("No VIDEO_TS folder found in image file %s" % (item), "TRANSCODER")
logger.error("No VIDEO_TS folder found in image file {0}".format(item), "TRANSCODER")
newFiles = [failure_dir]
except:
logger.error("Failed to extract from image file %s" % (item), "TRANSCODER")
logger.error("Failed to extract from image file {0}".format(item), "TRANSCODER")
newFiles = [failure_dir]
return newFiles
def combineVTS(vtsPath):
newFiles = []
combined = ''
@ -642,41 +643,45 @@ def combineVTS(vtsPath):
concat = ''
m = 1
while True:
vtsName = 'VTS_%02d_%d.VOB' % (n+1, m)
vtsName = 'VTS_{0:02d}_{1:d}.VOB'.format(n + 1, m)
if os.path.isfile(os.path.join(vtsPath, vtsName)):
concat = concat + os.path.join(vtsPath, vtsName) + '|'
concat += '{file}|'.format(file=os.path.join(vtsPath, vtsName))
m += 1
else:
break
if not concat:
break
if core.CONCAT:
combined = combined + concat + '|'
combined += '{files}|'.format(files=concat)
continue
newFiles.append('concat:%s' % concat[:-1])
newFiles.append('concat:{0}'.format(concat[:-1]))
if core.CONCAT:
newFiles.append('concat:%s' % combined[:-1])
newFiles.append('concat:{0}'.format(combined[:-1]))
return newFiles
def combineCD(combine):
newFiles = []
for item in set([ re.match("(.+)[cC][dD][0-9].",item).groups()[0] for item in combine ]):
for item in set([re.match("(.+)[cC][dD][0-9].", item).groups()[0] for item in combine]):
concat = ''
for n in range(99):
files = [ file for file in combine if n+1 == int(re.match(".+[cC][dD]([0-9]+).",file).groups()[0]) and item in file ]
files = [file for file in combine if
n + 1 == int(re.match(".+[cC][dD]([0-9]+).", file).groups()[0]) and item in file]
if files:
concat = concat + files[0] + '|'
concat += '{file}|'.format(file=files[0])
else:
break
if concat:
newFiles.append('concat:%s' % concat[:-1])
newFiles.append('concat:{0}'.format(concat[:-1]))
return newFiles
def print_cmd(command):
cmd = ""
for item in command:
cmd = cmd + " " + str(item)
logger.debug("calling command:%s" % (cmd))
cmd = "{cmd} {item}".format(cmd=cmd, item=item)
logger.debug("calling command:{0}".format(cmd))
def Transcode_directory(dirName):
if not core.FFMPEG:
@ -714,20 +719,20 @@ def Transcode_directory(dirName):
try: # Try to remove the file that we're transcoding to just in case. (ffmpeg will return an error if it already exists for some reason)
os.remove(newfilePath)
except OSError, e:
except OSError as e:
if e.errno != errno.ENOENT: # Ignore the error if it's just telling us that the file doesn't exist
logger.debug("Error when removing transcoding target: %s" % (e))
except Exception, e:
logger.debug("Error when removing transcoding target: %s" % (e))
logger.debug("Error when removing transcoding target: {0}".format(e))
except Exception as e:
logger.debug("Error when removing transcoding target: {0}".format(e))
logger.info("Transcoding video: %s" % (newfilePath))
logger.info("Transcoding video: {0}".format(newfilePath))
print_cmd(command)
result = 1 # set result to failed in case call fails.
try:
if isinstance(file, str):
proc = subprocess.Popen(command, stdout=bitbucket, stderr=bitbucket)
else:
img, data = file.iteritems().next()
img, data = iteritems(file).next()
proc = subprocess.Popen(command, stdout=bitbucket, stderr=bitbucket, stdin=subprocess.PIPE)
for vob in data['files']:
procin = zip_out(vob, img, bitbucket)
@ -737,7 +742,7 @@ def Transcode_directory(dirName):
proc.communicate()
result = proc.returncode
except:
logger.error("Transcoding of video %s has failed" % (newfilePath))
logger.error("Transcoding of video {0} has failed".format(newfilePath))
if core.SUBSDIR and result == 0 and isinstance(file, str):
for sub in get_subs(file):
@ -751,22 +756,25 @@ def Transcode_directory(dirName):
if result == 0:
try:
shutil.copymode(file, newfilePath)
except: pass
logger.info("Transcoding of video to %s succeeded" % (newfilePath))
except:
pass
logger.info("Transcoding of video to {0} succeeded".format(newfilePath))
if os.path.isfile(newfilePath) and (file in newList or not core.DUPLICATE):
try:
os.unlink(file)
except: pass
except:
pass
else:
logger.error("Transcoding of video to %s failed with result %s" % (newfilePath, str(result)))
logger.error("Transcoding of video to {0} failed with result {1}".format(newfilePath, result))
# this will be 0 (successful) it all are successful, else will return a positive integer for failure.
final_result = final_result + result
if final_result == 0 and not core.DUPLICATE:
for file in remList:
try:
os.unlink(file)
except: pass
if not os.listdir(newDir): #this is an empty directory and we didn't transcode into it.
except:
pass
if not os.listdir(newDir): # this is an empty directory and we didn't transcode into it.
os.rmdir(newDir)
newDir = dirName
if not core.PROCESSOUTPUT and core.DUPLICATE: # We postprocess the original files to CP/SB

74
core/transmissionrpc/client.py
View file

@ -18,13 +18,9 @@ from core.transmissionrpc.torrent import Torrent
from core.transmissionrpc.session import Session
from six import PY3, integer_types, string_types, iteritems
from six.moves.urllib_parse import urlparse
from six.moves.urllib_request import urlopen
if PY3:
from urllib.parse import urlparse
from urllib.request import urlopen
else:
from urlparse import urlparse
from urllib2 import urlopen
def debug_httperror(error):
"""
@ -49,6 +45,7 @@ def debug_httperror(error):
)
)
def parse_torrent_id(arg):
"""Parse an torrent id or torrent hashString."""
torrent_id = None
@ -62,7 +59,7 @@ def parse_torrent_id(arg):
elif isinstance(arg, string_types):
try:
torrent_id = int(arg)
if torrent_id >= 2**31:
if torrent_id >= 2 ** 31:
torrent_id = None
except (ValueError, TypeError):
pass
@ -75,6 +72,7 @@ def parse_torrent_id(arg):
pass
return torrent_id
def parse_torrent_ids(args):
"""
Take things and make them valid torrent identifiers
@ -102,19 +100,20 @@ def parse_torrent_ids(args):
except ValueError:
pass
if not addition:
raise ValueError('Invalid torrent id, \"%s\"' % item)
raise ValueError('Invalid torrent id, {item!r}'.format(item=item))
ids.extend(addition)
elif isinstance(args, (list, tuple)):
for item in args:
ids.extend(parse_torrent_ids(item))
else:
torrent_id = parse_torrent_id(args)
if torrent_id == None:
if torrent_id is None:
raise ValueError('Invalid torrent id')
else:
ids = [torrent_id]
return ids
"""
Torrent ids
@ -129,26 +128,27 @@ possible to provide a argument called ``timeout``. Timeout is only effective
when using Python 2.6 or later and the default timeout is 30 seconds.
"""
class Client(object):
"""
Client is the class handling the Transmission JSON-RPC client protocol.
"""
def __init__(self, address='localhost', port=DEFAULT_PORT, user=None, password=None, http_handler=None, timeout=None):
def __init__(self, address='localhost', port=DEFAULT_PORT, user=None, password=None, http_handler=None,
timeout=None):
if isinstance(timeout, (integer_types, float)):
self._query_timeout = float(timeout)
else:
self._query_timeout = DEFAULT_TIMEOUT
urlo = urlparse(address)
if urlo.scheme == '':
base_url = 'http://' + address + ':' + str(port)
self.url = base_url + '/transmission/rpc/'
if not urlo.scheme:
self.url = 'http://{host}:{port}/transmission/rpc/'.format(host=address, port=port)
else:
if urlo.port:
self.url = urlo.scheme + '://' + urlo.hostname + ':' + str(urlo.port) + urlo.path
self.url = '{url.scheme}://{url.hostname}:{url.port}{url.path}'.format(url=urlo)
else:
self.url = urlo.scheme + '://' + urlo.hostname + urlo.path
LOGGER.info('Using custom URL "' + self.url + '".')
self.url = '{url.scheme}://{url.hostname}{url.path}'.format(url=urlo)
LOGGER.info('Using custom URL {url!r}.'.format(url=self.url))
if urlo.username and urlo.password:
user = urlo.username
password = urlo.password
@ -204,7 +204,8 @@ class Client(object):
if timeout is None:
timeout = self._query_timeout
while True:
LOGGER.debug(json.dumps({'url': self.url, 'headers': headers, 'query': query, 'timeout': timeout}, indent=2))
LOGGER.debug(
json.dumps({'url': self.url, 'headers': headers, 'query': query, 'timeout': timeout}, indent=2))
try:
result = self.http_handler.request(self.url, query, headers, timeout)
break
@ -244,26 +245,25 @@ class Client(object):
elif require_ids:
raise ValueError('request require ids')
query = json.dumps({'tag': self._sequence, 'method': method
, 'arguments': arguments})
query = json.dumps({'tag': self._sequence, 'method': method, 'arguments': arguments})
self._sequence += 1
start = time.time()
http_data = self._http_query(query, timeout)
elapsed = time.time() - start
LOGGER.info('http request took %.3f s' % (elapsed))
LOGGER.info('http request took {time:.3f} s'.format(time=elapsed))
try:
data = json.loads(http_data)
except ValueError as error:
LOGGER.error('Error: ' + str(error))
LOGGER.error('Request: \"%s\"' % (query))
LOGGER.error('HTTP data: \"%s\"' % (http_data))
LOGGER.error('Error: {msg}'.format(msg=error))
LOGGER.error('Request: {request!r}'.format(request=query))
LOGGER.error('HTTP data: {data!r}'.format(data=http_data))
raise
LOGGER.debug(json.dumps(data, indent=2))
if 'result' in data:
if data['result'] != 'success':
raise TransmissionError('Query failed with result \"%s\".' % (data['result']))
raise TransmissionError('Query failed with result {result!r}.'.format(result=data['result']))
else:
raise TransmissionError('Query failed without result.')
@ -347,8 +347,9 @@ class Client(object):
Add a warning to the log if the Transmission RPC version is lower then the provided version.
"""
if self.rpc_version < version:
LOGGER.warning('Using feature not supported by server. RPC version for server %d, feature introduced in %d.'
% (self.rpc_version, version))
LOGGER.warning('Using feature not supported by server. '
'RPC version for server {x}, feature introduced in {y}.'.format
(x=self.rpc_version, y=version))
def add_torrent(self, torrent, timeout=None, **kwargs):
"""
@ -408,11 +409,8 @@ class Client(object):
pass
if might_be_base64:
torrent_data = torrent
args = {}
if torrent_data:
args = {'metainfo': torrent_data}
else:
args = {'filename': torrent}
args = {'metainfo': torrent_data} if torrent_data else {'filename': torrent}
for key, value in iteritems(kwargs):
argument = make_rpc_name(key)
(arg, val) = argument_value_convert('torrent-add', argument, value, self.rpc_version)
@ -476,7 +474,7 @@ class Client(object):
"""
self._rpc_version_warning(3)
self._request('torrent-remove',
{'delete-local-data':rpc_bool(delete_data)}, ids, True, timeout=timeout)
{'delete-local-data': rpc_bool(delete_data)}, ids, True, timeout=timeout)
def remove(self, ids, delete_data=False, timeout=None):
"""
@ -606,9 +604,9 @@ class Client(object):
the new methods. list returns a dictionary indexed by torrent id.
"""
warnings.warn('list has been deprecated, please use get_torrent or get_torrents instead.', DeprecationWarning)
fields = ['id', 'hashString', 'name', 'sizeWhenDone', 'leftUntilDone'
, 'eta', 'status', 'rateUpload', 'rateDownload', 'uploadedEver'
, 'downloadedEver', 'uploadRatio', 'queuePosition']
fields = ['id', 'hashString', 'name', 'sizeWhenDone', 'leftUntilDone',
'eta', 'status', 'rateUpload', 'rateDownload', 'uploadedEver',
'downloadedEver', 'uploadRatio', 'queuePosition']
return self._request('torrent-get', {'fields': fields}, timeout=timeout)
def get_files(self, ids=None, timeout=None):
@ -742,7 +740,7 @@ class Client(object):
args = {}
for key, value in iteritems(kwargs):
argument = make_rpc_name(key)
(arg, val) = argument_value_convert('torrent-set' , argument, value, self.rpc_version)
(arg, val) = argument_value_convert('torrent-set', argument, value, self.rpc_version)
args[arg] = val
if len(args) > 0:
@ -803,7 +801,7 @@ class Client(object):
raise ValueError("Target name cannot contain a path delimiter")
args = {'path': location, 'name': name}
result = self._request('torrent-rename-path', args, torrent_id, True, timeout=timeout)
return (result['path'], result['name'])
return result['path'], result['name']
def queue_top(self, ids, timeout=None):
"""Move transfer to the top of the queue."""
@ -895,7 +893,7 @@ class Client(object):
if key == 'encryption' and value not in ['required', 'preferred', 'tolerated']:
raise ValueError('Invalid encryption value')
argument = make_rpc_name(key)
(arg, val) = argument_value_convert('session-set' , argument, value, self.rpc_version)
(arg, val) = argument_value_convert('session-set', argument, value, self.rpc_version)
args[arg] = val
if len(args) > 0:
self._request('session-set', args, timeout=timeout)

113
core/transmissionrpc/constants.py
View file

@ -6,10 +6,10 @@ import logging
from core.transmissionrpc.six import iteritems
LOGGER = logging.getLogger('transmissionrpc')
LOGGER.setLevel(logging.ERROR)
def mirror_dict(source):
"""
Creates a dictionary with all values as keys and all keys as values.
@ -17,6 +17,7 @@ def mirror_dict(source):
source.update(dict((value, key) for key, value in iteritems(source)))
return source
DEFAULT_PORT = 9091
DEFAULT_TIMEOUT = 30.0
@ -26,9 +27,9 @@ TR_PRI_NORMAL = 0
TR_PRI_HIGH = 1
PRIORITY = mirror_dict({
'low' : TR_PRI_LOW,
'normal' : TR_PRI_NORMAL,
'high' : TR_PRI_HIGH
'low': TR_PRI_LOW,
'normal': TR_PRI_NORMAL,
'high': TR_PRI_HIGH
})
TR_RATIOLIMIT_GLOBAL = 0 # follow the global settings
@ -36,9 +37,9 @@ TR_RATIOLIMIT_SINGLE = 1 # override the global settings, seeding until a cert
TR_RATIOLIMIT_UNLIMITED = 2 # override the global settings, seeding regardless of ratio
RATIO_LIMIT = mirror_dict({
'global' : TR_RATIOLIMIT_GLOBAL,
'single' : TR_RATIOLIMIT_SINGLE,
'unlimited' : TR_RATIOLIMIT_UNLIMITED
'global': TR_RATIOLIMIT_GLOBAL,
'single': TR_RATIOLIMIT_SINGLE,
'unlimited': TR_RATIOLIMIT_UNLIMITED
})
TR_IDLELIMIT_GLOBAL = 0 # follow the global settings
@ -46,9 +47,9 @@ TR_IDLELIMIT_SINGLE = 1 # override the global settings, seeding until a cert
TR_IDLELIMIT_UNLIMITED = 2 # override the global settings, seeding regardless of activity
IDLE_LIMIT = mirror_dict({
'global' : TR_RATIOLIMIT_GLOBAL,
'single' : TR_RATIOLIMIT_SINGLE,
'unlimited' : TR_RATIOLIMIT_UNLIMITED
'global': TR_RATIOLIMIT_GLOBAL,
'single': TR_RATIOLIMIT_SINGLE,
'unlimited': TR_RATIOLIMIT_UNLIMITED
})
# A note on argument maps
@ -62,7 +63,7 @@ IDLE_LIMIT = mirror_dict({
# Arguments for torrent methods
TORRENT_ARGS = {
'get' : {
'get': {
'activityDate': ('number', 1, None, None, None, 'Last time of upload or download activity.'),
'addedDate': ('number', 1, None, None, None, 'The date when this torrent was first added.'),
'announceResponse': ('string', 1, 7, None, None, 'The announce message from the tracker.'),
@ -79,13 +80,19 @@ TORRENT_ARGS = {
'downloaders': ('number', 4, 7, None, None, 'Number of downloaders.'),
'downloadLimit': ('number', 1, None, None, None, 'Download limit in Kbps.'),
'downloadLimited': ('boolean', 5, None, None, None, 'Download limit is enabled'),
'downloadLimitMode': ('number', 1, 5, None, None, 'Download limit mode. 0 means global, 1 means signle, 2 unlimited.'),
'error': ('number', 1, None, None, None, 'Kind of error. 0 means OK, 1 means tracker warning, 2 means tracker error, 3 means local error.'),
'downloadLimitMode': (
'number', 1, 5, None, None, 'Download limit mode. 0 means global, 1 means signle, 2 unlimited.'),
'error': ('number', 1, None, None, None,
'Kind of error. 0 means OK, 1 means tracker warning, 2 means tracker error, 3 means local error.'),
'errorString': ('number', 1, None, None, None, 'Error message.'),
'eta': ('number', 1, None, None, None, 'Estimated number of seconds left when downloading or seeding. -1 means not available and -2 means unknown.'),
'etaIdle': ('number', 15, None, None, None, 'Estimated number of seconds left until the idle time limit is reached. -1 means not available and -2 means unknown.'),
'files': ('array', 1, None, None, None, 'Array of file object containing key, bytesCompleted, length and name.'),
'fileStats': ('array', 5, None, None, None, 'Aray of file statistics containing bytesCompleted, wanted and priority.'),
'eta': ('number', 1, None, None, None,
'Estimated number of seconds left when downloading or seeding. -1 means not available and -2 means unknown.'),
'etaIdle': ('number', 15, None, None, None,
'Estimated number of seconds left until the idle time limit is reached. -1 means not available and -2 means unknown.'),
'files': (
'array', 1, None, None, None, 'Array of file object containing key, bytesCompleted, length and name.'),
'fileStats': (
'array', 5, None, None, None, 'Aray of file statistics containing bytesCompleted, wanted and priority.'),
'hashString': ('string', 1, None, None, None, 'Hashstring unique for the torrent even between sessions.'),
'haveUnchecked': ('number', 1, None, None, None, 'Number of bytes of partial pieces.'),
'haveValid': ('number', 1, None, None, None, 'Number of bytes of checksum verified data.'),
@ -108,7 +115,8 @@ TORRENT_ARGS = {
'peer-limit': ('number', 5, None, None, None, 'Maximum number of peers.'),
'peers': ('array', 2, None, None, None, 'Array of peer objects.'),
'peersConnected': ('number', 1, None, None, None, 'Number of peers we are connected to.'),
'peersFrom': ('object', 1, None, None, None, 'Object containing download peers counts for different peer types.'),
'peersFrom': (
'object', 1, None, None, None, 'Object containing download peers counts for different peer types.'),
'peersGettingFromUs': ('number', 1, None, None, None, 'Number of peers we are sending data to.'),
'peersKnown': ('number', 1, 13, None, None, 'Number of peers that the tracker knows.'),
'peersSendingToUs': ('number', 1, None, None, None, 'Number of peers sending to us'),
@ -141,7 +149,8 @@ TORRENT_ARGS = {
'torrentFile': ('string', 5, None, None, None, 'Path to .torrent file.'),
'uploadedEver': ('number', 1, None, None, None, 'Number of bytes uploaded, ever.'),
'uploadLimit': ('number', 1, None, None, None, 'Upload limit in Kbps'),
'uploadLimitMode': ('number', 1, 5, None, None, 'Upload limit mode. 0 means global, 1 means signle, 2 unlimited.'),
'uploadLimitMode': (
'number', 1, 5, None, None, 'Upload limit mode. 0 means global, 1 means signle, 2 unlimited.'),
'uploadLimited': ('boolean', 5, None, None, None, 'Upload limit enabled.'),
'uploadRatio': ('double', 1, None, None, None, 'Seed ratio.'),
'wanted': ('array', 1, None, None, None, 'Array of booleans indicated wanted files.'),
@ -154,7 +163,8 @@ TORRENT_ARGS = {
'downloadLimited': ('boolean', 5, None, 'speed-limit-down-enabled', None, 'Enable download speed limiter.'),
'files-wanted': ('array', 1, None, None, None, "A list of file id's that should be downloaded."),
'files-unwanted': ('array', 1, None, None, None, "A list of file id's that shouldn't be downloaded."),
'honorsSessionLimits': ('boolean', 5, None, None, None, "Enables or disables the transfer to honour the upload limit set in the session."),
'honorsSessionLimits': ('boolean', 5, None, None, None,
"Enables or disables the transfer to honour the upload limit set in the session."),
'location': ('array', 1, None, None, None, 'Local download location.'),
'peer-limit': ('number', 1, None, None, None, 'The peer limit for the torrents.'),
'priority-high': ('array', 1, None, None, None, "A list of file id's that should have high priority."),
@ -162,22 +172,26 @@ TORRENT_ARGS = {
'priority-normal': ('array', 1, None, None, None, "A list of file id's that should have low priority."),
'queuePosition': ('number', 14, None, None, None, 'Position of this transfer in its queue.'),
'seedIdleLimit': ('number', 10, None, None, None, 'Seed inactivity limit in minutes.'),
'seedIdleMode': ('number', 10, None, None, None, 'Seed inactivity mode. 0 = Use session limit, 1 = Use transfer limit, 2 = Disable limit.'),
'seedIdleMode': ('number', 10, None, None, None,
'Seed inactivity mode. 0 = Use session limit, 1 = Use transfer limit, 2 = Disable limit.'),
'seedRatioLimit': ('double', 5, None, None, None, 'Seeding ratio.'),
'seedRatioMode': ('number', 5, None, None, None, 'Which ratio to use. 0 = Use session limit, 1 = Use transfer limit, 2 = Disable limit.'),
'seedRatioMode': ('number', 5, None, None, None,
'Which ratio to use. 0 = Use session limit, 1 = Use transfer limit, 2 = Disable limit.'),
'speed-limit-down': ('number', 1, 5, None, 'downloadLimit', 'Set the speed limit for download in Kib/s.'),
'speed-limit-down-enabled': ('boolean', 1, 5, None, 'downloadLimited', 'Enable download speed limiter.'),
'speed-limit-up': ('number', 1, 5, None, 'uploadLimit', 'Set the speed limit for upload in Kib/s.'),
'speed-limit-up-enabled': ('boolean', 1, 5, None, 'uploadLimited', 'Enable upload speed limiter.'),
'trackerAdd': ('array', 10, None, None, None, 'Array of string with announce URLs to add.'),
'trackerRemove': ('array', 10, None, None, None, 'Array of ids of trackers to remove.'),
'trackerReplace': ('array', 10, None, None, None, 'Array of (id, url) tuples where the announce URL should be replaced.'),
'trackerReplace': (
'array', 10, None, None, None, 'Array of (id, url) tuples where the announce URL should be replaced.'),
'uploadLimit': ('number', 5, None, 'speed-limit-up', None, 'Set the speed limit for upload in Kib/s.'),
'uploadLimited': ('boolean', 5, None, 'speed-limit-up-enabled', None, 'Enable upload speed limiter.'),
},
'add': {
'bandwidthPriority': ('number', 8, None, None, None, 'Priority for this transfer.'),
'download-dir': ('string', 1, None, None, None, 'The directory where the downloaded contents will be saved in.'),
'download-dir': (
'string', 1, None, None, None, 'The directory where the downloaded contents will be saved in.'),
'cookies': ('string', 13, None, None, None, 'One or more HTTP cookie(s).'),
'filename': ('string', 1, None, None, None, "A file path or URL to a torrent file or a magnet link."),
'files-wanted': ('array', 1, None, None, None, "A list of file id's that should be downloaded."),
@ -195,10 +209,13 @@ TORRENT_ARGS = {
SESSION_ARGS = {
'get': {
"alt-speed-down": ('number', 5, None, None, None, 'Alternate session download speed limit (in Kib/s).'),
"alt-speed-enabled": ('boolean', 5, None, None, None, 'True if alternate global download speed limiter is ebabled.'),
"alt-speed-time-begin": ('number', 5, None, None, None, 'Time when alternate speeds should be enabled. Minutes after midnight.'),
"alt-speed-enabled": (
'boolean', 5, None, None, None, 'True if alternate global download speed limiter is ebabled.'),
"alt-speed-time-begin": (
'number', 5, None, None, None, 'Time when alternate speeds should be enabled. Minutes after midnight.'),
"alt-speed-time-enabled": ('boolean', 5, None, None, None, 'True if alternate speeds scheduling is enabled.'),
"alt-speed-time-end": ('number', 5, None, None, None, 'Time when alternate speeds should be disabled. Minutes after midnight.'),
"alt-speed-time-end": (
'number', 5, None, None, None, 'Time when alternate speeds should be disabled. Minutes after midnight.'),
"alt-speed-time-day": ('number', 5, None, None, None, 'Days alternate speeds scheduling is enabled.'),
"alt-speed-up": ('number', 5, None, None, None, 'Alternate session upload speed limit (in Kib/s)'),
"blocklist-enabled": ('boolean', 5, None, None, None, 'True when blocklist is enabled.'),
@ -211,10 +228,12 @@ SESSION_ARGS = {
"download-dir-free-space": ('number', 12, None, None, None, 'Free space in the download directory, in bytes'),
"download-queue-size": ('number', 14, None, None, None, 'Number of slots in the download queue.'),
"download-queue-enabled": ('boolean', 14, None, None, None, 'True if the download queue is enabled.'),
"encryption": ('string', 1, None, None, None, 'Encryption mode, one of ``required``, ``preferred`` or ``tolerated``.'),
"encryption": (
'string', 1, None, None, None, 'Encryption mode, one of ``required``, ``preferred`` or ``tolerated``.'),
"idle-seeding-limit": ('number', 10, None, None, None, 'Seed inactivity limit in minutes.'),
"idle-seeding-limit-enabled": ('boolean', 10, None, None, None, 'True if the seed activity limit is enabled.'),
"incomplete-dir": ('string', 7, None, None, None, 'The path to the directory for incomplete torrent transfer data.'),
"incomplete-dir": (
'string', 7, None, None, None, 'The path to the directory for incomplete torrent transfer data.'),
"incomplete-dir-enabled": ('boolean', 7, None, None, None, 'True if the incomplete dir is enabled.'),
"lpd-enabled": ('boolean', 9, None, None, None, 'True if local peer discovery is enabled.'),
"peer-limit": ('number', 1, 5, None, 'peer-limit-global', 'Maximum number of peers.'),
@ -224,15 +243,18 @@ SESSION_ARGS = {
"pex-enabled": ('boolean', 5, None, 'pex-allowed', None, 'True if PEX is enabled.'),
"port": ('number', 1, 5, None, 'peer-port', 'Peer port.'),
"peer-port": ('number', 5, None, 'port', None, 'Peer port.'),
"peer-port-random-on-start": ('boolean', 5, None, None, None, 'Enables randomized peer port on start of Transmission.'),
"peer-port-random-on-start": (
'boolean', 5, None, None, None, 'Enables randomized peer port on start of Transmission.'),
"port-forwarding-enabled": ('boolean', 1, None, None, None, 'True if port forwarding is enabled.'),
"queue-stalled-minutes": ('number', 14, None, None, None, 'Number of minutes of idle that marks a transfer as stalled.'),
"queue-stalled-minutes": (
'number', 14, None, None, None, 'Number of minutes of idle that marks a transfer as stalled.'),
"queue-stalled-enabled": ('boolean', 14, None, None, None, 'True if stalled tracking of transfers is enabled.'),
"rename-partial-files": ('boolean', 8, None, None, None, 'True if ".part" is appended to incomplete files'),
"rpc-version": ('number', 4, None, None, None, 'Transmission RPC API Version.'),
"rpc-version-minimum": ('number', 4, None, None, None, 'Minimum accepted RPC API Version.'),
"script-torrent-done-enabled": ('boolean', 9, None, None, None, 'True if the done script is enabled.'),
"script-torrent-done-filename": ('string', 9, None, None, None, 'Filename of the script to run when the transfer is done.'),
"script-torrent-done-filename": (
'string', 9, None, None, None, 'Filename of the script to run when the transfer is done.'),
"seedRatioLimit": ('double', 5, None, None, None, 'Seed ratio limit. 1.0 means 1:1 download and upload ratio.'),
"seedRatioLimited": ('boolean', 5, None, None, None, 'True if seed ration limit is enabled.'),
"seed-queue-size": ('number', 14, None, None, None, 'Number of slots in the upload queue.'),
@ -242,7 +264,8 @@ SESSION_ARGS = {
"speed-limit-up": ('number', 1, None, None, None, 'Upload speed limit (in Kib/s).'),
"speed-limit-up-enabled": ('boolean', 1, None, None, None, 'True if the upload speed is limited.'),
"start-added-torrents": ('boolean', 9, None, None, None, 'When true uploaded torrents will start right away.'),
"trash-original-torrent-files": ('boolean', 9, None, None, None, 'When true added .torrent files will be deleted.'),
"trash-original-torrent-files": (
'boolean', 9, None, None, None, 'When true added .torrent files will be deleted.'),
'units': ('object', 10, None, None, None, 'An object containing units for size and speed.'),
'utp-enabled': ('boolean', 13, None, None, None, 'True if Micro Transport Protocol (UTP) is enabled.'),
"version": ('string', 3, None, None, None, 'Transmission version.'),
@ -250,9 +273,11 @@ SESSION_ARGS = {
'set': {
"alt-speed-down": ('number', 5, None, None, None, 'Alternate session download speed limit (in Kib/s).'),
"alt-speed-enabled": ('boolean', 5, None, None, None, 'Enables alternate global download speed limiter.'),
"alt-speed-time-begin": ('number', 5, None, None, None, 'Time when alternate speeds should be enabled. Minutes after midnight.'),
"alt-speed-time-begin": (
'number', 5, None, None, None, 'Time when alternate speeds should be enabled. Minutes after midnight.'),
"alt-speed-time-enabled": ('boolean', 5, None, None, None, 'Enables alternate speeds scheduling.'),
"alt-speed-time-end": ('number', 5, None, None, None, 'Time when alternate speeds should be disabled. Minutes after midnight.'),
"alt-speed-time-end": (
'number', 5, None, None, None, 'Time when alternate speeds should be disabled. Minutes after midnight.'),
"alt-speed-time-day": ('number', 5, None, None, None, 'Enables alternate speeds scheduling these days.'),
"alt-speed-up": ('number', 5, None, None, None, 'Alternate session upload speed limit (in Kib/s).'),
"blocklist-enabled": ('boolean', 5, None, None, None, 'Enables the block list'),
@ -262,11 +287,13 @@ SESSION_ARGS = {
"download-dir": ('string', 1, None, None, None, 'Set the session download directory.'),
"download-queue-size": ('number', 14, None, None, None, 'Number of slots in the download queue.'),
"download-queue-enabled": ('boolean', 14, None, None, None, 'Enables download queue.'),
"encryption": ('string', 1, None, None, None, 'Set the session encryption mode, one of ``required``, ``preferred`` or ``tolerated``.'),
"encryption": ('string', 1, None, None, None,
'Set the session encryption mode, one of ``required``, ``preferred`` or ``tolerated``.'),
"idle-seeding-limit": ('number', 10, None, None, None, 'The default seed inactivity limit in minutes.'),
"idle-seeding-limit-enabled": ('boolean', 10, None, None, None, 'Enables the default seed inactivity limit'),
"incomplete-dir": ('string', 7, None, None, None, 'The path to the directory of incomplete transfer data.'),
"incomplete-dir-enabled": ('boolean', 7, None, None, None, 'Enables the incomplete transfer data directory. Otherwise data for incomplete transfers are stored in the download target.'),
"incomplete-dir-enabled": ('boolean', 7, None, None, None,
'Enables the incomplete transfer data directory. Otherwise data for incomplete transfers are stored in the download target.'),
"lpd-enabled": ('boolean', 9, None, None, None, 'Enables local peer discovery for public torrents.'),
"peer-limit": ('number', 1, 5, None, 'peer-limit-global', 'Maximum number of peers.'),
"peer-limit-global": ('number', 5, None, 'peer-limit', None, 'Maximum number of peers.'),
@ -275,13 +302,16 @@ SESSION_ARGS = {
"pex-enabled": ('boolean', 5, None, 'pex-allowed', None, 'Allowing PEX in public torrents.'),
"port": ('number', 1, 5, None, 'peer-port', 'Peer port.'),
"peer-port": ('number', 5, None, 'port', None, 'Peer port.'),
"peer-port-random-on-start": ('boolean', 5, None, None, None, 'Enables randomized peer port on start of Transmission.'),
"peer-port-random-on-start": (
'boolean', 5, None, None, None, 'Enables randomized peer port on start of Transmission.'),
"port-forwarding-enabled": ('boolean', 1, None, None, None, 'Enables port forwarding.'),
"rename-partial-files": ('boolean', 8, None, None, None, 'Appends ".part" to incomplete files'),
"queue-stalled-minutes": ('number', 14, None, None, None, 'Number of minutes of idle that marks a transfer as stalled.'),
"queue-stalled-minutes": (
'number', 14, None, None, None, 'Number of minutes of idle that marks a transfer as stalled.'),
"queue-stalled-enabled": ('boolean', 14, None, None, None, 'Enable tracking of stalled transfers.'),
"script-torrent-done-enabled": ('boolean', 9, None, None, None, 'Whether or not to call the "done" script.'),
"script-torrent-done-filename": ('string', 9, None, None, None, 'Filename of the script to run when the transfer is done.'),
"script-torrent-done-filename": (
'string', 9, None, None, None, 'Filename of the script to run when the transfer is done.'),
"seed-queue-size": ('number', 14, None, None, None, 'Number of slots in the upload queue.'),
"seed-queue-enabled": ('boolean', 14, None, None, None, 'Enables upload queue.'),
"seedRatioLimit": ('double', 5, None, None, None, 'Seed ratio limit. 1.0 means 1:1 download and upload ratio.'),
@ -291,7 +321,8 @@ SESSION_ARGS = {
"speed-limit-up": ('number', 1, None, None, None, 'Upload speed limit (in Kib/s).'),
"speed-limit-up-enabled": ('boolean', 1, None, None, None, 'Enables upload speed limiting.'),
"start-added-torrents": ('boolean', 9, None, None, None, 'Added torrents will be started right away.'),
"trash-original-torrent-files": ('boolean', 9, None, None, None, 'The .torrent file of added torrents will be deleted.'),
"trash-original-torrent-files": (
'boolean', 9, None, None, None, 'The .torrent file of added torrents will be deleted.'),
'utp-enabled': ('boolean', 13, None, None, None, 'Enables Micro Transport Protocol (UTP).'),
},
}

12
core/transmissionrpc/error.py
View file

@ -4,11 +4,13 @@
from core.transmissionrpc.six import string_types, integer_types
class TransmissionError(Exception):
"""
This exception is raised when there has occurred an error related to
communication with Transmission. It is a subclass of Exception.
"""
def __init__(self, message='', original=None):
Exception.__init__(self)
self.message = message
@ -17,15 +19,17 @@ class TransmissionError(Exception):
def __str__(self):
if self.original:
original_name = type(self.original).__name__
return '%s Original exception: %s, "%s"' % (self.message, original_name, str(self.original))
return '{0} Original exception: {1}, "{2}"'.format(self.message, original_name, str(self.original))
else:
return self.message
class HTTPHandlerError(Exception):
"""
This exception is raised when there has occurred an error related to
the HTTP handler. It is a subclass of Exception.
"""
def __init__(self, httpurl=None, httpcode=None, httpmsg=None, httpheaders=None, httpdata=None):
Exception.__init__(self)
self.url = ''
@ -45,10 +49,10 @@ class HTTPHandlerError(Exception):
self.data = httpdata
def __repr__(self):
return '<HTTPHandlerError %d, %s>' % (self.code, self.message)
return '<HTTPHandlerError {0:d}, {1}>'.format(self.code, self.message)
def __str__(self):
return 'HTTPHandlerError %d: %s' % (self.code, self.message)
return 'HTTPHandlerError {0:d}: {1}'.format(self.code, self.message)
def __unicode__(self):
return 'HTTPHandlerError %d: %s' % (self.code, self.message)
return 'HTTPHandlerError {0:d}: {1}'.format(self.code, self.message)

28
core/transmissionrpc/httphandler.py
View file

@ -4,24 +4,22 @@
import sys
from core.transmissionrpc.error import HTTPHandlerError
from six import PY3
from six.moves.urllib_request import (
build_opener, install_opener,
HTTPBasicAuthHandler, HTTPDigestAuthHandler, HTTPPasswordMgrWithDefaultRealm,
Request,
)
from six.moves.urllib_error import HTTPError, URLError
from six.moves.http_client import BadStatusLine
from core.transmissionrpc.error import HTTPHandlerError
if PY3:
from urllib.request import Request, build_opener, \
HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler, HTTPDigestAuthHandler
from urllib.error import HTTPError, URLError
from http.client import BadStatusLine
else:
from urllib2 import Request, build_opener, \
HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler, HTTPDigestAuthHandler
from urllib2 import HTTPError, URLError
from httplib import BadStatusLine
class HTTPHandler(object):
"""
Prototype for HTTP handling.
"""
def set_authentication(self, uri, login, password):
"""
Transmission use basic authentication in earlier versions and digest
@ -44,10 +42,12 @@ class HTTPHandler(object):
"""
raise NotImplementedError("Bad HTTPHandler, failed to implement request.")
class DefaultHTTPHandler(HTTPHandler):
"""
The default HTTP handler provided with transmissionrpc.
"""
def __init__(self):
HTTPHandler.__init__(self)
self.http_opener = build_opener()
@ -75,7 +75,7 @@ class DefaultHTTPHandler(HTTPHandler):
if hasattr(error.reason, 'args') and isinstance(error.reason.args, tuple) and len(error.reason.args) == 2:
raise HTTPHandlerError(httpcode=error.reason.args[0], httpmsg=error.reason.args[1])
else:
raise HTTPHandlerError(httpmsg='urllib2.URLError: %s' % (error.reason))
raise HTTPHandlerError(httpmsg='urllib2.URLError: {error.reason}'.format(error=error))
except BadStatusLine as error:
raise HTTPHandlerError(httpmsg='httplib.BadStatusLine: %s' % (error.line))
raise HTTPHandlerError(httpmsg='httplib.BadStatusLine: {error.line}'.format(error=error))
return response.read().decode('utf-8')

5
core/transmissionrpc/session.py
View file

@ -6,6 +6,7 @@ from core.transmissionrpc.utils import Field
from core.transmissionrpc.six import iteritems, integer_types
class Session(object):
"""
Session is a class holding the session data for a Transmission daemon.
@ -26,12 +27,12 @@ class Session(object):
try:
return self._fields[name].value
except KeyError:
raise AttributeError('No attribute %s' % name)
raise AttributeError('No attribute {0}'.format(name))
def __str__(self):
text = ''
for key in sorted(self._fields.keys()):
text += "% 32s: %s\n" % (key[-32:], self._fields[key].value)
text += "{0:32}: {1}\n".format(key[-32:], self._fields[key].value)
return text
def _update_fields(self, other):

441
core/transmissionrpc/six.py
View file

@ -1,6 +1,6 @@
"""Utilities for writing code that runs on Python 2 and 3"""
# Copyright (c) 2010-2013 Benjamin Peterson
# Copyright (c) 2010-2015 Benjamin Peterson
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
@ -20,17 +20,22 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from __future__ import absolute_import
import functools
import itertools
import operator
import sys
import types
__author__ = "Benjamin Peterson <benjamin@python.org>"
__version__ = "1.4.1"
__version__ = "1.10.0"
# Useful for very coarse version differentiation.
PY2 = sys.version_info[0] == 2
PY3 = sys.version_info[0] == 3
PY34 = sys.version_info[0:2] >= (3, 4)
if PY3:
string_types = str,
@ -53,6 +58,7 @@ else:
else:
# It's possible to have sizeof(long) != sizeof(Py_ssize_t).
class X(object):
def __len__(self):
return 1 << 31
try:
@ -84,9 +90,13 @@ class _LazyDescr(object):
def __get__(self, obj, tp):
result = self._resolve()
setattr(obj, self.name, result)
# This is a bit ugly, but it avoids running this again.
delattr(tp, self.name)
setattr(obj, self.name, result) # Invokes __set__.
try:
# This is a bit ugly, but it avoids running this again by
# removing this descriptor.
delattr(obj.__class__, self.name)
except AttributeError:
pass
return result
@ -104,6 +114,27 @@ class MovedModule(_LazyDescr):
def _resolve(self):
return _import_module(self.mod)
def __getattr__(self, attr):
_module = self._resolve()
value = getattr(_module, attr)
setattr(self, attr, value)
return value
class _LazyModule(types.ModuleType):
def __init__(self, name):
super(_LazyModule, self).__init__(name)
self.__doc__ = self.__class__.__doc__
def __dir__(self):
attrs = ["__doc__", "__name__"]
attrs += [attr.name for attr in self._moved_attributes]
return attrs
# Subclasses should override this
_moved_attributes = []
class MovedAttribute(_LazyDescr):
@ -130,9 +161,75 @@ class MovedAttribute(_LazyDescr):
return getattr(module, self.attr)
class _SixMetaPathImporter(object):
"""
A meta path importer to import six.moves and its submodules.
This class implements a PEP302 finder and loader. It should be compatible
with Python 2.5 and all existing versions of Python3
"""
def __init__(self, six_module_name):
self.name = six_module_name
self.known_modules = {}
def _add_module(self, mod, *fullnames):
for fullname in fullnames:
self.known_modules[self.name + "." + fullname] = mod
def _get_module(self, fullname):
return self.known_modules[self.name + "." + fullname]
def find_module(self, fullname, path=None):
if fullname in self.known_modules:
return self
return None
def __get_module(self, fullname):
try:
return self.known_modules[fullname]
except KeyError:
raise ImportError("This loader does not know module " + fullname)
def load_module(self, fullname):
try:
# in case of a reload
return sys.modules[fullname]
except KeyError:
pass
mod = self.__get_module(fullname)
if isinstance(mod, MovedModule):
mod = mod._resolve()
else:
mod.__loader__ = self
sys.modules[fullname] = mod
return mod
def is_package(self, fullname):
"""
Return true, if the named module is a package.
We need this method to get correct spec objects with
Python 3.4 (see PEP451)
"""
return hasattr(self.__get_module(fullname), "__path__")
def get_code(self, fullname):
"""Return None
Required, if is_package is implemented"""
self.__get_module(fullname) # eventually raises ImportError
return None
get_source = get_code # same as get_code
_importer = _SixMetaPathImporter(__name__)
class _MovedItems(_LazyModule):
class _MovedItems(types.ModuleType):
"""Lazy loading of moved objects"""
__path__ = [] # mark as package
_moved_attributes = [
@ -140,25 +237,33 @@ _moved_attributes = [
MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"),
MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"),
MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"),
MovedAttribute("intern", "__builtin__", "sys"),
MovedAttribute("map", "itertools", "builtins", "imap", "map"),
MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"),
MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"),
MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"),
MovedAttribute("reload_module", "__builtin__", "imp", "reload"),
MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"),
MovedAttribute("reduce", "__builtin__", "functools"),
MovedAttribute("shlex_quote", "pipes", "shlex", "quote"),
MovedAttribute("StringIO", "StringIO", "io"),
MovedAttribute("UserDict", "UserDict", "collections"),
MovedAttribute("UserList", "UserList", "collections"),
MovedAttribute("UserString", "UserString", "collections"),
MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"),
MovedAttribute("zip", "itertools", "builtins", "izip", "zip"),
MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"),
MovedModule("builtins", "__builtin__"),
MovedModule("config", "config"),
MovedModule("configparser", "ConfigParser"),
MovedModule("copyreg", "copy_reg"),
MovedModule("dbm_gnu", "gdbm", "dbm.gnu"),
MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread"),
MovedModule("http_cookiejar", "cookielib", "http.cookiejar"),
MovedModule("http_cookies", "Cookie", "http.cookies"),
MovedModule("html_entities", "htmlentitydefs", "html.entities"),
MovedModule("html_parser", "HTMLParser", "html.parser"),
MovedModule("http_client", "httplib", "http.client"),
MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"),
MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"),
MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"),
MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"),
MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"),
@ -168,12 +273,14 @@ _moved_attributes = [
MovedModule("queue", "Queue"),
MovedModule("reprlib", "repr"),
MovedModule("socketserver", "SocketServer"),
MovedModule("_thread", "thread", "_thread"),
MovedModule("tkinter", "Tkinter"),
MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"),
MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"),
MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"),
MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"),
MovedModule("tkinter_tix", "Tix", "tkinter.tix"),
MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"),
MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"),
MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"),
MovedModule("tkinter_colorchooser", "tkColorChooser",
@ -189,22 +296,35 @@ _moved_attributes = [
MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"),
MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"),
MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"),
MovedModule("winreg", "_winreg"),
MovedModule("xmlrpc_client", "xmlrpclib", "xmlrpc.client"),
MovedModule("xmlrpc_server", "SimpleXMLRPCServer", "xmlrpc.server"),
]
# Add windows specific modules.
if sys.platform == "win32":
_moved_attributes += [
MovedModule("winreg", "_winreg"),
]
for attr in _moved_attributes:
setattr(_MovedItems, attr.name, attr)
if isinstance(attr, MovedModule):
_importer._add_module(attr, "moves." + attr.name)
del attr
moves = sys.modules[__name__ + ".moves"] = _MovedItems(__name__ + ".moves")
_MovedItems._moved_attributes = _moved_attributes
moves = _MovedItems(__name__ + ".moves")
_importer._add_module(moves, "moves")
class Module_six_moves_urllib_parse(_LazyModule):
class Module_six_moves_urllib_parse(types.ModuleType):
"""Lazy loading of moved objects in six.moves.urllib_parse"""
_urllib_parse_moved_attributes = [
MovedAttribute("ParseResult", "urlparse", "urllib.parse"),
MovedAttribute("SplitResult", "urlparse", "urllib.parse"),
MovedAttribute("parse_qs", "urlparse", "urllib.parse"),
MovedAttribute("parse_qsl", "urlparse", "urllib.parse"),
MovedAttribute("urldefrag", "urlparse", "urllib.parse"),
@ -218,16 +338,27 @@ _urllib_parse_moved_attributes = [
MovedAttribute("unquote", "urllib", "urllib.parse"),
MovedAttribute("unquote_plus", "urllib", "urllib.parse"),
MovedAttribute("urlencode", "urllib", "urllib.parse"),
MovedAttribute("splitquery", "urllib", "urllib.parse"),
MovedAttribute("splittag", "urllib", "urllib.parse"),
MovedAttribute("splituser", "urllib", "urllib.parse"),
MovedAttribute("uses_fragment", "urlparse", "urllib.parse"),
MovedAttribute("uses_netloc", "urlparse", "urllib.parse"),
MovedAttribute("uses_params", "urlparse", "urllib.parse"),
MovedAttribute("uses_query", "urlparse", "urllib.parse"),
MovedAttribute("uses_relative", "urlparse", "urllib.parse"),
]
for attr in _urllib_parse_moved_attributes:
setattr(Module_six_moves_urllib_parse, attr.name, attr)
del attr
sys.modules[__name__ + ".moves.urllib_parse"] = Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse")
sys.modules[__name__ + ".moves.urllib.parse"] = Module_six_moves_urllib_parse(__name__ + ".moves.urllib.parse")
Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes
_importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"),
"moves.urllib_parse", "moves.urllib.parse")
class Module_six_moves_urllib_error(types.ModuleType):
class Module_six_moves_urllib_error(_LazyModule):
"""Lazy loading of moved objects in six.moves.urllib_error"""
@ -240,11 +371,14 @@ for attr in _urllib_error_moved_attributes:
setattr(Module_six_moves_urllib_error, attr.name, attr)
del attr
sys.modules[__name__ + ".moves.urllib_error"] = Module_six_moves_urllib_error(__name__ + ".moves.urllib_error")
sys.modules[__name__ + ".moves.urllib.error"] = Module_six_moves_urllib_error(__name__ + ".moves.urllib.error")
Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes
_importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"),
"moves.urllib_error", "moves.urllib.error")
class Module_six_moves_urllib_request(types.ModuleType):
class Module_six_moves_urllib_request(_LazyModule):
"""Lazy loading of moved objects in six.moves.urllib_request"""
@ -281,16 +415,20 @@ _urllib_request_moved_attributes = [
MovedAttribute("urlcleanup", "urllib", "urllib.request"),
MovedAttribute("URLopener", "urllib", "urllib.request"),
MovedAttribute("FancyURLopener", "urllib", "urllib.request"),
MovedAttribute("proxy_bypass", "urllib", "urllib.request"),
]
for attr in _urllib_request_moved_attributes:
setattr(Module_six_moves_urllib_request, attr.name, attr)
del attr
sys.modules[__name__ + ".moves.urllib_request"] = Module_six_moves_urllib_request(__name__ + ".moves.urllib_request")
sys.modules[__name__ + ".moves.urllib.request"] = Module_six_moves_urllib_request(__name__ + ".moves.urllib.request")
Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes
_importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"),
"moves.urllib_request", "moves.urllib.request")
class Module_six_moves_urllib_response(types.ModuleType):
class Module_six_moves_urllib_response(_LazyModule):
"""Lazy loading of moved objects in six.moves.urllib_response"""
@ -304,11 +442,14 @@ for attr in _urllib_response_moved_attributes:
setattr(Module_six_moves_urllib_response, attr.name, attr)
del attr
sys.modules[__name__ + ".moves.urllib_response"] = Module_six_moves_urllib_response(__name__ + ".moves.urllib_response")
sys.modules[__name__ + ".moves.urllib.response"] = Module_six_moves_urllib_response(__name__ + ".moves.urllib.response")
Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes
_importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"),
"moves.urllib_response", "moves.urllib.response")
class Module_six_moves_urllib_robotparser(types.ModuleType):
class Module_six_moves_urllib_robotparser(_LazyModule):
"""Lazy loading of moved objects in six.moves.urllib_robotparser"""
@ -319,20 +460,27 @@ for attr in _urllib_robotparser_moved_attributes:
setattr(Module_six_moves_urllib_robotparser, attr.name, attr)
del attr
sys.modules[__name__ + ".moves.urllib_robotparser"] = Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib_robotparser")
sys.modules[__name__ + ".moves.urllib.robotparser"] = Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser")
Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes
_importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"),
"moves.urllib_robotparser", "moves.urllib.robotparser")
class Module_six_moves_urllib(types.ModuleType):
"""Create a six.moves.urllib namespace that resembles the Python 3 namespace"""
parse = sys.modules[__name__ + ".moves.urllib_parse"]
error = sys.modules[__name__ + ".moves.urllib_error"]
request = sys.modules[__name__ + ".moves.urllib_request"]
response = sys.modules[__name__ + ".moves.urllib_response"]
robotparser = sys.modules[__name__ + ".moves.urllib_robotparser"]
__path__ = [] # mark as package
parse = _importer._get_module("moves.urllib_parse")
error = _importer._get_module("moves.urllib_error")
request = _importer._get_module("moves.urllib_request")
response = _importer._get_module("moves.urllib_response")
robotparser = _importer._get_module("moves.urllib_robotparser")
def __dir__(self):
return ['parse', 'error', 'request', 'response', 'robotparser']
sys.modules[__name__ + ".moves.urllib"] = Module_six_moves_urllib(__name__ + ".moves.urllib")
_importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"),
"moves.urllib")
def add_move(move):
@ -359,11 +507,6 @@ if PY3:
_func_code = "__code__"
_func_defaults = "__defaults__"
_func_globals = "__globals__"
_iterkeys = "keys"
_itervalues = "values"
_iteritems = "items"
_iterlists = "lists"
else:
_meth_func = "im_func"
_meth_self = "im_self"
@ -373,11 +516,6 @@ else:
_func_defaults = "func_defaults"
_func_globals = "func_globals"
_iterkeys = "iterkeys"
_itervalues = "itervalues"
_iteritems = "iteritems"
_iterlists = "iterlists"
try:
advance_iterator = next
@ -400,6 +538,9 @@ if PY3:
create_bound_method = types.MethodType
def create_unbound_method(func, cls):
return func
Iterator = object
else:
def get_unbound_function(unbound):
@ -408,6 +549,9 @@ else:
def create_bound_method(func, obj):
return types.MethodType(func, obj, obj.__class__)
def create_unbound_method(func, cls):
return types.MethodType(func, None, cls)
class Iterator(object):
def next(self):
@ -426,74 +570,121 @@ get_function_defaults = operator.attrgetter(_func_defaults)
get_function_globals = operator.attrgetter(_func_globals)
def iterkeys(d, **kw):
"""Return an iterator over the keys of a dictionary."""
return iter(getattr(d, _iterkeys)(**kw))
if PY3:
def iterkeys(d, **kw):
return iter(d.keys(**kw))
def itervalues(d, **kw):
"""Return an iterator over the values of a dictionary."""
return iter(getattr(d, _itervalues)(**kw))
def itervalues(d, **kw):
return iter(d.values(**kw))
def iteritems(d, **kw):
"""Return an iterator over the (key, value) pairs of a dictionary."""
return iter(getattr(d, _iteritems)(**kw))
def iteritems(d, **kw):
return iter(d.items(**kw))
def iterlists(d, **kw):
"""Return an iterator over the (key, [values]) pairs of a dictionary."""
return iter(getattr(d, _iterlists)(**kw))
def iterlists(d, **kw):
return iter(d.lists(**kw))
viewkeys = operator.methodcaller("keys")
viewvalues = operator.methodcaller("values")
viewitems = operator.methodcaller("items")
else:
def iterkeys(d, **kw):
return d.iterkeys(**kw)
def itervalues(d, **kw):
return d.itervalues(**kw)
def iteritems(d, **kw):
return d.iteritems(**kw)
def iterlists(d, **kw):
return d.iterlists(**kw)
viewkeys = operator.methodcaller("viewkeys")
viewvalues = operator.methodcaller("viewvalues")
viewitems = operator.methodcaller("viewitems")
_add_doc(iterkeys, "Return an iterator over the keys of a dictionary.")
_add_doc(itervalues, "Return an iterator over the values of a dictionary.")
_add_doc(iteritems,
"Return an iterator over the (key, value) pairs of a dictionary.")
_add_doc(iterlists,
"Return an iterator over the (key, [values]) pairs of a dictionary.")
if PY3:
def b(s):
return s.encode("latin-1")
def u(s):
return s
unichr = chr
if sys.version_info[1] <= 1:
def int2byte(i):
return bytes((i,))
else:
# This is about 2x faster than the implementation above on 3.2+
int2byte = operator.methodcaller("to_bytes", 1, "big")
import struct
int2byte = struct.Struct(">B").pack
del struct
byte2int = operator.itemgetter(0)
indexbytes = operator.getitem
iterbytes = iter
import io
StringIO = io.StringIO
BytesIO = io.BytesIO
_assertCountEqual = "assertCountEqual"
if sys.version_info[1] <= 1:
_assertRaisesRegex = "assertRaisesRegexp"
_assertRegex = "assertRegexpMatches"
else:
_assertRaisesRegex = "assertRaisesRegex"
_assertRegex = "assertRegex"
else:
def b(s):
return s
# Workaround for standalone backslash
def u(s):
return unicode(s, "unicode_escape")
return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape")
unichr = unichr
int2byte = chr
def byte2int(bs):
return ord(bs[0])
def indexbytes(buf, i):
return ord(buf[i])
def iterbytes(buf):
return (ord(byte) for byte in buf)
iterbytes = functools.partial(itertools.imap, ord)
import StringIO
StringIO = BytesIO = StringIO.StringIO
_assertCountEqual = "assertItemsEqual"
_assertRaisesRegex = "assertRaisesRegexp"
_assertRegex = "assertRegexpMatches"
_add_doc(b, """Byte literal""")
_add_doc(u, """Text literal""")
if PY3:
import builtins
exec_ = getattr(builtins, "exec")
def assertCountEqual(self, *args, **kwargs):
return getattr(self, _assertCountEqual)(*args, **kwargs)
def assertRaisesRegex(self, *args, **kwargs):
return getattr(self, _assertRaisesRegex)(*args, **kwargs)
def assertRegex(self, *args, **kwargs):
return getattr(self, _assertRegex)(*args, **kwargs)
if PY3:
exec_ = getattr(moves.builtins, "exec")
def reraise(tp, value, tb=None):
if value is None:
value = tp()
if value.__traceback__ is not tb:
raise value.with_traceback(tb)
raise value
print_ = getattr(builtins, "print")
del builtins
else:
def exec_(_code_, _globs_=None, _locs_=None):
"""Execute code in a namespace."""
@ -507,20 +698,45 @@ else:
_locs_ = _globs_
exec("""exec _code_ in _globs_, _locs_""")
exec_("""def reraise(tp, value, tb=None):
raise tp, value, tb
""")
if sys.version_info[:2] == (3, 2):
exec_("""def raise_from(value, from_value):
if from_value is None:
raise value
raise value from from_value
""")
elif sys.version_info[:2] > (3, 2):
exec_("""def raise_from(value, from_value):
raise value from from_value
""")
else:
def raise_from(value, from_value):
raise value
print_ = getattr(moves.builtins, "print", None)
if print_ is None:
def print_(*args, **kwargs):
"""The new-style print function."""
"""The new-style print function for Python 2.4 and 2.5."""
fp = kwargs.pop("file", sys.stdout)
if fp is None:
return
def write(data):
if not isinstance(data, basestring):
data = str(data)
# If the file has an encoding, encode unicode with it.
if (isinstance(fp, file) and
isinstance(data, unicode) and
fp.encoding is not None):
errors = getattr(fp, "errors", None)
if errors is None:
errors = "strict"
data = data.encode(fp.encoding, errors)
fp.write(data)
want_unicode = False
sep = kwargs.pop("sep", None)
@ -557,21 +773,96 @@ else:
write(sep)
write(arg)
write(end)
if sys.version_info[:2] < (3, 3):
_print = print_
def print_(*args, **kwargs):
fp = kwargs.get("file", sys.stdout)
flush = kwargs.pop("flush", False)
_print(*args, **kwargs)
if flush and fp is not None:
fp.flush()
_add_doc(reraise, """Reraise an exception.""")
if sys.version_info[0:2] < (3, 4):
def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS,
updated=functools.WRAPPER_UPDATES):
def wrapper(f):
f = functools.wraps(wrapped, assigned, updated)(f)
f.__wrapped__ = wrapped
return f
return wrapper
else:
wraps = functools.wraps
def with_metaclass(meta, *bases):
"""Create a base class with a metaclass."""
return meta("NewBase", bases, {})
# This requires a bit of explanation: the basic idea is to make a dummy
# metaclass for one level of class instantiation that replaces itself with
# the actual metaclass.
class metaclass(meta):
def __new__(cls, name, this_bases, d):
return meta(name, bases, d)
return type.__new__(metaclass, 'temporary_class', (), {})
def add_metaclass(metaclass):
"""Class decorator for creating a class with a metaclass."""
def wrapper(cls):
orig_vars = cls.__dict__.copy()
slots = orig_vars.get('__slots__')
if slots is not None:
if isinstance(slots, str):
slots = [slots]
for slots_var in slots:
orig_vars.pop(slots_var)
orig_vars.pop('__dict__', None)
orig_vars.pop('__weakref__', None)
for slots_var in orig_vars.get('__slots__', ()):
orig_vars.pop(slots_var)
return metaclass(cls.__name__, cls.__bases__, orig_vars)
return wrapper
def python_2_unicode_compatible(klass):
"""
A decorator that defines __unicode__ and __str__ methods under Python 2.
Under Python 3 it does nothing.
To support Python 2 and 3 with a single code base, define a __str__ method
returning text and apply this decorator to the class.
"""
if PY2:
if '__str__' not in klass.__dict__:
raise ValueError("@python_2_unicode_compatible cannot be applied "
"to %s because it doesn't define __str__()." %
klass.__name__)
klass.__unicode__ = klass.__str__
klass.__str__ = lambda self: self.__unicode__().encode('utf-8')
return klass
# Complete the moves implementation.
# This code is at the end of this module to speed up module loading.
# Turn this module into a package.
__path__ = [] # required for PEP 302 and PEP 451
__package__ = __name__ # see PEP 366 @ReservedAssignment
if globals().get("__spec__") is not None:
__spec__.submodule_search_locations = [] # PEP 451 @UndefinedVariable
# Remove other six meta path importers, since they cause problems. This can
# happen if six is removed from sys.modules and then reloaded. (Setuptools does
# this for some reason.)
if sys.meta_path:
for i, importer in enumerate(sys.meta_path):
# Here's some real nastiness: Another "instance" of the six module might
# be floating around. Therefore, we can't use isinstance() to check for
# the six meta path importer, since the other six instance will have
# inserted an importer with different class.
if (type(importer).__name__ == "_SixMetaPathImporter" and
importer.name == __name__):
del sys.meta_path[i]
break
del i, importer
# Finally, add the importer to the meta path import hook.
sys.meta_path.append(_importer)

36
core/transmissionrpc/torrent.py
View file

@ -13,14 +13,15 @@ from six import integer_types, string_types, text_type, iteritems
def get_status_old(code):
"""Get the torrent status using old status codes"""
mapping = {
(1<<0): 'check pending',
(1<<1): 'checking',
(1<<2): 'downloading',
(1<<3): 'seeding',
(1<<4): 'stopped',
(1 << 0): 'check pending',
(1 << 1): 'checking',
(1 << 2): 'downloading',
(1 << 3): 'seeding',
(1 << 4): 'stopped',
}
return mapping[code]
def get_status_new(code):
"""Get the torrent status using new status codes"""
mapping = {
@ -34,6 +35,7 @@ def get_status_new(code):
}
return mapping[code]
class Torrent(object):
"""
Torrent is a class holding the data received from Transmission regarding a bittorrent transfer.
@ -71,14 +73,14 @@ class Torrent(object):
tid = self._fields['id'].value
name = self._get_name_string()
if isinstance(name, str):
return '<Torrent %d \"%s\">' % (tid, name)
return '<Torrent {0:d} \"{1}\">'.format(tid, name)
else:
return '<Torrent %d>' % (tid)
return '<Torrent {0:d}>'.format(tid)
def __str__(self):
name = self._get_name_string()
if isinstance(name, str):
return 'Torrent \"%s\"' % (name)
return 'Torrent \"{0}\"'.format(name)
else:
return 'Torrent'
@ -89,7 +91,7 @@ class Torrent(object):
try:
return self._fields[name].value
except KeyError:
raise AttributeError('No attribute %s' % name)
raise AttributeError('No attribute {0}'.format(name))
def _rpc_version(self):
"""Get the Transmission RPC API version."""
@ -99,8 +101,9 @@ class Torrent(object):
def _dirty_fields(self):
"""Enumerate changed fields"""
outgoing_keys = ['bandwidthPriority', 'downloadLimit', 'downloadLimited', 'peer_limit', 'queuePosition'
, 'seedIdleLimit', 'seedIdleMode', 'seedRatioLimit', 'seedRatioMode', 'uploadLimit', 'uploadLimited']
outgoing_keys = ['bandwidthPriority', 'downloadLimit', 'downloadLimited', 'peer_limit', 'queuePosition',
'seedIdleLimit', 'seedIdleMode', 'seedRatioLimit', 'seedRatioMode', 'uploadLimit',
'uploadLimited']
fields = []
for key in outgoing_keys:
if key in self._fields and self._fields[key].dirty:
@ -121,7 +124,6 @@ class Torrent(object):
"""
Update the torrent data from a Transmission JSON-RPC arguments dictionary
"""
fields = None
if isinstance(other, dict):
for key, value in iteritems(other):
self._fields[key.replace('-', '_')] = Field(value, False)
@ -264,13 +266,14 @@ class Torrent(object):
self._fields['downloadLimited'] = Field(True, True)
self._fields['downloadLimit'] = Field(limit, True)
self._push()
elif limit == None:
elif limit is None:
self._fields['downloadLimited'] = Field(False, True)
self._push()
else:
raise ValueError("Not a valid limit")
download_limit = property(_get_download_limit, _set_download_limit, None, "Download limit in Kbps or None. This is a mutator.")
download_limit = property(_get_download_limit, _set_download_limit, None,
"Download limit in Kbps or None. This is a mutator.")
def _get_peer_limit(self):
"""
@ -422,13 +425,14 @@ class Torrent(object):
self._fields['uploadLimited'] = Field(True, True)
self._fields['uploadLimit'] = Field(limit, True)
self._push()
elif limit == None:
elif limit is None:
self._fields['uploadLimited'] = Field(False, True)
self._push()
else:
raise ValueError("Not a valid limit")
upload_limit = property(_get_upload_limit, _set_upload_limit, None, "Upload limit in Kbps or None. This is a mutator.")
upload_limit = property(_get_upload_limit, _set_upload_limit, None,
"Upload limit in Kbps or None. This is a mutator.")
def _get_queue_position(self):
"""Get the queue position for this torrent."""

54
core/transmissionrpc/utils.py
View file

@ -2,14 +2,18 @@
# Copyright (c) 2008-2013 Erik Svensson <erik.public@gmail.com>
# Licensed under the MIT license.
import socket, datetime, logging, constants
import constants
import datetime
import logging
import socket
from collections import namedtuple
from constants import LOGGER
from constants import LOGGER
from six import string_types, iteritems
UNITS = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB']
def format_size(size):
"""
Format byte size into IEC prefixes, B, KiB, MiB ...
@ -19,14 +23,16 @@ def format_size(size):
while size >= 1024.0 and i < len(UNITS):
i += 1
size /= 1024.0
return (size, UNITS[i])
return size, UNITS[i]
def format_speed(size):
"""
Format bytes per second speed into IEC prefixes, B/s, KiB/s, MiB/s ...
"""
(size, unit) = format_size(size)
return (size, unit + '/s')
return size, '{unit}/s'.format(unit=unit)
def format_timedelta(delta):
"""
@ -34,7 +40,8 @@ def format_timedelta(delta):
"""
minutes, seconds = divmod(delta.seconds, 60)
hours, minutes = divmod(minutes, 60)
return '%d %02d:%02d:%02d' % (delta.days, hours, minutes, seconds)
return '{0:d} {1:02d}:{2:02d}:{3:02d}'.format(delta.days, hours, minutes, seconds)
def format_timestamp(timestamp, utc=False):
"""
@ -49,12 +56,14 @@ def format_timestamp(timestamp, utc=False):
else:
return '-'
class INetAddressError(Exception):
"""
Error parsing / generating a internet address.
"""
pass
def inet_address(address, default_port, default_address='localhost'):
"""
Parse internet address.
@ -71,18 +80,19 @@ def inet_address(address, default_port, default_address='localhost'):
try:
port = int(addr[1])
except ValueError:
raise INetAddressError('Invalid address "%s".' % address)
raise INetAddressError('Invalid address "{0}".'.format(address))
if len(addr[0]) == 0:
addr = default_address
else:
addr = addr[0]
else:
raise INetAddressError('Invalid address "%s".' % address)
raise INetAddressError('Invalid address "{0}".'.format(address))
try:
socket.getaddrinfo(addr, port, socket.AF_INET, socket.SOCK_STREAM)
except socket.gaierror:
raise INetAddressError('Cannot look up address "%s".' % address)
return (addr, port)
raise INetAddressError('Cannot look up address "{0}".'.format(address))
return addr, port
def rpc_bool(arg):
"""
@ -95,27 +105,31 @@ def rpc_bool(arg):
arg = arg.lower() in ['true', 'yes']
return 1 if bool(arg) else 0
TR_TYPE_MAP = {
'number' : int,
'string' : str,
'number': int,
'string': str,
'double': float,
'boolean' : rpc_bool,
'boolean': rpc_bool,
'array': list,
'object': dict
}
def make_python_name(name):
"""
Convert Transmission RPC name to python compatible name.
"""
return name.replace('-', '_')
def make_rpc_name(name):
"""
Convert python compatible name to Transmission RPC name.
"""
return name.replace('_', '-')
def argument_value_convert(method, argument, value, rpc_version):
"""
Check and fix Transmission RPC issues with regards to methods, arguments and values.
@ -125,7 +139,7 @@ def argument_value_convert(method, argument, value, rpc_version):
elif method in ('session-get', 'session-set'):
args = constants.SESSION_ARGS[method[-3:]]
else:
return ValueError('Method "%s" not supported' % (method))
return ValueError('Method "{0}" not supported'.format(method))
if argument in args:
info = args[argument]
invalid_version = True
@ -141,19 +155,18 @@ def argument_value_convert(method, argument, value, rpc_version):
if invalid_version:
if replacement:
LOGGER.warning(
'Replacing requested argument "%s" with "%s".'
% (argument, replacement))
'Replacing requested argument "{0}" with "{1}".'.format(argument, replacement))
argument = replacement
info = args[argument]
else:
raise ValueError(
'Method "%s" Argument "%s" does not exist in version %d.'
% (method, argument, rpc_version))
return (argument, TR_TYPE_MAP[info[0]](value))
'Method "{0}" Argument "{1}" does not exist in version {2:d}.'.format(method, argument, rpc_version))
return argument, TR_TYPE_MAP[info[0]](value)
else:
raise ValueError('Argument "%s" does not exists for method "%s".',
(argument, method))
def get_arguments(method, rpc_version):
"""
Get arguments for method in specified Transmission RPC version.
@ -163,7 +176,7 @@ def get_arguments(method, rpc_version):
elif method in ('session-get', 'session-set'):
args = constants.SESSION_ARGS[method[-3:]]
else:
return ValueError('Method "%s" not supported' % (method))
return ValueError('Method "{0}" not supported'.format(method))
accessible = []
for argument, info in iteritems(args):
valid_version = True
@ -175,6 +188,7 @@ def get_arguments(method, rpc_version):
accessible.append(argument)
return accessible
def add_stdout_logger(level='debug'):
"""
Add a stdout target for the transmissionrpc logging.
@ -189,6 +203,7 @@ def add_stdout_logger(level='debug'):
loghandler.setLevel(loglevel)
trpc_logger.addHandler(loghandler)
def add_file_logger(filepath, level='debug'):
"""
Add a stdout target for the transmissionrpc logging.
@ -203,4 +218,5 @@ def add_file_logger(filepath, level='debug'):
loghandler.setLevel(loglevel)
trpc_logger.addHandler(loghandler)
Field = namedtuple('Field', ['value', 'dirty'])

1
core/utorrent/__init__.py
View file

@ -0,0 +1 @@
# coding=utf-8

67
core/utorrent/client.py
View file

@ -1,96 +1,98 @@
#coding=utf8
import urllib
import urllib2
import urlparse
import cookielib
# coding=utf8
import json
import re
import StringIO
try:
import json
except ImportError:
import simplejson as json
from six import StringIO
from six.moves.http_cookiejar import CookieJar
from six.moves.urllib_error import HTTPError
from six.moves.urllib_parse import urljoin, urlencode
from six.moves.urllib_request import (
build_opener, install_opener,
HTTPBasicAuthHandler, HTTPCookieProcessor,
Request,
)
from core.utorrent.upload import MultiPartForm
class UTorrentClient(object):
class UTorrentClient(object):
def __init__(self, base_url, username, password):
self.base_url = base_url
self.username = username
self.password = password
self.opener = self._make_opener('uTorrent', base_url, username, password)
self.token = self._get_token()
#TODO refresh token, when necessary
# TODO refresh token, when necessary
def _make_opener(self, realm, base_url, username, password):
'''uTorrent API need HTTP Basic Auth and cookie support for token verify.'''
"""uTorrent API need HTTP Basic Auth and cookie support for token verify."""
auth_handler = urllib2.HTTPBasicAuthHandler()
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(realm=realm,
uri=base_url,
user=username,
passwd=password)
opener = urllib2.build_opener(auth_handler)
urllib2.install_opener(opener)
opener = build_opener(auth_handler)
install_opener(opener)
cookie_jar = cookielib.CookieJar()
cookie_handler = urllib2.HTTPCookieProcessor(cookie_jar)
cookie_jar = CookieJar()
cookie_handler = HTTPCookieProcessor(cookie_jar)
handlers = [auth_handler, cookie_handler]
opener = urllib2.build_opener(*handlers)
opener = build_opener(*handlers)
return opener
def _get_token(self):
url = urlparse.urljoin(self.base_url, 'token.html')
url = urljoin(self.base_url, 'token.html')
response = self.opener.open(url)
token_re = "<div id='token' style='display:none;'>([^<>]+)</div>"
match = re.search(token_re, response.read())
return match.group(1)
def list(self, **kwargs):
params = [('list', '1')]
params += kwargs.items()
return self._action(params)
def start(self, *hashes):
params = [('action', 'start'),]
params = [('action', 'start'), ]
for hash in hashes:
params.append(('hash', hash))
return self._action(params)
def stop(self, *hashes):
params = [('action', 'stop'),]
params = [('action', 'stop'), ]
for hash in hashes:
params.append(('hash', hash))
return self._action(params)
def pause(self, *hashes):
params = [('action', 'pause'),]
params = [('action', 'pause'), ]
for hash in hashes:
params.append(('hash', hash))
return self._action(params)
def forcestart(self, *hashes):
params = [('action', 'forcestart'),]
params = [('action', 'forcestart'), ]
for hash in hashes:
params.append(('hash', hash))
return self._action(params)
def remove(self, *hashes):
params = [('action', 'remove'),]
params = [('action', 'remove'), ]
for hash in hashes:
params.append(('hash', hash))
return self._action(params)
def removedata(self, *hashes):
params = [('action', 'removedata'),]
params = [('action', 'removedata'), ]
for hash in hashes:
params.append(('hash', hash))
return self._action(params)
def recheck(self, *hashes):
params = [('action', 'recheck'),]
params = [('action', 'recheck'), ]
for hash in hashes:
params.append(('hash', hash))
return self._action(params)
@ -124,9 +126,9 @@ class UTorrentClient(object):
return self._action(params, str(form), form.get_content_type())
def _action(self, params, body=None, content_type=None):
#about token, see https://github.com/bittorrent/webui/wiki/TokenSystem
url = self.base_url + '?token=' + self.token + '&' + urllib.urlencode(params)
request = urllib2.Request(url)
# about token, see https://github.com/bittorrent/webui/wiki/TokenSystem
url = '{url}?token={token}&{params}'.format(url=self.url, token=self.token, params=urlencode(params))
request = Request(url)
if body:
request.add_data(body)
@ -137,6 +139,5 @@ class UTorrentClient(object):
try:
response = self.opener.open(request)
return response.code, json.loads(response.read())
except urllib2.HTTPError,e:
except HTTPError:
raise

24
core/utorrent/upload.py
View file

@ -1,7 +1,8 @@
#code copied from http://www.doughellmann.com/PyMOTW/urllib2/
# coding=utf-8
# code copied from http://www.doughellmann.com/PyMOTW/urllib2/
from email.generator import _make_boundary as make_boundary
import itertools
import mimetools
import mimetypes
@ -11,11 +12,11 @@ class MultiPartForm(object):
def __init__(self):
self.form_fields = []
self.files = []
self.boundary = mimetools.choose_boundary()
self.boundary = make_boundary()
return
def get_content_type(self):
return 'multipart/form-data; boundary=%s' % self.boundary
return 'multipart/form-data; boundary={0}'.format(self.boundary)
def add_field(self, name, value):
"""Add a simple field to the form data."""
@ -37,12 +38,12 @@ class MultiPartForm(object):
# Once the list is built, return a string where each
# line is separated by '\r\n'.
parts = []
part_boundary = '--' + self.boundary
part_boundary = '--{boundary}'.format(boundary=self.boundary)
# Add the form fields
parts.extend(
[ part_boundary,
'Content-Disposition: form-data; name="%s"' % name,
[part_boundary,
'Content-Disposition: form-data; name="{0}"'.format(name),
'',
value,
]
@ -51,10 +52,9 @@ class MultiPartForm(object):
# Add the files to upload
parts.extend(
[ part_boundary,
'Content-Disposition: file; name="%s"; filename="%s"' % \
(field_name, filename),
'Content-Type: %s' % content_type,
[part_boundary,
'Content-Disposition: file; name="{0}"; filename="{1}"'.format(field_name, filename),
'Content-Type: {0}'.format(content_type),
'',
body,
]
@ -64,6 +64,6 @@ class MultiPartForm(object):
# Flatten the list and add closing boundary marker,
# then return CR+LF separated data
flattened = list(itertools.chain(*parts))
flattened.append('--' + self.boundary + '--')
flattened.append('--{boundary}--'.format(boundary=self.boundary))
flattened.append('')
return '\r\n'.join(flattened)

163
core/versionCheck.py
View file

@ -1,3 +1,4 @@
# coding=utf-8
# Author: Nic Wolfe <nic@wolfeden.ca>
# Modified by: echel0n
@ -15,7 +16,8 @@ import gh_api as github
import core
from core import logger
class CheckVersion():
class CheckVersion(object):
"""
Version check class meant to run as a thread object with the SB scheduler.
"""
@ -66,7 +68,7 @@ class CheckVersion():
logger.log(u"Version checking is disabled, not checking for the newest version")
return False
logger.log(u"Checking if " + self.install_type + " needs an update")
logger.log(u"Checking if {install} needs an update".format(install=self.install_type))
if not self.updater.need_update():
core.NEWEST_VERSION_STRING = None
logger.log(u"No update needed")
@ -79,7 +81,8 @@ class CheckVersion():
if self.updater.need_update():
return self.updater.update()
class UpdateManager():
class UpdateManager(object):
def get_github_repo_user(self):
return core.GIT_USER
@ -89,6 +92,7 @@ class UpdateManager():
def get_github_branch(self):
return core.GIT_BRANCH
class GitUpdateManager(UpdateManager):
def __init__(self):
self._git_path = self._find_working_git()
@ -102,24 +106,26 @@ class GitUpdateManager(UpdateManager):
self._num_commits_ahead = 0
def _git_error(self):
logger.debug('Unable to find your git executable - Set git_path in your autoProcessMedia.cfg OR delete your .git folder and run from source to enable updates.')
logger.debug(
'Unable to find your git executable - Set git_path in your autoProcessMedia.cfg OR delete your .git folder and run from source to enable updates.')
def _find_working_git(self):
test_cmd = 'version'
if core.GIT_PATH:
main_git = '"' + core.GIT_PATH + '"'
main_git = '"{git}"'.format(git=core.GIT_PATH)
else:
main_git = 'git'
logger.log(u"Checking if we can use git commands: " + main_git + ' ' + test_cmd, logger.DEBUG)
logger.log(u"Checking if we can use git commands: {git} {cmd}".format
(git=main_git, cmd=test_cmd), logger.DEBUG)
output, err, exit_status = self._run_git(main_git, test_cmd)
if exit_status == 0:
logger.log(u"Using: " + main_git, logger.DEBUG)
logger.log(u"Using: {git}".format(git=main_git), logger.DEBUG)
return main_git
else:
logger.log(u"Not using: " + main_git, logger.DEBUG)
logger.log(u"Not using: {git}".format(git=main_git), logger.DEBUG)
# trying alternatives
@ -137,33 +143,38 @@ class GitUpdateManager(UpdateManager):
logger.log(u"Trying known alternative git locations", logger.DEBUG)
for cur_git in alternative_git:
logger.log(u"Checking if we can use git commands: " + cur_git + ' ' + test_cmd, logger.DEBUG)
logger.log(u"Checking if we can use git commands: {git} {cmd}".format
(git=cur_git, cmd=test_cmd), logger.DEBUG)
output, err, exit_status = self._run_git(cur_git, test_cmd)
if exit_status == 0:
logger.log(u"Using: " + cur_git, logger.DEBUG)
logger.log(u"Using: {git}".format(git=cur_git), logger.DEBUG)
return cur_git
else:
logger.log(u"Not using: " + cur_git, logger.DEBUG)
logger.log(u"Not using: {git}".format(git=cur_git), logger.DEBUG)
# Still haven't found a working git
logger.debug('Unable to find your git executable - Set git_path in your autoProcessMedia.cfg OR delete your .git folder and run from source to enable updates.')
logger.debug('Unable to find your git executable - '
'Set git_path in your autoProcessMedia.cfg OR '
'delete your .git folder and run from source to enable updates.')
return None
def _run_git(self, git_path, args):
output = err = exit_status = None
output = None
err = None
if not git_path:
logger.log(u"No git specified, can't use git commands", logger.DEBUG)
exit_status = 1
return (output, err, exit_status)
return output, err, exit_status
cmd = git_path + ' ' + args
cmd = '{git} {args}'.format(git=git_path, args=args)
try:
logger.log(u"Executing " + cmd + " with your shell in " + core.PROGRAM_DIR, logger.DEBUG)
logger.log(u"Executing {cmd} with your shell in {directory}".format
(cmd=cmd, directory=core.PROGRAM_DIR), logger.DEBUG)
p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
shell=True, cwd=core.PROGRAM_DIR)
output, err = p.communicate()
@ -172,32 +183,26 @@ class GitUpdateManager(UpdateManager):
if output:
output = output.strip()
if core.LOG_GIT:
logger.log(u"git output: " + output, logger.DEBUG)
logger.log(u"git output: {output}".format(output=output), logger.DEBUG)
except OSError:
logger.log(u"Command " + cmd + " didn't work")
logger.log(u"Command {cmd} didn't work".format(cmd=cmd))
exit_status = 1
exit_status = 128 if ('fatal:' in output) or err else exit_status
if exit_status == 0:
logger.log(cmd + u" : returned successful", logger.DEBUG)
logger.log(u"{cmd} : returned successful".format(cmd=cmd), logger.DEBUG)
exit_status = 0
elif exit_status == 1:
if core.LOG_GIT:
logger.log(cmd + u" returned : " + output, logger.DEBUG)
exit_status = 1
elif exit_status == 128 or 'fatal:' in output or err:
if core.LOG_GIT:
logger.log(cmd + u" returned : " + output, logger.DEBUG)
exit_status = 128
elif core.LOG_GIT and exit_status in (1, 128):
logger.log(u"{cmd} returned : {output}".format
(cmd=cmd, output=output), logger.DEBUG)
else:
if core.LOG_GIT:
logger.log(cmd + u" returned : " + output + u", treat as error for now", logger.DEBUG)
logger.log(u"{cmd} returned : {output}, treat as error for now".format
(cmd=cmd, output=output), logger.DEBUG)
exit_status = 1
return (output, err, exit_status)
return output, err, exit_status
def _find_installed_version(self):
"""
@ -278,20 +283,18 @@ class GitUpdateManager(UpdateManager):
logger.log(u"git didn't return numbers for behind and ahead, not using it", logger.DEBUG)
return
logger.log(u"cur_commit = " + str(self._cur_commit_hash) + u" % (newest_commit)= " + str(self._newest_commit_hash)
+ u", num_commits_behind = " + str(self._num_commits_behind) + u", num_commits_ahead = " + str(
self._num_commits_ahead), logger.DEBUG)
logger.log(u"cur_commit = {current} % (newest_commit)= {new}, "
u"num_commits_behind = {x}, num_commits_ahead = {y}".format
(current=self._cur_commit_hash, new=self._newest_commit_hash,
x=self._num_commits_behind, y=self._num_commits_ahead), logger.DEBUG)
def set_newest_text(self):
if self._num_commits_ahead:
logger.log(u"Local branch is ahead of " + self.branch + ". Automatic update not possible.", logger.ERROR)
elif self._num_commits_behind > 0:
newest_text = 'There is a newer version available '
newest_text += " (you're " + str(self._num_commits_behind) + " commit"
if self._num_commits_behind > 1:
newest_text += 's'
newest_text += ' behind)'
logger.log(newest_text, logger.MESSAGE)
logger.log(u"Local branch is ahead of {branch}. Automatic update not possible.".format
(branch=self.branch), logger.ERROR)
elif self._num_commits_behind:
logger.log(u"There is a newer version available (you're {x} commit{s} behind)".format
(x=self._num_commits_behind, s=u's' if self._num_commits_behind > 1 else u''), logger.MESSAGE)
else:
return
@ -305,8 +308,8 @@ class GitUpdateManager(UpdateManager):
else:
try:
self._check_github_for_update()
except Exception, e:
logger.log(u"Unable to contact github, can't check for update: " + repr(e), logger.ERROR)
except Exception as error:
logger.log(u"Unable to contact github, can't check for update: {msg!r}".format(msg=error), logger.ERROR)
return False
if self._num_commits_behind > 0:
@ -320,7 +323,7 @@ class GitUpdateManager(UpdateManager):
on the call's success.
"""
output, err, exit_status = self._run_git(self._git_path, 'pull origin ' + self.branch) # @UnusedVariable
output, err, exit_status = self._run_git(self._git_path, 'pull origin {branch}'.format(branch=self.branch)) # @UnusedVariable
if exit_status == 0:
return True
@ -349,8 +352,8 @@ class SourceUpdateManager(UpdateManager):
try:
with open(version_file, 'r') as fp:
self._cur_commit_hash = fp.read().strip(' \n\r')
except EnvironmentError, e:
logger.log(u"Unable to open 'version.txt': " + str(e), logger.DEBUG)
except EnvironmentError as error:
logger.log(u"Unable to open 'version.txt': {msg}".format(msg=error), logger.DEBUG)
if not self._cur_commit_hash:
self._cur_commit_hash = None
@ -363,8 +366,8 @@ class SourceUpdateManager(UpdateManager):
try:
self._check_github_for_update()
except Exception, e:
logger.log(u"Unable to contact github, can't check for update: " + repr(e), logger.ERROR)
except Exception as error:
logger.log(u"Unable to contact github, can't check for update: {msg!r}".format(msg=error), logger.ERROR)
return False
if not self._cur_commit_hash or self._num_commits_behind > 0:
@ -410,8 +413,8 @@ class SourceUpdateManager(UpdateManager):
# when _cur_commit_hash doesn't match anything _num_commits_behind == 100
self._num_commits_behind += 1
logger.log(u"cur_commit = " + str(self._cur_commit_hash) + u" % (newest_commit)= " + str(self._newest_commit_hash)
+ u", num_commits_behind = " + str(self._num_commits_behind), logger.DEBUG)
logger.log(u"cur_commit = {current} % (newest_commit)= {new}, num_commits_behind = {x}".format
(current=self._cur_commit_hash, new=self._newest_commit_hash, x=self._num_commits_behind), logger.DEBUG)
def set_newest_text(self):
@ -421,12 +424,8 @@ class SourceUpdateManager(UpdateManager):
if not self._cur_commit_hash:
logger.log(u"Unknown current version number, don't know if we should update or not", logger.ERROR)
elif self._num_commits_behind > 0:
newest_text = 'There is a newer version available'
newest_text += " (you're " + str(self._num_commits_behind) + " commit"
if self._num_commits_behind > 1:
newest_text += "s"
newest_text += " behind)"
logger.log(newest_text, logger.MESSAGE)
logger.log(u"There is a newer version available (you're {x} commit{s} behind)".format
(x=self._num_commits_behind, s=u's' if self._num_commits_behind > 1 else u''), logger.MESSAGE)
else:
return
@ -434,8 +433,8 @@ class SourceUpdateManager(UpdateManager):
"""
Downloads the latest source tarball from github and installs it over the existing version.
"""
base_url = 'https://github.com/' + self.github_repo_user + '/' + self.github_repo
tar_download_url = base_url + '/tarball/' + self.branch
tar_download_url = 'https://github.com/{org}/{repo}/tarball/{branch}'.format(
org=self.github_repo_user, repo=self.github_repo, branch=self.branch)
version_path = os.path.join(core.PROGRAM_DIR, u'version.txt')
try:
@ -443,61 +442,65 @@ class SourceUpdateManager(UpdateManager):
sb_update_dir = os.path.join(core.PROGRAM_DIR, u'sb-update')
if os.path.isdir(sb_update_dir):
logger.log(u"Clearing out update folder " + sb_update_dir + " before extracting")
logger.log(u"Clearing out update folder {dir} before extracting".format(dir=sb_update_dir))
shutil.rmtree(sb_update_dir)
logger.log(u"Creating update folder " + sb_update_dir + " before extracting")
logger.log(u"Creating update folder {dir} before extracting".format(dir=sb_update_dir))
os.makedirs(sb_update_dir)
# retrieve file
logger.log(u"Downloading update from " + repr(tar_download_url))
logger.log(u"Downloading update from {url!r}".format(url=tar_download_url))
tar_download_path = os.path.join(sb_update_dir, u'nzbtomedia-update.tar')
urllib.urlretrieve(tar_download_url, tar_download_path)
if not os.path.isfile(tar_download_path):
logger.log(u"Unable to retrieve new version from " + tar_download_url + ", can't update", logger.ERROR)
logger.log(u"Unable to retrieve new version from {url}, can't update".format
(url=tar_download_url), logger.ERROR)
return False
if not tarfile.is_tarfile(tar_download_path):
logger.log(u"Retrieved version from " + tar_download_url + " is corrupt, can't update", logger.ERROR)
logger.log(u"Retrieved version from {url} is corrupt, can't update".format
(url=tar_download_url), logger.ERROR)
return False
# extract to sb-update dir
logger.log(u"Extracting file " + tar_download_path)
logger.log(u"Extracting file {path}".format(path=tar_download_path))
tar = tarfile.open(tar_download_path)
tar.extractall(sb_update_dir)
tar.close()
# delete .tar.gz
logger.log(u"Deleting file " + tar_download_path)
logger.log(u"Deleting file {path}".format(path=tar_download_path))
os.remove(tar_download_path)
# find update dir name
update_dir_contents = [x for x in os.listdir(sb_update_dir) if
os.path.isdir(os.path.join(sb_update_dir, x))]
if len(update_dir_contents) != 1:
logger.log(u"Invalid update data, update failed: " + str(update_dir_contents), logger.ERROR)
logger.log(u"Invalid update data, update failed: {0}".format(update_dir_contents), logger.ERROR)
return False
content_dir = os.path.join(sb_update_dir, update_dir_contents[0])
# walk temp folder and move files to main folder
logger.log(u"Moving files from " + content_dir + " to " + core.PROGRAM_DIR)
logger.log(u"Moving files from {source} to {destination}".format
(source=content_dir, destination=core.PROGRAM_DIR))
for dirname, dirnames, filenames in os.walk(content_dir): # @UnusedVariable
dirname = dirname[len(content_dir) + 1:]
for curfile in filenames:
old_path = os.path.join(content_dir, dirname, curfile)
new_path = os.path.join(core.PROGRAM_DIR, dirname, curfile)
#Avoid DLL access problem on WIN32/64
#These files needing to be updated manually
#or find a way to kill the access from memory
# Avoid DLL access problem on WIN32/64
# These files needing to be updated manually
# or find a way to kill the access from memory
if curfile in ('unrar.dll', 'unrar64.dll'):
try:
os.chmod(new_path, stat.S_IWRITE)
os.remove(new_path)
os.renames(old_path, new_path)
except Exception, e:
logger.log(u"Unable to update " + new_path + ': ' + str(e), logger.DEBUG)
except Exception as error:
logger.log(u"Unable to update {path}: {msg}".format
(path=new_path, msg=error), logger.DEBUG)
os.remove(old_path) # Trash the updated file without moving in new path
continue
@ -509,13 +512,15 @@ class SourceUpdateManager(UpdateManager):
try:
with open(version_path, 'w') as ver_file:
ver_file.write(self._newest_commit_hash)
except EnvironmentError, e:
logger.log(u"Unable to write version file, update not complete: " + str(e), logger.ERROR)
except EnvironmentError as error:
logger.log(u"Unable to write version file, update not complete: {msg}".format
(msg=error), logger.ERROR)
return False
except Exception, e:
logger.log(u"Error while trying to update: " + str(e), logger.ERROR)
logger.log(u"Traceback: " + traceback.format_exc(), logger.DEBUG)
except Exception as error:
logger.log(u"Error while trying to update: {msg}".format
(msg=error), logger.ERROR)
logger.log(u"Traceback: {error}".format(error=traceback.format_exc()), logger.DEBUG)
return False
return True

1
libs/backports.functools_lru_cache-1.2.1-py3.5-nspkg.pth Normal file
View file

@ -0,0 +1 @@
import sys, types, os;p = os.path.join(sys._getframe(1).f_locals['sitedir'], *('backports',));ie = os.path.exists(os.path.join(p,'__init__.py'));m = not ie and sys.modules.setdefault('backports', types.ModuleType('backports'));mp = (m or []) and m.__dict__.setdefault('__path__',[]);(p not in mp) and mp.append(p)

184
libs/backports/functools_lru_cache.py Normal file
View file

@ -0,0 +1,184 @@
from __future__ import absolute_import
import functools
from collections import namedtuple
from threading import RLock
_CacheInfo = namedtuple("CacheInfo", ["hits", "misses", "maxsize", "currsize"])
@functools.wraps(functools.update_wrapper)
def update_wrapper(wrapper,
wrapped,
assigned = functools.WRAPPER_ASSIGNMENTS,
updated = functools.WRAPPER_UPDATES):
"""
Patch two bugs in functools.update_wrapper.
"""
# workaround for http://bugs.python.org/issue3445
assigned = tuple(attr for attr in assigned if hasattr(wrapped, attr))
wrapper = functools.update_wrapper(wrapper, wrapped, assigned, updated)
# workaround for https://bugs.python.org/issue17482
wrapper.__wrapped__ = wrapped
return wrapper
class _HashedSeq(list):
__slots__ = 'hashvalue'
def __init__(self, tup, hash=hash):
self[:] = tup
self.hashvalue = hash(tup)
def __hash__(self):
return self.hashvalue
def _make_key(args, kwds, typed,
kwd_mark=(object(),),
fasttypes=set([int, str, frozenset, type(None)]),
sorted=sorted, tuple=tuple, type=type, len=len):
'Make a cache key from optionally typed positional and keyword arguments'
key = args
if kwds:
sorted_items = sorted(kwds.items())
key += kwd_mark
for item in sorted_items:
key += item
if typed:
key += tuple(type(v) for v in args)
if kwds:
key += tuple(type(v) for k, v in sorted_items)
elif len(key) == 1 and type(key[0]) in fasttypes:
return key[0]
return _HashedSeq(key)
def lru_cache(maxsize=100, typed=False):
"""Least-recently-used cache decorator.
If *maxsize* is set to None, the LRU features are disabled and the cache
can grow without bound.
If *typed* is True, arguments of different types will be cached separately.
For example, f(3.0) and f(3) will be treated as distinct calls with
distinct results.
Arguments to the cached function must be hashable.
View the cache statistics named tuple (hits, misses, maxsize, currsize) with
f.cache_info(). Clear the cache and statistics with f.cache_clear().
Access the underlying function with f.__wrapped__.
See: http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used
"""
# Users should only access the lru_cache through its public API:
# cache_info, cache_clear, and f.__wrapped__
# The internals of the lru_cache are encapsulated for thread safety and
# to allow the implementation to change (including a possible C version).
def decorating_function(user_function):
cache = dict()
stats = [0, 0] # make statistics updateable non-locally
HITS, MISSES = 0, 1 # names for the stats fields
make_key = _make_key
cache_get = cache.get # bound method to lookup key or return None
_len = len # localize the global len() function
lock = RLock() # because linkedlist updates aren't threadsafe
root = [] # root of the circular doubly linked list
root[:] = [root, root, None, None] # initialize by pointing to self
nonlocal_root = [root] # make updateable non-locally
PREV, NEXT, KEY, RESULT = 0, 1, 2, 3 # names for the link fields
if maxsize == 0:
def wrapper(*args, **kwds):
# no caching, just do a statistics update after a successful call
result = user_function(*args, **kwds)
stats[MISSES] += 1
return result
elif maxsize is None:
def wrapper(*args, **kwds):
# simple caching without ordering or size limit
key = make_key(args, kwds, typed)
result = cache_get(key, root) # root used here as a unique not-found sentinel
if result is not root:
stats[HITS] += 1
return result
result = user_function(*args, **kwds)
cache[key] = result
stats[MISSES] += 1
return result
else:
def wrapper(*args, **kwds):
# size limited caching that tracks accesses by recency
key = make_key(args, kwds, typed) if kwds or typed else args
with lock:
link = cache_get(key)
if link is not None:
# record recent use of the key by moving it to the front of the list
root, = nonlocal_root
link_prev, link_next, key, result = link
link_prev[NEXT] = link_next
link_next[PREV] = link_prev
last = root[PREV]
last[NEXT] = root[PREV] = link
link[PREV] = last
link[NEXT] = root
stats[HITS] += 1
return result
result = user_function(*args, **kwds)
with lock:
root, = nonlocal_root
if key in cache:
# getting here means that this same key was added to the
# cache while the lock was released. since the link
# update is already done, we need only return the
# computed result and update the count of misses.
pass
elif _len(cache) >= maxsize:
# use the old root to store the new key and result
oldroot = root
oldroot[KEY] = key
oldroot[RESULT] = result
# empty the oldest link and make it the new root
root = nonlocal_root[0] = oldroot[NEXT]
oldkey = root[KEY]
root[KEY] = root[RESULT] = None
# now update the cache dictionary for the new links
del cache[oldkey]
cache[key] = oldroot
else:
# put result in a new link at the front of the list
last = root[PREV]
link = [last, root, key, result]
last[NEXT] = root[PREV] = cache[key] = link
stats[MISSES] += 1
return result
def cache_info():
"""Report cache statistics"""
with lock:
return _CacheInfo(stats[HITS], stats[MISSES], maxsize, len(cache))
def cache_clear():
"""Clear the cache and cache statistics"""
with lock:
cache.clear()
root = nonlocal_root[0]
root[:] = [root, root, None, None]
stats[:] = [0, 0]
wrapper.__wrapped__ = user_function
wrapper.cache_info = cache_info
wrapper.cache_clear = cache_clear
return update_wrapper(wrapper, user_function)
return decorating_function

32
libs/beets/__init__.py
View file

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2013, Adrian Sampson.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
@ -12,15 +13,30 @@
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
# This particular version has been slightly modified to work with headphones
# https://github.com/rembo10/headphones
from __future__ import division, absolute_import, print_function
__version__ = '1.3.4'
__author__ = 'Adrian Sampson <adrian@radbox.org>'
import os
import beets.library
from beets.util import confit
Library = beets.library.Library
__version__ = u'1.3.18'
__author__ = u'Adrian Sampson <adrian@radbox.org>'
config = confit.LazyConfig('beets', __name__)
class IncludeLazyConfig(confit.LazyConfig):
"""A version of Confit's LazyConfig that also merges in data from
YAML files specified in an `include` setting.
"""
def read(self, user=True, defaults=True):
super(IncludeLazyConfig, self).read(user, defaults)
try:
for view in self['include']:
filename = view.as_filename()
if os.path.isfile(filename):
self.set_file(filename)
except confit.NotFoundError:
pass
config = IncludeLazyConfig('beets', __name__)

200
libs/beets/art.py Normal file
View file

@ -0,0 +1,200 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""High-level utilities for manipulating image files associated with
music and items' embedded album art.
"""
from __future__ import division, absolute_import, print_function
import subprocess
import platform
from tempfile import NamedTemporaryFile
import imghdr
import os
from beets.util import displayable_path, syspath
from beets.util.artresizer import ArtResizer
from beets import mediafile
def mediafile_image(image_path, maxwidth=None):
"""Return a `mediafile.Image` object for the path.
"""
with open(syspath(image_path), 'rb') as f:
data = f.read()
return mediafile.Image(data, type=mediafile.ImageType.front)
def get_art(log, item):
# Extract the art.
try:
mf = mediafile.MediaFile(syspath(item.path))
except mediafile.UnreadableFileError as exc:
log.warning(u'Could not extract art from {0}: {1}',
displayable_path(item.path), exc)
return
return mf.art
def embed_item(log, item, imagepath, maxwidth=None, itempath=None,
compare_threshold=0, ifempty=False, as_album=False):
"""Embed an image into the item's media file.
"""
# Conditions and filters.
if compare_threshold:
if not check_art_similarity(log, item, imagepath, compare_threshold):
log.info(u'Image not similar; skipping.')
return
if ifempty and get_art(log, item):
log.info(u'media file already contained art')
return
if maxwidth and not as_album:
imagepath = resize_image(log, imagepath, maxwidth)
# Get the `Image` object from the file.
try:
log.debug(u'embedding {0}', displayable_path(imagepath))
image = mediafile_image(imagepath, maxwidth)
except IOError as exc:
log.warning(u'could not read image file: {0}', exc)
return
# Make sure the image kind is safe (some formats only support PNG
# and JPEG).
if image.mime_type not in ('image/jpeg', 'image/png'):
log.info('not embedding image of unsupported type: {}',
image.mime_type)
return
item.try_write(path=itempath, tags={'images': [image]})
def embed_album(log, album, maxwidth=None, quiet=False,
compare_threshold=0, ifempty=False):
"""Embed album art into all of the album's items.
"""
imagepath = album.artpath
if not imagepath:
log.info(u'No album art present for {0}', album)
return
if not os.path.isfile(syspath(imagepath)):
log.info(u'Album art not found at {0} for {1}',
displayable_path(imagepath), album)
return
if maxwidth:
imagepath = resize_image(log, imagepath, maxwidth)
log.info(u'Embedding album art into {0}', album)
for item in album.items():
embed_item(log, item, imagepath, maxwidth, None,
compare_threshold, ifempty, as_album=True)
def resize_image(log, imagepath, maxwidth):
"""Returns path to an image resized to maxwidth.
"""
log.debug(u'Resizing album art to {0} pixels wide', maxwidth)
imagepath = ArtResizer.shared.resize(maxwidth, syspath(imagepath))
return imagepath
def check_art_similarity(log, item, imagepath, compare_threshold):
"""A boolean indicating if an image is similar to embedded item art.
"""
with NamedTemporaryFile(delete=True) as f:
art = extract(log, f.name, item)
if art:
is_windows = platform.system() == "Windows"
# Converting images to grayscale tends to minimize the weight
# of colors in the diff score.
convert_proc = subprocess.Popen(
[b'convert', syspath(imagepath), syspath(art),
b'-colorspace', b'gray', b'MIFF:-'],
stdout=subprocess.PIPE,
close_fds=not is_windows,
)
compare_proc = subprocess.Popen(
[b'compare', b'-metric', b'PHASH', b'-', b'null:'],
stdin=convert_proc.stdout,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=not is_windows,
)
convert_proc.stdout.close()
stdout, stderr = compare_proc.communicate()
if compare_proc.returncode:
if compare_proc.returncode != 1:
log.debug(u'IM phashes compare failed for {0}, {1}',
displayable_path(imagepath),
displayable_path(art))
return
out_str = stderr
else:
out_str = stdout
try:
phash_diff = float(out_str)
except ValueError:
log.debug(u'IM output is not a number: {0!r}', out_str)
return
log.debug(u'compare PHASH score is {0}', phash_diff)
return phash_diff <= compare_threshold
return True
def extract(log, outpath, item):
art = get_art(log, item)
if not art:
log.info(u'No album art present in {0}, skipping.', item)
return
# Add an extension to the filename.
ext = imghdr.what(None, h=art)
if not ext:
log.warning(u'Unknown image type in {0}.',
displayable_path(item.path))
return
outpath += b'.' + ext
log.info(u'Extracting album art from: {0} to: {1}',
item, displayable_path(outpath))
with open(syspath(outpath), 'wb') as f:
f.write(art)
return outpath
def extract_first(log, outpath, items):
for item in items:
real_path = extract(log, outpath, item)
if real_path:
return real_path
def clear(log, lib, query):
items = lib.items(query)
log.info(u'Clearing album art from {0} items', len(items))
for item in items:
log.debug(u'Clearing art for {0}', item)
item.try_write(tags={'images': None})

150
libs/beets/autotag/__init__.py
View file

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2013, Adrian Sampson.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
@ -14,135 +15,23 @@
"""Facilities for automatically determining files' correct metadata.
"""
import os
import logging
import re
from beets import library, mediafile, config
from beets.util import sorted_walk, ancestry, displayable_path
from __future__ import division, absolute_import, print_function
from beets import logging
from beets import config
# Parts of external interface.
from .hooks import AlbumInfo, TrackInfo, AlbumMatch, TrackMatch
from .match import tag_item, tag_album
from .match import recommendation
from .hooks import AlbumInfo, TrackInfo, AlbumMatch, TrackMatch # noqa
from .match import tag_item, tag_album # noqa
from .match import Recommendation # noqa
# Global logger.
log = logging.getLogger('beets')
# Constants for directory walker.
MULTIDISC_MARKERS = (r'dis[ck]', r'cd')
MULTIDISC_PAT_FMT = r'^(.*%s[\W_]*)\d'
# Additional utilities for the main interface.
def albums_in_dir(path):
"""Recursively searches the given directory and returns an iterable
of (paths, items) where paths is a list of directories and items is
a list of Items that is probably an album. Specifically, any folder
containing any media files is an album.
"""
collapse_pat = collapse_paths = collapse_items = None
for root, dirs, files in sorted_walk(path,
ignore=config['ignore'].as_str_seq(),
logger=log):
# Get a list of items in the directory.
items = []
for filename in files:
try:
i = library.Item.from_path(os.path.join(root, filename))
except mediafile.FileTypeError:
pass
except mediafile.UnreadableFileError:
log.warn(u'unreadable file: {0}'.format(
displayable_path(filename))
)
else:
items.append(i)
# If we're currently collapsing the constituent directories in a
# multi-disc album, check whether we should continue collapsing
# and add the current directory. If so, just add the directory
# and move on to the next directory. If not, stop collapsing.
if collapse_paths:
if (not collapse_pat and collapse_paths[0] in ancestry(root)) or \
(collapse_pat and
collapse_pat.match(os.path.basename(root))):
# Still collapsing.
collapse_paths.append(root)
collapse_items += items
continue
else:
# Collapse finished. Yield the collapsed directory and
# proceed to process the current one.
if collapse_items:
yield collapse_paths, collapse_items
collapse_pat = collapse_paths = collapse_items = None
# Check whether this directory looks like the *first* directory
# in a multi-disc sequence. There are two indicators: the file
# is named like part of a multi-disc sequence (e.g., "Title Disc
# 1") or it contains no items but only directories that are
# named in this way.
start_collapsing = False
for marker in MULTIDISC_MARKERS:
marker_pat = re.compile(MULTIDISC_PAT_FMT % marker, re.I)
match = marker_pat.match(os.path.basename(root))
# Is this directory the root of a nested multi-disc album?
if dirs and not items:
# Check whether all subdirectories have the same prefix.
start_collapsing = True
subdir_pat = None
for subdir in dirs:
# The first directory dictates the pattern for
# the remaining directories.
if not subdir_pat:
match = marker_pat.match(subdir)
if match:
subdir_pat = re.compile(r'^%s\d' %
re.escape(match.group(1)), re.I)
else:
start_collapsing = False
break
# Subsequent directories must match the pattern.
elif not subdir_pat.match(subdir):
start_collapsing = False
break
# If all subdirectories match, don't check other
# markers.
if start_collapsing:
break
# Is this directory the first in a flattened multi-disc album?
elif match:
start_collapsing = True
# Set the current pattern to match directories with the same
# prefix as this one, followed by a digit.
collapse_pat = re.compile(r'^%s\d' %
re.escape(match.group(1)), re.I)
break
# If either of the above heuristics indicated that this is the
# beginning of a multi-disc album, initialize the collapsed
# directory and item lists and check the next directory.
if start_collapsing:
# Start collapsing; continue to the next iteration.
collapse_paths = [root]
collapse_items = items
continue
# If it's nonempty, yield it.
if items:
yield [root], items
# Clear out any unfinished collapse.
if collapse_paths and collapse_items:
yield collapse_paths, collapse_items
def apply_item_metadata(item, track_info):
"""Set an item's metadata from its matched TrackInfo object.
"""
@ -153,9 +42,12 @@ def apply_item_metadata(item, track_info):
item.mb_trackid = track_info.track_id
if track_info.artist_id:
item.mb_artistid = track_info.artist_id
if track_info.data_source:
item.data_source = track_info.data_source
# At the moment, the other metadata is left intact (including album
# and track number). Perhaps these should be emptied?
def apply_metadata(album_info, mapping):
"""Set the items' metadata to match an AlbumInfo object using a
mapping from Items to TrackInfo objects.
@ -171,8 +63,8 @@ def apply_metadata(album_info, mapping):
# Artist sort and credit names.
item.artist_sort = track_info.artist_sort or album_info.artist_sort
item.artist_credit = track_info.artist_credit or \
album_info.artist_credit
item.artist_credit = (track_info.artist_credit or
album_info.artist_credit)
item.albumartist_sort = album_info.artist_sort
item.albumartist_credit = album_info.artist_credit
@ -203,7 +95,11 @@ def apply_metadata(album_info, mapping):
item.title = track_info.title
if config['per_disc_numbering']:
item.track = track_info.medium_index or track_info.index
# We want to let the track number be zero, but if the medium index
# is not provided we need to fall back to the overall index.
item.track = track_info.medium_index
if item.track is None:
item.track = track_info.index
item.tracktotal = track_info.medium_total or len(album_info.tracks)
else:
item.track = track_info.index
@ -235,13 +131,13 @@ def apply_metadata(album_info, mapping):
'language',
'country',
'albumstatus',
'media',
'albumdisambig'):
'albumdisambig',
'data_source',):
value = getattr(album_info, field)
if value is not None:
item[field] = value
if track_info.disctitle is not None:
item.disctitle = track_info.disctitle
# Headphones seal of approval
item.comments = 'tagged by headphones/beets'
if track_info.media is not None:
item.media = track_info.media

119
libs/beets/autotag/hooks.py
View file

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2013, Adrian Sampson.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
@ -13,14 +14,16 @@
# included in all copies or substantial portions of the Software.
"""Glue between metadata sources and the matching logic."""
import logging
from __future__ import division, absolute_import, print_function
from collections import namedtuple
import re
from beets import logging
from beets import plugins
from beets import config
from beets.autotag import mb
from beets.util import levenshtein
from jellyfish import levenshtein_distance
from unidecode import unidecode
log = logging.getLogger('beets')
@ -109,13 +112,14 @@ class AlbumInfo(object):
'catalognum', 'script', 'language', 'country',
'albumstatus', 'albumdisambig', 'artist_credit', 'media']:
value = getattr(self, fld)
if isinstance(value, str):
if isinstance(value, bytes):
setattr(self, fld, value.decode(codec, 'ignore'))
if self.tracks:
for track in self.tracks:
track.decode(codec)
class TrackInfo(object):
"""Describes a canonical track present on a release. Appears as part
of an AlbumInfo's ``tracks`` list. Consists of these data members:
@ -126,12 +130,15 @@ class TrackInfo(object):
- ``artist_id``
- ``length``: float: duration of the track in seconds
- ``index``: position on the entire release
- ``media``: delivery mechanism (Vinyl, etc.)
- ``medium``: the disc number this track appears on in the album
- ``medium_index``: the track's position on the disc
- ``medium_total``: the number of tracks on the item's disc
- ``artist_sort``: name of the track artist for sorting
- ``disctitle``: name of the individual medium (subtitle)
- ``artist_credit``: Recording-specific artist name
- ``data_source``: The original data source (MusicBrainz, Discogs, etc.)
- ``data_url``: The data source release URL.
Only ``title`` and ``track_id`` are required. The rest of the fields
may be None. The indices ``index``, ``medium``, and ``medium_index``
@ -140,13 +147,15 @@ class TrackInfo(object):
def __init__(self, title, track_id, artist=None, artist_id=None,
length=None, index=None, medium=None, medium_index=None,
medium_total=None, artist_sort=None, disctitle=None,
artist_credit=None, data_source=None, data_url=None):
artist_credit=None, data_source=None, data_url=None,
media=None):
self.title = title
self.track_id = track_id
self.artist = artist
self.artist_id = artist_id
self.length = length
self.index = index
self.media = media
self.medium = medium
self.medium_index = medium_index
self.medium_total = medium_total
@ -162,9 +171,9 @@ class TrackInfo(object):
to Unicode.
"""
for fld in ['title', 'artist', 'medium', 'artist_sort', 'disctitle',
'artist_credit']:
'artist_credit', 'media']:
value = getattr(self, fld)
if isinstance(value, str):
if isinstance(value, bytes):
setattr(self, fld, value.decode(codec, 'ignore'))
@ -187,27 +196,33 @@ SD_REPLACE = [
(r'&', 'and'),
]
def _string_dist_basic(str1, str2):
"""Basic edit distance between two strings, ignoring
non-alphanumeric characters and case. Comparisons are based on a
transliteration/lowering to ASCII characters. Normalized by string
length.
"""
str1 = unidecode(str1)
str2 = unidecode(str2)
assert isinstance(str1, unicode)
assert isinstance(str2, unicode)
str1 = unidecode(str1).decode('ascii')
str2 = unidecode(str2).decode('ascii')
str1 = re.sub(r'[^a-z0-9]', '', str1.lower())
str2 = re.sub(r'[^a-z0-9]', '', str2.lower())
if not str1 and not str2:
return 0.0
return levenshtein(str1, str2) / float(max(len(str1), len(str2)))
return levenshtein_distance(str1, str2) / float(max(len(str1), len(str2)))
def string_dist(str1, str2):
"""Gives an "intuitive" edit distance between two strings. This is
an edit distance, normalized by the string length, with a number of
tweaks that reflect intuition about text.
"""
if str1 == None and str2 == None: return 0.0
if str1 == None or str2 == None: return 1.0
if str1 is None and str2 is None:
return 0.0
if str1 is None or str2 is None:
return 1.0
str1 = str1.lower()
str2 = str2.lower()
@ -217,9 +232,9 @@ def string_dist(str1, str2):
# "something, the".
for word in SD_END_WORDS:
if str1.endswith(', %s' % word):
str1 = '%s %s' % (word, str1[:-len(word)-2])
str1 = '%s %s' % (word, str1[:-len(word) - 2])
if str2.endswith(', %s' % word):
str2 = '%s %s' % (word, str2[:-len(word)-2])
str2 = '%s %s' % (word, str2[:-len(word) - 2])
# Perform a couple of basic normalizing substitutions.
for pat, repl in SD_REPLACE:
@ -256,6 +271,23 @@ def string_dist(str1, str2):
return base_dist + penalty
class LazyClassProperty(object):
"""A decorator implementing a read-only property that is *lazy* in
the sense that the getter is only invoked once. Subsequent accesses
through *any* instance use the cached result.
"""
def __init__(self, getter):
self.getter = getter
self.computed = False
def __get__(self, obj, owner):
if not self.computed:
self.value = self.getter(owner)
self.computed = True
return self.value
class Distance(object):
"""Keeps track of multiple distance penalties. Provides a single
weighted distance for all penalties as well as a weighted distance
@ -264,11 +296,15 @@ class Distance(object):
def __init__(self):
self._penalties = {}
@LazyClassProperty
def _weights(cls): # noqa
"""A dictionary from keys to floating-point weights.
"""
weights_view = config['match']['distance_weights']
self._weights = {}
weights = {}
for key in weights_view.keys():
self._weights[key] = weights_view[key].as_number()
weights[key] = weights_view[key].as_number()
return weights
# Access the components and their aggregates.
@ -313,8 +349,10 @@ class Distance(object):
# Convert distance into a negative float we can sort items in
# ascending order (for keys, when the penalty is equal) and
# still get the items with the biggest distance first.
return sorted(list_, key=lambda (key, dist): (0-dist, key))
return sorted(
list_,
key=lambda key_and_dist: (-key_and_dist[1], key_and_dist[0])
)
# Behave like a float.
@ -323,12 +361,15 @@ class Distance(object):
def __float__(self):
return self.distance
def __sub__(self, other):
return self.distance - other
def __rsub__(self, other):
return other - self.distance
def __unicode__(self):
return "{0:.2f}".format(self.distance)
# Behave like a dict.
@ -355,11 +396,11 @@ class Distance(object):
"""
if not isinstance(dist, Distance):
raise ValueError(
'`dist` must be a Distance object. It is: %r' % dist)
u'`dist` must be a Distance object, not {0}'.format(type(dist))
)
for key, penalties in dist._penalties.iteritems():
self._penalties.setdefault(key, []).extend(penalties)
# Adding components.
def _eq(self, value1, value2):
@ -379,7 +420,8 @@ class Distance(object):
"""
if not 0.0 <= dist <= 1.0:
raise ValueError(
'`dist` must be between 0.0 and 1.0. It is: %r' % dist)
u'`dist` must be between 0.0 and 1.0, not {0}'.format(dist)
)
self._penalties.setdefault(key, []).append(dist)
def add_equality(self, key, value, options):
@ -472,31 +514,47 @@ def album_for_mbid(release_id):
if the ID is not found.
"""
try:
return mb.album_for_id(release_id)
album = mb.album_for_id(release_id)
if album:
plugins.send(u'albuminfo_received', info=album)
return album
except mb.MusicBrainzAPIError as exc:
exc.log(log)
def track_for_mbid(recording_id):
"""Get a TrackInfo object for a MusicBrainz recording ID. Return None
if the ID is not found.
"""
try:
return mb.track_for_id(recording_id)
track = mb.track_for_id(recording_id)
if track:
plugins.send(u'trackinfo_received', info=track)
return track
except mb.MusicBrainzAPIError as exc:
exc.log(log)
def albums_for_id(album_id):
"""Get a list of albums for an ID."""
candidates = [album_for_mbid(album_id)]
candidates.extend(plugins.album_for_id(album_id))
plugin_albums = plugins.album_for_id(album_id)
for a in plugin_albums:
plugins.send(u'albuminfo_received', info=a)
candidates.extend(plugin_albums)
return filter(None, candidates)
def tracks_for_id(track_id):
"""Get a list of tracks for an ID."""
candidates = [track_for_mbid(track_id)]
candidates.extend(plugins.track_for_id(track_id))
plugin_tracks = plugins.track_for_id(track_id)
for t in plugin_tracks:
plugins.send(u'trackinfo_received', info=t)
candidates.extend(plugin_tracks)
return filter(None, candidates)
def album_candidates(items, artist, album, va_likely):
"""Search for album matches. ``items`` is a list of Item objects
that make up the album. ``artist`` and ``album`` are the respective
@ -523,8 +581,13 @@ def album_candidates(items, artist, album, va_likely):
# Candidates from plugins.
out.extend(plugins.candidates(items, artist, album, va_likely))
# Notify subscribed plugins about fetched album info
for a in out:
plugins.send(u'albuminfo_received', info=a)
return out
def item_candidates(item, artist, title):
"""Search for item matches. ``item`` is the Item to be matched.
``artist`` and ``title`` are strings and either reflect the item or
@ -542,4 +605,8 @@ def item_candidates(item, artist, title):
# Plugin candidates.
out.extend(plugins.item_candidates(item, artist, title))
# Notify subscribed plugins about fetched track info
for i in out:
plugins.send(u'trackinfo_received', info=i)
return out

173
libs/beets/autotag/match.py
View file

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2013, Adrian Sampson.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
@ -15,21 +16,20 @@
"""Matches existing metadata with canonical information to identify
releases and tracks.
"""
from __future__ import division
from __future__ import division, absolute_import, print_function
import datetime
import logging
import re
from munkres import Munkres
from beets import logging
from beets import plugins
from beets import config
from beets.util import plurality
from beets.util.enumeration import enum
from beets.autotag import hooks
# Recommendation enumeration.
recommendation = enum('none', 'low', 'medium', 'strong', name='recommendation')
from beets.util.enumeration import OrderedEnum
from functools import reduce
# Artist signals that indicate "various artists". These are used at the
# album level to determine whether a given release is likely a VA
@ -41,6 +41,18 @@ VA_ARTISTS = (u'', u'various artists', u'various', u'va', u'unknown')
log = logging.getLogger('beets')
# Recommendation enumeration.
class Recommendation(OrderedEnum):
"""Indicates a qualitative suggestion to the user about what should
be done with a given match.
"""
none = 0
low = 1
medium = 2
strong = 3
# Primary matching functionality.
def current_metadata(items):
@ -56,10 +68,10 @@ def current_metadata(items):
fields = ['artist', 'album', 'albumartist', 'year', 'disctotal',
'mb_albumid', 'label', 'catalognum', 'country', 'media',
'albumdisambig']
for key in fields:
values = [getattr(item, key) for item in items if item]
likelies[key], freq = plurality(values)
consensus[key] = (freq == len(values))
for field in fields:
values = [item[field] for item in items if item]
likelies[field], freq = plurality(values)
consensus[field] = (freq == len(values))
# If there's an album artist consensus, use this for the artist.
if consensus['albumartist'] and likelies['albumartist']:
@ -67,6 +79,7 @@ def current_metadata(items):
return likelies, consensus
def assign_items(items, tracks):
"""Given a list of Items and a list of TrackInfo objects, find the
best mapping between them. Returns a mapping from Items to TrackInfo
@ -93,12 +106,14 @@ def assign_items(items, tracks):
extra_tracks.sort(key=lambda t: (t.index, t.title))
return mapping, extra_items, extra_tracks
def track_index_changed(item, track_info):
"""Returns True if the item and track info index is different. Tolerates
per disc and per release numbering.
"""
return item.track not in (track_info.medium_index, track_info.index)
def track_distance(item, track_info, incl_artist=False):
"""Determines the significance of a track metadata change. Returns a
Distance object. `incl_artist` indicates that a distance component should
@ -134,6 +149,7 @@ def track_distance(item, track_info, incl_artist=False):
return dist
def distance(items, album_info, mapping):
"""Determines how "significant" an album metadata change would be.
Returns a Distance object. `album_info` is an AlbumInfo object
@ -239,6 +255,7 @@ def distance(items, album_info, mapping):
return dist
def match_by_id(items):
"""If the items are tagged with a MusicBrainz album ID, returns an
AlbumInfo object for the corresponding album. Otherwise, returns
@ -247,16 +264,17 @@ def match_by_id(items):
# Is there a consensus on the MB album ID?
albumids = [item.mb_albumid for item in items if item.mb_albumid]
if not albumids:
log.debug('No album IDs found.')
log.debug(u'No album IDs found.')
return None
# If all album IDs are equal, look up the album.
if bool(reduce(lambda x,y: x if x==y else (), albumids)):
if bool(reduce(lambda x, y: x if x == y else (), albumids)):
albumid = albumids[0]
log.debug('Searching for discovered album ID: ' + albumid)
log.debug(u'Searching for discovered album ID: {0}', albumid)
return hooks.album_for_mbid(albumid)
else:
log.debug('No album ID consensus.')
log.debug(u'No album ID consensus.')
def _recommendation(results):
"""Given a sorted list of AlbumMatch or TrackMatch objects, return a
@ -268,26 +286,26 @@ def _recommendation(results):
"""
if not results:
# No candidates: no recommendation.
return recommendation.none
return Recommendation.none
# Basic distance thresholding.
min_dist = results[0].distance
if min_dist < config['match']['strong_rec_thresh'].as_number():
# Strong recommendation level.
rec = recommendation.strong
rec = Recommendation.strong
elif min_dist <= config['match']['medium_rec_thresh'].as_number():
# Medium recommendation level.
rec = recommendation.medium
rec = Recommendation.medium
elif len(results) == 1:
# Only a single candidate.
rec = recommendation.low
rec = Recommendation.low
elif results[1].distance - min_dist >= \
config['match']['rec_gap_thresh'].as_number():
# Gap between first two candidates is large.
rec = recommendation.low
rec = Recommendation.low
else:
# No conclusion. Return immediately. Can't be downgraded any further.
return recommendation.none
return Recommendation.none
# Downgrade to the max rec if it is lower than the current rec for an
# applied penalty.
@ -299,26 +317,38 @@ def _recommendation(results):
for key in keys:
if key in max_rec_view.keys():
max_rec = max_rec_view[key].as_choice({
'strong': recommendation.strong,
'medium': recommendation.medium,
'low': recommendation.low,
'none': recommendation.none,
'strong': Recommendation.strong,
'medium': Recommendation.medium,
'low': Recommendation.low,
'none': Recommendation.none,
})
rec = min(rec, max_rec)
return rec
def _add_candidate(items, results, info):
"""Given a candidate AlbumInfo object, attempt to add the candidate
to the output dictionary of AlbumMatch objects. This involves
checking the track count, ordering the items, checking for
duplicates, and calculating the distance.
"""
log.debug('Candidate: %s - %s' % (info.artist, info.album))
log.debug(u'Candidate: {0} - {1}', info.artist, info.album)
# Discard albums with zero tracks.
if not info.tracks:
log.debug(u'No tracks.')
return
# Don't duplicate.
if info.album_id in results:
log.debug('Duplicate.')
log.debug(u'Duplicate.')
return
# Discard matches without required tags.
for req_tag in config['match']['required'].as_str_seq():
if getattr(info, req_tag) is None:
log.debug(u'Ignored. Missing required tag: {0}', req_tag)
return
# Find mapping between the items and the track info.
@ -328,42 +358,53 @@ def _add_candidate(items, results, info):
dist = distance(items, info, mapping)
# Skip matches with ignored penalties.
penalties = [key for _, key in dist]
penalties = [key for key, _ in dist]
for penalty in config['match']['ignored'].as_str_seq():
if penalty in penalties:
log.debug('Ignored. Penalty: %s' % penalty)
log.debug(u'Ignored. Penalty: {0}', penalty)
return
log.debug('Success. Distance: %f' % dist)
log.debug(u'Success. Distance: {0}', dist)
results[info.album_id] = hooks.AlbumMatch(dist, info, mapping,
extra_items, extra_tracks)
def tag_album(items, search_artist=None, search_album=None,
search_id=None):
"""Bundles together the functionality used to infer tags for a
set of items comprised by an album. Returns everything relevant:
- The current artist.
- The current album.
- A list of AlbumMatch objects. The candidates are sorted by
distance (i.e., best match first).
- A recommendation.
If search_artist and search_album or search_id are provided, then
they are used as search terms in place of the current metadata.
search_ids=[]):
"""Return a tuple of a artist name, an album name, a list of
`AlbumMatch` candidates from the metadata backend, and a
`Recommendation`.
The artist and album are the most common values of these fields
among `items`.
The `AlbumMatch` objects are generated by searching the metadata
backends. By default, the metadata of the items is used for the
search. This can be customized by setting the parameters.
`search_ids` is a list of metadata backend IDs: if specified,
it will restrict the candidates to those IDs, ignoring
`search_artist` and `search album`. The `mapping` field of the
album has the matched `items` as keys.
The recommendation is calculated from the match quality of the
candidates.
"""
# Get current metadata.
likelies, consensus = current_metadata(items)
cur_artist = likelies['artist']
cur_album = likelies['album']
log.debug('Tagging %s - %s' % (cur_artist, cur_album))
log.debug(u'Tagging {0} - {1}', cur_artist, cur_album)
# The output result (distance, AlbumInfo) tuples (keyed by MB album
# ID).
candidates = {}
# Search by explicit ID.
if search_id is not None:
log.debug('Searching for album ID: ' + search_id)
search_cands = hooks.albums_for_id(search_id)
if search_ids:
search_cands = []
for search_id in search_ids:
log.debug(u'Searching for album ID: {0}', search_id)
search_cands.extend(hooks.albums_for_id(search_id))
# Use existing metadata or text search.
else:
@ -372,32 +413,32 @@ def tag_album(items, search_artist=None, search_album=None,
if id_info:
_add_candidate(items, candidates, id_info)
rec = _recommendation(candidates.values())
log.debug('Album ID match recommendation is ' + str(rec))
log.debug(u'Album ID match recommendation is {0}', rec)
if candidates and not config['import']['timid']:
# If we have a very good MBID match, return immediately.
# Otherwise, this match will compete against metadata-based
# matches.
if rec == recommendation.strong:
log.debug('ID match.')
if rec == Recommendation.strong:
log.debug(u'ID match.')
return cur_artist, cur_album, candidates.values(), rec
# Search terms.
if not (search_artist and search_album):
# No explicit search terms -- use current metadata.
search_artist, search_album = cur_artist, cur_album
log.debug(u'Search terms: %s - %s' % (search_artist, search_album))
log.debug(u'Search terms: {0} - {1}', search_artist, search_album)
# Is this album likely to be a "various artist" release?
va_likely = ((not consensus['artist']) or
(search_artist.lower() in VA_ARTISTS) or
any(item.comp for item in items))
log.debug(u'Album might be VA: %s' % str(va_likely))
log.debug(u'Album might be VA: {0}', va_likely)
# Get the results from the data sources.
search_cands = hooks.album_candidates(items, search_artist,
search_album, va_likely)
log.debug(u'Evaluating %i candidates.' % len(search_cands))
log.debug(u'Evaluating {0} candidates.', len(search_cands))
for info in search_cands:
_add_candidate(items, candidates, info)
@ -406,43 +447,47 @@ def tag_album(items, search_artist=None, search_album=None,
rec = _recommendation(candidates)
return cur_artist, cur_album, candidates, rec
def tag_item(item, search_artist=None, search_title=None,
search_id=None):
search_ids=[]):
"""Attempts to find metadata for a single track. Returns a
`(candidates, recommendation)` pair where `candidates` is a list of
TrackMatch objects. `search_artist` and `search_title` may be used
to override the current metadata for the purposes of the MusicBrainz
title; likewise `search_id`.
title. `search_ids` may be used for restricting the search to a list
of metadata backend IDs.
"""
# Holds candidates found so far: keys are MBIDs; values are
# (distance, TrackInfo) pairs.
candidates = {}
# First, try matching by MusicBrainz ID.
trackid = search_id or item.mb_trackid
if trackid:
log.debug('Searching for track ID: ' + trackid)
trackids = search_ids or filter(None, [item.mb_trackid])
if trackids:
for trackid in trackids:
log.debug(u'Searching for track ID: {0}', trackid)
for track_info in hooks.tracks_for_id(trackid):
dist = track_distance(item, track_info, incl_artist=True)
candidates[track_info.track_id] = \
hooks.TrackMatch(dist, track_info)
# If this is a good match, then don't keep searching.
rec = _recommendation(candidates.values())
if rec == recommendation.strong and not config['import']['timid']:
log.debug('Track ID match.')
return candidates.values(), rec
rec = _recommendation(sorted(candidates.itervalues()))
if rec == Recommendation.strong and \
not config['import']['timid']:
log.debug(u'Track ID match.')
return sorted(candidates.itervalues()), rec
# If we're searching by ID, don't proceed.
if search_id is not None:
if search_ids:
if candidates:
return candidates.values(), rec
return sorted(candidates.itervalues()), rec
else:
return [], recommendation.none
return [], Recommendation.none
# Search terms.
if not (search_artist and search_title):
search_artist, search_title = item.artist, item.title
log.debug(u'Item search terms: %s - %s' % (search_artist, search_title))
log.debug(u'Item search terms: {0} - {1}', search_artist, search_title)
# Get and evaluate candidate metadata.
for track_info in hooks.item_candidates(item, search_artist, search_title):
@ -450,7 +495,7 @@ def tag_item(item, search_artist=None, search_title=None,
candidates[track_info.track_id] = hooks.TrackMatch(dist, track_info)
# Sort by distance and return with recommendation.
log.debug('Found %i candidates.' % len(candidates))
log.debug(u'Found {0} candidates.', len(candidates))
candidates = sorted(candidates.itervalues())
rec = _recommendation(candidates)
return candidates, rec

87
libs/beets/autotag/mb.py
View file

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2013, Adrian Sampson.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
@ -14,23 +15,25 @@
"""Searches for albums in the MusicBrainz database.
"""
import logging
from __future__ import division, absolute_import, print_function
import musicbrainzngs
import re
import traceback
from urlparse import urljoin
from beets import logging
import beets.autotag.hooks
import beets
from beets import util
from beets import config
SEARCH_LIMIT = 5
VARIOUS_ARTISTS_ID = '89ad4ac3-39f7-470e-963a-56509c546377'
BASE_URL = 'http://musicbrainz.org/'
musicbrainzngs.set_useragent('beets', beets.__version__,
'http://beets.radbox.org/')
'http://beets.io/')
class MusicBrainzAPIError(util.HumanReadableException):
"""An error while talking to MusicBrainz. The `query` field is the
@ -38,10 +41,12 @@ class MusicBrainzAPIError(util.HumanReadableException):
"""
def __init__(self, reason, verb, query, tb=None):
self.query = query
if isinstance(reason, musicbrainzngs.WebServiceError):
reason = u'MusicBrainz not reachable'
super(MusicBrainzAPIError, self).__init__(reason, verb, tb)
def get_message(self):
return u'"{0}" in {1} with query {2}'.format(
return u'{0} in {1} with query {2}'.format(
self._reasonstr(), self.verb, repr(self.query)
)
@ -51,12 +56,15 @@ RELEASE_INCLUDES = ['artists', 'media', 'recordings', 'release-groups',
'labels', 'artist-credits', 'aliases']
TRACK_INCLUDES = ['artists', 'aliases']
def track_url(trackid):
return urljoin(BASE_URL, 'recording/' + trackid)
def album_url(albumid):
return urljoin(BASE_URL, 'release/' + albumid)
def configure():
"""Set up the python-musicbrainz-ngs module according to settings
from the beets configuration. This should be called at startup.
@ -67,6 +75,7 @@ def configure():
config['musicbrainz']['ratelimit'].get(int),
)
def _preferred_alias(aliases):
"""Given an list of alias structures for an artist credit, select
and return the user's preferred alias alias or None if no matching
@ -81,13 +90,15 @@ def _preferred_alias(aliases):
# Search configured locales in order.
for locale in config['import']['languages'].as_str_seq():
# Find matching primary aliases for this locale.
matches = [a for a in aliases if a['locale'] == locale and 'primary' in a]
matches = [a for a in aliases
if a['locale'] == locale and 'primary' in a]
# Skip to the next locale if we have no matches
if not matches:
continue
return matches[0]
def _flatten_artist_credit(credit):
"""Given a list representing an ``artist-credit`` block, flatten the
data into a triple of joined artist name strings: canonical, sort, and
@ -133,6 +144,7 @@ def _flatten_artist_credit(credit):
''.join(artist_credit_parts),
)
def track_info(recording, index=None, medium=None, medium_index=None,
medium_total=None):
"""Translates a MusicBrainz recording result dictionary into a beets
@ -149,6 +161,7 @@ def track_info(recording, index=None, medium=None, medium_index=None,
medium=medium,
medium_index=medium_index,
medium_total=medium_total,
data_source=u'MusicBrainz',
data_url=track_url(recording['id']),
)
@ -167,6 +180,7 @@ def track_info(recording, index=None, medium=None, medium_index=None,
info.decode()
return info
def _set_date_str(info, date_str, original=False):
"""Given a (possibly partial) YYYY-MM-DD string and an AlbumInfo
object, set the object's release date fields appropriately. If
@ -186,6 +200,7 @@ def _set_date_str(info, date_str, original=False):
key = 'original_' + key
setattr(info, key, date_num)
def album_info(release):
"""Takes a MusicBrainz release result dictionary and returns a beets
AlbumInfo object containing the interesting data about that release.
@ -199,7 +214,13 @@ def album_info(release):
index = 0
for medium in release['medium-list']:
disctitle = medium.get('title')
for track in medium['track-list']:
format = medium.get('format')
all_tracks = medium['track-list']
if 'pregap' in medium:
all_tracks.insert(0, medium['pregap'])
for track in all_tracks:
# Basic information from the recording.
index += 1
ti = track_info(
@ -210,6 +231,7 @@ def album_info(release):
len(medium['track-list']),
)
ti.disctitle = disctitle
ti.media = format
# Prefer track data, where present, over recording data.
if track.get('title'):
@ -233,10 +255,12 @@ def album_info(release):
mediums=len(release['medium-list']),
artist_sort=artist_sort_name,
artist_credit=artist_credit_name,
data_source='MusicBrainz',
data_source=u'MusicBrainz',
data_url=album_url(release['id']),
)
info.va = info.artist_id == VARIOUS_ARTISTS_ID
if info.va:
info.artist = config['va_name'].get(unicode)
info.asin = release.get('asin')
info.releasegroup_id = release['release-group']['id']
info.country = release.get('country')
@ -288,7 +312,8 @@ def album_info(release):
info.decode()
return info
def match_album(artist, album, tracks=None, limit=SEARCH_LIMIT):
def match_album(artist, album, tracks=None):
"""Searches for a single album ("release" in MusicBrainz parlance)
and returns an iterator over AlbumInfo objects. May raise a
MusicBrainzAPIError.
@ -297,21 +322,22 @@ def match_album(artist, album, tracks=None, limit=SEARCH_LIMIT):
optionally, a number of tracks on the album.
"""
# Build search criteria.
criteria = {'release': album.lower()}
criteria = {'release': album.lower().strip()}
if artist is not None:
criteria['artist'] = artist.lower()
criteria['artist'] = artist.lower().strip()
else:
# Various Artists search.
criteria['arid'] = VARIOUS_ARTISTS_ID
if tracks is not None:
criteria['tracks'] = str(tracks)
criteria['tracks'] = unicode(tracks)
# Abort if we have no search terms.
if not any(criteria.itervalues()):
return
try:
res = musicbrainzngs.search_releases(limit=limit, **criteria)
res = musicbrainzngs.search_releases(
limit=config['musicbrainz']['searchlimit'].get(int), **criteria)
except musicbrainzngs.MusicBrainzError as exc:
raise MusicBrainzAPIError(exc, 'release search', criteria,
traceback.format_exc())
@ -322,69 +348,74 @@ def match_album(artist, album, tracks=None, limit=SEARCH_LIMIT):
if albuminfo is not None:
yield albuminfo
def match_track(artist, title, limit=SEARCH_LIMIT):
def match_track(artist, title):
"""Searches for a single track and returns an iterable of TrackInfo
objects. May raise a MusicBrainzAPIError.
"""
criteria = {
'artist': artist.lower(),
'recording': title.lower(),
'artist': artist.lower().strip(),
'recording': title.lower().strip(),
}
if not any(criteria.itervalues()):
return
try:
res = musicbrainzngs.search_recordings(limit=limit, **criteria)
res = musicbrainzngs.search_recordings(
limit=config['musicbrainz']['searchlimit'].get(int), **criteria)
except musicbrainzngs.MusicBrainzError as exc:
raise MusicBrainzAPIError(exc, 'recording search', criteria,
traceback.format_exc())
for recording in res['recording-list']:
yield track_info(recording)
def _parse_id(s):
"""Search for a MusicBrainz ID in the given string and return it. If
no ID can be found, return None.
"""
# Find the first thing that looks like a UUID/MBID.
match = re.search('[a-f0-9]{8}(-[a-f0-9]{4}){3}-[a-f0-9]{12}', s)
match = re.search(ur'[a-f0-9]{8}(-[a-f0-9]{4}){3}-[a-f0-9]{12}', s)
if match:
return match.group()
def album_for_id(albumid):
def album_for_id(releaseid):
"""Fetches an album by its MusicBrainz ID and returns an AlbumInfo
object or None if the album is not found. May raise a
MusicBrainzAPIError.
"""
albumid = _parse_id(albumid)
albumid = _parse_id(releaseid)
if not albumid:
log.error('Invalid MBID.')
log.debug(u'Invalid MBID ({0}).', releaseid)
return
try:
res = musicbrainzngs.get_release_by_id(albumid,
RELEASE_INCLUDES)
except musicbrainzngs.ResponseError:
log.debug('Album ID match failed.')
log.debug(u'Album ID match failed.')
return None
except musicbrainzngs.MusicBrainzError as exc:
raise MusicBrainzAPIError(exc, 'get release by ID', albumid,
raise MusicBrainzAPIError(exc, u'get release by ID', albumid,
traceback.format_exc())
return album_info(res['release'])
def track_for_id(trackid):
def track_for_id(releaseid):
"""Fetches a track by its MusicBrainz ID. Returns a TrackInfo object
or None if no track is found. May raise a MusicBrainzAPIError.
"""
trackid = _parse_id(trackid)
trackid = _parse_id(releaseid)
if not trackid:
log.error('Invalid MBID.')
log.debug(u'Invalid MBID ({0}).', releaseid)
return
try:
res = musicbrainzngs.get_recording_by_id(trackid, TRACK_INCLUDES)
except musicbrainzngs.ResponseError:
log.debug('Track ID match failed.')
log.debug(u'Track ID match failed.')
return None
except musicbrainzngs.MusicBrainzError as exc:
raise MusicBrainzAPIError(exc, 'get recording by ID', trackid,
raise MusicBrainzAPIError(exc, u'get recording by ID', trackid,
traceback.format_exc())
return track_info(res['recording'])

34
libs/beets/config_default.yaml
View file

@ -5,6 +5,7 @@ import:
write: yes
copy: yes
move: no
link: no
delete: no
resume: ask
incremental: no
@ -20,9 +21,13 @@ import:
detail: no
flat: no
group_albums: no
pretend: false
search_ids: []
clutter: ["Thumbs.DB", ".DS_Store"]
ignore: [".*", "*~", "System Volume Information"]
ignore: [".*", "*~", "System Volume Information", "lost+found"]
ignore_hidden: yes
replace:
'[\\/]': _
'^\.': _
@ -32,27 +37,42 @@ replace:
'\s+$': ''
'^\s+': ''
path_sep_replace: _
asciify_paths: false
art_filename: cover
max_filename_length: 0
plugins: []
pluginpath: []
threaded: yes
color: yes
timeout: 5.0
per_disc_numbering: no
verbose: no
terminal_encoding: utf8
verbose: 0
terminal_encoding:
original_date: no
id3v23: no
va_name: "Various Artists"
ui:
terminal_width: 80
length_diff_thresh: 10.0
color: yes
colors:
text_success: green
text_warning: yellow
text_error: red
text_highlight: red
text_highlight_minor: lightgray
action_default: turquoise
action: blue
list_format_item: $artist - $album - $title
list_format_album: $albumartist - $album
format_item: $artist - $album - $title
format_album: $albumartist - $album
time_format: '%Y-%m-%d %H:%M:%S'
format_raw_length: no
sort_album: albumartist+ album+
sort_item: artist+ album+ disc+ track+
sort_case_insensitive: yes
paths:
default: $albumartist/$album%aunique{}/$track $title
@ -65,6 +85,7 @@ musicbrainz:
host: musicbrainz.org
ratelimit: 1
ratelimit_interval: 1.0
searchlimit: 5
match:
strong_rec_thresh: 0.04
@ -98,5 +119,6 @@ match:
media: []
original_year: no
ignored: []
required: []
track_length_grace: 10
track_length_max: 30

11
libs/beets/dbcore/__init__.py
View file

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2014, Adrian Sampson.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
@ -15,6 +16,14 @@
"""DBCore is an abstract database package that forms the basis for beets'
Library.
"""
from __future__ import division, absolute_import, print_function
from .db import Model, Database
from .query import Query, FieldQuery, MatchQuery, AndQuery, OrQuery
from .types import Type
from .queryparse import query_from_strings
from .queryparse import sort_from_strings
from .queryparse import parse_sorted_query
from .query import InvalidQueryError
# flake8: noqa

370
libs/beets/dbcore/db.py
View file

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2014, Adrian Sampson.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
@ -14,22 +15,70 @@
"""The central Model and Database constructs for DBCore.
"""
from __future__ import division, absolute_import, print_function
import time
import os
from collections import defaultdict
import threading
import sqlite3
import contextlib
import collections
import beets
from beets.util.functemplate import Template
from .query import MatchQuery
from beets.dbcore import types
from .query import MatchQuery, NullSort, TrueQuery
class FormattedMapping(collections.Mapping):
"""A `dict`-like formatted view of a model.
The accessor `mapping[key]` returns the formatted version of
`model[key]` as a unicode string.
If `for_path` is true, all path separators in the formatted values
are replaced.
"""
def __init__(self, model, for_path=False):
self.for_path = for_path
self.model = model
self.model_keys = model.keys(True)
def __getitem__(self, key):
if key in self.model_keys:
return self._get_formatted(self.model, key)
else:
raise KeyError(key)
def __iter__(self):
return iter(self.model_keys)
def __len__(self):
return len(self.model_keys)
def get(self, key, default=None):
if default is None:
default = self.model._type(key).format(None)
return super(FormattedMapping, self).get(key, default)
def _get_formatted(self, model, key):
value = model._type(key).format(model.get(key))
if isinstance(value, bytes):
value = value.decode('utf8', 'ignore')
if self.for_path:
sep_repl = beets.config['path_sep_replace'].get(unicode)
for sep in (os.path.sep, os.path.altsep):
if sep:
value = value.replace(sep, sep_repl)
return value
# Abstract base for model classes.
class Model(object):
"""An abstract object representing an object in the database. Model
objects act like dictionaries (i.e., the allow subscript access like
@ -66,12 +115,7 @@ class Model(object):
_fields = {}
"""A mapping indicating available "fixed" fields on this type. The
keys are field names and the values are Type objects.
"""
_bytes_keys = ()
"""Keys whose values should be stored as raw bytes blobs rather than
strings.
keys are field names and the values are `Type` objects.
"""
_search_fields = ()
@ -79,6 +123,21 @@ class Model(object):
terms.
"""
_types = {}
"""Optional Types for non-fixed (i.e., flexible and computed) fields.
"""
_sorts = {}
"""Optional named sort criteria. The keys are strings and the values
are subclasses of `Sort`.
"""
_always_dirty = False
"""By default, fields only become "dirty" when their value actually
changes. Enabling this flag marks fields as dirty even when the new
value is the same as the old value (e.g., `o.f = o.f`).
"""
@classmethod
def _getters(cls):
"""Return a mapping from field names to getter functions.
@ -94,7 +153,6 @@ class Model(object):
# As above: we could consider caching this result.
raise NotImplementedError()
# Basic operation.
def __init__(self, db=None, **values):
@ -110,6 +168,20 @@ class Model(object):
self.update(values)
self.clear_dirty()
@classmethod
def _awaken(cls, db=None, fixed_values={}, flex_values={}):
"""Create an object with values drawn from the database.
This is a performance optimization: the checks involved with
ordinary construction are bypassed.
"""
obj = cls(db)
for key, value in fixed_values.iteritems():
obj._values_fixed[key] = cls._type(key).from_sql(value)
for key, value in flex_values.iteritems():
obj._values_flex[key] = cls._type(key).from_sql(value)
return obj
def __repr__(self):
return '{0}({1})'.format(
type(self).__name__,
@ -128,13 +200,23 @@ class Model(object):
exception is raised otherwise.
"""
if not self._db:
raise ValueError('{0} has no database'.format(type(self).__name__))
raise ValueError(
u'{0} has no database'.format(type(self).__name__)
)
if need_id and not self.id:
raise ValueError('{0} has no id'.format(type(self).__name__))
raise ValueError(u'{0} has no id'.format(type(self).__name__))
# Essential field accessors.
@classmethod
def _type(cls, key):
"""Get the type of a field, a `Type` instance.
If the field has no explicit type, it is given the base `Type`,
which does no conversion.
"""
return cls._fields.get(key) or cls._types.get(key) or types.DEFAULT
def __getitem__(self, key):
"""Get the value for a field. Raise a KeyError if the field is
not available.
@ -152,11 +234,19 @@ class Model(object):
def __setitem__(self, key, value):
"""Assign the value for a field.
"""
source = self._values_fixed if key in self._fields \
else self._values_flex
# Choose where to place the value.
if key in self._fields:
source = self._values_fixed
else:
source = self._values_flex
# If the field has a type, filter the value.
value = self._type(key).normalize(value)
# Assign value and possibly mark as dirty.
old_value = source.get(key)
source[key] = value
if old_value != value:
if self._always_dirty or old_value != value:
self._dirty.add(key)
def __delitem__(self, key):
@ -166,11 +256,11 @@ class Model(object):
del self._values_flex[key]
self._dirty.add(key) # Mark for dropping on store.
elif key in self._getters(): # Computed.
raise KeyError('computed field {0} cannot be deleted'.format(key))
raise KeyError(u'computed field {0} cannot be deleted'.format(key))
elif key in self._fields: # Fixed.
raise KeyError('fixed field {0} cannot be deleted'.format(key))
raise KeyError(u'fixed field {0} cannot be deleted'.format(key))
else:
raise KeyError('no such field {0}'.format(key))
raise KeyError(u'no such field {0}'.format(key))
def keys(self, computed=False):
"""Get a list of available field names for this object. The
@ -183,6 +273,12 @@ class Model(object):
else:
return base_keys
@classmethod
def all_keys(cls):
"""Get a list of available keys for objects of this type.
Includes fixed and computed fields.
"""
return list(cls._fields) + cls._getters().keys()
# Act like a dictionary.
@ -219,17 +315,16 @@ class Model(object):
"""
return iter(self.keys())
# Convenient attribute access.
def __getattr__(self, key):
if key.startswith('_'):
raise AttributeError('model has no attribute {0!r}'.format(key))
raise AttributeError(u'model has no attribute {0!r}'.format(key))
else:
try:
return self[key]
except KeyError:
raise AttributeError('no such field {0!r}'.format(key))
raise AttributeError(u'no such field {0!r}'.format(key))
def __setattr__(self, key, value):
if key.startswith('_'):
@ -243,7 +338,6 @@ class Model(object):
else:
del self[key]
# Database interaction (CRUD methods).
def store(self):
@ -252,19 +346,15 @@ class Model(object):
self._check_db()
# Build assignments for query.
assignments = ''
assignments = []
subvars = []
for key in self._fields:
if key != 'id' and key in self._dirty:
self._dirty.remove(key)
assignments += key + '=?,'
value = self[key]
# Wrap path strings in buffers so they get stored
# "in the raw".
if key in self._bytes_keys and isinstance(value, str):
value = buffer(value)
assignments.append(key + '=?')
value = self._type(key).to_sql(self[key])
subvars.append(value)
assignments = assignments[:-1] # Knock off last ,
assignments = ','.join(assignments)
with self._db.transaction() as tx:
# Main table update.
@ -301,7 +391,9 @@ class Model(object):
"""
self._check_db()
stored_obj = self._db._get(type(self), self.id)
assert stored_obj is not None, "object {0} not in DB".format(self.id)
assert stored_obj is not None, u"object {0} not in DB".format(self.id)
self._values_fixed = {}
self._values_flex = {}
self.update(dict(stored_obj))
self.clear_dirty()
@ -344,76 +436,26 @@ class Model(object):
self._dirty.add(key)
self.store()
# Formatting and templating.
@classmethod
def _format(cls, key, value, for_path=False):
"""Format a value as the given field for this model.
"""
# Format the value as a string according to its type, if any.
if key in cls._fields:
value = cls._fields[key].format(value)
# Formatting must result in a string. To deal with
# Python2isms, implicitly convert ASCII strings.
assert isinstance(value, basestring), \
u'field formatter must produce strings'
if isinstance(value, bytes):
value = value.decode('utf8', 'ignore')
_formatter = FormattedMapping
elif not isinstance(value, unicode):
# Fallback formatter. Convert to unicode at all cost.
if value is None:
value = u''
elif isinstance(value, basestring):
if isinstance(value, bytes):
value = value.decode('utf8', 'ignore')
else:
value = unicode(value)
if for_path:
sep_repl = beets.config['path_sep_replace'].get(unicode)
for sep in (os.path.sep, os.path.altsep):
if sep:
value = value.replace(sep, sep_repl)
return value
def _get_formatted(self, key, for_path=False):
"""Get a field value formatted as a string (`unicode` object)
for display to the user. If `for_path` is true, then the value
will be sanitized for inclusion in a pathname (i.e., path
separators will be removed from the value).
"""
return self._format(key, self.get(key), for_path)
def _formatted_mapping(self, for_path=False):
def formatted(self, for_path=False):
"""Get a mapping containing all values on this object formatted
as human-readable strings.
as human-readable unicode strings.
"""
# In the future, this could be made "lazy" to avoid computing
# fields unnecessarily.
out = {}
for key in self.keys(True):
out[key] = self._get_formatted(key, for_path)
return out
return self._formatter(self, for_path)
def evaluate_template(self, template, for_path=False):
"""Evaluate a template (a string or a `Template` object) using
the object's fields. If `for_path` is true, then no new path
separators will be added to the template.
"""
# Build value mapping.
mapping = self._formatted_mapping(for_path)
# Get template functions.
funcs = self._template_funcs()
# Perform substitution.
if isinstance(template, basestring):
template = Template(template)
return template.substitute(mapping, funcs)
return template.substitute(self.formatted(for_path),
self._template_funcs())
# Parsing.
@ -422,42 +464,98 @@ class Model(object):
"""Parse a string as a value for the given key.
"""
if not isinstance(string, basestring):
raise TypeError("_parse() argument must be a string")
raise TypeError(u"_parse() argument must be a string")
typ = cls._fields.get(key)
if typ:
return typ.parse(string)
else:
# Fall back to unparsed string.
return string
return cls._type(key).parse(string)
def set_parse(self, key, string):
"""Set the object's key to a value represented by a string.
"""
self[key] = self._parse(key, string)
# Database controller and supporting interfaces.
class Results(object):
"""An item query result set. Iterating over the collection lazily
constructs LibModel objects that reflect database rows.
"""
def __init__(self, model_class, rows, db, query=None):
def __init__(self, model_class, rows, db, query=None, sort=None):
"""Create a result set that will construct objects of type
`model_class`, which should be a subclass of `LibModel`, out of
the query result mapping in `rows`. The new objects are
associated with the database `db`. If `query` is provided, it is
used as a predicate to filter the results for a "slow query" that
cannot be evaluated by the database directly.
`model_class`.
`model_class` is a subclass of `LibModel` that will be
constructed. `rows` is a query result: a list of mappings. The
new objects will be associated with the database `db`.
If `query` is provided, it is used as a predicate to filter the
results for a "slow query" that cannot be evaluated by the
database directly. If `sort` is provided, it is used to sort the
full list of results before returning. This means it is a "slow
sort" and all objects must be built before returning the first
one.
"""
self.model_class = model_class
self.rows = rows
self.db = db
self.query = query
self.sort = sort
# We keep a queue of rows we haven't yet consumed for
# materialization. We preserve the original total number of
# rows.
self._rows = rows
self._row_count = len(rows)
# The materialized objects corresponding to rows that have been
# consumed.
self._objects = []
def _get_objects(self):
"""Construct and generate Model objects for they query. The
objects are returned in the order emitted from the database; no
slow sort is applied.
For performance, this generator caches materialized objects to
avoid constructing them more than once. This way, iterating over
a `Results` object a second time should be much faster than the
first.
"""
index = 0 # Position in the materialized objects.
while index < len(self._objects) or self._rows:
# Are there previously-materialized objects to produce?
if index < len(self._objects):
yield self._objects[index]
index += 1
# Otherwise, we consume another row, materialize its object
# and produce it.
else:
while self._rows:
row = self._rows.pop(0)
obj = self._make_model(row)
# If there is a slow-query predicate, ensurer that the
# object passes it.
if not self.query or self.query.match(obj):
self._objects.append(obj)
index += 1
yield obj
break
def __iter__(self):
"""Construct Python objects for all rows that pass the query
predicate.
"""Construct and generate Model objects for all matching
objects, in sorted order.
"""
for row in self.rows:
if self.sort:
# Slow sort. Must build the full list first.
objects = self.sort.sort(list(self._get_objects()))
return iter(objects)
else:
# Objects are pre-sorted (i.e., by the database).
return self._get_objects()
def _make_model(self, row):
# Get the flexible attributes for the object.
with self.db.transaction() as tx:
flex_rows = tx.query(
@ -466,21 +564,24 @@ class Results(object):
),
(row['id'],)
)
values = dict(row)
values.update(
dict((row['key'], row['value']) for row in flex_rows)
)
# Construct the Python object and yield it if it passes the
# predicate.
obj = self.model_class(self.db, **values)
if not self.query or self.query.match(obj):
yield obj
cols = dict(row)
values = dict((k, v) for (k, v) in cols.items()
if not k[:4] == 'flex')
flex_values = dict((row['key'], row['value']) for row in flex_rows)
# Construct the Python object
obj = self.model_class._awaken(self.db, values, flex_values)
return obj
def __len__(self):
"""Get the number of matching objects.
"""
if self.query:
if not self._rows:
# Fully materialized. Just count the objects.
return len(self._objects)
elif self.query:
# A slow query. Fall back to testing every object.
count = 0
for obj in self:
@ -489,7 +590,7 @@ class Results(object):
else:
# A fast query. Just count the rows.
return len(self.rows)
return self._row_count
def __nonzero__(self):
"""Does this result contain any objects?
@ -500,13 +601,18 @@ class Results(object):
"""Get the nth item in this result set. This is inefficient: all
items up to n are materialized and thrown away.
"""
if not self._rows and not self.sort:
# Fully materialized and already in order. Just look up the
# object.
return self._objects[n]
it = iter(self)
try:
for i in range(n):
it.next()
return it.next()
next(it)
return next(it)
except StopIteration:
raise IndexError('result index {0} out of range'.format(n))
raise IndexError(u'result index {0} out of range'.format(n))
def get(self):
"""Return the first matching object, or None if no objects
@ -514,7 +620,7 @@ class Results(object):
"""
it = iter(self)
try:
return it.next()
return next(it)
except StopIteration:
return None
@ -604,7 +710,6 @@ class Database(object):
self._make_table(model_cls._table, model_cls._fields)
self._make_attribute_table(model_cls._flex_table)
# Primitive access control: connections and transactions.
def _connection(self):
@ -644,7 +749,6 @@ class Database(object):
"""
return Transaction(self)
# Schema setup and migration.
def _make_table(self, table, fields):
@ -698,27 +802,33 @@ class Database(object):
ON {0} (entity_id);
""".format(flex_table))
# Querying.
def _fetch(self, model_cls, query, order_by=None):
def _fetch(self, model_cls, query=None, sort=None):
"""Fetch the objects of type `model_cls` matching the given
query. The query may be given as a string, string sequence, a
Query object, or None (to fetch everything). If provided,
`order_by` is a SQLite ORDER BY clause for sorting.
Query object, or None (to fetch everything). `sort` is an
`Sort` object.
"""
query = query or TrueQuery() # A null query.
sort = sort or NullSort() # Unsorted.
where, subvals = query.clause()
order_by = sort.order_clause()
sql = "SELECT * FROM {0} WHERE {1}".format(
sql = ("SELECT * FROM {0} WHERE {1} {2}").format(
model_cls._table,
where or '1',
"ORDER BY {0}".format(order_by) if order_by else '',
)
if order_by:
sql += " ORDER BY {0}".format(order_by)
with self.transaction() as tx:
rows = tx.query(sql, subvals)
return Results(model_cls, rows, self, None if where else query)
return Results(
model_cls, rows, self,
None if where else query, # Slow query component.
sort if sort.is_slow() else None, # Slow sort component.
)
def _get(self, model_cls, id):
"""Get a Model object by its id or None if the id does not

426
libs/beets/dbcore/query.py
View file

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2014, Adrian Sampson.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
@ -14,9 +15,45 @@
"""The Query type hierarchy for DBCore.
"""
from __future__ import division, absolute_import, print_function
import re
from operator import mul
from beets import util
from datetime import datetime, timedelta
import unicodedata
from functools import reduce
class ParsingError(ValueError):
"""Abstract class for any unparseable user-requested album/query
specification.
"""
class InvalidQueryError(ParsingError):
"""Represent any kind of invalid query.
The query should be a unicode string or a list, which will be space-joined.
"""
def __init__(self, query, explanation):
if isinstance(query, list):
query = " ".join(query)
message = u"'{0}': {1}".format(query, explanation)
super(InvalidQueryError, self).__init__(message)
class InvalidQueryArgumentTypeError(ParsingError):
"""Represent a query argument that could not be converted as expected.
It exists to be caught in upper stack levels so a meaningful (i.e. with the
query) InvalidQueryError can be raised.
"""
def __init__(self, what, expected, detail=None):
message = u"'{0}' is not {1}".format(what, expected)
if detail:
message = u"{0}: {1}".format(message, detail)
super(InvalidQueryArgumentTypeError, self).__init__(message)
class Query(object):
@ -24,9 +61,8 @@ class Query(object):
"""
def clause(self):
"""Generate an SQLite expression implementing the query.
Return a clause string, a sequence of substitution values for
the clause, and a Query object representing the "remainder"
Returns (clause, subvals) where clause is a valid sqlite
Return (clause, subvals) where clause is a valid sqlite
WHERE clause implementing the query and subvals is a list of
items to be substituted for ?s in the clause.
"""
@ -38,6 +74,15 @@ class Query(object):
"""
raise NotImplementedError
def __repr__(self):
return "{0.__class__.__name__}()".format(self)
def __eq__(self, other):
return type(self) == type(other)
def __hash__(self):
return 0
class FieldQuery(Query):
"""An abstract query that searches in a specific field for a
@ -71,6 +116,17 @@ class FieldQuery(Query):
def match(self, item):
return self.value_match(self.pattern, item.get(self.field))
def __repr__(self):
return ("{0.__class__.__name__}({0.field!r}, {0.pattern!r}, "
"{0.fast})".format(self))
def __eq__(self, other):
return super(FieldQuery, self).__eq__(other) and \
self.field == other.field and self.pattern == other.pattern
def __hash__(self):
return hash((self.field, hash(self.pattern)))
class MatchQuery(FieldQuery):
"""A query that looks for exact matches in an item field."""
@ -82,6 +138,25 @@ class MatchQuery(FieldQuery):
return pattern == value
class NoneQuery(FieldQuery):
def __init__(self, field, fast=True):
super(NoneQuery, self).__init__(field, None, fast)
def col_clause(self):
return self.field + " IS NULL", ()
@classmethod
def match(cls, item):
try:
return item[cls.field] is None
except KeyError:
return True
def __repr__(self):
return "{0.__class__.__name__}({0.field!r}, {0.fast})".format(self)
class StringFieldQuery(FieldQuery):
"""A FieldQuery that converts values to strings before matching
them.
@ -104,8 +179,11 @@ class StringFieldQuery(FieldQuery):
class SubstringQuery(StringFieldQuery):
"""A query that matches a substring in a specific item field."""
def col_clause(self):
search = '%' + (self.pattern.replace('\\','\\\\').replace('%','\\%')
.replace('_','\\_')) + '%'
pattern = (self.pattern
.replace('\\', '\\\\')
.replace('%', '\\%')
.replace('_', '\\_'))
search = '%' + pattern + '%'
clause = self.field + " like ? escape '\\'"
subvals = [search]
return clause, subvals
@ -118,15 +196,31 @@ class SubstringQuery(StringFieldQuery):
class RegexpQuery(StringFieldQuery):
"""A query that matches a regular expression in a specific item
field.
Raises InvalidQueryError when the pattern is not a valid regular
expression.
"""
def __init__(self, field, pattern, fast=True):
super(RegexpQuery, self).__init__(field, pattern, fast)
pattern = self._normalize(pattern)
try:
self.pattern = re.compile(self.pattern)
except re.error as exc:
# Invalid regular expression.
raise InvalidQueryArgumentTypeError(pattern,
u"a regular expression",
format(exc))
@staticmethod
def _normalize(s):
"""Normalize a Unicode string's representation (used on both
patterns and matched values).
"""
return unicodedata.normalize('NFC', s)
@classmethod
def string_match(cls, pattern, value):
try:
res = re.search(pattern, value)
except re.error:
# Invalid regular expression.
return False
return res is not None
return pattern.search(cls._normalize(value)) is not None
class BooleanQuery(MatchQuery):
@ -142,7 +236,7 @@ class BooleanQuery(MatchQuery):
class BytesQuery(MatchQuery):
"""Match a raw bytes field (i.e., a path). This is a necessary hack
to work around the `sqlite3` module's desire to treat `str` and
to work around the `sqlite3` module's desire to treat `bytes` and
`unicode` equivalently in Python 2. Always use this query instead of
`MatchQuery` when matching on BLOB values.
"""
@ -170,19 +264,26 @@ class NumericQuery(FieldQuery):
"""Matches numeric fields. A syntax using Ruby-style range ellipses
(``..``) lets users specify one- or two-sided ranges. For example,
``year:2001..`` finds music released since the turn of the century.
Raises InvalidQueryError when the pattern does not represent an int or
a float.
"""
def _convert(self, s):
"""Convert a string to a numeric type (float or int). If the
string cannot be converted, return None.
"""Convert a string to a numeric type (float or int).
Return None if `s` is empty.
Raise an InvalidQueryError if the string cannot be converted.
"""
# This is really just a bit of fun premature optimization.
if not s:
return None
try:
return int(s)
except ValueError:
try:
return float(s)
except ValueError:
return None
raise InvalidQueryArgumentTypeError(s, u"an int or a float")
def __init__(self, field, pattern, fast=True):
super(NumericQuery, self).__init__(field, pattern, fast)
@ -200,7 +301,9 @@ class NumericQuery(FieldQuery):
self.rangemax = self._convert(parts[1])
def match(self, item):
value = getattr(item, self.field)
if self.field not in item:
return False
value = item[self.field]
if isinstance(value, basestring):
value = self._convert(value)
@ -225,7 +328,7 @@ class NumericQuery(FieldQuery):
elif self.rangemax is not None:
return u'{0} <= ?'.format(self.field), (self.rangemax,)
else:
return '1', ()
return u'1', ()
class CollectionQuery(Query):
@ -236,17 +339,21 @@ class CollectionQuery(Query):
self.subqueries = subqueries
# Act like a sequence.
def __len__(self):
return len(self.subqueries)
def __getitem__(self, key):
return self.subqueries[key]
def __iter__(self):
return iter(self.subqueries)
def __contains__(self, item):
return item in self.subqueries
def clause_with_joiner(self, joiner):
"""Returns a clause created by joining together the clauses of
"""Return a clause created by joining together the clauses of
all subqueries with the string joiner (padded by spaces).
"""
clause_parts = []
@ -261,6 +368,19 @@ class CollectionQuery(Query):
clause = (' ' + joiner + ' ').join(clause_parts)
return clause, subvals
def __repr__(self):
return "{0.__class__.__name__}({0.subqueries!r})".format(self)
def __eq__(self, other):
return super(CollectionQuery, self).__eq__(other) and \
self.subqueries == other.subqueries
def __hash__(self):
"""Since subqueries are mutable, this object should not be hashable.
However and for conveniences purposes, it can be hashed.
"""
return reduce(mul, map(hash, self.subqueries), 1)
class AnyFieldQuery(CollectionQuery):
"""A query that matches if a given FieldQuery subclass matches in
@ -286,6 +406,17 @@ class AnyFieldQuery(CollectionQuery):
return True
return False
def __repr__(self):
return ("{0.__class__.__name__}({0.pattern!r}, {0.fields!r}, "
"{0.query_class.__name__})".format(self))
def __eq__(self, other):
return super(AnyFieldQuery, self).__eq__(other) and \
self.query_class == other.query_class
def __hash__(self):
return hash((self.pattern, tuple(self.fields), self.query_class))
class MutableCollectionQuery(CollectionQuery):
"""A collection query whose subqueries may be modified after the
@ -316,6 +447,36 @@ class OrQuery(MutableCollectionQuery):
return any([q.match(item) for q in self.subqueries])
class NotQuery(Query):
"""A query that matches the negation of its `subquery`, as a shorcut for
performing `not(subquery)` without using regular expressions.
"""
def __init__(self, subquery):
self.subquery = subquery
def clause(self):
clause, subvals = self.subquery.clause()
if clause:
return 'not ({0})'.format(clause), subvals
else:
# If there is no clause, there is nothing to negate. All the logic
# is handled by match() for slow queries.
return clause, subvals
def match(self, item):
return not self.subquery.match(item)
def __repr__(self):
return "{0.__class__.__name__}({0.subquery!r})".format(self)
def __eq__(self, other):
return super(NotQuery, self).__eq__(other) and \
self.subquery == other.subquery
def __hash__(self):
return hash(('not', hash(self.subquery)))
class TrueQuery(Query):
"""A query that always matches."""
def clause(self):
@ -334,21 +495,15 @@ class FalseQuery(Query):
return False
# Time/date queries.
def _to_epoch_time(date):
"""Convert a `datetime` object to an integer number of seconds since
the (local) Unix epoch.
"""
epoch = datetime.fromtimestamp(0)
delta = date - epoch
try:
return int(delta.total_seconds())
except AttributeError:
# datetime.timedelta.total_seconds() is not available on Python 2.6
return delta.seconds + delta.days * 24 * 3600
def _parse_periods(pattern):
@ -380,7 +535,7 @@ class Period(object):
precision (a string, one of "year", "month", or "day").
"""
if precision not in Period.precisions:
raise ValueError('Invalid precision ' + str(precision))
raise ValueError(u'Invalid precision {0}'.format(precision))
self.date = date
self.precision = precision
@ -393,10 +548,14 @@ class Period(object):
return None
ordinal = string.count('-')
if ordinal >= len(cls.date_formats):
raise ValueError('date is not in one of the formats '
+ ', '.join(cls.date_formats))
# Too many components.
return None
date_format = cls.date_formats[ordinal]
try:
date = datetime.strptime(string, date_format)
except ValueError:
# Parsing failed.
return None
precision = cls.precisions[ordinal]
return cls(date, precision)
@ -416,7 +575,7 @@ class Period(object):
elif 'day' == precision:
return date + timedelta(days=1)
else:
raise ValueError('unhandled precision ' + str(precision))
raise ValueError(u'unhandled precision {0}'.format(precision))
class DateInterval(object):
@ -428,7 +587,7 @@ class DateInterval(object):
def __init__(self, start, end):
if start is not None and end is not None and not start < end:
raise ValueError("start date {0} is not before end date {1}"
raise ValueError(u"start date {0} is not before end date {1}"
.format(start, end))
self.start = start
self.end = end
@ -449,7 +608,7 @@ class DateInterval(object):
return True
def __str__(self):
return'[{0}, {1})'.format(self.start, self.end)
return '[{0}, {1})'.format(self.start, self.end)
class DateQuery(FieldQuery):
@ -492,3 +651,208 @@ class DateQuery(FieldQuery):
# Match any date.
clause = '1'
return clause, subvals
class DurationQuery(NumericQuery):
"""NumericQuery that allow human-friendly (M:SS) time interval formats.
Converts the range(s) to a float value, and delegates on NumericQuery.
Raises InvalidQueryError when the pattern does not represent an int, float
or M:SS time interval.
"""
def _convert(self, s):
"""Convert a M:SS or numeric string to a float.
Return None if `s` is empty.
Raise an InvalidQueryError if the string cannot be converted.
"""
if not s:
return None
try:
return util.raw_seconds_short(s)
except ValueError:
try:
return float(s)
except ValueError:
raise InvalidQueryArgumentTypeError(
s,
u"a M:SS string or a float")
# Sorting.
class Sort(object):
"""An abstract class representing a sort operation for a query into
the item database.
"""
def order_clause(self):
"""Generates a SQL fragment to be used in a ORDER BY clause, or
None if no fragment is used (i.e., this is a slow sort).
"""
return None
def sort(self, items):
"""Sort the list of objects and return a list.
"""
return sorted(items)
def is_slow(self):
"""Indicate whether this query is *slow*, meaning that it cannot
be executed in SQL and must be executed in Python.
"""
return False
def __hash__(self):
return 0
def __eq__(self, other):
return type(self) == type(other)
class MultipleSort(Sort):
"""Sort that encapsulates multiple sub-sorts.
"""
def __init__(self, sorts=None):
self.sorts = sorts or []
def add_sort(self, sort):
self.sorts.append(sort)
def _sql_sorts(self):
"""Return the list of sub-sorts for which we can be (at least
partially) fast.
A contiguous suffix of fast (SQL-capable) sub-sorts are
executable in SQL. The remaining, even if they are fast
independently, must be executed slowly.
"""
sql_sorts = []
for sort in reversed(self.sorts):
if not sort.order_clause() is None:
sql_sorts.append(sort)
else:
break
sql_sorts.reverse()
return sql_sorts
def order_clause(self):
order_strings = []
for sort in self._sql_sorts():
order = sort.order_clause()
order_strings.append(order)
return ", ".join(order_strings)
def is_slow(self):
for sort in self.sorts:
if sort.is_slow():
return True
return False
def sort(self, items):
slow_sorts = []
switch_slow = False
for sort in reversed(self.sorts):
if switch_slow:
slow_sorts.append(sort)
elif sort.order_clause() is None:
switch_slow = True
slow_sorts.append(sort)
else:
pass
for sort in slow_sorts:
items = sort.sort(items)
return items
def __repr__(self):
return 'MultipleSort({!r})'.format(self.sorts)
def __hash__(self):
return hash(tuple(self.sorts))
def __eq__(self, other):
return super(MultipleSort, self).__eq__(other) and \
self.sorts == other.sorts
class FieldSort(Sort):
"""An abstract sort criterion that orders by a specific field (of
any kind).
"""
def __init__(self, field, ascending=True, case_insensitive=True):
self.field = field
self.ascending = ascending
self.case_insensitive = case_insensitive
def sort(self, objs):
# TODO: Conversion and null-detection here. In Python 3,
# comparisons with None fail. We should also support flexible
# attributes with different types without falling over.
def key(item):
field_val = item.get(self.field, '')
if self.case_insensitive and isinstance(field_val, unicode):
field_val = field_val.lower()
return field_val
return sorted(objs, key=key, reverse=not self.ascending)
def __repr__(self):
return '<{0}: {1}{2}>'.format(
type(self).__name__,
self.field,
'+' if self.ascending else '-',
)
def __hash__(self):
return hash((self.field, self.ascending))
def __eq__(self, other):
return super(FieldSort, self).__eq__(other) and \
self.field == other.field and \
self.ascending == other.ascending
class FixedFieldSort(FieldSort):
"""Sort object to sort on a fixed field.
"""
def order_clause(self):
order = "ASC" if self.ascending else "DESC"
if self.case_insensitive:
field = '(CASE ' \
'WHEN TYPEOF({0})="text" THEN LOWER({0}) ' \
'WHEN TYPEOF({0})="blob" THEN LOWER({0}) ' \
'ELSE {0} END)'.format(self.field)
else:
field = self.field
return "{0} {1}".format(field, order)
class SlowFieldSort(FieldSort):
"""A sort criterion by some model field other than a fixed field:
i.e., a computed or flexible field.
"""
def is_slow(self):
return True
class NullSort(Sort):
"""No sorting. Leave results unsorted."""
def sort(self, items):
return items
def __nonzero__(self):
return self.__bool__()
def __bool__(self):
return False
def __eq__(self, other):
return type(self) == type(other) or other is None
def __hash__(self):
return 0

250
libs/beets/dbcore/queryparse.py Normal file
View file

@ -0,0 +1,250 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Parsing of strings into DBCore queries.
"""
from __future__ import division, absolute_import, print_function
import re
import itertools
from . import query
import beets
PARSE_QUERY_PART_REGEX = re.compile(
# Non-capturing optional segment for the keyword.
r'(-|\^)?' # Negation prefixes.
r'(?:'
r'(\S+?)' # The field key.
r'(?<!\\):' # Unescaped :
r')?'
r'(.*)', # The term itself.
re.I # Case-insensitive.
)
def parse_query_part(part, query_classes={}, prefixes={},
default_class=query.SubstringQuery):
"""Parse a single *query part*, which is a chunk of a complete query
string representing a single criterion.
A query part is a string consisting of:
- A *pattern*: the value to look for.
- Optionally, a *field name* preceding the pattern, separated by a
colon. So in `foo:bar`, `foo` is the field name and `bar` is the
pattern.
- Optionally, a *query prefix* just before the pattern (and after the
optional colon) indicating the type of query that should be used. For
example, in `~foo`, `~` might be a prefix. (The set of prefixes to
look for is given in the `prefixes` parameter.)
- Optionally, a negation indicator, `-` or `^`, at the very beginning.
Both prefixes and the separating `:` character may be escaped with a
backslash to avoid their normal meaning.
The function returns a tuple consisting of:
- The field name: a string or None if it's not present.
- The pattern, a string.
- The query class to use, which inherits from the base
:class:`Query` type.
- A negation flag, a bool.
The three optional parameters determine which query class is used (i.e.,
the third return value). They are:
- `query_classes`, which maps field names to query classes. These
are used when no explicit prefix is present.
- `prefixes`, which maps prefix strings to query classes.
- `default_class`, the fallback when neither the field nor a prefix
indicates a query class.
So the precedence for determining which query class to return is:
prefix, followed by field, and finally the default.
For example, assuming the `:` prefix is used for `RegexpQuery`:
- `'stapler'` -> `(None, 'stapler', SubstringQuery, False)`
- `'color:red'` -> `('color', 'red', SubstringQuery, False)`
- `':^Quiet'` -> `(None, '^Quiet', RegexpQuery, False)`, because
the `^` follows the `:`
- `'color::b..e'` -> `('color', 'b..e', RegexpQuery, False)`
- `'-color:red'` -> `('color', 'red', SubstringQuery, True)`
"""
# Apply the regular expression and extract the components.
part = part.strip()
match = PARSE_QUERY_PART_REGEX.match(part)
assert match # Regex should always match
negate = bool(match.group(1))
key = match.group(2)
term = match.group(3).replace('\:', ':')
# Check whether there's a prefix in the query and use the
# corresponding query type.
for pre, query_class in prefixes.items():
if term.startswith(pre):
return key, term[len(pre):], query_class, negate
# No matching prefix, so use either the query class determined by
# the field or the default as a fallback.
query_class = query_classes.get(key, default_class)
return key, term, query_class, negate
def construct_query_part(model_cls, prefixes, query_part):
"""Parse a *query part* string and return a :class:`Query` object.
:param model_cls: The :class:`Model` class that this is a query for.
This is used to determine the appropriate query types for the
model's fields.
:param prefixes: A map from prefix strings to :class:`Query` types.
:param query_part: The string to parse.
See the documentation for `parse_query_part` for more information on
query part syntax.
"""
# A shortcut for empty query parts.
if not query_part:
return query.TrueQuery()
# Use `model_cls` to build up a map from field names to `Query`
# classes.
query_classes = {}
for k, t in itertools.chain(model_cls._fields.items(),
model_cls._types.items()):
query_classes[k] = t.query
# Parse the string.
key, pattern, query_class, negate = \
parse_query_part(query_part, query_classes, prefixes)
# If there's no key (field name) specified, this is a "match
# anything" query.
if key is None:
if issubclass(query_class, query.FieldQuery):
# The query type matches a specific field, but none was
# specified. So we use a version of the query that matches
# any field.
q = query.AnyFieldQuery(pattern, model_cls._search_fields,
query_class)
if negate:
return query.NotQuery(q)
else:
return q
else:
# Non-field query type.
if negate:
return query.NotQuery(query_class(pattern))
else:
return query_class(pattern)
# Otherwise, this must be a `FieldQuery`. Use the field name to
# construct the query object.
key = key.lower()
q = query_class(key.lower(), pattern, key in model_cls._fields)
if negate:
return query.NotQuery(q)
return q
def query_from_strings(query_cls, model_cls, prefixes, query_parts):
"""Creates a collection query of type `query_cls` from a list of
strings in the format used by parse_query_part. `model_cls`
determines how queries are constructed from strings.
"""
subqueries = []
for part in query_parts:
subqueries.append(construct_query_part(model_cls, prefixes, part))
if not subqueries: # No terms in query.
subqueries = [query.TrueQuery()]
return query_cls(subqueries)
def construct_sort_part(model_cls, part):
"""Create a `Sort` from a single string criterion.
`model_cls` is the `Model` being queried. `part` is a single string
ending in ``+`` or ``-`` indicating the sort.
"""
assert part, "part must be a field name and + or -"
field = part[:-1]
assert field, "field is missing"
direction = part[-1]
assert direction in ('+', '-'), "part must end with + or -"
is_ascending = direction == '+'
case_insensitive = beets.config['sort_case_insensitive'].get(bool)
if field in model_cls._sorts:
sort = model_cls._sorts[field](model_cls, is_ascending,
case_insensitive)
elif field in model_cls._fields:
sort = query.FixedFieldSort(field, is_ascending, case_insensitive)
else:
# Flexible or computed.
sort = query.SlowFieldSort(field, is_ascending, case_insensitive)
return sort
def sort_from_strings(model_cls, sort_parts):
"""Create a `Sort` from a list of sort criteria (strings).
"""
if not sort_parts:
sort = query.NullSort()
elif len(sort_parts) == 1:
sort = construct_sort_part(model_cls, sort_parts[0])
else:
sort = query.MultipleSort()
for part in sort_parts:
sort.add_sort(construct_sort_part(model_cls, part))
return sort
def parse_sorted_query(model_cls, parts, prefixes={}):
"""Given a list of strings, create the `Query` and `Sort` that they
represent.
"""
# Separate query token and sort token.
query_parts = []
sort_parts = []
# Split up query in to comma-separated subqueries, each representing
# an AndQuery, which need to be joined together in one OrQuery
subquery_parts = []
for part in parts + [u',']:
if part.endswith(u','):
# Ensure we can catch "foo, bar" as well as "foo , bar"
last_subquery_part = part[:-1]
if last_subquery_part:
subquery_parts.append(last_subquery_part)
# Parse the subquery in to a single AndQuery
# TODO: Avoid needlessly wrapping AndQueries containing 1 subquery?
query_parts.append(query_from_strings(
query.AndQuery, model_cls, prefixes, subquery_parts
))
del subquery_parts[:]
else:
# Sort parts (1) end in + or -, (2) don't have a field, and
# (3) consist of more than just the + or -.
if part.endswith((u'+', u'-')) \
and u':' not in part \
and len(part) > 1:
sort_parts.append(part)
else:
subquery_parts.append(part)
# Avoid needlessly wrapping single statements in an OR
q = query.OrQuery(query_parts) if len(query_parts) > 1 else query_parts[0]
s = sort_from_strings(model_cls, sort_parts)
return q, s

133
libs/beets/dbcore/types.py
View file

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2014, Adrian Sampson.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
@ -14,59 +15,117 @@
"""Representation of type information for DBCore model fields.
"""
from __future__ import division, absolute_import, print_function
from . import query
from beets.util import str2bool
# Abstract base.
class Type(object):
"""An object encapsulating the type of a model field. Includes
information about how to store the value in the database, query,
format, and parse a given field.
information about how to store, query, format, and parse a given
field.
"""
sql = None
sql = u'TEXT'
"""The SQLite column type for the value.
"""
query = None
query = query.SubstringQuery
"""The `Query` subclass to be used when querying the field.
"""
model_type = unicode
"""The Python type that is used to represent the value in the model.
The model is guaranteed to return a value of this type if the field
is accessed. To this end, the constructor is used by the `normalize`
and `from_sql` methods and the `default` property.
"""
@property
def null(self):
"""The value to be exposed when the underlying value is None.
"""
return self.model_type()
def format(self, value):
"""Given a value of this type, produce a Unicode string
representing the value. This is used in template evaluation.
"""
raise NotImplementedError()
if value is None:
value = self.null
# `self.null` might be `None`
if value is None:
value = u''
if isinstance(value, bytes):
value = value.decode('utf8', 'ignore')
return unicode(value)
def parse(self, string):
"""Parse a (possibly human-written) string and return the
indicated value of this type.
"""
raise NotImplementedError()
try:
return self.model_type(string)
except ValueError:
return self.null
def normalize(self, value):
"""Given a value that will be assigned into a field of this
type, normalize the value to have the appropriate type. This
base implementation only reinterprets `None`.
"""
if value is None:
return self.null
else:
# TODO This should eventually be replaced by
# `self.model_type(value)`
return value
def from_sql(self, sql_value):
"""Receives the value stored in the SQL backend and return the
value to be stored in the model.
For fixed fields the type of `value` is determined by the column
type affinity given in the `sql` property and the SQL to Python
mapping of the database adapter. For more information see:
http://www.sqlite.org/datatype3.html
https://docs.python.org/2/library/sqlite3.html#sqlite-and-python-types
Flexible fields have the type affinity `TEXT`. This means the
`sql_value` is either a `buffer` or a `unicode` object` and the
method must handle these in addition.
"""
if isinstance(sql_value, buffer):
sql_value = bytes(sql_value).decode('utf8', 'ignore')
if isinstance(sql_value, unicode):
return self.parse(sql_value)
else:
return self.normalize(sql_value)
def to_sql(self, model_value):
"""Convert a value as stored in the model object to a value used
by the database adapter.
"""
return model_value
# Reusable types.
class Default(Type):
null = None
class Integer(Type):
"""A basic integer type.
"""
sql = u'INTEGER'
query = query.NumericQuery
def format(self, value):
return unicode(value or 0)
def parse(self, string):
try:
return int(string)
except ValueError:
return 0
model_type = int
class PaddedInt(Integer):
@ -93,9 +152,14 @@ class ScaledInt(Integer):
class Id(Integer):
"""An integer used as the row key for a SQLite table.
"""An integer used as the row id or a foreign key in a SQLite table.
This type is nullable: None values are not translated to zero.
"""
sql = u'INTEGER PRIMARY KEY'
null = None
def __init__(self, primary=True):
if primary:
self.sql = u'INTEGER PRIMARY KEY'
class Float(Type):
@ -103,15 +167,16 @@ class Float(Type):
"""
sql = u'REAL'
query = query.NumericQuery
model_type = float
def format(self, value):
return u'{0:.1f}'.format(value or 0.0)
def parse(self, string):
try:
return float(string)
except ValueError:
return 0.0
class NullFloat(Float):
"""Same as `Float`, but does not normalize `None` to `0.0`.
"""
null = None
class String(Type):
@ -120,21 +185,27 @@ class String(Type):
sql = u'TEXT'
query = query.SubstringQuery
def format(self, value):
return unicode(value) if value else u''
def parse(self, string):
return string
class Boolean(Type):
"""A boolean type.
"""
sql = u'INTEGER'
query = query.BooleanQuery
model_type = bool
def format(self, value):
return unicode(bool(value))
def parse(self, string):
return str2bool(string)
# Shared instances of common types.
DEFAULT = Default()
INTEGER = Integer()
PRIMARY_ID = Id(True)
FOREIGN_ID = Id(False)
FLOAT = Float()
NULL_FLOAT = NullFloat()
STRING = String()
BOOLEAN = Boolean()

1842
libs/beets/importer.py
View file

File diff suppressed because it is too large Load diff

1138
libs/beets/library.py
View file

File diff suppressed because it is too large Load diff

133
libs/beets/logging.py Normal file
View file

@ -0,0 +1,133 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""A drop-in replacement for the standard-library `logging` module that
allows {}-style log formatting on Python 2 and 3.
Provides everything the "logging" module does. The only difference is
that when getLogger(name) instantiates a logger that logger uses
{}-style formatting.
"""
from __future__ import division, absolute_import, print_function
from copy import copy
from logging import * # noqa
import subprocess
import threading
def logsafe(val):
"""Coerce a potentially "problematic" value so it can be formatted
in a Unicode log string.
This works around a number of pitfalls when logging objects in
Python 2:
- Logging path names, which must be byte strings, requires
conversion for output.
- Some objects, including some exceptions, will crash when you call
`unicode(v)` while `str(v)` works fine. CalledProcessError is an
example.
"""
# Already Unicode.
if isinstance(val, unicode):
return val
# Bytestring: needs decoding.
elif isinstance(val, bytes):
# Blindly convert with UTF-8. Eventually, it would be nice to
# (a) only do this for paths, if they can be given a distinct
# type, and (b) warn the developer if they do this for other
# bytestrings.
return val.decode('utf8', 'replace')
# A "problem" object: needs a workaround.
elif isinstance(val, subprocess.CalledProcessError):
try:
return unicode(val)
except UnicodeDecodeError:
# An object with a broken __unicode__ formatter. Use __str__
# instead.
return str(val).decode('utf8', 'replace')
# Other objects are used as-is so field access, etc., still works in
# the format string.
else:
return val
class StrFormatLogger(Logger):
"""A version of `Logger` that uses `str.format`-style formatting
instead of %-style formatting.
"""
class _LogMessage(object):
def __init__(self, msg, args, kwargs):
self.msg = msg
self.args = args
self.kwargs = kwargs
def __str__(self):
args = [logsafe(a) for a in self.args]
kwargs = dict((k, logsafe(v)) for (k, v) in self.kwargs.items())
return self.msg.format(*args, **kwargs)
def _log(self, level, msg, args, exc_info=None, extra=None, **kwargs):
"""Log msg.format(*args, **kwargs)"""
m = self._LogMessage(msg, args, kwargs)
return super(StrFormatLogger, self)._log(level, m, (), exc_info, extra)
class ThreadLocalLevelLogger(Logger):
"""A version of `Logger` whose level is thread-local instead of shared.
"""
def __init__(self, name, level=NOTSET):
self._thread_level = threading.local()
self.default_level = NOTSET
super(ThreadLocalLevelLogger, self).__init__(name, level)
@property
def level(self):
try:
return self._thread_level.level
except AttributeError:
self._thread_level.level = self.default_level
return self.level
@level.setter
def level(self, value):
self._thread_level.level = value
def set_global_level(self, level):
"""Set the level on the current thread + the default value for all
threads.
"""
self.default_level = level
self.setLevel(level)
class BeetsLogger(ThreadLocalLevelLogger, StrFormatLogger):
pass
my_manager = copy(Logger.manager)
my_manager.loggerClass = BeetsLogger
def getLogger(name=None): # noqa
if name:
return my_manager.getLogger(name)
else:
return Logger.root

886
libs/beets/mediafile.py
View file

File diff suppressed because it is too large Load diff

246
libs/beets/plugins.py
View file

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2013, Adrian Sampson.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
@ -14,11 +15,17 @@
"""Support for beets plugins."""
import logging
from __future__ import division, absolute_import, print_function
import inspect
import traceback
import re
from collections import defaultdict
from functools import wraps
import beets
from beets import logging
from beets import mediafile
PLUGIN_NAMESPACE = 'beetsplug'
@ -30,6 +37,31 @@ LASTFM_KEY = '2dc3914abf35f0d9c92d97d8f8e42b43'
log = logging.getLogger('beets')
class PluginConflictException(Exception):
"""Indicates that the services provided by one plugin conflict with
those of another.
For example two plugins may define different types for flexible fields.
"""
class PluginLogFilter(logging.Filter):
"""A logging filter that identifies the plugin that emitted a log
message.
"""
def __init__(self, plugin):
self.prefix = u'{0}: '.format(plugin.name)
def filter(self, record):
if hasattr(record.msg, 'msg') and isinstance(record.msg.msg,
basestring):
# A _LogMessage from our hacked-up Logging replacement.
record.msg.msg = self.prefix + record.msg.msg
elif isinstance(record.msg, basestring):
record.msg = self.prefix + record.msg
return True
# Managing the plugins themselves.
class BeetsPlugin(object):
@ -40,8 +72,6 @@ class BeetsPlugin(object):
def __init__(self, name=None):
"""Perform one-time plugin setup.
"""
_add_media_fields(self.item_fields())
self.import_stages = []
self.name = name or self.__module__.split('.')[-1]
self.config = beets.config[self.name]
if not self.template_funcs:
@ -50,6 +80,12 @@ class BeetsPlugin(object):
self.template_fields = {}
if not self.album_template_fields:
self.album_template_fields = {}
self.import_stages = []
self._log = log.getChild(self.name)
self._log.setLevel(logging.NOTSET) # Use `beets` logger level.
if not any(isinstance(f, PluginLogFilter) for f in self._log.filters):
self._log.addFilter(PluginLogFilter(self))
def commands(self):
"""Should return a list of beets.ui.Subcommand objects for
@ -57,6 +93,46 @@ class BeetsPlugin(object):
"""
return ()
def get_import_stages(self):
"""Return a list of functions that should be called as importer
pipelines stages.
The callables are wrapped versions of the functions in
`self.import_stages`. Wrapping provides some bookkeeping for the
plugin: specifically, the logging level is adjusted to WARNING.
"""
return [self._set_log_level_and_params(logging.WARNING, import_stage)
for import_stage in self.import_stages]
def _set_log_level_and_params(self, base_log_level, func):
"""Wrap `func` to temporarily set this plugin's logger level to
`base_log_level` + config options (and restore it to its previous
value after the function returns). Also determines which params may not
be sent for backwards-compatibility.
"""
argspec = inspect.getargspec(func)
@wraps(func)
def wrapper(*args, **kwargs):
assert self._log.level == logging.NOTSET
verbosity = beets.config['verbose'].get(int)
log_level = max(logging.DEBUG, base_log_level - 10 * verbosity)
self._log.setLevel(log_level)
try:
try:
return func(*args, **kwargs)
except TypeError as exc:
if exc.args[0].startswith(func.__name__):
# caused by 'func' and not stuff internal to 'func'
kwargs = dict((arg, val) for arg, val in kwargs.items()
if arg in argspec.args)
return func(*args, **kwargs)
else:
raise
finally:
self._log.setLevel(logging.NOTSET)
return wrapper
def queries(self):
"""Should return a dict mapping prefixes to Query subclasses.
"""
@ -86,14 +162,6 @@ class BeetsPlugin(object):
"""
return ()
def item_fields(self):
"""Returns field descriptors to be added to the MediaFile class,
in the form of a dictionary whose keys are field names and whose
values are descriptor (e.g., MediaField) instances. The Library
database schema is not (currently) extended.
"""
return {}
def album_for_id(self, album_id):
"""Return an AlbumInfo object or None if no matching release was
found.
@ -106,38 +174,36 @@ class BeetsPlugin(object):
"""
return None
def add_media_field(self, name, descriptor):
"""Add a field that is synchronized between media files and items.
When a media field is added ``item.write()`` will set the name
property of the item's MediaFile to ``item[name]`` and save the
changes. Similarly ``item.read()`` will set ``item[name]`` to
the value of the name property of the media file.
``descriptor`` must be an instance of ``mediafile.MediaField``.
"""
# Defer impor to prevent circular dependency
from beets import library
mediafile.MediaFile.add_field(name, descriptor)
library.Item._media_fields.add(name)
_raw_listeners = None
listeners = None
@classmethod
def register_listener(cls, event, func):
"""Add a function as a listener for the specified event. (An
imperative alternative to the @listen decorator.)
def register_listener(self, event, func):
"""Add a function as a listener for the specified event.
"""
if cls.listeners is None:
wrapped_func = self._set_log_level_and_params(logging.WARNING, func)
cls = self.__class__
if cls.listeners is None or cls._raw_listeners is None:
cls._raw_listeners = defaultdict(list)
cls.listeners = defaultdict(list)
cls.listeners[event].append(func)
@classmethod
def listen(cls, event):
"""Decorator that adds a function as an event handler for the
specified event (as a string). The parameters passed to function
will vary depending on what event occurred.
The function should respond to named parameters.
function(**kwargs) will trap all arguments in a dictionary.
Example:
>>> @MyPlugin.listen("imported")
>>> def importListener(**kwargs):
>>> pass
"""
def helper(func):
if cls.listeners is None:
cls.listeners = defaultdict(list)
cls.listeners[event].append(func)
return func
return helper
if func not in cls._raw_listeners[event]:
cls._raw_listeners[event].append(func)
cls.listeners[event].append(wrapped_func)
template_funcs = None
template_fields = None
@ -170,7 +236,10 @@ class BeetsPlugin(object):
return func
return helper
_classes = set()
def load_plugins(names=()):
"""Imports the modules for a sequence of plugin names. Each name
must be the name of a Python module under the "beetsplug" namespace
@ -178,14 +247,14 @@ def load_plugins(names=()):
BeetsPlugin subclasses desired.
"""
for name in names:
modname = '%s.%s' % (PLUGIN_NAMESPACE, name)
modname = '{0}.{1}'.format(PLUGIN_NAMESPACE, name)
try:
try:
namespace = __import__(modname, None, None)
except ImportError as exc:
# Again, this is hacky:
if exc.args[0].endswith(' ' + name):
log.warn('** plugin %s not found' % name)
log.warn(u'** plugin {0} not found', name)
else:
raise
else:
@ -195,10 +264,16 @@ def load_plugins(names=()):
_classes.add(obj)
except:
log.warn('** error loading plugin %s' % name)
log.warn(traceback.format_exc())
log.warn(
u'** error loading plugin {}:\n{}',
name,
traceback.format_exc(),
)
_instances = {}
def find_plugins():
"""Returns a list of BeetsPlugin subclass instances from all
currently loaded beets plugins. Loads the default plugin set
@ -224,6 +299,7 @@ def commands():
out += plugin.commands()
return out
def queries():
"""Returns a dict mapping prefix strings to Query subclasses all loaded
plugins.
@ -233,6 +309,24 @@ def queries():
out.update(plugin.queries())
return out
def types(model_cls):
# Gives us `item_types` and `album_types`
attr_name = '{0}_types'.format(model_cls.__name__.lower())
types = {}
for plugin in find_plugins():
plugin_types = getattr(plugin, attr_name, {})
for field in plugin_types:
if field in types and plugin_types[field] != types[field]:
raise PluginConflictException(
u'Plugin {0} defines flexible field {1} '
u'which has already been defined with '
u'another type.'.format(plugin.name, field)
)
types.update(plugin_types)
return types
def track_distance(item, info):
"""Gets the track distance calculated by all loaded plugins.
Returns a Distance object.
@ -243,6 +337,7 @@ def track_distance(item, info):
dist.update(plugin.track_distance(item, info))
return dist
def album_distance(items, album_info, mapping):
"""Returns the album distance calculated by plugins."""
from beets.autotag.hooks import Distance
@ -251,6 +346,7 @@ def album_distance(items, album_info, mapping):
dist.update(plugin.album_distance(items, album_info, mapping))
return dist
def candidates(items, artist, album, va_likely):
"""Gets MusicBrainz candidates for an album from each plugin.
"""
@ -259,6 +355,7 @@ def candidates(items, artist, album, va_likely):
out.extend(plugin.candidates(items, artist, album, va_likely))
return out
def item_candidates(item, artist, title):
"""Gets MusicBrainz candidates for an item from the plugins.
"""
@ -267,6 +364,7 @@ def item_candidates(item, artist, title):
out.extend(plugin.item_candidates(item, artist, title))
return out
def album_for_id(album_id):
"""Get AlbumInfo objects for a given ID string.
"""
@ -277,6 +375,7 @@ def album_for_id(album_id):
out.append(res)
return out
def track_for_id(track_id):
"""Get TrackInfo objects for a given ID string.
"""
@ -287,6 +386,7 @@ def track_for_id(track_id):
out.append(res)
return out
def template_funcs():
"""Get all the template functions declared by plugins as a
dictionary.
@ -297,19 +397,12 @@ def template_funcs():
funcs.update(plugin.template_funcs)
return funcs
def _add_media_fields(fields):
"""Adds a {name: descriptor} dictionary of fields to the MediaFile
class. Called during the plugin initialization.
"""
for key, value in fields.iteritems():
setattr(mediafile.MediaFile, key, value)
def import_stages():
"""Get a list of import stage functions defined by plugins."""
stages = []
for plugin in find_plugins():
if hasattr(plugin, 'import_stages'):
stages += plugin.import_stages
stages += plugin.get_import_stages()
return stages
@ -325,6 +418,7 @@ def item_field_getters():
funcs.update(plugin.template_fields)
return funcs
def album_field_getters():
"""As above, for album fields.
"""
@ -348,12 +442,48 @@ def event_handlers():
all_handlers[event] += handlers
return all_handlers
def send(event, **arguments):
"""Sends an event to all assigned event listeners. Event is the
name of the event to send, all other named arguments go to the
event handler(s).
Returns a list of return values from the handlers.
def send(event, **arguments):
"""Send an event to all assigned event listeners.
`event` is the name of the event to send, all other named arguments
are passed along to the handlers.
Return a list of non-None values returned from the handlers.
"""
log.debug('Sending event: %s' % event)
return [handler(**arguments) for handler in event_handlers()[event]]
log.debug(u'Sending event: {0}', event)
results = []
for handler in event_handlers()[event]:
result = handler(**arguments)
if result is not None:
results.append(result)
return results
def feat_tokens(for_artist=True):
"""Return a regular expression that matches phrases like "featuring"
that separate a main artist or a song title from secondary artists.
The `for_artist` option determines whether the regex should be
suitable for matching artist fields (the default) or title fields.
"""
feat_words = ['ft', 'featuring', 'feat', 'feat.', 'ft.']
if for_artist:
feat_words += ['with', 'vs', 'and', 'con', '&']
return '(?<=\s)(?:{0})(?=\s)'.format(
'|'.join(re.escape(x) for x in feat_words)
)
def sanitize_choices(choices, choices_all):
"""Clean up a stringlist configuration attribute: keep only choices
elements present in choices_all, remove duplicate elements, expand '*'
wildcard while keeping original stringlist order.
"""
seen = set()
others = [x for x in choices_all if x not in choices]
res = []
for s in choices:
if s in list(choices_all) + ['*']:
if not (s in seen or seen.add(s)):
res.extend(list(others) if s == '*' else [s])
return res

826
libs/beets/ui/__init__.py
View file

File diff suppressed because it is too large Load diff

1227
libs/beets/ui/commands.py
View file

File diff suppressed because it is too large Load diff

401
libs/beets/ui/migrate.py
View file

@ -1,401 +0,0 @@
# This file is part of beets.
# Copyright 2013, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Conversion from legacy (pre-1.1) configuration to Confit/YAML
configuration.
"""
import os
import ConfigParser
import codecs
import yaml
import logging
import time
import itertools
import re
import beets
from beets import util
from beets import ui
from beets.util import confit
CONFIG_PATH_VAR = 'BEETSCONFIG'
DEFAULT_CONFIG_FILENAME_UNIX = '.beetsconfig'
DEFAULT_CONFIG_FILENAME_WINDOWS = 'beetsconfig.ini'
DEFAULT_LIBRARY_FILENAME_UNIX = '.beetsmusic.blb'
DEFAULT_LIBRARY_FILENAME_WINDOWS = 'beetsmusic.blb'
WINDOWS_BASEDIR = os.environ.get('APPDATA') or '~'
OLD_CONFIG_SUFFIX = '.old'
PLUGIN_NAMES = {
'rdm': 'random',
'fuzzy_search': 'fuzzy',
}
AUTO_KEYS = ('automatic', 'autofetch', 'autoembed', 'autoscrub')
IMPORTFEEDS_PREFIX = 'feeds_'
CONFIG_MIGRATED_MESSAGE = u"""
You appear to be upgrading from beets 1.0 (or earlier) to 1.1. Your
configuration file has been migrated automatically to:
{newconfig}
Edit this file to configure beets. You might want to remove your
old-style ".beetsconfig" file now. See the documentation for more
details on the new configuration system:
http://beets.readthedocs.org/page/reference/config.html
""".strip()
DB_MIGRATED_MESSAGE = u'Your database file has also been copied to:\n{newdb}'
YAML_COMMENT = '# Automatically migrated from legacy .beetsconfig.\n\n'
log = logging.getLogger('beets')
# An itertools recipe.
def grouper(n, iterable):
args = [iter(iterable)] * n
return itertools.izip_longest(*args)
def _displace(fn):
"""Move a file aside using a timestamp suffix so a new file can be
put in its place.
"""
util.move(
fn,
u'{0}.old.{1}'.format(fn, int(time.time())),
True
)
def default_paths():
"""Produces the appropriate default config and library database
paths for the current system. On Unix, this is always in ~. On
Windows, tries ~ first and then $APPDATA for the config and library
files (for backwards compatibility).
"""
windows = os.path.__name__ == 'ntpath'
if windows:
windata = os.environ.get('APPDATA') or '~'
# Shorthand for joining paths.
def exp(*vals):
return os.path.expanduser(os.path.join(*vals))
config = exp('~', DEFAULT_CONFIG_FILENAME_UNIX)
if windows and not os.path.exists(config):
config = exp(windata, DEFAULT_CONFIG_FILENAME_WINDOWS)
libpath = exp('~', DEFAULT_LIBRARY_FILENAME_UNIX)
if windows and not os.path.exists(libpath):
libpath = exp(windata, DEFAULT_LIBRARY_FILENAME_WINDOWS)
return config, libpath
def get_config():
"""Using the same logic as beets 1.0, locate and read the
.beetsconfig file. Return a ConfigParser instance or None if no
config is found.
"""
default_config, default_libpath = default_paths()
if CONFIG_PATH_VAR in os.environ:
configpath = os.path.expanduser(os.environ[CONFIG_PATH_VAR])
else:
configpath = default_config
config = ConfigParser.SafeConfigParser()
if os.path.exists(util.syspath(configpath)):
with codecs.open(configpath, 'r', encoding='utf-8') as f:
config.readfp(f)
return config, configpath
else:
return None, configpath
def flatten_config(config):
"""Given a ConfigParser, flatten the values into a dict-of-dicts
representation where each section gets its own dictionary of values.
"""
out = confit.OrderedDict()
for section in config.sections():
sec_dict = out[section] = confit.OrderedDict()
for option in config.options(section):
sec_dict[option] = config.get(section, option, True)
return out
def transform_value(value):
"""Given a string read as the value of a config option, return a
massaged version of that value (possibly with a different type).
"""
# Booleans.
if value.lower() in ('false', 'no', 'off'):
return False
elif value.lower() in ('true', 'yes', 'on'):
return True
# Integers.
try:
return int(value)
except ValueError:
pass
# Floats.
try:
return float(value)
except ValueError:
pass
return value
def transform_data(data):
"""Given a dict-of-dicts representation of legacy config data, tweak
the data into a new form. This new form is suitable for dumping as
YAML.
"""
out = confit.OrderedDict()
for section, pairs in data.items():
if section == 'beets':
# The "main" section. In the new config system, these values
# are in the "root": no section at all.
for key, value in pairs.items():
value = transform_value(value)
if key.startswith('import_'):
# Importer config is now under an "import:" key.
if 'import' not in out:
out['import'] = confit.OrderedDict()
out['import'][key[7:]] = value
elif key == 'plugins':
# Renamed plugins.
plugins = value.split()
new_plugins = [PLUGIN_NAMES.get(p, p) for p in plugins]
out['plugins'] = ' '.join(new_plugins)
elif key == 'replace':
# YAMLy representation for character replacements.
replacements = confit.OrderedDict()
for pat, repl in grouper(2, value.split()):
if repl == '<strip>':
repl = ''
replacements[pat] = repl
out['replace'] = replacements
elif key == 'pluginpath':
# Used to be a colon-separated string. Now a list.
out['pluginpath'] = value.split(':')
else:
out[key] = value
elif pairs:
# Other sections (plugins, etc).
sec_out = out[section] = confit.OrderedDict()
for key, value in pairs.items():
# Standardized "auto" option.
if key in AUTO_KEYS:
key = 'auto'
# Unnecessary : hack in queries.
if section == 'paths':
key = key.replace('_', ':')
# Changed option names for importfeeds plugin.
if section == 'importfeeds':
if key.startswith(IMPORTFEEDS_PREFIX):
key = key[len(IMPORTFEEDS_PREFIX):]
sec_out[key] = transform_value(value)
return out
class Dumper(yaml.SafeDumper):
"""A PyYAML Dumper that represents OrderedDicts as ordinary mappings
(in order, of course).
"""
# From http://pyyaml.org/attachment/ticket/161/use_ordered_dict.py
def represent_mapping(self, tag, mapping, flow_style=None):
value = []
node = yaml.MappingNode(tag, value, flow_style=flow_style)
if self.alias_key is not None:
self.represented_objects[self.alias_key] = node
best_style = True
if hasattr(mapping, 'items'):
mapping = list(mapping.items())
for item_key, item_value in mapping:
node_key = self.represent_data(item_key)
node_value = self.represent_data(item_value)
if not (isinstance(node_key, yaml.ScalarNode) and \
not node_key.style):
best_style = False
if not (isinstance(node_value, yaml.ScalarNode) and \
not node_value.style):
best_style = False
value.append((node_key, node_value))
if flow_style is None:
if self.default_flow_style is not None:
node.flow_style = self.default_flow_style
else:
node.flow_style = best_style
return node
Dumper.add_representer(confit.OrderedDict, Dumper.represent_dict)
def migrate_config(replace=False):
"""Migrate a legacy beetsconfig file to a new-style config.yaml file
in an appropriate place. If `replace` is enabled, then any existing
config.yaml will be moved aside. Otherwise, the process is aborted
when the file exists.
"""
# Load legacy configuration data, if any.
config, configpath = get_config()
if not config:
log.debug(u'no config file found at {0}'.format(
util.displayable_path(configpath)
))
return
# Get the new configuration file path and possibly move it out of
# the way.
destfn = os.path.join(beets.config.config_dir(), confit.CONFIG_FILENAME)
if os.path.exists(destfn):
if replace:
log.debug(u'moving old config aside: {0}'.format(
util.displayable_path(destfn)
))
_displace(destfn)
else:
# File exists and we won't replace it. We're done.
return
log.debug(u'migrating config file {0}'.format(
util.displayable_path(configpath)
))
# Convert the configuration to a data structure ready to be dumped
# as the new Confit file.
data = transform_data(flatten_config(config))
# Encode result as YAML.
yaml_out = yaml.dump(
data,
Dumper=Dumper,
default_flow_style=False,
indent=4,
width=1000,
)
# A ridiculous little hack to add some whitespace between "sections"
# in the YAML output. I hope this doesn't break any YAML syntax.
yaml_out = re.sub(r'(\n\w+:\n [^-\s])', '\n\\1', yaml_out)
yaml_out = YAML_COMMENT + yaml_out
# Write the data to the new config destination.
log.debug(u'writing migrated config to {0}'.format(
util.displayable_path(destfn)
))
with open(destfn, 'w') as f:
f.write(yaml_out)
return destfn
def migrate_db(replace=False):
"""Copy the beets library database file to the new location (e.g.,
from ~/.beetsmusic.blb to ~/.config/beets/library.db).
"""
_, srcfn = default_paths()
destfn = beets.config['library'].as_filename()
if not os.path.exists(srcfn) or srcfn == destfn:
# Old DB does not exist or we're configured to point to the same
# database. Do nothing.
return
if os.path.exists(destfn):
if replace:
log.debug(u'moving old database aside: {0}'.format(
util.displayable_path(destfn)
))
_displace(destfn)
else:
return
log.debug(u'copying database from {0} to {1}'.format(
util.displayable_path(srcfn), util.displayable_path(destfn)
))
util.copy(srcfn, destfn)
return destfn
def migrate_state(replace=False):
"""Copy the beets runtime state file from the old path (i.e.,
~/.beetsstate) to the new path (i.e., ~/.config/beets/state.pickle).
"""
srcfn = os.path.expanduser(os.path.join('~', '.beetsstate'))
if not os.path.exists(srcfn):
return
destfn = beets.config['statefile'].as_filename()
if os.path.exists(destfn):
if replace:
_displace(destfn)
else:
return
log.debug(u'copying state file from {0} to {1}'.format(
util.displayable_path(srcfn), util.displayable_path(destfn)
))
util.copy(srcfn, destfn)
return destfn
# Automatic migration when beets starts.
def automigrate():
"""Migrate the configuration, database, and state files. If any
migration occurs, print out a notice with some helpful next steps.
"""
config_fn = migrate_config()
db_fn = migrate_db()
migrate_state()
if config_fn:
ui.print_(ui.colorize('fuchsia', u'MIGRATED CONFIGURATION'))
ui.print_(CONFIG_MIGRATED_MESSAGE.format(
newconfig=util.displayable_path(config_fn))
)
if db_fn:
ui.print_(DB_MIGRATED_MESSAGE.format(
newdb=util.displayable_path(db_fn)
))
ui.input_(ui.colorize('fuchsia', u'Press ENTER to continue:'))
ui.print_()
# CLI command for explicit migration.
migrate_cmd = ui.Subcommand('migrate', help='convert legacy config')
def migrate_func(lib, opts, args):
"""Explicit command for migrating files. Existing files in each
destination are moved aside.
"""
config_fn = migrate_config(replace=True)
if config_fn:
log.info(u'Migrated configuration to: {0}'.format(
util.displayable_path(config_fn)
))
db_fn = migrate_db(replace=True)
if db_fn:
log.info(u'Migrated library database to: {0}'.format(
util.displayable_path(db_fn)
))
state_fn = migrate_state(replace=True)
if state_fn:
log.info(u'Migrated state file to: {0}'.format(
util.displayable_path(state_fn)
))
migrate_cmd.func = migrate_func

412
libs/beets/util/__init__.py
View file

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2013, Adrian Sampson.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
@ -13,20 +14,25 @@
# included in all copies or substantial portions of the Software.
"""Miscellaneous utility functions."""
from __future__ import division
from __future__ import division, absolute_import, print_function
import os
import sys
import re
import shutil
import fnmatch
from collections import defaultdict
from collections import Counter
import traceback
import subprocess
import platform
import shlex
from beets.util import hidden
MAX_FILENAME_LENGTH = 200
WINDOWS_MAGIC_PREFIX = u'\\\\?\\'
class HumanReadableException(Exception):
"""An Exception that can include a human-readable error message to
be logged without a traceback. Can preserve a traceback for
@ -51,10 +57,10 @@ class HumanReadableException(Exception):
def _gerund(self):
"""Generate a (likely) gerund form of the English verb.
"""
if ' ' in self.verb:
if u' ' in self.verb:
return self.verb
gerund = self.verb[:-1] if self.verb.endswith('e') else self.verb
gerund += 'ing'
gerund = self.verb[:-1] if self.verb.endswith(u'e') else self.verb
gerund += u'ing'
return gerund
def _reasonstr(self):
@ -80,7 +86,8 @@ class HumanReadableException(Exception):
"""
if self.tb:
logger.debug(self.tb)
logger.error(u'{0}: {1}'.format(self.error_kind, self.args[0]))
logger.error(u'{0}: {1}', self.error_kind, self.args[0])
class FilesystemError(HumanReadableException):
"""An error that occurred while performing a filesystem manipulation
@ -111,6 +118,7 @@ class FilesystemError(HumanReadableException):
return u'{0} {1}'.format(self._reasonstr(), clause)
def normpath(path):
"""Provide the canonical form of the path suitable for storing in
the database.
@ -119,6 +127,7 @@ def normpath(path):
path = os.path.normpath(os.path.abspath(os.path.expanduser(path)))
return bytestring_path(path)
def ancestry(path):
"""Return a list consisting of path's parent directory, its
grandparent, and so on. For instance:
@ -137,11 +146,13 @@ def ancestry(path):
break
last_path = path
if path: # don't yield ''
if path:
# don't yield ''
out.insert(0, path)
return out
def sorted_walk(path, ignore=(), logger=None):
def sorted_walk(path, ignore=(), ignore_hidden=False, logger=None):
"""Like `os.walk`, but yields things in case-insensitive sorted,
breadth-first order. Directory and file names matching any glob
pattern in `ignore` are skipped. If `logger` is provided, then
@ -175,6 +186,7 @@ def sorted_walk(path, ignore=(), logger=None):
# Add to output as either a file or a directory.
cur = os.path.join(path, base)
if (ignore_hidden and not hidden.is_hidden(cur)) or not ignore_hidden:
if os.path.isdir(syspath(cur)):
dirs.append(base)
else:
@ -189,9 +201,10 @@ def sorted_walk(path, ignore=(), logger=None):
for base in dirs:
cur = os.path.join(path, base)
# yield from sorted_walk(...)
for res in sorted_walk(cur, ignore, logger):
for res in sorted_walk(cur, ignore, ignore_hidden, logger):
yield res
def mkdirall(path):
"""Make all the enclosing directories of path (like mkdir -p on the
parent).
@ -204,6 +217,7 @@ def mkdirall(path):
raise FilesystemError(exc, 'create', (ancestor,),
traceback.format_exc())
def fnmatch_all(names, patterns):
"""Determine whether all strings in `names` match at least one of
the `patterns`, which should be shell glob expressions.
@ -218,6 +232,7 @@ def fnmatch_all(names, patterns):
return False
return True
def prune_dirs(path, root=None, clutter=('.DS_Store', 'Thumbs.db')):
"""If path is an empty directory, then remove it. Recursively remove
path's ancestry up to root (which is never removed) where there are
@ -236,7 +251,7 @@ def prune_dirs(path, root=None, clutter=('.DS_Store', 'Thumbs.db')):
ancestors = []
elif root in ancestors:
# Only remove directories below the root.
ancestors = ancestors[ancestors.index(root)+1:]
ancestors = ancestors[ancestors.index(root) + 1:]
else:
# Remove nothing.
return
@ -258,6 +273,7 @@ def prune_dirs(path, root=None, clutter=('.DS_Store', 'Thumbs.db')):
else:
break
def components(path):
"""Return a list of the path components in path. For instance:
@ -281,6 +297,7 @@ def components(path):
return comps
def _fsencoding():
"""Get the system's filesystem encoding. On Windows, this is always
UTF-8 (not MBCS).
@ -295,12 +312,13 @@ def _fsencoding():
encoding = 'utf8'
return encoding
def bytestring_path(path):
"""Given a path, which is either a str or a unicode, returns a str
"""Given a path, which is either a bytes or a unicode, returns a str
path (ensuring that we never deal with Unicode pathnames).
"""
# Pass through bytestrings.
if isinstance(path, str):
if isinstance(path, bytes):
return path
# On Windows, remove the magic prefix added by `syspath`. This makes
@ -315,6 +333,7 @@ def bytestring_path(path):
except (UnicodeError, LookupError):
return path.encode('utf8')
def displayable_path(path, separator=u'; '):
"""Attempts to decode a bytestring path to a unicode object for the
purpose of displaying it to the user. If the `path` argument is a
@ -324,7 +343,7 @@ def displayable_path(path, separator=u'; '):
return separator.join(displayable_path(p) for p in path)
elif isinstance(path, unicode):
return path
elif not isinstance(path, str):
elif not isinstance(path, bytes):
# A non-string object: just get its unicode representation.
return unicode(path)
@ -333,6 +352,7 @@ def displayable_path(path, separator=u'; '):
except (UnicodeError, LookupError):
return path.decode('utf8', 'ignore')
def syspath(path, prefix=True):
"""Convert a path for use by the operating system. In particular,
paths on Windows must receive a magic prefix and must be converted
@ -356,16 +376,22 @@ def syspath(path, prefix=True):
encoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
path = path.decode(encoding, 'replace')
# Add the magic prefix if it isn't already there
# Add the magic prefix if it isn't already there.
# http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247.aspx
if prefix and not path.startswith(WINDOWS_MAGIC_PREFIX):
if path.startswith(u'\\\\'):
# UNC path. Final path should look like \\?\UNC\...
path = u'UNC' + path[1:]
path = WINDOWS_MAGIC_PREFIX + path
return path
def samefile(p1, p2):
"""Safer equality for paths."""
return shutil._samefile(syspath(p1), syspath(p2))
def remove(path, soft=True):
"""Remove the file. If `soft`, then no error will be raised if the
file does not exist.
@ -378,6 +404,7 @@ def remove(path, soft=True):
except (OSError, IOError) as exc:
raise FilesystemError(exc, 'delete', (path,), traceback.format_exc())
def copy(path, dest, replace=False):
"""Copy a plain file. Permissions are not copied. If `dest` already
exists, raises a FilesystemError unless `replace` is True. Has no
@ -389,13 +416,14 @@ def copy(path, dest, replace=False):
path = syspath(path)
dest = syspath(dest)
if not replace and os.path.exists(dest):
raise FilesystemError('file exists', 'copy', (path, dest))
raise FilesystemError(u'file exists', 'copy', (path, dest))
try:
shutil.copyfile(path, dest)
except (OSError, IOError) as exc:
raise FilesystemError(exc, 'copy', (path, dest),
traceback.format_exc())
def move(path, dest, replace=False):
"""Rename a file. `dest` may not be a directory. If `dest` already
exists, raises an OSError unless `replace` is True. Has no effect if
@ -409,7 +437,7 @@ def move(path, dest, replace=False):
path = syspath(path)
dest = syspath(dest)
if os.path.exists(dest) and not replace:
raise FilesystemError('file exists', 'rename', (path, dest),
raise FilesystemError(u'file exists', 'rename', (path, dest),
traceback.format_exc())
# First, try renaming the file.
@ -424,6 +452,27 @@ def move(path, dest, replace=False):
raise FilesystemError(exc, 'move', (path, dest),
traceback.format_exc())
def link(path, dest, replace=False):
"""Create a symbolic link from path to `dest`. Raises an OSError if
`dest` already exists, unless `replace` is True. Does nothing if
`path` == `dest`."""
if (samefile(path, dest)):
return
path = syspath(path)
dest = syspath(dest)
if os.path.exists(dest) and not replace:
raise FilesystemError(u'file exists', 'rename', (path, dest),
traceback.format_exc())
try:
os.symlink(path, dest)
except OSError:
raise FilesystemError(u'Operating system does not support symbolic '
u'links.', 'link', (path, dest),
traceback.format_exc())
def unique_path(path):
"""Returns a version of ``path`` that does not exist on the
filesystem. Specifically, if ``path` itself already exists, then
@ -433,7 +482,7 @@ def unique_path(path):
return path
base, ext = os.path.splitext(path)
match = re.search(r'\.(\d)+$', base)
match = re.search(br'\.(\d)+$', base)
if match:
num = int(match.group(1))
base = base[:match.start()]
@ -441,7 +490,7 @@ def unique_path(path):
num = 0
while True:
num += 1
new_path = '%s.%i%s' % (base, num, ext)
new_path = b'%s.%i%s' % (base, num, ext)
if not os.path.exists(new_path):
return new_path
@ -450,13 +499,15 @@ def unique_path(path):
# shares, which are sufficiently common as to cause frequent problems.
# http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247.aspx
CHAR_REPLACE = [
(re.compile(ur'[\\/]'), u'_'), # / and \ -- forbidden everywhere.
(re.compile(ur'^\.'), u'_'), # Leading dot (hidden files on Unix).
(re.compile(ur'[\x00-\x1f]'), u''), # Control characters.
(re.compile(ur'[<>:"\?\*\|]'), u'_'), # Windows "reserved characters".
(re.compile(ur'\.$'), u'_'), # Trailing dots.
(re.compile(ur'\s+$'), u''), # Trailing whitespace.
(re.compile(r'[\\/]'), u'_'), # / and \ -- forbidden everywhere.
(re.compile(r'^\.'), u'_'), # Leading dot (hidden files on Unix).
(re.compile(r'[\x00-\x1f]'), u''), # Control characters.
(re.compile(r'[<>:"\?\*\|]'), u'_'), # Windows "reserved characters".
(re.compile(r'\.$'), u'_'), # Trailing dots.
(re.compile(r'\s+$'), u''), # Trailing whitespace.
]
def sanitize_path(path, replacements=None):
"""Takes a path (as a Unicode string) and makes sure that it is
legal. Returns a new path. Only works with fragments; won't work
@ -477,6 +528,7 @@ def sanitize_path(path, replacements=None):
comps[i] = comp
return os.path.join(*comps)
def truncate_path(path, length=MAX_FILENAME_LENGTH):
"""Given a bytestring path or a Unicode path fragment, truncate the
components to a legal length. In the last component, the extension
@ -493,12 +545,83 @@ def truncate_path(path, length=MAX_FILENAME_LENGTH):
return os.path.join(*out)
def _legalize_stage(path, replacements, length, extension, fragment):
"""Perform a single round of path legalization steps
(sanitation/replacement, encoding from Unicode to bytes,
extension-appending, and truncation). Return the path (Unicode if
`fragment` is set, `bytes` otherwise) and whether truncation was
required.
"""
# Perform an initial sanitization including user replacements.
path = sanitize_path(path, replacements)
# Encode for the filesystem.
if not fragment:
path = bytestring_path(path)
# Preserve extension.
path += extension.lower()
# Truncate too-long components.
pre_truncate_path = path
path = truncate_path(path, length)
return path, path != pre_truncate_path
def legalize_path(path, replacements, length, extension, fragment):
"""Given a path-like Unicode string, produce a legal path. Return
the path and a flag indicating whether some replacements had to be
ignored (see below).
The legalization process (see `_legalize_stage`) consists of
applying the sanitation rules in `replacements`, encoding the string
to bytes (unless `fragment` is set), truncating components to
`length`, appending the `extension`.
This function performs up to three calls to `_legalize_stage` in
case truncation conflicts with replacements (as can happen when
truncation creates whitespace at the end of the string, for
example). The limited number of iterations iterations avoids the
possibility of an infinite loop of sanitation and truncation
operations, which could be caused by replacement rules that make the
string longer. The flag returned from this function indicates that
the path has to be truncated twice (indicating that replacements
made the string longer again after it was truncated); the
application should probably log some sort of warning.
"""
if fragment:
# Outputting Unicode.
extension = extension.decode('utf8', 'ignore')
first_stage_path, _ = _legalize_stage(
path, replacements, length, extension, fragment
)
# Convert back to Unicode with extension removed.
first_stage_path, _ = os.path.splitext(displayable_path(first_stage_path))
# Re-sanitize following truncation (including user replacements).
second_stage_path, retruncated = _legalize_stage(
first_stage_path, replacements, length, extension, fragment
)
# If the path was once again truncated, discard user replacements
# and run through one last legalization stage.
if retruncated:
second_stage_path, _ = _legalize_stage(
first_stage_path, None, length, extension, fragment
)
return second_stage_path, retruncated
def str2bool(value):
"""Returns a boolean reflecting a human-entered string."""
if value.lower() in ('yes', '1', 'true', 't', 'y'):
return True
else:
return False
return value.lower() in (u'yes', u'1', u'true', u't', u'y')
def as_string(value):
"""Convert a value to a Unicode object for matching with a query.
@ -507,56 +630,23 @@ def as_string(value):
if value is None:
return u''
elif isinstance(value, buffer):
return str(value).decode('utf8', 'ignore')
elif isinstance(value, str):
return bytes(value).decode('utf8', 'ignore')
elif isinstance(value, bytes):
return value.decode('utf8', 'ignore')
else:
return unicode(value)
def levenshtein(s1, s2):
"""A nice DP edit distance implementation from Wikibooks:
http://en.wikibooks.org/wiki/Algorithm_implementation/Strings/
Levenshtein_distance#Python
"""
if len(s1) < len(s2):
return levenshtein(s2, s1)
if not s1:
return len(s2)
previous_row = xrange(len(s2) + 1)
for i, c1 in enumerate(s1):
current_row = [i + 1]
for j, c2 in enumerate(s2):
insertions = previous_row[j + 1] + 1
deletions = current_row[j] + 1
substitutions = previous_row[j] + (c1 != c2)
current_row.append(min(insertions, deletions, substitutions))
previous_row = current_row
return previous_row[-1]
def plurality(objs):
"""Given a sequence of comparable objects, returns the object that
is most common in the set and the frequency of that object. The
"""Given a sequence of hashble objects, returns the object that
is most common in the set and the its number of appearance. The
sequence must contain at least one object.
"""
# Calculate frequencies.
freqs = defaultdict(int)
for obj in objs:
freqs[obj] += 1
c = Counter(objs)
if not c:
raise ValueError(u'sequence must be non-empty')
return c.most_common(1)[0]
if not freqs:
raise ValueError('sequence must be non-empty')
# Find object with maximum frequency.
max_freq = 0
res = None
for obj, freq in freqs.items():
if freq > max_freq:
max_freq = freq
res = obj
return res, max_freq
def cpu_count():
"""Return the number of hardware thread contexts (cores or SMT
@ -571,8 +661,8 @@ def cpu_count():
num = 0
elif sys.platform == 'darwin':
try:
num = int(os.popen('sysctl -n hw.ncpu').read())
except ValueError:
num = int(command_output([b'/usr/sbin/sysctl', b'-n', b'hw.ncpu']))
except (ValueError, OSError, subprocess.CalledProcessError):
num = 0
else:
try:
@ -584,23 +674,38 @@ def cpu_count():
else:
return 1
def command_output(cmd):
"""Wraps the `subprocess` module to invoke a command (given as a
list of arguments starting with the command name) and collect
stdout. The stderr stream is ignored. May raise
`subprocess.CalledProcessError` or an `OSError`.
This replaces `subprocess.check_output`, which isn't available in
Python 2.6 and which can have problems if lots of output is sent to
stderr.
def command_output(cmd, shell=False):
"""Runs the command and returns its output after it has exited.
``cmd`` is a list of byte string arguments starting with the command names.
If ``shell`` is true, ``cmd`` is assumed to be a string and passed to a
shell to execute.
If the process exits with a non-zero return code
``subprocess.CalledProcessError`` is raised. May also raise
``OSError``.
This replaces `subprocess.check_output` which can have problems if lots of
output is sent to stderr.
"""
with open(os.devnull, 'w') as devnull:
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=devnull)
stdout, _ = proc.communicate()
proc = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=platform.system() != 'Windows',
shell=shell
)
stdout, stderr = proc.communicate()
if proc.returncode:
raise subprocess.CalledProcessError(proc.returncode, cmd)
raise subprocess.CalledProcessError(
returncode=proc.returncode,
cmd=b' '.join(cmd),
output=stdout + stderr,
)
return stdout
def max_filename_length(path, limit=MAX_FILENAME_LENGTH):
"""Attempt to determine the maximum filename length for the
filesystem containing `path`. If the value is greater than `limit`,
@ -616,3 +721,142 @@ def max_filename_length(path, limit=MAX_FILENAME_LENGTH):
return min(res[9], limit)
else:
return limit
def open_anything():
"""Return the system command that dispatches execution to the correct
program.
"""
sys_name = platform.system()
if sys_name == 'Darwin':
base_cmd = 'open'
elif sys_name == 'Windows':
base_cmd = 'start'
else: # Assume Unix
base_cmd = 'xdg-open'
return base_cmd
def editor_command():
"""Get a command for opening a text file.
Use the `EDITOR` environment variable by default. If it is not
present, fall back to `open_anything()`, the platform-specific tool
for opening files in general.
"""
editor = os.environ.get('EDITOR')
if editor:
return editor
return open_anything()
def shlex_split(s):
"""Split a Unicode or bytes string according to shell lexing rules.
Raise `ValueError` if the string is not a well-formed shell string.
This is a workaround for a bug in some versions of Python.
"""
if isinstance(s, bytes):
# Shlex works fine.
return shlex.split(s)
elif isinstance(s, unicode):
# Work around a Python bug.
# http://bugs.python.org/issue6988
bs = s.encode('utf8')
return [c.decode('utf8') for c in shlex.split(bs)]
else:
raise TypeError(u'shlex_split called with non-string')
def interactive_open(targets, command):
"""Open the files in `targets` by `exec`ing a new `command`, given
as a Unicode string. (The new program takes over, and Python
execution ends: this does not fork a subprocess.)
Can raise `OSError`.
"""
assert command
# Split the command string into its arguments.
try:
args = shlex_split(command)
except ValueError: # Malformed shell tokens.
args = [command]
args.insert(0, args[0]) # for argv[0]
args += targets
return os.execlp(*args)
def _windows_long_path_name(short_path):
"""Use Windows' `GetLongPathNameW` via ctypes to get the canonical,
long path given a short filename.
"""
if not isinstance(short_path, unicode):
short_path = unicode(short_path)
import ctypes
buf = ctypes.create_unicode_buffer(260)
get_long_path_name_w = ctypes.windll.kernel32.GetLongPathNameW
return_value = get_long_path_name_w(short_path, buf, 260)
if return_value == 0 or return_value > 260:
# An error occurred
return short_path
else:
long_path = buf.value
# GetLongPathNameW does not change the case of the drive
# letter.
if len(long_path) > 1 and long_path[1] == ':':
long_path = long_path[0].upper() + long_path[1:]
return long_path
def case_sensitive(path):
"""Check whether the filesystem at the given path is case sensitive.
To work best, the path should point to a file or a directory. If the path
does not exist, assume a case sensitive file system on every platform
except Windows.
"""
# A fallback in case the path does not exist.
if not os.path.exists(syspath(path)):
# By default, the case sensitivity depends on the platform.
return platform.system() != 'Windows'
# If an upper-case version of the path exists but a lower-case
# version does not, then the filesystem must be case-sensitive.
# (Otherwise, we have more work to do.)
if not (os.path.exists(syspath(path.lower())) and
os.path.exists(syspath(path.upper()))):
return True
# Both versions of the path exist on the file system. Check whether
# they refer to different files by their inodes. Alas,
# `os.path.samefile` is only available on Unix systems on Python 2.
if platform.system() != 'Windows':
return not os.path.samefile(syspath(path.lower()),
syspath(path.upper()))
# On Windows, we check whether the canonical, long filenames for the
# files are the same.
lower = _windows_long_path_name(path.lower())
upper = _windows_long_path_name(path.upper())
return lower != upper
def raw_seconds_short(string):
"""Formats a human-readable M:SS string as a float (number of seconds).
Raises ValueError if the conversion cannot take place due to `string` not
being in the right format.
"""
match = re.match(r'^(\d+):([0-5]\d)$', string)
if not match:
raise ValueError(u'String not in M:SS format')
minutes, seconds = map(int, match.groups())
return float(minutes * 60 + seconds)

174
libs/beets/util/artresizer.py
View file

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2013, Fabrice Laporte
# Copyright 2016, Fabrice Laporte
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
@ -15,11 +16,15 @@
"""Abstraction layer to resize images using PIL, ImageMagick, or a
public resizing proxy if neither is available.
"""
from __future__ import division, absolute_import, print_function
import urllib
import subprocess
import os
import re
from tempfile import NamedTemporaryFile
import logging
from beets import logging
from beets import util
# Resizing methods
@ -37,8 +42,8 @@ def resize_url(url, maxwidth):
maxwidth (preserving aspect ratio).
"""
return '{0}?{1}'.format(PROXY_URL, urllib.urlencode({
'url': url.replace('http://',''),
'w': str(maxwidth),
'url': url.replace('http://', ''),
'w': bytes(maxwidth),
}))
@ -57,9 +62,8 @@ def pil_resize(maxwidth, path_in, path_out=None):
"""
path_out = path_out or temp_file_for(path_in)
from PIL import Image
log.debug(u'artresizer: PIL resizing {0} to {1}'.format(
util.displayable_path(path_in), util.displayable_path(path_out)
))
log.debug(u'artresizer: PIL resizing {0} to {1}',
util.displayable_path(path_in), util.displayable_path(path_out))
try:
im = Image.open(util.syspath(path_in))
@ -68,20 +72,18 @@ def pil_resize(maxwidth, path_in, path_out=None):
im.save(path_out)
return path_out
except IOError:
log.error(u"PIL cannot create thumbnail for '{0}'".format(
util.displayable_path(path_in)
))
log.error(u"PIL cannot create thumbnail for '{0}'",
util.displayable_path(path_in))
return path_in
def im_resize(maxwidth, path_in, path_out=None):
"""Resize using ImageMagick's ``convert`` tool.
tool. Return the output path of resized image.
Return the output path of resized image.
"""
path_out = path_out or temp_file_for(path_in)
log.debug(u'artresizer: ImageMagick resizing {0} to {1}'.format(
util.displayable_path(path_in), util.displayable_path(path_out)
))
log.debug(u'artresizer: ImageMagick resizing {0} to {1}',
util.displayable_path(path_in), util.displayable_path(path_out))
# "-resize widthxheight>" shrinks images with dimension(s) larger
# than the corresponding width and/or height dimension(s). The >
@ -89,13 +91,13 @@ def im_resize(maxwidth, path_in, path_out=None):
# compatibility.
try:
util.command_output([
'convert', util.syspath(path_in),
'-resize', '{0}x^>'.format(maxwidth), path_out
b'convert', util.syspath(path_in, prefix=False),
b'-resize', b'{0}x^>'.format(maxwidth),
util.syspath(path_out, prefix=False),
])
except subprocess.CalledProcessError:
log.warn(u'artresizer: IM convert failed for {0}'.format(
util.displayable_path(path_in)
))
log.warn(u'artresizer: IM convert failed for {0}',
util.displayable_path(path_in))
return path_in
return path_out
@ -106,21 +108,56 @@ BACKEND_FUNCS = {
}
def pil_getsize(path_in):
from PIL import Image
try:
im = Image.open(util.syspath(path_in))
return im.size
except IOError as exc:
log.error(u"PIL could not read file {}: {}",
util.displayable_path(path_in), exc)
def im_getsize(path_in):
cmd = [b'identify', b'-format', b'%w %h',
util.syspath(path_in, prefix=False)]
try:
out = util.command_output(cmd)
except subprocess.CalledProcessError as exc:
log.warn(u'ImageMagick size query failed')
log.debug(
u'`convert` exited with (status {}) when '
u'getting size with command {}:\n{}',
exc.returncode, cmd, exc.output.strip()
)
return
try:
return tuple(map(int, out.split(b' ')))
except IndexError:
log.warn(u'Could not understand IM output: {0!r}', out)
BACKEND_GET_SIZE = {
PIL: pil_getsize,
IMAGEMAGICK: im_getsize,
}
class Shareable(type):
"""A pseudo-singleton metaclass that allows both shared and
non-shared instances. The ``MyClass.shared`` property holds a
lazily-created shared instance of ``MyClass`` while calling
``MyClass()`` to construct a new object works as usual.
"""
def __init__(cls, name, bases, dict):
super(Shareable, cls).__init__(name, bases, dict)
cls._instance = None
def __init__(self, name, bases, dict):
super(Shareable, self).__init__(name, bases, dict)
self._instance = None
@property
def shared(cls):
if cls._instance is None:
cls._instance = cls()
return cls._instance
def shared(self):
if self._instance is None:
self._instance = self()
return self._instance
class ArtResizer(object):
@ -128,12 +165,12 @@ class ArtResizer(object):
"""
__metaclass__ = Shareable
def __init__(self, method=None):
"""Create a resizer object for the given method or, if none is
specified, with an inferred method.
def __init__(self):
"""Create a resizer object with an inferred method.
"""
self.method = method or self._guess_method()
log.debug(u"artresizer: method is {0}".format(self.method))
self.method = self._check_method()
log.debug(u"artresizer: method is {0}", self.method)
self.can_compare = self._can_compare()
def resize(self, maxwidth, path_in, path_out=None):
"""Manipulate an image file according to the method, returning a
@ -141,7 +178,7 @@ class ArtResizer(object):
temporary file. For WEBPROXY, returns `path_in` unmodified.
"""
if self.local:
func = BACKEND_FUNCS[self.method]
func = BACKEND_FUNCS[self.method[0]]
return func(maxwidth, path_in, path_out)
else:
return path_in
@ -159,30 +196,63 @@ class ArtResizer(object):
@property
def local(self):
"""A boolean indicating whether the resizing method is performed
locally (i.e., PIL or IMAGEMAGICK).
locally (i.e., PIL or ImageMagick).
"""
return self.method in BACKEND_FUNCS
return self.method[0] in BACKEND_FUNCS
def get_size(self, path_in):
"""Return the size of an image file as an int couple (width, height)
in pixels.
Only available locally
"""
if self.local:
func = BACKEND_GET_SIZE[self.method[0]]
return func(path_in)
def _can_compare(self):
"""A boolean indicating whether image comparison is available"""
return self.method[0] == IMAGEMAGICK and self.method[1] > (6, 8, 7)
@staticmethod
def _guess_method():
"""Determine which resizing method to use. Returns PIL,
IMAGEMAGICK, or WEBPROXY depending on available dependencies.
"""
# Try importing PIL.
try:
__import__('PIL', fromlist=['Image'])
return PIL
except ImportError:
pass
def _check_method():
"""Return a tuple indicating an available method and its version."""
version = get_im_version()
if version:
return IMAGEMAGICK, version
# Try invoking ImageMagick's "convert".
version = get_pil_version()
if version:
return PIL, version
return WEBPROXY, (0)
def get_im_version():
"""Return Image Magick version or None if it is unavailable
Try invoking ImageMagick's "convert"."""
try:
out = util.command_output(['convert', '--version'])
out = util.command_output([b'identify', b'--version'])
if 'imagemagick' in out.lower():
# system32/convert.exe may be interfering
return IMAGEMAGICK
except (subprocess.CalledProcessError, OSError):
pass
pattern = r".+ (\d+)\.(\d+)\.(\d+).*"
match = re.search(pattern, out)
if match:
return (int(match.group(1)),
int(match.group(2)),
int(match.group(3)))
return (0,)
# Fall back to Web proxy method.
return WEBPROXY
except (subprocess.CalledProcessError, OSError):
return None
def get_pil_version():
"""Return Image Magick version or None if it is unavailable
Try importing PIL."""
try:
__import__('PIL', fromlist=[str('Image')])
return (0,)
except ImportError:
return None

40
libs/beets/util/bluelet.py
View file

@ -1,3 +1,5 @@
# -*- coding: utf-8 -*-
"""Extremely simple pure-Python implementation of coroutine-style
asynchronous socket I/O. Inspired by, but inferior to, Eventlet.
Bluelet can also be thought of as a less-terrible replacement for
@ -5,6 +7,8 @@ asyncore.
Bluelet: easy concurrency without all the messy parallelism.
"""
from __future__ import division, absolute_import, print_function
import socket
import select
import sys
@ -38,6 +42,7 @@ class Event(object):
"""
pass
class WaitableEvent(Event):
"""A waitable event is one encapsulating an action that can be
waited for using a select() call. That is, it's an event with an
@ -57,21 +62,25 @@ class WaitableEvent(Event):
"""
pass
class ValueEvent(Event):
"""An event that does nothing but return a fixed value."""
def __init__(self, value):
self.value = value
class ExceptionEvent(Event):
"""Raise an exception at the yield point. Used internally."""
def __init__(self, exc_info):
self.exc_info = exc_info
class SpawnEvent(Event):
"""Add a new coroutine thread to the scheduler."""
def __init__(self, coro):
self.spawned = coro
class JoinEvent(Event):
"""Suspend the thread until the specified child thread has
completed.
@ -79,11 +88,13 @@ class JoinEvent(Event):
def __init__(self, child):
self.child = child
class KillEvent(Event):
"""Unschedule a child thread."""
def __init__(self, child):
self.child = child
class DelegationEvent(Event):
"""Suspend execution of the current thread, start a new thread and,
once the child thread finished, return control to the parent
@ -92,6 +103,7 @@ class DelegationEvent(Event):
def __init__(self, coro):
self.spawned = coro
class ReturnEvent(Event):
"""Return a value the current thread's delegator at the point of
delegation. Ends the current (delegate) thread.
@ -99,6 +111,7 @@ class ReturnEvent(Event):
def __init__(self, value):
self.value = value
class SleepEvent(WaitableEvent):
"""Suspend the thread for a given duration.
"""
@ -108,6 +121,7 @@ class SleepEvent(WaitableEvent):
def time_left(self):
return max(self.wakeup_time - time.time(), 0.0)
class ReadEvent(WaitableEvent):
"""Reads from a file-like object."""
def __init__(self, fd, bufsize):
@ -120,6 +134,7 @@ class ReadEvent(WaitableEvent):
def fire(self):
return self.fd.read(self.bufsize)
class WriteEvent(WaitableEvent):
"""Writes to a file-like object."""
def __init__(self, fd, data):
@ -192,15 +207,19 @@ def _event_select(events):
return ready_events
class ThreadException(Exception):
def __init__(self, coro, exc_info):
self.coro = coro
self.exc_info = exc_info
def reraise(self):
_reraise(self.exc_info[0], self.exc_info[1], self.exc_info[2])
SUSPENDED = Event() # Special sentinel placeholder for suspended threads.
class Delegated(Event):
"""Placeholder indicating that a thread has delegated execution to a
different thread.
@ -208,6 +227,7 @@ class Delegated(Event):
def __init__(self, child):
self.child = child
def run(root_coro):
"""Schedules a coroutine, running it to completion. This
encapsulates the Bluelet scheduler, which the root coroutine can
@ -329,7 +349,7 @@ def run(root_coro):
break
# Wait and fire.
event2coro = dict((v,k) for k,v in threads.items())
event2coro = dict((v, k) for k, v in threads.items())
for event in _event_select(threads.values()):
# Run the IO operation, but catch socket errors.
try:
@ -378,6 +398,7 @@ def run(root_coro):
class SocketClosedError(Exception):
pass
class Listener(object):
"""A socket wrapper object for listening sockets.
"""
@ -407,6 +428,7 @@ class Listener(object):
self._closed = True
self.sock.close()
class Connection(object):
"""A socket wrapper object for connected sockets.
"""
@ -468,6 +490,7 @@ class Connection(object):
yield ReturnEvent(line)
break
class AcceptEvent(WaitableEvent):
"""An event for Listener objects (listening sockets) that suspends
execution until the socket gets a connection.
@ -482,6 +505,7 @@ class AcceptEvent(WaitableEvent):
sock, addr = self.listener.sock.accept()
return Connection(sock, addr)
class ReceiveEvent(WaitableEvent):
"""An event for Connection objects (connected sockets) for
asynchronously reading data.
@ -496,6 +520,7 @@ class ReceiveEvent(WaitableEvent):
def fire(self):
return self.conn.sock.recv(self.bufsize)
class SendEvent(WaitableEvent):
"""An event for Connection objects (connected sockets) for
asynchronously writing data.
@ -523,29 +548,33 @@ def null():
"""
return ValueEvent(None)
def spawn(coro):
"""Event: add another coroutine to the scheduler. Both the parent
and child coroutines run concurrently.
"""
if not isinstance(coro, types.GeneratorType):
raise ValueError('%s is not a coroutine' % str(coro))
raise ValueError(u'%s is not a coroutine' % coro)
return SpawnEvent(coro)
def call(coro):
"""Event: delegate to another coroutine. The current coroutine
is resumed once the sub-coroutine finishes. If the sub-coroutine
returns a value using end(), then this event returns that value.
"""
if not isinstance(coro, types.GeneratorType):
raise ValueError('%s is not a coroutine' % str(coro))
raise ValueError(u'%s is not a coroutine' % coro)
return DelegationEvent(coro)
def end(value=None):
"""Event: ends the coroutine and returns a value to its
delegator.
"""
return ReturnEvent(value)
def read(fd, bufsize=None):
"""Event: read from a file descriptor asynchronously."""
if bufsize is None:
@ -563,10 +592,12 @@ def read(fd, bufsize=None):
else:
return ReadEvent(fd, bufsize)
def write(fd, data):
"""Event: write to a file descriptor asynchronously."""
return WriteEvent(fd, data)
def connect(host, port):
"""Event: connect to a network address and return a Connection
object for communicating on the socket.
@ -575,17 +606,20 @@ def connect(host, port):
sock = socket.create_connection(addr)
return ValueEvent(Connection(sock, addr))
def sleep(duration):
"""Event: suspend the thread for ``duration`` seconds.
"""
return SleepEvent(duration)
def join(coro):
"""Suspend the thread until another, previously `spawn`ed thread
completes.
"""
return JoinEvent(coro)
def kill(coro):
"""Halt the execution of a different `spawn`ed thread.
"""

825
libs/beets/util/confit.py
View file

File diff suppressed because it is too large Load diff

177
libs/beets/util/enumeration.py
View file

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2013, Adrian Sampson.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
@ -12,167 +13,31 @@
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""A metaclass for enumerated types that really are types.
from __future__ import division, absolute_import, print_function
You can create enumerations with `enum(values, [name])` and they work
how you would expect them to.
from enum import Enum
>>> from enumeration import enum
>>> Direction = enum('north east south west', name='Direction')
>>> Direction.west
Direction.west
>>> Direction.west == Direction.west
True
>>> Direction.west == Direction.east
False
>>> isinstance(Direction.west, Direction)
True
>>> Direction[3]
Direction.west
>>> Direction['west']
Direction.west
>>> Direction.west.name
'west'
>>> Direction.north < Direction.west
True
Enumerations are classes; their instances represent the possible values
of the enumeration. Because Python classes must have names, you may
provide a `name` parameter to `enum`; if you don't, a meaningless one
will be chosen for you.
"""
import random
class Enumeration(type):
"""A metaclass whose classes are enumerations.
The `values` attribute of the class is used to populate the
enumeration. Values may either be a list of enumerated names or a
string containing a space-separated list of names. When the class
is created, it is instantiated for each name value in `values`.
Each such instance is the name of the enumerated item as the sole
argument.
The `Enumerated` class is a good choice for a superclass.
class OrderedEnum(Enum):
"""
def __init__(cls, name, bases, dic):
super(Enumeration, cls).__init__(name, bases, dic)
if 'values' not in dic:
# Do nothing if no values are provided (i.e., with
# Enumerated itself).
return
# May be called with a single string, in which case we split on
# whitespace for convenience.
values = dic['values']
if isinstance(values, basestring):
values = values.split()
# Create the Enumerated instances for each value. We have to use
# super's __setattr__ here because we disallow setattr below.
super(Enumeration, cls).__setattr__('_items_dict', {})
super(Enumeration, cls).__setattr__('_items_list', [])
for value in values:
item = cls(value, len(cls._items_list))
cls._items_dict[value] = item
cls._items_list.append(item)
def __getattr__(cls, key):
try:
return cls._items_dict[key]
except KeyError:
raise AttributeError("enumeration '" + cls.__name__ +
"' has no item '" + key + "'")
def __setattr__(cls, key, val):
raise TypeError("enumerations do not support attribute assignment")
def __getitem__(cls, key):
if isinstance(key, int):
return cls._items_list[key]
else:
return getattr(cls, key)
def __len__(cls):
return len(cls._items_list)
def __iter__(cls):
return iter(cls._items_list)
def __nonzero__(cls):
# Ensures that __len__ doesn't get called before __init__ by
# pydoc.
return True
class Enumerated(object):
"""An item in an enumeration.
Contains instance methods inherited by enumerated objects. The
metaclass is preset to `Enumeration` for your convenience.
Instance attributes:
name -- The name of the item.
index -- The index of the item in its enumeration.
>>> from enumeration import Enumerated
>>> class Garment(Enumerated):
... values = 'hat glove belt poncho lederhosen suspenders'
... def wear(self):
... print('now wearing a ' + self.name)
...
>>> Garment.poncho.wear()
now wearing a poncho
An Enum subclass that allows comparison of members.
"""
__metaclass__ = Enumeration
def __init__(self, name, index):
self.name = name
self.index = index
def __str__(self):
return type(self).__name__ + '.' + self.name
def __repr__(self):
return str(self)
def __cmp__(self, other):
if type(self) is type(other):
# Note that we're assuming that the items are direct
# instances of the same Enumeration (i.e., no fancy
# subclassing), which is probably okay.
return cmp(self.index, other.index)
else:
def __ge__(self, other):
if self.__class__ is other.__class__:
return self.value >= other.value
return NotImplemented
def enum(*values, **kwargs):
"""Shorthand for creating a new Enumeration class.
def __gt__(self, other):
if self.__class__ is other.__class__:
return self.value > other.value
return NotImplemented
Call with enumeration values as a list, a space-delimited string, or
just an argument list. To give the class a name, pass it as the
`name` keyword argument. Otherwise, a name will be chosen for you.
def __le__(self, other):
if self.__class__ is other.__class__:
return self.value <= other.value
return NotImplemented
The following are all equivalent:
enum('pinkie ring middle index thumb')
enum('pinkie', 'ring', 'middle', 'index', 'thumb')
enum(['pinkie', 'ring', 'middle', 'index', 'thumb'])
"""
if ('name' not in kwargs) or kwargs['name'] is None:
# Create a probably-unique name. It doesn't really have to be
# unique, but getting distinct names each time helps with
# identification in debugging.
name = 'Enumeration' + hex(random.randint(0,0xfffffff))[2:].upper()
else:
name = kwargs['name']
if len(values) == 1:
# If there's only one value, we have a couple of alternate calling
# styles.
if isinstance(values[0], basestring) or hasattr(values[0], '__iter__'):
values = values[0]
return type(name, (Enumerated,), {'values': values})
def __lt__(self, other):
if self.__class__ is other.__class__:
return self.value < other.value
return NotImplemented

42
libs/beets/util/functemplate.py
View file

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2013, Adrian Sampson.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
@ -25,13 +26,16 @@ library: unknown symbols are left intact.
This is sort of like a tiny, horrible degeneration of a real templating
engine like Jinja2 or Mustache.
"""
from __future__ import print_function
from __future__ import division, absolute_import, print_function
import re
import ast
import dis
import types
from .confit import NUMERIC_TYPES
SYMBOL_DELIM = u'$'
FUNC_DELIM = u'%'
GROUP_OPEN = u'{'
@ -42,6 +46,7 @@ ESCAPE_CHAR = u'$'
VARIABLE_PREFIX = '__var_'
FUNCTION_PREFIX = '__func_'
class Environment(object):
"""Contains the values and functions to be substituted into a
template.
@ -57,23 +62,26 @@ def ex_lvalue(name):
"""A variable load expression."""
return ast.Name(name, ast.Store())
def ex_rvalue(name):
"""A variable store expression."""
return ast.Name(name, ast.Load())
def ex_literal(val):
"""An int, float, long, bool, string, or None literal with the given
value.
"""
if val is None:
return ast.Name('None', ast.Load())
elif isinstance(val, (int, float, long)):
elif isinstance(val, NUMERIC_TYPES):
return ast.Num(val)
elif isinstance(val, bool):
return ast.Name(str(val), ast.Load())
return ast.Name(bytes(val), ast.Load())
elif isinstance(val, basestring):
return ast.Str(val)
raise TypeError('no literal for {0}'.format(type(val)))
raise TypeError(u'no literal for {0}'.format(type(val)))
def ex_varassign(name, expr):
"""Assign an expression into a single variable. The expression may
@ -83,6 +91,7 @@ def ex_varassign(name, expr):
expr = ex_literal(expr)
return ast.Assign([ex_lvalue(name)], expr)
def ex_call(func, args):
"""A function-call expression with only positional parameters. The
function may be an expression or the name of a function. Each
@ -98,13 +107,14 @@ def ex_call(func, args):
return ast.Call(func, args, [], None, None)
def compile_func(arg_names, statements, name='_the_func', debug=False):
"""Compile a list of statements as the body of a function and return
the resulting Python function. If `debug`, then print out the
bytecode of the compiled function.
"""
func_def = ast.FunctionDef(
name,
name.encode('utf8'),
ast.arguments(
[ast.Name(n, ast.Param()) for n in arg_names],
None, None,
@ -126,7 +136,7 @@ def compile_func(arg_names, statements, name='_the_func', debug=False):
dis.dis(const)
the_locals = {}
exec prog in {}, the_locals
exec(prog, {}, the_locals)
return the_locals[name]
@ -157,6 +167,7 @@ class Symbol(object):
expr = ex_rvalue(VARIABLE_PREFIX + self.ident.encode('utf8'))
return [expr], set([self.ident.encode('utf8')]), set()
class Call(object):
"""A function call in a template."""
def __init__(self, ident, args, original):
@ -214,6 +225,7 @@ class Call(object):
)
return [subexpr_call], varnames, funcnames
class Expression(object):
"""Top-level template construct: contains a list of text blobs,
Symbols, and Calls.
@ -259,6 +271,7 @@ class Expression(object):
class ParseError(Exception):
pass
class Parser(object):
"""Parses a template expression string. Instantiate the class with
the template source and call ``parse_expression``. The ``pos`` field
@ -280,7 +293,7 @@ class Parser(object):
# Common parsing resources.
special_chars = (SYMBOL_DELIM, FUNC_DELIM, GROUP_OPEN, GROUP_CLOSE,
ARG_SEP, ESCAPE_CHAR)
special_char_re = re.compile(ur'[%s]|$' %
special_char_re = re.compile(r'[%s]|$' %
u''.join(re.escape(c) for c in special_chars))
def parse_expression(self):
@ -298,8 +311,8 @@ class Parser(object):
# A non-special character. Skip to the next special
# character, treating the interstice as literal text.
next_pos = (
self.special_char_re.search(self.string[self.pos:]).start()
+ self.pos
self.special_char_re.search(
self.string[self.pos:]).start() + self.pos
)
text_parts.append(self.string[self.pos:next_pos])
self.pos = next_pos
@ -468,10 +481,11 @@ class Parser(object):
Updates ``pos``.
"""
remainder = self.string[self.pos:]
ident = re.match(ur'\w*', remainder).group(0)
ident = re.match(r'\w*', remainder).group(0)
self.pos += len(ident)
return ident
def _parse(template):
"""Parse a top-level template string Expression. Any extraneous text
is considered literal text.
@ -522,9 +536,9 @@ class Template(object):
argnames = []
for varname in varnames:
argnames.append(VARIABLE_PREFIX.encode('utf8') + varname)
argnames.append(VARIABLE_PREFIX + varname)
for funcname in funcnames:
argnames.append(FUNCTION_PREFIX.encode('utf8') + funcname)
argnames.append(FUNCTION_PREFIX + funcname)
func = compile_func(
argnames,
@ -558,4 +572,4 @@ if __name__ == '__main__':
'from __main__ import _tmpl, _vars, _funcs',
number=10000)
print(comp_time)
print('Speedup:', interp_time / comp_time)
print(u'Speedup:', interp_time / comp_time)

88
libs/beets/util/hidden.py Normal file
View file

@ -0,0 +1,88 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Simple library to work out if a file is hidden on different platforms."""
from __future__ import division, absolute_import, print_function
import os
import stat
import ctypes
import sys
def _is_hidden_osx(path):
"""Return whether or not a file is hidden on OS X.
This uses os.lstat to work out if a file has the "hidden" flag.
"""
file_stat = os.lstat(path)
if hasattr(file_stat, 'st_flags') and hasattr(stat, 'UF_HIDDEN'):
return bool(file_stat.st_flags & stat.UF_HIDDEN)
else:
return False
def _is_hidden_win(path):
"""Return whether or not a file is hidden on Windows.
This uses GetFileAttributes to work out if a file has the "hidden" flag
(FILE_ATTRIBUTE_HIDDEN).
"""
# FILE_ATTRIBUTE_HIDDEN = 2 (0x2) from GetFileAttributes documentation.
hidden_mask = 2
# Retrieve the attributes for the file.
attrs = ctypes.windll.kernel32.GetFileAttributesW(path)
# Ensure we have valid attribues and compare them against the mask.
return attrs >= 0 and attrs & hidden_mask
def _is_hidden_dot(path):
"""Return whether or not a file starts with a dot.
Files starting with a dot are seen as "hidden" files on Unix-based OSes.
"""
return os.path.basename(path).startswith('.')
def is_hidden(path):
"""Return whether or not a file is hidden.
This method works differently depending on the platform it is called on.
On OS X, it uses both the result of `is_hidden_osx` and `is_hidden_dot` to
work out if a file is hidden.
On Windows, it uses the result of `is_hidden_win` to work out if a file is
hidden.
On any other operating systems (i.e. Linux), it uses `is_hidden_dot` to
work out if a file is hidden.
"""
# Convert the path to unicode if it is not already.
if not isinstance(path, unicode):
path = path.decode('utf-8')
# Run platform specific functions depending on the platform
if sys.platform == 'darwin':
return _is_hidden_osx(path) or _is_hidden_dot(path)
elif sys.platform == 'win32':
return _is_hidden_win(path)
else:
return _is_hidden_dot(path)
__all__ = ['is_hidden']

111
libs/beets/util/pipeline.py
View file

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2013, Adrian Sampson.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
@ -30,18 +31,19 @@ up a bottleneck stage by dividing its work among multiple threads.
To do so, pass an iterable of coroutines to the Pipeline constructor
in place of any single coroutine.
"""
from __future__ import print_function
from __future__ import division, absolute_import, print_function
import Queue
from threading import Thread, Lock
import sys
import types
BUBBLE = '__PIPELINE_BUBBLE__'
POISON = '__PIPELINE_POISON__'
DEFAULT_QUEUE_SIZE = 16
def _invalidate_queue(q, val=None, sync=True):
"""Breaks a Queue such that it never blocks, always has size 1,
and has no maximum size. get()ing from the queue returns `val`,
@ -50,8 +52,10 @@ def _invalidate_queue(q, val=None, sync=True):
"""
def _qsize(len=len):
return 1
def _put(item):
pass
def _get():
return val
@ -70,6 +74,7 @@ def _invalidate_queue(q, val=None, sync=True):
if sync:
q.mutex.release()
class CountedQueue(Queue.Queue):
"""A queue that keeps track of the number of threads that are
still feeding into it. The queue is poisoned when all threads are
@ -104,6 +109,7 @@ class CountedQueue(Queue.Queue):
# Replacement _get invalidates when no items remain.
_old_get = self._get
def _get():
out = _old_get()
if not self.queue:
@ -117,18 +123,67 @@ class CountedQueue(Queue.Queue):
# No items. Invalidate immediately.
_invalidate_queue(self, POISON, False)
class MultiMessage(object):
"""A message yielded by a pipeline stage encapsulating multiple
values to be sent to the next stage.
"""
def __init__(self, messages):
self.messages = messages
def multiple(messages):
"""Yield multiple([message, ..]) from a pipeline stage to send
multiple values to the next pipeline stage.
"""
return MultiMessage(messages)
def stage(func):
"""Decorate a function to become a simple stage.
>>> @stage
... def add(n, i):
... return i + n
>>> pipe = Pipeline([
... iter([1, 2, 3]),
... add(2),
... ])
>>> list(pipe.pull())
[3, 4, 5]
"""
def coro(*args):
task = None
while True:
task = yield task
task = func(*(args + (task,)))
return coro
def mutator_stage(func):
"""Decorate a function that manipulates items in a coroutine to
become a simple stage.
>>> @mutator_stage
... def setkey(key, item):
... item[key] = True
>>> pipe = Pipeline([
... iter([{'x': False}, {'a': False}]),
... setkey('x'),
... ])
>>> list(pipe.pull())
[{'x': True}, {'a': False, 'x': True}]
"""
def coro(*args):
task = None
while True:
task = yield task
func(*(args + (task,)))
return coro
def _allmsgs(obj):
"""Returns a list of all the messages encapsulated in obj. If obj
is a MultiMessage, returns its enclosed messages. If obj is BUBBLE,
@ -141,6 +196,7 @@ def _allmsgs(obj):
else:
return [obj]
class PipelineThread(Thread):
"""Abstract base class for pipeline-stage threads."""
def __init__(self, all_threads):
@ -169,6 +225,7 @@ class PipelineThread(Thread):
for thread in self.all_threads:
thread.abort()
class FirstPipelineThread(PipelineThread):
"""The thread running the first stage in a parallel pipeline setup.
The coroutine should just be a generator.
@ -191,7 +248,7 @@ class FirstPipelineThread(PipelineThread):
# Get the value from the generator.
try:
msg = self.coro.next()
msg = next(self.coro)
except StopIteration:
break
@ -209,6 +266,7 @@ class FirstPipelineThread(PipelineThread):
# Generator finished; shut down the pipeline.
self.out_queue.release()
class MiddlePipelineThread(PipelineThread):
"""A thread running any stage in the pipeline except the first or
last.
@ -223,7 +281,7 @@ class MiddlePipelineThread(PipelineThread):
def run(self):
try:
# Prime the coroutine.
self.coro.next()
next(self.coro)
while True:
with self.abort_lock:
@ -256,6 +314,7 @@ class MiddlePipelineThread(PipelineThread):
# Pipeline is shutting down normally.
self.out_queue.release()
class LastPipelineThread(PipelineThread):
"""A thread running the last stage in a pipeline. The coroutine
should yield nothing.
@ -267,7 +326,7 @@ class LastPipelineThread(PipelineThread):
def run(self):
# Prime the coroutine.
self.coro.next()
next(self.coro)
try:
while True:
@ -291,6 +350,7 @@ class LastPipelineThread(PipelineThread):
self.abort_all(sys.exc_info())
return
class Pipeline(object):
"""Represents a staged pattern of work. Each stage in the pipeline
is a coroutine that receives messages from the previous stage and
@ -301,7 +361,7 @@ class Pipeline(object):
be at least two stages.
"""
if len(stages) < 2:
raise ValueError('pipeline must have at least two stages')
raise ValueError(u'pipeline must have at least two stages')
self.stages = []
for stage in stages:
if isinstance(stage, (list, tuple)):
@ -322,7 +382,8 @@ class Pipeline(object):
messages between the stages are stored in queues of the given
size.
"""
queues = [CountedQueue(queue_size) for i in range(len(self.stages)-1)]
queue_count = len(self.stages) - 1
queues = [CountedQueue(queue_size) for i in range(queue_count)]
threads = []
# Set up first stage.
@ -330,10 +391,10 @@ class Pipeline(object):
threads.append(FirstPipelineThread(coro, queues[0], threads))
# Middle stages.
for i in range(1, len(self.stages)-1):
for i in range(1, queue_count):
for coro in self.stages[i]:
threads.append(MiddlePipelineThread(
coro, queues[i-1], queues[i], threads
coro, queues[i - 1], queues[i], threads
))
# Last stage.
@ -383,7 +444,7 @@ class Pipeline(object):
# "Prime" the coroutines.
for coro in coros[1:]:
coro.next()
next(coro)
# Begin the pipeline.
for out in coros[0]:
@ -405,20 +466,23 @@ if __name__ == '__main__':
# in parallel.
def produce():
for i in range(5):
print('generating %i' % i)
print(u'generating %i' % i)
time.sleep(1)
yield i
def work():
num = yield
while True:
print('processing %i' % num)
print(u'processing %i' % num)
time.sleep(2)
num = yield num*2
num = yield num * 2
def consume():
while True:
num = yield
time.sleep(1)
print('received %i' % num)
print(u'received %i' % num)
ts_start = time.time()
Pipeline([produce(), work(), consume()]).run_sequential()
ts_seq = time.time()
@ -426,29 +490,30 @@ if __name__ == '__main__':
ts_par = time.time()
Pipeline([produce(), (work(), work()), consume()]).run_parallel()
ts_end = time.time()
print('Sequential time:', ts_seq - ts_start)
print('Parallel time:', ts_par - ts_seq)
print('Multiply-parallel time:', ts_end - ts_par)
print(u'Sequential time:', ts_seq - ts_start)
print(u'Parallel time:', ts_par - ts_seq)
print(u'Multiply-parallel time:', ts_end - ts_par)
print()
# Test a pipeline that raises an exception.
def exc_produce():
for i in range(10):
print('generating %i' % i)
print(u'generating %i' % i)
time.sleep(1)
yield i
def exc_work():
num = yield
while True:
print('processing %i' % num)
print(u'processing %i' % num)
time.sleep(3)
if num == 3:
raise Exception()
num = yield num * 2
def exc_consume():
while True:
num = yield
#if num == 4:
# raise Exception()
print('received %i' % num)
print(u'received %i' % num)
Pipeline([exc_produce(), exc_work(), exc_consume()]).run_parallel(1)

7
libs/beets/vfs.py
View file

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2013, Adrian Sampson.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
@ -15,11 +16,14 @@
"""A simple utility for constructing filesystem-like trees from beets
libraries.
"""
from __future__ import division, absolute_import, print_function
from collections import namedtuple
from beets import util
Node = namedtuple('Node', ['files', 'dirs'])
def _insert(node, path, itemid):
"""Insert an item into a virtual filesystem node."""
if len(path) == 1:
@ -33,6 +37,7 @@ def _insert(node, path, itemid):
node.dirs[dirname] = Node({}, {})
_insert(node.dirs[dirname], rest, itemid)
def libtree(lib):
"""Generates a filesystem-like directory tree for the files
contained in `lib`. Filesystem nodes are (files, dirs) named

22
libs/beetsplug/__init__.py Normal file
View file

@ -0,0 +1,22 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""A namespace package for beets plugins."""
from __future__ import division, absolute_import, print_function
# Make this a namespace package.
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)

165
libs/beetsplug/acousticbrainz.py Normal file
View file

@ -0,0 +1,165 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2015-2016, Ohm Patel.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Fetch various AcousticBrainz metadata using MBID.
"""
from __future__ import division, absolute_import, print_function
import requests
import operator
from beets import plugins, ui
from functools import reduce
ACOUSTIC_BASE = "https://acousticbrainz.org/"
LEVELS = ["/low-level", "/high-level"]
class AcousticPlugin(plugins.BeetsPlugin):
def __init__(self):
super(AcousticPlugin, self).__init__()
self.config.add({'auto': True})
if self.config['auto']:
self.register_listener('import_task_files',
self.import_task_files)
def commands(self):
cmd = ui.Subcommand('acousticbrainz',
help=u"fetch metadata from AcousticBrainz")
def func(lib, opts, args):
items = lib.items(ui.decargs(args))
fetch_info(self._log, items, ui.should_write())
cmd.func = func
return [cmd]
def import_task_files(self, session, task):
"""Function is called upon beet import.
"""
items = task.imported_items()
fetch_info(self._log, items, False)
def fetch_info(log, items, write):
"""Get data from AcousticBrainz for the items.
"""
def get_value(*map_path):
try:
return reduce(operator.getitem, map_path, data)
except KeyError:
log.debug(u'Invalid Path: {}', map_path)
for item in items:
if item.mb_trackid:
log.info(u'getting data for: {}', item)
# Fetch the data from the AB API.
urls = [generate_url(item.mb_trackid, path) for path in LEVELS]
log.debug(u'fetching URLs: {}', ' '.join(urls))
try:
res = [requests.get(url) for url in urls]
except requests.RequestException as exc:
log.info(u'request error: {}', exc)
continue
# Check for missing tracks.
if any(r.status_code == 404 for r in res):
log.info(u'recording ID {} not found', item.mb_trackid)
continue
# Parse the JSON response.
try:
data = res[0].json()
data.update(res[1].json())
except ValueError:
log.debug(u'Invalid Response: {} & {}', [r.text for r in res])
# Get each field and assign it on the item.
item.danceable = get_value(
"highlevel", "danceability", "all", "danceable",
)
item.gender = get_value(
"highlevel", "gender", "value",
)
item.genre_rosamerica = get_value(
"highlevel", "genre_rosamerica", "value"
)
item.mood_acoustic = get_value(
"highlevel", "mood_acoustic", "all", "acoustic"
)
item.mood_aggressive = get_value(
"highlevel", "mood_aggressive", "all", "aggressive"
)
item.mood_electronic = get_value(
"highlevel", "mood_electronic", "all", "electronic"
)
item.mood_happy = get_value(
"highlevel", "mood_happy", "all", "happy"
)
item.mood_party = get_value(
"highlevel", "mood_party", "all", "party"
)
item.mood_relaxed = get_value(
"highlevel", "mood_relaxed", "all", "relaxed"
)
item.mood_sad = get_value(
"highlevel", "mood_sad", "all", "sad"
)
item.rhythm = get_value(
"highlevel", "ismir04_rhythm", "value"
)
item.tonal = get_value(
"highlevel", "tonal_atonal", "all", "tonal"
)
item.voice_instrumental = get_value(
"highlevel", "voice_instrumental", "value"
)
item.average_loudness = get_value(
"lowlevel", "average_loudness"
)
item.chords_changes_rate = get_value(
"tonal", "chords_changes_rate"
)
item.chords_key = get_value(
"tonal", "chords_key"
)
item.chords_number_rate = get_value(
"tonal", "chords_number_rate"
)
item.chords_scale = get_value(
"tonal", "chords_scale"
)
item.initial_key = '{} {}'.format(
get_value("tonal", "key_key"),
get_value("tonal", "key_scale")
)
item.key_strength = get_value(
"tonal", "key_strength"
)
# Store the data.
item.store()
if write:
item.try_write()
def generate_url(mbid, level):
"""Generates AcousticBrainz end point url for given MBID.
"""
return ACOUSTIC_BASE + mbid + level

120
libs/beetsplug/badfiles.py Normal file
View file

@ -0,0 +1,120 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016, François-Xavier Thomas.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Use command-line tools to check for audio file corruption.
"""
from __future__ import division, absolute_import, print_function
from beets.plugins import BeetsPlugin
from beets.ui import Subcommand
from beets.util import displayable_path, confit
from beets import ui
from subprocess import check_output, CalledProcessError, list2cmdline, STDOUT
import shlex
import os
import errno
import sys
class BadFiles(BeetsPlugin):
def run_command(self, cmd):
self._log.debug(u"running command: {}",
displayable_path(list2cmdline(cmd)))
try:
output = check_output(cmd, stderr=STDOUT)
errors = 0
status = 0
except CalledProcessError as e:
output = e.output
errors = 1
status = e.returncode
except OSError as e:
if e.errno == errno.ENOENT:
ui.print_(u"command not found: {}".format(cmd[0]))
sys.exit(1)
else:
raise
output = output.decode(sys.getfilesystemencoding())
return status, errors, [line for line in output.split("\n") if line]
def check_mp3val(self, path):
status, errors, output = self.run_command(["mp3val", path])
if status == 0:
output = [line for line in output if line.startswith("WARNING:")]
errors = len(output)
return status, errors, output
def check_flac(self, path):
return self.run_command(["flac", "-wst", path])
def check_custom(self, command):
def checker(path):
cmd = shlex.split(command)
cmd.append(path)
return self.run_command(cmd)
return checker
def get_checker(self, ext):
ext = ext.lower()
try:
command = self.config['commands'].get(dict).get(ext)
except confit.NotFoundError:
command = None
if command:
return self.check_custom(command)
elif ext == "mp3":
return self.check_mp3val
elif ext == "flac":
return self.check_flac
def check_bad(self, lib, opts, args):
for item in lib.items(ui.decargs(args)):
# First, check whether the path exists. If not, the user
# should probably run `beet update` to cleanup your library.
dpath = displayable_path(item.path)
self._log.debug(u"checking path: {}", dpath)
if not os.path.exists(item.path):
ui.print_(u"{}: file does not exist".format(
ui.colorize('text_error', dpath)))
# Run the checker against the file if one is found
ext = os.path.splitext(item.path)[1][1:]
checker = self.get_checker(ext)
if not checker:
continue
path = item.path
if not isinstance(path, unicode):
path = item.path.decode(sys.getfilesystemencoding())
status, errors, output = checker(path)
if status > 0:
ui.print_(u"{}: checker exited withs status {}"
.format(ui.colorize('text_error', dpath), status))
for line in output:
ui.print_(" {}".format(displayable_path(line)))
elif errors > 0:
ui.print_(u"{}: checker found {} errors or warnings"
.format(ui.colorize('text_warning', dpath), errors))
for line in output:
ui.print_(u" {}".format(displayable_path(line)))
else:
ui.print_(u"{}: ok".format(ui.colorize('text_success', dpath)))
def commands(self):
bad_command = Subcommand('bad',
help=u'check for corrupt or missing files')
bad_command.func = self.check_bad
return [bad_command]

109
libs/beetsplug/bench.py Normal file
View file

@ -0,0 +1,109 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Some simple performance benchmarks for beets.
"""
from __future__ import division, absolute_import, print_function
from beets.plugins import BeetsPlugin
from beets import ui
from beets import vfs
from beets import library
from beets.util.functemplate import Template
from beets.autotag import match
from beets import plugins
from beets import importer
import cProfile
import timeit
def aunique_benchmark(lib, prof):
def _build_tree():
vfs.libtree(lib)
# Measure path generation performance with %aunique{} included.
lib.path_formats = [
(library.PF_KEY_DEFAULT,
Template('$albumartist/$album%aunique{}/$track $title')),
]
if prof:
cProfile.runctx('_build_tree()', {}, {'_build_tree': _build_tree},
'paths.withaunique.prof')
else:
interval = timeit.timeit(_build_tree, number=1)
print('With %aunique:', interval)
# And with %aunique replaceed with a "cheap" no-op function.
lib.path_formats = [
(library.PF_KEY_DEFAULT,
Template('$albumartist/$album%lower{}/$track $title')),
]
if prof:
cProfile.runctx('_build_tree()', {}, {'_build_tree': _build_tree},
'paths.withoutaunique.prof')
else:
interval = timeit.timeit(_build_tree, number=1)
print('Without %aunique:', interval)
def match_benchmark(lib, prof, query=None, album_id=None):
# If no album ID is provided, we'll match against a suitably huge
# album.
if not album_id:
album_id = '9c5c043e-bc69-4edb-81a4-1aaf9c81e6dc'
# Get an album from the library to use as the source for the match.
items = lib.albums(query).get().items()
# Ensure fingerprinting is invoked (if enabled).
plugins.send('import_task_start',
task=importer.ImportTask(None, None, items),
session=importer.ImportSession(lib, None, None, None))
# Run the match.
def _run_match():
match.tag_album(items, search_ids=[album_id])
if prof:
cProfile.runctx('_run_match()', {}, {'_run_match': _run_match},
'match.prof')
else:
interval = timeit.timeit(_run_match, number=1)
print('match duration:', interval)
class BenchmarkPlugin(BeetsPlugin):
"""A plugin for performing some simple performance benchmarks.
"""
def commands(self):
aunique_bench_cmd = ui.Subcommand('bench_aunique',
help='benchmark for %aunique{}')
aunique_bench_cmd.parser.add_option('-p', '--profile',
action='store_true', default=False,
help='performance profiling')
aunique_bench_cmd.func = lambda lib, opts, args: \
aunique_benchmark(lib, opts.profile)
match_bench_cmd = ui.Subcommand('bench_match',
help='benchmark for track matching')
match_bench_cmd.parser.add_option('-p', '--profile',
action='store_true', default=False,
help='performance profiling')
match_bench_cmd.parser.add_option('-i', '--id', default=None,
help='album ID to match against')
match_bench_cmd.func = lambda lib, opts, args: \
match_benchmark(lib, opts.profile, ui.decargs(args), opts.id)
return [aunique_bench_cmd, match_bench_cmd]

1193
libs/beetsplug/bpd/__init__.py Normal file
View file

File diff suppressed because it is too large Load diff

223
libs/beetsplug/bpd/gstplayer.py Normal file
View file

@ -0,0 +1,223 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""A wrapper for the GStreamer Python bindings that exposes a simple
music player.
"""
from __future__ import division, absolute_import, print_function
import sys
import time
import gobject
import thread
import os
import copy
import urllib
import pygst
pygst.require('0.10')
import gst # noqa
class GstPlayer(object):
"""A music player abstracting GStreamer's Playbin element.
Create a player object, then call run() to start a thread with a
runloop. Then call play_file to play music. Use player.playing
to check whether music is currently playing.
A basic play queue is also implemented (just a Python list,
player.queue, whose last element is next to play). To use it,
just call enqueue() and then play(). When a track finishes and
another is available on the queue, it is played automatically.
"""
def __init__(self, finished_callback=None):
"""Initialize a player.
If a finished_callback is provided, it is called every time a
track started with play_file finishes.
Once the player has been created, call run() to begin the main
runloop in a separate thread.
"""
# Set up the Gstreamer player. From the pygst tutorial:
# http://pygstdocs.berlios.de/pygst-tutorial/playbin.html
self.player = gst.element_factory_make("playbin2", "player")
fakesink = gst.element_factory_make("fakesink", "fakesink")
self.player.set_property("video-sink", fakesink)
bus = self.player.get_bus()
bus.add_signal_watch()
bus.connect("message", self._handle_message)
# Set up our own stuff.
self.playing = False
self.finished_callback = finished_callback
self.cached_time = None
self._volume = 1.0
def _get_state(self):
"""Returns the current state flag of the playbin."""
# gst's get_state function returns a 3-tuple; we just want the
# status flag in position 1.
return self.player.get_state()[1]
def _handle_message(self, bus, message):
"""Callback for status updates from GStreamer."""
if message.type == gst.MESSAGE_EOS:
# file finished playing
self.player.set_state(gst.STATE_NULL)
self.playing = False
self.cached_time = None
if self.finished_callback:
self.finished_callback()
elif message.type == gst.MESSAGE_ERROR:
# error
self.player.set_state(gst.STATE_NULL)
err, debug = message.parse_error()
print(u"Error: {0}".format(err))
self.playing = False
def _set_volume(self, volume):
"""Set the volume level to a value in the range [0, 1.5]."""
# And the volume for the playbin.
self._volume = volume
self.player.set_property("volume", volume)
def _get_volume(self):
"""Get the volume as a float in the range [0, 1.5]."""
return self._volume
volume = property(_get_volume, _set_volume)
def play_file(self, path):
"""Immediately begin playing the audio file at the given
path.
"""
self.player.set_state(gst.STATE_NULL)
if isinstance(path, unicode):
path = path.encode('utf8')
uri = 'file://' + urllib.quote(path)
self.player.set_property("uri", uri)
self.player.set_state(gst.STATE_PLAYING)
self.playing = True
def play(self):
"""If paused, resume playback."""
if self._get_state() == gst.STATE_PAUSED:
self.player.set_state(gst.STATE_PLAYING)
self.playing = True
def pause(self):
"""Pause playback."""
self.player.set_state(gst.STATE_PAUSED)
def stop(self):
"""Halt playback."""
self.player.set_state(gst.STATE_NULL)
self.playing = False
self.cached_time = None
def run(self):
"""Start a new thread for the player.
Call this function before trying to play any music with
play_file() or play().
"""
# If we don't use the MainLoop, messages are never sent.
gobject.threads_init()
def start():
loop = gobject.MainLoop()
loop.run()
thread.start_new_thread(start, ())
def time(self):
"""Returns a tuple containing (position, length) where both
values are integers in seconds. If no stream is available,
returns (0, 0).
"""
fmt = gst.Format(gst.FORMAT_TIME)
try:
pos = self.player.query_position(fmt, None)[0] / (10 ** 9)
length = self.player.query_duration(fmt, None)[0] / (10 ** 9)
self.cached_time = (pos, length)
return (pos, length)
except gst.QueryError:
# Stream not ready. For small gaps of time, for instance
# after seeking, the time values are unavailable. For this
# reason, we cache recent.
if self.playing and self.cached_time:
return self.cached_time
else:
return (0, 0)
def seek(self, position):
"""Seeks to position (in seconds)."""
cur_pos, cur_len = self.time()
if position > cur_len:
self.stop()
return
fmt = gst.Format(gst.FORMAT_TIME)
ns = position * 10 ** 9 # convert to nanoseconds
self.player.seek_simple(fmt, gst.SEEK_FLAG_FLUSH, ns)
# save new cached time
self.cached_time = (position, cur_len)
def block(self):
"""Block until playing finishes."""
while self.playing:
time.sleep(1)
def play_simple(paths):
"""Play the files in paths in a straightforward way, without
using the player's callback function.
"""
p = GstPlayer()
p.run()
for path in paths:
p.play_file(path)
p.block()
def play_complicated(paths):
"""Play the files in the path one after the other by using the
callback function to advance to the next song.
"""
my_paths = copy.copy(paths)
def next_song():
my_paths.pop(0)
p.play_file(my_paths[0])
p = GstPlayer(next_song)
p.run()
p.play_file(my_paths[0])
while my_paths:
time.sleep(1)
if __name__ == '__main__':
# A very simple command-line player. Just give it names of audio
# files on the command line; these are all played in sequence.
paths = [os.path.abspath(os.path.expanduser(p))
for p in sys.argv[1:]]
# play_simple(paths)
play_complicated(paths)

87
libs/beetsplug/bpm.py Normal file
View file

@ -0,0 +1,87 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016, aroquen
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Determine BPM by pressing a key to the rhythm."""
from __future__ import division, absolute_import, print_function
import time
from beets import ui
from beets.plugins import BeetsPlugin
def bpm(max_strokes):
"""Returns average BPM (possibly of a playing song)
listening to Enter keystrokes.
"""
t0 = None
dt = []
for i in range(max_strokes):
# Press enter to the rhythm...
s = raw_input()
if s == '':
t1 = time.time()
# Only start measuring at the second stroke
if t0:
dt.append(t1 - t0)
t0 = t1
else:
break
# Return average BPM
# bpm = (max_strokes-1) / sum(dt) * 60
ave = sum([1.0 / dti * 60 for dti in dt]) / len(dt)
return ave
class BPMPlugin(BeetsPlugin):
def __init__(self):
super(BPMPlugin, self).__init__()
self.config.add({
u'max_strokes': 3,
u'overwrite': True,
})
def commands(self):
cmd = ui.Subcommand('bpm',
help=u'determine bpm of a song by pressing '
u'a key to the rhythm')
cmd.func = self.command
return [cmd]
def command(self, lib, opts, args):
self.get_bpm(lib.items(ui.decargs(args)))
def get_bpm(self, items, write=False):
overwrite = self.config['overwrite'].get(bool)
if len(items) > 1:
raise ValueError(u'Can only get bpm of one song at time')
item = items[0]
if item['bpm']:
self._log.info(u'Found bpm {0}', item['bpm'])
if not overwrite:
return
self._log.info(u'Press Enter {0} times to the rhythm or Ctrl-D '
u'to exit', self.config['max_strokes'].get(int))
new_bpm = bpm(self.config['max_strokes'].get(int))
item['bpm'] = int(new_bpm)
if write:
item.try_write()
item.store()
self._log.info(u'Added new bpm {0}', item['bpm'])

243
libs/beetsplug/bucket.py Normal file
View file

@ -0,0 +1,243 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016, Fabrice Laporte.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Provides the %bucket{} function for path formatting.
"""
from __future__ import division, absolute_import, print_function
from datetime import datetime
import re
import string
from itertools import tee, izip
from beets import plugins, ui
ASCII_DIGITS = string.digits + string.ascii_lowercase
class BucketError(Exception):
pass
def pairwise(iterable):
"s -> (s0,s1), (s1,s2), (s2, s3), ..."
a, b = tee(iterable)
next(b, None)
return izip(a, b)
def span_from_str(span_str):
"""Build a span dict from the span string representation.
"""
def normalize_year(d, yearfrom):
"""Convert string to a 4 digits year
"""
if yearfrom < 100:
raise BucketError(u"%d must be expressed on 4 digits" % yearfrom)
# if two digits only, pick closest year that ends by these two
# digits starting from yearfrom
if d < 100:
if (d % 100) < (yearfrom % 100):
d = (yearfrom - yearfrom % 100) + 100 + d
else:
d = (yearfrom - yearfrom % 100) + d
return d
years = [int(x) for x in re.findall('\d+', span_str)]
if not years:
raise ui.UserError(u"invalid range defined for year bucket '%s': no "
u"year found" % span_str)
try:
years = [normalize_year(x, years[0]) for x in years]
except BucketError as exc:
raise ui.UserError(u"invalid range defined for year bucket '%s': %s" %
(span_str, exc))
res = {'from': years[0], 'str': span_str}
if len(years) > 1:
res['to'] = years[-1]
return res
def complete_year_spans(spans):
"""Set the `to` value of spans if empty and sort them chronologically.
"""
spans.sort(key=lambda x: x['from'])
for (x, y) in pairwise(spans):
if 'to' not in x:
x['to'] = y['from'] - 1
if spans and 'to' not in spans[-1]:
spans[-1]['to'] = datetime.now().year
def extend_year_spans(spans, spanlen, start=1900, end=2014):
"""Add new spans to given spans list so that every year of [start,end]
belongs to a span.
"""
extended_spans = spans[:]
for (x, y) in pairwise(spans):
# if a gap between two spans, fill the gap with as much spans of
# spanlen length as necessary
for span_from in range(x['to'] + 1, y['from'], spanlen):
extended_spans.append({'from': span_from})
# Create spans prior to declared ones
for span_from in range(spans[0]['from'] - spanlen, start, -spanlen):
extended_spans.append({'from': span_from})
# Create spans after the declared ones
for span_from in range(spans[-1]['to'] + 1, end, spanlen):
extended_spans.append({'from': span_from})
complete_year_spans(extended_spans)
return extended_spans
def build_year_spans(year_spans_str):
"""Build a chronologically ordered list of spans dict from unordered spans
stringlist.
"""
spans = []
for elem in year_spans_str:
spans.append(span_from_str(elem))
complete_year_spans(spans)
return spans
def str2fmt(s):
"""Deduces formatting syntax from a span string.
"""
regex = re.compile(r"(?P<bef>\D*)(?P<fromyear>\d+)(?P<sep>\D*)"
r"(?P<toyear>\d*)(?P<after>\D*)")
m = re.match(regex, s)
res = {'fromnchars': len(m.group('fromyear')),
'tonchars': len(m.group('toyear'))}
res['fmt'] = "%s%%s%s%s%s" % (m.group('bef'),
m.group('sep'),
'%s' if res['tonchars'] else '',
m.group('after'))
return res
def format_span(fmt, yearfrom, yearto, fromnchars, tonchars):
"""Return a span string representation.
"""
args = (bytes(yearfrom)[-fromnchars:])
if tonchars:
args = (bytes(yearfrom)[-fromnchars:], bytes(yearto)[-tonchars:])
return fmt % args
def extract_modes(spans):
"""Extract the most common spans lengths and representation formats
"""
rangelen = sorted([x['to'] - x['from'] + 1 for x in spans])
deflen = sorted(rangelen, key=rangelen.count)[-1]
reprs = [str2fmt(x['str']) for x in spans]
deffmt = sorted(reprs, key=reprs.count)[-1]
return deflen, deffmt
def build_alpha_spans(alpha_spans_str, alpha_regexs):
"""Extract alphanumerics from string and return sorted list of chars
[from...to]
"""
spans = []
for elem in alpha_spans_str:
if elem in alpha_regexs:
spans.append(re.compile(alpha_regexs[elem]))
else:
bucket = sorted([x for x in elem.lower() if x.isalnum()])
if bucket:
begin_index = ASCII_DIGITS.index(bucket[0])
end_index = ASCII_DIGITS.index(bucket[-1])
else:
raise ui.UserError(u"invalid range defined for alpha bucket "
u"'%s': no alphanumeric character found" %
elem)
spans.append(
re.compile(
"^[" + ASCII_DIGITS[begin_index:end_index + 1] +
ASCII_DIGITS[begin_index:end_index + 1].upper() + "]"
)
)
return spans
class BucketPlugin(plugins.BeetsPlugin):
def __init__(self):
super(BucketPlugin, self).__init__()
self.template_funcs['bucket'] = self._tmpl_bucket
self.config.add({
'bucket_year': [],
'bucket_alpha': [],
'bucket_alpha_regex': {},
'extrapolate': False
})
self.setup()
def setup(self):
"""Setup plugin from config options
"""
self.year_spans = build_year_spans(self.config['bucket_year'].get())
if self.year_spans and self.config['extrapolate']:
[self.ys_len_mode,
self.ys_repr_mode] = extract_modes(self.year_spans)
self.year_spans = extend_year_spans(self.year_spans,
self.ys_len_mode)
self.alpha_spans = build_alpha_spans(
self.config['bucket_alpha'].get(),
self.config['bucket_alpha_regex'].get()
)
def find_bucket_year(self, year):
"""Return bucket that matches given year or return the year
if no matching bucket.
"""
for ys in self.year_spans:
if ys['from'] <= int(year) <= ys['to']:
if 'str' in ys:
return ys['str']
else:
return format_span(self.ys_repr_mode['fmt'],
ys['from'], ys['to'],
self.ys_repr_mode['fromnchars'],
self.ys_repr_mode['tonchars'])
return year
def find_bucket_alpha(self, s):
"""Return alpha-range bucket that matches given string or return the
string initial if no matching bucket.
"""
for (i, span) in enumerate(self.alpha_spans):
if span.match(s):
return self.config['bucket_alpha'].get()[i]
return s[0].upper()
def _tmpl_bucket(self, text, field=None):
if not field and len(text) == 4 and text.isdigit():
field = 'year'
if field == 'year':
func = self.find_bucket_year
else:
func = self.find_bucket_alpha
return func(text)

308
libs/beetsplug/chroma.py Normal file
View file

@ -0,0 +1,308 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Adds Chromaprint/Acoustid acoustic fingerprinting support to the
autotagger. Requires the pyacoustid library.
"""
from __future__ import division, absolute_import, print_function
from beets import plugins
from beets import ui
from beets import util
from beets import config
from beets.util import confit
from beets.autotag import hooks
import acoustid
from collections import defaultdict
API_KEY = '1vOwZtEn'
SCORE_THRESH = 0.5
TRACK_ID_WEIGHT = 10.0
COMMON_REL_THRESH = 0.6 # How many tracks must have an album in common?
MAX_RECORDINGS = 5
MAX_RELEASES = 5
# Stores the Acoustid match information for each track. This is
# populated when an import task begins and then used when searching for
# candidates. It maps audio file paths to (recording_ids, release_ids)
# pairs. If a given path is not present in the mapping, then no match
# was found.
_matches = {}
# Stores the fingerprint and Acoustid ID for each track. This is stored
# as metadata for each track for later use but is not relevant for
# autotagging.
_fingerprints = {}
_acoustids = {}
def prefix(it, count):
"""Truncate an iterable to at most `count` items.
"""
for i, v in enumerate(it):
if i >= count:
break
yield v
def acoustid_match(log, path):
"""Gets metadata for a file from Acoustid and populates the
_matches, _fingerprints, and _acoustids dictionaries accordingly.
"""
try:
duration, fp = acoustid.fingerprint_file(util.syspath(path))
except acoustid.FingerprintGenerationError as exc:
log.error(u'fingerprinting of {0} failed: {1}',
util.displayable_path(repr(path)), exc)
return None
_fingerprints[path] = fp
try:
res = acoustid.lookup(API_KEY, fp, duration,
meta='recordings releases')
except acoustid.AcoustidError as exc:
log.debug(u'fingerprint matching {0} failed: {1}',
util.displayable_path(repr(path)), exc)
return None
log.debug(u'chroma: fingerprinted {0}',
util.displayable_path(repr(path)))
# Ensure the response is usable and parse it.
if res['status'] != 'ok' or not res.get('results'):
log.debug(u'no match found')
return None
result = res['results'][0] # Best match.
if result['score'] < SCORE_THRESH:
log.debug(u'no results above threshold')
return None
_acoustids[path] = result['id']
# Get recording and releases from the result.
if not result.get('recordings'):
log.debug(u'no recordings found')
return None
recording_ids = []
release_ids = []
for recording in result['recordings']:
recording_ids.append(recording['id'])
if 'releases' in recording:
release_ids += [rel['id'] for rel in recording['releases']]
log.debug(u'matched recordings {0} on releases {1}',
recording_ids, release_ids)
_matches[path] = recording_ids, release_ids
# Plugin structure and autotagging logic.
def _all_releases(items):
"""Given an iterable of Items, determines (according to Acoustid)
which releases the items have in common. Generates release IDs.
"""
# Count the number of "hits" for each release.
relcounts = defaultdict(int)
for item in items:
if item.path not in _matches:
continue
_, release_ids = _matches[item.path]
for release_id in release_ids:
relcounts[release_id] += 1
for release_id, count in relcounts.iteritems():
if float(count) / len(items) > COMMON_REL_THRESH:
yield release_id
class AcoustidPlugin(plugins.BeetsPlugin):
def __init__(self):
super(AcoustidPlugin, self).__init__()
self.config.add({
'auto': True,
})
config['acoustid']['apikey'].redact = True
if self.config['auto']:
self.register_listener('import_task_start', self.fingerprint_task)
self.register_listener('import_task_apply', apply_acoustid_metadata)
def fingerprint_task(self, task, session):
return fingerprint_task(self._log, task, session)
def track_distance(self, item, info):
dist = hooks.Distance()
if item.path not in _matches or not info.track_id:
# Match failed or no track ID.
return dist
recording_ids, _ = _matches[item.path]
dist.add_expr('track_id', info.track_id not in recording_ids)
return dist
def candidates(self, items, artist, album, va_likely):
albums = []
for relid in prefix(_all_releases(items), MAX_RELEASES):
album = hooks.album_for_mbid(relid)
if album:
albums.append(album)
self._log.debug(u'acoustid album candidates: {0}', len(albums))
return albums
def item_candidates(self, item, artist, title):
if item.path not in _matches:
return []
recording_ids, _ = _matches[item.path]
tracks = []
for recording_id in prefix(recording_ids, MAX_RECORDINGS):
track = hooks.track_for_mbid(recording_id)
if track:
tracks.append(track)
self._log.debug(u'acoustid item candidates: {0}', len(tracks))
return tracks
def commands(self):
submit_cmd = ui.Subcommand('submit',
help=u'submit Acoustid fingerprints')
def submit_cmd_func(lib, opts, args):
try:
apikey = config['acoustid']['apikey'].get(unicode)
except confit.NotFoundError:
raise ui.UserError(u'no Acoustid user API key provided')
submit_items(self._log, apikey, lib.items(ui.decargs(args)))
submit_cmd.func = submit_cmd_func
fingerprint_cmd = ui.Subcommand(
'fingerprint',
help=u'generate fingerprints for items without them'
)
def fingerprint_cmd_func(lib, opts, args):
for item in lib.items(ui.decargs(args)):
fingerprint_item(self._log, item, write=ui.should_write())
fingerprint_cmd.func = fingerprint_cmd_func
return [submit_cmd, fingerprint_cmd]
# Hooks into import process.
def fingerprint_task(log, task, session):
"""Fingerprint each item in the task for later use during the
autotagging candidate search.
"""
items = task.items if task.is_album else [task.item]
for item in items:
acoustid_match(log, item.path)
def apply_acoustid_metadata(task, session):
"""Apply Acoustid metadata (fingerprint and ID) to the task's items.
"""
for item in task.imported_items():
if item.path in _fingerprints:
item.acoustid_fingerprint = _fingerprints[item.path]
if item.path in _acoustids:
item.acoustid_id = _acoustids[item.path]
# UI commands.
def submit_items(log, userkey, items, chunksize=64):
"""Submit fingerprints for the items to the Acoustid server.
"""
data = [] # The running list of dictionaries to submit.
def submit_chunk():
"""Submit the current accumulated fingerprint data."""
log.info(u'submitting {0} fingerprints', len(data))
try:
acoustid.submit(API_KEY, userkey, data)
except acoustid.AcoustidError as exc:
log.warn(u'acoustid submission error: {0}', exc)
del data[:]
for item in items:
fp = fingerprint_item(log, item)
# Construct a submission dictionary for this item.
item_data = {
'duration': int(item.length),
'fingerprint': fp,
}
if item.mb_trackid:
item_data['mbid'] = item.mb_trackid
log.debug(u'submitting MBID')
else:
item_data.update({
'track': item.title,
'artist': item.artist,
'album': item.album,
'albumartist': item.albumartist,
'year': item.year,
'trackno': item.track,
'discno': item.disc,
})
log.debug(u'submitting textual metadata')
data.append(item_data)
# If we have enough data, submit a chunk.
if len(data) >= chunksize:
submit_chunk()
# Submit remaining data in a final chunk.
if data:
submit_chunk()
def fingerprint_item(log, item, write=False):
"""Get the fingerprint for an Item. If the item already has a
fingerprint, it is not regenerated. If fingerprint generation fails,
return None. If the items are associated with a library, they are
saved to the database. If `write` is set, then the new fingerprints
are also written to files' metadata.
"""
# Get a fingerprint and length for this track.
if not item.length:
log.info(u'{0}: no duration available',
util.displayable_path(item.path))
elif item.acoustid_fingerprint:
if write:
log.info(u'{0}: fingerprint exists, skipping',
util.displayable_path(item.path))
else:
log.info(u'{0}: using existing fingerprint',
util.displayable_path(item.path))
return item.acoustid_fingerprint
else:
log.info(u'{0}: fingerprinting',
util.displayable_path(item.path))
try:
_, fp = acoustid.fingerprint_file(item.path)
item.acoustid_fingerprint = fp
if write:
log.info(u'{0}: writing fingerprint',
util.displayable_path(item.path))
item.try_write()
if item._db:
item.store()
return item.acoustid_fingerprint
except acoustid.FingerprintGenerationError as exc:
log.info(u'fingerprint generation failed: {0}', exc)

449
libs/beetsplug/convert.py Normal file
View file

@ -0,0 +1,449 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016, Jakob Schnitzer.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Converts tracks or albums to external directory
"""
from __future__ import division, absolute_import, print_function
import os
import threading
import subprocess
import tempfile
import shlex
from string import Template
from beets import ui, util, plugins, config
from beets.plugins import BeetsPlugin
from beets.util.confit import ConfigTypeError
from beets import art
from beets.util.artresizer import ArtResizer
_fs_lock = threading.Lock()
_temp_files = [] # Keep track of temporary transcoded files for deletion.
# Some convenient alternate names for formats.
ALIASES = {
u'wma': u'windows media',
u'vorbis': u'ogg',
}
LOSSLESS_FORMATS = ['ape', 'flac', 'alac', 'wav', 'aiff']
def replace_ext(path, ext):
"""Return the path with its extension replaced by `ext`.
The new extension must not contain a leading dot.
"""
return os.path.splitext(path)[0] + b'.' + ext
def get_format(fmt=None):
"""Return the command template and the extension from the config.
"""
if not fmt:
fmt = config['convert']['format'].get(unicode).lower()
fmt = ALIASES.get(fmt, fmt)
try:
format_info = config['convert']['formats'][fmt].get(dict)
command = format_info['command']
extension = format_info.get('extension', fmt)
except KeyError:
raise ui.UserError(
u'convert: format {0} needs the "command" field'
.format(fmt)
)
except ConfigTypeError:
command = config['convert']['formats'][fmt].get(bytes)
extension = fmt
# Convenience and backwards-compatibility shortcuts.
keys = config['convert'].keys()
if 'command' in keys:
command = config['convert']['command'].get(unicode)
elif 'opts' in keys:
# Undocumented option for backwards compatibility with < 1.3.1.
command = u'ffmpeg -i $source -y {0} $dest'.format(
config['convert']['opts'].get(unicode)
)
if 'extension' in keys:
extension = config['convert']['extension'].get(unicode)
return (command.encode('utf8'), extension.encode('utf8'))
def should_transcode(item, fmt):
"""Determine whether the item should be transcoded as part of
conversion (i.e., its bitrate is high or it has the wrong format).
"""
if config['convert']['never_convert_lossy_files'] and \
not (item.format.lower() in LOSSLESS_FORMATS):
return False
maxbr = config['convert']['max_bitrate'].get(int)
return fmt.lower() != item.format.lower() or \
item.bitrate >= 1000 * maxbr
class ConvertPlugin(BeetsPlugin):
def __init__(self):
super(ConvertPlugin, self).__init__()
self.config.add({
u'dest': None,
u'pretend': False,
u'threads': util.cpu_count(),
u'format': u'mp3',
u'formats': {
u'aac': {
u'command': u'ffmpeg -i $source -y -vn -acodec libfaac '
u'-aq 100 $dest',
u'extension': u'm4a',
},
u'alac': {
u'command': u'ffmpeg -i $source -y -vn -acodec alac $dest',
u'extension': u'm4a',
},
u'flac': u'ffmpeg -i $source -y -vn -acodec flac $dest',
u'mp3': u'ffmpeg -i $source -y -vn -aq 2 $dest',
u'opus':
u'ffmpeg -i $source -y -vn -acodec libopus -ab 96k $dest',
u'ogg':
u'ffmpeg -i $source -y -vn -acodec libvorbis -aq 3 $dest',
u'wma':
u'ffmpeg -i $source -y -vn -acodec wmav2 -vn $dest',
},
u'max_bitrate': 500,
u'auto': False,
u'tmpdir': None,
u'quiet': False,
u'embed': True,
u'paths': {},
u'never_convert_lossy_files': False,
u'copy_album_art': False,
u'album_art_maxwidth': 0,
})
self.import_stages = [self.auto_convert]
self.register_listener('import_task_files', self._cleanup)
def commands(self):
cmd = ui.Subcommand('convert', help=u'convert to external location')
cmd.parser.add_option('-p', '--pretend', action='store_true',
help=u'show actions but do nothing')
cmd.parser.add_option('-t', '--threads', action='store', type='int',
help=u'change the number of threads, \
defaults to maximum available processors')
cmd.parser.add_option('-k', '--keep-new', action='store_true',
dest='keep_new', help=u'keep only the converted \
and move the old files')
cmd.parser.add_option('-d', '--dest', action='store',
help=u'set the destination directory')
cmd.parser.add_option('-f', '--format', action='store', dest='format',
help=u'set the target format of the tracks')
cmd.parser.add_option('-y', '--yes', action='store_true', dest='yes',
help=u'do not ask for confirmation')
cmd.parser.add_album_option()
cmd.func = self.convert_func
return [cmd]
def auto_convert(self, config, task):
if self.config['auto']:
for item in task.imported_items():
self.convert_on_import(config.lib, item)
# Utilities converted from functions to methods on logging overhaul
def encode(self, command, source, dest, pretend=False):
"""Encode `source` to `dest` using command template `command`.
Raises `subprocess.CalledProcessError` if the command exited with a
non-zero status code.
"""
# The paths and arguments must be bytes.
assert isinstance(command, bytes)
assert isinstance(source, bytes)
assert isinstance(dest, bytes)
quiet = self.config['quiet'].get(bool)
if not quiet and not pretend:
self._log.info(u'Encoding {0}', util.displayable_path(source))
# Substitute $source and $dest in the argument list.
args = shlex.split(command)
for i, arg in enumerate(args):
args[i] = Template(arg).safe_substitute({
'source': source,
'dest': dest,
})
if pretend:
self._log.info(u' '.join(ui.decargs(args)))
return
try:
util.command_output(args)
except subprocess.CalledProcessError as exc:
# Something went wrong (probably Ctrl+C), remove temporary files
self._log.info(u'Encoding {0} failed. Cleaning up...',
util.displayable_path(source))
self._log.debug(u'Command {0} exited with status {1}',
exc.cmd.decode('utf8', 'ignore'),
exc.returncode)
util.remove(dest)
util.prune_dirs(os.path.dirname(dest))
raise
except OSError as exc:
raise ui.UserError(
u"convert: couldn't invoke '{0}': {1}".format(
u' '.join(ui.decargs(args)), exc
)
)
if not quiet and not pretend:
self._log.info(u'Finished encoding {0}',
util.displayable_path(source))
def convert_item(self, dest_dir, keep_new, path_formats, fmt,
pretend=False):
command, ext = get_format(fmt)
item, original, converted = None, None, None
while True:
item = yield (item, original, converted)
dest = item.destination(basedir=dest_dir,
path_formats=path_formats)
# When keeping the new file in the library, we first move the
# current (pristine) file to the destination. We'll then copy it
# back to its old path or transcode it to a new path.
if keep_new:
original = dest
converted = item.path
if should_transcode(item, fmt):
converted = replace_ext(converted, ext)
else:
original = item.path
if should_transcode(item, fmt):
dest = replace_ext(dest, ext)
converted = dest
# Ensure that only one thread tries to create directories at a
# time. (The existence check is not atomic with the directory
# creation inside this function.)
if not pretend:
with _fs_lock:
util.mkdirall(dest)
if os.path.exists(util.syspath(dest)):
self._log.info(u'Skipping {0} (target file exists)',
util.displayable_path(item.path))
continue
if keep_new:
if pretend:
self._log.info(u'mv {0} {1}',
util.displayable_path(item.path),
util.displayable_path(original))
else:
self._log.info(u'Moving to {0}',
util.displayable_path(original))
util.move(item.path, original)
if should_transcode(item, fmt):
try:
self.encode(command, original, converted, pretend)
except subprocess.CalledProcessError:
continue
else:
if pretend:
self._log.info(u'cp {0} {1}',
util.displayable_path(original),
util.displayable_path(converted))
else:
# No transcoding necessary.
self._log.info(u'Copying {0}',
util.displayable_path(item.path))
util.copy(original, converted)
if pretend:
continue
# Write tags from the database to the converted file.
item.try_write(path=converted)
if keep_new:
# If we're keeping the transcoded file, read it again (after
# writing) to get new bitrate, duration, etc.
item.path = converted
item.read()
item.store() # Store new path and audio data.
if self.config['embed']:
album = item.get_album()
if album and album.artpath:
self._log.debug(u'embedding album art from {}',
util.displayable_path(album.artpath))
art.embed_item(self._log, item, album.artpath,
itempath=converted)
if keep_new:
plugins.send('after_convert', item=item,
dest=dest, keepnew=True)
else:
plugins.send('after_convert', item=item,
dest=converted, keepnew=False)
def copy_album_art(self, album, dest_dir, path_formats, pretend=False):
"""Copies or converts the associated cover art of the album. Album must
have at least one track.
"""
if not album or not album.artpath:
return
album_item = album.items().get()
# Album shouldn't be empty.
if not album_item:
return
# Get the destination of the first item (track) of the album, we use
# this function to format the path accordingly to path_formats.
dest = album_item.destination(basedir=dest_dir,
path_formats=path_formats)
# Remove item from the path.
dest = os.path.join(*util.components(dest)[:-1])
dest = album.art_destination(album.artpath, item_dir=dest)
if album.artpath == dest:
return
if not pretend:
util.mkdirall(dest)
if os.path.exists(util.syspath(dest)):
self._log.info(u'Skipping {0} (target file exists)',
util.displayable_path(album.artpath))
return
# Decide whether we need to resize the cover-art image.
resize = False
maxwidth = None
if self.config['album_art_maxwidth']:
maxwidth = self.config['album_art_maxwidth'].get(int)
size = ArtResizer.shared.get_size(album.artpath)
self._log.debug('image size: {}', size)
if size:
resize = size[0] > maxwidth
else:
self._log.warning(u'Could not get size of image (please see '
u'documentation for dependencies).')
# Either copy or resize (while copying) the image.
if resize:
self._log.info(u'Resizing cover art from {0} to {1}',
util.displayable_path(album.artpath),
util.displayable_path(dest))
if not pretend:
ArtResizer.shared.resize(maxwidth, album.artpath, dest)
else:
if pretend:
self._log.info(u'cp {0} {1}',
util.displayable_path(album.artpath),
util.displayable_path(dest))
else:
self._log.info(u'Copying cover art to {0}',
util.displayable_path(album.artpath),
util.displayable_path(dest))
util.copy(album.artpath, dest)
def convert_func(self, lib, opts, args):
if not opts.dest:
opts.dest = self.config['dest'].get()
if not opts.dest:
raise ui.UserError(u'no convert destination set')
opts.dest = util.bytestring_path(opts.dest)
if not opts.threads:
opts.threads = self.config['threads'].get(int)
if self.config['paths']:
path_formats = ui.get_path_formats(self.config['paths'])
else:
path_formats = ui.get_path_formats()
if not opts.format:
opts.format = self.config['format'].get(unicode).lower()
pretend = opts.pretend if opts.pretend is not None else \
self.config['pretend'].get(bool)
if not pretend:
ui.commands.list_items(lib, ui.decargs(args), opts.album)
if not (opts.yes or ui.input_yn(u"Convert? (Y/n)")):
return
if opts.album:
albums = lib.albums(ui.decargs(args))
items = (i for a in albums for i in a.items())
if self.config['copy_album_art']:
for album in albums:
self.copy_album_art(album, opts.dest, path_formats,
pretend)
else:
items = iter(lib.items(ui.decargs(args)))
convert = [self.convert_item(opts.dest,
opts.keep_new,
path_formats,
opts.format,
pretend)
for _ in range(opts.threads)]
pipe = util.pipeline.Pipeline([items, convert])
pipe.run_parallel()
def convert_on_import(self, lib, item):
"""Transcode a file automatically after it is imported into the
library.
"""
fmt = self.config['format'].get(unicode).lower()
if should_transcode(item, fmt):
command, ext = get_format()
# Create a temporary file for the conversion.
tmpdir = self.config['tmpdir'].get()
fd, dest = tempfile.mkstemp('.' + ext, dir=tmpdir)
os.close(fd)
dest = util.bytestring_path(dest)
_temp_files.append(dest) # Delete the transcode later.
# Convert.
try:
self.encode(command, item.path, dest)
except subprocess.CalledProcessError:
return
# Change the newly-imported database entry to point to the
# converted file.
item.path = dest
item.write()
item.read() # Load new audio information data.
item.store()
def _cleanup(self, task, session):
for path in task.old_paths:
if path in _temp_files:
if os.path.isfile(path):
util.remove(path)
_temp_files.remove(path)

57
libs/beetsplug/cue.py Normal file
View file

@ -0,0 +1,57 @@
# -*- coding: utf-8 -*-
# Copyright 2016 Bruno Cauet
# Split an album-file in tracks thanks a cue file
from __future__ import division, absolute_import, print_function
import subprocess
from os import path
from glob import glob
from beets.util import command_output, displayable_path
from beets.plugins import BeetsPlugin
from beets.autotag import TrackInfo
class CuePlugin(BeetsPlugin):
def __init__(self):
super(CuePlugin, self).__init__()
# this does not seem supported by shnsplit
self.config.add({
'keep_before': .1,
'keep_after': .9,
})
# self.register_listener('import_task_start', self.look_for_cues)
def candidates(self, items, artist, album, va_likely):
import pdb
pdb.set_trace()
def item_candidates(self, item, artist, album):
dir = path.dirname(item.path)
cues = glob.glob(path.join(dir, "*.cue"))
if not cues:
return
if len(cues) > 1:
self._log.info(u"Found multiple cue files doing nothing: {0}",
map(displayable_path, cues))
cue_file = cues[0]
self._log.info("Found {} for {}", displayable_path(cue_file), item)
try:
# careful: will ask for input in case of conflicts
command_output(['shnsplit', '-f', cue_file, item.path])
except (subprocess.CalledProcessError, OSError):
self._log.exception(u'shnsplit execution failed')
return
tracks = glob(path.join(dir, "*.wav"))
self._log.info("Generated {0} tracks", len(tracks))
for t in tracks:
title = "dunno lol"
track_id = "wtf"
index = int(path.basename(t)[len("split-track"):-len(".wav")])
yield TrackInfo(title, track_id, index=index, artist=artist)
# generate TrackInfo instances

350
libs/beetsplug/discogs.py Normal file
View file

@ -0,0 +1,350 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Adds Discogs album search support to the autotagger. Requires the
discogs-client library.
"""
from __future__ import division, absolute_import, print_function
import beets.ui
from beets import logging
from beets import config
from beets.autotag.hooks import AlbumInfo, TrackInfo, Distance
from beets.plugins import BeetsPlugin
from beets.util import confit
from discogs_client import Release, Client
from discogs_client.exceptions import DiscogsAPIError
from requests.exceptions import ConnectionError
import beets
import re
import time
import json
import socket
import httplib
import os
# Silence spurious INFO log lines generated by urllib3.
urllib3_logger = logging.getLogger('requests.packages.urllib3')
urllib3_logger.setLevel(logging.CRITICAL)
USER_AGENT = u'beets/{0} +http://beets.io/'.format(beets.__version__)
# Exceptions that discogs_client should really handle but does not.
CONNECTION_ERRORS = (ConnectionError, socket.error, httplib.HTTPException,
ValueError, # JSON decoding raises a ValueError.
DiscogsAPIError)
class DiscogsPlugin(BeetsPlugin):
def __init__(self):
super(DiscogsPlugin, self).__init__()
self.config.add({
'apikey': 'rAzVUQYRaoFjeBjyWuWZ',
'apisecret': 'plxtUTqoCzwxZpqdPysCwGuBSmZNdZVy',
'tokenfile': 'discogs_token.json',
'source_weight': 0.5,
})
self.config['apikey'].redact = True
self.config['apisecret'].redact = True
self.discogs_client = None
self.register_listener('import_begin', self.setup)
def setup(self, session=None):
"""Create the `discogs_client` field. Authenticate if necessary.
"""
c_key = self.config['apikey'].get(unicode)
c_secret = self.config['apisecret'].get(unicode)
# Get the OAuth token from a file or log in.
try:
with open(self._tokenfile()) as f:
tokendata = json.load(f)
except IOError:
# No token yet. Generate one.
token, secret = self.authenticate(c_key, c_secret)
else:
token = tokendata['token']
secret = tokendata['secret']
self.discogs_client = Client(USER_AGENT, c_key, c_secret,
token, secret)
def reset_auth(self):
"""Delete toke file & redo the auth steps.
"""
os.remove(self._tokenfile())
self.setup()
def _tokenfile(self):
"""Get the path to the JSON file for storing the OAuth token.
"""
return self.config['tokenfile'].get(confit.Filename(in_app_dir=True))
def authenticate(self, c_key, c_secret):
# Get the link for the OAuth page.
auth_client = Client(USER_AGENT, c_key, c_secret)
try:
_, _, url = auth_client.get_authorize_url()
except CONNECTION_ERRORS as e:
self._log.debug(u'connection error: {0}', e)
raise beets.ui.UserError(u'communication with Discogs failed')
beets.ui.print_(u"To authenticate with Discogs, visit:")
beets.ui.print_(url)
# Ask for the code and validate it.
code = beets.ui.input_(u"Enter the code:")
try:
token, secret = auth_client.get_access_token(code)
except DiscogsAPIError:
raise beets.ui.UserError(u'Discogs authorization failed')
except CONNECTION_ERRORS as e:
self._log.debug(u'connection error: {0}', e)
raise beets.ui.UserError(u'Discogs token request failed')
# Save the token for later use.
self._log.debug(u'Discogs token {0}, secret {1}', token, secret)
with open(self._tokenfile(), 'w') as f:
json.dump({'token': token, 'secret': secret}, f)
return token, secret
def album_distance(self, items, album_info, mapping):
"""Returns the album distance.
"""
dist = Distance()
if album_info.data_source == 'Discogs':
dist.add('source', self.config['source_weight'].as_number())
return dist
def candidates(self, items, artist, album, va_likely):
"""Returns a list of AlbumInfo objects for discogs search results
matching an album and artist (if not various).
"""
if not self.discogs_client:
return
if va_likely:
query = album
else:
query = '%s %s' % (artist, album)
try:
return self.get_albums(query)
except DiscogsAPIError as e:
self._log.debug(u'API Error: {0} (query: {1})', e, query)
if e.status_code == 401:
self.reset_auth()
return self.candidates(items, artist, album, va_likely)
else:
return []
except CONNECTION_ERRORS:
self._log.debug(u'Connection error in album search', exc_info=True)
return []
def album_for_id(self, album_id):
"""Fetches an album by its Discogs ID and returns an AlbumInfo object
or None if the album is not found.
"""
if not self.discogs_client:
return
self._log.debug(u'Searching for release {0}', album_id)
# Discogs-IDs are simple integers. We only look for those at the end
# of an input string as to avoid confusion with other metadata plugins.
# An optional bracket can follow the integer, as this is how discogs
# displays the release ID on its webpage.
match = re.search(r'(^|\[*r|discogs\.com/.+/release/)(\d+)($|\])',
album_id)
if not match:
return None
result = Release(self.discogs_client, {'id': int(match.group(2))})
# Try to obtain title to verify that we indeed have a valid Release
try:
getattr(result, 'title')
except DiscogsAPIError as e:
if e.status_code != 404:
self._log.debug(u'API Error: {0} (query: {1})', e, result._uri)
if e.status_code == 401:
self.reset_auth()
return self.album_for_id(album_id)
return None
except CONNECTION_ERRORS:
self._log.debug(u'Connection error in album lookup', exc_info=True)
return None
return self.get_album_info(result)
def get_albums(self, query):
"""Returns a list of AlbumInfo objects for a discogs search query.
"""
# Strip non-word characters from query. Things like "!" and "-" can
# cause a query to return no results, even if they match the artist or
# album title. Use `re.UNICODE` flag to avoid stripping non-english
# word characters.
# TEMPORARY: Encode as ASCII to work around a bug:
# https://github.com/beetbox/beets/issues/1051
# When the library is fixed, we should encode as UTF-8.
query = re.sub(r'(?u)\W+', ' ', query).encode('ascii', "replace")
# Strip medium information from query, Things like "CD1" and "disk 1"
# can also negate an otherwise positive result.
query = re.sub(r'(?i)\b(CD|disc)\s*\d+', '', query)
try:
releases = self.discogs_client.search(query,
type='release').page(1)
except CONNECTION_ERRORS:
self._log.debug(u"Communication error while searching for {0!r}",
query, exc_info=True)
return []
return [self.get_album_info(release) for release in releases[:5]]
def get_album_info(self, result):
"""Returns an AlbumInfo object for a discogs Release object.
"""
artist, artist_id = self.get_artist([a.data for a in result.artists])
album = re.sub(r' +', ' ', result.title)
album_id = result.data['id']
# Use `.data` to access the tracklist directly instead of the
# convenient `.tracklist` property, which will strip out useful artist
# information and leave us with skeleton `Artist` objects that will
# each make an API call just to get the same data back.
tracks = self.get_tracks(result.data['tracklist'])
albumtype = ', '.join(
result.data['formats'][0].get('descriptions', [])) or None
va = result.data['artists'][0]['name'].lower() == 'various'
if va:
artist = config['va_name'].get(unicode)
year = result.data['year']
label = result.data['labels'][0]['name']
mediums = len(set(t.medium for t in tracks))
catalogno = result.data['labels'][0]['catno']
if catalogno == 'none':
catalogno = None
country = result.data.get('country')
media = result.data['formats'][0]['name']
data_url = result.data['uri']
return AlbumInfo(album, album_id, artist, artist_id, tracks, asin=None,
albumtype=albumtype, va=va, year=year, month=None,
day=None, label=label, mediums=mediums,
artist_sort=None, releasegroup_id=None,
catalognum=catalogno, script=None, language=None,
country=country, albumstatus=None, media=media,
albumdisambig=None, artist_credit=None,
original_year=None, original_month=None,
original_day=None, data_source='Discogs',
data_url=data_url)
def get_artist(self, artists):
"""Returns an artist string (all artists) and an artist_id (the main
artist) for a list of discogs album or track artists.
"""
artist_id = None
bits = []
for i, artist in enumerate(artists):
if not artist_id:
artist_id = artist['id']
name = artist['name']
# Strip disambiguation number.
name = re.sub(r' \(\d+\)$', '', name)
# Move articles to the front.
name = re.sub(r'(?i)^(.*?), (a|an|the)$', r'\2 \1', name)
bits.append(name)
if artist['join'] and i < len(artists) - 1:
bits.append(artist['join'])
artist = ' '.join(bits).replace(' ,', ',') or None
return artist, artist_id
def get_tracks(self, tracklist):
"""Returns a list of TrackInfo objects for a discogs tracklist.
"""
tracks = []
index_tracks = {}
index = 0
for track in tracklist:
# Only real tracks have `position`. Otherwise, it's an index track.
if track['position']:
index += 1
tracks.append(self.get_track_info(track, index))
else:
index_tracks[index + 1] = track['title']
# Fix up medium and medium_index for each track. Discogs position is
# unreliable, but tracks are in order.
medium = None
medium_count, index_count = 0, 0
for track in tracks:
# Handle special case where a different medium does not indicate a
# new disc, when there is no medium_index and the ordinal of medium
# is not sequential. For example, I, II, III, IV, V. Assume these
# are the track index, not the medium.
medium_is_index = track.medium and not track.medium_index and (
len(track.medium) != 1 or
ord(track.medium) - 64 != medium_count + 1
)
if not medium_is_index and medium != track.medium:
# Increment medium_count and reset index_count when medium
# changes.
medium = track.medium
medium_count += 1
index_count = 0
index_count += 1
track.medium, track.medium_index = medium_count, index_count
# Get `disctitle` from Discogs index tracks. Assume that an index track
# before the first track of each medium is a disc title.
for track in tracks:
if track.medium_index == 1:
if track.index in index_tracks:
disctitle = index_tracks[track.index]
else:
disctitle = None
track.disctitle = disctitle
return tracks
def get_track_info(self, track, index):
"""Returns a TrackInfo object for a discogs track.
"""
title = track['title']
track_id = None
medium, medium_index = self.get_track_index(track['position'])
artist, artist_id = self.get_artist(track.get('artists', []))
length = self.get_track_length(track['duration'])
return TrackInfo(title, track_id, artist, artist_id, length, index,
medium, medium_index, artist_sort=None,
disctitle=None, artist_credit=None)
def get_track_index(self, position):
"""Returns the medium and medium index for a discogs track position.
"""
# medium_index is a number at the end of position. medium is everything
# else. E.g. (A)(1), (Side A, Track )(1), (A)(), ()(1), etc.
match = re.match(r'^(.*?)(\d*)$', position.upper())
if match:
medium, index = match.groups()
else:
self._log.debug(u'Invalid position: {0}', position)
medium = index = None
return medium or None, index or None
def get_track_length(self, duration):
"""Returns the track length in seconds for a discogs duration.
"""
try:
length = time.strptime(duration, '%M:%S')
except ValueError:
return None
return length.tm_min * 60 + length.tm_sec

337
libs/beetsplug/duplicates.py Normal file
View file

@ -0,0 +1,337 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016, Pedro Silva.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""List duplicate tracks or albums.
"""
from __future__ import division, absolute_import, print_function
import shlex
from beets.plugins import BeetsPlugin
from beets.ui import decargs, print_, vararg_callback, Subcommand, UserError
from beets.util import command_output, displayable_path, subprocess
from beets.library import Item, Album
PLUGIN = 'duplicates'
class DuplicatesPlugin(BeetsPlugin):
"""List duplicate tracks or albums
"""
def __init__(self):
super(DuplicatesPlugin, self).__init__()
self.config.add({
'album': False,
'checksum': '',
'copy': '',
'count': False,
'delete': False,
'format': '',
'full': False,
'keys': [],
'merge': False,
'move': '',
'path': False,
'tiebreak': {},
'strict': False,
'tag': '',
})
self._command = Subcommand('duplicates',
help=__doc__,
aliases=['dup'])
self._command.parser.add_option(
u'-c', u'--count', dest='count',
action='store_true',
help=u'show duplicate counts',
)
self._command.parser.add_option(
u'-C', u'--checksum', dest='checksum',
action='store', metavar='PROG',
help=u'report duplicates based on arbitrary command',
)
self._command.parser.add_option(
u'-d', u'--delete', dest='delete',
action='store_true',
help=u'delete items from library and disk',
)
self._command.parser.add_option(
u'-F', u'--full', dest='full',
action='store_true',
help=u'show all versions of duplicate tracks or albums',
)
self._command.parser.add_option(
u'-s', u'--strict', dest='strict',
action='store_true',
help=u'report duplicates only if all attributes are set',
)
self._command.parser.add_option(
u'-k', u'--keys', dest='keys',
action='callback', metavar='KEY1 KEY2',
callback=vararg_callback,
help=u'report duplicates based on keys',
)
self._command.parser.add_option(
u'-M', u'--merge', dest='merge',
action='store_true',
help=u'merge duplicate items',
)
self._command.parser.add_option(
u'-m', u'--move', dest='move',
action='store', metavar='DEST',
help=u'move items to dest',
)
self._command.parser.add_option(
u'-o', u'--copy', dest='copy',
action='store', metavar='DEST',
help=u'copy items to dest',
)
self._command.parser.add_option(
u'-t', u'--tag', dest='tag',
action='store',
help=u'tag matched items with \'k=v\' attribute',
)
self._command.parser.add_all_common_options()
def commands(self):
def _dup(lib, opts, args):
self.config.set_args(opts)
album = self.config['album'].get(bool)
checksum = self.config['checksum'].get(str)
copy = self.config['copy'].get(str)
count = self.config['count'].get(bool)
delete = self.config['delete'].get(bool)
fmt = self.config['format'].get(str)
full = self.config['full'].get(bool)
keys = self.config['keys'].get(list)
merge = self.config['merge'].get(bool)
move = self.config['move'].get(str)
path = self.config['path'].get(bool)
tiebreak = self.config['tiebreak'].get(dict)
strict = self.config['strict'].get(bool)
tag = self.config['tag'].get(str)
if album:
if not keys:
keys = ['mb_albumid']
items = lib.albums(decargs(args))
else:
if not keys:
keys = ['mb_trackid', 'mb_albumid']
items = lib.items(decargs(args))
if path:
fmt = '$path'
# Default format string for count mode.
if count and not fmt:
if album:
fmt = '$albumartist - $album'
else:
fmt = '$albumartist - $album - $title'
fmt += ': {0}'
if checksum:
for i in items:
k, _ = self._checksum(i, checksum)
keys = [k]
for obj_id, obj_count, objs in self._duplicates(items,
keys=keys,
full=full,
strict=strict,
tiebreak=tiebreak,
merge=merge):
if obj_id: # Skip empty IDs.
for o in objs:
self._process_item(o,
copy=copy,
move=move,
delete=delete,
tag=tag,
fmt=fmt.format(obj_count))
self._command.func = _dup
return [self._command]
def _process_item(self, item, copy=False, move=False, delete=False,
tag=False, fmt=''):
"""Process Item `item`.
"""
print_(format(item, fmt))
if copy:
item.move(basedir=copy, copy=True)
item.store()
if move:
item.move(basedir=move, copy=False)
item.store()
if delete:
item.remove(delete=True)
if tag:
try:
k, v = tag.split('=')
except:
raise UserError(
u"{}: can't parse k=v tag: {}".format(PLUGIN, tag)
)
setattr(item, k, v)
item.store()
def _checksum(self, item, prog):
"""Run external `prog` on file path associated with `item`, cache
output as flexattr on a key that is the name of the program, and
return the key, checksum tuple.
"""
args = [p.format(file=item.path) for p in shlex.split(prog)]
key = args[0]
checksum = getattr(item, key, False)
if not checksum:
self._log.debug(u'key {0} on item {1} not cached:'
u'computing checksum',
key, displayable_path(item.path))
try:
checksum = command_output(args)
setattr(item, key, checksum)
item.store()
self._log.debug(u'computed checksum for {0} using {1}',
item.title, key)
except subprocess.CalledProcessError as e:
self._log.debug(u'failed to checksum {0}: {1}',
displayable_path(item.path), e)
else:
self._log.debug(u'key {0} on item {1} cached:'
u'not computing checksum',
key, displayable_path(item.path))
return key, checksum
def _group_by(self, objs, keys, strict):
"""Return a dictionary with keys arbitrary concatenations of attributes
and values lists of objects (Albums or Items) with those keys.
If strict, all attributes must be defined for a duplicate match.
"""
import collections
counts = collections.defaultdict(list)
for obj in objs:
values = [getattr(obj, k, None) for k in keys]
values = [v for v in values if v not in (None, '')]
if strict and len(values) < len(keys):
self._log.debug(u'some keys {0} on item {1} are null or empty:'
u' skipping',
keys, displayable_path(obj.path))
elif (not strict and not len(values)):
self._log.debug(u'all keys {0} on item {1} are null or empty:'
u' skipping',
keys, displayable_path(obj.path))
else:
key = tuple(values)
counts[key].append(obj)
return counts
def _order(self, objs, tiebreak=None):
"""Return the objects (Items or Albums) sorted by descending
order of priority.
If provided, the `tiebreak` dict indicates the field to use to
prioritize the objects. Otherwise, Items are placed in order of
"completeness" (objects with more non-null fields come first)
and Albums are ordered by their track count.
"""
if tiebreak:
kind = 'items' if all(isinstance(o, Item)
for o in objs) else 'albums'
key = lambda x: tuple(getattr(x, k) for k in tiebreak[kind])
else:
kind = Item if all(isinstance(o, Item) for o in objs) else Album
if kind is Item:
def truthy(v):
# Avoid a Unicode warning by avoiding comparison
# between a bytes object and the empty Unicode
# string ''.
return v is not None and \
(v != '' if isinstance(v, unicode) else True)
fields = kind.all_keys()
key = lambda x: sum(1 for f in fields if truthy(getattr(x, f)))
else:
key = lambda x: len(x.items())
return sorted(objs, key=key, reverse=True)
def _merge_items(self, objs):
"""Merge Item objs by copying missing fields from items in the tail to
the head item.
Return same number of items, with the head item modified.
"""
fields = Item.all_keys()
for f in fields:
for o in objs[1:]:
if getattr(objs[0], f, None) in (None, ''):
value = getattr(o, f, None)
if value:
self._log.debug(u'key {0} on item {1} is null '
u'or empty: setting from item {2}',
f, displayable_path(objs[0].path),
displayable_path(o.path))
setattr(objs[0], f, value)
objs[0].store()
break
return objs
def _merge_albums(self, objs):
"""Merge Album objs by copying missing items from albums in the tail
to the head album.
Return same number of albums, with the head album modified."""
ids = [i.mb_trackid for i in objs[0].items()]
for o in objs[1:]:
for i in o.items():
if i.mb_trackid not in ids:
missing = Item.from_path(i.path)
missing.album_id = objs[0].id
missing.add(i._db)
self._log.debug(u'item {0} missing from album {1}:'
u' merging from {2} into {3}',
missing,
objs[0],
displayable_path(o.path),
displayable_path(missing.destination()))
missing.move(copy=True)
return objs
def _merge(self, objs):
"""Merge duplicate items. See ``_merge_items`` and ``_merge_albums``
for the relevant strategies.
"""
kind = Item if all(isinstance(o, Item) for o in objs) else Album
if kind is Item:
objs = self._merge_items(objs)
else:
objs = self._merge_albums(objs)
return objs
def _duplicates(self, objs, keys, full, strict, tiebreak, merge):
"""Generate triples of keys, duplicate counts, and constituent objects.
"""
offset = 0 if full else 1
for k, objs in self._group_by(objs, keys, strict).iteritems():
if len(objs) > 1:
objs = self._order(objs, tiebreak)
if merge:
objs = self._merge(objs)
yield (k, len(objs) - offset, objs[offset:])

392
libs/beetsplug/edit.py Normal file
View file

@ -0,0 +1,392 @@
# This file is part of beets.
# Copyright 2016
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Open metadata information in a text editor to let the user edit it.
"""
from __future__ import division, absolute_import, print_function
from beets import plugins
from beets import util
from beets import ui
from beets.dbcore import types
from beets.importer import action
from beets.ui.commands import _do_query, PromptChoice
from copy import deepcopy
import subprocess
import yaml
from tempfile import NamedTemporaryFile
import os
# These "safe" types can avoid the format/parse cycle that most fields go
# through: they are safe to edit with native YAML types.
SAFE_TYPES = (types.Float, types.Integer, types.Boolean)
class ParseError(Exception):
"""The modified file is unreadable. The user should be offered a chance to
fix the error.
"""
def edit(filename, log):
"""Open `filename` in a text editor.
"""
cmd = util.shlex_split(util.editor_command())
cmd.append(filename)
log.debug(u'invoking editor command: {!r}', cmd)
try:
subprocess.call(cmd)
except OSError as exc:
raise ui.UserError(u'could not run editor command {!r}: {}'.format(
cmd[0], exc
))
def dump(arg):
"""Dump a sequence of dictionaries as YAML for editing.
"""
return yaml.safe_dump_all(
arg,
allow_unicode=True,
default_flow_style=False,
)
def load(s):
"""Read a sequence of YAML documents back to a list of dictionaries
with string keys.
Can raise a `ParseError`.
"""
try:
out = []
for d in yaml.load_all(s):
if not isinstance(d, dict):
raise ParseError(
u'each entry must be a dictionary; found {}'.format(
type(d).__name__
)
)
# Convert all keys to strings. They started out as strings,
# but the user may have inadvertently messed this up.
out.append({unicode(k): v for k, v in d.items()})
except yaml.YAMLError as e:
raise ParseError(u'invalid YAML: {}'.format(e))
return out
def _safe_value(obj, key, value):
"""Check whether the `value` is safe to represent in YAML and trust as
returned from parsed YAML.
This ensures that values do not change their type when the user edits their
YAML representation.
"""
typ = obj._type(key)
return isinstance(typ, SAFE_TYPES) and isinstance(value, typ.model_type)
def flatten(obj, fields):
"""Represent `obj`, a `dbcore.Model` object, as a dictionary for
serialization. Only include the given `fields` if provided;
otherwise, include everything.
The resulting dictionary's keys are strings and the values are
safely YAML-serializable types.
"""
# Format each value.
d = {}
for key in obj.keys():
value = obj[key]
if _safe_value(obj, key, value):
# A safe value that is faithfully representable in YAML.
d[key] = value
else:
# A value that should be edited as a string.
d[key] = obj.formatted()[key]
# Possibly filter field names.
if fields:
return {k: v for k, v in d.items() if k in fields}
else:
return d
def apply_(obj, data):
"""Set the fields of a `dbcore.Model` object according to a
dictionary.
This is the opposite of `flatten`. The `data` dictionary should have
strings as values.
"""
for key, value in data.items():
if _safe_value(obj, key, value):
# A safe value *stayed* represented as a safe type. Assign it
# directly.
obj[key] = value
else:
# Either the field was stringified originally or the user changed
# it from a safe type to an unsafe one. Parse it as a string.
obj.set_parse(key, unicode(value))
class EditPlugin(plugins.BeetsPlugin):
def __init__(self):
super(EditPlugin, self).__init__()
self.config.add({
# The default fields to edit.
'albumfields': 'album albumartist',
'itemfields': 'track title artist album',
# Silently ignore any changes to these fields.
'ignore_fields': 'id path',
})
self.register_listener('before_choose_candidate',
self.before_choose_candidate_listener)
def commands(self):
edit_command = ui.Subcommand(
'edit',
help=u'interactively edit metadata'
)
edit_command.parser.add_option(
u'-f', u'--field',
metavar='FIELD',
action='append',
help=u'edit this field also',
)
edit_command.parser.add_option(
u'--all',
action='store_true', dest='all',
help=u'edit all fields',
)
edit_command.parser.add_album_option()
edit_command.func = self._edit_command
return [edit_command]
def _edit_command(self, lib, opts, args):
"""The CLI command function for the `beet edit` command.
"""
# Get the objects to edit.
query = ui.decargs(args)
items, albums = _do_query(lib, query, opts.album, False)
objs = albums if opts.album else items
if not objs:
ui.print_(u'Nothing to edit.')
return
# Get the fields to edit.
if opts.all:
fields = None
else:
fields = self._get_fields(opts.album, opts.field)
self.edit(opts.album, objs, fields)
def _get_fields(self, album, extra):
"""Get the set of fields to edit.
"""
# Start with the configured base fields.
if album:
fields = self.config['albumfields'].as_str_seq()
else:
fields = self.config['itemfields'].as_str_seq()
# Add the requested extra fields.
if extra:
fields += extra
# Ensure we always have the `id` field for identification.
fields.append('id')
return set(fields)
def edit(self, album, objs, fields):
"""The core editor function.
- `album`: A flag indicating whether we're editing Items or Albums.
- `objs`: The `Item`s or `Album`s to edit.
- `fields`: The set of field names to edit (or None to edit
everything).
"""
# Present the YAML to the user and let her change it.
success = self.edit_objects(objs, fields)
# Save the new data.
if success:
self.save_changes(objs)
def edit_objects(self, objs, fields):
"""Dump a set of Model objects to a file as text, ask the user
to edit it, and apply any changes to the objects.
Return a boolean indicating whether the edit succeeded.
"""
# Get the content to edit as raw data structures.
old_data = [flatten(o, fields) for o in objs]
# Set up a temporary file with the initial data for editing.
new = NamedTemporaryFile(suffix='.yaml', delete=False)
old_str = dump(old_data)
new.write(old_str)
new.close()
# Loop until we have parseable data and the user confirms.
try:
while True:
# Ask the user to edit the data.
edit(new.name, self._log)
# Read the data back after editing and check whether anything
# changed.
with open(new.name) as f:
new_str = f.read()
if new_str == old_str:
ui.print_(u"No changes; aborting.")
return False
# Parse the updated data.
try:
new_data = load(new_str)
except ParseError as e:
ui.print_(u"Could not read data: {}".format(e))
if ui.input_yn(u"Edit again to fix? (Y/n)", True):
continue
else:
return False
# Show the changes.
# If the objects are not on the DB yet, we need a copy of their
# original state for show_model_changes.
objs_old = [deepcopy(obj) if not obj._db else None
for obj in objs]
self.apply_data(objs, old_data, new_data)
changed = False
for obj, obj_old in zip(objs, objs_old):
changed |= ui.show_model_changes(obj, obj_old)
if not changed:
ui.print_(u'No changes to apply.')
return False
# Confirm the changes.
choice = ui.input_options(
(u'continue Editing', u'apply', u'cancel')
)
if choice == u'a': # Apply.
return True
elif choice == u'c': # Cancel.
return False
elif choice == u'e': # Keep editing.
# Reset the temporary changes to the objects.
for obj in objs:
obj.read()
continue
# Remove the temporary file before returning.
finally:
os.remove(new.name)
def apply_data(self, objs, old_data, new_data):
"""Take potentially-updated data and apply it to a set of Model
objects.
The objects are not written back to the database, so the changes
are temporary.
"""
if len(old_data) != len(new_data):
self._log.warn(u'number of objects changed from {} to {}',
len(old_data), len(new_data))
obj_by_id = {o.id: o for o in objs}
ignore_fields = self.config['ignore_fields'].as_str_seq()
for old_dict, new_dict in zip(old_data, new_data):
# Prohibit any changes to forbidden fields to avoid
# clobbering `id` and such by mistake.
forbidden = False
for key in ignore_fields:
if old_dict.get(key) != new_dict.get(key):
self._log.warn(u'ignoring object whose {} changed', key)
forbidden = True
break
if forbidden:
continue
id_ = int(old_dict['id'])
apply_(obj_by_id[id_], new_dict)
def save_changes(self, objs):
"""Save a list of updated Model objects to the database.
"""
# Save to the database and possibly write tags.
for ob in objs:
if ob._dirty:
self._log.debug(u'saving changes to {}', ob)
ob.try_sync(ui.should_write(), ui.should_move())
# Methods for interactive importer execution.
def before_choose_candidate_listener(self, session, task):
"""Append an "Edit" choice and an "edit Candidates" choice (if
there are candidates) to the interactive importer prompt.
"""
choices = [PromptChoice('d', 'eDit', self.importer_edit)]
if task.candidates:
choices.append(PromptChoice('c', 'edit Candidates',
self.importer_edit_candidate))
return choices
def importer_edit(self, session, task):
"""Callback for invoking the functionality during an interactive
import session on the *original* item tags.
"""
# Assign temporary ids to the Items.
for i, obj in enumerate(task.items):
obj.id = i + 1
# Present the YAML to the user and let her change it.
fields = self._get_fields(album=False, extra=[])
success = self.edit_objects(task.items, fields)
# Remove temporary ids.
for obj in task.items:
obj.id = None
# Save the new data.
if success:
# Return action.RETAG, which makes the importer write the tags
# to the files if needed without re-applying metadata.
return action.RETAG
else:
# Edit cancelled / no edits made. Revert changes.
for obj in task.items:
obj.read()
def importer_edit_candidate(self, session, task):
"""Callback for invoking the functionality during an interactive
import session on a *candidate*. The candidate's metadata is
applied to the original items.
"""
# Prompt the user for a candidate.
sel = ui.input_options([], numrange=(1, len(task.candidates)))
# Force applying the candidate on the items.
task.match = task.candidates[sel - 1]
task.apply_metadata()
return self.importer_edit(session, task)

154
libs/beetsplug/embedart.py Normal file
View file

@ -0,0 +1,154 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Allows beets to embed album art into file metadata."""
from __future__ import division, absolute_import, print_function
import os.path
from beets.plugins import BeetsPlugin
from beets import ui
from beets.ui import decargs
from beets.util import syspath, normpath, displayable_path, bytestring_path
from beets.util.artresizer import ArtResizer
from beets import config
from beets import art
class EmbedCoverArtPlugin(BeetsPlugin):
"""Allows albumart to be embedded into the actual files.
"""
def __init__(self):
super(EmbedCoverArtPlugin, self).__init__()
self.config.add({
'maxwidth': 0,
'auto': True,
'compare_threshold': 0,
'ifempty': False,
'remove_art_file': False
})
if self.config['maxwidth'].get(int) and not ArtResizer.shared.local:
self.config['maxwidth'] = 0
self._log.warning(u"ImageMagick or PIL not found; "
u"'maxwidth' option ignored")
if self.config['compare_threshold'].get(int) and not \
ArtResizer.shared.can_compare:
self.config['compare_threshold'] = 0
self._log.warning(u"ImageMagick 6.8.7 or higher not installed; "
u"'compare_threshold' option ignored")
self.register_listener('art_set', self.process_album)
def commands(self):
# Embed command.
embed_cmd = ui.Subcommand(
'embedart', help=u'embed image files into file metadata'
)
embed_cmd.parser.add_option(
u'-f', u'--file', metavar='PATH', help=u'the image file to embed'
)
maxwidth = self.config['maxwidth'].get(int)
compare_threshold = self.config['compare_threshold'].get(int)
ifempty = self.config['ifempty'].get(bool)
def embed_func(lib, opts, args):
if opts.file:
imagepath = normpath(opts.file)
if not os.path.isfile(syspath(imagepath)):
raise ui.UserError(u'image file {0} not found'.format(
displayable_path(imagepath)
))
for item in lib.items(decargs(args)):
art.embed_item(self._log, item, imagepath, maxwidth, None,
compare_threshold, ifempty)
else:
for album in lib.albums(decargs(args)):
art.embed_album(self._log, album, maxwidth, False,
compare_threshold, ifempty)
self.remove_artfile(album)
embed_cmd.func = embed_func
# Extract command.
extract_cmd = ui.Subcommand(
'extractart',
help=u'extract an image from file metadata',
)
extract_cmd.parser.add_option(
u'-o', dest='outpath',
help=u'image output file',
)
extract_cmd.parser.add_option(
u'-n', dest='filename',
help=u'image filename to create for all matched albums',
)
extract_cmd.parser.add_option(
'-a', dest='associate', action='store_true',
help='associate the extracted images with the album',
)
def extract_func(lib, opts, args):
if opts.outpath:
art.extract_first(self._log, normpath(opts.outpath),
lib.items(decargs(args)))
else:
filename = bytestring_path(opts.filename or
config['art_filename'].get())
if os.path.dirname(filename) != '':
self._log.error(
u"Only specify a name rather than a path for -n")
return
for album in lib.albums(decargs(args)):
artpath = normpath(os.path.join(album.path, filename))
artpath = art.extract_first(self._log, artpath,
album.items())
if artpath and opts.associate:
album.set_art(artpath)
album.store()
extract_cmd.func = extract_func
# Clear command.
clear_cmd = ui.Subcommand(
'clearart',
help=u'remove images from file metadata',
)
def clear_func(lib, opts, args):
art.clear(self._log, lib, decargs(args))
clear_cmd.func = clear_func
return [embed_cmd, extract_cmd, clear_cmd]
def process_album(self, album):
"""Automatically embed art after art has been set
"""
if self.config['auto'] and ui.should_write():
max_width = self.config['maxwidth'].get(int)
art.embed_album(self._log, album, max_width, True,
self.config['compare_threshold'].get(int),
self.config['ifempty'].get(bool))
self.remove_artfile(album)
def remove_artfile(self, album):
"""Possibly delete the album art file for an album (if the
appropriate configuration option is enabled.
"""
if self.config['remove_art_file'] and album.artpath:
if os.path.isfile(album.artpath):
self._log.debug(u'Removing album art file for {0}', album)
os.remove(album.artpath)
album.artpath = None
album.store()

135
libs/beetsplug/embyupdate.py Normal file
View file

@ -0,0 +1,135 @@
# -*- coding: utf-8 -*-
"""Updates the Emby Library whenever the beets library is changed.
emby:
host: localhost
port: 8096
username: user
password: password
"""
from __future__ import division, absolute_import, print_function
from beets import config
from beets.plugins import BeetsPlugin
from urllib import urlencode
from urlparse import urljoin, parse_qs, urlsplit, urlunsplit
import hashlib
import requests
def api_url(host, port, endpoint):
"""Returns a joined url.
"""
joined = urljoin('http://{0}:{1}'.format(host, port), endpoint)
scheme, netloc, path, query_string, fragment = urlsplit(joined)
query_params = parse_qs(query_string)
query_params['format'] = ['json']
new_query_string = urlencode(query_params, doseq=True)
return urlunsplit((scheme, netloc, path, new_query_string, fragment))
def password_data(username, password):
"""Returns a dict with username and its encoded password.
"""
return {
'username': username,
'password': hashlib.sha1(password).hexdigest(),
'passwordMd5': hashlib.md5(password).hexdigest()
}
def create_headers(user_id, token=None):
"""Return header dict that is needed to talk to the Emby API.
"""
headers = {
'Authorization': 'MediaBrowser',
'UserId': user_id,
'Client': 'other',
'Device': 'empy',
'DeviceId': 'beets',
'Version': '0.0.0'
}
if token:
headers['X-MediaBrowser-Token'] = token
return headers
def get_token(host, port, headers, auth_data):
"""Return token for a user.
"""
url = api_url(host, port, '/Users/AuthenticateByName')
r = requests.post(url, headers=headers, data=auth_data)
return r.json().get('AccessToken')
def get_user(host, port, username):
"""Return user dict from server or None if there is no user.
"""
url = api_url(host, port, '/Users/Public')
r = requests.get(url)
user = [i for i in r.json() if i['Name'] == username]
return user
class EmbyUpdate(BeetsPlugin):
def __init__(self):
super(EmbyUpdate, self).__init__()
# Adding defaults.
config['emby'].add({
u'host': u'localhost',
u'port': 8096
})
self.register_listener('database_change', self.listen_for_db_change)
def listen_for_db_change(self, lib, model):
"""Listens for beets db change and register the update for the end.
"""
self.register_listener('cli_exit', self.update)
def update(self, lib):
"""When the client exists try to send refresh request to Emby.
"""
self._log.info(u'Updating Emby library...')
host = config['emby']['host'].get()
port = config['emby']['port'].get()
username = config['emby']['username'].get()
password = config['emby']['password'].get()
# Get user information from the Emby API.
user = get_user(host, port, username)
if not user:
self._log.warning(u'User {0} could not be found.'.format(username))
return
# Create Authentication data and headers.
auth_data = password_data(username, password)
headers = create_headers(user[0]['Id'])
# Get authentication token.
token = get_token(host, port, headers, auth_data)
if not token:
self._log.warning(
u'Could not get token for user {0}', username
)
return
# Recreate headers with a token.
headers = create_headers(user[0]['Id'], token=token)
# Trigger the Update.
url = api_url(host, port, '/Library/Refresh')
r = requests.post(url, headers=headers)
if r.status_code != 204:
self._log.warning(u'Update could not be triggered')
else:
self._log.info(u'Update triggered.')

151
libs/beetsplug/export.py Normal file
View file

@ -0,0 +1,151 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Exports data from beets
"""
from __future__ import division, absolute_import, print_function
import sys
import json
import codecs
from datetime import datetime, date
from beets.plugins import BeetsPlugin
from beets import ui
from beets import mediafile
from beetsplug.info import make_key_filter, library_data, tag_data
class ExportEncoder(json.JSONEncoder):
"""Deals with dates because JSON doesn't have a standard"""
def default(self, o):
if isinstance(o, datetime) or isinstance(o, date):
return o.isoformat()
return json.JSONEncoder.default(self, o)
class ExportPlugin(BeetsPlugin):
def __init__(self):
super(ExportPlugin, self).__init__()
self.config.add({
'default_format': 'json',
'json': {
# json module formatting options
'formatting': {
'ensure_ascii': False,
'indent': 4,
'separators': (',', ': '),
'sort_keys': True
}
},
# TODO: Use something like the edit plugin
# 'item_fields': []
})
def commands(self):
# TODO: Add option to use albums
cmd = ui.Subcommand('export', help=u'export data from beets')
cmd.func = self.run
cmd.parser.add_option(
u'-l', u'--library', action='store_true',
help=u'show library fields instead of tags',
)
cmd.parser.add_option(
u'--append', action='store_true', default=False,
help=u'if should append data to the file',
)
cmd.parser.add_option(
u'-i', u'--include-keys', default=[],
action='append', dest='included_keys',
help=u'comma separated list of keys to show',
)
cmd.parser.add_option(
u'-o', u'--output',
help=u'path for the output file. If not given, will print the data'
)
return [cmd]
def run(self, lib, opts, args):
file_path = opts.output
file_format = self.config['default_format'].get(str)
file_mode = 'a' if opts.append else 'w'
format_options = self.config[file_format]['formatting'].get(dict)
export_format = ExportFormat.factory(
file_format, **{
'file_path': file_path,
'file_mode': file_mode
}
)
items = []
data_collector = library_data if opts.library else tag_data
included_keys = []
for keys in opts.included_keys:
included_keys.extend(keys.split(','))
key_filter = make_key_filter(included_keys)
for data_emitter in data_collector(lib, ui.decargs(args)):
try:
data, item = data_emitter()
except (mediafile.UnreadableFileError, IOError) as ex:
self._log.error(u'cannot read file: {0}', ex)
continue
data = key_filter(data)
items += [data]
export_format.export(items, **format_options)
class ExportFormat(object):
"""The output format type"""
@classmethod
def factory(cls, type, **kwargs):
if type == "json":
if kwargs['file_path']:
return JsonFileFormat(**kwargs)
else:
return JsonPrintFormat()
raise NotImplementedError()
def export(self, data, **kwargs):
raise NotImplementedError()
class JsonPrintFormat(ExportFormat):
"""Outputs to the console"""
def export(self, data, **kwargs):
json.dump(data, sys.stdout, cls=ExportEncoder, **kwargs)
class JsonFileFormat(ExportFormat):
"""Saves in a json file"""
def __init__(self, file_path, file_mode=u'w', encoding=u'utf-8'):
self.path = file_path
self.mode = file_mode
self.encoding = encoding
def export(self, data, **kwargs):
with codecs.open(self.path, self.mode, self.encoding) as f:
json.dump(data, f, cls=ExportEncoder, **kwargs)

861
libs/beetsplug/fetchart.py Normal file
View file

@ -0,0 +1,861 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Fetches album art.
"""
from __future__ import division, absolute_import, print_function
from contextlib import closing
import os
import re
from tempfile import NamedTemporaryFile
import requests
from beets import plugins
from beets import importer
from beets import ui
from beets import util
from beets import config
from beets.util.artresizer import ArtResizer
from beets.util import confit
try:
import itunes
HAVE_ITUNES = True
except ImportError:
HAVE_ITUNES = False
IMAGE_EXTENSIONS = ['png', 'jpg', 'jpeg']
CONTENT_TYPES = ('image/jpeg', 'image/png')
DOWNLOAD_EXTENSION = '.jpg'
class Candidate(object):
"""Holds information about a matching artwork, deals with validation of
dimension restrictions and resizing.
"""
CANDIDATE_BAD = 0
CANDIDATE_EXACT = 1
CANDIDATE_DOWNSCALE = 2
MATCH_EXACT = 0
MATCH_FALLBACK = 1
def __init__(self, log, path=None, url=None, source=u'',
match=None, size=None):
self._log = log
self.path = path
self.url = url
self.source = source
self.check = None
self.match = match
self.size = size
def _validate(self, extra):
"""Determine whether the candidate artwork is valid based on
its dimensions (width and ratio).
Return `CANDIDATE_BAD` if the file is unusable.
Return `CANDIDATE_EXACT` if the file is usable as-is.
Return `CANDIDATE_DOWNSCALE` if the file must be resized.
"""
if not self.path:
return self.CANDIDATE_BAD
if not (extra['enforce_ratio'] or
extra['minwidth'] or
extra['maxwidth']):
return self.CANDIDATE_EXACT
# get_size returns None if no local imaging backend is available
if not self.size:
self.size = ArtResizer.shared.get_size(self.path)
self._log.debug(u'image size: {}', self.size)
if not self.size:
self._log.warning(u'Could not get size of image (please see '
u'documentation for dependencies). '
u'The configuration options `minwidth` and '
u'`enforce_ratio` may be violated.')
return self.CANDIDATE_EXACT
short_edge = min(self.size)
long_edge = max(self.size)
# Check minimum size.
if extra['minwidth'] and self.size[0] < extra['minwidth']:
self._log.debug(u'image too small ({} < {})',
self.size[0], extra['minwidth'])
return self.CANDIDATE_BAD
# Check aspect ratio.
edge_diff = long_edge - short_edge
if extra['enforce_ratio']:
if extra['margin_px']:
if edge_diff > extra['margin_px']:
self._log.debug(u'image is not close enough to being '
u'square, ({} - {} > {})',
long_edge, short_edge, extra['margin_px'])
return self.CANDIDATE_BAD
elif extra['margin_percent']:
margin_px = extra['margin_percent'] * long_edge
if edge_diff > margin_px:
self._log.debug(u'image is not close enough to being '
u'square, ({} - {} > {})',
long_edge, short_edge, margin_px)
return self.CANDIDATE_BAD
elif edge_diff:
# also reached for margin_px == 0 and margin_percent == 0.0
self._log.debug(u'image is not square ({} != {})',
self.size[0], self.size[1])
return self.CANDIDATE_BAD
# Check maximum size.
if extra['maxwidth'] and self.size[0] > extra['maxwidth']:
self._log.debug(u'image needs resizing ({} > {})',
self.size[0], extra['maxwidth'])
return self.CANDIDATE_DOWNSCALE
return self.CANDIDATE_EXACT
def validate(self, extra):
self.check = self._validate(extra)
return self.check
def resize(self, extra):
if extra['maxwidth'] and self.check == self.CANDIDATE_DOWNSCALE:
self.path = ArtResizer.shared.resize(extra['maxwidth'], self.path)
def _logged_get(log, *args, **kwargs):
"""Like `requests.get`, but logs the effective URL to the specified
`log` at the `DEBUG` level.
Use the optional `message` parameter to specify what to log before
the URL. By default, the string is "getting URL".
Also sets the User-Agent header to indicate beets.
"""
# Use some arguments with the `send` call but most with the
# `Request` construction. This is a cheap, magic-filled way to
# emulate `requests.get` or, more pertinently,
# `requests.Session.request`.
req_kwargs = kwargs
send_kwargs = {}
for arg in ('stream', 'verify', 'proxies', 'cert', 'timeout'):
if arg in kwargs:
send_kwargs[arg] = req_kwargs.pop(arg)
# Our special logging message parameter.
if 'message' in kwargs:
message = kwargs.pop('message')
else:
message = 'getting URL'
req = requests.Request('GET', *args, **req_kwargs)
with requests.Session() as s:
s.headers = {'User-Agent': 'beets'}
prepped = s.prepare_request(req)
log.debug('{}: {}', message, prepped.url)
return s.send(prepped, **send_kwargs)
class RequestMixin(object):
"""Adds a Requests wrapper to the class that uses the logger, which
must be named `self._log`.
"""
def request(self, *args, **kwargs):
"""Like `requests.get`, but uses the logger `self._log`.
See also `_logged_get`.
"""
return _logged_get(self._log, *args, **kwargs)
# ART SOURCES ################################################################
class ArtSource(RequestMixin):
def __init__(self, log, config):
self._log = log
self._config = config
def get(self, album, extra):
raise NotImplementedError()
def _candidate(self, **kwargs):
return Candidate(source=self, log=self._log, **kwargs)
def fetch_image(self, candidate, extra):
raise NotImplementedError()
class LocalArtSource(ArtSource):
IS_LOCAL = True
LOC_STR = u'local'
def fetch_image(self, candidate, extra):
pass
class RemoteArtSource(ArtSource):
IS_LOCAL = False
LOC_STR = u'remote'
def fetch_image(self, candidate, extra):
"""Downloads an image from a URL and checks whether it seems to
actually be an image. If so, returns a path to the downloaded image.
Otherwise, returns None.
"""
if extra['maxwidth']:
candidate.url = ArtResizer.shared.proxy_url(extra['maxwidth'],
candidate.url)
try:
with closing(self.request(candidate.url, stream=True,
message=u'downloading image')) as resp:
if 'Content-Type' not in resp.headers \
or resp.headers['Content-Type'] not in CONTENT_TYPES:
self._log.debug(
u'not a supported image: {}',
resp.headers.get('Content-Type') or u'no content type',
)
candidate.path = None
return
# Generate a temporary file with the correct extension.
with NamedTemporaryFile(suffix=DOWNLOAD_EXTENSION,
delete=False) as fh:
for chunk in resp.iter_content(chunk_size=1024):
fh.write(chunk)
self._log.debug(u'downloaded art to: {0}',
util.displayable_path(fh.name))
candidate.path = fh.name
return
except (IOError, requests.RequestException, TypeError) as exc:
# Handling TypeError works around a urllib3 bug:
# https://github.com/shazow/urllib3/issues/556
self._log.debug(u'error fetching art: {}', exc)
candidate.path = None
return
class CoverArtArchive(RemoteArtSource):
NAME = u"Cover Art Archive"
URL = 'http://coverartarchive.org/release/{mbid}/front'
GROUP_URL = 'http://coverartarchive.org/release-group/{mbid}/front'
def get(self, album, extra):
"""Return the Cover Art Archive and Cover Art Archive release group URLs
using album MusicBrainz release ID and release group ID.
"""
if album.mb_albumid:
yield self._candidate(url=self.URL.format(mbid=album.mb_albumid),
match=Candidate.MATCH_EXACT)
if album.mb_releasegroupid:
yield self._candidate(
url=self.GROUP_URL.format(mbid=album.mb_releasegroupid),
match=Candidate.MATCH_FALLBACK)
class Amazon(RemoteArtSource):
NAME = u"Amazon"
URL = 'http://images.amazon.com/images/P/%s.%02i.LZZZZZZZ.jpg'
INDICES = (1, 2)
def get(self, album, extra):
"""Generate URLs using Amazon ID (ASIN) string.
"""
if album.asin:
for index in self.INDICES:
yield self._candidate(url=self.URL % (album.asin, index),
match=Candidate.MATCH_EXACT)
class AlbumArtOrg(RemoteArtSource):
NAME = u"AlbumArt.org scraper"
URL = 'http://www.albumart.org/index_detail.php'
PAT = r'href\s*=\s*"([^>"]*)"[^>]*title\s*=\s*"View larger image"'
def get(self, album, extra):
"""Return art URL from AlbumArt.org using album ASIN.
"""
if not album.asin:
return
# Get the page from albumart.org.
try:
resp = self.request(self.URL, params={'asin': album.asin})
self._log.debug(u'scraped art URL: {0}', resp.url)
except requests.RequestException:
self._log.debug(u'error scraping art page')
return
# Search the page for the image URL.
m = re.search(self.PAT, resp.text)
if m:
image_url = m.group(1)
yield self._candidate(url=image_url, match=Candidate.MATCH_EXACT)
else:
self._log.debug(u'no image found on page')
class GoogleImages(RemoteArtSource):
NAME = u"Google Images"
URL = u'https://www.googleapis.com/customsearch/v1'
def __init__(self, *args, **kwargs):
super(GoogleImages, self).__init__(*args, **kwargs)
self.key = self._config['google_key'].get(),
self.cx = self._config['google_engine'].get(),
def get(self, album, extra):
"""Return art URL from google custom search engine
given an album title and interpreter.
"""
if not (album.albumartist and album.album):
return
search_string = (album.albumartist + ',' + album.album).encode('utf-8')
response = self.request(self.URL, params={
'key': self.key,
'cx': self.cx,
'q': search_string,
'searchType': 'image'
})
# Get results using JSON.
try:
data = response.json()
except ValueError:
self._log.debug(u'google: error loading response: {}'
.format(response.text))
return
if 'error' in data:
reason = data['error']['errors'][0]['reason']
self._log.debug(u'google fetchart error: {0}', reason)
return
if 'items' in data.keys():
for item in data['items']:
yield self._candidate(url=item['link'],
match=Candidate.MATCH_EXACT)
class FanartTV(RemoteArtSource):
"""Art from fanart.tv requested using their API"""
NAME = u"fanart.tv"
API_URL = 'http://webservice.fanart.tv/v3/'
API_ALBUMS = API_URL + 'music/albums/'
PROJECT_KEY = '61a7d0ab4e67162b7a0c7c35915cd48e'
def __init__(self, *args, **kwargs):
super(FanartTV, self).__init__(*args, **kwargs)
self.client_key = self._config['fanarttv_key'].get()
def get(self, album, extra):
if not album.mb_releasegroupid:
return
response = self.request(
self.API_ALBUMS + album.mb_releasegroupid,
headers={'api-key': self.PROJECT_KEY,
'client-key': self.client_key})
try:
data = response.json()
except ValueError:
self._log.debug(u'fanart.tv: error loading response: {}',
response.text)
return
if u'status' in data and data[u'status'] == u'error':
if u'not found' in data[u'error message'].lower():
self._log.debug(u'fanart.tv: no image found')
elif u'api key' in data[u'error message'].lower():
self._log.warning(u'fanart.tv: Invalid API key given, please '
u'enter a valid one in your config file.')
else:
self._log.debug(u'fanart.tv: error on request: {}',
data[u'error message'])
return
matches = []
# can there be more than one releasegroupid per response?
for mbid, art in data.get(u'albums', dict()).items():
# there might be more art referenced, e.g. cdart, and an albumcover
# might not be present, even if the request was succesful
if album.mb_releasegroupid == mbid and u'albumcover' in art:
matches.extend(art[u'albumcover'])
# can this actually occur?
else:
self._log.debug(u'fanart.tv: unexpected mb_releasegroupid in '
u'response!')
matches.sort(key=lambda x: x[u'likes'], reverse=True)
for item in matches:
# fanart.tv has a strict size requirement for album art to be
# uploaded
yield self._candidate(url=item[u'url'],
match=Candidate.MATCH_EXACT,
size=(1000, 1000))
class ITunesStore(RemoteArtSource):
NAME = u"iTunes Store"
def get(self, album, extra):
"""Return art URL from iTunes Store given an album title.
"""
if not (album.albumartist and album.album):
return
search_string = (album.albumartist + ' ' + album.album).encode('utf-8')
try:
# Isolate bugs in the iTunes library while searching.
try:
results = itunes.search_album(search_string)
except Exception as exc:
self._log.debug(u'iTunes search failed: {0}', exc)
return
# Get the first match.
if results:
itunes_album = results[0]
else:
self._log.debug(u'iTunes search for {:r} got no results',
search_string)
return
if itunes_album.get_artwork()['100']:
small_url = itunes_album.get_artwork()['100']
big_url = small_url.replace('100x100', '1200x1200')
yield self._candidate(url=big_url, match=Candidate.MATCH_EXACT)
else:
self._log.debug(u'album has no artwork in iTunes Store')
except IndexError:
self._log.debug(u'album not found in iTunes Store')
class Wikipedia(RemoteArtSource):
NAME = u"Wikipedia (queried through DBpedia)"
DBPEDIA_URL = 'http://dbpedia.org/sparql'
WIKIPEDIA_URL = 'http://en.wikipedia.org/w/api.php'
SPARQL_QUERY = u'''PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX dbpprop: <http://dbpedia.org/property/>
PREFIX owl: <http://dbpedia.org/ontology/>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
SELECT DISTINCT ?pageId ?coverFilename WHERE {{
?subject owl:wikiPageID ?pageId .
?subject dbpprop:name ?name .
?subject rdfs:label ?label .
{{ ?subject dbpprop:artist ?artist }}
UNION
{{ ?subject owl:artist ?artist }}
{{ ?artist foaf:name "{artist}"@en }}
UNION
{{ ?artist dbpprop:name "{artist}"@en }}
?subject rdf:type <http://dbpedia.org/ontology/Album> .
?subject dbpprop:cover ?coverFilename .
FILTER ( regex(?name, "{album}", "i") )
}}
Limit 1'''
def get(self, album, extra):
if not (album.albumartist and album.album):
return
# Find the name of the cover art filename on DBpedia
cover_filename, page_id = None, None
dbpedia_response = self.request(
self.DBPEDIA_URL,
params={
'format': 'application/sparql-results+json',
'timeout': 2500,
'query': self.SPARQL_QUERY.format(
artist=album.albumartist.title(), album=album.album)
},
headers={'content-type': 'application/json'},
)
try:
data = dbpedia_response.json()
results = data['results']['bindings']
if results:
cover_filename = 'File:' + results[0]['coverFilename']['value']
page_id = results[0]['pageId']['value']
else:
self._log.debug(u'wikipedia: album not found on dbpedia')
except (ValueError, KeyError, IndexError):
self._log.debug(u'wikipedia: error scraping dbpedia response: {}',
dbpedia_response.text)
# Ensure we have a filename before attempting to query wikipedia
if not (cover_filename and page_id):
return
# DBPedia sometimes provides an incomplete cover_filename, indicated
# by the filename having a space before the extension, e.g., 'foo .bar'
# An additional Wikipedia call can help to find the real filename.
# This may be removed once the DBPedia issue is resolved, see:
# https://github.com/dbpedia/extraction-framework/issues/396
if ' .' in cover_filename and \
'.' not in cover_filename.split(' .')[-1]:
self._log.debug(
u'wikipedia: dbpedia provided incomplete cover_filename'
)
lpart, rpart = cover_filename.rsplit(' .', 1)
# Query all the images in the page
wikipedia_response = self.request(
self.WIKIPEDIA_URL,
params={
'format': 'json',
'action': 'query',
'continue': '',
'prop': 'images',
'pageids': page_id,
},
headers={'content-type': 'application/json'},
)
# Try to see if one of the images on the pages matches our
# imcomplete cover_filename
try:
data = wikipedia_response.json()
results = data['query']['pages'][page_id]['images']
for result in results:
if re.match(re.escape(lpart) + r'.*?\.' + re.escape(rpart),
result['title']):
cover_filename = result['title']
break
except (ValueError, KeyError):
self._log.debug(
u'wikipedia: failed to retrieve a cover_filename'
)
return
# Find the absolute url of the cover art on Wikipedia
wikipedia_response = self.request(
self.WIKIPEDIA_URL,
params={
'format': 'json',
'action': 'query',
'continue': '',
'prop': 'imageinfo',
'iiprop': 'url',
'titles': cover_filename.encode('utf-8'),
},
headers={'content-type': 'application/json'},
)
try:
data = wikipedia_response.json()
results = data['query']['pages']
for _, result in results.iteritems():
image_url = result['imageinfo'][0]['url']
yield self._candidate(url=image_url,
match=Candidate.MATCH_EXACT)
except (ValueError, KeyError, IndexError):
self._log.debug(u'wikipedia: error scraping imageinfo')
return
class FileSystem(LocalArtSource):
NAME = u"Filesystem"
@staticmethod
def filename_priority(filename, cover_names):
"""Sort order for image names.
Return indexes of cover names found in the image filename. This
means that images with lower-numbered and more keywords will have
higher priority.
"""
return [idx for (idx, x) in enumerate(cover_names) if x in filename]
def get(self, album, extra):
"""Look for album art files in the specified directories.
"""
paths = extra['paths']
if not paths:
return
cover_names = extra['cover_names']
cover_pat = br"(\b|_)({0})(\b|_)".format(b'|'.join(cover_names))
cautious = extra['cautious']
for path in paths:
if not os.path.isdir(path):
continue
# Find all files that look like images in the directory.
images = []
for fn in os.listdir(path):
for ext in IMAGE_EXTENSIONS:
if fn.lower().endswith(b'.' + ext.encode('utf8')) and \
os.path.isfile(os.path.join(path, fn)):
images.append(fn)
# Look for "preferred" filenames.
images = sorted(images,
key=lambda x:
self.filename_priority(x, cover_names))
remaining = []
for fn in images:
if re.search(cover_pat, os.path.splitext(fn)[0], re.I):
self._log.debug(u'using well-named art file {0}',
util.displayable_path(fn))
yield self._candidate(path=os.path.join(path, fn),
match=Candidate.MATCH_EXACT)
else:
remaining.append(fn)
# Fall back to any image in the folder.
if remaining and not cautious:
self._log.debug(u'using fallback art file {0}',
util.displayable_path(remaining[0]))
yield self._candidate(path=os.path.join(path, remaining[0]),
match=Candidate.MATCH_FALLBACK)
# Try each source in turn.
SOURCES_ALL = [u'filesystem',
u'coverart', u'itunes', u'amazon', u'albumart',
u'wikipedia', u'google', u'fanarttv']
ART_SOURCES = {
u'filesystem': FileSystem,
u'coverart': CoverArtArchive,
u'itunes': ITunesStore,
u'albumart': AlbumArtOrg,
u'amazon': Amazon,
u'wikipedia': Wikipedia,
u'google': GoogleImages,
u'fanarttv': FanartTV,
}
SOURCE_NAMES = {v: k for k, v in ART_SOURCES.items()}
# PLUGIN LOGIC ###############################################################
class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin):
PAT_PX = r"(0|[1-9][0-9]*)px"
PAT_PERCENT = r"(100(\.00?)?|[1-9]?[0-9](\.[0-9]{1,2})?)%"
def __init__(self):
super(FetchArtPlugin, self).__init__()
# Holds candidates corresponding to downloaded images between
# fetching them and placing them in the filesystem.
self.art_candidates = {}
self.config.add({
'auto': True,
'minwidth': 0,
'maxwidth': 0,
'enforce_ratio': False,
'cautious': False,
'cover_names': ['cover', 'front', 'art', 'album', 'folder'],
'sources': ['filesystem',
'coverart', 'itunes', 'amazon', 'albumart'],
'google_key': None,
'google_engine': u'001442825323518660753:hrh5ch1gjzm',
'fanarttv_key': None,
'store_source': False,
})
self.config['google_key'].redact = True
self.config['fanarttv_key'].redact = True
self.minwidth = self.config['minwidth'].get(int)
self.maxwidth = self.config['maxwidth'].get(int)
# allow both pixel and percentage-based margin specifications
self.enforce_ratio = self.config['enforce_ratio'].get(
confit.OneOf([bool,
confit.String(pattern=self.PAT_PX),
confit.String(pattern=self.PAT_PERCENT)]))
self.margin_px = None
self.margin_percent = None
if type(self.enforce_ratio) is unicode:
if self.enforce_ratio[-1] == u'%':
self.margin_percent = float(self.enforce_ratio[:-1]) / 100
elif self.enforce_ratio[-2:] == u'px':
self.margin_px = int(self.enforce_ratio[:-2])
else:
# shouldn't happen
raise confit.ConfigValueError()
self.enforce_ratio = True
cover_names = self.config['cover_names'].as_str_seq()
self.cover_names = map(util.bytestring_path, cover_names)
self.cautious = self.config['cautious'].get(bool)
self.store_source = self.config['store_source'].get(bool)
self.src_removed = (config['import']['delete'].get(bool) or
config['import']['move'].get(bool))
if self.config['auto']:
# Enable two import hooks when fetching is enabled.
self.import_stages = [self.fetch_art]
self.register_listener('import_task_files', self.assign_art)
available_sources = list(SOURCES_ALL)
if not HAVE_ITUNES and u'itunes' in available_sources:
available_sources.remove(u'itunes')
if not self.config['google_key'].get() and \
u'google' in available_sources:
available_sources.remove(u'google')
sources_name = plugins.sanitize_choices(
self.config['sources'].as_str_seq(), available_sources)
if 'remote_priority' in self.config:
self._log.warning(
u'The `fetch_art.remote_priority` configuration option has '
u'been deprecated, see the documentation.')
if self.config['remote_priority'].get(bool):
try:
sources_name.remove(u'filesystem')
sources_name.append(u'filesystem')
except ValueError:
pass
self.sources = [ART_SOURCES[s](self._log, self.config)
for s in sources_name]
# Asynchronous; after music is added to the library.
def fetch_art(self, session, task):
"""Find art for the album being imported."""
if task.is_album: # Only fetch art for full albums.
if task.album.artpath and os.path.isfile(task.album.artpath):
# Album already has art (probably a re-import); skip it.
return
if task.choice_flag == importer.action.ASIS:
# For as-is imports, don't search Web sources for art.
local = True
elif task.choice_flag == importer.action.APPLY:
# Search everywhere for art.
local = False
else:
# For any other choices (e.g., TRACKS), do nothing.
return
candidate = self.art_for_album(task.album, task.paths, local)
if candidate:
self.art_candidates[task] = candidate
def _set_art(self, album, candidate, delete=False):
album.set_art(candidate.path, delete)
if self.store_source:
# store the source of the chosen artwork in a flexible field
self._log.debug(
u"Storing art_source for {0.albumartist} - {0.album}",
album)
album.art_source = SOURCE_NAMES[type(candidate.source)]
album.store()
# Synchronous; after music files are put in place.
def assign_art(self, session, task):
"""Place the discovered art in the filesystem."""
if task in self.art_candidates:
candidate = self.art_candidates.pop(task)
self._set_art(task.album, candidate, not self.src_removed)
if self.src_removed:
task.prune(candidate.path)
# Manual album art fetching.
def commands(self):
cmd = ui.Subcommand('fetchart', help='download album art')
cmd.parser.add_option(
u'-f', u'--force', dest='force',
action='store_true', default=False,
help=u're-download art when already present'
)
def func(lib, opts, args):
self.batch_fetch_art(lib, lib.albums(ui.decargs(args)), opts.force)
cmd.func = func
return [cmd]
# Utilities converted from functions to methods on logging overhaul
def art_for_album(self, album, paths, local_only=False):
"""Given an Album object, returns a path to downloaded art for the
album (or None if no art is found). If `maxwidth`, then images are
resized to this maximum pixel size. If `local_only`, then only local
image files from the filesystem are returned; no network requests
are made.
"""
out = None
# all the information any of the sources might need
extra = {'paths': paths,
'cover_names': self.cover_names,
'cautious': self.cautious,
'enforce_ratio': self.enforce_ratio,
'margin_px': self.margin_px,
'margin_percent': self.margin_percent,
'minwidth': self.minwidth,
'maxwidth': self.maxwidth}
for source in self.sources:
if source.IS_LOCAL or not local_only:
self._log.debug(
u'trying source {0} for album {1.albumartist} - {1.album}',
SOURCE_NAMES[type(source)],
album,
)
# URLs might be invalid at this point, or the image may not
# fulfill the requirements
for candidate in source.get(album, extra):
source.fetch_image(candidate, extra)
if candidate.validate(extra):
out = candidate
self._log.debug(
u'using {0.LOC_STR} image {1}'.format(
source, util.displayable_path(out.path)))
break
if out:
break
if out:
out.resize(extra)
return out
def batch_fetch_art(self, lib, albums, force):
"""Fetch album art for each of the albums. This implements the manual
fetchart CLI command.
"""
for album in albums:
if album.artpath and not force and os.path.isfile(album.artpath):
message = ui.colorize('text_highlight_minor', u'has album art')
else:
# In ordinary invocations, look for images on the
# filesystem. When forcing, however, always go to the Web
# sources.
local_paths = None if force else [album.path]
candidate = self.art_for_album(album, local_paths)
if candidate:
self._set_art(album, candidate)
message = ui.colorize('text_success', u'found album art')
else:
message = ui.colorize('text_error', u'no art found')
self._log.info(u'{0}: {1}', album, message)

78
libs/beetsplug/filefilter.py Normal file
View file

@ -0,0 +1,78 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016, Malte Ried.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Filter imported files using a regular expression.
"""
from __future__ import division, absolute_import, print_function
import re
from beets import config
from beets.plugins import BeetsPlugin
from beets.importer import SingletonImportTask
class FileFilterPlugin(BeetsPlugin):
def __init__(self):
super(FileFilterPlugin, self).__init__()
self.register_listener('import_task_created',
self.import_task_created_event)
self.config.add({
'path': '.*'
})
self.path_album_regex = \
self.path_singleton_regex = \
re.compile(self.config['path'].get())
if 'album_path' in self.config:
self.path_album_regex = re.compile(self.config['album_path'].get())
if 'singleton_path' in self.config:
self.path_singleton_regex = re.compile(
self.config['singleton_path'].get())
def import_task_created_event(self, session, task):
if task.items and len(task.items) > 0:
items_to_import = []
for item in task.items:
if self.file_filter(item['path']):
items_to_import.append(item)
if len(items_to_import) > 0:
task.items = items_to_import
else:
# Returning an empty list of tasks from the handler
# drops the task from the rest of the importer pipeline.
return []
elif isinstance(task, SingletonImportTask):
if not self.file_filter(task.item['path']):
return []
# If not filtered, return the original task unchanged.
return [task]
def file_filter(self, full_path):
"""Checks if the configured regular expressions allow the import
of the file given in full_path.
"""
import_config = dict(config['import'])
if 'singletons' not in import_config or not import_config[
'singletons']:
# Album
return self.path_album_regex.match(full_path) is not None
else:
# Singleton
return self.path_singleton_regex.match(full_path) is not None

37
libs/beetsplug/freedesktop.py Normal file
View file

@ -0,0 +1,37 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016, Matt Lichtenberg.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Creates freedesktop.org-compliant .directory files on an album level.
"""
from __future__ import division, absolute_import, print_function
from beets.plugins import BeetsPlugin
from beets import ui
class FreedesktopPlugin(BeetsPlugin):
def commands(self):
deprecated = ui.Subcommand(
"freedesktop",
help=u"Print a message to redirect to thumbnails --dolphin")
deprecated.func = self.deprecation_message
return [deprecated]
def deprecation_message(self, lib, opts, args):
ui.print_(u"This plugin is deprecated. Its functionality is "
u"superseded by the 'thumbnails' plugin")
ui.print_(u"'thumbnails --dolphin' replaces freedesktop. See doc & "
u"changelog for more information")

173
libs/beetsplug/fromfilename.py Normal file
View file

@ -0,0 +1,173 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016, Jan-Erik Dahlin
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""If the title is empty, try to extract track and title from the
filename.
"""
from __future__ import division, absolute_import, print_function
from beets import plugins
from beets.util import displayable_path
import os
import re
# Filename field extraction patterns.
PATTERNS = [
# "01 - Track 01" and "01": do nothing
r'^(\d+)\s*-\s*track\s*\d$',
r'^\d+$',
# Useful patterns.
r'^(?P<artist>.+)-(?P<title>.+)-(?P<tag>.*)$',
r'^(?P<track>\d+)\s*-(?P<artist>.+)-(?P<title>.+)-(?P<tag>.*)$',
r'^(?P<track>\d+)\s(?P<artist>.+)-(?P<title>.+)-(?P<tag>.*)$',
r'^(?P<artist>.+)-(?P<title>.+)$',
r'^(?P<track>\d+)\.\s*(?P<artist>.+)-(?P<title>.+)$',
r'^(?P<track>\d+)\s*-\s*(?P<artist>.+)-(?P<title>.+)$',
r'^(?P<track>\d+)\s*-(?P<artist>.+)-(?P<title>.+)$',
r'^(?P<track>\d+)\s(?P<artist>.+)-(?P<title>.+)$',
r'^(?P<title>.+)$',
r'^(?P<track>\d+)\.\s*(?P<title>.+)$',
r'^(?P<track>\d+)\s*-\s*(?P<title>.+)$',
r'^(?P<track>\d+)\s(?P<title>.+)$',
r'^(?P<title>.+) by (?P<artist>.+)$',
]
# Titles considered "empty" and in need of replacement.
BAD_TITLE_PATTERNS = [
r'^$',
r'\d+?\s?-?\s*track\s*\d+',
]
def equal(seq):
"""Determine whether a sequence holds identical elements.
"""
return len(set(seq)) <= 1
def equal_fields(matchdict, field):
"""Do all items in `matchdict`, whose values are dictionaries, have
the same value for `field`? (If they do, the field is probably not
the title.)
"""
return equal(m[field] for m in matchdict.values())
def all_matches(names, pattern):
"""If all the filenames in the item/filename mapping match the
pattern, return a dictionary mapping the items to dictionaries
giving the value for each named subpattern in the match. Otherwise,
return None.
"""
matches = {}
for item, name in names.items():
m = re.match(pattern, name, re.IGNORECASE)
if m and m.groupdict():
# Only yield a match when the regex applies *and* has
# capture groups. Otherwise, no information can be extracted
# from the filename.
matches[item] = m.groupdict()
else:
return None
return matches
def bad_title(title):
"""Determine whether a given title is "bad" (empty or otherwise
meaningless) and in need of replacement.
"""
for pat in BAD_TITLE_PATTERNS:
if re.match(pat, title, re.IGNORECASE):
return True
return False
def apply_matches(d):
"""Given a mapping from items to field dicts, apply the fields to
the objects.
"""
some_map = d.values()[0]
keys = some_map.keys()
# Only proceed if the "tag" field is equal across all filenames.
if 'tag' in keys and not equal_fields(d, 'tag'):
return
# Given both an "artist" and "title" field, assume that one is
# *actually* the artist, which must be uniform, and use the other
# for the title. This, of course, won't work for VA albums.
if 'artist' in keys:
if equal_fields(d, 'artist'):
artist = some_map['artist']
title_field = 'title'
elif equal_fields(d, 'title'):
artist = some_map['title']
title_field = 'artist'
else:
# Both vary. Abort.
return
for item in d:
if not item.artist:
item.artist = artist
# No artist field: remaining field is the title.
else:
title_field = 'title'
# Apply the title and track.
for item in d:
if bad_title(item.title):
item.title = unicode(d[item][title_field])
if 'track' in d[item] and item.track == 0:
item.track = int(d[item]['track'])
# Plugin structure and hook into import process.
class FromFilenamePlugin(plugins.BeetsPlugin):
def __init__(self):
super(FromFilenamePlugin, self).__init__()
self.register_listener('import_task_start', filename_task)
def filename_task(task, session):
"""Examine each item in the task to see if we can extract a title
from the filename. Try to match all filenames to a number of
regexps, starting with the most complex patterns and successively
trying less complex patterns. As soon as all filenames match the
same regex we can make an educated guess of which part of the
regex that contains the title.
"""
items = task.items if task.is_album else [task.item]
# Look for suspicious (empty or meaningless) titles.
missing_titles = sum(bad_title(i.title) for i in items)
if missing_titles:
# Get the base filenames (no path or extension).
names = {}
for item in items:
path = displayable_path(item.path)
name, _ = os.path.splitext(os.path.basename(path))
names[item] = name
# Look for useful information in the filenames.
for pattern in PATTERNS:
d = all_matches(names, pattern)
if d:
apply_matches(d)

169
libs/beetsplug/ftintitle.py Normal file
View file

@ -0,0 +1,169 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016, Verrus, <github.com/Verrus/beets-plugin-featInTitle>
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Moves "featured" artists to the title from the artist field.
"""
from __future__ import division, absolute_import, print_function
import re
from beets import plugins
from beets import ui
from beets.util import displayable_path
def split_on_feat(artist):
"""Given an artist string, split the "main" artist from any artist
on the right-hand side of a string like "feat". Return the main
artist, which is always a string, and the featuring artist, which
may be a string or None if none is present.
"""
# split on the first "feat".
regex = re.compile(plugins.feat_tokens(), re.IGNORECASE)
parts = [s.strip() for s in regex.split(artist, 1)]
if len(parts) == 1:
return parts[0], None
else:
return tuple(parts)
def contains_feat(title):
"""Determine whether the title contains a "featured" marker.
"""
return bool(re.search(plugins.feat_tokens(), title, flags=re.IGNORECASE))
def find_feat_part(artist, albumartist):
"""Attempt to find featured artists in the item's artist fields and
return the results. Returns None if no featured artist found.
"""
feat_part = None
# Look for the album artist in the artist field. If it's not
# present, give up.
albumartist_split = artist.split(albumartist, 1)
if len(albumartist_split) <= 1:
return feat_part
# If the last element of the split (the right-hand side of the
# album artist) is nonempty, then it probably contains the
# featured artist.
elif albumartist_split[-1] != '':
# Extract the featured artist from the right-hand side.
_, feat_part = split_on_feat(albumartist_split[-1])
# Otherwise, if there's nothing on the right-hand side, look for a
# featuring artist on the left-hand side.
else:
lhs, rhs = split_on_feat(albumartist_split[0])
if lhs:
feat_part = lhs
return feat_part
class FtInTitlePlugin(plugins.BeetsPlugin):
def __init__(self):
super(FtInTitlePlugin, self).__init__()
self.config.add({
'auto': True,
'drop': False,
'format': u'feat. {0}',
})
self._command = ui.Subcommand(
'ftintitle',
help=u'move featured artists to the title field')
self._command.parser.add_option(
u'-d', u'--drop', dest='drop',
action='store_true', default=False,
help=u'drop featuring from artists and ignore title update')
if self.config['auto']:
self.import_stages = [self.imported]
def commands(self):
def func(lib, opts, args):
self.config.set_args(opts)
drop_feat = self.config['drop'].get(bool)
write = ui.should_write()
for item in lib.items(ui.decargs(args)):
self.ft_in_title(item, drop_feat)
item.store()
if write:
item.try_write()
self._command.func = func
return [self._command]
def imported(self, session, task):
"""Import hook for moving featuring artist automatically.
"""
drop_feat = self.config['drop'].get(bool)
for item in task.imported_items():
self.ft_in_title(item, drop_feat)
item.store()
def update_metadata(self, item, feat_part, drop_feat):
"""Choose how to add new artists to the title and set the new
metadata. Also, print out messages about any changes that are made.
If `drop_feat` is set, then do not add the artist to the title; just
remove it from the artist field.
"""
# In all cases, update the artist fields.
self._log.info(u'artist: {0} -> {1}', item.artist, item.albumartist)
item.artist = item.albumartist
if item.artist_sort:
# Just strip the featured artist from the sort name.
item.artist_sort, _ = split_on_feat(item.artist_sort)
# Only update the title if it does not already contain a featured
# artist and if we do not drop featuring information.
if not drop_feat and not contains_feat(item.title):
feat_format = self.config['format'].get(unicode)
new_format = feat_format.format(feat_part)
new_title = u"{0} {1}".format(item.title, new_format)
self._log.info(u'title: {0} -> {1}', item.title, new_title)
item.title = new_title
def ft_in_title(self, item, drop_feat):
"""Look for featured artists in the item's artist fields and move
them to the title.
"""
artist = item.artist.strip()
albumartist = item.albumartist.strip()
# Check whether there is a featured artist on this track and the
# artist field does not exactly match the album artist field. In
# that case, we attempt to move the featured artist to the title.
_, featured = split_on_feat(artist)
if featured and albumartist != artist and albumartist:
self._log.info('{}', displayable_path(item.path))
feat_part = None
# Attempt to find the featured artist.
feat_part = find_feat_part(artist, albumartist)
# If we have a featuring artist, move it to the title.
if feat_part:
self.update_metadata(item, feat_part, drop_feat)
else:
self._log.info(u'no featuring artists found')

48
libs/beetsplug/fuzzy.py Normal file
View file

@ -0,0 +1,48 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016, Philippe Mongeau.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Provides a fuzzy matching query.
"""
from __future__ import division, absolute_import, print_function
from beets.plugins import BeetsPlugin
from beets.dbcore.query import StringFieldQuery
from beets import config
import difflib
class FuzzyQuery(StringFieldQuery):
@classmethod
def string_match(cls, pattern, val):
# smartcase
if pattern.islower():
val = val.lower()
query_matcher = difflib.SequenceMatcher(None, pattern, val)
threshold = config['fuzzy']['threshold'].as_number()
return query_matcher.quick_ratio() >= threshold
class FuzzyPlugin(BeetsPlugin):
def __init__(self):
super(FuzzyPlugin, self).__init__()
self.config.add({
'prefix': '~',
'threshold': 0.7,
})
def queries(self):
prefix = self.config['prefix'].get(basestring)
return {prefix: FuzzyQuery}

108
libs/beetsplug/hook.py Normal file
View file

@ -0,0 +1,108 @@
# This file is part of beets.
# Copyright 2015, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Allows custom commands to be run when an event is emitted by beets"""
from __future__ import division, absolute_import, print_function
import string
import subprocess
from beets.plugins import BeetsPlugin
from beets.ui import _arg_encoding
from beets.util import shlex_split
class CodingFormatter(string.Formatter):
"""A custom string formatter that decodes the format string and it's
fields.
"""
def __init__(self, coding):
"""Creates a new coding formatter with the provided coding."""
self._coding = coding
def format(self, format_string, *args, **kwargs):
"""Formats the provided string using the provided arguments and keyword
arguments.
This method decodes the format string using the formatter's coding.
See str.format and string.Formatter.format.
"""
try:
format_string = format_string.decode(self._coding)
except UnicodeEncodeError:
pass
return super(CodingFormatter, self).format(format_string, *args,
**kwargs)
def convert_field(self, value, conversion):
"""Converts the provided value given a conversion type.
This method decodes the converted value using the formatter's coding.
See string.Formatter.convert_field.
"""
converted = super(CodingFormatter, self).convert_field(value,
conversion)
try:
converted = converted.decode(self._coding)
except UnicodeEncodeError:
pass
return converted
class HookPlugin(BeetsPlugin):
"""Allows custom commands to be run when an event is emitted by beets"""
def __init__(self):
super(HookPlugin, self).__init__()
self.config.add({
'hooks': []
})
hooks = self.config['hooks'].get(list)
for hook_index in range(len(hooks)):
hook = self.config['hooks'][hook_index]
hook_event = hook['event'].get(unicode)
hook_command = hook['command'].get(unicode)
self.create_and_register_hook(hook_event, hook_command)
def create_and_register_hook(self, event, command):
def hook_function(**kwargs):
if command is None or len(command) == 0:
self._log.error('invalid command "{0}"', command)
return
formatter = CodingFormatter(_arg_encoding())
command_pieces = shlex_split(command)
for i, piece in enumerate(command_pieces):
command_pieces[i] = formatter.format(piece, event=event,
**kwargs)
self._log.debug(u'running command "{0}" for event {1}',
u' '.join(command_pieces), event)
try:
subprocess.Popen(command_pieces).wait()
except OSError as exc:
self._log.error(u'hook for {0} failed: {1}', event, exc)
self.register_listener(event, hook_function)

82
libs/beetsplug/ihate.py Normal file
View file

@ -0,0 +1,82 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016, Blemjhoo Tezoulbr <baobab@heresiarch.info>.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
from __future__ import division, absolute_import, print_function
"""Warns you about things you hate (or even blocks import)."""
from beets.plugins import BeetsPlugin
from beets.importer import action
from beets.library import parse_query_string
from beets.library import Item
from beets.library import Album
__author__ = 'baobab@heresiarch.info'
__version__ = '2.0'
def summary(task):
"""Given an ImportTask, produce a short string identifying the
object.
"""
if task.is_album:
return u'{0} - {1}'.format(task.cur_artist, task.cur_album)
else:
return u'{0} - {1}'.format(task.item.artist, task.item.title)
class IHatePlugin(BeetsPlugin):
def __init__(self):
super(IHatePlugin, self).__init__()
self.register_listener('import_task_choice',
self.import_task_choice_event)
self.config.add({
'warn': [],
'skip': [],
})
@classmethod
def do_i_hate_this(cls, task, action_patterns):
"""Process group of patterns (warn or skip) and returns True if
task is hated and not whitelisted.
"""
if action_patterns:
for query_string in action_patterns:
query, _ = parse_query_string(
query_string,
Album if task.is_album else Item,
)
if any(query.match(item) for item in task.imported_items()):
return True
return False
def import_task_choice_event(self, session, task):
skip_queries = self.config['skip'].as_str_seq()
warn_queries = self.config['warn'].as_str_seq()
if task.choice_flag == action.APPLY:
if skip_queries or warn_queries:
self._log.debug(u'processing your hate')
if self.do_i_hate_this(task, skip_queries):
task.choice_flag = action.SKIP
self._log.info(u'skipped: {0}', summary(task))
return
if self.do_i_hate_this(task, warn_queries):
self._log.info(u'you may hate this: {0}', summary(task))
else:
self._log.debug(u'nothing to do')
else:
self._log.debug(u'user made a decision, nothing to do')

Some files were not shown because too many files have changed in this diff Show more