github/nzbToMedia
1
0
Fork 0
mirror of https://github.com/clinton-hall/nzbToMedia.git synced 2025-07-06 05:01:10 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #1907 from clinton-hall/vendor

Browse source 
Update vendored libraries
This commit is contained in:
Clinton Hall 2022-12-03 13:38:44 +13:00 • committed by GitHub
commit 18ac3575ba
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1512 changed files with 132579 additions and 47313 deletions
Download patch file Download diff file Expand all files Collapse all files

3
.gitignore vendored
View file

@ -1,8 +1,7 @@
*.cfg *.cfg
!.bumpversion.cfg !.bumpversion.cfg
*.cfg.old *.cfg.old
*.pyc *.py[cod]
*.pyo
*.log *.log
*.pid *.pid
*.db *.db

BIN
libs/common/_yaml.cp37-win32.pyd
View file

Binary file not shown.

33
libs/common/_yaml/__init__.py Normal file
View file

@ -0,0 +1,33 @@
# This is a stub package designed to roughly emulate the _yaml
# extension module, which previously existed as a standalone module
# and has been moved into the `yaml` package namespace.
# It does not perfectly mimic its old counterpart, but should get
# close enough for anyone who's relying on it even when they shouldn't.
import yaml
# in some circumstances, the yaml module we imoprted may be from a different version, so we need
# to tread carefully when poking at it here (it may not have the attributes we expect)
if not getattr(yaml, '__with_libyaml__', False):
from sys import version_info
exc = ModuleNotFoundError if version_info >= (3, 6) else ImportError
raise exc("No module named '_yaml'")
else:
from yaml._yaml import *
import warnings
warnings.warn(
'The _yaml extension module is now located at yaml._yaml'
' and its location is subject to change. To use the'
' LibYAML-based parser and emitter, import from `yaml`:'
' `from yaml import CLoader as Loader, CDumper as Dumper`.',
DeprecationWarning
)
del warnings
# Don't `del yaml` here because yaml is actually an existing
# namespace member of _yaml.
__name__ = '_yaml'
# If the module is top-level (i.e. not a part of any specific package)
# then the attribute should be set to ''.
# https://docs.python.org/3.8/library/types.html
__package__ = ''

4
libs/common/appdirs.py
View file

@ -13,8 +13,8 @@ See <http://github.com/ActiveState/appdirs> for details and usage.
# - Mac OS X: http://developer.apple.com/documentation/MacOSX/Conceptual/BPFileSystem/index.html # - Mac OS X: http://developer.apple.com/documentation/MacOSX/Conceptual/BPFileSystem/index.html
# - XDG spec for Un*x: http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html # - XDG spec for Un*x: http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html
__version_info__ = (1, 4, 3) __version__ = "1.4.4"
__version__ = '.'.join(map(str, __version_info__)) __version_info__ = tuple(int(segment) for segment in __version__.split("."))
import sys import sys

26
libs/common/beets/__init__.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson. # Copyright 2016, Adrian Sampson.
# #
@ -13,30 +12,29 @@
# The above copyright notice and this permission notice shall be # The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software. # included in all copies or substantial portions of the Software.
from __future__ import division, absolute_import, print_function
import os import confuse
from sys import stderr
from beets.util import confit __version__ = '1.6.0'
__author__ = 'Adrian Sampson <adrian@radbox.org>'
__version__ = u'1.4.7'
__author__ = u'Adrian Sampson <adrian@radbox.org>'
class IncludeLazyConfig(confit.LazyConfig): class IncludeLazyConfig(confuse.LazyConfig):
"""A version of Confit's LazyConfig that also merges in data from """A version of Confuse's LazyConfig that also merges in data from
YAML files specified in an `include` setting. YAML files specified in an `include` setting.
""" """
def read(self, user=True, defaults=True): def read(self, user=True, defaults=True):
super(IncludeLazyConfig, self).read(user, defaults) super().read(user, defaults)
try: try:
for view in self['include']: for view in self['include']:
filename = view.as_filename() self.set_file(view.as_filename())
if os.path.isfile(filename): except confuse.NotFoundError:
self.set_file(filename)
except confit.NotFoundError:
pass pass
except confuse.ConfigReadError as err:
stderr.write("configuration `import` failed: {}"
.format(err.reason))
config = IncludeLazyConfig('beets', __name__) config = IncludeLazyConfig('beets', __name__)

2
libs/common/beets/__main__.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2017, Adrian Sampson. # Copyright 2017, Adrian Sampson.
# #
@ -17,7 +16,6 @@
`python -m beets`. `python -m beets`.
""" """
from __future__ import division, absolute_import, print_function
import sys import sys
from .ui import main from .ui import main

72
libs/common/beets/art.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson. # Copyright 2016, Adrian Sampson.
# #
@ -17,7 +16,6 @@
music and items' embedded album art. music and items' embedded album art.
""" """
from __future__ import division, absolute_import, print_function
import subprocess import subprocess
import platform import platform
@ -26,7 +24,7 @@ import os
from beets.util import displayable_path, syspath, bytestring_path from beets.util import displayable_path, syspath, bytestring_path
from beets.util.artresizer import ArtResizer from beets.util.artresizer import ArtResizer
from beets import mediafile import mediafile
def mediafile_image(image_path, maxwidth=None): def mediafile_image(image_path, maxwidth=None):
@ -43,7 +41,7 @@ def get_art(log, item):
try: try:
mf = mediafile.MediaFile(syspath(item.path)) mf = mediafile.MediaFile(syspath(item.path))
except mediafile.UnreadableFileError as exc: except mediafile.UnreadableFileError as exc:
log.warning(u'Could not extract art from {0}: {1}', log.warning('Could not extract art from {0}: {1}',
displayable_path(item.path), exc) displayable_path(item.path), exc)
return return
@ -51,26 +49,27 @@ def get_art(log, item):
def embed_item(log, item, imagepath, maxwidth=None, itempath=None, def embed_item(log, item, imagepath, maxwidth=None, itempath=None,
compare_threshold=0, ifempty=False, as_album=False): compare_threshold=0, ifempty=False, as_album=False, id3v23=None,
quality=0):
"""Embed an image into the item's media file. """Embed an image into the item's media file.
""" """
# Conditions and filters. # Conditions and filters.
if compare_threshold: if compare_threshold:
if not check_art_similarity(log, item, imagepath, compare_threshold): if not check_art_similarity(log, item, imagepath, compare_threshold):
log.info(u'Image not similar; skipping.') log.info('Image not similar; skipping.')
return return
if ifempty and get_art(log, item): if ifempty and get_art(log, item):
log.info(u'media file already contained art') log.info('media file already contained art')
return return
if maxwidth and not as_album: if maxwidth and not as_album:
imagepath = resize_image(log, imagepath, maxwidth) imagepath = resize_image(log, imagepath, maxwidth, quality)
# Get the `Image` object from the file. # Get the `Image` object from the file.
try: try:
log.debug(u'embedding {0}', displayable_path(imagepath)) log.debug('embedding {0}', displayable_path(imagepath))
image = mediafile_image(imagepath, maxwidth) image = mediafile_image(imagepath, maxwidth)
except IOError as exc: except OSError as exc:
log.warning(u'could not read image file: {0}', exc) log.warning('could not read image file: {0}', exc)
return return
# Make sure the image kind is safe (some formats only support PNG # Make sure the image kind is safe (some formats only support PNG
@ -80,36 +79,39 @@ def embed_item(log, item, imagepath, maxwidth=None, itempath=None,
image.mime_type) image.mime_type)
return return
item.try_write(path=itempath, tags={'images': [image]}) item.try_write(path=itempath, tags={'images': [image]}, id3v23=id3v23)
def embed_album(log, album, maxwidth=None, quiet=False, def embed_album(log, album, maxwidth=None, quiet=False, compare_threshold=0,
compare_threshold=0, ifempty=False): ifempty=False, quality=0):
"""Embed album art into all of the album's items. """Embed album art into all of the album's items.
""" """
imagepath = album.artpath imagepath = album.artpath
if not imagepath: if not imagepath:
log.info(u'No album art present for {0}', album) log.info('No album art present for {0}', album)
return return
if not os.path.isfile(syspath(imagepath)): if not os.path.isfile(syspath(imagepath)):
log.info(u'Album art not found at {0} for {1}', log.info('Album art not found at {0} for {1}',
displayable_path(imagepath), album) displayable_path(imagepath), album)
return return
if maxwidth: if maxwidth:
imagepath = resize_image(log, imagepath, maxwidth) imagepath = resize_image(log, imagepath, maxwidth, quality)
log.info(u'Embedding album art into {0}', album) log.info('Embedding album art into {0}', album)
for item in album.items(): for item in album.items():
embed_item(log, item, imagepath, maxwidth, None, embed_item(log, item, imagepath, maxwidth, None, compare_threshold,
compare_threshold, ifempty, as_album=True) ifempty, as_album=True, quality=quality)
def resize_image(log, imagepath, maxwidth): def resize_image(log, imagepath, maxwidth, quality):
"""Returns path to an image resized to maxwidth. """Returns path to an image resized to maxwidth and encoded with the
specified quality level.
""" """
log.debug(u'Resizing album art to {0} pixels wide', maxwidth) log.debug('Resizing album art to {0} pixels wide and encoding at quality \
imagepath = ArtResizer.shared.resize(maxwidth, syspath(imagepath)) level {1}', maxwidth, quality)
imagepath = ArtResizer.shared.resize(maxwidth, syspath(imagepath),
quality=quality)
return imagepath return imagepath
@ -131,7 +133,7 @@ def check_art_similarity(log, item, imagepath, compare_threshold):
syspath(art, prefix=False), syspath(art, prefix=False),
'-colorspace', 'gray', 'MIFF:-'] '-colorspace', 'gray', 'MIFF:-']
compare_cmd = ['compare', '-metric', 'PHASH', '-', 'null:'] compare_cmd = ['compare', '-metric', 'PHASH', '-', 'null:']
log.debug(u'comparing images with pipeline {} | {}', log.debug('comparing images with pipeline {} | {}',
convert_cmd, compare_cmd) convert_cmd, compare_cmd)
convert_proc = subprocess.Popen( convert_proc = subprocess.Popen(
convert_cmd, convert_cmd,
@ -155,7 +157,7 @@ def check_art_similarity(log, item, imagepath, compare_threshold):
convert_proc.wait() convert_proc.wait()
if convert_proc.returncode: if convert_proc.returncode:
log.debug( log.debug(
u'ImageMagick convert failed with status {}: {!r}', 'ImageMagick convert failed with status {}: {!r}',
convert_proc.returncode, convert_proc.returncode,
convert_stderr, convert_stderr,
) )
@ -165,7 +167,7 @@ def check_art_similarity(log, item, imagepath, compare_threshold):
stdout, stderr = compare_proc.communicate() stdout, stderr = compare_proc.communicate()
if compare_proc.returncode: if compare_proc.returncode:
if compare_proc.returncode != 1: if compare_proc.returncode != 1:
log.debug(u'ImageMagick compare failed: {0}, {1}', log.debug('ImageMagick compare failed: {0}, {1}',
displayable_path(imagepath), displayable_path(imagepath),
displayable_path(art)) displayable_path(art))
return return
@ -176,10 +178,10 @@ def check_art_similarity(log, item, imagepath, compare_threshold):
try: try:
phash_diff = float(out_str) phash_diff = float(out_str)
except ValueError: except ValueError:
log.debug(u'IM output is not a number: {0!r}', out_str) log.debug('IM output is not a number: {0!r}', out_str)
return return
log.debug(u'ImageMagick compare score: {0}', phash_diff) log.debug('ImageMagick compare score: {0}', phash_diff)
return phash_diff <= compare_threshold return phash_diff <= compare_threshold
return True return True
@ -189,18 +191,18 @@ def extract(log, outpath, item):
art = get_art(log, item) art = get_art(log, item)
outpath = bytestring_path(outpath) outpath = bytestring_path(outpath)
if not art: if not art:
log.info(u'No album art present in {0}, skipping.', item) log.info('No album art present in {0}, skipping.', item)
return return
# Add an extension to the filename. # Add an extension to the filename.
ext = mediafile.image_extension(art) ext = mediafile.image_extension(art)
if not ext: if not ext:
log.warning(u'Unknown image type in {0}.', log.warning('Unknown image type in {0}.',
displayable_path(item.path)) displayable_path(item.path))
return return
outpath += bytestring_path('.' + ext) outpath += bytestring_path('.' + ext)
log.info(u'Extracting album art from: {0} to: {1}', log.info('Extracting album art from: {0} to: {1}',
item, displayable_path(outpath)) item, displayable_path(outpath))
with open(syspath(outpath), 'wb') as f: with open(syspath(outpath), 'wb') as f:
f.write(art) f.write(art)
@ -216,7 +218,7 @@ def extract_first(log, outpath, items):
def clear(log, lib, query): def clear(log, lib, query):
items = lib.items(query) items = lib.items(query)
log.info(u'Clearing album art from {0} items', len(items)) log.info('Clearing album art from {0} items', len(items))
for item in items: for item in items:
log.debug(u'Clearing art for {0}', item) log.debug('Clearing art for {0}', item)
item.try_write(tags={'images': None}) item.try_write(tags={'images': None})

111
libs/common/beets/autotag/__init__.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson. # Copyright 2016, Adrian Sampson.
# #
@ -16,19 +15,59 @@
"""Facilities for automatically determining files' correct metadata. """Facilities for automatically determining files' correct metadata.
""" """
from __future__ import division, absolute_import, print_function
from beets import logging from beets import logging
from beets import config from beets import config
# Parts of external interface. # Parts of external interface.
from .hooks import AlbumInfo, TrackInfo, AlbumMatch, TrackMatch # noqa from .hooks import ( # noqa
AlbumInfo,
TrackInfo,
AlbumMatch,
TrackMatch,
Distance,
)
from .match import tag_item, tag_album, Proposal # noqa from .match import tag_item, tag_album, Proposal # noqa
from .match import Recommendation # noqa from .match import Recommendation # noqa
# Global logger. # Global logger.
log = logging.getLogger('beets') log = logging.getLogger('beets')
# Metadata fields that are already hardcoded, or where the tag name changes.
SPECIAL_FIELDS = {
'album': (
'va',
'releasegroup_id',
'artist_id',
'album_id',
'mediums',
'tracks',
'year',
'month',
'day',
'artist',
'artist_credit',
'artist_sort',
'data_url'
),
'track': (
'track_alt',
'artist_id',
'release_track_id',
'medium',
'index',
'medium_index',
'title',
'artist_credit',
'artist_sort',
'artist',
'track_id',
'medium_total',
'data_url',
'length'
)
}
# Additional utilities for the main interface. # Additional utilities for the main interface.
@ -43,17 +82,14 @@ def apply_item_metadata(item, track_info):
item.mb_releasetrackid = track_info.release_track_id item.mb_releasetrackid = track_info.release_track_id
if track_info.artist_id: if track_info.artist_id:
item.mb_artistid = track_info.artist_id item.mb_artistid = track_info.artist_id
if track_info.data_source:
item.data_source = track_info.data_source
if track_info.lyricist is not None: for field, value in track_info.items():
item.lyricist = track_info.lyricist # We only overwrite fields that are not already hardcoded.
if track_info.composer is not None: if field in SPECIAL_FIELDS['track']:
item.composer = track_info.composer continue
if track_info.composer_sort is not None: if value is None:
item.composer_sort = track_info.composer_sort continue
if track_info.arranger is not None: item[field] = value
item.arranger = track_info.arranger
# At the moment, the other metadata is left intact (including album # At the moment, the other metadata is left intact (including album
# and track number). Perhaps these should be emptied? # and track number). Perhaps these should be emptied?
@ -142,33 +178,24 @@ def apply_metadata(album_info, mapping):
# Compilation flag. # Compilation flag.
item.comp = album_info.va item.comp = album_info.va
# Miscellaneous metadata. # Track alt.
for field in ('albumtype',
'label',
'asin',
'catalognum',
'script',
'language',
'country',
'albumstatus',
'albumdisambig',
'data_source',):
value = getattr(album_info, field)
if value is not None:
item[field] = value
if track_info.disctitle is not None:
item.disctitle = track_info.disctitle
if track_info.media is not None:
item.media = track_info.media
if track_info.lyricist is not None:
item.lyricist = track_info.lyricist
if track_info.composer is not None:
item.composer = track_info.composer
if track_info.composer_sort is not None:
item.composer_sort = track_info.composer_sort
if track_info.arranger is not None:
item.arranger = track_info.arranger
item.track_alt = track_info.track_alt item.track_alt = track_info.track_alt
# Don't overwrite fields with empty values unless the
# field is explicitly allowed to be overwritten
for field, value in album_info.items():
if field in SPECIAL_FIELDS['album']:
continue
clobber = field in config['overwrite_null']['album'].as_str_seq()
if value is None and not clobber:
continue
item[field] = value
for field, value in track_info.items():
if field in SPECIAL_FIELDS['track']:
continue
clobber = field in config['overwrite_null']['track'].as_str_seq()
value = getattr(track_info, field)
if value is None and not clobber:
continue
item[field] = value

209
libs/common/beets/autotag/hooks.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson. # Copyright 2016, Adrian Sampson.
# #
@ -14,7 +13,6 @@
# included in all copies or substantial portions of the Software. # included in all copies or substantial portions of the Software.
"""Glue between metadata sources and the matching logic.""" """Glue between metadata sources and the matching logic."""
from __future__ import division, absolute_import, print_function
from collections import namedtuple from collections import namedtuple
from functools import total_ordering from functools import total_ordering
@ -27,14 +25,36 @@ from beets.util import as_string
from beets.autotag import mb from beets.autotag import mb
from jellyfish import levenshtein_distance from jellyfish import levenshtein_distance
from unidecode import unidecode from unidecode import unidecode
import six
log = logging.getLogger('beets') log = logging.getLogger('beets')
# The name of the type for patterns in re changed in Python 3.7.
try:
Pattern = re._pattern_type
except AttributeError:
Pattern = re.Pattern
# Classes used to represent candidate options. # Classes used to represent candidate options.
class AttrDict(dict):
"""A dictionary that supports attribute ("dot") access, so `d.field`
is equivalent to `d['field']`.
"""
class AlbumInfo(object): def __getattr__(self, attr):
if attr in self:
return self.get(attr)
else:
raise AttributeError
def __setattr__(self, key, value):
self.__setitem__(key, value)
def __hash__(self):
return id(self)
class AlbumInfo(AttrDict):
"""Describes a canonical release that may be used to match a release """Describes a canonical release that may be used to match a release
in the library. Consists of these data members: in the library. Consists of these data members:
@ -43,38 +63,22 @@ class AlbumInfo(object):
- ``artist``: name of the release's primary artist - ``artist``: name of the release's primary artist
- ``artist_id`` - ``artist_id``
- ``tracks``: list of TrackInfo objects making up the release - ``tracks``: list of TrackInfo objects making up the release
- ``asin``: Amazon ASIN
- ``albumtype``: string describing the kind of release
- ``va``: boolean: whether the release has "various artists"
- ``year``: release year
- ``month``: release month
- ``day``: release day
- ``label``: music label responsible for the release
- ``mediums``: the number of discs in this release
- ``artist_sort``: name of the release's artist for sorting
- ``releasegroup_id``: MBID for the album's release group
- ``catalognum``: the label's catalog number for the release
- ``script``: character set used for metadata
- ``language``: human language of the metadata
- ``country``: the release country
- ``albumstatus``: MusicBrainz release status (Official, etc.)
- ``media``: delivery mechanism (Vinyl, etc.)
- ``albumdisambig``: MusicBrainz release disambiguation comment
- ``artist_credit``: Release-specific artist name
- ``data_source``: The original data source (MusicBrainz, Discogs, etc.)
- ``data_url``: The data source release URL.
The fields up through ``tracks`` are required. The others are ``mediums`` along with the fields up through ``tracks`` are required.
optional and may be None. The others are optional and may be None.
""" """
def __init__(self, album, album_id, artist, artist_id, tracks, asin=None,
albumtype=None, va=False, year=None, month=None, day=None, def __init__(self, tracks, album=None, album_id=None, artist=None,
label=None, mediums=None, artist_sort=None, artist_id=None, asin=None, albumtype=None, va=False,
releasegroup_id=None, catalognum=None, script=None, year=None, month=None, day=None, label=None, mediums=None,
language=None, country=None, albumstatus=None, media=None, artist_sort=None, releasegroup_id=None, catalognum=None,
albumdisambig=None, artist_credit=None, original_year=None, script=None, language=None, country=None, style=None,
original_month=None, original_day=None, data_source=None, genre=None, albumstatus=None, media=None, albumdisambig=None,
data_url=None): releasegroupdisambig=None, artist_credit=None,
original_year=None, original_month=None,
original_day=None, data_source=None, data_url=None,
discogs_albumid=None, discogs_labelid=None,
discogs_artistid=None, **kwargs):
self.album = album self.album = album
self.album_id = album_id self.album_id = album_id
self.artist = artist self.artist = artist
@ -94,15 +98,22 @@ class AlbumInfo(object):
self.script = script self.script = script
self.language = language self.language = language
self.country = country self.country = country
self.style = style
self.genre = genre
self.albumstatus = albumstatus self.albumstatus = albumstatus
self.media = media self.media = media
self.albumdisambig = albumdisambig self.albumdisambig = albumdisambig
self.releasegroupdisambig = releasegroupdisambig
self.artist_credit = artist_credit self.artist_credit = artist_credit
self.original_year = original_year self.original_year = original_year
self.original_month = original_month self.original_month = original_month
self.original_day = original_day self.original_day = original_day
self.data_source = data_source self.data_source = data_source
self.data_url = data_url self.data_url = data_url
self.discogs_albumid = discogs_albumid
self.discogs_labelid = discogs_labelid
self.discogs_artistid = discogs_artistid
self.update(kwargs)
# Work around a bug in python-musicbrainz-ngs that causes some # Work around a bug in python-musicbrainz-ngs that causes some
# strings to be bytes rather than Unicode. # strings to be bytes rather than Unicode.
@ -112,54 +123,46 @@ class AlbumInfo(object):
constituent `TrackInfo` objects, are decoded to Unicode. constituent `TrackInfo` objects, are decoded to Unicode.
""" """
for fld in ['album', 'artist', 'albumtype', 'label', 'artist_sort', for fld in ['album', 'artist', 'albumtype', 'label', 'artist_sort',
'catalognum', 'script', 'language', 'country', 'catalognum', 'script', 'language', 'country', 'style',
'albumstatus', 'albumdisambig', 'artist_credit', 'media']: 'genre', 'albumstatus', 'albumdisambig',
'releasegroupdisambig', 'artist_credit',
'media', 'discogs_albumid', 'discogs_labelid',
'discogs_artistid']:
value = getattr(self, fld) value = getattr(self, fld)
if isinstance(value, bytes): if isinstance(value, bytes):
setattr(self, fld, value.decode(codec, 'ignore')) setattr(self, fld, value.decode(codec, 'ignore'))
if self.tracks: for track in self.tracks:
for track in self.tracks: track.decode(codec)
track.decode(codec)
def copy(self):
dupe = AlbumInfo([])
dupe.update(self)
dupe.tracks = [track.copy() for track in self.tracks]
return dupe
class TrackInfo(object): class TrackInfo(AttrDict):
"""Describes a canonical track present on a release. Appears as part """Describes a canonical track present on a release. Appears as part
of an AlbumInfo's ``tracks`` list. Consists of these data members: of an AlbumInfo's ``tracks`` list. Consists of these data members:
- ``title``: name of the track - ``title``: name of the track
- ``track_id``: MusicBrainz ID; UUID fragment only - ``track_id``: MusicBrainz ID; UUID fragment only
- ``release_track_id``: MusicBrainz ID respective to a track on a
particular release; UUID fragment only
- ``artist``: individual track artist name
- ``artist_id``
- ``length``: float: duration of the track in seconds
- ``index``: position on the entire release
- ``media``: delivery mechanism (Vinyl, etc.)
- ``medium``: the disc number this track appears on in the album
- ``medium_index``: the track's position on the disc
- ``medium_total``: the number of tracks on the item's disc
- ``artist_sort``: name of the track artist for sorting
- ``disctitle``: name of the individual medium (subtitle)
- ``artist_credit``: Recording-specific artist name
- ``data_source``: The original data source (MusicBrainz, Discogs, etc.)
- ``data_url``: The data source release URL.
- ``lyricist``: individual track lyricist name
- ``composer``: individual track composer name
- ``composer_sort``: individual track composer sort name
- ``arranger`: individual track arranger name
- ``track_alt``: alternative track number (tape, vinyl, etc.)
Only ``title`` and ``track_id`` are required. The rest of the fields Only ``title`` and ``track_id`` are required. The rest of the fields
may be None. The indices ``index``, ``medium``, and ``medium_index`` may be None. The indices ``index``, ``medium``, and ``medium_index``
are all 1-based. are all 1-based.
""" """
def __init__(self, title, track_id, release_track_id=None, artist=None,
artist_id=None, length=None, index=None, medium=None, def __init__(self, title=None, track_id=None, release_track_id=None,
medium_index=None, medium_total=None, artist_sort=None, artist=None, artist_id=None, length=None, index=None,
disctitle=None, artist_credit=None, data_source=None, medium=None, medium_index=None, medium_total=None,
data_url=None, media=None, lyricist=None, composer=None, artist_sort=None, disctitle=None, artist_credit=None,
composer_sort=None, arranger=None, track_alt=None): data_source=None, data_url=None, media=None, lyricist=None,
composer=None, composer_sort=None, arranger=None,
track_alt=None, work=None, mb_workid=None,
work_disambig=None, bpm=None, initial_key=None, genre=None,
**kwargs):
self.title = title self.title = title
self.track_id = track_id self.track_id = track_id
self.release_track_id = release_track_id self.release_track_id = release_track_id
@ -181,6 +184,13 @@ class TrackInfo(object):
self.composer_sort = composer_sort self.composer_sort = composer_sort
self.arranger = arranger self.arranger = arranger
self.track_alt = track_alt self.track_alt = track_alt
self.work = work
self.mb_workid = mb_workid
self.work_disambig = work_disambig
self.bpm = bpm
self.initial_key = initial_key
self.genre = genre
self.update(kwargs)
# As above, work around a bug in python-musicbrainz-ngs. # As above, work around a bug in python-musicbrainz-ngs.
def decode(self, codec='utf-8'): def decode(self, codec='utf-8'):
@ -193,6 +203,11 @@ class TrackInfo(object):
if isinstance(value, bytes): if isinstance(value, bytes):
setattr(self, fld, value.decode(codec, 'ignore')) setattr(self, fld, value.decode(codec, 'ignore'))
def copy(self):
dupe = TrackInfo()
dupe.update(self)
return dupe
# Candidate distance scoring. # Candidate distance scoring.
@ -220,8 +235,8 @@ def _string_dist_basic(str1, str2):
transliteration/lowering to ASCII characters. Normalized by string transliteration/lowering to ASCII characters. Normalized by string
length. length.
""" """
assert isinstance(str1, six.text_type) assert isinstance(str1, str)
assert isinstance(str2, six.text_type) assert isinstance(str2, str)
str1 = as_string(unidecode(str1)) str1 = as_string(unidecode(str1))
str2 = as_string(unidecode(str2)) str2 = as_string(unidecode(str2))
str1 = re.sub(r'[^a-z0-9]', '', str1.lower()) str1 = re.sub(r'[^a-z0-9]', '', str1.lower())
@ -249,9 +264,9 @@ def string_dist(str1, str2):
# "something, the". # "something, the".
for word in SD_END_WORDS: for word in SD_END_WORDS:
if str1.endswith(', %s' % word): if str1.endswith(', %s' % word):
str1 = '%s %s' % (word, str1[:-len(word) - 2]) str1 = '{} {}'.format(word, str1[:-len(word) - 2])
if str2.endswith(', %s' % word): if str2.endswith(', %s' % word):
str2 = '%s %s' % (word, str2[:-len(word) - 2]) str2 = '{} {}'.format(word, str2[:-len(word) - 2])
# Perform a couple of basic normalizing substitutions. # Perform a couple of basic normalizing substitutions.
for pat, repl in SD_REPLACE: for pat, repl in SD_REPLACE:
@ -289,11 +304,12 @@ def string_dist(str1, str2):
return base_dist + penalty return base_dist + penalty
class LazyClassProperty(object): class LazyClassProperty:
"""A decorator implementing a read-only property that is *lazy* in """A decorator implementing a read-only property that is *lazy* in
the sense that the getter is only invoked once. Subsequent accesses the sense that the getter is only invoked once. Subsequent accesses
through *any* instance use the cached result. through *any* instance use the cached result.
""" """
def __init__(self, getter): def __init__(self, getter):
self.getter = getter self.getter = getter
self.computed = False self.computed = False
@ -306,17 +322,17 @@ class LazyClassProperty(object):
@total_ordering @total_ordering
@six.python_2_unicode_compatible class Distance:
class Distance(object):
"""Keeps track of multiple distance penalties. Provides a single """Keeps track of multiple distance penalties. Provides a single
weighted distance for all penalties as well as a weighted distance weighted distance for all penalties as well as a weighted distance
for each individual penalty. for each individual penalty.
""" """
def __init__(self): def __init__(self):
self._penalties = {} self._penalties = {}
@LazyClassProperty @LazyClassProperty
def _weights(cls): # noqa def _weights(cls): # noqa: N805
"""A dictionary from keys to floating-point weights. """A dictionary from keys to floating-point weights.
""" """
weights_view = config['match']['distance_weights'] weights_view = config['match']['distance_weights']
@ -394,7 +410,7 @@ class Distance(object):
return other - self.distance return other - self.distance
def __str__(self): def __str__(self):
return "{0:.2f}".format(self.distance) return f"{self.distance:.2f}"
# Behave like a dict. # Behave like a dict.
@ -421,7 +437,7 @@ class Distance(object):
""" """
if not isinstance(dist, Distance): if not isinstance(dist, Distance):
raise ValueError( raise ValueError(
u'`dist` must be a Distance object, not {0}'.format(type(dist)) '`dist` must be a Distance object, not {}'.format(type(dist))
) )
for key, penalties in dist._penalties.items(): for key, penalties in dist._penalties.items():
self._penalties.setdefault(key, []).extend(penalties) self._penalties.setdefault(key, []).extend(penalties)
@ -433,7 +449,7 @@ class Distance(object):
be a compiled regular expression, in which case it will be be a compiled regular expression, in which case it will be
matched against `value2`. matched against `value2`.
""" """
if isinstance(value1, re._pattern_type): if isinstance(value1, Pattern):
return bool(value1.match(value2)) return bool(value1.match(value2))
return value1 == value2 return value1 == value2
@ -445,7 +461,7 @@ class Distance(object):
""" """
if not 0.0 <= dist <= 1.0: if not 0.0 <= dist <= 1.0:
raise ValueError( raise ValueError(
u'`dist` must be between 0.0 and 1.0, not {0}'.format(dist) f'`dist` must be between 0.0 and 1.0, not {dist}'
) )
self._penalties.setdefault(key, []).append(dist) self._penalties.setdefault(key, []).append(dist)
@ -541,7 +557,7 @@ def album_for_mbid(release_id):
try: try:
album = mb.album_for_id(release_id) album = mb.album_for_id(release_id)
if album: if album:
plugins.send(u'albuminfo_received', info=album) plugins.send('albuminfo_received', info=album)
return album return album
except mb.MusicBrainzAPIError as exc: except mb.MusicBrainzAPIError as exc:
exc.log(log) exc.log(log)
@ -554,7 +570,7 @@ def track_for_mbid(recording_id):
try: try:
track = mb.track_for_id(recording_id) track = mb.track_for_id(recording_id)
if track: if track:
plugins.send(u'trackinfo_received', info=track) plugins.send('trackinfo_received', info=track)
return track return track
except mb.MusicBrainzAPIError as exc: except mb.MusicBrainzAPIError as exc:
exc.log(log) exc.log(log)
@ -567,7 +583,7 @@ def albums_for_id(album_id):
yield a yield a
for a in plugins.album_for_id(album_id): for a in plugins.album_for_id(album_id):
if a: if a:
plugins.send(u'albuminfo_received', info=a) plugins.send('albuminfo_received', info=a)
yield a yield a
@ -578,40 +594,43 @@ def tracks_for_id(track_id):
yield t yield t
for t in plugins.track_for_id(track_id): for t in plugins.track_for_id(track_id):
if t: if t:
plugins.send(u'trackinfo_received', info=t) plugins.send('trackinfo_received', info=t)
yield t yield t
@plugins.notify_info_yielded(u'albuminfo_received') @plugins.notify_info_yielded('albuminfo_received')
def album_candidates(items, artist, album, va_likely): def album_candidates(items, artist, album, va_likely, extra_tags):
"""Search for album matches. ``items`` is a list of Item objects """Search for album matches. ``items`` is a list of Item objects
that make up the album. ``artist`` and ``album`` are the respective that make up the album. ``artist`` and ``album`` are the respective
names (strings), which may be derived from the item list or may be names (strings), which may be derived from the item list or may be
entered by the user. ``va_likely`` is a boolean indicating whether entered by the user. ``va_likely`` is a boolean indicating whether
the album is likely to be a "various artists" release. the album is likely to be a "various artists" release. ``extra_tags``
is an optional dictionary of additional tags used to further
constrain the search.
""" """
# Base candidates if we have album and artist to match. # Base candidates if we have album and artist to match.
if artist and album: if artist and album:
try: try:
for candidate in mb.match_album(artist, album, len(items)): yield from mb.match_album(artist, album, len(items),
yield candidate extra_tags)
except mb.MusicBrainzAPIError as exc: except mb.MusicBrainzAPIError as exc:
exc.log(log) exc.log(log)
# Also add VA matches from MusicBrainz where appropriate. # Also add VA matches from MusicBrainz where appropriate.
if va_likely and album: if va_likely and album:
try: try:
for candidate in mb.match_album(None, album, len(items)): yield from mb.match_album(None, album, len(items),
yield candidate extra_tags)
except mb.MusicBrainzAPIError as exc: except mb.MusicBrainzAPIError as exc:
exc.log(log) exc.log(log)
# Candidates from plugins. # Candidates from plugins.
for candidate in plugins.candidates(items, artist, album, va_likely): yield from plugins.candidates(items, artist, album, va_likely,
yield candidate extra_tags)
@plugins.notify_info_yielded(u'trackinfo_received') @plugins.notify_info_yielded('trackinfo_received')
def item_candidates(item, artist, title): def item_candidates(item, artist, title):
"""Search for item matches. ``item`` is the Item to be matched. """Search for item matches. ``item`` is the Item to be matched.
``artist`` and ``title`` are strings and either reflect the item or ``artist`` and ``title`` are strings and either reflect the item or
@ -621,11 +640,9 @@ def item_candidates(item, artist, title):
# MusicBrainz candidates. # MusicBrainz candidates.
if artist and title: if artist and title:
try: try:
for candidate in mb.match_track(artist, title): yield from mb.match_track(artist, title)
yield candidate
except mb.MusicBrainzAPIError as exc: except mb.MusicBrainzAPIError as exc:
exc.log(log) exc.log(log)
# Plugin candidates. # Plugin candidates.
for candidate in plugins.item_candidates(item, artist, title): yield from plugins.item_candidates(item, artist, title)
yield candidate

55
libs/common/beets/autotag/match.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson. # Copyright 2016, Adrian Sampson.
# #
@ -17,7 +16,6 @@
releases and tracks. releases and tracks.
""" """
from __future__ import division, absolute_import, print_function
import datetime import datetime
import re import re
@ -35,7 +33,7 @@ from beets.util.enumeration import OrderedEnum
# album level to determine whether a given release is likely a VA # album level to determine whether a given release is likely a VA
# release and also on the track level to to remove the penalty for # release and also on the track level to to remove the penalty for
# differing artists. # differing artists.
VA_ARTISTS = (u'', u'various artists', u'various', u'va', u'unknown') VA_ARTISTS = ('', 'various artists', 'various', 'va', 'unknown')
# Global logger. # Global logger.
log = logging.getLogger('beets') log = logging.getLogger('beets')
@ -108,7 +106,7 @@ def assign_items(items, tracks):
log.debug('...done.') log.debug('...done.')
# Produce the output matching. # Produce the output matching.
mapping = dict((items[i], tracks[j]) for (i, j) in matching) mapping = {items[i]: tracks[j] for (i, j) in matching}
extra_items = list(set(items) - set(mapping.keys())) extra_items = list(set(items) - set(mapping.keys()))
extra_items.sort(key=lambda i: (i.disc, i.track, i.title)) extra_items.sort(key=lambda i: (i.disc, i.track, i.title))
extra_tracks = list(set(tracks) - set(mapping.values())) extra_tracks = list(set(tracks) - set(mapping.values()))
@ -276,16 +274,16 @@ def match_by_id(items):
try: try:
first = next(albumids) first = next(albumids)
except StopIteration: except StopIteration:
log.debug(u'No album ID found.') log.debug('No album ID found.')
return None return None
# Is there a consensus on the MB album ID? # Is there a consensus on the MB album ID?
for other in albumids: for other in albumids:
if other != first: if other != first:
log.debug(u'No album ID consensus.') log.debug('No album ID consensus.')
return None return None
# If all album IDs are equal, look up the album. # If all album IDs are equal, look up the album.
log.debug(u'Searching for discovered album ID: {0}', first) log.debug('Searching for discovered album ID: {0}', first)
return hooks.album_for_mbid(first) return hooks.album_for_mbid(first)
@ -351,23 +349,23 @@ def _add_candidate(items, results, info):
checking the track count, ordering the items, checking for checking the track count, ordering the items, checking for
duplicates, and calculating the distance. duplicates, and calculating the distance.
""" """
log.debug(u'Candidate: {0} - {1} ({2})', log.debug('Candidate: {0} - {1} ({2})',
info.artist, info.album, info.album_id) info.artist, info.album, info.album_id)
# Discard albums with zero tracks. # Discard albums with zero tracks.
if not info.tracks: if not info.tracks:
log.debug(u'No tracks.') log.debug('No tracks.')
return return
# Don't duplicate. # Don't duplicate.
if info.album_id in results: if info.album_id in results:
log.debug(u'Duplicate.') log.debug('Duplicate.')
return return
# Discard matches without required tags. # Discard matches without required tags.
for req_tag in config['match']['required'].as_str_seq(): for req_tag in config['match']['required'].as_str_seq():
if getattr(info, req_tag) is None: if getattr(info, req_tag) is None:
log.debug(u'Ignored. Missing required tag: {0}', req_tag) log.debug('Ignored. Missing required tag: {0}', req_tag)
return return
# Find mapping between the items and the track info. # Find mapping between the items and the track info.
@ -380,10 +378,10 @@ def _add_candidate(items, results, info):
penalties = [key for key, _ in dist] penalties = [key for key, _ in dist]
for penalty in config['match']['ignored'].as_str_seq(): for penalty in config['match']['ignored'].as_str_seq():
if penalty in penalties: if penalty in penalties:
log.debug(u'Ignored. Penalty: {0}', penalty) log.debug('Ignored. Penalty: {0}', penalty)
return return
log.debug(u'Success. Distance: {0}', dist) log.debug('Success. Distance: {0}', dist)
results[info.album_id] = hooks.AlbumMatch(dist, info, mapping, results[info.album_id] = hooks.AlbumMatch(dist, info, mapping,
extra_items, extra_tracks) extra_items, extra_tracks)
@ -411,7 +409,7 @@ def tag_album(items, search_artist=None, search_album=None,
likelies, consensus = current_metadata(items) likelies, consensus = current_metadata(items)
cur_artist = likelies['artist'] cur_artist = likelies['artist']
cur_album = likelies['album'] cur_album = likelies['album']
log.debug(u'Tagging {0} - {1}', cur_artist, cur_album) log.debug('Tagging {0} - {1}', cur_artist, cur_album)
# The output result (distance, AlbumInfo) tuples (keyed by MB album # The output result (distance, AlbumInfo) tuples (keyed by MB album
# ID). # ID).
@ -420,7 +418,7 @@ def tag_album(items, search_artist=None, search_album=None,
# Search by explicit ID. # Search by explicit ID.
if search_ids: if search_ids:
for search_id in search_ids: for search_id in search_ids:
log.debug(u'Searching for album ID: {0}', search_id) log.debug('Searching for album ID: {0}', search_id)
for id_candidate in hooks.albums_for_id(search_id): for id_candidate in hooks.albums_for_id(search_id):
_add_candidate(items, candidates, id_candidate) _add_candidate(items, candidates, id_candidate)
@ -431,13 +429,13 @@ def tag_album(items, search_artist=None, search_album=None,
if id_info: if id_info:
_add_candidate(items, candidates, id_info) _add_candidate(items, candidates, id_info)
rec = _recommendation(list(candidates.values())) rec = _recommendation(list(candidates.values()))
log.debug(u'Album ID match recommendation is {0}', rec) log.debug('Album ID match recommendation is {0}', rec)
if candidates and not config['import']['timid']: if candidates and not config['import']['timid']:
# If we have a very good MBID match, return immediately. # If we have a very good MBID match, return immediately.
# Otherwise, this match will compete against metadata-based # Otherwise, this match will compete against metadata-based
# matches. # matches.
if rec == Recommendation.strong: if rec == Recommendation.strong:
log.debug(u'ID match.') log.debug('ID match.')
return cur_artist, cur_album, \ return cur_artist, cur_album, \
Proposal(list(candidates.values()), rec) Proposal(list(candidates.values()), rec)
@ -445,22 +443,29 @@ def tag_album(items, search_artist=None, search_album=None,
if not (search_artist and search_album): if not (search_artist and search_album):
# No explicit search terms -- use current metadata. # No explicit search terms -- use current metadata.
search_artist, search_album = cur_artist, cur_album search_artist, search_album = cur_artist, cur_album
log.debug(u'Search terms: {0} - {1}', search_artist, search_album) log.debug('Search terms: {0} - {1}', search_artist, search_album)
extra_tags = None
if config['musicbrainz']['extra_tags']:
tag_list = config['musicbrainz']['extra_tags'].get()
extra_tags = {k: v for (k, v) in likelies.items() if k in tag_list}
log.debug('Additional search terms: {0}', extra_tags)
# Is this album likely to be a "various artist" release? # Is this album likely to be a "various artist" release?
va_likely = ((not consensus['artist']) or va_likely = ((not consensus['artist']) or
(search_artist.lower() in VA_ARTISTS) or (search_artist.lower() in VA_ARTISTS) or
any(item.comp for item in items)) any(item.comp for item in items))
log.debug(u'Album might be VA: {0}', va_likely) log.debug('Album might be VA: {0}', va_likely)
# Get the results from the data sources. # Get the results from the data sources.
for matched_candidate in hooks.album_candidates(items, for matched_candidate in hooks.album_candidates(items,
search_artist, search_artist,
search_album, search_album,
va_likely): va_likely,
extra_tags):
_add_candidate(items, candidates, matched_candidate) _add_candidate(items, candidates, matched_candidate)
log.debug(u'Evaluating {0} candidates.', len(candidates)) log.debug('Evaluating {0} candidates.', len(candidates))
# Sort and get the recommendation. # Sort and get the recommendation.
candidates = _sort_candidates(candidates.values()) candidates = _sort_candidates(candidates.values())
rec = _recommendation(candidates) rec = _recommendation(candidates)
@ -485,7 +490,7 @@ def tag_item(item, search_artist=None, search_title=None,
trackids = search_ids or [t for t in [item.mb_trackid] if t] trackids = search_ids or [t for t in [item.mb_trackid] if t]
if trackids: if trackids:
for trackid in trackids: for trackid in trackids:
log.debug(u'Searching for track ID: {0}', trackid) log.debug('Searching for track ID: {0}', trackid)
for track_info in hooks.tracks_for_id(trackid): for track_info in hooks.tracks_for_id(trackid):
dist = track_distance(item, track_info, incl_artist=True) dist = track_distance(item, track_info, incl_artist=True)
candidates[track_info.track_id] = \ candidates[track_info.track_id] = \
@ -494,7 +499,7 @@ def tag_item(item, search_artist=None, search_title=None,
rec = _recommendation(_sort_candidates(candidates.values())) rec = _recommendation(_sort_candidates(candidates.values()))
if rec == Recommendation.strong and \ if rec == Recommendation.strong and \
not config['import']['timid']: not config['import']['timid']:
log.debug(u'Track ID match.') log.debug('Track ID match.')
return Proposal(_sort_candidates(candidates.values()), rec) return Proposal(_sort_candidates(candidates.values()), rec)
# If we're searching by ID, don't proceed. # If we're searching by ID, don't proceed.
@ -507,7 +512,7 @@ def tag_item(item, search_artist=None, search_title=None,
# Search terms. # Search terms.
if not (search_artist and search_title): if not (search_artist and search_title):
search_artist, search_title = item.artist, item.title search_artist, search_title = item.artist, item.title
log.debug(u'Item search terms: {0} - {1}', search_artist, search_title) log.debug('Item search terms: {0} - {1}', search_artist, search_title)
# Get and evaluate candidate metadata. # Get and evaluate candidate metadata.
for track_info in hooks.item_candidates(item, search_artist, search_title): for track_info in hooks.item_candidates(item, search_artist, search_title):
@ -515,7 +520,7 @@ def tag_item(item, search_artist=None, search_title=None,
candidates[track_info.track_id] = hooks.TrackMatch(dist, track_info) candidates[track_info.track_id] = hooks.TrackMatch(dist, track_info)
# Sort by distance and return with recommendation. # Sort by distance and return with recommendation.
log.debug(u'Found {0} candidates.', len(candidates)) log.debug('Found {0} candidates.', len(candidates))
candidates = _sort_candidates(candidates.values()) candidates = _sort_candidates(candidates.values())
rec = _recommendation(candidates) rec = _recommendation(candidates)
return Proposal(candidates, rec) return Proposal(candidates, rec)

186
libs/common/beets/autotag/mb.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson. # Copyright 2016, Adrian Sampson.
# #
@ -15,57 +14,72 @@
"""Searches for albums in the MusicBrainz database. """Searches for albums in the MusicBrainz database.
""" """
from __future__ import division, absolute_import, print_function
import musicbrainzngs import musicbrainzngs
import re import re
import traceback import traceback
from six.moves.urllib.parse import urljoin
from beets import logging from beets import logging
from beets import plugins
import beets.autotag.hooks import beets.autotag.hooks
import beets import beets
from beets import util from beets import util
from beets import config from beets import config
import six from collections import Counter
from urllib.parse import urljoin
VARIOUS_ARTISTS_ID = '89ad4ac3-39f7-470e-963a-56509c546377' VARIOUS_ARTISTS_ID = '89ad4ac3-39f7-470e-963a-56509c546377'
if util.SNI_SUPPORTED: BASE_URL = 'https://musicbrainz.org/'
BASE_URL = 'https://musicbrainz.org/'
else:
BASE_URL = 'http://musicbrainz.org/'
SKIPPED_TRACKS = ['[data track]'] SKIPPED_TRACKS = ['[data track]']
FIELDS_TO_MB_KEYS = {
'catalognum': 'catno',
'country': 'country',
'label': 'label',
'media': 'format',
'year': 'date',
}
musicbrainzngs.set_useragent('beets', beets.__version__, musicbrainzngs.set_useragent('beets', beets.__version__,
'http://beets.io/') 'https://beets.io/')
class MusicBrainzAPIError(util.HumanReadableException): class MusicBrainzAPIError(util.HumanReadableException):
"""An error while talking to MusicBrainz. The `query` field is the """An error while talking to MusicBrainz. The `query` field is the
parameter to the action and may have any type. parameter to the action and may have any type.
""" """
def __init__(self, reason, verb, query, tb=None): def __init__(self, reason, verb, query, tb=None):
self.query = query self.query = query
if isinstance(reason, musicbrainzngs.WebServiceError): if isinstance(reason, musicbrainzngs.WebServiceError):
reason = u'MusicBrainz not reachable' reason = 'MusicBrainz not reachable'
super(MusicBrainzAPIError, self).__init__(reason, verb, tb) super().__init__(reason, verb, tb)
def get_message(self): def get_message(self):
return u'{0} in {1} with query {2}'.format( return '{} in {} with query {}'.format(
self._reasonstr(), self.verb, repr(self.query) self._reasonstr(), self.verb, repr(self.query)
) )
log = logging.getLogger('beets') log = logging.getLogger('beets')
RELEASE_INCLUDES = ['artists', 'media', 'recordings', 'release-groups', RELEASE_INCLUDES = ['artists', 'media', 'recordings', 'release-groups',
'labels', 'artist-credits', 'aliases', 'labels', 'artist-credits', 'aliases',
'recording-level-rels', 'work-rels', 'recording-level-rels', 'work-rels',
'work-level-rels', 'artist-rels'] 'work-level-rels', 'artist-rels', 'isrcs']
TRACK_INCLUDES = ['artists', 'aliases'] BROWSE_INCLUDES = ['artist-credits', 'work-rels',
'artist-rels', 'recording-rels', 'release-rels']
if "work-level-rels" in musicbrainzngs.VALID_BROWSE_INCLUDES['recording']:
BROWSE_INCLUDES.append("work-level-rels")
BROWSE_CHUNKSIZE = 100
BROWSE_MAXTRACKS = 500
TRACK_INCLUDES = ['artists', 'aliases', 'isrcs']
if 'work-level-rels' in musicbrainzngs.VALID_INCLUDES['recording']: if 'work-level-rels' in musicbrainzngs.VALID_INCLUDES['recording']:
TRACK_INCLUDES += ['work-level-rels', 'artist-rels'] TRACK_INCLUDES += ['work-level-rels', 'artist-rels']
if 'genres' in musicbrainzngs.VALID_INCLUDES['recording']:
RELEASE_INCLUDES += ['genres']
def track_url(trackid): def track_url(trackid):
@ -81,7 +95,11 @@ def configure():
from the beets configuration. This should be called at startup. from the beets configuration. This should be called at startup.
""" """
hostname = config['musicbrainz']['host'].as_str() hostname = config['musicbrainz']['host'].as_str()
musicbrainzngs.set_hostname(hostname) https = config['musicbrainz']['https'].get(bool)
# Only call set_hostname when a custom server is configured. Since
# musicbrainz-ngs connects to musicbrainz.org with HTTPS by default
if hostname != "musicbrainz.org":
musicbrainzngs.set_hostname(hostname, https)
musicbrainzngs.set_rate_limit( musicbrainzngs.set_rate_limit(
config['musicbrainz']['ratelimit_interval'].as_number(), config['musicbrainz']['ratelimit_interval'].as_number(),
config['musicbrainz']['ratelimit'].get(int), config['musicbrainz']['ratelimit'].get(int),
@ -138,7 +156,7 @@ def _flatten_artist_credit(credit):
artist_sort_parts = [] artist_sort_parts = []
artist_credit_parts = [] artist_credit_parts = []
for el in credit: for el in credit:
if isinstance(el, six.string_types): if isinstance(el, str):
# Join phrase. # Join phrase.
artist_parts.append(el) artist_parts.append(el)
artist_credit_parts.append(el) artist_credit_parts.append(el)
@ -185,13 +203,13 @@ def track_info(recording, index=None, medium=None, medium_index=None,
the number of tracks on the medium. Each number is a 1-based index. the number of tracks on the medium. Each number is a 1-based index.
""" """
info = beets.autotag.hooks.TrackInfo( info = beets.autotag.hooks.TrackInfo(
recording['title'], title=recording['title'],
recording['id'], track_id=recording['id'],
index=index, index=index,
medium=medium, medium=medium,
medium_index=medium_index, medium_index=medium_index,
medium_total=medium_total, medium_total=medium_total,
data_source=u'MusicBrainz', data_source='MusicBrainz',
data_url=track_url(recording['id']), data_url=track_url(recording['id']),
) )
@ -207,12 +225,22 @@ def track_info(recording, index=None, medium=None, medium_index=None,
if recording.get('length'): if recording.get('length'):
info.length = int(recording['length']) / (1000.0) info.length = int(recording['length']) / (1000.0)
info.trackdisambig = recording.get('disambiguation')
if recording.get('isrc-list'):
info.isrc = ';'.join(recording['isrc-list'])
lyricist = [] lyricist = []
composer = [] composer = []
composer_sort = [] composer_sort = []
for work_relation in recording.get('work-relation-list', ()): for work_relation in recording.get('work-relation-list', ()):
if work_relation['type'] != 'performance': if work_relation['type'] != 'performance':
continue continue
info.work = work_relation['work']['title']
info.mb_workid = work_relation['work']['id']
if 'disambiguation' in work_relation['work']:
info.work_disambig = work_relation['work']['disambiguation']
for artist_relation in work_relation['work'].get( for artist_relation in work_relation['work'].get(
'artist-relation-list', ()): 'artist-relation-list', ()):
if 'type' in artist_relation: if 'type' in artist_relation:
@ -224,10 +252,10 @@ def track_info(recording, index=None, medium=None, medium_index=None,
composer_sort.append( composer_sort.append(
artist_relation['artist']['sort-name']) artist_relation['artist']['sort-name'])
if lyricist: if lyricist:
info.lyricist = u', '.join(lyricist) info.lyricist = ', '.join(lyricist)
if composer: if composer:
info.composer = u', '.join(composer) info.composer = ', '.join(composer)
info.composer_sort = u', '.join(composer_sort) info.composer_sort = ', '.join(composer_sort)
arranger = [] arranger = []
for artist_relation in recording.get('artist-relation-list', ()): for artist_relation in recording.get('artist-relation-list', ()):
@ -236,7 +264,12 @@ def track_info(recording, index=None, medium=None, medium_index=None,
if type == 'arranger': if type == 'arranger':
arranger.append(artist_relation['artist']['name']) arranger.append(artist_relation['artist']['name'])
if arranger: if arranger:
info.arranger = u', '.join(arranger) info.arranger = ', '.join(arranger)
# Supplementary fields provided by plugins
extra_trackdatas = plugins.send('mb_track_extract', data=recording)
for extra_trackdata in extra_trackdatas:
info.update(extra_trackdata)
info.decode() info.decode()
return info return info
@ -270,6 +303,26 @@ def album_info(release):
artist_name, artist_sort_name, artist_credit_name = \ artist_name, artist_sort_name, artist_credit_name = \
_flatten_artist_credit(release['artist-credit']) _flatten_artist_credit(release['artist-credit'])
ntracks = sum(len(m['track-list']) for m in release['medium-list'])
# The MusicBrainz API omits 'artist-relation-list' and 'work-relation-list'
# when the release has more than 500 tracks. So we use browse_recordings
# on chunks of tracks to recover the same information in this case.
if ntracks > BROWSE_MAXTRACKS:
log.debug('Album {} has too many tracks', release['id'])
recording_list = []
for i in range(0, ntracks, BROWSE_CHUNKSIZE):
log.debug('Retrieving tracks starting at {}', i)
recording_list.extend(musicbrainzngs.browse_recordings(
release=release['id'], limit=BROWSE_CHUNKSIZE,
includes=BROWSE_INCLUDES,
offset=i)['recording-list'])
track_map = {r['id']: r for r in recording_list}
for medium in release['medium-list']:
for recording in medium['track-list']:
recording_info = track_map[recording['recording']['id']]
recording['recording'] = recording_info
# Basic info. # Basic info.
track_infos = [] track_infos = []
index = 0 index = 0
@ -281,7 +334,8 @@ def album_info(release):
continue continue
all_tracks = medium['track-list'] all_tracks = medium['track-list']
if 'data-track-list' in medium: if ('data-track-list' in medium
and not config['match']['ignore_data_tracks']):
all_tracks += medium['data-track-list'] all_tracks += medium['data-track-list']
track_count = len(all_tracks) track_count = len(all_tracks)
@ -327,15 +381,15 @@ def album_info(release):
track_infos.append(ti) track_infos.append(ti)
info = beets.autotag.hooks.AlbumInfo( info = beets.autotag.hooks.AlbumInfo(
release['title'], album=release['title'],
release['id'], album_id=release['id'],
artist_name, artist=artist_name,
release['artist-credit'][0]['artist']['id'], artist_id=release['artist-credit'][0]['artist']['id'],
track_infos, tracks=track_infos,
mediums=len(release['medium-list']), mediums=len(release['medium-list']),
artist_sort=artist_sort_name, artist_sort=artist_sort_name,
artist_credit=artist_credit_name, artist_credit=artist_credit_name,
data_source=u'MusicBrainz', data_source='MusicBrainz',
data_url=album_url(release['id']), data_url=album_url(release['id']),
) )
info.va = info.artist_id == VARIOUS_ARTISTS_ID info.va = info.artist_id == VARIOUS_ARTISTS_ID
@ -345,13 +399,12 @@ def album_info(release):
info.releasegroup_id = release['release-group']['id'] info.releasegroup_id = release['release-group']['id']
info.albumstatus = release.get('status') info.albumstatus = release.get('status')
# Build up the disambiguation string from the release group and release. # Get the disambiguation strings at the release and release group level.
disambig = []
if release['release-group'].get('disambiguation'): if release['release-group'].get('disambiguation'):
disambig.append(release['release-group'].get('disambiguation')) info.releasegroupdisambig = \
release['release-group'].get('disambiguation')
if release.get('disambiguation'): if release.get('disambiguation'):
disambig.append(release.get('disambiguation')) info.albumdisambig = release.get('disambiguation')
info.albumdisambig = u', '.join(disambig)
# Get the "classic" Release type. This data comes from a legacy API # Get the "classic" Release type. This data comes from a legacy API
# feature before MusicBrainz supported multiple release types. # feature before MusicBrainz supported multiple release types.
@ -360,18 +413,17 @@ def album_info(release):
if reltype: if reltype:
info.albumtype = reltype.lower() info.albumtype = reltype.lower()
# Log the new-style "primary" and "secondary" release types. # Set the new-style "primary" and "secondary" release types.
# Eventually, we'd like to actually store this data, but we just log albumtypes = []
# it for now to help understand the differences.
if 'primary-type' in release['release-group']: if 'primary-type' in release['release-group']:
rel_primarytype = release['release-group']['primary-type'] rel_primarytype = release['release-group']['primary-type']
if rel_primarytype: if rel_primarytype:
log.debug('primary MB release type: ' + rel_primarytype.lower()) albumtypes.append(rel_primarytype.lower())
if 'secondary-type-list' in release['release-group']: if 'secondary-type-list' in release['release-group']:
if release['release-group']['secondary-type-list']: if release['release-group']['secondary-type-list']:
log.debug('secondary MB release type(s): ' + ', '.join( for sec_type in release['release-group']['secondary-type-list']:
[secondarytype.lower() for secondarytype in albumtypes.append(sec_type.lower())
release['release-group']['secondary-type-list']])) info.albumtypes = '; '.join(albumtypes)
# Release events. # Release events.
info.country, release_date = _preferred_release_event(release) info.country, release_date = _preferred_release_event(release)
@ -402,17 +454,33 @@ def album_info(release):
first_medium = release['medium-list'][0] first_medium = release['medium-list'][0]
info.media = first_medium.get('format') info.media = first_medium.get('format')
if config['musicbrainz']['genres']:
sources = [
release['release-group'].get('genre-list', []),
release.get('genre-list', []),
]
genres = Counter()
for source in sources:
for genreitem in source:
genres[genreitem['name']] += int(genreitem['count'])
info.genre = '; '.join(g[0] for g in sorted(genres.items(),
key=lambda g: -g[1]))
extra_albumdatas = plugins.send('mb_album_extract', data=release)
for extra_albumdata in extra_albumdatas:
info.update(extra_albumdata)
info.decode() info.decode()
return info return info
def match_album(artist, album, tracks=None): def match_album(artist, album, tracks=None, extra_tags=None):
"""Searches for a single album ("release" in MusicBrainz parlance) """Searches for a single album ("release" in MusicBrainz parlance)
and returns an iterator over AlbumInfo objects. May raise a and returns an iterator over AlbumInfo objects. May raise a
MusicBrainzAPIError. MusicBrainzAPIError.
The query consists of an artist name, an album name, and, The query consists of an artist name, an album name, and,
optionally, a number of tracks on the album. optionally, a number of tracks on the album and any other extra tags.
""" """
# Build search criteria. # Build search criteria.
criteria = {'release': album.lower().strip()} criteria = {'release': album.lower().strip()}
@ -422,14 +490,24 @@ def match_album(artist, album, tracks=None):
# Various Artists search. # Various Artists search.
criteria['arid'] = VARIOUS_ARTISTS_ID criteria['arid'] = VARIOUS_ARTISTS_ID
if tracks is not None: if tracks is not None:
criteria['tracks'] = six.text_type(tracks) criteria['tracks'] = str(tracks)
# Additional search cues from existing metadata.
if extra_tags:
for tag in extra_tags:
key = FIELDS_TO_MB_KEYS[tag]
value = str(extra_tags.get(tag, '')).lower().strip()
if key == 'catno':
value = value.replace(' ', '')
if value:
criteria[key] = value
# Abort if we have no search terms. # Abort if we have no search terms.
if not any(criteria.values()): if not any(criteria.values()):
return return
try: try:
log.debug(u'Searching for MusicBrainz releases with: {!r}', criteria) log.debug('Searching for MusicBrainz releases with: {!r}', criteria)
res = musicbrainzngs.search_releases( res = musicbrainzngs.search_releases(
limit=config['musicbrainz']['searchlimit'].get(int), **criteria) limit=config['musicbrainz']['searchlimit'].get(int), **criteria)
except musicbrainzngs.MusicBrainzError as exc: except musicbrainzngs.MusicBrainzError as exc:
@ -470,7 +548,7 @@ def _parse_id(s):
no ID can be found, return None. no ID can be found, return None.
""" """
# Find the first thing that looks like a UUID/MBID. # Find the first thing that looks like a UUID/MBID.
match = re.search(u'[a-f0-9]{8}(-[a-f0-9]{4}){3}-[a-f0-9]{12}', s) match = re.search('[a-f0-9]{8}(-[a-f0-9]{4}){3}-[a-f0-9]{12}', s)
if match: if match:
return match.group() return match.group()
@ -480,19 +558,19 @@ def album_for_id(releaseid):
object or None if the album is not found. May raise a object or None if the album is not found. May raise a
MusicBrainzAPIError. MusicBrainzAPIError.
""" """
log.debug(u'Requesting MusicBrainz release {}', releaseid) log.debug('Requesting MusicBrainz release {}', releaseid)
albumid = _parse_id(releaseid) albumid = _parse_id(releaseid)
if not albumid: if not albumid:
log.debug(u'Invalid MBID ({0}).', releaseid) log.debug('Invalid MBID ({0}).', releaseid)
return return
try: try:
res = musicbrainzngs.get_release_by_id(albumid, res = musicbrainzngs.get_release_by_id(albumid,
RELEASE_INCLUDES) RELEASE_INCLUDES)
except musicbrainzngs.ResponseError: except musicbrainzngs.ResponseError:
log.debug(u'Album ID match failed.') log.debug('Album ID match failed.')
return None return None
except musicbrainzngs.MusicBrainzError as exc: except musicbrainzngs.MusicBrainzError as exc:
raise MusicBrainzAPIError(exc, u'get release by ID', albumid, raise MusicBrainzAPIError(exc, 'get release by ID', albumid,
traceback.format_exc()) traceback.format_exc())
return album_info(res['release']) return album_info(res['release'])
@ -503,14 +581,14 @@ def track_for_id(releaseid):
""" """
trackid = _parse_id(releaseid) trackid = _parse_id(releaseid)
if not trackid: if not trackid:
log.debug(u'Invalid MBID ({0}).', releaseid) log.debug('Invalid MBID ({0}).', releaseid)
return return
try: try:
res = musicbrainzngs.get_recording_by_id(trackid, TRACK_INCLUDES) res = musicbrainzngs.get_recording_by_id(trackid, TRACK_INCLUDES)
except musicbrainzngs.ResponseError: except musicbrainzngs.ResponseError:
log.debug(u'Track ID match failed.') log.debug('Track ID match failed.')
return None return None
except musicbrainzngs.MusicBrainzError as exc: except musicbrainzngs.MusicBrainzError as exc:
raise MusicBrainzAPIError(exc, u'get recording by ID', trackid, raise MusicBrainzAPIError(exc, 'get recording by ID', trackid,
traceback.format_exc()) traceback.format_exc())
return track_info(res['recording']) return track_info(res['recording'])

15
libs/common/beets/config_default.yaml
View file

@ -7,6 +7,7 @@ import:
move: no move: no
link: no link: no
hardlink: no hardlink: no
reflink: no
delete: no delete: no
resume: ask resume: ask
incremental: no incremental: no
@ -44,10 +45,20 @@ replace:
'^\s+': '' '^\s+': ''
'^-': _ '^-': _
path_sep_replace: _ path_sep_replace: _
drive_sep_replace: _
asciify_paths: false asciify_paths: false
art_filename: cover art_filename: cover
max_filename_length: 0 max_filename_length: 0
aunique:
keys: albumartist album
disambiguators: albumtype year label catalognum albumdisambig releasegroupdisambig
bracket: '[]'
overwrite_null:
album: []
track: []
plugins: [] plugins: []
pluginpath: [] pluginpath: []
threaded: yes threaded: yes
@ -91,9 +102,12 @@ statefile: state.pickle
musicbrainz: musicbrainz:
host: musicbrainz.org host: musicbrainz.org
https: no
ratelimit: 1 ratelimit: 1
ratelimit_interval: 1.0 ratelimit_interval: 1.0
searchlimit: 5 searchlimit: 5
extra_tags: []
genres: no
match: match:
strong_rec_thresh: 0.04 strong_rec_thresh: 0.04
@ -129,6 +143,7 @@ match:
ignored: [] ignored: []
required: [] required: []
ignored_media: [] ignored_media: []
ignore_data_tracks: yes
ignore_video_tracks: yes ignore_video_tracks: yes
track_length_grace: 10 track_length_grace: 10
track_length_max: 30 track_length_max: 30

2
libs/common/beets/dbcore/__init__.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson. # Copyright 2016, Adrian Sampson.
# #
@ -16,7 +15,6 @@
"""DBCore is an abstract database package that forms the basis for beets' """DBCore is an abstract database package that forms the basis for beets'
Library. Library.
""" """
from __future__ import division, absolute_import, print_function
from .db import Model, Database from .db import Model, Database
from .query import Query, FieldQuery, MatchQuery, AndQuery, OrQuery from .query import Query, FieldQuery, MatchQuery, AndQuery, OrQuery

372
libs/common/beets/dbcore/db.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson. # Copyright 2016, Adrian Sampson.
# #
@ -15,22 +14,21 @@
"""The central Model and Database constructs for DBCore. """The central Model and Database constructs for DBCore.
""" """
from __future__ import division, absolute_import, print_function
import time import time
import os import os
import re
from collections import defaultdict from collections import defaultdict
import threading import threading
import sqlite3 import sqlite3
import contextlib import contextlib
import collections
import beets import beets
from beets.util.functemplate import Template from beets.util import functemplate
from beets.util import py3_path from beets.util import py3_path
from beets.dbcore import types from beets.dbcore import types
from .query import MatchQuery, NullSort, TrueQuery from .query import MatchQuery, NullSort, TrueQuery
import six from collections.abc import Mapping
class DBAccessError(Exception): class DBAccessError(Exception):
@ -42,20 +40,30 @@ class DBAccessError(Exception):
""" """
class FormattedMapping(collections.Mapping): class FormattedMapping(Mapping):
"""A `dict`-like formatted view of a model. """A `dict`-like formatted view of a model.
The accessor `mapping[key]` returns the formatted version of The accessor `mapping[key]` returns the formatted version of
`model[key]` as a unicode string. `model[key]` as a unicode string.
The `included_keys` parameter allows filtering the fields that are
returned. By default all fields are returned. Limiting to specific keys can
avoid expensive per-item database queries.
If `for_path` is true, all path separators in the formatted values If `for_path` is true, all path separators in the formatted values
are replaced. are replaced.
""" """
def __init__(self, model, for_path=False): ALL_KEYS = '*'
def __init__(self, model, included_keys=ALL_KEYS, for_path=False):
self.for_path = for_path self.for_path = for_path
self.model = model self.model = model
self.model_keys = model.keys(True) if included_keys == self.ALL_KEYS:
# Performance note: this triggers a database query.
self.model_keys = self.model.keys(True)
else:
self.model_keys = included_keys
def __getitem__(self, key): def __getitem__(self, key):
if key in self.model_keys: if key in self.model_keys:
@ -72,7 +80,7 @@ class FormattedMapping(collections.Mapping):
def get(self, key, default=None): def get(self, key, default=None):
if default is None: if default is None:
default = self.model._type(key).format(None) default = self.model._type(key).format(None)
return super(FormattedMapping, self).get(key, default) return super().get(key, default)
def _get_formatted(self, model, key): def _get_formatted(self, model, key):
value = model._type(key).format(model.get(key)) value = model._type(key).format(model.get(key))
@ -81,6 +89,11 @@ class FormattedMapping(collections.Mapping):
if self.for_path: if self.for_path:
sep_repl = beets.config['path_sep_replace'].as_str() sep_repl = beets.config['path_sep_replace'].as_str()
sep_drive = beets.config['drive_sep_replace'].as_str()
if re.match(r'^\w:', value):
value = re.sub(r'(?<=^\w):', sep_drive, value)
for sep in (os.path.sep, os.path.altsep): for sep in (os.path.sep, os.path.altsep):
if sep: if sep:
value = value.replace(sep, sep_repl) value = value.replace(sep, sep_repl)
@ -88,11 +101,105 @@ class FormattedMapping(collections.Mapping):
return value return value
class LazyConvertDict:
"""Lazily convert types for attributes fetched from the database
"""
def __init__(self, model_cls):
"""Initialize the object empty
"""
self.data = {}
self.model_cls = model_cls
self._converted = {}
def init(self, data):
"""Set the base data that should be lazily converted
"""
self.data = data
def _convert(self, key, value):
"""Convert the attribute type according the the SQL type
"""
return self.model_cls._type(key).from_sql(value)
def __setitem__(self, key, value):
"""Set an attribute value, assume it's already converted
"""
self._converted[key] = value
def __getitem__(self, key):
"""Get an attribute value, converting the type on demand
if needed
"""
if key in self._converted:
return self._converted[key]
elif key in self.data:
value = self._convert(key, self.data[key])
self._converted[key] = value
return value
def __delitem__(self, key):
"""Delete both converted and base data
"""
if key in self._converted:
del self._converted[key]
if key in self.data:
del self.data[key]
def keys(self):
"""Get a list of available field names for this object.
"""
return list(self._converted.keys()) + list(self.data.keys())
def copy(self):
"""Create a copy of the object.
"""
new = self.__class__(self.model_cls)
new.data = self.data.copy()
new._converted = self._converted.copy()
return new
# Act like a dictionary.
def update(self, values):
"""Assign all values in the given dict.
"""
for key, value in values.items():
self[key] = value
def items(self):
"""Iterate over (key, value) pairs that this object contains.
Computed fields are not included.
"""
for key in self:
yield key, self[key]
def get(self, key, default=None):
"""Get the value for a given key or `default` if it does not
exist.
"""
if key in self:
return self[key]
else:
return default
def __contains__(self, key):
"""Determine whether `key` is an attribute on this object.
"""
return key in self.keys()
def __iter__(self):
"""Iterate over the available field names (excluding computed
fields).
"""
return iter(self.keys())
# Abstract base for model classes. # Abstract base for model classes.
class Model(object): class Model:
"""An abstract object representing an object in the database. Model """An abstract object representing an object in the database. Model
objects act like dictionaries (i.e., the allow subscript access like objects act like dictionaries (i.e., they allow subscript access like
``obj['field']``). The same field set is available via attribute ``obj['field']``). The same field set is available via attribute
access as a shortcut (i.e., ``obj.field``). Three kinds of attributes are access as a shortcut (i.e., ``obj.field``). Three kinds of attributes are
available: available:
@ -143,12 +250,22 @@ class Model(object):
are subclasses of `Sort`. are subclasses of `Sort`.
""" """
_queries = {}
"""Named queries that use a field-like `name:value` syntax but which
do not relate to any specific field.
"""
_always_dirty = False _always_dirty = False
"""By default, fields only become "dirty" when their value actually """By default, fields only become "dirty" when their value actually
changes. Enabling this flag marks fields as dirty even when the new changes. Enabling this flag marks fields as dirty even when the new
value is the same as the old value (e.g., `o.f = o.f`). value is the same as the old value (e.g., `o.f = o.f`).
""" """
_revision = -1
"""A revision number from when the model was loaded from or written
to the database.
"""
@classmethod @classmethod
def _getters(cls): def _getters(cls):
"""Return a mapping from field names to getter functions. """Return a mapping from field names to getter functions.
@ -172,8 +289,8 @@ class Model(object):
""" """
self._db = db self._db = db
self._dirty = set() self._dirty = set()
self._values_fixed = {} self._values_fixed = LazyConvertDict(self)
self._values_flex = {} self._values_flex = LazyConvertDict(self)
# Initial contents. # Initial contents.
self.update(values) self.update(values)
@ -187,23 +304,25 @@ class Model(object):
ordinary construction are bypassed. ordinary construction are bypassed.
""" """
obj = cls(db) obj = cls(db)
for key, value in fixed_values.items():
obj._values_fixed[key] = cls._type(key).from_sql(value) obj._values_fixed.init(fixed_values)
for key, value in flex_values.items(): obj._values_flex.init(flex_values)
obj._values_flex[key] = cls._type(key).from_sql(value)
return obj return obj
def __repr__(self): def __repr__(self):
return '{0}({1})'.format( return '{}({})'.format(
type(self).__name__, type(self).__name__,
', '.join('{0}={1!r}'.format(k, v) for k, v in dict(self).items()), ', '.join(f'{k}={v!r}' for k, v in dict(self).items()),
) )
def clear_dirty(self): def clear_dirty(self):
"""Mark all fields as *clean* (i.e., not needing to be stored to """Mark all fields as *clean* (i.e., not needing to be stored to
the database). the database). Also update the revision.
""" """
self._dirty = set() self._dirty = set()
if self._db:
self._revision = self._db.revision
def _check_db(self, need_id=True): def _check_db(self, need_id=True):
"""Ensure that this object is associated with a database row: it """Ensure that this object is associated with a database row: it
@ -212,10 +331,10 @@ class Model(object):
""" """
if not self._db: if not self._db:
raise ValueError( raise ValueError(
u'{0} has no database'.format(type(self).__name__) '{} has no database'.format(type(self).__name__)
) )
if need_id and not self.id: if need_id and not self.id:
raise ValueError(u'{0} has no id'.format(type(self).__name__)) raise ValueError('{} has no id'.format(type(self).__name__))
def copy(self): def copy(self):
"""Create a copy of the model object. """Create a copy of the model object.
@ -243,19 +362,32 @@ class Model(object):
""" """
return cls._fields.get(key) or cls._types.get(key) or types.DEFAULT return cls._fields.get(key) or cls._types.get(key) or types.DEFAULT
def __getitem__(self, key): def _get(self, key, default=None, raise_=False):
"""Get the value for a field. Raise a KeyError if the field is """Get the value for a field, or `default`. Alternatively,
not available. raise a KeyError if the field is not available.
""" """
getters = self._getters() getters = self._getters()
if key in getters: # Computed. if key in getters: # Computed.
return getters[key](self) return getters[key](self)
elif key in self._fields: # Fixed. elif key in self._fields: # Fixed.
return self._values_fixed.get(key, self._type(key).null) if key in self._values_fixed:
return self._values_fixed[key]
else:
return self._type(key).null
elif key in self._values_flex: # Flexible. elif key in self._values_flex: # Flexible.
return self._values_flex[key] return self._values_flex[key]
else: elif raise_:
raise KeyError(key) raise KeyError(key)
else:
return default
get = _get
def __getitem__(self, key):
"""Get the value for a field. Raise a KeyError if the field is
not available.
"""
return self._get(key, raise_=True)
def _setitem(self, key, value): def _setitem(self, key, value):
"""Assign the value for a field, return whether new and old value """Assign the value for a field, return whether new and old value
@ -290,12 +422,12 @@ class Model(object):
if key in self._values_flex: # Flexible. if key in self._values_flex: # Flexible.
del self._values_flex[key] del self._values_flex[key]
self._dirty.add(key) # Mark for dropping on store. self._dirty.add(key) # Mark for dropping on store.
elif key in self._fields: # Fixed
setattr(self, key, self._type(key).null)
elif key in self._getters(): # Computed. elif key in self._getters(): # Computed.
raise KeyError(u'computed field {0} cannot be deleted'.format(key)) raise KeyError(f'computed field {key} cannot be deleted')
elif key in self._fields: # Fixed.
raise KeyError(u'fixed field {0} cannot be deleted'.format(key))
else: else:
raise KeyError(u'no such field {0}'.format(key)) raise KeyError(f'no such field {key}')
def keys(self, computed=False): def keys(self, computed=False):
"""Get a list of available field names for this object. The """Get a list of available field names for this object. The
@ -330,19 +462,10 @@ class Model(object):
for key in self: for key in self:
yield key, self[key] yield key, self[key]
def get(self, key, default=None):
"""Get the value for a given key or `default` if it does not
exist.
"""
if key in self:
return self[key]
else:
return default
def __contains__(self, key): def __contains__(self, key):
"""Determine whether `key` is an attribute on this object. """Determine whether `key` is an attribute on this object.
""" """
return key in self.keys(True) return key in self.keys(computed=True)
def __iter__(self): def __iter__(self):
"""Iterate over the available field names (excluding computed """Iterate over the available field names (excluding computed
@ -354,22 +477,22 @@ class Model(object):
def __getattr__(self, key): def __getattr__(self, key):
if key.startswith('_'): if key.startswith('_'):
raise AttributeError(u'model has no attribute {0!r}'.format(key)) raise AttributeError(f'model has no attribute {key!r}')
else: else:
try: try:
return self[key] return self[key]
except KeyError: except KeyError:
raise AttributeError(u'no such field {0!r}'.format(key)) raise AttributeError(f'no such field {key!r}')
def __setattr__(self, key, value): def __setattr__(self, key, value):
if key.startswith('_'): if key.startswith('_'):
super(Model, self).__setattr__(key, value) super().__setattr__(key, value)
else: else:
self[key] = value self[key] = value
def __delattr__(self, key): def __delattr__(self, key):
if key.startswith('_'): if key.startswith('_'):
super(Model, self).__delattr__(key) super().__delattr__(key)
else: else:
del self[key] del self[key]
@ -398,7 +521,7 @@ class Model(object):
with self._db.transaction() as tx: with self._db.transaction() as tx:
# Main table update. # Main table update.
if assignments: if assignments:
query = 'UPDATE {0} SET {1} WHERE id=?'.format( query = 'UPDATE {} SET {} WHERE id=?'.format(
self._table, assignments self._table, assignments
) )
subvars.append(self.id) subvars.append(self.id)
@ -409,7 +532,7 @@ class Model(object):
if key in self._dirty: if key in self._dirty:
self._dirty.remove(key) self._dirty.remove(key)
tx.mutate( tx.mutate(
'INSERT INTO {0} ' 'INSERT INTO {} '
'(entity_id, key, value) ' '(entity_id, key, value) '
'VALUES (?, ?, ?);'.format(self._flex_table), 'VALUES (?, ?, ?);'.format(self._flex_table),
(self.id, key, value), (self.id, key, value),
@ -418,7 +541,7 @@ class Model(object):
# Deleted flexible attributes. # Deleted flexible attributes.
for key in self._dirty: for key in self._dirty:
tx.mutate( tx.mutate(
'DELETE FROM {0} ' 'DELETE FROM {} '
'WHERE entity_id=? AND key=?'.format(self._flex_table), 'WHERE entity_id=? AND key=?'.format(self._flex_table),
(self.id, key) (self.id, key)
) )
@ -427,12 +550,18 @@ class Model(object):
def load(self): def load(self):
"""Refresh the object's metadata from the library database. """Refresh the object's metadata from the library database.
If check_revision is true, the database is only queried loaded when a
transaction has been committed since the item was last loaded.
""" """
self._check_db() self._check_db()
if not self._dirty and self._db.revision == self._revision:
# Exit early
return
stored_obj = self._db._get(type(self), self.id) stored_obj = self._db._get(type(self), self.id)
assert stored_obj is not None, u"object {0} not in DB".format(self.id) assert stored_obj is not None, f"object {self.id} not in DB"
self._values_fixed = {} self._values_fixed = LazyConvertDict(self)
self._values_flex = {} self._values_flex = LazyConvertDict(self)
self.update(dict(stored_obj)) self.update(dict(stored_obj))
self.clear_dirty() self.clear_dirty()
@ -442,11 +571,11 @@ class Model(object):
self._check_db() self._check_db()
with self._db.transaction() as tx: with self._db.transaction() as tx:
tx.mutate( tx.mutate(
'DELETE FROM {0} WHERE id=?'.format(self._table), f'DELETE FROM {self._table} WHERE id=?',
(self.id,) (self.id,)
) )
tx.mutate( tx.mutate(
'DELETE FROM {0} WHERE entity_id=?'.format(self._flex_table), f'DELETE FROM {self._flex_table} WHERE entity_id=?',
(self.id,) (self.id,)
) )
@ -464,7 +593,7 @@ class Model(object):
with self._db.transaction() as tx: with self._db.transaction() as tx:
new_id = tx.mutate( new_id = tx.mutate(
'INSERT INTO {0} DEFAULT VALUES'.format(self._table) f'INSERT INTO {self._table} DEFAULT VALUES'
) )
self.id = new_id self.id = new_id
self.added = time.time() self.added = time.time()
@ -479,11 +608,11 @@ class Model(object):
_formatter = FormattedMapping _formatter = FormattedMapping
def formatted(self, for_path=False): def formatted(self, included_keys=_formatter.ALL_KEYS, for_path=False):
"""Get a mapping containing all values on this object formatted """Get a mapping containing all values on this object formatted
as human-readable unicode strings. as human-readable unicode strings.
""" """
return self._formatter(self, for_path) return self._formatter(self, included_keys, for_path)
def evaluate_template(self, template, for_path=False): def evaluate_template(self, template, for_path=False):
"""Evaluate a template (a string or a `Template` object) using """Evaluate a template (a string or a `Template` object) using
@ -491,9 +620,9 @@ class Model(object):
separators will be added to the template. separators will be added to the template.
""" """
# Perform substitution. # Perform substitution.
if isinstance(template, six.string_types): if isinstance(template, str):
template = Template(template) template = functemplate.template(template)
return template.substitute(self.formatted(for_path), return template.substitute(self.formatted(for_path=for_path),
self._template_funcs()) self._template_funcs())
# Parsing. # Parsing.
@ -502,8 +631,8 @@ class Model(object):
def _parse(cls, key, string): def _parse(cls, key, string):
"""Parse a string as a value for the given key. """Parse a string as a value for the given key.
""" """
if not isinstance(string, six.string_types): if not isinstance(string, str):
raise TypeError(u"_parse() argument must be a string") raise TypeError("_parse() argument must be a string")
return cls._type(key).parse(string) return cls._type(key).parse(string)
@ -515,11 +644,13 @@ class Model(object):
# Database controller and supporting interfaces. # Database controller and supporting interfaces.
class Results(object): class Results:
"""An item query result set. Iterating over the collection lazily """An item query result set. Iterating over the collection lazily
constructs LibModel objects that reflect database rows. constructs LibModel objects that reflect database rows.
""" """
def __init__(self, model_class, rows, db, query=None, sort=None):
def __init__(self, model_class, rows, db, flex_rows,
query=None, sort=None):
"""Create a result set that will construct objects of type """Create a result set that will construct objects of type
`model_class`. `model_class`.
@ -539,6 +670,7 @@ class Results(object):
self.db = db self.db = db
self.query = query self.query = query
self.sort = sort self.sort = sort
self.flex_rows = flex_rows
# We keep a queue of rows we haven't yet consumed for # We keep a queue of rows we haven't yet consumed for
# materialization. We preserve the original total number of # materialization. We preserve the original total number of
@ -560,6 +692,10 @@ class Results(object):
a `Results` object a second time should be much faster than the a `Results` object a second time should be much faster than the
first. first.
""" """
# Index flexible attributes by the item ID, so we have easier access
flex_attrs = self._get_indexed_flex_attrs()
index = 0 # Position in the materialized objects. index = 0 # Position in the materialized objects.
while index < len(self._objects) or self._rows: while index < len(self._objects) or self._rows:
# Are there previously-materialized objects to produce? # Are there previously-materialized objects to produce?
@ -572,7 +708,7 @@ class Results(object):
else: else:
while self._rows: while self._rows:
row = self._rows.pop(0) row = self._rows.pop(0)
obj = self._make_model(row) obj = self._make_model(row, flex_attrs.get(row['id'], {}))
# If there is a slow-query predicate, ensurer that the # If there is a slow-query predicate, ensurer that the
# object passes it. # object passes it.
if not self.query or self.query.match(obj): if not self.query or self.query.match(obj):
@ -594,20 +730,24 @@ class Results(object):
# Objects are pre-sorted (i.e., by the database). # Objects are pre-sorted (i.e., by the database).
return self._get_objects() return self._get_objects()
def _make_model(self, row): def _get_indexed_flex_attrs(self):
# Get the flexible attributes for the object. """ Index flexible attributes by the entity id they belong to
with self.db.transaction() as tx: """
flex_rows = tx.query( flex_values = {}
'SELECT * FROM {0} WHERE entity_id=?'.format( for row in self.flex_rows:
self.model_class._flex_table if row['entity_id'] not in flex_values:
), flex_values[row['entity_id']] = {}
(row['id'],)
)
flex_values[row['entity_id']][row['key']] = row['value']
return flex_values
def _make_model(self, row, flex_values={}):
""" Create a Model object for the given row
"""
cols = dict(row) cols = dict(row)
values = dict((k, v) for (k, v) in cols.items() values = {k: v for (k, v) in cols.items()
if not k[:4] == 'flex') if not k[:4] == 'flex'}
flex_values = dict((row['key'], row['value']) for row in flex_rows)
# Construct the Python object # Construct the Python object
obj = self.model_class._awaken(self.db, values, flex_values) obj = self.model_class._awaken(self.db, values, flex_values)
@ -656,7 +796,7 @@ class Results(object):
next(it) next(it)
return next(it) return next(it)
except StopIteration: except StopIteration:
raise IndexError(u'result index {0} out of range'.format(n)) raise IndexError(f'result index {n} out of range')
def get(self): def get(self):
"""Return the first matching object, or None if no objects """Return the first matching object, or None if no objects
@ -669,10 +809,16 @@ class Results(object):
return None return None
class Transaction(object): class Transaction:
"""A context manager for safe, concurrent access to the database. """A context manager for safe, concurrent access to the database.
All SQL commands should be executed through a transaction. All SQL commands should be executed through a transaction.
""" """
_mutated = False
"""A flag storing whether a mutation has been executed in the
current transaction.
"""
def __init__(self, db): def __init__(self, db):
self.db = db self.db = db
@ -694,12 +840,15 @@ class Transaction(object):
entered but not yet exited transaction. If it is the last active entered but not yet exited transaction. If it is the last active
transaction, the database updates are committed. transaction, the database updates are committed.
""" """
# Beware of races; currently secured by db._db_lock
self.db.revision += self._mutated
with self.db._tx_stack() as stack: with self.db._tx_stack() as stack:
assert stack.pop() is self assert stack.pop() is self
empty = not stack empty = not stack
if empty: if empty:
# Ending a "root" transaction. End the SQLite transaction. # Ending a "root" transaction. End the SQLite transaction.
self.db._connection().commit() self.db._connection().commit()
self._mutated = False
self.db._db_lock.release() self.db._db_lock.release()
def query(self, statement, subvals=()): def query(self, statement, subvals=()):
@ -715,7 +864,6 @@ class Transaction(object):
""" """
try: try:
cursor = self.db._connection().execute(statement, subvals) cursor = self.db._connection().execute(statement, subvals)
return cursor.lastrowid
except sqlite3.OperationalError as e: except sqlite3.OperationalError as e:
# In two specific cases, SQLite reports an error while accessing # In two specific cases, SQLite reports an error while accessing
# the underlying database file. We surface these exceptions as # the underlying database file. We surface these exceptions as
@ -725,26 +873,41 @@ class Transaction(object):
raise DBAccessError(e.args[0]) raise DBAccessError(e.args[0])
else: else:
raise raise
else:
self._mutated = True
return cursor.lastrowid
def script(self, statements): def script(self, statements):
"""Execute a string containing multiple SQL statements.""" """Execute a string containing multiple SQL statements."""
# We don't know whether this mutates, but quite likely it does.
self._mutated = True
self.db._connection().executescript(statements) self.db._connection().executescript(statements)
class Database(object): class Database:
"""A container for Model objects that wraps an SQLite database as """A container for Model objects that wraps an SQLite database as
the backend. the backend.
""" """
_models = () _models = ()
"""The Model subclasses representing tables in this database. """The Model subclasses representing tables in this database.
""" """
supports_extensions = hasattr(sqlite3.Connection, 'enable_load_extension')
"""Whether or not the current version of SQLite supports extensions"""
revision = 0
"""The current revision of the database. To be increased whenever
data is written in a transaction.
"""
def __init__(self, path, timeout=5.0): def __init__(self, path, timeout=5.0):
self.path = path self.path = path
self.timeout = timeout self.timeout = timeout
self._connections = {} self._connections = {}
self._tx_stacks = defaultdict(list) self._tx_stacks = defaultdict(list)
self._extensions = []
# A lock to protect the _connections and _tx_stacks maps, which # A lock to protect the _connections and _tx_stacks maps, which
# both map thread IDs to private resources. # both map thread IDs to private resources.
@ -794,6 +957,13 @@ class Database(object):
py3_path(self.path), timeout=self.timeout py3_path(self.path), timeout=self.timeout
) )
if self.supports_extensions:
conn.enable_load_extension(True)
# Load any extension that are already loaded for other connections.
for path in self._extensions:
conn.load_extension(path)
# Access SELECT results like dictionaries. # Access SELECT results like dictionaries.
conn.row_factory = sqlite3.Row conn.row_factory = sqlite3.Row
return conn return conn
@ -822,6 +992,18 @@ class Database(object):
""" """
return Transaction(self) return Transaction(self)
def load_extension(self, path):
"""Load an SQLite extension into all open connections."""
if not self.supports_extensions:
raise ValueError(
'this sqlite3 installation does not support extensions')
self._extensions.append(path)
# Load the extension into every open connection.
for conn in self._connections.values():
conn.load_extension(path)
# Schema setup and migration. # Schema setup and migration.
def _make_table(self, table, fields): def _make_table(self, table, fields):
@ -831,7 +1013,7 @@ class Database(object):
# Get current schema. # Get current schema.
with self.transaction() as tx: with self.transaction() as tx:
rows = tx.query('PRAGMA table_info(%s)' % table) rows = tx.query('PRAGMA table_info(%s)' % table)
current_fields = set([row[1] for row in rows]) current_fields = {row[1] for row in rows}
field_names = set(fields.keys()) field_names = set(fields.keys())
if current_fields.issuperset(field_names): if current_fields.issuperset(field_names):
@ -842,9 +1024,9 @@ class Database(object):
# No table exists. # No table exists.
columns = [] columns = []
for name, typ in fields.items(): for name, typ in fields.items():
columns.append('{0} {1}'.format(name, typ.sql)) columns.append(f'{name} {typ.sql}')
setup_sql = 'CREATE TABLE {0} ({1});\n'.format(table, setup_sql = 'CREATE TABLE {} ({});\n'.format(table,
', '.join(columns)) ', '.join(columns))
else: else:
# Table exists does not match the field set. # Table exists does not match the field set.
@ -852,7 +1034,7 @@ class Database(object):
for name, typ in fields.items(): for name, typ in fields.items():
if name in current_fields: if name in current_fields:
continue continue
setup_sql += 'ALTER TABLE {0} ADD COLUMN {1} {2};\n'.format( setup_sql += 'ALTER TABLE {} ADD COLUMN {} {};\n'.format(
table, name, typ.sql table, name, typ.sql
) )
@ -888,17 +1070,31 @@ class Database(object):
where, subvals = query.clause() where, subvals = query.clause()
order_by = sort.order_clause() order_by = sort.order_clause()
sql = ("SELECT * FROM {0} WHERE {1} {2}").format( sql = ("SELECT * FROM {} WHERE {} {}").format(
model_cls._table, model_cls._table,
where or '1', where or '1',
"ORDER BY {0}".format(order_by) if order_by else '', f"ORDER BY {order_by}" if order_by else '',
)
# Fetch flexible attributes for items matching the main query.
# Doing the per-item filtering in python is faster than issuing
# one query per item to sqlite.
flex_sql = ("""
SELECT * FROM {} WHERE entity_id IN
(SELECT id FROM {} WHERE {});
""".format(
model_cls._flex_table,
model_cls._table,
where or '1',
)
) )
with self.transaction() as tx: with self.transaction() as tx:
rows = tx.query(sql, subvals) rows = tx.query(sql, subvals)
flex_rows = tx.query(flex_sql, subvals)
return Results( return Results(
model_cls, rows, self, model_cls, rows, self, flex_rows,
None if where else query, # Slow query component. None if where else query, # Slow query component.
sort if sort.is_slow() else None, # Slow sort component. sort if sort.is_slow() else None, # Slow sort component.
) )

108
libs/common/beets/dbcore/query.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson. # Copyright 2016, Adrian Sampson.
# #
@ -15,7 +14,6 @@
"""The Query type hierarchy for DBCore. """The Query type hierarchy for DBCore.
""" """
from __future__ import division, absolute_import, print_function
import re import re
from operator import mul from operator import mul
@ -23,10 +21,6 @@ from beets import util
from datetime import datetime, timedelta from datetime import datetime, timedelta
import unicodedata import unicodedata
from functools import reduce from functools import reduce
import six
if not six.PY2:
buffer = memoryview # sqlite won't accept memoryview in python 2
class ParsingError(ValueError): class ParsingError(ValueError):
@ -44,8 +38,8 @@ class InvalidQueryError(ParsingError):
def __init__(self, query, explanation): def __init__(self, query, explanation):
if isinstance(query, list): if isinstance(query, list):
query = " ".join(query) query = " ".join(query)
message = u"'{0}': {1}".format(query, explanation) message = f"'{query}': {explanation}"
super(InvalidQueryError, self).__init__(message) super().__init__(message)
class InvalidQueryArgumentValueError(ParsingError): class InvalidQueryArgumentValueError(ParsingError):
@ -56,13 +50,13 @@ class InvalidQueryArgumentValueError(ParsingError):
""" """
def __init__(self, what, expected, detail=None): def __init__(self, what, expected, detail=None):
message = u"'{0}' is not {1}".format(what, expected) message = f"'{what}' is not {expected}"
if detail: if detail:
message = u"{0}: {1}".format(message, detail) message = f"{message}: {detail}"
super(InvalidQueryArgumentValueError, self).__init__(message) super().__init__(message)
class Query(object): class Query:
"""An abstract class representing a query into the item database. """An abstract class representing a query into the item database.
""" """
@ -82,7 +76,7 @@ class Query(object):
raise NotImplementedError raise NotImplementedError
def __repr__(self): def __repr__(self):
return "{0.__class__.__name__}()".format(self) return f"{self.__class__.__name__}()"
def __eq__(self, other): def __eq__(self, other):
return type(self) == type(other) return type(self) == type(other)
@ -129,7 +123,7 @@ class FieldQuery(Query):
"{0.fast})".format(self)) "{0.fast})".format(self))
def __eq__(self, other): def __eq__(self, other):
return super(FieldQuery, self).__eq__(other) and \ return super().__eq__(other) and \
self.field == other.field and self.pattern == other.pattern self.field == other.field and self.pattern == other.pattern
def __hash__(self): def __hash__(self):
@ -151,17 +145,13 @@ class NoneQuery(FieldQuery):
"""A query that checks whether a field is null.""" """A query that checks whether a field is null."""
def __init__(self, field, fast=True): def __init__(self, field, fast=True):
super(NoneQuery, self).__init__(field, None, fast) super().__init__(field, None, fast)
def col_clause(self): def col_clause(self):
return self.field + " IS NULL", () return self.field + " IS NULL", ()
@classmethod def match(self, item):
def match(cls, item): return item.get(self.field) is None
try:
return item[cls.field] is None
except KeyError:
return True
def __repr__(self): def __repr__(self):
return "{0.__class__.__name__}({0.field!r}, {0.fast})".format(self) return "{0.__class__.__name__}({0.field!r}, {0.fast})".format(self)
@ -214,14 +204,14 @@ class RegexpQuery(StringFieldQuery):
""" """
def __init__(self, field, pattern, fast=True): def __init__(self, field, pattern, fast=True):
super(RegexpQuery, self).__init__(field, pattern, fast) super().__init__(field, pattern, fast)
pattern = self._normalize(pattern) pattern = self._normalize(pattern)
try: try:
self.pattern = re.compile(self.pattern) self.pattern = re.compile(self.pattern)
except re.error as exc: except re.error as exc:
# Invalid regular expression. # Invalid regular expression.
raise InvalidQueryArgumentValueError(pattern, raise InvalidQueryArgumentValueError(pattern,
u"a regular expression", "a regular expression",
format(exc)) format(exc))
@staticmethod @staticmethod
@ -242,8 +232,8 @@ class BooleanQuery(MatchQuery):
""" """
def __init__(self, field, pattern, fast=True): def __init__(self, field, pattern, fast=True):
super(BooleanQuery, self).__init__(field, pattern, fast) super().__init__(field, pattern, fast)
if isinstance(pattern, six.string_types): if isinstance(pattern, str):
self.pattern = util.str2bool(pattern) self.pattern = util.str2bool(pattern)
self.pattern = int(self.pattern) self.pattern = int(self.pattern)
@ -256,16 +246,16 @@ class BytesQuery(MatchQuery):
""" """
def __init__(self, field, pattern): def __init__(self, field, pattern):
super(BytesQuery, self).__init__(field, pattern) super().__init__(field, pattern)
# Use a buffer/memoryview representation of the pattern for SQLite # Use a buffer/memoryview representation of the pattern for SQLite
# matching. This instructs SQLite to treat the blob as binary # matching. This instructs SQLite to treat the blob as binary
# rather than encoded Unicode. # rather than encoded Unicode.
if isinstance(self.pattern, (six.text_type, bytes)): if isinstance(self.pattern, (str, bytes)):
if isinstance(self.pattern, six.text_type): if isinstance(self.pattern, str):
self.pattern = self.pattern.encode('utf-8') self.pattern = self.pattern.encode('utf-8')
self.buf_pattern = buffer(self.pattern) self.buf_pattern = memoryview(self.pattern)
elif isinstance(self.pattern, buffer): elif isinstance(self.pattern, memoryview):
self.buf_pattern = self.pattern self.buf_pattern = self.pattern
self.pattern = bytes(self.pattern) self.pattern = bytes(self.pattern)
@ -297,10 +287,10 @@ class NumericQuery(FieldQuery):
try: try:
return float(s) return float(s)
except ValueError: except ValueError:
raise InvalidQueryArgumentValueError(s, u"an int or a float") raise InvalidQueryArgumentValueError(s, "an int or a float")
def __init__(self, field, pattern, fast=True): def __init__(self, field, pattern, fast=True):
super(NumericQuery, self).__init__(field, pattern, fast) super().__init__(field, pattern, fast)
parts = pattern.split('..', 1) parts = pattern.split('..', 1)
if len(parts) == 1: if len(parts) == 1:
@ -318,7 +308,7 @@ class NumericQuery(FieldQuery):
if self.field not in item: if self.field not in item:
return False return False
value = item[self.field] value = item[self.field]
if isinstance(value, six.string_types): if isinstance(value, str):
value = self._convert(value) value = self._convert(value)
if self.point is not None: if self.point is not None:
@ -335,14 +325,14 @@ class NumericQuery(FieldQuery):
return self.field + '=?', (self.point,) return self.field + '=?', (self.point,)
else: else:
if self.rangemin is not None and self.rangemax is not None: if self.rangemin is not None and self.rangemax is not None:
return (u'{0} >= ? AND {0} <= ?'.format(self.field), return ('{0} >= ? AND {0} <= ?'.format(self.field),
(self.rangemin, self.rangemax)) (self.rangemin, self.rangemax))
elif self.rangemin is not None: elif self.rangemin is not None:
return u'{0} >= ?'.format(self.field), (self.rangemin,) return f'{self.field} >= ?', (self.rangemin,)
elif self.rangemax is not None: elif self.rangemax is not None:
return u'{0} <= ?'.format(self.field), (self.rangemax,) return f'{self.field} <= ?', (self.rangemax,)
else: else:
return u'1', () return '1', ()
class CollectionQuery(Query): class CollectionQuery(Query):
@ -387,7 +377,7 @@ class CollectionQuery(Query):
return "{0.__class__.__name__}({0.subqueries!r})".format(self) return "{0.__class__.__name__}({0.subqueries!r})".format(self)
def __eq__(self, other): def __eq__(self, other):
return super(CollectionQuery, self).__eq__(other) and \ return super().__eq__(other) and \
self.subqueries == other.subqueries self.subqueries == other.subqueries
def __hash__(self): def __hash__(self):
@ -411,7 +401,7 @@ class AnyFieldQuery(CollectionQuery):
subqueries = [] subqueries = []
for field in self.fields: for field in self.fields:
subqueries.append(cls(field, pattern, True)) subqueries.append(cls(field, pattern, True))
super(AnyFieldQuery, self).__init__(subqueries) super().__init__(subqueries)
def clause(self): def clause(self):
return self.clause_with_joiner('or') return self.clause_with_joiner('or')
@ -427,7 +417,7 @@ class AnyFieldQuery(CollectionQuery):
"{0.query_class.__name__})".format(self)) "{0.query_class.__name__})".format(self))
def __eq__(self, other): def __eq__(self, other):
return super(AnyFieldQuery, self).__eq__(other) and \ return super().__eq__(other) and \
self.query_class == other.query_class self.query_class == other.query_class
def __hash__(self): def __hash__(self):
@ -453,7 +443,7 @@ class AndQuery(MutableCollectionQuery):
return self.clause_with_joiner('and') return self.clause_with_joiner('and')
def match(self, item): def match(self, item):
return all([q.match(item) for q in self.subqueries]) return all(q.match(item) for q in self.subqueries)
class OrQuery(MutableCollectionQuery): class OrQuery(MutableCollectionQuery):
@ -463,7 +453,7 @@ class OrQuery(MutableCollectionQuery):
return self.clause_with_joiner('or') return self.clause_with_joiner('or')
def match(self, item): def match(self, item):
return any([q.match(item) for q in self.subqueries]) return any(q.match(item) for q in self.subqueries)
class NotQuery(Query): class NotQuery(Query):
@ -477,7 +467,7 @@ class NotQuery(Query):
def clause(self): def clause(self):
clause, subvals = self.subquery.clause() clause, subvals = self.subquery.clause()
if clause: if clause:
return 'not ({0})'.format(clause), subvals return f'not ({clause})', subvals
else: else:
# If there is no clause, there is nothing to negate. All the logic # If there is no clause, there is nothing to negate. All the logic
# is handled by match() for slow queries. # is handled by match() for slow queries.
@ -490,7 +480,7 @@ class NotQuery(Query):
return "{0.__class__.__name__}({0.subquery!r})".format(self) return "{0.__class__.__name__}({0.subquery!r})".format(self)
def __eq__(self, other): def __eq__(self, other):
return super(NotQuery, self).__eq__(other) and \ return super().__eq__(other) and \
self.subquery == other.subquery self.subquery == other.subquery
def __hash__(self): def __hash__(self):
@ -546,7 +536,7 @@ def _parse_periods(pattern):
return (start, end) return (start, end)
class Period(object): class Period:
"""A period of time given by a date, time and precision. """A period of time given by a date, time and precision.
Example: 2014-01-01 10:50:30 with precision 'month' represents all Example: 2014-01-01 10:50:30 with precision 'month' represents all
@ -572,7 +562,7 @@ class Period(object):
or "second"). or "second").
""" """
if precision not in Period.precisions: if precision not in Period.precisions:
raise ValueError(u'Invalid precision {0}'.format(precision)) raise ValueError(f'Invalid precision {precision}')
self.date = date self.date = date
self.precision = precision self.precision = precision
@ -653,10 +643,10 @@ class Period(object):
elif 'second' == precision: elif 'second' == precision:
return date + timedelta(seconds=1) return date + timedelta(seconds=1)
else: else:
raise ValueError(u'unhandled precision {0}'.format(precision)) raise ValueError(f'unhandled precision {precision}')
class DateInterval(object): class DateInterval:
"""A closed-open interval of dates. """A closed-open interval of dates.
A left endpoint of None means since the beginning of time. A left endpoint of None means since the beginning of time.
@ -665,7 +655,7 @@ class DateInterval(object):
def __init__(self, start, end): def __init__(self, start, end):
if start is not None and end is not None and not start < end: if start is not None and end is not None and not start < end:
raise ValueError(u"start date {0} is not before end date {1}" raise ValueError("start date {} is not before end date {}"
.format(start, end)) .format(start, end))
self.start = start self.start = start
self.end = end self.end = end
@ -686,7 +676,7 @@ class DateInterval(object):
return True return True
def __str__(self): def __str__(self):
return '[{0}, {1})'.format(self.start, self.end) return f'[{self.start}, {self.end})'
class DateQuery(FieldQuery): class DateQuery(FieldQuery):
@ -700,7 +690,7 @@ class DateQuery(FieldQuery):
""" """
def __init__(self, field, pattern, fast=True): def __init__(self, field, pattern, fast=True):
super(DateQuery, self).__init__(field, pattern, fast) super().__init__(field, pattern, fast)
start, end = _parse_periods(pattern) start, end = _parse_periods(pattern)
self.interval = DateInterval.from_periods(start, end) self.interval = DateInterval.from_periods(start, end)
@ -759,12 +749,12 @@ class DurationQuery(NumericQuery):
except ValueError: except ValueError:
raise InvalidQueryArgumentValueError( raise InvalidQueryArgumentValueError(
s, s,
u"a M:SS string or a float") "a M:SS string or a float")
# Sorting. # Sorting.
class Sort(object): class Sort:
"""An abstract class representing a sort operation for a query into """An abstract class representing a sort operation for a query into
the item database. the item database.
""" """
@ -851,13 +841,13 @@ class MultipleSort(Sort):
return items return items
def __repr__(self): def __repr__(self):
return 'MultipleSort({!r})'.format(self.sorts) return f'MultipleSort({self.sorts!r})'
def __hash__(self): def __hash__(self):
return hash(tuple(self.sorts)) return hash(tuple(self.sorts))
def __eq__(self, other): def __eq__(self, other):
return super(MultipleSort, self).__eq__(other) and \ return super().__eq__(other) and \
self.sorts == other.sorts self.sorts == other.sorts
@ -878,14 +868,14 @@ class FieldSort(Sort):
def key(item): def key(item):
field_val = item.get(self.field, '') field_val = item.get(self.field, '')
if self.case_insensitive and isinstance(field_val, six.text_type): if self.case_insensitive and isinstance(field_val, str):
field_val = field_val.lower() field_val = field_val.lower()
return field_val return field_val
return sorted(objs, key=key, reverse=not self.ascending) return sorted(objs, key=key, reverse=not self.ascending)
def __repr__(self): def __repr__(self):
return '<{0}: {1}{2}>'.format( return '<{}: {}{}>'.format(
type(self).__name__, type(self).__name__,
self.field, self.field,
'+' if self.ascending else '-', '+' if self.ascending else '-',
@ -895,7 +885,7 @@ class FieldSort(Sort):
return hash((self.field, self.ascending)) return hash((self.field, self.ascending))
def __eq__(self, other): def __eq__(self, other):
return super(FieldSort, self).__eq__(other) and \ return super().__eq__(other) and \
self.field == other.field and \ self.field == other.field and \
self.ascending == other.ascending self.ascending == other.ascending
@ -913,7 +903,7 @@ class FixedFieldSort(FieldSort):
'ELSE {0} END)'.format(self.field) 'ELSE {0} END)'.format(self.field)
else: else:
field = self.field field = self.field
return "{0} {1}".format(field, order) return f"{field} {order}"
class SlowFieldSort(FieldSort): class SlowFieldSort(FieldSort):

70
libs/common/beets/dbcore/queryparse.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson. # Copyright 2016, Adrian Sampson.
# #
@ -15,12 +14,10 @@
"""Parsing of strings into DBCore queries. """Parsing of strings into DBCore queries.
""" """
from __future__ import division, absolute_import, print_function
import re import re
import itertools import itertools
from . import query from . import query
import beets
PARSE_QUERY_PART_REGEX = re.compile( PARSE_QUERY_PART_REGEX = re.compile(
# Non-capturing optional segment for the keyword. # Non-capturing optional segment for the keyword.
@ -89,7 +86,7 @@ def parse_query_part(part, query_classes={}, prefixes={},
assert match # Regex should always match assert match # Regex should always match
negate = bool(match.group(1)) negate = bool(match.group(1))
key = match.group(2) key = match.group(2)
term = match.group(3).replace('\:', ':') term = match.group(3).replace('\\:', ':')
# Check whether there's a prefix in the query and use the # Check whether there's a prefix in the query and use the
# corresponding query type. # corresponding query type.
@ -119,12 +116,13 @@ def construct_query_part(model_cls, prefixes, query_part):
if not query_part: if not query_part:
return query.TrueQuery() return query.TrueQuery()
# Use `model_cls` to build up a map from field names to `Query` # Use `model_cls` to build up a map from field (or query) names to
# classes. # `Query` classes.
query_classes = {} query_classes = {}
for k, t in itertools.chain(model_cls._fields.items(), for k, t in itertools.chain(model_cls._fields.items(),
model_cls._types.items()): model_cls._types.items()):
query_classes[k] = t.query query_classes[k] = t.query
query_classes.update(model_cls._queries) # Non-field queries.
# Parse the string. # Parse the string.
key, pattern, query_class, negate = \ key, pattern, query_class, negate = \
@ -137,26 +135,27 @@ def construct_query_part(model_cls, prefixes, query_part):
# The query type matches a specific field, but none was # The query type matches a specific field, but none was
# specified. So we use a version of the query that matches # specified. So we use a version of the query that matches
# any field. # any field.
q = query.AnyFieldQuery(pattern, model_cls._search_fields, out_query = query.AnyFieldQuery(pattern, model_cls._search_fields,
query_class) query_class)
if negate:
return query.NotQuery(q)
else:
return q
else: else:
# Non-field query type. # Non-field query type.
if negate: out_query = query_class(pattern)
return query.NotQuery(query_class(pattern))
else:
return query_class(pattern)
# Otherwise, this must be a `FieldQuery`. Use the field name to # Field queries get constructed according to the name of the field
# construct the query object. # they are querying.
key = key.lower() elif issubclass(query_class, query.FieldQuery):
q = query_class(key.lower(), pattern, key in model_cls._fields) key = key.lower()
out_query = query_class(key.lower(), pattern, key in model_cls._fields)
# Non-field (named) query.
else:
out_query = query_class(pattern)
# Apply negation.
if negate: if negate:
return query.NotQuery(q) return query.NotQuery(out_query)
return q else:
return out_query
def query_from_strings(query_cls, model_cls, prefixes, query_parts): def query_from_strings(query_cls, model_cls, prefixes, query_parts):
@ -172,11 +171,13 @@ def query_from_strings(query_cls, model_cls, prefixes, query_parts):
return query_cls(subqueries) return query_cls(subqueries)
def construct_sort_part(model_cls, part): def construct_sort_part(model_cls, part, case_insensitive=True):
"""Create a `Sort` from a single string criterion. """Create a `Sort` from a single string criterion.
`model_cls` is the `Model` being queried. `part` is a single string `model_cls` is the `Model` being queried. `part` is a single string
ending in ``+`` or ``-`` indicating the sort. ending in ``+`` or ``-`` indicating the sort. `case_insensitive`
indicates whether or not the sort should be performed in a case
sensitive manner.
""" """
assert part, "part must be a field name and + or -" assert part, "part must be a field name and + or -"
field = part[:-1] field = part[:-1]
@ -185,7 +186,6 @@ def construct_sort_part(model_cls, part):
assert direction in ('+', '-'), "part must end with + or -" assert direction in ('+', '-'), "part must end with + or -"
is_ascending = direction == '+' is_ascending = direction == '+'
case_insensitive = beets.config['sort_case_insensitive'].get(bool)
if field in model_cls._sorts: if field in model_cls._sorts:
sort = model_cls._sorts[field](model_cls, is_ascending, sort = model_cls._sorts[field](model_cls, is_ascending,
case_insensitive) case_insensitive)
@ -197,21 +197,23 @@ def construct_sort_part(model_cls, part):
return sort return sort
def sort_from_strings(model_cls, sort_parts): def sort_from_strings(model_cls, sort_parts, case_insensitive=True):
"""Create a `Sort` from a list of sort criteria (strings). """Create a `Sort` from a list of sort criteria (strings).
""" """
if not sort_parts: if not sort_parts:
sort = query.NullSort() sort = query.NullSort()
elif len(sort_parts) == 1: elif len(sort_parts) == 1:
sort = construct_sort_part(model_cls, sort_parts[0]) sort = construct_sort_part(model_cls, sort_parts[0], case_insensitive)
else: else:
sort = query.MultipleSort() sort = query.MultipleSort()
for part in sort_parts: for part in sort_parts:
sort.add_sort(construct_sort_part(model_cls, part)) sort.add_sort(construct_sort_part(model_cls, part,
case_insensitive))
return sort return sort
def parse_sorted_query(model_cls, parts, prefixes={}): def parse_sorted_query(model_cls, parts, prefixes={},
case_insensitive=True):
"""Given a list of strings, create the `Query` and `Sort` that they """Given a list of strings, create the `Query` and `Sort` that they
represent. represent.
""" """
@ -222,8 +224,8 @@ def parse_sorted_query(model_cls, parts, prefixes={}):
# Split up query in to comma-separated subqueries, each representing # Split up query in to comma-separated subqueries, each representing
# an AndQuery, which need to be joined together in one OrQuery # an AndQuery, which need to be joined together in one OrQuery
subquery_parts = [] subquery_parts = []
for part in parts + [u',']: for part in parts + [',']:
if part.endswith(u','): if part.endswith(','):
# Ensure we can catch "foo, bar" as well as "foo , bar" # Ensure we can catch "foo, bar" as well as "foo , bar"
last_subquery_part = part[:-1] last_subquery_part = part[:-1]
if last_subquery_part: if last_subquery_part:
@ -237,8 +239,8 @@ def parse_sorted_query(model_cls, parts, prefixes={}):
else: else:
# Sort parts (1) end in + or -, (2) don't have a field, and # Sort parts (1) end in + or -, (2) don't have a field, and
# (3) consist of more than just the + or -. # (3) consist of more than just the + or -.
if part.endswith((u'+', u'-')) \ if part.endswith(('+', '-')) \
and u':' not in part \ and ':' not in part \
and len(part) > 1: and len(part) > 1:
sort_parts.append(part) sort_parts.append(part)
else: else:
@ -246,5 +248,5 @@ def parse_sorted_query(model_cls, parts, prefixes={}):
# Avoid needlessly wrapping single statements in an OR # Avoid needlessly wrapping single statements in an OR
q = query.OrQuery(query_parts) if len(query_parts) > 1 else query_parts[0] q = query.OrQuery(query_parts) if len(query_parts) > 1 else query_parts[0]
s = sort_from_strings(model_cls, sort_parts) s = sort_from_strings(model_cls, sort_parts, case_insensitive)
return q, s return q, s

70
libs/common/beets/dbcore/types.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson. # Copyright 2016, Adrian Sampson.
# #
@ -15,25 +14,20 @@
"""Representation of type information for DBCore model fields. """Representation of type information for DBCore model fields.
""" """
from __future__ import division, absolute_import, print_function
from . import query from . import query
from beets.util import str2bool from beets.util import str2bool
import six
if not six.PY2:
buffer = memoryview # sqlite won't accept memoryview in python 2
# Abstract base. # Abstract base.
class Type(object): class Type:
"""An object encapsulating the type of a model field. Includes """An object encapsulating the type of a model field. Includes
information about how to store, query, format, and parse a given information about how to store, query, format, and parse a given
field. field.
""" """
sql = u'TEXT' sql = 'TEXT'
"""The SQLite column type for the value. """The SQLite column type for the value.
""" """
@ -41,7 +35,7 @@ class Type(object):
"""The `Query` subclass to be used when querying the field. """The `Query` subclass to be used when querying the field.
""" """
model_type = six.text_type model_type = str
"""The Python type that is used to represent the value in the model. """The Python type that is used to represent the value in the model.
The model is guaranteed to return a value of this type if the field The model is guaranteed to return a value of this type if the field
@ -63,11 +57,11 @@ class Type(object):
value = self.null value = self.null
# `self.null` might be `None` # `self.null` might be `None`
if value is None: if value is None:
value = u'' value = ''
if isinstance(value, bytes): if isinstance(value, bytes):
value = value.decode('utf-8', 'ignore') value = value.decode('utf-8', 'ignore')
return six.text_type(value) return str(value)
def parse(self, string): def parse(self, string):
"""Parse a (possibly human-written) string and return the """Parse a (possibly human-written) string and return the
@ -97,16 +91,16 @@ class Type(object):
For fixed fields the type of `value` is determined by the column For fixed fields the type of `value` is determined by the column
type affinity given in the `sql` property and the SQL to Python type affinity given in the `sql` property and the SQL to Python
mapping of the database adapter. For more information see: mapping of the database adapter. For more information see:
http://www.sqlite.org/datatype3.html https://www.sqlite.org/datatype3.html
https://docs.python.org/2/library/sqlite3.html#sqlite-and-python-types https://docs.python.org/2/library/sqlite3.html#sqlite-and-python-types
Flexible fields have the type affinity `TEXT`. This means the Flexible fields have the type affinity `TEXT`. This means the
`sql_value` is either a `buffer`/`memoryview` or a `unicode` object` `sql_value` is either a `memoryview` or a `unicode` object`
and the method must handle these in addition. and the method must handle these in addition.
""" """
if isinstance(sql_value, buffer): if isinstance(sql_value, memoryview):
sql_value = bytes(sql_value).decode('utf-8', 'ignore') sql_value = bytes(sql_value).decode('utf-8', 'ignore')
if isinstance(sql_value, six.text_type): if isinstance(sql_value, str):
return self.parse(sql_value) return self.parse(sql_value)
else: else:
return self.normalize(sql_value) return self.normalize(sql_value)
@ -127,10 +121,18 @@ class Default(Type):
class Integer(Type): class Integer(Type):
"""A basic integer type. """A basic integer type.
""" """
sql = u'INTEGER' sql = 'INTEGER'
query = query.NumericQuery query = query.NumericQuery
model_type = int model_type = int
def normalize(self, value):
try:
return self.model_type(round(float(value)))
except ValueError:
return self.null
except TypeError:
return self.null
class PaddedInt(Integer): class PaddedInt(Integer):
"""An integer field that is formatted with a given number of digits, """An integer field that is formatted with a given number of digits,
@ -140,19 +142,25 @@ class PaddedInt(Integer):
self.digits = digits self.digits = digits
def format(self, value): def format(self, value):
return u'{0:0{1}d}'.format(value or 0, self.digits) return '{0:0{1}d}'.format(value or 0, self.digits)
class NullPaddedInt(PaddedInt):
"""Same as `PaddedInt`, but does not normalize `None` to `0.0`.
"""
null = None
class ScaledInt(Integer): class ScaledInt(Integer):
"""An integer whose formatting operation scales the number by a """An integer whose formatting operation scales the number by a
constant and adds a suffix. Good for units with large magnitudes. constant and adds a suffix. Good for units with large magnitudes.
""" """
def __init__(self, unit, suffix=u''): def __init__(self, unit, suffix=''):
self.unit = unit self.unit = unit
self.suffix = suffix self.suffix = suffix
def format(self, value): def format(self, value):
return u'{0}{1}'.format((value or 0) // self.unit, self.suffix) return '{}{}'.format((value or 0) // self.unit, self.suffix)
class Id(Integer): class Id(Integer):
@ -163,18 +171,22 @@ class Id(Integer):
def __init__(self, primary=True): def __init__(self, primary=True):
if primary: if primary:
self.sql = u'INTEGER PRIMARY KEY' self.sql = 'INTEGER PRIMARY KEY'
class Float(Type): class Float(Type):
"""A basic floating-point type. """A basic floating-point type. The `digits` parameter specifies how
many decimal places to use in the human-readable representation.
""" """
sql = u'REAL' sql = 'REAL'
query = query.NumericQuery query = query.NumericQuery
model_type = float model_type = float
def __init__(self, digits=1):
self.digits = digits
def format(self, value): def format(self, value):
return u'{0:.1f}'.format(value or 0.0) return '{0:.{1}f}'.format(value or 0, self.digits)
class NullFloat(Float): class NullFloat(Float):
@ -186,19 +198,25 @@ class NullFloat(Float):
class String(Type): class String(Type):
"""A Unicode string type. """A Unicode string type.
""" """
sql = u'TEXT' sql = 'TEXT'
query = query.SubstringQuery query = query.SubstringQuery
def normalize(self, value):
if value is None:
return self.null
else:
return self.model_type(value)
class Boolean(Type): class Boolean(Type):
"""A boolean type. """A boolean type.
""" """
sql = u'INTEGER' sql = 'INTEGER'
query = query.BooleanQuery query = query.BooleanQuery
model_type = bool model_type = bool
def format(self, value): def format(self, value):
return six.text_type(bool(value)) return str(bool(value))
def parse(self, string): def parse(self, string):
return str2bool(string) return str2bool(string)

204
libs/common/beets/importer.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson. # Copyright 2016, Adrian Sampson.
# #
@ -13,7 +12,6 @@
# The above copyright notice and this permission notice shall be # The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software. # included in all copies or substantial portions of the Software.
from __future__ import division, absolute_import, print_function
"""Provides the basic, interface-agnostic workflow for importing and """Provides the basic, interface-agnostic workflow for importing and
autotagging music files. autotagging music files.
@ -40,7 +38,7 @@ from beets import config
from beets.util import pipeline, sorted_walk, ancestry, MoveOperation from beets.util import pipeline, sorted_walk, ancestry, MoveOperation
from beets.util import syspath, normpath, displayable_path from beets.util import syspath, normpath, displayable_path
from enum import Enum from enum import Enum
from beets import mediafile import mediafile
action = Enum('action', action = Enum('action',
['SKIP', 'ASIS', 'TRACKS', 'APPLY', 'ALBUMS', 'RETAG']) ['SKIP', 'ASIS', 'TRACKS', 'APPLY', 'ALBUMS', 'RETAG'])
@ -75,7 +73,7 @@ def _open_state():
# unpickling, including ImportError. We use a catch-all # unpickling, including ImportError. We use a catch-all
# exception to avoid enumerating them all (the docs don't even have a # exception to avoid enumerating them all (the docs don't even have a
# full list!). # full list!).
log.debug(u'state file could not be read: {0}', exc) log.debug('state file could not be read: {0}', exc)
return {} return {}
@ -84,8 +82,8 @@ def _save_state(state):
try: try:
with open(config['statefile'].as_filename(), 'wb') as f: with open(config['statefile'].as_filename(), 'wb') as f:
pickle.dump(state, f) pickle.dump(state, f)
except IOError as exc: except OSError as exc:
log.error(u'state file could not be written: {0}', exc) log.error('state file could not be written: {0}', exc)
# Utilities for reading and writing the beets progress file, which # Utilities for reading and writing the beets progress file, which
@ -174,10 +172,11 @@ def history_get():
# Abstract session class. # Abstract session class.
class ImportSession(object): class ImportSession:
"""Controls an import action. Subclasses should implement methods to """Controls an import action. Subclasses should implement methods to
communicate with the user or otherwise make decisions. communicate with the user or otherwise make decisions.
""" """
def __init__(self, lib, loghandler, paths, query): def __init__(self, lib, loghandler, paths, query):
"""Create a session. `lib` is a Library object. `loghandler` is a """Create a session. `lib` is a Library object. `loghandler` is a
logging.Handler. Either `paths` or `query` is non-null and indicates logging.Handler. Either `paths` or `query` is non-null and indicates
@ -187,7 +186,7 @@ class ImportSession(object):
self.logger = self._setup_logging(loghandler) self.logger = self._setup_logging(loghandler)
self.paths = paths self.paths = paths
self.query = query self.query = query
self._is_resuming = dict() self._is_resuming = {}
self._merged_items = set() self._merged_items = set()
self._merged_dirs = set() self._merged_dirs = set()
@ -222,19 +221,31 @@ class ImportSession(object):
iconfig['resume'] = False iconfig['resume'] = False
iconfig['incremental'] = False iconfig['incremental'] = False
# Copy, move, link, and hardlink are mutually exclusive. if iconfig['reflink']:
iconfig['reflink'] = iconfig['reflink'] \
.as_choice(['auto', True, False])
# Copy, move, reflink, link, and hardlink are mutually exclusive.
if iconfig['move']: if iconfig['move']:
iconfig['copy'] = False iconfig['copy'] = False
iconfig['link'] = False iconfig['link'] = False
iconfig['hardlink'] = False iconfig['hardlink'] = False
iconfig['reflink'] = False
elif iconfig['link']: elif iconfig['link']:
iconfig['copy'] = False iconfig['copy'] = False
iconfig['move'] = False iconfig['move'] = False
iconfig['hardlink'] = False iconfig['hardlink'] = False
iconfig['reflink'] = False
elif iconfig['hardlink']: elif iconfig['hardlink']:
iconfig['copy'] = False iconfig['copy'] = False
iconfig['move'] = False iconfig['move'] = False
iconfig['link'] = False iconfig['link'] = False
iconfig['reflink'] = False
elif iconfig['reflink']:
iconfig['copy'] = False
iconfig['move'] = False
iconfig['link'] = False
iconfig['hardlink'] = False
# Only delete when copying. # Only delete when copying.
if not iconfig['copy']: if not iconfig['copy']:
@ -246,7 +257,7 @@ class ImportSession(object):
"""Log a message about a given album to the importer log. The status """Log a message about a given album to the importer log. The status
should reflect the reason the album couldn't be tagged. should reflect the reason the album couldn't be tagged.
""" """
self.logger.info(u'{0} {1}', status, displayable_path(paths)) self.logger.info('{0} {1}', status, displayable_path(paths))
def log_choice(self, task, duplicate=False): def log_choice(self, task, duplicate=False):
"""Logs the task's current choice if it should be logged. If """Logs the task's current choice if it should be logged. If
@ -257,17 +268,17 @@ class ImportSession(object):
if duplicate: if duplicate:
# Duplicate: log all three choices (skip, keep both, and trump). # Duplicate: log all three choices (skip, keep both, and trump).
if task.should_remove_duplicates: if task.should_remove_duplicates:
self.tag_log(u'duplicate-replace', paths) self.tag_log('duplicate-replace', paths)
elif task.choice_flag in (action.ASIS, action.APPLY): elif task.choice_flag in (action.ASIS, action.APPLY):
self.tag_log(u'duplicate-keep', paths) self.tag_log('duplicate-keep', paths)
elif task.choice_flag is (action.SKIP): elif task.choice_flag is (action.SKIP):
self.tag_log(u'duplicate-skip', paths) self.tag_log('duplicate-skip', paths)
else: else:
# Non-duplicate: log "skip" and "asis" choices. # Non-duplicate: log "skip" and "asis" choices.
if task.choice_flag is action.ASIS: if task.choice_flag is action.ASIS:
self.tag_log(u'asis', paths) self.tag_log('asis', paths)
elif task.choice_flag is action.SKIP: elif task.choice_flag is action.SKIP:
self.tag_log(u'skip', paths) self.tag_log('skip', paths)
def should_resume(self, path): def should_resume(self, path):
raise NotImplementedError raise NotImplementedError
@ -284,7 +295,7 @@ class ImportSession(object):
def run(self): def run(self):
"""Run the import task. """Run the import task.
""" """
self.logger.info(u'import started {0}', time.asctime()) self.logger.info('import started {0}', time.asctime())
self.set_config(config['import']) self.set_config(config['import'])
# Set up the pipeline. # Set up the pipeline.
@ -368,8 +379,8 @@ class ImportSession(object):
"""Mark paths and directories as merged for future reimport tasks. """Mark paths and directories as merged for future reimport tasks.
""" """
self._merged_items.update(paths) self._merged_items.update(paths)
dirs = set([os.path.dirname(path) if os.path.isfile(path) else path dirs = {os.path.dirname(path) if os.path.isfile(path) else path
for path in paths]) for path in paths}
self._merged_dirs.update(dirs) self._merged_dirs.update(dirs)
def is_resuming(self, toppath): def is_resuming(self, toppath):
@ -389,7 +400,7 @@ class ImportSession(object):
# Either accept immediately or prompt for input to decide. # Either accept immediately or prompt for input to decide.
if self.want_resume is True or \ if self.want_resume is True or \
self.should_resume(toppath): self.should_resume(toppath):
log.warning(u'Resuming interrupted import of {0}', log.warning('Resuming interrupted import of {0}',
util.displayable_path(toppath)) util.displayable_path(toppath))
self._is_resuming[toppath] = True self._is_resuming[toppath] = True
else: else:
@ -399,11 +410,12 @@ class ImportSession(object):
# The importer task class. # The importer task class.
class BaseImportTask(object): class BaseImportTask:
"""An abstract base class for importer tasks. """An abstract base class for importer tasks.
Tasks flow through the importer pipeline. Each stage can update Tasks flow through the importer pipeline. Each stage can update
them. """ them. """
def __init__(self, toppath, paths, items): def __init__(self, toppath, paths, items):
"""Create a task. The primary fields that define a task are: """Create a task. The primary fields that define a task are:
@ -457,8 +469,9 @@ class ImportTask(BaseImportTask):
* `finalize()` Update the import progress and cleanup the file * `finalize()` Update the import progress and cleanup the file
system. system.
""" """
def __init__(self, toppath, paths, items): def __init__(self, toppath, paths, items):
super(ImportTask, self).__init__(toppath, paths, items) super().__init__(toppath, paths, items)
self.choice_flag = None self.choice_flag = None
self.cur_album = None self.cur_album = None
self.cur_artist = None self.cur_artist = None
@ -550,28 +563,34 @@ class ImportTask(BaseImportTask):
def remove_duplicates(self, lib): def remove_duplicates(self, lib):
duplicate_items = self.duplicate_items(lib) duplicate_items = self.duplicate_items(lib)
log.debug(u'removing {0} old duplicated items', len(duplicate_items)) log.debug('removing {0} old duplicated items', len(duplicate_items))
for item in duplicate_items: for item in duplicate_items:
item.remove() item.remove()
if lib.directory in util.ancestry(item.path): if lib.directory in util.ancestry(item.path):
log.debug(u'deleting duplicate {0}', log.debug('deleting duplicate {0}',
util.displayable_path(item.path)) util.displayable_path(item.path))
util.remove(item.path) util.remove(item.path)
util.prune_dirs(os.path.dirname(item.path), util.prune_dirs(os.path.dirname(item.path),
lib.directory) lib.directory)
def set_fields(self): def set_fields(self, lib):
"""Sets the fields given at CLI or configuration to the specified """Sets the fields given at CLI or configuration to the specified
values. values, for both the album and all its items.
""" """
items = self.imported_items()
for field, view in config['import']['set_fields'].items(): for field, view in config['import']['set_fields'].items():
value = view.get() value = view.get()
log.debug(u'Set field {1}={2} for {0}', log.debug('Set field {1}={2} for {0}',
displayable_path(self.paths), displayable_path(self.paths),
field, field,
value) value)
self.album[field] = value self.album[field] = value
self.album.store() for item in items:
item[field] = value
with lib.transaction():
for item in items:
item.store()
self.album.store()
def finalize(self, session): def finalize(self, session):
"""Save progress, clean up files, and emit plugin event. """Save progress, clean up files, and emit plugin event.
@ -655,7 +674,7 @@ class ImportTask(BaseImportTask):
return [] return []
duplicates = [] duplicates = []
task_paths = set(i.path for i in self.items if i) task_paths = {i.path for i in self.items if i}
duplicate_query = dbcore.AndQuery(( duplicate_query = dbcore.AndQuery((
dbcore.MatchQuery('albumartist', artist), dbcore.MatchQuery('albumartist', artist),
dbcore.MatchQuery('album', album), dbcore.MatchQuery('album', album),
@ -665,7 +684,7 @@ class ImportTask(BaseImportTask):
# Check whether the album paths are all present in the task # Check whether the album paths are all present in the task
# i.e. album is being completely re-imported by the task, # i.e. album is being completely re-imported by the task,
# in which case it is not a duplicate (will be replaced). # in which case it is not a duplicate (will be replaced).
album_paths = set(i.path for i in album.items()) album_paths = {i.path for i in album.items()}
if not (album_paths <= task_paths): if not (album_paths <= task_paths):
duplicates.append(album) duplicates.append(album)
return duplicates return duplicates
@ -707,7 +726,7 @@ class ImportTask(BaseImportTask):
item.update(changes) item.update(changes)
def manipulate_files(self, operation=None, write=False, session=None): def manipulate_files(self, operation=None, write=False, session=None):
""" Copy, move, link or hardlink (depending on `operation`) the files """ Copy, move, link, hardlink or reflink (depending on `operation`) the files
as well as write metadata. as well as write metadata.
`operation` should be an instance of `util.MoveOperation`. `operation` should be an instance of `util.MoveOperation`.
@ -754,6 +773,8 @@ class ImportTask(BaseImportTask):
self.record_replaced(lib) self.record_replaced(lib)
self.remove_replaced(lib) self.remove_replaced(lib)
self.album = lib.add_album(self.imported_items()) self.album = lib.add_album(self.imported_items())
if 'data_source' in self.imported_items()[0]:
self.album.data_source = self.imported_items()[0].data_source
self.reimport_metadata(lib) self.reimport_metadata(lib)
def record_replaced(self, lib): def record_replaced(self, lib):
@ -772,7 +793,7 @@ class ImportTask(BaseImportTask):
if (not dup_item.album_id or if (not dup_item.album_id or
dup_item.album_id in replaced_album_ids): dup_item.album_id in replaced_album_ids):
continue continue
replaced_album = dup_item.get_album() replaced_album = dup_item._cached_album
if replaced_album: if replaced_album:
replaced_album_ids.add(dup_item.album_id) replaced_album_ids.add(dup_item.album_id)
self.replaced_albums[replaced_album.path] = replaced_album self.replaced_albums[replaced_album.path] = replaced_album
@ -789,8 +810,8 @@ class ImportTask(BaseImportTask):
self.album.artpath = replaced_album.artpath self.album.artpath = replaced_album.artpath
self.album.store() self.album.store()
log.debug( log.debug(
u'Reimported album: added {0}, flexible ' 'Reimported album: added {0}, flexible '
u'attributes {1} from album {2} for {3}', 'attributes {1} from album {2} for {3}',
self.album.added, self.album.added,
replaced_album._values_flex.keys(), replaced_album._values_flex.keys(),
replaced_album.id, replaced_album.id,
@ -803,16 +824,16 @@ class ImportTask(BaseImportTask):
if dup_item.added and dup_item.added != item.added: if dup_item.added and dup_item.added != item.added:
item.added = dup_item.added item.added = dup_item.added
log.debug( log.debug(
u'Reimported item added {0} ' 'Reimported item added {0} '
u'from item {1} for {2}', 'from item {1} for {2}',
item.added, item.added,
dup_item.id, dup_item.id,
displayable_path(item.path) displayable_path(item.path)
) )
item.update(dup_item._values_flex) item.update(dup_item._values_flex)
log.debug( log.debug(
u'Reimported item flexible attributes {0} ' 'Reimported item flexible attributes {0} '
u'from item {1} for {2}', 'from item {1} for {2}',
dup_item._values_flex.keys(), dup_item._values_flex.keys(),
dup_item.id, dup_item.id,
displayable_path(item.path) displayable_path(item.path)
@ -825,10 +846,10 @@ class ImportTask(BaseImportTask):
""" """
for item in self.imported_items(): for item in self.imported_items():
for dup_item in self.replaced_items[item]: for dup_item in self.replaced_items[item]:
log.debug(u'Replacing item {0}: {1}', log.debug('Replacing item {0}: {1}',
dup_item.id, displayable_path(item.path)) dup_item.id, displayable_path(item.path))
dup_item.remove() dup_item.remove()
log.debug(u'{0} of {1} items replaced', log.debug('{0} of {1} items replaced',
sum(bool(l) for l in self.replaced_items.values()), sum(bool(l) for l in self.replaced_items.values()),
len(self.imported_items())) len(self.imported_items()))
@ -866,7 +887,7 @@ class SingletonImportTask(ImportTask):
""" """
def __init__(self, toppath, item): def __init__(self, toppath, item):
super(SingletonImportTask, self).__init__(toppath, [item.path], [item]) super().__init__(toppath, [item.path], [item])
self.item = item self.item = item
self.is_album = False self.is_album = False
self.paths = [item.path] self.paths = [item.path]
@ -932,13 +953,13 @@ class SingletonImportTask(ImportTask):
def reload(self): def reload(self):
self.item.load() self.item.load()
def set_fields(self): def set_fields(self, lib):
"""Sets the fields given at CLI or configuration to the specified """Sets the fields given at CLI or configuration to the specified
values. values, for the singleton item.
""" """
for field, view in config['import']['set_fields'].items(): for field, view in config['import']['set_fields'].items():
value = view.get() value = view.get()
log.debug(u'Set field {1}={2} for {0}', log.debug('Set field {1}={2} for {0}',
displayable_path(self.paths), displayable_path(self.paths),
field, field,
value) value)
@ -959,7 +980,7 @@ class SentinelImportTask(ImportTask):
""" """
def __init__(self, toppath, paths): def __init__(self, toppath, paths):
super(SentinelImportTask, self).__init__(toppath, paths, ()) super().__init__(toppath, paths, ())
# TODO Remove the remaining attributes eventually # TODO Remove the remaining attributes eventually
self.should_remove_duplicates = False self.should_remove_duplicates = False
self.is_album = True self.is_album = True
@ -1003,7 +1024,7 @@ class ArchiveImportTask(SentinelImportTask):
""" """
def __init__(self, toppath): def __init__(self, toppath):
super(ArchiveImportTask, self).__init__(toppath, ()) super().__init__(toppath, ())
self.extracted = False self.extracted = False
@classmethod @classmethod
@ -1032,14 +1053,20 @@ class ArchiveImportTask(SentinelImportTask):
cls._handlers = [] cls._handlers = []
from zipfile import is_zipfile, ZipFile from zipfile import is_zipfile, ZipFile
cls._handlers.append((is_zipfile, ZipFile)) cls._handlers.append((is_zipfile, ZipFile))
from tarfile import is_tarfile, TarFile import tarfile
cls._handlers.append((is_tarfile, TarFile)) cls._handlers.append((tarfile.is_tarfile, tarfile.open))
try: try:
from rarfile import is_rarfile, RarFile from rarfile import is_rarfile, RarFile
except ImportError: except ImportError:
pass pass
else: else:
cls._handlers.append((is_rarfile, RarFile)) cls._handlers.append((is_rarfile, RarFile))
try:
from py7zr import is_7zfile, SevenZipFile
except ImportError:
pass
else:
cls._handlers.append((is_7zfile, SevenZipFile))
return cls._handlers return cls._handlers
@ -1047,7 +1074,7 @@ class ArchiveImportTask(SentinelImportTask):
"""Removes the temporary directory the archive was extracted to. """Removes the temporary directory the archive was extracted to.
""" """
if self.extracted: if self.extracted:
log.debug(u'Removing extracted directory: {0}', log.debug('Removing extracted directory: {0}',
displayable_path(self.toppath)) displayable_path(self.toppath))
shutil.rmtree(self.toppath) shutil.rmtree(self.toppath)
@ -1059,9 +1086,9 @@ class ArchiveImportTask(SentinelImportTask):
if path_test(util.py3_path(self.toppath)): if path_test(util.py3_path(self.toppath)):
break break
extract_to = mkdtemp()
archive = handler_class(util.py3_path(self.toppath), mode='r')
try: try:
extract_to = mkdtemp()
archive = handler_class(util.py3_path(self.toppath), mode='r')
archive.extractall(extract_to) archive.extractall(extract_to)
finally: finally:
archive.close() archive.close()
@ -1069,10 +1096,11 @@ class ArchiveImportTask(SentinelImportTask):
self.toppath = extract_to self.toppath = extract_to
class ImportTaskFactory(object): class ImportTaskFactory:
"""Generate album and singleton import tasks for all media files """Generate album and singleton import tasks for all media files
indicated by a path. indicated by a path.
""" """
def __init__(self, toppath, session): def __init__(self, toppath, session):
"""Create a new task factory. """Create a new task factory.
@ -1110,14 +1138,12 @@ class ImportTaskFactory(object):
if self.session.config['singletons']: if self.session.config['singletons']:
for path in paths: for path in paths:
tasks = self._create(self.singleton(path)) tasks = self._create(self.singleton(path))
for task in tasks: yield from tasks
yield task
yield self.sentinel(dirs) yield self.sentinel(dirs)
else: else:
tasks = self._create(self.album(paths, dirs)) tasks = self._create(self.album(paths, dirs))
for task in tasks: yield from tasks
yield task
# Produce the final sentinel for this toppath to indicate that # Produce the final sentinel for this toppath to indicate that
# it is finished. This is usually just a SentinelImportTask, but # it is finished. This is usually just a SentinelImportTask, but
@ -1165,7 +1191,7 @@ class ImportTaskFactory(object):
"""Return a `SingletonImportTask` for the music file. """Return a `SingletonImportTask` for the music file.
""" """
if self.session.already_imported(self.toppath, [path]): if self.session.already_imported(self.toppath, [path]):
log.debug(u'Skipping previously-imported path: {0}', log.debug('Skipping previously-imported path: {0}',
displayable_path(path)) displayable_path(path))
self.skipped += 1 self.skipped += 1
return None return None
@ -1186,10 +1212,10 @@ class ImportTaskFactory(object):
return None return None
if dirs is None: if dirs is None:
dirs = list(set(os.path.dirname(p) for p in paths)) dirs = list({os.path.dirname(p) for p in paths})
if self.session.already_imported(self.toppath, dirs): if self.session.already_imported(self.toppath, dirs):
log.debug(u'Skipping previously-imported path: {0}', log.debug('Skipping previously-imported path: {0}',
displayable_path(dirs)) displayable_path(dirs))
self.skipped += 1 self.skipped += 1
return None return None
@ -1219,22 +1245,22 @@ class ImportTaskFactory(object):
if not (self.session.config['move'] or if not (self.session.config['move'] or
self.session.config['copy']): self.session.config['copy']):
log.warning(u"Archive importing requires either " log.warning("Archive importing requires either "
u"'copy' or 'move' to be enabled.") "'copy' or 'move' to be enabled.")
return return
log.debug(u'Extracting archive: {0}', log.debug('Extracting archive: {0}',
displayable_path(self.toppath)) displayable_path(self.toppath))
archive_task = ArchiveImportTask(self.toppath) archive_task = ArchiveImportTask(self.toppath)
try: try:
archive_task.extract() archive_task.extract()
except Exception as exc: except Exception as exc:
log.error(u'extraction failed: {0}', exc) log.error('extraction failed: {0}', exc)
return return
# Now read albums from the extracted directory. # Now read albums from the extracted directory.
self.toppath = archive_task.toppath self.toppath = archive_task.toppath
log.debug(u'Archive extracted to: {0}', self.toppath) log.debug('Archive extracted to: {0}', self.toppath)
return archive_task return archive_task
def read_item(self, path): def read_item(self, path):
@ -1250,9 +1276,9 @@ class ImportTaskFactory(object):
# Silently ignore non-music files. # Silently ignore non-music files.
pass pass
elif isinstance(exc.reason, mediafile.UnreadableFileError): elif isinstance(exc.reason, mediafile.UnreadableFileError):
log.warning(u'unreadable file: {0}', displayable_path(path)) log.warning('unreadable file: {0}', displayable_path(path))
else: else:
log.error(u'error reading {0}: {1}', log.error('error reading {0}: {1}',
displayable_path(path), exc) displayable_path(path), exc)
@ -1291,17 +1317,16 @@ def read_tasks(session):
# Generate tasks. # Generate tasks.
task_factory = ImportTaskFactory(toppath, session) task_factory = ImportTaskFactory(toppath, session)
for t in task_factory.tasks(): yield from task_factory.tasks()
yield t
skipped += task_factory.skipped skipped += task_factory.skipped
if not task_factory.imported: if not task_factory.imported:
log.warning(u'No files imported from {0}', log.warning('No files imported from {0}',
displayable_path(toppath)) displayable_path(toppath))
# Show skipped directories (due to incremental/resume). # Show skipped directories (due to incremental/resume).
if skipped: if skipped:
log.info(u'Skipped {0} paths.', skipped) log.info('Skipped {0} paths.', skipped)
def query_tasks(session): def query_tasks(session):
@ -1319,7 +1344,7 @@ def query_tasks(session):
else: else:
# Search for albums. # Search for albums.
for album in session.lib.albums(session.query): for album in session.lib.albums(session.query):
log.debug(u'yielding album {0}: {1} - {2}', log.debug('yielding album {0}: {1} - {2}',
album.id, album.albumartist, album.album) album.id, album.albumartist, album.album)
items = list(album.items()) items = list(album.items())
_freshen_items(items) _freshen_items(items)
@ -1342,7 +1367,7 @@ def lookup_candidates(session, task):
return return
plugins.send('import_task_start', session=session, task=task) plugins.send('import_task_start', session=session, task=task)
log.debug(u'Looking up: {0}', displayable_path(task.paths)) log.debug('Looking up: {0}', displayable_path(task.paths))
# Restrict the initial lookup to IDs specified by the user via the -m # Restrict the initial lookup to IDs specified by the user via the -m
# option. Currently all the IDs are passed onto the tasks directly. # option. Currently all the IDs are passed onto the tasks directly.
@ -1381,8 +1406,7 @@ def user_query(session, task):
def emitter(task): def emitter(task):
for item in task.items: for item in task.items:
task = SingletonImportTask(task.toppath, item) task = SingletonImportTask(task.toppath, item)
for new_task in task.handle_created(session): yield from task.handle_created(session)
yield new_task
yield SentinelImportTask(task.toppath, task.paths) yield SentinelImportTask(task.toppath, task.paths)
return _extend_pipeline(emitter(task), return _extend_pipeline(emitter(task),
@ -1428,30 +1452,30 @@ def resolve_duplicates(session, task):
if task.choice_flag in (action.ASIS, action.APPLY, action.RETAG): if task.choice_flag in (action.ASIS, action.APPLY, action.RETAG):
found_duplicates = task.find_duplicates(session.lib) found_duplicates = task.find_duplicates(session.lib)
if found_duplicates: if found_duplicates:
log.debug(u'found duplicates: {}'.format( log.debug('found duplicates: {}'.format(
[o.id for o in found_duplicates] [o.id for o in found_duplicates]
)) ))
# Get the default action to follow from config. # Get the default action to follow from config.
duplicate_action = config['import']['duplicate_action'].as_choice({ duplicate_action = config['import']['duplicate_action'].as_choice({
u'skip': u's', 'skip': 's',
u'keep': u'k', 'keep': 'k',
u'remove': u'r', 'remove': 'r',
u'merge': u'm', 'merge': 'm',
u'ask': u'a', 'ask': 'a',
}) })
log.debug(u'default action for duplicates: {0}', duplicate_action) log.debug('default action for duplicates: {0}', duplicate_action)
if duplicate_action == u's': if duplicate_action == 's':
# Skip new. # Skip new.
task.set_choice(action.SKIP) task.set_choice(action.SKIP)
elif duplicate_action == u'k': elif duplicate_action == 'k':
# Keep both. Do nothing; leave the choice intact. # Keep both. Do nothing; leave the choice intact.
pass pass
elif duplicate_action == u'r': elif duplicate_action == 'r':
# Remove old. # Remove old.
task.should_remove_duplicates = True task.should_remove_duplicates = True
elif duplicate_action == u'm': elif duplicate_action == 'm':
# Merge duplicates together # Merge duplicates together
task.should_merge_duplicates = True task.should_merge_duplicates = True
else: else:
@ -1471,7 +1495,7 @@ def import_asis(session, task):
if task.skip: if task.skip:
return return
log.info(u'{}', displayable_path(task.paths)) log.info('{}', displayable_path(task.paths))
task.set_choice(action.ASIS) task.set_choice(action.ASIS)
apply_choice(session, task) apply_choice(session, task)
@ -1496,7 +1520,7 @@ def apply_choice(session, task):
# because then the ``ImportTask`` won't have an `album` for which # because then the ``ImportTask`` won't have an `album` for which
# it can set the fields. # it can set the fields.
if config['import']['set_fields']: if config['import']['set_fields']:
task.set_fields() task.set_fields(session.lib)
@pipeline.mutator_stage @pipeline.mutator_stage
@ -1534,6 +1558,8 @@ def manipulate_files(session, task):
operation = MoveOperation.LINK operation = MoveOperation.LINK
elif session.config['hardlink']: elif session.config['hardlink']:
operation = MoveOperation.HARDLINK operation = MoveOperation.HARDLINK
elif session.config['reflink']:
operation = MoveOperation.REFLINK
else: else:
operation = None operation = None
@ -1552,11 +1578,11 @@ def log_files(session, task):
"""A coroutine (pipeline stage) to log each file to be imported. """A coroutine (pipeline stage) to log each file to be imported.
""" """
if isinstance(task, SingletonImportTask): if isinstance(task, SingletonImportTask):
log.info(u'Singleton: {0}', displayable_path(task.item['path'])) log.info('Singleton: {0}', displayable_path(task.item['path']))
elif task.items: elif task.items:
log.info(u'Album: {0}', displayable_path(task.paths[0])) log.info('Album: {0}', displayable_path(task.paths[0]))
for item in task.items: for item in task.items:
log.info(u' {0}', displayable_path(item['path'])) log.info(' {0}', displayable_path(item['path']))
def group_albums(session): def group_albums(session):

574
libs/common/beets/library.py
View file

File diff suppressed because it is too large Load diff

16
libs/common/beets/logging.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson. # Copyright 2016, Adrian Sampson.
# #
@ -21,13 +20,11 @@ that when getLogger(name) instantiates a logger that logger uses
{}-style formatting. {}-style formatting.
""" """
from __future__ import division, absolute_import, print_function
from copy import copy from copy import copy
from logging import * # noqa from logging import * # noqa
import subprocess import subprocess
import threading import threading
import six
def logsafe(val): def logsafe(val):
@ -43,7 +40,7 @@ def logsafe(val):
example. example.
""" """
# Already Unicode. # Already Unicode.
if isinstance(val, six.text_type): if isinstance(val, str):
return val return val
# Bytestring: needs decoding. # Bytestring: needs decoding.
@ -57,7 +54,7 @@ def logsafe(val):
# A "problem" object: needs a workaround. # A "problem" object: needs a workaround.
elif isinstance(val, subprocess.CalledProcessError): elif isinstance(val, subprocess.CalledProcessError):
try: try:
return six.text_type(val) return str(val)
except UnicodeDecodeError: except UnicodeDecodeError:
# An object with a broken __unicode__ formatter. Use __str__ # An object with a broken __unicode__ formatter. Use __str__
# instead. # instead.
@ -74,7 +71,7 @@ class StrFormatLogger(Logger):
instead of %-style formatting. instead of %-style formatting.
""" """
class _LogMessage(object): class _LogMessage:
def __init__(self, msg, args, kwargs): def __init__(self, msg, args, kwargs):
self.msg = msg self.msg = msg
self.args = args self.args = args
@ -82,22 +79,23 @@ class StrFormatLogger(Logger):
def __str__(self): def __str__(self):
args = [logsafe(a) for a in self.args] args = [logsafe(a) for a in self.args]
kwargs = dict((k, logsafe(v)) for (k, v) in self.kwargs.items()) kwargs = {k: logsafe(v) for (k, v) in self.kwargs.items()}
return self.msg.format(*args, **kwargs) return self.msg.format(*args, **kwargs)
def _log(self, level, msg, args, exc_info=None, extra=None, **kwargs): def _log(self, level, msg, args, exc_info=None, extra=None, **kwargs):
"""Log msg.format(*args, **kwargs)""" """Log msg.format(*args, **kwargs)"""
m = self._LogMessage(msg, args, kwargs) m = self._LogMessage(msg, args, kwargs)
return super(StrFormatLogger, self)._log(level, m, (), exc_info, extra) return super()._log(level, m, (), exc_info, extra)
class ThreadLocalLevelLogger(Logger): class ThreadLocalLevelLogger(Logger):
"""A version of `Logger` whose level is thread-local instead of shared. """A version of `Logger` whose level is thread-local instead of shared.
""" """
def __init__(self, name, level=NOTSET): def __init__(self, name, level=NOTSET):
self._thread_level = threading.local() self._thread_level = threading.local()
self.default_level = NOTSET self.default_level = NOTSET
super(ThreadLocalLevelLogger, self).__init__(name, level) super().__init__(name, level)
@property @property
def level(self): def level(self):

2098
libs/common/beets/mediafile.py
View file

File diff suppressed because it is too large Load diff

271
libs/common/beets/plugins.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson. # Copyright 2016, Adrian Sampson.
# #
@ -15,19 +14,19 @@
"""Support for beets plugins.""" """Support for beets plugins."""
from __future__ import division, absolute_import, print_function
import inspect
import traceback import traceback
import re import re
import inspect
import abc
from collections import defaultdict from collections import defaultdict
from functools import wraps from functools import wraps
import beets import beets
from beets import logging from beets import logging
from beets import mediafile import mediafile
import six
PLUGIN_NAMESPACE = 'beetsplug' PLUGIN_NAMESPACE = 'beetsplug'
@ -50,26 +49,28 @@ class PluginLogFilter(logging.Filter):
"""A logging filter that identifies the plugin that emitted a log """A logging filter that identifies the plugin that emitted a log
message. message.
""" """
def __init__(self, plugin): def __init__(self, plugin):
self.prefix = u'{0}: '.format(plugin.name) self.prefix = f'{plugin.name}: '
def filter(self, record): def filter(self, record):
if hasattr(record.msg, 'msg') and isinstance(record.msg.msg, if hasattr(record.msg, 'msg') and isinstance(record.msg.msg,
six.string_types): str):
# A _LogMessage from our hacked-up Logging replacement. # A _LogMessage from our hacked-up Logging replacement.
record.msg.msg = self.prefix + record.msg.msg record.msg.msg = self.prefix + record.msg.msg
elif isinstance(record.msg, six.string_types): elif isinstance(record.msg, str):
record.msg = self.prefix + record.msg record.msg = self.prefix + record.msg
return True return True
# Managing the plugins themselves. # Managing the plugins themselves.
class BeetsPlugin(object): class BeetsPlugin:
"""The base class for all beets plugins. Plugins provide """The base class for all beets plugins. Plugins provide
functionality by defining a subclass of BeetsPlugin and overriding functionality by defining a subclass of BeetsPlugin and overriding
the abstract methods defined here. the abstract methods defined here.
""" """
def __init__(self, name=None): def __init__(self, name=None):
"""Perform one-time plugin setup. """Perform one-time plugin setup.
""" """
@ -127,27 +128,24 @@ class BeetsPlugin(object):
value after the function returns). Also determines which params may not value after the function returns). Also determines which params may not
be sent for backwards-compatibility. be sent for backwards-compatibility.
""" """
argspec = inspect.getargspec(func) argspec = inspect.getfullargspec(func)
@wraps(func) @wraps(func)
def wrapper(*args, **kwargs): def wrapper(*args, **kwargs):
assert self._log.level == logging.NOTSET assert self._log.level == logging.NOTSET
verbosity = beets.config['verbose'].get(int) verbosity = beets.config['verbose'].get(int)
log_level = max(logging.DEBUG, base_log_level - 10 * verbosity) log_level = max(logging.DEBUG, base_log_level - 10 * verbosity)
self._log.setLevel(log_level) self._log.setLevel(log_level)
if argspec.varkw is None:
kwargs = {k: v for k, v in kwargs.items()
if k in argspec.args}
try: try:
try: return func(*args, **kwargs)
return func(*args, **kwargs)
except TypeError as exc:
if exc.args[0].startswith(func.__name__):
# caused by 'func' and not stuff internal to 'func'
kwargs = dict((arg, val) for arg, val in kwargs.items()
if arg in argspec.args)
return func(*args, **kwargs)
else:
raise
finally: finally:
self._log.setLevel(logging.NOTSET) self._log.setLevel(logging.NOTSET)
return wrapper return wrapper
def queries(self): def queries(self):
@ -167,7 +165,7 @@ class BeetsPlugin(object):
""" """
return beets.autotag.hooks.Distance() return beets.autotag.hooks.Distance()
def candidates(self, items, artist, album, va_likely): def candidates(self, items, artist, album, va_likely, extra_tags=None):
"""Should return a sequence of AlbumInfo objects that match the """Should return a sequence of AlbumInfo objects that match the
album whose items are provided. album whose items are provided.
""" """
@ -201,7 +199,7 @@ class BeetsPlugin(object):
``descriptor`` must be an instance of ``mediafile.MediaField``. ``descriptor`` must be an instance of ``mediafile.MediaField``.
""" """
# Defer impor to prevent circular dependency # Defer import to prevent circular dependency
from beets import library from beets import library
mediafile.MediaFile.add_field(name, descriptor) mediafile.MediaFile.add_field(name, descriptor)
library.Item._media_fields.add(name) library.Item._media_fields.add(name)
@ -264,14 +262,14 @@ def load_plugins(names=()):
BeetsPlugin subclasses desired. BeetsPlugin subclasses desired.
""" """
for name in names: for name in names:
modname = '{0}.{1}'.format(PLUGIN_NAMESPACE, name) modname = f'{PLUGIN_NAMESPACE}.{name}'
try: try:
try: try:
namespace = __import__(modname, None, None) namespace = __import__(modname, None, None)
except ImportError as exc: except ImportError as exc:
# Again, this is hacky: # Again, this is hacky:
if exc.args[0].endswith(' ' + name): if exc.args[0].endswith(' ' + name):
log.warning(u'** plugin {0} not found', name) log.warning('** plugin {0} not found', name)
else: else:
raise raise
else: else:
@ -282,7 +280,7 @@ def load_plugins(names=()):
except Exception: except Exception:
log.warning( log.warning(
u'** error loading plugin {}:\n{}', '** error loading plugin {}:\n{}',
name, name,
traceback.format_exc(), traceback.format_exc(),
) )
@ -296,6 +294,11 @@ def find_plugins():
currently loaded beets plugins. Loads the default plugin set currently loaded beets plugins. Loads the default plugin set
first. first.
""" """
if _instances:
# After the first call, use cached instances for performance reasons.
# See https://github.com/beetbox/beets/pull/3810
return list(_instances.values())
load_plugins() load_plugins()
plugins = [] plugins = []
for cls in _classes: for cls in _classes:
@ -329,21 +332,31 @@ def queries():
def types(model_cls): def types(model_cls):
# Gives us `item_types` and `album_types` # Gives us `item_types` and `album_types`
attr_name = '{0}_types'.format(model_cls.__name__.lower()) attr_name = f'{model_cls.__name__.lower()}_types'
types = {} types = {}
for plugin in find_plugins(): for plugin in find_plugins():
plugin_types = getattr(plugin, attr_name, {}) plugin_types = getattr(plugin, attr_name, {})
for field in plugin_types: for field in plugin_types:
if field in types and plugin_types[field] != types[field]: if field in types and plugin_types[field] != types[field]:
raise PluginConflictException( raise PluginConflictException(
u'Plugin {0} defines flexible field {1} ' 'Plugin {} defines flexible field {} '
u'which has already been defined with ' 'which has already been defined with '
u'another type.'.format(plugin.name, field) 'another type.'.format(plugin.name, field)
) )
types.update(plugin_types) types.update(plugin_types)
return types return types
def named_queries(model_cls):
# Gather `item_queries` and `album_queries` from the plugins.
attr_name = f'{model_cls.__name__.lower()}_queries'
queries = {}
for plugin in find_plugins():
plugin_queries = getattr(plugin, attr_name, {})
queries.update(plugin_queries)
return queries
def track_distance(item, info): def track_distance(item, info):
"""Gets the track distance calculated by all loaded plugins. """Gets the track distance calculated by all loaded plugins.
Returns a Distance object. Returns a Distance object.
@ -364,20 +377,19 @@ def album_distance(items, album_info, mapping):
return dist return dist
def candidates(items, artist, album, va_likely): def candidates(items, artist, album, va_likely, extra_tags=None):
"""Gets MusicBrainz candidates for an album from each plugin. """Gets MusicBrainz candidates for an album from each plugin.
""" """
for plugin in find_plugins(): for plugin in find_plugins():
for candidate in plugin.candidates(items, artist, album, va_likely): yield from plugin.candidates(items, artist, album, va_likely,
yield candidate extra_tags)
def item_candidates(item, artist, title): def item_candidates(item, artist, title):
"""Gets MusicBrainz candidates for an item from the plugins. """Gets MusicBrainz candidates for an item from the plugins.
""" """
for plugin in find_plugins(): for plugin in find_plugins():
for item_candidate in plugin.item_candidates(item, artist, title): yield from plugin.item_candidates(item, artist, title)
yield item_candidate
def album_for_id(album_id): def album_for_id(album_id):
@ -470,7 +482,7 @@ def send(event, **arguments):
Return a list of non-None values returned from the handlers. Return a list of non-None values returned from the handlers.
""" """
log.debug(u'Sending event: {0}', event) log.debug('Sending event: {0}', event)
results = [] results = []
for handler in event_handlers()[event]: for handler in event_handlers()[event]:
result = handler(**arguments) result = handler(**arguments)
@ -488,7 +500,7 @@ def feat_tokens(for_artist=True):
feat_words = ['ft', 'featuring', 'feat', 'feat.', 'ft.'] feat_words = ['ft', 'featuring', 'feat', 'feat.', 'ft.']
if for_artist: if for_artist:
feat_words += ['with', 'vs', 'and', 'con', '&'] feat_words += ['with', 'vs', 'and', 'con', '&']
return '(?<=\s)(?:{0})(?=\s)'.format( return r'(?<=\s)(?:{})(?=\s)'.format(
'|'.join(re.escape(x) for x in feat_words) '|'.join(re.escape(x) for x in feat_words)
) )
@ -513,7 +525,7 @@ def sanitize_choices(choices, choices_all):
def sanitize_pairs(pairs, pairs_all): def sanitize_pairs(pairs, pairs_all):
"""Clean up a single-element mapping configuration attribute as returned """Clean up a single-element mapping configuration attribute as returned
by `confit`'s `Pairs` template: keep only two-element tuples present in by Confuse's `Pairs` template: keep only two-element tuples present in
pairs_all, remove duplicate elements, expand ('str', '*') and ('*', '*') pairs_all, remove duplicate elements, expand ('str', '*') and ('*', '*')
wildcards while keeping the original order. Note that ('*', '*') and wildcards while keeping the original order. Note that ('*', '*') and
('*', 'whatever') have the same effect. ('*', 'whatever') have the same effect.
@ -563,3 +575,188 @@ def notify_info_yielded(event):
yield v yield v
return decorated return decorated
return decorator return decorator
def get_distance(config, data_source, info):
"""Returns the ``data_source`` weight and the maximum source weight
for albums or individual tracks.
"""
dist = beets.autotag.Distance()
if info.data_source == data_source:
dist.add('source', config['source_weight'].as_number())
return dist
def apply_item_changes(lib, item, move, pretend, write):
"""Store, move, and write the item according to the arguments.
:param lib: beets library.
:type lib: beets.library.Library
:param item: Item whose changes to apply.
:type item: beets.library.Item
:param move: Move the item if it's in the library.
:type move: bool
:param pretend: Return without moving, writing, or storing the item's
metadata.
:type pretend: bool
:param write: Write the item's metadata to its media file.
:type write: bool
"""
if pretend:
return
from beets import util
# Move the item if it's in the library.
if move and lib.directory in util.ancestry(item.path):
item.move(with_album=False)
if write:
item.try_write()
item.store()
class MetadataSourcePlugin(metaclass=abc.ABCMeta):
def __init__(self):
super().__init__()
self.config.add({'source_weight': 0.5})
@abc.abstractproperty
def id_regex(self):
raise NotImplementedError
@abc.abstractproperty
def data_source(self):
raise NotImplementedError
@abc.abstractproperty
def search_url(self):
raise NotImplementedError
@abc.abstractproperty
def album_url(self):
raise NotImplementedError
@abc.abstractproperty
def track_url(self):
raise NotImplementedError
@abc.abstractmethod
def _search_api(self, query_type, filters, keywords=''):
raise NotImplementedError
@abc.abstractmethod
def album_for_id(self, album_id):
raise NotImplementedError
@abc.abstractmethod
def track_for_id(self, track_id=None, track_data=None):
raise NotImplementedError
@staticmethod
def get_artist(artists, id_key='id', name_key='name'):
"""Returns an artist string (all artists) and an artist_id (the main
artist) for a list of artist object dicts.
For each artist, this function moves articles (such as 'a', 'an',
and 'the') to the front and strips trailing disambiguation numbers. It
returns a tuple containing the comma-separated string of all
normalized artists and the ``id`` of the main/first artist.
:param artists: Iterable of artist dicts or lists returned by API.
:type artists: list[dict] or list[list]
:param id_key: Key or index corresponding to the value of ``id`` for
the main/first artist. Defaults to 'id'.
:type id_key: str or int
:param name_key: Key or index corresponding to values of names
to concatenate for the artist string (containing all artists).
Defaults to 'name'.
:type name_key: str or int
:return: Normalized artist string.
:rtype: str
"""
artist_id = None
artist_names = []
for artist in artists:
if not artist_id:
artist_id = artist[id_key]
name = artist[name_key]
# Strip disambiguation number.
name = re.sub(r' \(\d+\)$', '', name)
# Move articles to the front.
name = re.sub(r'^(.*?), (a|an|the)$', r'\2 \1', name, flags=re.I)
artist_names.append(name)
artist = ', '.join(artist_names).replace(' ,', ',') or None
return artist, artist_id
def _get_id(self, url_type, id_):
"""Parse an ID from its URL if necessary.
:param url_type: Type of URL. Either 'album' or 'track'.
:type url_type: str
:param id_: Album/track ID or URL.
:type id_: str
:return: Album/track ID.
:rtype: str
"""
self._log.debug(
"Searching {} for {} '{}'", self.data_source, url_type, id_
)
match = re.search(self.id_regex['pattern'].format(url_type), str(id_))
if match:
id_ = match.group(self.id_regex['match_group'])
if id_:
return id_
return None
def candidates(self, items, artist, album, va_likely, extra_tags=None):
"""Returns a list of AlbumInfo objects for Search API results
matching an ``album`` and ``artist`` (if not various).
:param items: List of items comprised by an album to be matched.
:type items: list[beets.library.Item]
:param artist: The artist of the album to be matched.
:type artist: str
:param album: The name of the album to be matched.
:type album: str
:param va_likely: True if the album to be matched likely has
Various Artists.
:type va_likely: bool
:return: Candidate AlbumInfo objects.
:rtype: list[beets.autotag.hooks.AlbumInfo]
"""
query_filters = {'album': album}
if not va_likely:
query_filters['artist'] = artist
results = self._search_api(query_type='album', filters=query_filters)
albums = [self.album_for_id(album_id=r['id']) for r in results]
return [a for a in albums if a is not None]
def item_candidates(self, item, artist, title):
"""Returns a list of TrackInfo objects for Search API results
matching ``title`` and ``artist``.
:param item: Singleton item to be matched.
:type item: beets.library.Item
:param artist: The artist of the track to be matched.
:type artist: str
:param title: The title of the track to be matched.
:type title: str
:return: Candidate TrackInfo objects.
:rtype: list[beets.autotag.hooks.TrackInfo]
"""
tracks = self._search_api(
query_type='track', keywords=title, filters={'artist': artist}
)
return [self.track_for_id(track_data=track) for track in tracks]
def album_distance(self, items, album_info, mapping):
return get_distance(
data_source=self.data_source, info=album_info, config=self.config
)
def track_distance(self, item, track_info):
return get_distance(
data_source=self.data_source, info=track_info, config=self.config
)

113
libs/common/beets/random.py Normal file
View file

@ -0,0 +1,113 @@
# This file is part of beets.
# Copyright 2016, Philippe Mongeau.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Get a random song or album from the library.
"""
import random
from operator import attrgetter
from itertools import groupby
def _length(obj, album):
"""Get the duration of an item or album.
"""
if album:
return sum(i.length for i in obj.items())
else:
return obj.length
def _equal_chance_permutation(objs, field='albumartist', random_gen=None):
"""Generate (lazily) a permutation of the objects where every group
with equal values for `field` have an equal chance of appearing in
any given position.
"""
rand = random_gen or random
# Group the objects by artist so we can sample from them.
key = attrgetter(field)
objs.sort(key=key)
objs_by_artists = {}
for artist, v in groupby(objs, key):
objs_by_artists[artist] = list(v)
# While we still have artists with music to choose from, pick one
# randomly and pick a track from that artist.
while objs_by_artists:
# Choose an artist and an object for that artist, removing
# this choice from the pool.
artist = rand.choice(list(objs_by_artists.keys()))
objs_from_artist = objs_by_artists[artist]
i = rand.randint(0, len(objs_from_artist) - 1)
yield objs_from_artist.pop(i)
# Remove the artist if we've used up all of its objects.
if not objs_from_artist:
del objs_by_artists[artist]
def _take(iter, num):
"""Return a list containing the first `num` values in `iter` (or
fewer, if the iterable ends early).
"""
out = []
for val in iter:
out.append(val)
num -= 1
if num <= 0:
break
return out
def _take_time(iter, secs, album):
"""Return a list containing the first values in `iter`, which should
be Item or Album objects, that add up to the given amount of time in
seconds.
"""
out = []
total_time = 0.0
for obj in iter:
length = _length(obj, album)
if total_time + length <= secs:
out.append(obj)
total_time += length
return out
def random_objs(objs, album, number=1, time=None, equal_chance=False,
random_gen=None):
"""Get a random subset of the provided `objs`.
If `number` is provided, produce that many matches. Otherwise, if
`time` is provided, instead select a list whose total time is close
to that number of minutes. If `equal_chance` is true, give each
artist an equal chance of being included so that artists with more
songs are not represented disproportionately.
"""
rand = random_gen or random
# Permute the objects either in a straightforward way or an
# artist-balanced way.
if equal_chance:
perm = _equal_chance_permutation(objs)
else:
perm = objs
rand.shuffle(perm) # N.B. This shuffles the original list.
# Select objects by time our count.
if time:
return _take_time(perm, time * 60, album)
else:
return _take(perm, number)

351
libs/common/beets/ui/__init__.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson. # Copyright 2016, Adrian Sampson.
# #
@ -18,7 +17,6 @@ interface. To invoke the CLI, just call beets.ui.main(). The actual
CLI commands are implemented in the ui.commands module. CLI commands are implemented in the ui.commands module.
""" """
from __future__ import division, absolute_import, print_function
import optparse import optparse
import textwrap import textwrap
@ -30,19 +28,18 @@ import re
import struct import struct
import traceback import traceback
import os.path import os.path
from six.moves import input
from beets import logging from beets import logging
from beets import library from beets import library
from beets import plugins from beets import plugins
from beets import util from beets import util
from beets.util.functemplate import Template from beets.util.functemplate import template
from beets import config from beets import config
from beets.util import confit, as_string from beets.util import as_string
from beets.autotag import mb from beets.autotag import mb
from beets.dbcore import query as db_query from beets.dbcore import query as db_query
from beets.dbcore import db from beets.dbcore import db
import six import confuse
# On Windows platforms, use colorama to support "ANSI" terminal colors. # On Windows platforms, use colorama to support "ANSI" terminal colors.
if sys.platform == 'win32': if sys.platform == 'win32':
@ -61,8 +58,8 @@ log.propagate = False # Don't propagate to root handler.
PF_KEY_QUERIES = { PF_KEY_QUERIES = {
'comp': u'comp:true', 'comp': 'comp:true',
'singleton': u'singleton:true', 'singleton': 'singleton:true',
} }
@ -112,10 +109,7 @@ def decargs(arglist):
"""Given a list of command-line argument bytestrings, attempts to """Given a list of command-line argument bytestrings, attempts to
decode them to Unicode strings when running under Python 2. decode them to Unicode strings when running under Python 2.
""" """
if six.PY2: return arglist
return [s.decode(util.arg_encoding()) for s in arglist]
else:
return arglist
def print_(*strings, **kwargs): def print_(*strings, **kwargs):
@ -130,30 +124,25 @@ def print_(*strings, **kwargs):
(it defaults to a newline). (it defaults to a newline).
""" """
if not strings: if not strings:
strings = [u''] strings = ['']
assert isinstance(strings[0], six.text_type) assert isinstance(strings[0], str)
txt = u' '.join(strings) txt = ' '.join(strings)
txt += kwargs.get('end', u'\n') txt += kwargs.get('end', '\n')
# Encode the string and write it to stdout. # Encode the string and write it to stdout.
if six.PY2: # On Python 3, sys.stdout expects text strings and uses the
# On Python 2, sys.stdout expects bytes. # exception-throwing encoding error policy. To avoid throwing
# errors and use our configurable encoding override, we use the
# underlying bytes buffer instead.
if hasattr(sys.stdout, 'buffer'):
out = txt.encode(_out_encoding(), 'replace') out = txt.encode(_out_encoding(), 'replace')
sys.stdout.write(out) sys.stdout.buffer.write(out)
sys.stdout.buffer.flush()
else: else:
# On Python 3, sys.stdout expects text strings and uses the # In our test harnesses (e.g., DummyOut), sys.stdout.buffer
# exception-throwing encoding error policy. To avoid throwing # does not exist. We instead just record the text string.
# errors and use our configurable encoding override, we use the sys.stdout.write(txt)
# underlying bytes buffer instead.
if hasattr(sys.stdout, 'buffer'):
out = txt.encode(_out_encoding(), 'replace')
sys.stdout.buffer.write(out)
sys.stdout.buffer.flush()
else:
# In our test harnesses (e.g., DummyOut), sys.stdout.buffer
# does not exist. We instead just record the text string.
sys.stdout.write(txt)
# Configuration wrappers. # Configuration wrappers.
@ -203,19 +192,16 @@ def input_(prompt=None):
""" """
# raw_input incorrectly sends prompts to stderr, not stdout, so we # raw_input incorrectly sends prompts to stderr, not stdout, so we
# use print_() explicitly to display prompts. # use print_() explicitly to display prompts.
# http://bugs.python.org/issue1927 # https://bugs.python.org/issue1927
if prompt: if prompt:
print_(prompt, end=u' ') print_(prompt, end=' ')
try: try:
resp = input() resp = input()
except EOFError: except EOFError:
raise UserError(u'stdin stream ended while input required') raise UserError('stdin stream ended while input required')
if six.PY2: return resp
return resp.decode(_in_encoding(), 'ignore')
else:
return resp
def input_options(options, require=False, prompt=None, fallback_prompt=None, def input_options(options, require=False, prompt=None, fallback_prompt=None,
@ -259,7 +245,7 @@ def input_options(options, require=False, prompt=None, fallback_prompt=None,
found_letter = letter found_letter = letter
break break
else: else:
raise ValueError(u'no unambiguous lettering found') raise ValueError('no unambiguous lettering found')
letters[found_letter.lower()] = option letters[found_letter.lower()] = option
index = option.index(found_letter) index = option.index(found_letter)
@ -267,7 +253,7 @@ def input_options(options, require=False, prompt=None, fallback_prompt=None,
# Mark the option's shortcut letter for display. # Mark the option's shortcut letter for display.
if not require and ( if not require and (
(default is None and not numrange and first) or (default is None and not numrange and first) or
(isinstance(default, six.string_types) and (isinstance(default, str) and
found_letter.lower() == default.lower())): found_letter.lower() == default.lower())):
# The first option is the default; mark it. # The first option is the default; mark it.
show_letter = '[%s]' % found_letter.upper() show_letter = '[%s]' % found_letter.upper()
@ -303,11 +289,11 @@ def input_options(options, require=False, prompt=None, fallback_prompt=None,
prompt_part_lengths = [] prompt_part_lengths = []
if numrange: if numrange:
if isinstance(default, int): if isinstance(default, int):
default_name = six.text_type(default) default_name = str(default)
default_name = colorize('action_default', default_name) default_name = colorize('action_default', default_name)
tmpl = '# selection (default %s)' tmpl = '# selection (default %s)'
prompt_parts.append(tmpl % default_name) prompt_parts.append(tmpl % default_name)
prompt_part_lengths.append(len(tmpl % six.text_type(default))) prompt_part_lengths.append(len(tmpl % str(default)))
else: else:
prompt_parts.append('# selection') prompt_parts.append('# selection')
prompt_part_lengths.append(len(prompt_parts[-1])) prompt_part_lengths.append(len(prompt_parts[-1]))
@ -342,9 +328,9 @@ def input_options(options, require=False, prompt=None, fallback_prompt=None,
# Make a fallback prompt too. This is displayed if the user enters # Make a fallback prompt too. This is displayed if the user enters
# something that is not recognized. # something that is not recognized.
if not fallback_prompt: if not fallback_prompt:
fallback_prompt = u'Enter one of ' fallback_prompt = 'Enter one of '
if numrange: if numrange:
fallback_prompt += u'%i-%i, ' % numrange fallback_prompt += '%i-%i, ' % numrange
fallback_prompt += ', '.join(display_letters) + ':' fallback_prompt += ', '.join(display_letters) + ':'
resp = input_(prompt) resp = input_(prompt)
@ -383,34 +369,41 @@ def input_yn(prompt, require=False):
"yes" unless `require` is `True`, in which case there is no default. "yes" unless `require` is `True`, in which case there is no default.
""" """
sel = input_options( sel = input_options(
('y', 'n'), require, prompt, u'Enter Y or N:' ('y', 'n'), require, prompt, 'Enter Y or N:'
) )
return sel == u'y' return sel == 'y'
def input_select_objects(prompt, objs, rep): def input_select_objects(prompt, objs, rep, prompt_all=None):
"""Prompt to user to choose all, none, or some of the given objects. """Prompt to user to choose all, none, or some of the given objects.
Return the list of selected objects. Return the list of selected objects.
`prompt` is the prompt string to use for each question (it should be `prompt` is the prompt string to use for each question (it should be
phrased as an imperative verb). `rep` is a function to call on each phrased as an imperative verb). If `prompt_all` is given, it is used
object to print it out when confirming objects individually. instead of `prompt` for the first (yes(/no/select) question.
`rep` is a function to call on each object to print it out when confirming
objects individually.
""" """
choice = input_options( choice = input_options(
(u'y', u'n', u's'), False, ('y', 'n', 's'), False,
u'%s? (Yes/no/select)' % prompt) '%s? (Yes/no/select)' % (prompt_all or prompt))
print() # Blank line. print() # Blank line.
if choice == u'y': # Yes. if choice == 'y': # Yes.
return objs return objs
elif choice == u's': # Select. elif choice == 's': # Select.
out = [] out = []
for obj in objs: for obj in objs:
rep(obj) rep(obj)
if input_yn(u'%s? (yes/no)' % prompt, True): answer = input_options(
('y', 'n', 'q'), True, '%s? (yes/no/quit)' % prompt,
'Enter Y or N:'
)
if answer == 'y':
out.append(obj) out.append(obj)
print() # go to a new line elif answer == 'q':
return out
return out return out
else: # No. else: # No.
@ -421,14 +414,14 @@ def input_select_objects(prompt, objs, rep):
def human_bytes(size): def human_bytes(size):
"""Formats size, a number of bytes, in a human-readable way.""" """Formats size, a number of bytes, in a human-readable way."""
powers = [u'', u'K', u'M', u'G', u'T', u'P', u'E', u'Z', u'Y', u'H'] powers = ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y', 'H']
unit = 'B' unit = 'B'
for power in powers: for power in powers:
if size < 1024: if size < 1024:
return u"%3.1f %s%s" % (size, power, unit) return f"{size:3.1f} {power}{unit}"
size /= 1024.0 size /= 1024.0
unit = u'iB' unit = 'iB'
return u"big" return "big"
def human_seconds(interval): def human_seconds(interval):
@ -436,13 +429,13 @@ def human_seconds(interval):
interval using English words. interval using English words.
""" """
units = [ units = [
(1, u'second'), (1, 'second'),
(60, u'minute'), (60, 'minute'),
(60, u'hour'), (60, 'hour'),
(24, u'day'), (24, 'day'),
(7, u'week'), (7, 'week'),
(52, u'year'), (52, 'year'),
(10, u'decade'), (10, 'decade'),
] ]
for i in range(len(units) - 1): for i in range(len(units) - 1):
increment, suffix = units[i] increment, suffix = units[i]
@ -455,7 +448,7 @@ def human_seconds(interval):
increment, suffix = units[-1] increment, suffix = units[-1]
interval /= float(increment) interval /= float(increment)
return u"%3.1f %ss" % (interval, suffix) return f"{interval:3.1f} {suffix}s"
def human_seconds_short(interval): def human_seconds_short(interval):
@ -463,13 +456,13 @@ def human_seconds_short(interval):
string. string.
""" """
interval = int(interval) interval = int(interval)
return u'%i:%02i' % (interval // 60, interval % 60) return '%i:%02i' % (interval // 60, interval % 60)
# Colorization. # Colorization.
# ANSI terminal colorization code heavily inspired by pygments: # ANSI terminal colorization code heavily inspired by pygments:
# http://dev.pocoo.org/hg/pygments-main/file/b2deea5b5030/pygments/console.py # https://bitbucket.org/birkenfeld/pygments-main/src/default/pygments/console.py
# (pygments is by Tim Hatch, Armin Ronacher, et al.) # (pygments is by Tim Hatch, Armin Ronacher, et al.)
COLOR_ESCAPE = "\x1b[" COLOR_ESCAPE = "\x1b["
DARK_COLORS = { DARK_COLORS = {
@ -516,7 +509,7 @@ def _colorize(color, text):
elif color in LIGHT_COLORS: elif color in LIGHT_COLORS:
escape = COLOR_ESCAPE + "%i;01m" % (LIGHT_COLORS[color] + 30) escape = COLOR_ESCAPE + "%i;01m" % (LIGHT_COLORS[color] + 30)
else: else:
raise ValueError(u'no such color %s', color) raise ValueError('no such color %s', color)
return escape + text + RESET_COLOR return escape + text + RESET_COLOR
@ -524,22 +517,22 @@ def colorize(color_name, text):
"""Colorize text if colored output is enabled. (Like _colorize but """Colorize text if colored output is enabled. (Like _colorize but
conditional.) conditional.)
""" """
if config['ui']['color']: if not config['ui']['color'] or 'NO_COLOR' in os.environ.keys():
global COLORS
if not COLORS:
COLORS = dict((name,
config['ui']['colors'][name].as_str())
for name in COLOR_NAMES)
# In case a 3rd party plugin is still passing the actual color ('red')
# instead of the abstract color name ('text_error')
color = COLORS.get(color_name)
if not color:
log.debug(u'Invalid color_name: {0}', color_name)
color = color_name
return _colorize(color, text)
else:
return text return text
global COLORS
if not COLORS:
COLORS = {name:
config['ui']['colors'][name].as_str()
for name in COLOR_NAMES}
# In case a 3rd party plugin is still passing the actual color ('red')
# instead of the abstract color name ('text_error')
color = COLORS.get(color_name)
if not color:
log.debug('Invalid color_name: {0}', color_name)
color = color_name
return _colorize(color, text)
def _colordiff(a, b, highlight='text_highlight', def _colordiff(a, b, highlight='text_highlight',
minor_highlight='text_highlight_minor'): minor_highlight='text_highlight_minor'):
@ -548,11 +541,11 @@ def _colordiff(a, b, highlight='text_highlight',
highlighted intelligently to show differences; other values are highlighted intelligently to show differences; other values are
stringified and highlighted in their entirety. stringified and highlighted in their entirety.
""" """
if not isinstance(a, six.string_types) \ if not isinstance(a, str) \
or not isinstance(b, six.string_types): or not isinstance(b, str):
# Non-strings: use ordinary equality. # Non-strings: use ordinary equality.
a = six.text_type(a) a = str(a)
b = six.text_type(b) b = str(b)
if a == b: if a == b:
return a, b return a, b
else: else:
@ -590,7 +583,7 @@ def _colordiff(a, b, highlight='text_highlight',
else: else:
assert(False) assert(False)
return u''.join(a_out), u''.join(b_out) return ''.join(a_out), ''.join(b_out)
def colordiff(a, b, highlight='text_highlight'): def colordiff(a, b, highlight='text_highlight'):
@ -600,7 +593,7 @@ def colordiff(a, b, highlight='text_highlight'):
if config['ui']['color']: if config['ui']['color']:
return _colordiff(a, b, highlight) return _colordiff(a, b, highlight)
else: else:
return six.text_type(a), six.text_type(b) return str(a), str(b)
def get_path_formats(subview=None): def get_path_formats(subview=None):
@ -611,12 +604,12 @@ def get_path_formats(subview=None):
subview = subview or config['paths'] subview = subview or config['paths']
for query, view in subview.items(): for query, view in subview.items():
query = PF_KEY_QUERIES.get(query, query) # Expand common queries. query = PF_KEY_QUERIES.get(query, query) # Expand common queries.
path_formats.append((query, Template(view.as_str()))) path_formats.append((query, template(view.as_str())))
return path_formats return path_formats
def get_replacements(): def get_replacements():
"""Confit validation function that reads regex/string pairs. """Confuse validation function that reads regex/string pairs.
""" """
replacements = [] replacements = []
for pattern, repl in config['replace'].get(dict).items(): for pattern, repl in config['replace'].get(dict).items():
@ -625,7 +618,7 @@ def get_replacements():
replacements.append((re.compile(pattern), repl)) replacements.append((re.compile(pattern), repl))
except re.error: except re.error:
raise UserError( raise UserError(
u'malformed regular expression in replace: {0}'.format( 'malformed regular expression in replace: {}'.format(
pattern pattern
) )
) )
@ -646,7 +639,7 @@ def term_width():
try: try:
buf = fcntl.ioctl(0, termios.TIOCGWINSZ, ' ' * 4) buf = fcntl.ioctl(0, termios.TIOCGWINSZ, ' ' * 4)
except IOError: except OSError:
return fallback return fallback
try: try:
height, width = struct.unpack('hh', buf) height, width = struct.unpack('hh', buf)
@ -658,10 +651,10 @@ def term_width():
FLOAT_EPSILON = 0.01 FLOAT_EPSILON = 0.01
def _field_diff(field, old, new): def _field_diff(field, old, old_fmt, new, new_fmt):
"""Given two Model objects, format their values for `field` and """Given two Model objects and their formatted views, format their values
highlight changes among them. Return a human-readable string. If the for `field` and highlight changes among them. Return a human-readable
value has not changed, return None instead. string. If the value has not changed, return None instead.
""" """
oldval = old.get(field) oldval = old.get(field)
newval = new.get(field) newval = new.get(field)
@ -674,18 +667,18 @@ def _field_diff(field, old, new):
return None return None
# Get formatted values for output. # Get formatted values for output.
oldstr = old.formatted().get(field, u'') oldstr = old_fmt.get(field, '')
newstr = new.formatted().get(field, u'') newstr = new_fmt.get(field, '')
# For strings, highlight changes. For others, colorize the whole # For strings, highlight changes. For others, colorize the whole
# thing. # thing.
if isinstance(oldval, six.string_types): if isinstance(oldval, str):
oldstr, newstr = colordiff(oldval, newstr) oldstr, newstr = colordiff(oldval, newstr)
else: else:
oldstr = colorize('text_error', oldstr) oldstr = colorize('text_error', oldstr)
newstr = colorize('text_error', newstr) newstr = colorize('text_error', newstr)
return u'{0} -> {1}'.format(oldstr, newstr) return f'{oldstr} -> {newstr}'
def show_model_changes(new, old=None, fields=None, always=False): def show_model_changes(new, old=None, fields=None, always=False):
@ -700,6 +693,11 @@ def show_model_changes(new, old=None, fields=None, always=False):
""" """
old = old or new._db._get(type(new), new.id) old = old or new._db._get(type(new), new.id)
# Keep the formatted views around instead of re-creating them in each
# iteration step
old_fmt = old.formatted()
new_fmt = new.formatted()
# Build up lines showing changed fields. # Build up lines showing changed fields.
changes = [] changes = []
for field in old: for field in old:
@ -708,25 +706,25 @@ def show_model_changes(new, old=None, fields=None, always=False):
continue continue
# Detect and show difference for this field. # Detect and show difference for this field.
line = _field_diff(field, old, new) line = _field_diff(field, old, old_fmt, new, new_fmt)
if line: if line:
changes.append(u' {0}: {1}'.format(field, line)) changes.append(f' {field}: {line}')
# New fields. # New fields.
for field in set(new) - set(old): for field in set(new) - set(old):
if fields and field not in fields: if fields and field not in fields:
continue continue
changes.append(u' {0}: {1}'.format( changes.append(' {}: {}'.format(
field, field,
colorize('text_highlight', new.formatted()[field]) colorize('text_highlight', new_fmt[field])
)) ))
# Print changes. # Print changes.
if changes or always: if changes or always:
print_(format(old)) print_(format(old))
if changes: if changes:
print_(u'\n'.join(changes)) print_('\n'.join(changes))
return bool(changes) return bool(changes)
@ -759,15 +757,21 @@ def show_path_changes(path_changes):
if max_width > col_width: if max_width > col_width:
# Print every change over two lines # Print every change over two lines
for source, dest in zip(sources, destinations): for source, dest in zip(sources, destinations):
log.info(u'{0} \n -> {1}', source, dest) color_source, color_dest = colordiff(source, dest)
print_('{0} \n -> {1}'.format(color_source, color_dest))
else: else:
# Print every change on a single line, and add a header # Print every change on a single line, and add a header
title_pad = max_width - len('Source ') + len(' -> ') title_pad = max_width - len('Source ') + len(' -> ')
log.info(u'Source {0} Destination', ' ' * title_pad) print_('Source {0} Destination'.format(' ' * title_pad))
for source, dest in zip(sources, destinations): for source, dest in zip(sources, destinations):
pad = max_width - len(source) pad = max_width - len(source)
log.info(u'{0} {1} -> {2}', source, ' ' * pad, dest) color_source, color_dest = colordiff(source, dest)
print_('{0} {1} -> {2}'.format(
color_source,
' ' * pad,
color_dest,
))
# Helper functions for option parsing. # Helper functions for option parsing.
@ -783,22 +787,25 @@ def _store_dict(option, opt_str, value, parser):
if option_values is None: if option_values is None:
# This is the first supplied ``key=value`` pair of option. # This is the first supplied ``key=value`` pair of option.
# Initialize empty dictionary and get a reference to it. # Initialize empty dictionary and get a reference to it.
setattr(parser.values, dest, dict()) setattr(parser.values, dest, {})
option_values = getattr(parser.values, dest) option_values = getattr(parser.values, dest)
# Decode the argument using the platform's argument encoding.
value = util.text_string(value, util.arg_encoding())
try: try:
key, value = map(lambda s: util.text_string(s), value.split('=')) key, value = value.split('=', 1)
if not (key and value): if not (key and value):
raise ValueError raise ValueError
except ValueError: except ValueError:
raise UserError( raise UserError(
"supplied argument `{0}' is not of the form `key=value'" "supplied argument `{}' is not of the form `key=value'"
.format(value)) .format(value))
option_values[key] = value option_values[key] = value
class CommonOptionsParser(optparse.OptionParser, object): class CommonOptionsParser(optparse.OptionParser):
"""Offers a simple way to add common formatting options. """Offers a simple way to add common formatting options.
Options available include: Options available include:
@ -813,8 +820,9 @@ class CommonOptionsParser(optparse.OptionParser, object):
Each method is fully documented in the related method. Each method is fully documented in the related method.
""" """
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
super(CommonOptionsParser, self).__init__(*args, **kwargs) super().__init__(*args, **kwargs)
self._album_flags = False self._album_flags = False
# this serves both as an indicator that we offer the feature AND allows # this serves both as an indicator that we offer the feature AND allows
# us to check whether it has been specified on the CLI - bypassing the # us to check whether it has been specified on the CLI - bypassing the
@ -828,7 +836,7 @@ class CommonOptionsParser(optparse.OptionParser, object):
Sets the album property on the options extracted from the CLI. Sets the album property on the options extracted from the CLI.
""" """
album = optparse.Option(*flags, action='store_true', album = optparse.Option(*flags, action='store_true',
help=u'match albums instead of tracks') help='match albums instead of tracks')
self.add_option(album) self.add_option(album)
self._album_flags = set(flags) self._album_flags = set(flags)
@ -846,7 +854,7 @@ class CommonOptionsParser(optparse.OptionParser, object):
elif value: elif value:
value, = decargs([value]) value, = decargs([value])
else: else:
value = u'' value = ''
parser.values.format = value parser.values.format = value
if target: if target:
@ -873,14 +881,14 @@ class CommonOptionsParser(optparse.OptionParser, object):
By default this affects both items and albums. If add_album_option() By default this affects both items and albums. If add_album_option()
is used then the target will be autodetected. is used then the target will be autodetected.
Sets the format property to u'$path' on the options extracted from the Sets the format property to '$path' on the options extracted from the
CLI. CLI.
""" """
path = optparse.Option(*flags, nargs=0, action='callback', path = optparse.Option(*flags, nargs=0, action='callback',
callback=self._set_format, callback=self._set_format,
callback_kwargs={'fmt': u'$path', callback_kwargs={'fmt': '$path',
'store_true': True}, 'store_true': True},
help=u'print paths for matched items or albums') help='print paths for matched items or albums')
self.add_option(path) self.add_option(path)
def add_format_option(self, flags=('-f', '--format'), target=None): def add_format_option(self, flags=('-f', '--format'), target=None):
@ -900,7 +908,7 @@ class CommonOptionsParser(optparse.OptionParser, object):
""" """
kwargs = {} kwargs = {}
if target: if target:
if isinstance(target, six.string_types): if isinstance(target, str):
target = {'item': library.Item, target = {'item': library.Item,
'album': library.Album}[target] 'album': library.Album}[target]
kwargs['target'] = target kwargs['target'] = target
@ -908,7 +916,7 @@ class CommonOptionsParser(optparse.OptionParser, object):
opt = optparse.Option(*flags, action='callback', opt = optparse.Option(*flags, action='callback',
callback=self._set_format, callback=self._set_format,
callback_kwargs=kwargs, callback_kwargs=kwargs,
help=u'print with custom format') help='print with custom format')
self.add_option(opt) self.add_option(opt)
def add_all_common_options(self): def add_all_common_options(self):
@ -923,14 +931,15 @@ class CommonOptionsParser(optparse.OptionParser, object):
# #
# This is a fairly generic subcommand parser for optparse. It is # This is a fairly generic subcommand parser for optparse. It is
# maintained externally here: # maintained externally here:
# http://gist.github.com/462717 # https://gist.github.com/462717
# There you will also find a better description of the code and a more # There you will also find a better description of the code and a more
# succinct example program. # succinct example program.
class Subcommand(object): class Subcommand:
"""A subcommand of a root command-line application that may be """A subcommand of a root command-line application that may be
invoked by a SubcommandOptionParser. invoked by a SubcommandOptionParser.
""" """
def __init__(self, name, parser=None, help='', aliases=(), hide=False): def __init__(self, name, parser=None, help='', aliases=(), hide=False):
"""Creates a new subcommand. name is the primary way to invoke """Creates a new subcommand. name is the primary way to invoke
the subcommand; aliases are alternate names. parser is an the subcommand; aliases are alternate names. parser is an
@ -958,7 +967,7 @@ class Subcommand(object):
@root_parser.setter @root_parser.setter
def root_parser(self, root_parser): def root_parser(self, root_parser):
self._root_parser = root_parser self._root_parser = root_parser
self.parser.prog = '{0} {1}'.format( self.parser.prog = '{} {}'.format(
as_string(root_parser.get_prog_name()), self.name) as_string(root_parser.get_prog_name()), self.name)
@ -974,13 +983,13 @@ class SubcommandsOptionParser(CommonOptionsParser):
""" """
# A more helpful default usage. # A more helpful default usage.
if 'usage' not in kwargs: if 'usage' not in kwargs:
kwargs['usage'] = u""" kwargs['usage'] = """
%prog COMMAND [ARGS...] %prog COMMAND [ARGS...]
%prog help COMMAND""" %prog help COMMAND"""
kwargs['add_help_option'] = False kwargs['add_help_option'] = False
# Super constructor. # Super constructor.
super(SubcommandsOptionParser, self).__init__(*args, **kwargs) super().__init__(*args, **kwargs)
# Our root parser needs to stop on the first unrecognized argument. # Our root parser needs to stop on the first unrecognized argument.
self.disable_interspersed_args() self.disable_interspersed_args()
@ -997,7 +1006,7 @@ class SubcommandsOptionParser(CommonOptionsParser):
# Add the list of subcommands to the help message. # Add the list of subcommands to the help message.
def format_help(self, formatter=None): def format_help(self, formatter=None):
# Get the original help message, to which we will append. # Get the original help message, to which we will append.
out = super(SubcommandsOptionParser, self).format_help(formatter) out = super().format_help(formatter)
if formatter is None: if formatter is None:
formatter = self.formatter formatter = self.formatter
@ -1083,7 +1092,7 @@ class SubcommandsOptionParser(CommonOptionsParser):
cmdname = args.pop(0) cmdname = args.pop(0)
subcommand = self._subcommand_for_name(cmdname) subcommand = self._subcommand_for_name(cmdname)
if not subcommand: if not subcommand:
raise UserError(u"unknown command '{0}'".format(cmdname)) raise UserError(f"unknown command '{cmdname}'")
suboptions, subargs = subcommand.parse_args(args) suboptions, subargs = subcommand.parse_args(args)
return subcommand, suboptions, subargs return subcommand, suboptions, subargs
@ -1094,26 +1103,32 @@ optparse.Option.ALWAYS_TYPED_ACTIONS += ('callback',)
# The main entry point and bootstrapping. # The main entry point and bootstrapping.
def _load_plugins(config): def _load_plugins(options, config):
"""Load the plugins specified in the configuration. """Load the plugins specified on the command line or in the configuration.
""" """
paths = config['pluginpath'].as_str_seq(split=False) paths = config['pluginpath'].as_str_seq(split=False)
paths = [util.normpath(p) for p in paths] paths = [util.normpath(p) for p in paths]
log.debug(u'plugin paths: {0}', util.displayable_path(paths)) log.debug('plugin paths: {0}', util.displayable_path(paths))
# On Python 3, the search paths need to be unicode. # On Python 3, the search paths need to be unicode.
paths = [util.py3_path(p) for p in paths] paths = [util.py3_path(p) for p in paths]
# Extend the `beetsplug` package to include the plugin paths. # Extend the `beetsplug` package to include the plugin paths.
import beetsplug import beetsplug
beetsplug.__path__ = paths + beetsplug.__path__ beetsplug.__path__ = paths + list(beetsplug.__path__)
# For backwards compatibility, also support plugin paths that # For backwards compatibility, also support plugin paths that
# *contain* a `beetsplug` package. # *contain* a `beetsplug` package.
sys.path += paths sys.path += paths
plugins.load_plugins(config['plugins'].as_str_seq()) # If we were given any plugins on the command line, use those.
plugins.send("pluginload") if options.plugins is not None:
plugin_list = (options.plugins.split(',')
if len(options.plugins) > 0 else [])
else:
plugin_list = config['plugins'].as_str_seq()
plugins.load_plugins(plugin_list)
return plugins return plugins
@ -1127,7 +1142,20 @@ def _setup(options, lib=None):
config = _configure(options) config = _configure(options)
plugins = _load_plugins(config) plugins = _load_plugins(options, config)
# Add types and queries defined by plugins.
plugin_types_album = plugins.types(library.Album)
library.Album._types.update(plugin_types_album)
item_types = plugin_types_album.copy()
item_types.update(library.Item._types)
item_types.update(plugins.types(library.Item))
library.Item._types = item_types
library.Item._queries.update(plugins.named_queries(library.Item))
library.Album._queries.update(plugins.named_queries(library.Album))
plugins.send("pluginload")
# Get the default subcommands. # Get the default subcommands.
from beets.ui.commands import default_commands from beets.ui.commands import default_commands
@ -1138,8 +1166,6 @@ def _setup(options, lib=None):
if lib is None: if lib is None:
lib = _open_library(config) lib = _open_library(config)
plugins.send("library_opened", lib=lib) plugins.send("library_opened", lib=lib)
library.Item._types.update(plugins.types(library.Item))
library.Album._types.update(plugins.types(library.Album))
return subcommands, plugins, lib return subcommands, plugins, lib
@ -1165,18 +1191,18 @@ def _configure(options):
log.set_global_level(logging.INFO) log.set_global_level(logging.INFO)
if overlay_path: if overlay_path:
log.debug(u'overlaying configuration: {0}', log.debug('overlaying configuration: {0}',
util.displayable_path(overlay_path)) util.displayable_path(overlay_path))
config_path = config.user_config_path() config_path = config.user_config_path()
if os.path.isfile(config_path): if os.path.isfile(config_path):
log.debug(u'user configuration: {0}', log.debug('user configuration: {0}',
util.displayable_path(config_path)) util.displayable_path(config_path))
else: else:
log.debug(u'no user configuration found at {0}', log.debug('no user configuration found at {0}',
util.displayable_path(config_path)) util.displayable_path(config_path))
log.debug(u'data directory: {0}', log.debug('data directory: {0}',
util.displayable_path(config.config_dir())) util.displayable_path(config.config_dir()))
return config return config
@ -1193,13 +1219,14 @@ def _open_library(config):
get_replacements(), get_replacements(),
) )
lib.get_item(0) # Test database connection. lib.get_item(0) # Test database connection.
except (sqlite3.OperationalError, sqlite3.DatabaseError): except (sqlite3.OperationalError, sqlite3.DatabaseError) as db_error:
log.debug(u'{}', traceback.format_exc()) log.debug('{}', traceback.format_exc())
raise UserError(u"database file {0} could not be opened".format( raise UserError("database file {} cannot not be opened: {}".format(
util.displayable_path(dbpath) util.displayable_path(dbpath),
db_error
)) ))
log.debug(u'library database: {0}\n' log.debug('library database: {0}\n'
u'library directory: {1}', 'library directory: {1}',
util.displayable_path(lib.path), util.displayable_path(lib.path),
util.displayable_path(lib.directory)) util.displayable_path(lib.directory))
return lib return lib
@ -1213,15 +1240,17 @@ def _raw_main(args, lib=None):
parser.add_format_option(flags=('--format-item',), target=library.Item) parser.add_format_option(flags=('--format-item',), target=library.Item)
parser.add_format_option(flags=('--format-album',), target=library.Album) parser.add_format_option(flags=('--format-album',), target=library.Album)
parser.add_option('-l', '--library', dest='library', parser.add_option('-l', '--library', dest='library',
help=u'library database file to use') help='library database file to use')
parser.add_option('-d', '--directory', dest='directory', parser.add_option('-d', '--directory', dest='directory',
help=u"destination music directory") help="destination music directory")
parser.add_option('-v', '--verbose', dest='verbose', action='count', parser.add_option('-v', '--verbose', dest='verbose', action='count',
help=u'log more details (use twice for even more)') help='log more details (use twice for even more)')
parser.add_option('-c', '--config', dest='config', parser.add_option('-c', '--config', dest='config',
help=u'path to configuration file') help='path to configuration file')
parser.add_option('-p', '--plugins', dest='plugins',
help='a comma-separated list of plugins to load')
parser.add_option('-h', '--help', dest='help', action='store_true', parser.add_option('-h', '--help', dest='help', action='store_true',
help=u'show this help message and exit') help='show this help message and exit')
parser.add_option('--version', dest='version', action='store_true', parser.add_option('--version', dest='version', action='store_true',
help=optparse.SUPPRESS_HELP) help=optparse.SUPPRESS_HELP)
@ -1256,7 +1285,7 @@ def main(args=None):
_raw_main(args) _raw_main(args)
except UserError as exc: except UserError as exc:
message = exc.args[0] if exc.args else None message = exc.args[0] if exc.args else None
log.error(u'error: {0}', message) log.error('error: {0}', message)
sys.exit(1) sys.exit(1)
except util.HumanReadableException as exc: except util.HumanReadableException as exc:
exc.log(log) exc.log(log)
@ -1267,13 +1296,13 @@ def main(args=None):
log.debug('{}', traceback.format_exc()) log.debug('{}', traceback.format_exc())
log.error('{}', exc) log.error('{}', exc)
sys.exit(1) sys.exit(1)
except confit.ConfigError as exc: except confuse.ConfigError as exc:
log.error(u'configuration error: {0}', exc) log.error('configuration error: {0}', exc)
sys.exit(1) sys.exit(1)
except db_query.InvalidQueryError as exc: except db_query.InvalidQueryError as exc:
log.error(u'invalid query: {0}', exc) log.error('invalid query: {0}', exc)
sys.exit(1) sys.exit(1)
except IOError as exc: except OSError as exc:
if exc.errno == errno.EPIPE: if exc.errno == errno.EPIPE:
# "Broken pipe". End silently. # "Broken pipe". End silently.
sys.stderr.close() sys.stderr.close()
@ -1281,11 +1310,11 @@ def main(args=None):
raise raise
except KeyboardInterrupt: except KeyboardInterrupt:
# Silently ignore ^C except in verbose mode. # Silently ignore ^C except in verbose mode.
log.debug(u'{}', traceback.format_exc()) log.debug('{}', traceback.format_exc())
except db.DBAccessError as exc: except db.DBAccessError as exc:
log.error( log.error(
u'database access error: {0}\n' 'database access error: {0}\n'
u'the library file might have a permissions problem', 'the library file might have a permissions problem',
exc exc
) )
sys.exit(1) sys.exit(1)

727
libs/common/beets/ui/commands.py
View file

File diff suppressed because it is too large Load diff

292
libs/common/beets/util/__init__.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson. # Copyright 2016, Adrian Sampson.
# #
@ -15,28 +14,28 @@
"""Miscellaneous utility functions.""" """Miscellaneous utility functions."""
from __future__ import division, absolute_import, print_function
import os import os
import sys import sys
import errno import errno
import locale import locale
import re import re
import tempfile
import shutil import shutil
import fnmatch import fnmatch
from collections import Counter import functools
from collections import Counter, namedtuple
from multiprocessing.pool import ThreadPool
import traceback import traceback
import subprocess import subprocess
import platform import platform
import shlex import shlex
from beets.util import hidden from beets.util import hidden
import six
from unidecode import unidecode from unidecode import unidecode
from enum import Enum from enum import Enum
MAX_FILENAME_LENGTH = 200 MAX_FILENAME_LENGTH = 200
WINDOWS_MAGIC_PREFIX = u'\\\\?\\' WINDOWS_MAGIC_PREFIX = '\\\\?\\'
SNI_SUPPORTED = sys.version_info >= (2, 7, 9)
class HumanReadableException(Exception): class HumanReadableException(Exception):
@ -58,27 +57,27 @@ class HumanReadableException(Exception):
self.reason = reason self.reason = reason
self.verb = verb self.verb = verb
self.tb = tb self.tb = tb
super(HumanReadableException, self).__init__(self.get_message()) super().__init__(self.get_message())
def _gerund(self): def _gerund(self):
"""Generate a (likely) gerund form of the English verb. """Generate a (likely) gerund form of the English verb.
""" """
if u' ' in self.verb: if ' ' in self.verb:
return self.verb return self.verb
gerund = self.verb[:-1] if self.verb.endswith(u'e') else self.verb gerund = self.verb[:-1] if self.verb.endswith('e') else self.verb
gerund += u'ing' gerund += 'ing'
return gerund return gerund
def _reasonstr(self): def _reasonstr(self):
"""Get the reason as a string.""" """Get the reason as a string."""
if isinstance(self.reason, six.text_type): if isinstance(self.reason, str):
return self.reason return self.reason
elif isinstance(self.reason, bytes): elif isinstance(self.reason, bytes):
return self.reason.decode('utf-8', 'ignore') return self.reason.decode('utf-8', 'ignore')
elif hasattr(self.reason, 'strerror'): # i.e., EnvironmentError elif hasattr(self.reason, 'strerror'): # i.e., EnvironmentError
return self.reason.strerror return self.reason.strerror
else: else:
return u'"{0}"'.format(six.text_type(self.reason)) return '"{}"'.format(str(self.reason))
def get_message(self): def get_message(self):
"""Create the human-readable description of the error, sans """Create the human-readable description of the error, sans
@ -92,7 +91,7 @@ class HumanReadableException(Exception):
""" """
if self.tb: if self.tb:
logger.debug(self.tb) logger.debug(self.tb)
logger.error(u'{0}: {1}', self.error_kind, self.args[0]) logger.error('{0}: {1}', self.error_kind, self.args[0])
class FilesystemError(HumanReadableException): class FilesystemError(HumanReadableException):
@ -100,29 +99,30 @@ class FilesystemError(HumanReadableException):
via a function in this module. The `paths` field is a sequence of via a function in this module. The `paths` field is a sequence of
pathnames involved in the operation. pathnames involved in the operation.
""" """
def __init__(self, reason, verb, paths, tb=None): def __init__(self, reason, verb, paths, tb=None):
self.paths = paths self.paths = paths
super(FilesystemError, self).__init__(reason, verb, tb) super().__init__(reason, verb, tb)
def get_message(self): def get_message(self):
# Use a nicer English phrasing for some specific verbs. # Use a nicer English phrasing for some specific verbs.
if self.verb in ('move', 'copy', 'rename'): if self.verb in ('move', 'copy', 'rename'):
clause = u'while {0} {1} to {2}'.format( clause = 'while {} {} to {}'.format(
self._gerund(), self._gerund(),
displayable_path(self.paths[0]), displayable_path(self.paths[0]),
displayable_path(self.paths[1]) displayable_path(self.paths[1])
) )
elif self.verb in ('delete', 'write', 'create', 'read'): elif self.verb in ('delete', 'write', 'create', 'read'):
clause = u'while {0} {1}'.format( clause = 'while {} {}'.format(
self._gerund(), self._gerund(),
displayable_path(self.paths[0]) displayable_path(self.paths[0])
) )
else: else:
clause = u'during {0} of paths {1}'.format( clause = 'during {} of paths {}'.format(
self.verb, u', '.join(displayable_path(p) for p in self.paths) self.verb, ', '.join(displayable_path(p) for p in self.paths)
) )
return u'{0} {1}'.format(self._reasonstr(), clause) return f'{self._reasonstr()} {clause}'
class MoveOperation(Enum): class MoveOperation(Enum):
@ -132,6 +132,8 @@ class MoveOperation(Enum):
COPY = 1 COPY = 1
LINK = 2 LINK = 2
HARDLINK = 3 HARDLINK = 3
REFLINK = 4
REFLINK_AUTO = 5
def normpath(path): def normpath(path):
@ -182,7 +184,7 @@ def sorted_walk(path, ignore=(), ignore_hidden=False, logger=None):
contents = os.listdir(syspath(path)) contents = os.listdir(syspath(path))
except OSError as exc: except OSError as exc:
if logger: if logger:
logger.warning(u'could not list directory {0}: {1}'.format( logger.warning('could not list directory {}: {}'.format(
displayable_path(path), exc.strerror displayable_path(path), exc.strerror
)) ))
return return
@ -195,6 +197,10 @@ def sorted_walk(path, ignore=(), ignore_hidden=False, logger=None):
skip = False skip = False
for pat in ignore: for pat in ignore:
if fnmatch.fnmatch(base, pat): if fnmatch.fnmatch(base, pat):
if logger:
logger.debug('ignoring {} due to ignore rule {}'.format(
base, pat
))
skip = True skip = True
break break
if skip: if skip:
@ -217,8 +223,14 @@ def sorted_walk(path, ignore=(), ignore_hidden=False, logger=None):
for base in dirs: for base in dirs:
cur = os.path.join(path, base) cur = os.path.join(path, base)
# yield from sorted_walk(...) # yield from sorted_walk(...)
for res in sorted_walk(cur, ignore, ignore_hidden, logger): yield from sorted_walk(cur, ignore, ignore_hidden, logger)
yield res
def path_as_posix(path):
"""Return the string representation of the path with forward (/)
slashes.
"""
return path.replace(b'\\', b'/')
def mkdirall(path): def mkdirall(path):
@ -229,7 +241,7 @@ def mkdirall(path):
if not os.path.isdir(syspath(ancestor)): if not os.path.isdir(syspath(ancestor)):
try: try:
os.mkdir(syspath(ancestor)) os.mkdir(syspath(ancestor))
except (OSError, IOError) as exc: except OSError as exc:
raise FilesystemError(exc, 'create', (ancestor,), raise FilesystemError(exc, 'create', (ancestor,),
traceback.format_exc()) traceback.format_exc())
@ -282,13 +294,13 @@ def prune_dirs(path, root=None, clutter=('.DS_Store', 'Thumbs.db')):
continue continue
clutter = [bytestring_path(c) for c in clutter] clutter = [bytestring_path(c) for c in clutter]
match_paths = [bytestring_path(d) for d in os.listdir(directory)] match_paths = [bytestring_path(d) for d in os.listdir(directory)]
if fnmatch_all(match_paths, clutter): try:
# Directory contains only clutter (or nothing). if fnmatch_all(match_paths, clutter):
try: # Directory contains only clutter (or nothing).
shutil.rmtree(directory) shutil.rmtree(directory)
except OSError: else:
break break
else: except OSError:
break break
@ -367,18 +379,18 @@ def bytestring_path(path):
PATH_SEP = bytestring_path(os.sep) PATH_SEP = bytestring_path(os.sep)
def displayable_path(path, separator=u'; '): def displayable_path(path, separator='; '):
"""Attempts to decode a bytestring path to a unicode object for the """Attempts to decode a bytestring path to a unicode object for the
purpose of displaying it to the user. If the `path` argument is a purpose of displaying it to the user. If the `path` argument is a
list or a tuple, the elements are joined with `separator`. list or a tuple, the elements are joined with `separator`.
""" """
if isinstance(path, (list, tuple)): if isinstance(path, (list, tuple)):
return separator.join(displayable_path(p) for p in path) return separator.join(displayable_path(p) for p in path)
elif isinstance(path, six.text_type): elif isinstance(path, str):
return path return path
elif not isinstance(path, bytes): elif not isinstance(path, bytes):
# A non-string object: just get its unicode representation. # A non-string object: just get its unicode representation.
return six.text_type(path) return str(path)
try: try:
return path.decode(_fsencoding(), 'ignore') return path.decode(_fsencoding(), 'ignore')
@ -397,7 +409,7 @@ def syspath(path, prefix=True):
if os.path.__name__ != 'ntpath': if os.path.__name__ != 'ntpath':
return path return path
if not isinstance(path, six.text_type): if not isinstance(path, str):
# Beets currently represents Windows paths internally with UTF-8 # Beets currently represents Windows paths internally with UTF-8
# arbitrarily. But earlier versions used MBCS because it is # arbitrarily. But earlier versions used MBCS because it is
# reported as the FS encoding by Windows. Try both. # reported as the FS encoding by Windows. Try both.
@ -410,11 +422,11 @@ def syspath(path, prefix=True):
path = path.decode(encoding, 'replace') path = path.decode(encoding, 'replace')
# Add the magic prefix if it isn't already there. # Add the magic prefix if it isn't already there.
# http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247.aspx # https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247.aspx
if prefix and not path.startswith(WINDOWS_MAGIC_PREFIX): if prefix and not path.startswith(WINDOWS_MAGIC_PREFIX):
if path.startswith(u'\\\\'): if path.startswith('\\\\'):
# UNC path. Final path should look like \\?\UNC\... # UNC path. Final path should look like \\?\UNC\...
path = u'UNC' + path[1:] path = 'UNC' + path[1:]
path = WINDOWS_MAGIC_PREFIX + path path = WINDOWS_MAGIC_PREFIX + path
return path return path
@ -436,7 +448,7 @@ def remove(path, soft=True):
return return
try: try:
os.remove(path) os.remove(path)
except (OSError, IOError) as exc: except OSError as exc:
raise FilesystemError(exc, 'delete', (path,), traceback.format_exc()) raise FilesystemError(exc, 'delete', (path,), traceback.format_exc())
@ -451,10 +463,10 @@ def copy(path, dest, replace=False):
path = syspath(path) path = syspath(path)
dest = syspath(dest) dest = syspath(dest)
if not replace and os.path.exists(dest): if not replace and os.path.exists(dest):
raise FilesystemError(u'file exists', 'copy', (path, dest)) raise FilesystemError('file exists', 'copy', (path, dest))
try: try:
shutil.copyfile(path, dest) shutil.copyfile(path, dest)
except (OSError, IOError) as exc: except OSError as exc:
raise FilesystemError(exc, 'copy', (path, dest), raise FilesystemError(exc, 'copy', (path, dest),
traceback.format_exc()) traceback.format_exc())
@ -467,24 +479,37 @@ def move(path, dest, replace=False):
instead, in which case metadata will *not* be preserved. Paths are instead, in which case metadata will *not* be preserved. Paths are
translated to system paths. translated to system paths.
""" """
if os.path.isdir(path):
raise FilesystemError(u'source is directory', 'move', (path, dest))
if os.path.isdir(dest):
raise FilesystemError(u'destination is directory', 'move',
(path, dest))
if samefile(path, dest): if samefile(path, dest):
return return
path = syspath(path) path = syspath(path)
dest = syspath(dest) dest = syspath(dest)
if os.path.exists(dest) and not replace: if os.path.exists(dest) and not replace:
raise FilesystemError(u'file exists', 'rename', (path, dest)) raise FilesystemError('file exists', 'rename', (path, dest))
# First, try renaming the file. # First, try renaming the file.
try: try:
os.rename(path, dest) os.replace(path, dest)
except OSError: except OSError:
# Otherwise, copy and delete the original. tmp = tempfile.mktemp(suffix='.beets',
prefix=py3_path(b'.' + os.path.basename(dest)),
dir=py3_path(os.path.dirname(dest)))
tmp = syspath(tmp)
try: try:
shutil.copyfile(path, dest) shutil.copyfile(path, tmp)
os.replace(tmp, dest)
tmp = None
os.remove(path) os.remove(path)
except (OSError, IOError) as exc: except OSError as exc:
raise FilesystemError(exc, 'move', (path, dest), raise FilesystemError(exc, 'move', (path, dest),
traceback.format_exc()) traceback.format_exc())
finally:
if tmp is not None:
os.remove(tmp)
def link(path, dest, replace=False): def link(path, dest, replace=False):
@ -496,18 +521,18 @@ def link(path, dest, replace=False):
return return
if os.path.exists(syspath(dest)) and not replace: if os.path.exists(syspath(dest)) and not replace:
raise FilesystemError(u'file exists', 'rename', (path, dest)) raise FilesystemError('file exists', 'rename', (path, dest))
try: try:
os.symlink(syspath(path), syspath(dest)) os.symlink(syspath(path), syspath(dest))
except NotImplementedError: except NotImplementedError:
# raised on python >= 3.2 and Windows versions before Vista # raised on python >= 3.2 and Windows versions before Vista
raise FilesystemError(u'OS does not support symbolic links.' raise FilesystemError('OS does not support symbolic links.'
'link', (path, dest), traceback.format_exc()) 'link', (path, dest), traceback.format_exc())
except OSError as exc: except OSError as exc:
# TODO: Windows version checks can be removed for python 3 # TODO: Windows version checks can be removed for python 3
if hasattr('sys', 'getwindowsversion'): if hasattr('sys', 'getwindowsversion'):
if sys.getwindowsversion()[0] < 6: # is before Vista if sys.getwindowsversion()[0] < 6: # is before Vista
exc = u'OS does not support symbolic links.' exc = 'OS does not support symbolic links.'
raise FilesystemError(exc, 'link', (path, dest), raise FilesystemError(exc, 'link', (path, dest),
traceback.format_exc()) traceback.format_exc())
@ -521,21 +546,50 @@ def hardlink(path, dest, replace=False):
return return
if os.path.exists(syspath(dest)) and not replace: if os.path.exists(syspath(dest)) and not replace:
raise FilesystemError(u'file exists', 'rename', (path, dest)) raise FilesystemError('file exists', 'rename', (path, dest))
try: try:
os.link(syspath(path), syspath(dest)) os.link(syspath(path), syspath(dest))
except NotImplementedError: except NotImplementedError:
raise FilesystemError(u'OS does not support hard links.' raise FilesystemError('OS does not support hard links.'
'link', (path, dest), traceback.format_exc()) 'link', (path, dest), traceback.format_exc())
except OSError as exc: except OSError as exc:
if exc.errno == errno.EXDEV: if exc.errno == errno.EXDEV:
raise FilesystemError(u'Cannot hard link across devices.' raise FilesystemError('Cannot hard link across devices.'
'link', (path, dest), traceback.format_exc()) 'link', (path, dest), traceback.format_exc())
else: else:
raise FilesystemError(exc, 'link', (path, dest), raise FilesystemError(exc, 'link', (path, dest),
traceback.format_exc()) traceback.format_exc())
def reflink(path, dest, replace=False, fallback=False):
"""Create a reflink from `dest` to `path`.
Raise an `OSError` if `dest` already exists, unless `replace` is
True. If `path` == `dest`, then do nothing.
If reflinking fails and `fallback` is enabled, try copying the file
instead. Otherwise, raise an error without trying a plain copy.
May raise an `ImportError` if the `reflink` module is not available.
"""
import reflink as pyreflink
if samefile(path, dest):
return
if os.path.exists(syspath(dest)) and not replace:
raise FilesystemError('file exists', 'rename', (path, dest))
try:
pyreflink.reflink(path, dest)
except (NotImplementedError, pyreflink.ReflinkImpossibleError):
if fallback:
copy(path, dest, replace)
else:
raise FilesystemError('OS/filesystem does not support reflinks.',
'link', (path, dest), traceback.format_exc())
def unique_path(path): def unique_path(path):
"""Returns a version of ``path`` that does not exist on the """Returns a version of ``path`` that does not exist on the
filesystem. Specifically, if ``path` itself already exists, then filesystem. Specifically, if ``path` itself already exists, then
@ -553,22 +607,23 @@ def unique_path(path):
num = 0 num = 0
while True: while True:
num += 1 num += 1
suffix = u'.{}'.format(num).encode() + ext suffix = f'.{num}'.encode() + ext
new_path = base + suffix new_path = base + suffix
if not os.path.exists(new_path): if not os.path.exists(new_path):
return new_path return new_path
# Note: The Windows "reserved characters" are, of course, allowed on # Note: The Windows "reserved characters" are, of course, allowed on
# Unix. They are forbidden here because they cause problems on Samba # Unix. They are forbidden here because they cause problems on Samba
# shares, which are sufficiently common as to cause frequent problems. # shares, which are sufficiently common as to cause frequent problems.
# http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247.aspx # https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247.aspx
CHAR_REPLACE = [ CHAR_REPLACE = [
(re.compile(r'[\\/]'), u'_'), # / and \ -- forbidden everywhere. (re.compile(r'[\\/]'), '_'), # / and \ -- forbidden everywhere.
(re.compile(r'^\.'), u'_'), # Leading dot (hidden files on Unix). (re.compile(r'^\.'), '_'), # Leading dot (hidden files on Unix).
(re.compile(r'[\x00-\x1f]'), u''), # Control characters. (re.compile(r'[\x00-\x1f]'), ''), # Control characters.
(re.compile(r'[<>:"\?\*\|]'), u'_'), # Windows "reserved characters". (re.compile(r'[<>:"\?\*\|]'), '_'), # Windows "reserved characters".
(re.compile(r'\.$'), u'_'), # Trailing dots. (re.compile(r'\.$'), '_'), # Trailing dots.
(re.compile(r'\s+$'), u''), # Trailing whitespace. (re.compile(r'\s+$'), ''), # Trailing whitespace.
] ]
@ -692,36 +747,29 @@ def py3_path(path):
it is. So this function helps us "smuggle" the true bytes data it is. So this function helps us "smuggle" the true bytes data
through APIs that took Python 3's Unicode mandate too seriously. through APIs that took Python 3's Unicode mandate too seriously.
""" """
if isinstance(path, six.text_type): if isinstance(path, str):
return path return path
assert isinstance(path, bytes) assert isinstance(path, bytes)
if six.PY2:
return path
return os.fsdecode(path) return os.fsdecode(path)
def str2bool(value): def str2bool(value):
"""Returns a boolean reflecting a human-entered string.""" """Returns a boolean reflecting a human-entered string."""
return value.lower() in (u'yes', u'1', u'true', u't', u'y') return value.lower() in ('yes', '1', 'true', 't', 'y')
def as_string(value): def as_string(value):
"""Convert a value to a Unicode object for matching with a query. """Convert a value to a Unicode object for matching with a query.
None becomes the empty string. Bytestrings are silently decoded. None becomes the empty string. Bytestrings are silently decoded.
""" """
if six.PY2:
buffer_types = buffer, memoryview # noqa: F821
else:
buffer_types = memoryview
if value is None: if value is None:
return u'' return ''
elif isinstance(value, buffer_types): elif isinstance(value, memoryview):
return bytes(value).decode('utf-8', 'ignore') return bytes(value).decode('utf-8', 'ignore')
elif isinstance(value, bytes): elif isinstance(value, bytes):
return value.decode('utf-8', 'ignore') return value.decode('utf-8', 'ignore')
else: else:
return six.text_type(value) return str(value)
def text_string(value, encoding='utf-8'): def text_string(value, encoding='utf-8'):
@ -744,7 +792,7 @@ def plurality(objs):
""" """
c = Counter(objs) c = Counter(objs)
if not c: if not c:
raise ValueError(u'sequence must be non-empty') raise ValueError('sequence must be non-empty')
return c.most_common(1)[0] return c.most_common(1)[0]
@ -761,7 +809,11 @@ def cpu_count():
num = 0 num = 0
elif sys.platform == 'darwin': elif sys.platform == 'darwin':
try: try:
num = int(command_output(['/usr/sbin/sysctl', '-n', 'hw.ncpu'])) num = int(command_output([
'/usr/sbin/sysctl',
'-n',
'hw.ncpu',
]).stdout)
except (ValueError, OSError, subprocess.CalledProcessError): except (ValueError, OSError, subprocess.CalledProcessError):
num = 0 num = 0
else: else:
@ -781,20 +833,23 @@ def convert_command_args(args):
assert isinstance(args, list) assert isinstance(args, list)
def convert(arg): def convert(arg):
if six.PY2: if isinstance(arg, bytes):
if isinstance(arg, six.text_type): arg = arg.decode(arg_encoding(), 'surrogateescape')
arg = arg.encode(arg_encoding())
else:
if isinstance(arg, bytes):
arg = arg.decode(arg_encoding(), 'surrogateescape')
return arg return arg
return [convert(a) for a in args] return [convert(a) for a in args]
# stdout and stderr as bytes
CommandOutput = namedtuple("CommandOutput", ("stdout", "stderr"))
def command_output(cmd, shell=False): def command_output(cmd, shell=False):
"""Runs the command and returns its output after it has exited. """Runs the command and returns its output after it has exited.
Returns a CommandOutput. The attributes ``stdout`` and ``stderr`` contain
byte strings of the respective output streams.
``cmd`` is a list of arguments starting with the command names. The ``cmd`` is a list of arguments starting with the command names. The
arguments are bytes on Unix and strings on Windows. arguments are bytes on Unix and strings on Windows.
If ``shell`` is true, ``cmd`` is assumed to be a string and passed to a If ``shell`` is true, ``cmd`` is assumed to be a string and passed to a
@ -829,7 +884,7 @@ def command_output(cmd, shell=False):
cmd=' '.join(cmd), cmd=' '.join(cmd),
output=stdout + stderr, output=stdout + stderr,
) )
return stdout return CommandOutput(stdout, stderr)
def max_filename_length(path, limit=MAX_FILENAME_LENGTH): def max_filename_length(path, limit=MAX_FILENAME_LENGTH):
@ -876,25 +931,6 @@ def editor_command():
return open_anything() return open_anything()
def shlex_split(s):
"""Split a Unicode or bytes string according to shell lexing rules.
Raise `ValueError` if the string is not a well-formed shell string.
This is a workaround for a bug in some versions of Python.
"""
if not six.PY2 or isinstance(s, bytes): # Shlex works fine.
return shlex.split(s)
elif isinstance(s, six.text_type):
# Work around a Python bug.
# http://bugs.python.org/issue6988
bs = s.encode('utf-8')
return [c.decode('utf-8') for c in shlex.split(bs)]
else:
raise TypeError(u'shlex_split called with non-string')
def interactive_open(targets, command): def interactive_open(targets, command):
"""Open the files in `targets` by `exec`ing a new `command`, given """Open the files in `targets` by `exec`ing a new `command`, given
as a Unicode string. (The new program takes over, and Python as a Unicode string. (The new program takes over, and Python
@ -906,7 +942,7 @@ def interactive_open(targets, command):
# Split the command string into its arguments. # Split the command string into its arguments.
try: try:
args = shlex_split(command) args = shlex.split(command)
except ValueError: # Malformed shell tokens. except ValueError: # Malformed shell tokens.
args = [command] args = [command]
@ -921,7 +957,7 @@ def _windows_long_path_name(short_path):
"""Use Windows' `GetLongPathNameW` via ctypes to get the canonical, """Use Windows' `GetLongPathNameW` via ctypes to get the canonical,
long path given a short filename. long path given a short filename.
""" """
if not isinstance(short_path, six.text_type): if not isinstance(short_path, str):
short_path = short_path.decode(_fsencoding()) short_path = short_path.decode(_fsencoding())
import ctypes import ctypes
@ -982,7 +1018,7 @@ def raw_seconds_short(string):
""" """
match = re.match(r'^(\d+):([0-5]\d)$', string) match = re.match(r'^(\d+):([0-5]\d)$', string)
if not match: if not match:
raise ValueError(u'String not in M:SS format') raise ValueError('String not in M:SS format')
minutes, seconds = map(int, match.groups()) minutes, seconds = map(int, match.groups())
return float(minutes * 60 + seconds) return float(minutes * 60 + seconds)
@ -1009,3 +1045,59 @@ def asciify_path(path, sep_replace):
sep_replace sep_replace
) )
return os.sep.join(path_components) return os.sep.join(path_components)
def par_map(transform, items):
"""Apply the function `transform` to all the elements in the
iterable `items`, like `map(transform, items)` but with no return
value. The map *might* happen in parallel: it's parallel on Python 3
and sequential on Python 2.
The parallelism uses threads (not processes), so this is only useful
for IO-bound `transform`s.
"""
pool = ThreadPool()
pool.map(transform, items)
pool.close()
pool.join()
def lazy_property(func):
"""A decorator that creates a lazily evaluated property. On first access,
the property is assigned the return value of `func`. This first value is
stored, so that future accesses do not have to evaluate `func` again.
This behaviour is useful when `func` is expensive to evaluate, and it is
not certain that the result will be needed.
"""
field_name = '_' + func.__name__
@property
@functools.wraps(func)
def wrapper(self):
if hasattr(self, field_name):
return getattr(self, field_name)
value = func(self)
setattr(self, field_name, value)
return value
return wrapper
def decode_commandline_path(path):
"""Prepare a path for substitution into commandline template.
On Python 3, we need to construct the subprocess commands to invoke as a
Unicode string. On Unix, this is a little unfortunate---the OS is
expecting bytes---so we use surrogate escaping and decode with the
argument encoding, which is the same encoding that will then be
*reversed* to recover the same bytes before invoking the OS. On
Windows, we want to preserve the Unicode filename "as is."
"""
# On Python 3, the template is a Unicode string, which only supports
# substitution of Unicode variables.
if platform.system() == 'Windows':
return path.decode(_fsencoding())
else:
return path.decode(arg_encoding(), 'surrogateescape')

361
libs/common/beets/util/artresizer.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Fabrice Laporte # Copyright 2016, Fabrice Laporte
# #
@ -16,38 +15,39 @@
"""Abstraction layer to resize images using PIL, ImageMagick, or a """Abstraction layer to resize images using PIL, ImageMagick, or a
public resizing proxy if neither is available. public resizing proxy if neither is available.
""" """
from __future__ import division, absolute_import, print_function
import subprocess import subprocess
import os import os
import os.path
import re import re
from tempfile import NamedTemporaryFile from tempfile import NamedTemporaryFile
from six.moves.urllib.parse import urlencode from urllib.parse import urlencode
from beets import logging from beets import logging
from beets import util from beets import util
import six
# Resizing methods # Resizing methods
PIL = 1 PIL = 1
IMAGEMAGICK = 2 IMAGEMAGICK = 2
WEBPROXY = 3 WEBPROXY = 3
if util.SNI_SUPPORTED: PROXY_URL = 'https://images.weserv.nl/'
PROXY_URL = 'https://images.weserv.nl/'
else:
PROXY_URL = 'http://images.weserv.nl/'
log = logging.getLogger('beets') log = logging.getLogger('beets')
def resize_url(url, maxwidth): def resize_url(url, maxwidth, quality=0):
"""Return a proxied image URL that resizes the original image to """Return a proxied image URL that resizes the original image to
maxwidth (preserving aspect ratio). maxwidth (preserving aspect ratio).
""" """
return '{0}?{1}'.format(PROXY_URL, urlencode({ params = {
'url': url.replace('http://', ''), 'url': url.replace('http://', ''),
'w': maxwidth, 'w': maxwidth,
})) }
if quality > 0:
params['q'] = quality
return '{}?{}'.format(PROXY_URL, urlencode(params))
def temp_file_for(path): def temp_file_for(path):
@ -59,48 +59,102 @@ def temp_file_for(path):
return util.bytestring_path(f.name) return util.bytestring_path(f.name)
def pil_resize(maxwidth, path_in, path_out=None): def pil_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0):
"""Resize using Python Imaging Library (PIL). Return the output path """Resize using Python Imaging Library (PIL). Return the output path
of resized image. of resized image.
""" """
path_out = path_out or temp_file_for(path_in) path_out = path_out or temp_file_for(path_in)
from PIL import Image from PIL import Image
log.debug(u'artresizer: PIL resizing {0} to {1}',
log.debug('artresizer: PIL resizing {0} to {1}',
util.displayable_path(path_in), util.displayable_path(path_out)) util.displayable_path(path_in), util.displayable_path(path_out))
try: try:
im = Image.open(util.syspath(path_in)) im = Image.open(util.syspath(path_in))
size = maxwidth, maxwidth size = maxwidth, maxwidth
im.thumbnail(size, Image.ANTIALIAS) im.thumbnail(size, Image.ANTIALIAS)
im.save(path_out)
return path_out if quality == 0:
except IOError: # Use PIL's default quality.
log.error(u"PIL cannot create thumbnail for '{0}'", quality = -1
# progressive=False only affects JPEGs and is the default,
# but we include it here for explicitness.
im.save(util.py3_path(path_out), quality=quality, progressive=False)
if max_filesize > 0:
# If maximum filesize is set, we attempt to lower the quality of
# jpeg conversion by a proportional amount, up to 3 attempts
# First, set the maximum quality to either provided, or 95
if quality > 0:
lower_qual = quality
else:
lower_qual = 95
for i in range(5):
# 5 attempts is an abitrary choice
filesize = os.stat(util.syspath(path_out)).st_size
log.debug("PIL Pass {0} : Output size: {1}B", i, filesize)
if filesize <= max_filesize:
return path_out
# The relationship between filesize & quality will be
# image dependent.
lower_qual -= 10
# Restrict quality dropping below 10
if lower_qual < 10:
lower_qual = 10
# Use optimize flag to improve filesize decrease
im.save(util.py3_path(path_out), quality=lower_qual,
optimize=True, progressive=False)
log.warning("PIL Failed to resize file to below {0}B",
max_filesize)
return path_out
else:
return path_out
except OSError:
log.error("PIL cannot create thumbnail for '{0}'",
util.displayable_path(path_in)) util.displayable_path(path_in))
return path_in return path_in
def im_resize(maxwidth, path_in, path_out=None): def im_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0):
"""Resize using ImageMagick's ``convert`` tool. """Resize using ImageMagick.
Return the output path of resized image.
Use the ``magick`` program or ``convert`` on older versions. Return
the output path of resized image.
""" """
path_out = path_out or temp_file_for(path_in) path_out = path_out or temp_file_for(path_in)
log.debug(u'artresizer: ImageMagick resizing {0} to {1}', log.debug('artresizer: ImageMagick resizing {0} to {1}',
util.displayable_path(path_in), util.displayable_path(path_out)) util.displayable_path(path_in), util.displayable_path(path_out))
# "-resize WIDTHx>" shrinks images with the width larger # "-resize WIDTHx>" shrinks images with the width larger
# than the given width while maintaining the aspect ratio # than the given width while maintaining the aspect ratio
# with regards to the height. # with regards to the height.
# ImageMagick already seems to default to no interlace, but we include it
# here for the sake of explicitness.
cmd = ArtResizer.shared.im_convert_cmd + [
util.syspath(path_in, prefix=False),
'-resize', f'{maxwidth}x>',
'-interlace', 'none',
]
if quality > 0:
cmd += ['-quality', f'{quality}']
# "-define jpeg:extent=SIZEb" sets the target filesize for imagemagick to
# SIZE in bytes.
if max_filesize > 0:
cmd += ['-define', f'jpeg:extent={max_filesize}b']
cmd.append(util.syspath(path_out, prefix=False))
try: try:
util.command_output([ util.command_output(cmd)
'convert', util.syspath(path_in, prefix=False),
'-resize', '{0}x>'.format(maxwidth),
util.syspath(path_out, prefix=False),
])
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
log.warning(u'artresizer: IM convert failed for {0}', log.warning('artresizer: IM convert failed for {0}',
util.displayable_path(path_in)) util.displayable_path(path_in))
return path_in return path_in
return path_out return path_out
@ -112,31 +166,33 @@ BACKEND_FUNCS = {
def pil_getsize(path_in): def pil_getsize(path_in):
from PIL import Image from PIL import Image
try: try:
im = Image.open(util.syspath(path_in)) im = Image.open(util.syspath(path_in))
return im.size return im.size
except IOError as exc: except OSError as exc:
log.error(u"PIL could not read file {}: {}", log.error("PIL could not read file {}: {}",
util.displayable_path(path_in), exc) util.displayable_path(path_in), exc)
def im_getsize(path_in): def im_getsize(path_in):
cmd = ['identify', '-format', '%w %h', cmd = ArtResizer.shared.im_identify_cmd + \
util.syspath(path_in, prefix=False)] ['-format', '%w %h', util.syspath(path_in, prefix=False)]
try: try:
out = util.command_output(cmd) out = util.command_output(cmd).stdout
except subprocess.CalledProcessError as exc: except subprocess.CalledProcessError as exc:
log.warning(u'ImageMagick size query failed') log.warning('ImageMagick size query failed')
log.debug( log.debug(
u'`convert` exited with (status {}) when ' '`convert` exited with (status {}) when '
u'getting size with command {}:\n{}', 'getting size with command {}:\n{}',
exc.returncode, cmd, exc.output.strip() exc.returncode, cmd, exc.output.strip()
) )
return return
try: try:
return tuple(map(int, out.split(b' '))) return tuple(map(int, out.split(b' ')))
except IndexError: except IndexError:
log.warning(u'Could not understand IM output: {0!r}', out) log.warning('Could not understand IM output: {0!r}', out)
BACKEND_GET_SIZE = { BACKEND_GET_SIZE = {
@ -145,14 +201,115 @@ BACKEND_GET_SIZE = {
} }
def pil_deinterlace(path_in, path_out=None):
path_out = path_out or temp_file_for(path_in)
from PIL import Image
try:
im = Image.open(util.syspath(path_in))
im.save(util.py3_path(path_out), progressive=False)
return path_out
except IOError:
return path_in
def im_deinterlace(path_in, path_out=None):
path_out = path_out or temp_file_for(path_in)
cmd = ArtResizer.shared.im_convert_cmd + [
util.syspath(path_in, prefix=False),
'-interlace', 'none',
util.syspath(path_out, prefix=False),
]
try:
util.command_output(cmd)
return path_out
except subprocess.CalledProcessError:
return path_in
DEINTERLACE_FUNCS = {
PIL: pil_deinterlace,
IMAGEMAGICK: im_deinterlace,
}
def im_get_format(filepath):
cmd = ArtResizer.shared.im_identify_cmd + [
'-format', '%[magick]',
util.syspath(filepath)
]
try:
return util.command_output(cmd).stdout
except subprocess.CalledProcessError:
return None
def pil_get_format(filepath):
from PIL import Image, UnidentifiedImageError
try:
with Image.open(util.syspath(filepath)) as im:
return im.format
except (ValueError, TypeError, UnidentifiedImageError, FileNotFoundError):
log.exception("failed to detect image format for {}", filepath)
return None
BACKEND_GET_FORMAT = {
PIL: pil_get_format,
IMAGEMAGICK: im_get_format,
}
def im_convert_format(source, target, deinterlaced):
cmd = ArtResizer.shared.im_convert_cmd + [
util.syspath(source),
*(["-interlace", "none"] if deinterlaced else []),
util.syspath(target),
]
try:
subprocess.check_call(
cmd,
stderr=subprocess.DEVNULL,
stdout=subprocess.DEVNULL
)
return target
except subprocess.CalledProcessError:
return source
def pil_convert_format(source, target, deinterlaced):
from PIL import Image, UnidentifiedImageError
try:
with Image.open(util.syspath(source)) as im:
im.save(util.py3_path(target), progressive=not deinterlaced)
return target
except (ValueError, TypeError, UnidentifiedImageError, FileNotFoundError,
OSError):
log.exception("failed to convert image {} -> {}", source, target)
return source
BACKEND_CONVERT_IMAGE_FORMAT = {
PIL: pil_convert_format,
IMAGEMAGICK: im_convert_format,
}
class Shareable(type): class Shareable(type):
"""A pseudo-singleton metaclass that allows both shared and """A pseudo-singleton metaclass that allows both shared and
non-shared instances. The ``MyClass.shared`` property holds a non-shared instances. The ``MyClass.shared`` property holds a
lazily-created shared instance of ``MyClass`` while calling lazily-created shared instance of ``MyClass`` while calling
``MyClass()`` to construct a new object works as usual. ``MyClass()`` to construct a new object works as usual.
""" """
def __init__(cls, name, bases, dict): def __init__(cls, name, bases, dict):
super(Shareable, cls).__init__(name, bases, dict) super().__init__(name, bases, dict)
cls._instance = None cls._instance = None
@property @property
@ -162,7 +319,7 @@ class Shareable(type):
return cls._instance return cls._instance
class ArtResizer(six.with_metaclass(Shareable, object)): class ArtResizer(metaclass=Shareable):
"""A singleton class that performs image resizes. """A singleton class that performs image resizes.
""" """
@ -170,21 +327,44 @@ class ArtResizer(six.with_metaclass(Shareable, object)):
"""Create a resizer object with an inferred method. """Create a resizer object with an inferred method.
""" """
self.method = self._check_method() self.method = self._check_method()
log.debug(u"artresizer: method is {0}", self.method) log.debug("artresizer: method is {0}", self.method)
self.can_compare = self._can_compare() self.can_compare = self._can_compare()
def resize(self, maxwidth, path_in, path_out=None): # Use ImageMagick's magick binary when it's available. If it's
# not, fall back to the older, separate convert and identify
# commands.
if self.method[0] == IMAGEMAGICK:
self.im_legacy = self.method[2]
if self.im_legacy:
self.im_convert_cmd = ['convert']
self.im_identify_cmd = ['identify']
else:
self.im_convert_cmd = ['magick']
self.im_identify_cmd = ['magick', 'identify']
def resize(
self, maxwidth, path_in, path_out=None, quality=0, max_filesize=0
):
"""Manipulate an image file according to the method, returning a """Manipulate an image file according to the method, returning a
new path. For PIL or IMAGEMAGIC methods, resizes the image to a new path. For PIL or IMAGEMAGIC methods, resizes the image to a
temporary file. For WEBPROXY, returns `path_in` unmodified. temporary file and encodes with the specified quality level.
For WEBPROXY, returns `path_in` unmodified.
""" """
if self.local: if self.local:
func = BACKEND_FUNCS[self.method[0]] func = BACKEND_FUNCS[self.method[0]]
return func(maxwidth, path_in, path_out) return func(maxwidth, path_in, path_out,
quality=quality, max_filesize=max_filesize)
else: else:
return path_in return path_in
def proxy_url(self, maxwidth, url): def deinterlace(self, path_in, path_out=None):
if self.local:
func = DEINTERLACE_FUNCS[self.method[0]]
return func(path_in, path_out)
else:
return path_in
def proxy_url(self, maxwidth, url, quality=0):
"""Modifies an image URL according the method, returning a new """Modifies an image URL according the method, returning a new
URL. For WEBPROXY, a URL on the proxy server is returned. URL. For WEBPROXY, a URL on the proxy server is returned.
Otherwise, the URL is returned unmodified. Otherwise, the URL is returned unmodified.
@ -192,7 +372,7 @@ class ArtResizer(six.with_metaclass(Shareable, object)):
if self.local: if self.local:
return url return url
else: else:
return resize_url(url, maxwidth) return resize_url(url, maxwidth, quality)
@property @property
def local(self): def local(self):
@ -205,12 +385,50 @@ class ArtResizer(six.with_metaclass(Shareable, object)):
"""Return the size of an image file as an int couple (width, height) """Return the size of an image file as an int couple (width, height)
in pixels. in pixels.
Only available locally Only available locally.
""" """
if self.local: if self.local:
func = BACKEND_GET_SIZE[self.method[0]] func = BACKEND_GET_SIZE[self.method[0]]
return func(path_in) return func(path_in)
def get_format(self, path_in):
"""Returns the format of the image as a string.
Only available locally.
"""
if self.local:
func = BACKEND_GET_FORMAT[self.method[0]]
return func(path_in)
def reformat(self, path_in, new_format, deinterlaced=True):
"""Converts image to desired format, updating its extension, but
keeping the same filename.
Only available locally.
"""
if not self.local:
return path_in
new_format = new_format.lower()
# A nonexhaustive map of image "types" to extensions overrides
new_format = {
'jpeg': 'jpg',
}.get(new_format, new_format)
fname, ext = os.path.splitext(path_in)
path_new = fname + b'.' + new_format.encode('utf8')
func = BACKEND_CONVERT_IMAGE_FORMAT[self.method[0]]
# allows the exception to propagate, while still making sure a changed
# file path was removed
result_path = path_in
try:
result_path = func(path_in, path_new, deinterlaced)
finally:
if result_path != path_in:
os.unlink(path_in)
return result_path
def _can_compare(self): def _can_compare(self):
"""A boolean indicating whether image comparison is available""" """A boolean indicating whether image comparison is available"""
@ -218,10 +436,20 @@ class ArtResizer(six.with_metaclass(Shareable, object)):
@staticmethod @staticmethod
def _check_method(): def _check_method():
"""Return a tuple indicating an available method and its version.""" """Return a tuple indicating an available method and its version.
The result has at least two elements:
- The method, eitehr WEBPROXY, PIL, or IMAGEMAGICK.
- The version.
If the method is IMAGEMAGICK, there is also a third element: a
bool flag indicating whether to use the `magick` binary or
legacy single-purpose executables (`convert`, `identify`, etc.)
"""
version = get_im_version() version = get_im_version()
if version: if version:
return IMAGEMAGICK, version version, legacy = version
return IMAGEMAGICK, version, legacy
version = get_pil_version() version = get_pil_version()
if version: if version:
@ -231,31 +459,34 @@ class ArtResizer(six.with_metaclass(Shareable, object)):
def get_im_version(): def get_im_version():
"""Return Image Magick version or None if it is unavailable """Get the ImageMagick version and legacy flag as a pair. Or return
Try invoking ImageMagick's "convert". None if ImageMagick is not available.
""" """
try: for cmd_name, legacy in ((['magick'], False), (['convert'], True)):
out = util.command_output(['convert', '--version']) cmd = cmd_name + ['--version']
if b'imagemagick' in out.lower(): try:
pattern = br".+ (\d+)\.(\d+)\.(\d+).*" out = util.command_output(cmd).stdout
match = re.search(pattern, out) except (subprocess.CalledProcessError, OSError) as exc:
if match: log.debug('ImageMagick version check failed: {}', exc)
return (int(match.group(1)), else:
int(match.group(2)), if b'imagemagick' in out.lower():
int(match.group(3))) pattern = br".+ (\d+)\.(\d+)\.(\d+).*"
return (0,) match = re.search(pattern, out)
if match:
version = (int(match.group(1)),
int(match.group(2)),
int(match.group(3)))
return version, legacy
except (subprocess.CalledProcessError, OSError) as exc: return None
log.debug(u'ImageMagick check `convert --version` failed: {}', exc)
return None
def get_pil_version(): def get_pil_version():
"""Return Image Magick version or None if it is unavailable """Get the PIL/Pillow version, or None if it is unavailable.
Try importing PIL.""" """
try: try:
__import__('PIL', fromlist=[str('Image')]) __import__('PIL', fromlist=['Image'])
return (0,) return (0,)
except ImportError: except ImportError:
return None return None

24
libs/common/beets/util/bluelet.py
View file

@ -1,5 +1,3 @@
# -*- coding: utf-8 -*-
"""Extremely simple pure-Python implementation of coroutine-style """Extremely simple pure-Python implementation of coroutine-style
asynchronous socket I/O. Inspired by, but inferior to, Eventlet. asynchronous socket I/O. Inspired by, but inferior to, Eventlet.
Bluelet can also be thought of as a less-terrible replacement for Bluelet can also be thought of as a less-terrible replacement for
@ -7,9 +5,7 @@ asyncore.
Bluelet: easy concurrency without all the messy parallelism. Bluelet: easy concurrency without all the messy parallelism.
""" """
from __future__ import division, absolute_import, print_function
import six
import socket import socket
import select import select
import sys import sys
@ -22,7 +18,7 @@ import collections
# Basic events used for thread scheduling. # Basic events used for thread scheduling.
class Event(object): class Event:
"""Just a base class identifying Bluelet events. An event is an """Just a base class identifying Bluelet events. An event is an
object yielded from a Bluelet thread coroutine to suspend operation object yielded from a Bluelet thread coroutine to suspend operation
and communicate with the scheduler. and communicate with the scheduler.
@ -201,7 +197,7 @@ class ThreadException(Exception):
self.exc_info = exc_info self.exc_info = exc_info
def reraise(self): def reraise(self):
six.reraise(self.exc_info[0], self.exc_info[1], self.exc_info[2]) raise self.exc_info[1].with_traceback(self.exc_info[2])
SUSPENDED = Event() # Special sentinel placeholder for suspended threads. SUSPENDED = Event() # Special sentinel placeholder for suspended threads.
@ -336,16 +332,20 @@ def run(root_coro):
break break
# Wait and fire. # Wait and fire.
event2coro = dict((v, k) for k, v in threads.items()) event2coro = {v: k for k, v in threads.items()}
for event in _event_select(threads.values()): for event in _event_select(threads.values()):
# Run the IO operation, but catch socket errors. # Run the IO operation, but catch socket errors.
try: try:
value = event.fire() value = event.fire()
except socket.error as exc: except OSError as exc:
if isinstance(exc.args, tuple) and \ if isinstance(exc.args, tuple) and \
exc.args[0] == errno.EPIPE: exc.args[0] == errno.EPIPE:
# Broken pipe. Remote host disconnected. # Broken pipe. Remote host disconnected.
pass pass
elif isinstance(exc.args, tuple) and \
exc.args[0] == errno.ECONNRESET:
# Connection was reset by peer.
pass
else: else:
traceback.print_exc() traceback.print_exc()
# Abort the coroutine. # Abort the coroutine.
@ -386,7 +386,7 @@ class SocketClosedError(Exception):
pass pass
class Listener(object): class Listener:
"""A socket wrapper object for listening sockets. """A socket wrapper object for listening sockets.
""" """
def __init__(self, host, port): def __init__(self, host, port):
@ -416,7 +416,7 @@ class Listener(object):
self.sock.close() self.sock.close()
class Connection(object): class Connection:
"""A socket wrapper object for connected sockets. """A socket wrapper object for connected sockets.
""" """
def __init__(self, sock, addr): def __init__(self, sock, addr):
@ -541,7 +541,7 @@ def spawn(coro):
and child coroutines run concurrently. and child coroutines run concurrently.
""" """
if not isinstance(coro, types.GeneratorType): if not isinstance(coro, types.GeneratorType):
raise ValueError(u'%s is not a coroutine' % coro) raise ValueError('%s is not a coroutine' % coro)
return SpawnEvent(coro) return SpawnEvent(coro)
@ -551,7 +551,7 @@ def call(coro):
returns a value using end(), then this event returns that value. returns a value using end(), then this event returns that value.
""" """
if not isinstance(coro, types.GeneratorType): if not isinstance(coro, types.GeneratorType):
raise ValueError(u'%s is not a coroutine' % coro) raise ValueError('%s is not a coroutine' % coro)
return DelegationEvent(coro) return DelegationEvent(coro)

1507
libs/common/beets/util/confit.py
View file

File diff suppressed because it is too large Load diff

2
libs/common/beets/util/enumeration.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson. # Copyright 2016, Adrian Sampson.
# #
@ -13,7 +12,6 @@
# The above copyright notice and this permission notice shall be # The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software. # included in all copies or substantial portions of the Software.
from __future__ import division, absolute_import, print_function
from enum import Enum from enum import Enum

169
libs/common/beets/util/functemplate.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson. # Copyright 2016, Adrian Sampson.
# #
@ -27,30 +26,30 @@ This is sort of like a tiny, horrible degeneration of a real templating
engine like Jinja2 or Mustache. engine like Jinja2 or Mustache.
""" """
from __future__ import division, absolute_import, print_function
import re import re
import ast import ast
import dis import dis
import types import types
import sys import sys
import six import functools
SYMBOL_DELIM = u'$' SYMBOL_DELIM = '$'
FUNC_DELIM = u'%' FUNC_DELIM = '%'
GROUP_OPEN = u'{' GROUP_OPEN = '{'
GROUP_CLOSE = u'}' GROUP_CLOSE = '}'
ARG_SEP = u',' ARG_SEP = ','
ESCAPE_CHAR = u'$' ESCAPE_CHAR = '$'
VARIABLE_PREFIX = '__var_' VARIABLE_PREFIX = '__var_'
FUNCTION_PREFIX = '__func_' FUNCTION_PREFIX = '__func_'
class Environment(object): class Environment:
"""Contains the values and functions to be substituted into a """Contains the values and functions to be substituted into a
template. template.
""" """
def __init__(self, values, functions): def __init__(self, values, functions):
self.values = values self.values = values
self.functions = functions self.functions = functions
@ -72,15 +71,7 @@ def ex_literal(val):
"""An int, float, long, bool, string, or None literal with the given """An int, float, long, bool, string, or None literal with the given
value. value.
""" """
if val is None: return ast.Constant(val)
return ast.Name('None', ast.Load())
elif isinstance(val, six.integer_types):
return ast.Num(val)
elif isinstance(val, bool):
return ast.Name(bytes(val), ast.Load())
elif isinstance(val, six.string_types):
return ast.Str(val)
raise TypeError(u'no literal for {0}'.format(type(val)))
def ex_varassign(name, expr): def ex_varassign(name, expr):
@ -97,7 +88,7 @@ def ex_call(func, args):
function may be an expression or the name of a function. Each function may be an expression or the name of a function. Each
argument may be an expression or a value to be used as a literal. argument may be an expression or a value to be used as a literal.
""" """
if isinstance(func, six.string_types): if isinstance(func, str):
func = ex_rvalue(func) func = ex_rvalue(func)
args = list(args) args = list(args)
@ -105,10 +96,7 @@ def ex_call(func, args):
if not isinstance(args[i], ast.expr): if not isinstance(args[i], ast.expr):
args[i] = ex_literal(args[i]) args[i] = ex_literal(args[i])
if sys.version_info[:2] < (3, 5): return ast.Call(func, args, [])
return ast.Call(func, args, [], None, None)
else:
return ast.Call(func, args, [])
def compile_func(arg_names, statements, name='_the_func', debug=False): def compile_func(arg_names, statements, name='_the_func', debug=False):
@ -116,32 +104,30 @@ def compile_func(arg_names, statements, name='_the_func', debug=False):
the resulting Python function. If `debug`, then print out the the resulting Python function. If `debug`, then print out the
bytecode of the compiled function. bytecode of the compiled function.
""" """
if six.PY2: args_fields = {
func_def = ast.FunctionDef( 'args': [ast.arg(arg=n, annotation=None) for n in arg_names],
name=name.encode('utf-8'), 'kwonlyargs': [],
args=ast.arguments( 'kw_defaults': [],
args=[ast.Name(n, ast.Param()) for n in arg_names], 'defaults': [ex_literal(None) for _ in arg_names],
vararg=None, }
kwarg=None, if 'posonlyargs' in ast.arguments._fields: # Added in Python 3.8.
defaults=[ex_literal(None) for _ in arg_names], args_fields['posonlyargs'] = []
), args = ast.arguments(**args_fields)
body=statements,
decorator_list=[], func_def = ast.FunctionDef(
) name=name,
else: args=args,
func_def = ast.FunctionDef( body=statements,
name=name, decorator_list=[],
args=ast.arguments( )
args=[ast.arg(arg=n, annotation=None) for n in arg_names],
kwonlyargs=[], # The ast.Module signature changed in 3.8 to accept a list of types to
kw_defaults=[], # ignore.
defaults=[ex_literal(None) for _ in arg_names], if sys.version_info >= (3, 8):
), mod = ast.Module([func_def], [])
body=statements, else:
decorator_list=[], mod = ast.Module([func_def])
)
mod = ast.Module([func_def])
ast.fix_missing_locations(mod) ast.fix_missing_locations(mod)
prog = compile(mod, '<generated>', 'exec') prog = compile(mod, '<generated>', 'exec')
@ -160,14 +146,15 @@ def compile_func(arg_names, statements, name='_the_func', debug=False):
# AST nodes for the template language. # AST nodes for the template language.
class Symbol(object): class Symbol:
"""A variable-substitution symbol in a template.""" """A variable-substitution symbol in a template."""
def __init__(self, ident, original): def __init__(self, ident, original):
self.ident = ident self.ident = ident
self.original = original self.original = original
def __repr__(self): def __repr__(self):
return u'Symbol(%s)' % repr(self.ident) return 'Symbol(%s)' % repr(self.ident)
def evaluate(self, env): def evaluate(self, env):
"""Evaluate the symbol in the environment, returning a Unicode """Evaluate the symbol in the environment, returning a Unicode
@ -182,24 +169,22 @@ class Symbol(object):
def translate(self): def translate(self):
"""Compile the variable lookup.""" """Compile the variable lookup."""
if six.PY2: ident = self.ident
ident = self.ident.encode('utf-8')
else:
ident = self.ident
expr = ex_rvalue(VARIABLE_PREFIX + ident) expr = ex_rvalue(VARIABLE_PREFIX + ident)
return [expr], set([ident]), set() return [expr], {ident}, set()
class Call(object): class Call:
"""A function call in a template.""" """A function call in a template."""
def __init__(self, ident, args, original): def __init__(self, ident, args, original):
self.ident = ident self.ident = ident
self.args = args self.args = args
self.original = original self.original = original
def __repr__(self): def __repr__(self):
return u'Call(%s, %s, %s)' % (repr(self.ident), repr(self.args), return 'Call({}, {}, {})'.format(repr(self.ident), repr(self.args),
repr(self.original)) repr(self.original))
def evaluate(self, env): def evaluate(self, env):
"""Evaluate the function call in the environment, returning a """Evaluate the function call in the environment, returning a
@ -212,19 +197,15 @@ class Call(object):
except Exception as exc: except Exception as exc:
# Function raised exception! Maybe inlining the name of # Function raised exception! Maybe inlining the name of
# the exception will help debug. # the exception will help debug.
return u'<%s>' % six.text_type(exc) return '<%s>' % str(exc)
return six.text_type(out) return str(out)
else: else:
return self.original return self.original
def translate(self): def translate(self):
"""Compile the function call.""" """Compile the function call."""
varnames = set() varnames = set()
if six.PY2: funcnames = {self.ident}
ident = self.ident.encode('utf-8')
else:
ident = self.ident
funcnames = set([ident])
arg_exprs = [] arg_exprs = []
for arg in self.args: for arg in self.args:
@ -235,32 +216,33 @@ class Call(object):
# Create a subexpression that joins the result components of # Create a subexpression that joins the result components of
# the arguments. # the arguments.
arg_exprs.append(ex_call( arg_exprs.append(ex_call(
ast.Attribute(ex_literal(u''), 'join', ast.Load()), ast.Attribute(ex_literal(''), 'join', ast.Load()),
[ex_call( [ex_call(
'map', 'map',
[ [
ex_rvalue(six.text_type.__name__), ex_rvalue(str.__name__),
ast.List(subexprs, ast.Load()), ast.List(subexprs, ast.Load()),
] ]
)], )],
)) ))
subexpr_call = ex_call( subexpr_call = ex_call(
FUNCTION_PREFIX + ident, FUNCTION_PREFIX + self.ident,
arg_exprs arg_exprs
) )
return [subexpr_call], varnames, funcnames return [subexpr_call], varnames, funcnames
class Expression(object): class Expression:
"""Top-level template construct: contains a list of text blobs, """Top-level template construct: contains a list of text blobs,
Symbols, and Calls. Symbols, and Calls.
""" """
def __init__(self, parts): def __init__(self, parts):
self.parts = parts self.parts = parts
def __repr__(self): def __repr__(self):
return u'Expression(%s)' % (repr(self.parts)) return 'Expression(%s)' % (repr(self.parts))
def evaluate(self, env): def evaluate(self, env):
"""Evaluate the entire expression in the environment, returning """Evaluate the entire expression in the environment, returning
@ -268,11 +250,11 @@ class Expression(object):
""" """
out = [] out = []
for part in self.parts: for part in self.parts:
if isinstance(part, six.string_types): if isinstance(part, str):
out.append(part) out.append(part)
else: else:
out.append(part.evaluate(env)) out.append(part.evaluate(env))
return u''.join(map(six.text_type, out)) return ''.join(map(str, out))
def translate(self): def translate(self):
"""Compile the expression to a list of Python AST expressions, a """Compile the expression to a list of Python AST expressions, a
@ -282,7 +264,7 @@ class Expression(object):
varnames = set() varnames = set()
funcnames = set() funcnames = set()
for part in self.parts: for part in self.parts:
if isinstance(part, six.string_types): if isinstance(part, str):
expressions.append(ex_literal(part)) expressions.append(ex_literal(part))
else: else:
e, v, f = part.translate() e, v, f = part.translate()
@ -298,7 +280,7 @@ class ParseError(Exception):
pass pass
class Parser(object): class Parser:
"""Parses a template expression string. Instantiate the class with """Parses a template expression string. Instantiate the class with
the template source and call ``parse_expression``. The ``pos`` field the template source and call ``parse_expression``. The ``pos`` field
will indicate the character after the expression finished and will indicate the character after the expression finished and
@ -311,6 +293,7 @@ class Parser(object):
replaced with a real, accepted parsing technique (PEG, parser replaced with a real, accepted parsing technique (PEG, parser
generator, etc.). generator, etc.).
""" """
def __init__(self, string, in_argument=False): def __init__(self, string, in_argument=False):
""" Create a new parser. """ Create a new parser.
:param in_arguments: boolean that indicates the parser is to be :param in_arguments: boolean that indicates the parser is to be
@ -326,7 +309,7 @@ class Parser(object):
special_chars = (SYMBOL_DELIM, FUNC_DELIM, GROUP_OPEN, GROUP_CLOSE, special_chars = (SYMBOL_DELIM, FUNC_DELIM, GROUP_OPEN, GROUP_CLOSE,
ESCAPE_CHAR) ESCAPE_CHAR)
special_char_re = re.compile(r'[%s]|\Z' % special_char_re = re.compile(r'[%s]|\Z' %
u''.join(re.escape(c) for c in special_chars)) ''.join(re.escape(c) for c in special_chars))
escapable_chars = (SYMBOL_DELIM, FUNC_DELIM, GROUP_CLOSE, ARG_SEP) escapable_chars = (SYMBOL_DELIM, FUNC_DELIM, GROUP_CLOSE, ARG_SEP)
terminator_chars = (GROUP_CLOSE,) terminator_chars = (GROUP_CLOSE,)
@ -343,7 +326,7 @@ class Parser(object):
if self.in_argument: if self.in_argument:
extra_special_chars = (ARG_SEP,) extra_special_chars = (ARG_SEP,)
special_char_re = re.compile( special_char_re = re.compile(
r'[%s]|\Z' % u''.join( r'[%s]|\Z' % ''.join(
re.escape(c) for c in re.escape(c) for c in
self.special_chars + extra_special_chars self.special_chars + extra_special_chars
) )
@ -387,7 +370,7 @@ class Parser(object):
# Shift all characters collected so far into a single string. # Shift all characters collected so far into a single string.
if text_parts: if text_parts:
self.parts.append(u''.join(text_parts)) self.parts.append(''.join(text_parts))
text_parts = [] text_parts = []
if char == SYMBOL_DELIM: if char == SYMBOL_DELIM:
@ -409,7 +392,7 @@ class Parser(object):
# If any parsed characters remain, shift them into a string. # If any parsed characters remain, shift them into a string.
if text_parts: if text_parts:
self.parts.append(u''.join(text_parts)) self.parts.append(''.join(text_parts))
def parse_symbol(self): def parse_symbol(self):
"""Parse a variable reference (like ``$foo`` or ``${foo}``) """Parse a variable reference (like ``$foo`` or ``${foo}``)
@ -547,11 +530,27 @@ def _parse(template):
return Expression(parts) return Expression(parts)
# External interface. def cached(func):
"""Like the `functools.lru_cache` decorator, but works (as a no-op)
on Python < 3.2.
"""
if hasattr(functools, 'lru_cache'):
return functools.lru_cache(maxsize=128)(func)
else:
# Do nothing when lru_cache is not available.
return func
class Template(object):
@cached
def template(fmt):
return Template(fmt)
# External interface.
class Template:
"""A string template, including text, Symbols, and Calls. """A string template, including text, Symbols, and Calls.
""" """
def __init__(self, template): def __init__(self, template):
self.expr = _parse(template) self.expr = _parse(template)
self.original = template self.original = template
@ -600,7 +599,7 @@ class Template(object):
for funcname in funcnames: for funcname in funcnames:
args[FUNCTION_PREFIX + funcname] = functions[funcname] args[FUNCTION_PREFIX + funcname] = functions[funcname]
parts = func(**args) parts = func(**args)
return u''.join(parts) return ''.join(parts)
return wrapper_func return wrapper_func
@ -609,9 +608,9 @@ class Template(object):
if __name__ == '__main__': if __name__ == '__main__':
import timeit import timeit
_tmpl = Template(u'foo $bar %baz{foozle $bar barzle} $bar') _tmpl = Template('foo $bar %baz{foozle $bar barzle} $bar')
_vars = {'bar': 'qux'} _vars = {'bar': 'qux'}
_funcs = {'baz': six.text_type.upper} _funcs = {'baz': str.upper}
interp_time = timeit.timeit('_tmpl.interpret(_vars, _funcs)', interp_time = timeit.timeit('_tmpl.interpret(_vars, _funcs)',
'from __main__ import _tmpl, _vars, _funcs', 'from __main__ import _tmpl, _vars, _funcs',
number=10000) number=10000)
@ -620,4 +619,4 @@ if __name__ == '__main__':
'from __main__ import _tmpl, _vars, _funcs', 'from __main__ import _tmpl, _vars, _funcs',
number=10000) number=10000)
print(comp_time) print(comp_time)
print(u'Speedup:', interp_time / comp_time) print('Speedup:', interp_time / comp_time)

2
libs/common/beets/util/hidden.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson. # Copyright 2016, Adrian Sampson.
# #
@ -14,7 +13,6 @@
# included in all copies or substantial portions of the Software. # included in all copies or substantial portions of the Software.
"""Simple library to work out if a file is hidden on different platforms.""" """Simple library to work out if a file is hidden on different platforms."""
from __future__ import division, absolute_import, print_function
import os import os
import stat import stat

50
libs/common/beets/util/pipeline.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson. # Copyright 2016, Adrian Sampson.
# #
@ -32,12 +31,10 @@ To do so, pass an iterable of coroutines to the Pipeline constructor
in place of any single coroutine. in place of any single coroutine.
""" """
from __future__ import division, absolute_import, print_function
from six.moves import queue import queue
from threading import Thread, Lock from threading import Thread, Lock
import sys import sys
import six
BUBBLE = '__PIPELINE_BUBBLE__' BUBBLE = '__PIPELINE_BUBBLE__'
POISON = '__PIPELINE_POISON__' POISON = '__PIPELINE_POISON__'
@ -91,6 +88,7 @@ class CountedQueue(queue.Queue):
still feeding into it. The queue is poisoned when all threads are still feeding into it. The queue is poisoned when all threads are
finished with the queue. finished with the queue.
""" """
def __init__(self, maxsize=0): def __init__(self, maxsize=0):
queue.Queue.__init__(self, maxsize) queue.Queue.__init__(self, maxsize)
self.nthreads = 0 self.nthreads = 0
@ -135,10 +133,11 @@ class CountedQueue(queue.Queue):
_invalidate_queue(self, POISON, False) _invalidate_queue(self, POISON, False)
class MultiMessage(object): class MultiMessage:
"""A message yielded by a pipeline stage encapsulating multiple """A message yielded by a pipeline stage encapsulating multiple
values to be sent to the next stage. values to be sent to the next stage.
""" """
def __init__(self, messages): def __init__(self, messages):
self.messages = messages self.messages = messages
@ -210,8 +209,9 @@ def _allmsgs(obj):
class PipelineThread(Thread): class PipelineThread(Thread):
"""Abstract base class for pipeline-stage threads.""" """Abstract base class for pipeline-stage threads."""
def __init__(self, all_threads): def __init__(self, all_threads):
super(PipelineThread, self).__init__() super().__init__()
self.abort_lock = Lock() self.abort_lock = Lock()
self.abort_flag = False self.abort_flag = False
self.all_threads = all_threads self.all_threads = all_threads
@ -241,15 +241,13 @@ class FirstPipelineThread(PipelineThread):
"""The thread running the first stage in a parallel pipeline setup. """The thread running the first stage in a parallel pipeline setup.
The coroutine should just be a generator. The coroutine should just be a generator.
""" """
def __init__(self, coro, out_queue, all_threads): def __init__(self, coro, out_queue, all_threads):
super(FirstPipelineThread, self).__init__(all_threads) super().__init__(all_threads)
self.coro = coro self.coro = coro
self.out_queue = out_queue self.out_queue = out_queue
self.out_queue.acquire() self.out_queue.acquire()
self.abort_lock = Lock()
self.abort_flag = False
def run(self): def run(self):
try: try:
while True: while True:
@ -282,8 +280,9 @@ class MiddlePipelineThread(PipelineThread):
"""A thread running any stage in the pipeline except the first or """A thread running any stage in the pipeline except the first or
last. last.
""" """
def __init__(self, coro, in_queue, out_queue, all_threads): def __init__(self, coro, in_queue, out_queue, all_threads):
super(MiddlePipelineThread, self).__init__(all_threads) super().__init__(all_threads)
self.coro = coro self.coro = coro
self.in_queue = in_queue self.in_queue = in_queue
self.out_queue = out_queue self.out_queue = out_queue
@ -330,8 +329,9 @@ class LastPipelineThread(PipelineThread):
"""A thread running the last stage in a pipeline. The coroutine """A thread running the last stage in a pipeline. The coroutine
should yield nothing. should yield nothing.
""" """
def __init__(self, coro, in_queue, all_threads): def __init__(self, coro, in_queue, all_threads):
super(LastPipelineThread, self).__init__(all_threads) super().__init__(all_threads)
self.coro = coro self.coro = coro
self.in_queue = in_queue self.in_queue = in_queue
@ -362,17 +362,18 @@ class LastPipelineThread(PipelineThread):
return return
class Pipeline(object): class Pipeline:
"""Represents a staged pattern of work. Each stage in the pipeline """Represents a staged pattern of work. Each stage in the pipeline
is a coroutine that receives messages from the previous stage and is a coroutine that receives messages from the previous stage and
yields messages to be sent to the next stage. yields messages to be sent to the next stage.
""" """
def __init__(self, stages): def __init__(self, stages):
"""Makes a new pipeline from a list of coroutines. There must """Makes a new pipeline from a list of coroutines. There must
be at least two stages. be at least two stages.
""" """
if len(stages) < 2: if len(stages) < 2:
raise ValueError(u'pipeline must have at least two stages') raise ValueError('pipeline must have at least two stages')
self.stages = [] self.stages = []
for stage in stages: for stage in stages:
if isinstance(stage, (list, tuple)): if isinstance(stage, (list, tuple)):
@ -442,7 +443,7 @@ class Pipeline(object):
exc_info = thread.exc_info exc_info = thread.exc_info
if exc_info: if exc_info:
# Make the exception appear as it was raised originally. # Make the exception appear as it was raised originally.
six.reraise(exc_info[0], exc_info[1], exc_info[2]) raise exc_info[1].with_traceback(exc_info[2])
def pull(self): def pull(self):
"""Yield elements from the end of the pipeline. Runs the stages """Yield elements from the end of the pipeline. Runs the stages
@ -469,6 +470,7 @@ class Pipeline(object):
for msg in msgs: for msg in msgs:
yield msg yield msg
# Smoke test. # Smoke test.
if __name__ == '__main__': if __name__ == '__main__':
import time import time
@ -477,14 +479,14 @@ if __name__ == '__main__':
# in parallel. # in parallel.
def produce(): def produce():
for i in range(5): for i in range(5):
print(u'generating %i' % i) print('generating %i' % i)
time.sleep(1) time.sleep(1)
yield i yield i
def work(): def work():
num = yield num = yield
while True: while True:
print(u'processing %i' % num) print('processing %i' % num)
time.sleep(2) time.sleep(2)
num = yield num * 2 num = yield num * 2
@ -492,7 +494,7 @@ if __name__ == '__main__':
while True: while True:
num = yield num = yield
time.sleep(1) time.sleep(1)
print(u'received %i' % num) print('received %i' % num)
ts_start = time.time() ts_start = time.time()
Pipeline([produce(), work(), consume()]).run_sequential() Pipeline([produce(), work(), consume()]).run_sequential()
@ -501,22 +503,22 @@ if __name__ == '__main__':
ts_par = time.time() ts_par = time.time()
Pipeline([produce(), (work(), work()), consume()]).run_parallel() Pipeline([produce(), (work(), work()), consume()]).run_parallel()
ts_end = time.time() ts_end = time.time()
print(u'Sequential time:', ts_seq - ts_start) print('Sequential time:', ts_seq - ts_start)
print(u'Parallel time:', ts_par - ts_seq) print('Parallel time:', ts_par - ts_seq)
print(u'Multiply-parallel time:', ts_end - ts_par) print('Multiply-parallel time:', ts_end - ts_par)
print() print()
# Test a pipeline that raises an exception. # Test a pipeline that raises an exception.
def exc_produce(): def exc_produce():
for i in range(10): for i in range(10):
print(u'generating %i' % i) print('generating %i' % i)
time.sleep(1) time.sleep(1)
yield i yield i
def exc_work(): def exc_work():
num = yield num = yield
while True: while True:
print(u'processing %i' % num) print('processing %i' % num)
time.sleep(3) time.sleep(3)
if num == 3: if num == 3:
raise Exception() raise Exception()
@ -525,6 +527,6 @@ if __name__ == '__main__':
def exc_consume(): def exc_consume():
while True: while True:
num = yield num = yield
print(u'received %i' % num) print('received %i' % num)
Pipeline([exc_produce(), exc_work(), exc_consume()]).run_parallel(1) Pipeline([exc_produce(), exc_work(), exc_consume()]).run_parallel(1)

2
libs/common/beets/vfs.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson. # Copyright 2016, Adrian Sampson.
# #
@ -16,7 +15,6 @@
"""A simple utility for constructing filesystem-like trees from beets """A simple utility for constructing filesystem-like trees from beets
libraries. libraries.
""" """
from __future__ import division, absolute_import, print_function
from collections import namedtuple from collections import namedtuple
from beets import util from beets import util

2
libs/common/beetsplug/__init__.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson. # Copyright 2016, Adrian Sampson.
# #
@ -15,7 +14,6 @@
"""A namespace package for beets plugins.""" """A namespace package for beets plugins."""
from __future__ import division, absolute_import, print_function
# Make this a namespace package. # Make this a namespace package.
from pkgutil import extend_path from pkgutil import extend_path

77
libs/common/beetsplug/absubmit.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Pieter Mulder. # Copyright 2016, Pieter Mulder.
# #
@ -16,7 +15,6 @@
"""Calculate acoustic information and submit to AcousticBrainz. """Calculate acoustic information and submit to AcousticBrainz.
""" """
from __future__ import division, absolute_import, print_function
import errno import errno
import hashlib import hashlib
@ -32,6 +30,9 @@ from beets import plugins
from beets import util from beets import util
from beets import ui from beets import ui
# We use this field to check whether AcousticBrainz info is present.
PROBE_FIELD = 'mood_acoustic'
class ABSubmitError(Exception): class ABSubmitError(Exception):
"""Raised when failing to analyse file with extractor.""" """Raised when failing to analyse file with extractor."""
@ -43,19 +44,23 @@ def call(args):
Raise a AnalysisABSubmitError on failure. Raise a AnalysisABSubmitError on failure.
""" """
try: try:
return util.command_output(args) return util.command_output(args).stdout
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
raise ABSubmitError( raise ABSubmitError(
u'{0} exited with status {1}'.format(args[0], e.returncode) '{} exited with status {}'.format(args[0], e.returncode)
) )
class AcousticBrainzSubmitPlugin(plugins.BeetsPlugin): class AcousticBrainzSubmitPlugin(plugins.BeetsPlugin):
def __init__(self): def __init__(self):
super(AcousticBrainzSubmitPlugin, self).__init__() super().__init__()
self.config.add({'extractor': u''}) self.config.add({
'extractor': '',
'force': False,
'pretend': False
})
self.extractor = self.config['extractor'].as_str() self.extractor = self.config['extractor'].as_str()
if self.extractor: if self.extractor:
@ -63,7 +68,7 @@ class AcousticBrainzSubmitPlugin(plugins.BeetsPlugin):
# Expicit path to extractor # Expicit path to extractor
if not os.path.isfile(self.extractor): if not os.path.isfile(self.extractor):
raise ui.UserError( raise ui.UserError(
u'Extractor command does not exist: {0}.'. 'Extractor command does not exist: {0}.'.
format(self.extractor) format(self.extractor)
) )
else: else:
@ -73,8 +78,8 @@ class AcousticBrainzSubmitPlugin(plugins.BeetsPlugin):
call([self.extractor]) call([self.extractor])
except OSError: except OSError:
raise ui.UserError( raise ui.UserError(
u'No extractor command found: please install the ' 'No extractor command found: please install the extractor'
u'extractor binary from http://acousticbrainz.org/download' ' binary from https://acousticbrainz.org/download'
) )
except ABSubmitError: except ABSubmitError:
# Extractor found, will exit with an error if not called with # Extractor found, will exit with an error if not called with
@ -96,7 +101,18 @@ class AcousticBrainzSubmitPlugin(plugins.BeetsPlugin):
def commands(self): def commands(self):
cmd = ui.Subcommand( cmd = ui.Subcommand(
'absubmit', 'absubmit',
help=u'calculate and submit AcousticBrainz analysis' help='calculate and submit AcousticBrainz analysis'
)
cmd.parser.add_option(
'-f', '--force', dest='force_refetch',
action='store_true', default=False,
help='re-download data when already present'
)
cmd.parser.add_option(
'-p', '--pretend', dest='pretend_fetch',
action='store_true', default=False,
help='pretend to perform action, but show \
only files which would be processed'
) )
cmd.func = self.command cmd.func = self.command
return [cmd] return [cmd]
@ -104,17 +120,30 @@ class AcousticBrainzSubmitPlugin(plugins.BeetsPlugin):
def command(self, lib, opts, args): def command(self, lib, opts, args):
# Get items from arguments # Get items from arguments
items = lib.items(ui.decargs(args)) items = lib.items(ui.decargs(args))
for item in items: self.opts = opts
analysis = self._get_analysis(item) util.par_map(self.analyze_submit, items)
if analysis:
self._submit_data(item, analysis) def analyze_submit(self, item):
analysis = self._get_analysis(item)
if analysis:
self._submit_data(item, analysis)
def _get_analysis(self, item): def _get_analysis(self, item):
mbid = item['mb_trackid'] mbid = item['mb_trackid']
# If file has no mbid skip it.
# Avoid re-analyzing files that already have AB data.
if not self.opts.force_refetch and not self.config['force']:
if item.get(PROBE_FIELD):
return None
# If file has no MBID, skip it.
if not mbid: if not mbid:
self._log.info(u'Not analysing {}, missing ' self._log.info('Not analysing {}, missing '
u'musicbrainz track id.', item) 'musicbrainz track id.', item)
return None
if self.opts.pretend_fetch or self.config['pretend']:
self._log.info('pretend action - extract item: {}', item)
return None return None
# Temporary file to save extractor output to, extractor only works # Temporary file to save extractor output to, extractor only works
@ -129,11 +158,11 @@ class AcousticBrainzSubmitPlugin(plugins.BeetsPlugin):
call([self.extractor, util.syspath(item.path), filename]) call([self.extractor, util.syspath(item.path), filename])
except ABSubmitError as e: except ABSubmitError as e:
self._log.warning( self._log.warning(
u'Failed to analyse {item} for AcousticBrainz: {error}', 'Failed to analyse {item} for AcousticBrainz: {error}',
item=item, error=e item=item, error=e
) )
return None return None
with open(filename, 'rb') as tmp_file: with open(filename) as tmp_file:
analysis = json.load(tmp_file) analysis = json.load(tmp_file)
# Add the hash to the output. # Add the hash to the output.
analysis['metadata']['version']['essentia_build_sha'] = \ analysis['metadata']['version']['essentia_build_sha'] = \
@ -157,11 +186,11 @@ class AcousticBrainzSubmitPlugin(plugins.BeetsPlugin):
try: try:
message = response.json()['message'] message = response.json()['message']
except (ValueError, KeyError) as e: except (ValueError, KeyError) as e:
message = u'unable to get error message: {}'.format(e) message = f'unable to get error message: {e}'
self._log.error( self._log.error(
u'Failed to submit AcousticBrainz analysis of {item}: ' 'Failed to submit AcousticBrainz analysis of {item}: '
u'{message}).', item=item, message=message '{message}).', item=item, message=message
) )
else: else:
self._log.debug(u'Successfully submitted AcousticBrainz analysis ' self._log.debug('Successfully submitted AcousticBrainz analysis '
u'for {}.', item) 'for {}.', item)

87
libs/common/beetsplug/acousticbrainz.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2015-2016, Ohm Patel. # Copyright 2015-2016, Ohm Patel.
# #
@ -15,12 +14,13 @@
"""Fetch various AcousticBrainz metadata using MBID. """Fetch various AcousticBrainz metadata using MBID.
""" """
from __future__ import division, absolute_import, print_function
from collections import defaultdict
import requests import requests
from collections import defaultdict
from beets import plugins, ui from beets import plugins, ui
from beets.dbcore import types
ACOUSTIC_BASE = "https://acousticbrainz.org/" ACOUSTIC_BASE = "https://acousticbrainz.org/"
LEVELS = ["/low-level", "/high-level"] LEVELS = ["/low-level", "/high-level"]
@ -72,6 +72,9 @@ ABSCHEME = {
'sad': 'mood_sad' 'sad': 'mood_sad'
} }
}, },
'moods_mirex': {
'value': 'moods_mirex'
},
'ismir04_rhythm': { 'ismir04_rhythm': {
'value': 'rhythm' 'value': 'rhythm'
}, },
@ -80,6 +83,9 @@ ABSCHEME = {
'tonal': 'tonal' 'tonal': 'tonal'
} }
}, },
'timbre': {
'value': 'timbre'
},
'voice_instrumental': { 'voice_instrumental': {
'value': 'voice_instrumental' 'value': 'voice_instrumental'
}, },
@ -104,8 +110,33 @@ ABSCHEME = {
class AcousticPlugin(plugins.BeetsPlugin): class AcousticPlugin(plugins.BeetsPlugin):
item_types = {
'average_loudness': types.Float(6),
'chords_changes_rate': types.Float(6),
'chords_key': types.STRING,
'chords_number_rate': types.Float(6),
'chords_scale': types.STRING,
'danceable': types.Float(6),
'gender': types.STRING,
'genre_rosamerica': types.STRING,
'initial_key': types.STRING,
'key_strength': types.Float(6),
'mood_acoustic': types.Float(6),
'mood_aggressive': types.Float(6),
'mood_electronic': types.Float(6),
'mood_happy': types.Float(6),
'mood_party': types.Float(6),
'mood_relaxed': types.Float(6),
'mood_sad': types.Float(6),
'moods_mirex': types.STRING,
'rhythm': types.Float(6),
'timbre': types.STRING,
'tonal': types.Float(6),
'voice_instrumental': types.STRING,
}
def __init__(self): def __init__(self):
super(AcousticPlugin, self).__init__() super().__init__()
self.config.add({ self.config.add({
'auto': True, 'auto': True,
@ -119,11 +150,11 @@ class AcousticPlugin(plugins.BeetsPlugin):
def commands(self): def commands(self):
cmd = ui.Subcommand('acousticbrainz', cmd = ui.Subcommand('acousticbrainz',
help=u"fetch metadata from AcousticBrainz") help="fetch metadata from AcousticBrainz")
cmd.parser.add_option( cmd.parser.add_option(
u'-f', u'--force', dest='force_refetch', '-f', '--force', dest='force_refetch',
action='store_true', default=False, action='store_true', default=False,
help=u're-download data when already present' help='re-download data when already present'
) )
def func(lib, opts, args): def func(lib, opts, args):
@ -142,22 +173,22 @@ class AcousticPlugin(plugins.BeetsPlugin):
def _get_data(self, mbid): def _get_data(self, mbid):
data = {} data = {}
for url in _generate_urls(mbid): for url in _generate_urls(mbid):
self._log.debug(u'fetching URL: {}', url) self._log.debug('fetching URL: {}', url)
try: try:
res = requests.get(url) res = requests.get(url)
except requests.RequestException as exc: except requests.RequestException as exc:
self._log.info(u'request error: {}', exc) self._log.info('request error: {}', exc)
return {} return {}
if res.status_code == 404: if res.status_code == 404:
self._log.info(u'recording ID {} not found', mbid) self._log.info('recording ID {} not found', mbid)
return {} return {}
try: try:
data.update(res.json()) data.update(res.json())
except ValueError: except ValueError:
self._log.debug(u'Invalid Response: {}', res.text) self._log.debug('Invalid Response: {}', res.text)
return {} return {}
return data return data
@ -172,28 +203,28 @@ class AcousticPlugin(plugins.BeetsPlugin):
# representative field name to check for previously fetched # representative field name to check for previously fetched
# data. # data.
if not force: if not force:
mood_str = item.get('mood_acoustic', u'') mood_str = item.get('mood_acoustic', '')
if mood_str: if mood_str:
self._log.info(u'data already present for: {}', item) self._log.info('data already present for: {}', item)
continue continue
# We can only fetch data for tracks with MBIDs. # We can only fetch data for tracks with MBIDs.
if not item.mb_trackid: if not item.mb_trackid:
continue continue
self._log.info(u'getting data for: {}', item) self._log.info('getting data for: {}', item)
data = self._get_data(item.mb_trackid) data = self._get_data(item.mb_trackid)
if data: if data:
for attr, val in self._map_data_to_scheme(data, ABSCHEME): for attr, val in self._map_data_to_scheme(data, ABSCHEME):
if not tags or attr in tags: if not tags or attr in tags:
self._log.debug(u'attribute {} of {} set to {}', self._log.debug('attribute {} of {} set to {}',
attr, attr,
item, item,
val) val)
setattr(item, attr, val) setattr(item, attr, val)
else: else:
self._log.debug(u'skipping attribute {} of {}' self._log.debug('skipping attribute {} of {}'
u' (value {}) due to config', ' (value {}) due to config',
attr, attr,
item, item,
val) val)
@ -255,10 +286,9 @@ class AcousticPlugin(plugins.BeetsPlugin):
# The recursive traversal. # The recursive traversal.
composites = defaultdict(list) composites = defaultdict(list)
for attr, val in self._data_to_scheme_child(data, yield from self._data_to_scheme_child(data,
scheme, scheme,
composites): composites)
yield attr, val
# When composites has been populated, yield the composite attributes # When composites has been populated, yield the composite attributes
# by joining their parts. # by joining their parts.
@ -278,10 +308,9 @@ class AcousticPlugin(plugins.BeetsPlugin):
for k, v in subscheme.items(): for k, v in subscheme.items():
if k in subdata: if k in subdata:
if type(v) == dict: if type(v) == dict:
for attr, val in self._data_to_scheme_child(subdata[k], yield from self._data_to_scheme_child(subdata[k],
v, v,
composites): composites)
yield attr, val
elif type(v) == tuple: elif type(v) == tuple:
composite_attribute, part_number = v composite_attribute, part_number = v
attribute_parts = composites[composite_attribute] attribute_parts = composites[composite_attribute]
@ -292,10 +321,10 @@ class AcousticPlugin(plugins.BeetsPlugin):
else: else:
yield v, subdata[k] yield v, subdata[k]
else: else:
self._log.warning(u'Acousticbrainz did not provide info' self._log.warning('Acousticbrainz did not provide info'
u'about {}', k) 'about {}', k)
self._log.debug(u'Data {} could not be mapped to scheme {} ' self._log.debug('Data {} could not be mapped to scheme {} '
u'because key {} was not found', subdata, v, k) 'because key {} was not found', subdata, v, k)
def _generate_urls(mbid): def _generate_urls(mbid):

65
libs/common/beetsplug/albumtypes.py Normal file
View file

@ -0,0 +1,65 @@
# This file is part of beets.
# Copyright 2021, Edgars Supe.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Adds an album template field for formatted album types."""
from beets.autotag.mb import VARIOUS_ARTISTS_ID
from beets.library import Album
from beets.plugins import BeetsPlugin
class AlbumTypesPlugin(BeetsPlugin):
"""Adds an album template field for formatted album types."""
def __init__(self):
"""Init AlbumTypesPlugin."""
super().__init__()
self.album_template_fields['atypes'] = self._atypes
self.config.add({
'types': [
('ep', 'EP'),
('single', 'Single'),
('soundtrack', 'OST'),
('live', 'Live'),
('compilation', 'Anthology'),
('remix', 'Remix')
],
'ignore_va': ['compilation'],
'bracket': '[]'
})
def _atypes(self, item: Album):
"""Returns a formatted string based on album's types."""
types = self.config['types'].as_pairs()
ignore_va = self.config['ignore_va'].as_str_seq()
bracket = self.config['bracket'].as_str()
# Assign a left and right bracket or leave blank if argument is empty.
if len(bracket) == 2:
bracket_l = bracket[0]
bracket_r = bracket[1]
else:
bracket_l = ''
bracket_r = ''
res = ''
albumtypes = item.albumtypes.split('; ')
is_va = item.mb_albumartistid == VARIOUS_ARTISTS_ID
for type in types:
if type[0] in albumtypes and type[1]:
if not is_va or (type[0] not in ignore_va and is_va):
res += f'{bracket_l}{type[1]}{bracket_r}'
return res

984
libs/common/beetsplug/aura.py Normal file
View file

@ -0,0 +1,984 @@
# This file is part of beets.
# Copyright 2020, Callum Brown.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""An AURA server using Flask."""
from mimetypes import guess_type
import re
import os.path
from os.path import isfile, getsize
from beets.plugins import BeetsPlugin
from beets.ui import Subcommand, _open_library
from beets import config
from beets.util import py3_path
from beets.library import Item, Album
from beets.dbcore.query import (
MatchQuery,
NotQuery,
RegexpQuery,
AndQuery,
FixedFieldSort,
SlowFieldSort,
MultipleSort,
)
from flask import (
Blueprint,
Flask,
current_app,
send_file,
make_response,
request,
)
# Constants
# AURA server information
# TODO: Add version information
SERVER_INFO = {
"aura-version": "0",
"server": "beets-aura",
"server-version": "0.1",
"auth-required": False,
"features": ["albums", "artists", "images"],
}
# Maps AURA Track attribute to beets Item attribute
TRACK_ATTR_MAP = {
# Required
"title": "title",
"artist": "artist",
# Optional
"album": "album",
"track": "track", # Track number on album
"tracktotal": "tracktotal",
"disc": "disc",
"disctotal": "disctotal",
"year": "year",
"month": "month",
"day": "day",
"bpm": "bpm",
"genre": "genre",
"recording-mbid": "mb_trackid", # beets trackid is MB recording
"track-mbid": "mb_releasetrackid",
"composer": "composer",
"albumartist": "albumartist",
"comments": "comments",
# Optional for Audio Metadata
# TODO: Support the mimetype attribute, format != mime type
# "mimetype": track.format,
"duration": "length",
"framerate": "samplerate",
# I don't think beets has a framecount field
# "framecount": ???,
"channels": "channels",
"bitrate": "bitrate",
"bitdepth": "bitdepth",
"size": "filesize",
}
# Maps AURA Album attribute to beets Album attribute
ALBUM_ATTR_MAP = {
# Required
"title": "album",
"artist": "albumartist",
# Optional
"tracktotal": "albumtotal",
"disctotal": "disctotal",
"year": "year",
"month": "month",
"day": "day",
"genre": "genre",
"release-mbid": "mb_albumid",
"release-group-mbid": "mb_releasegroupid",
}
# Maps AURA Artist attribute to beets Item field
# Artists are not first-class in beets, so information is extracted from
# beets Items.
ARTIST_ATTR_MAP = {
# Required
"name": "artist",
# Optional
"artist-mbid": "mb_artistid",
}
class AURADocument:
"""Base class for building AURA documents."""
@staticmethod
def error(status, title, detail):
"""Make a response for an error following the JSON:API spec.
Args:
status: An HTTP status code string, e.g. "404 Not Found".
title: A short, human-readable summary of the problem.
detail: A human-readable explanation specific to this
occurrence of the problem.
"""
document = {
"errors": [{"status": status, "title": title, "detail": detail}]
}
return make_response(document, status)
def translate_filters(self):
"""Translate filters from request arguments to a beets Query."""
# The format of each filter key in the request parameter is:
# filter[<attribute>]. This regex extracts <attribute>.
pattern = re.compile(r"filter\[(?P<attribute>[a-zA-Z0-9_-]+)\]")
queries = []
for key, value in request.args.items():
match = pattern.match(key)
if match:
# Extract attribute name from key
aura_attr = match.group("attribute")
# Get the beets version of the attribute name
beets_attr = self.attribute_map.get(aura_attr, aura_attr)
converter = self.get_attribute_converter(beets_attr)
value = converter(value)
# Add exact match query to list
# Use a slow query so it works with all fields
queries.append(MatchQuery(beets_attr, value, fast=False))
# NOTE: AURA doesn't officially support multiple queries
return AndQuery(queries)
def translate_sorts(self, sort_arg):
"""Translate an AURA sort parameter into a beets Sort.
Args:
sort_arg: The value of the 'sort' query parameter; a comma
separated list of fields to sort by, in order.
E.g. "-year,title".
"""
# Change HTTP query parameter to a list
aura_sorts = sort_arg.strip(",").split(",")
sorts = []
for aura_attr in aura_sorts:
if aura_attr[0] == "-":
ascending = False
# Remove leading "-"
aura_attr = aura_attr[1:]
else:
# JSON:API default
ascending = True
# Get the beets version of the attribute name
beets_attr = self.attribute_map.get(aura_attr, aura_attr)
# Use slow sort so it works with all fields (inc. computed)
sorts.append(SlowFieldSort(beets_attr, ascending=ascending))
return MultipleSort(sorts)
def paginate(self, collection):
"""Get a page of the collection and the URL to the next page.
Args:
collection: The raw data from which resource objects can be
built. Could be an sqlite3.Cursor object (tracks and
albums) or a list of strings (artists).
"""
# Pages start from zero
page = request.args.get("page", 0, int)
# Use page limit defined in config by default.
default_limit = config["aura"]["page_limit"].get(int)
limit = request.args.get("limit", default_limit, int)
# start = offset of first item to return
start = page * limit
# end = offset of last item + 1
end = start + limit
if end > len(collection):
end = len(collection)
next_url = None
else:
# Not the last page so work out links.next url
if not request.args:
# No existing arguments, so current page is 0
next_url = request.url + "?page=1"
elif not request.args.get("page", None):
# No existing page argument, so add one to the end
next_url = request.url + "&page=1"
else:
# Increment page token by 1
next_url = request.url.replace(
f"page={page}", "page={}".format(page + 1)
)
# Get only the items in the page range
data = [self.resource_object(collection[i]) for i in range(start, end)]
return data, next_url
def get_included(self, data, include_str):
"""Build a list of resource objects for inclusion.
Args:
data: An array of dicts in the form of resource objects.
include_str: A comma separated list of resource types to
include. E.g. "tracks,images".
"""
# Change HTTP query parameter to a list
to_include = include_str.strip(",").split(",")
# Build a list of unique type and id combinations
# For each resource object in the primary data, iterate over it's
# relationships. If a relationship matches one of the types
# requested for inclusion (e.g. "albums") then add each type-id pair
# under the "data" key to unique_identifiers, checking first that
# it has not already been added. This ensures that no resources are
# included more than once.
unique_identifiers = []
for res_obj in data:
for rel_name, rel_obj in res_obj["relationships"].items():
if rel_name in to_include:
# NOTE: Assumes relationship is to-many
for identifier in rel_obj["data"]:
if identifier not in unique_identifiers:
unique_identifiers.append(identifier)
# TODO: I think this could be improved
included = []
for identifier in unique_identifiers:
res_type = identifier["type"]
if res_type == "track":
track_id = int(identifier["id"])
track = current_app.config["lib"].get_item(track_id)
included.append(TrackDocument.resource_object(track))
elif res_type == "album":
album_id = int(identifier["id"])
album = current_app.config["lib"].get_album(album_id)
included.append(AlbumDocument.resource_object(album))
elif res_type == "artist":
artist_id = identifier["id"]
included.append(ArtistDocument.resource_object(artist_id))
elif res_type == "image":
image_id = identifier["id"]
included.append(ImageDocument.resource_object(image_id))
else:
raise ValueError(f"Invalid resource type: {res_type}")
return included
def all_resources(self):
"""Build document for /tracks, /albums or /artists."""
query = self.translate_filters()
sort_arg = request.args.get("sort", None)
if sort_arg:
sort = self.translate_sorts(sort_arg)
# For each sort field add a query which ensures all results
# have a non-empty, non-zero value for that field.
for s in sort.sorts:
query.subqueries.append(
NotQuery(
# Match empty fields (^$) or zero fields, (^0$)
RegexpQuery(s.field, "(^$|^0$)", fast=False)
)
)
else:
sort = None
# Get information from the library
collection = self.get_collection(query=query, sort=sort)
# Convert info to AURA form and paginate it
data, next_url = self.paginate(collection)
document = {"data": data}
# If there are more pages then provide a way to access them
if next_url:
document["links"] = {"next": next_url}
# Include related resources for each element in "data"
include_str = request.args.get("include", None)
if include_str:
document["included"] = self.get_included(data, include_str)
return document
def single_resource_document(self, resource_object):
"""Build document for a specific requested resource.
Args:
resource_object: A dictionary in the form of a JSON:API
resource object.
"""
document = {"data": resource_object}
include_str = request.args.get("include", None)
if include_str:
# [document["data"]] is because arg needs to be list
document["included"] = self.get_included(
[document["data"]], include_str
)
return document
class TrackDocument(AURADocument):
"""Class for building documents for /tracks endpoints."""
attribute_map = TRACK_ATTR_MAP
def get_collection(self, query=None, sort=None):
"""Get Item objects from the library.
Args:
query: A beets Query object or a beets query string.
sort: A beets Sort object.
"""
return current_app.config["lib"].items(query, sort)
def get_attribute_converter(self, beets_attr):
"""Work out what data type an attribute should be for beets.
Args:
beets_attr: The name of the beets attribute, e.g. "title".
"""
# filesize is a special field (read from disk not db?)
if beets_attr == "filesize":
converter = int
else:
try:
# Look for field in list of Item fields
# and get python type of database type.
# See beets.library.Item and beets.dbcore.types
converter = Item._fields[beets_attr].model_type
except KeyError:
# Fall back to string (NOTE: probably not good)
converter = str
return converter
@staticmethod
def resource_object(track):
"""Construct a JSON:API resource object from a beets Item.
Args:
track: A beets Item object.
"""
attributes = {}
# Use aura => beets attribute map, e.g. size => filesize
for aura_attr, beets_attr in TRACK_ATTR_MAP.items():
a = getattr(track, beets_attr)
# Only set attribute if it's not None, 0, "", etc.
# NOTE: This could result in required attributes not being set
if a:
attributes[aura_attr] = a
# JSON:API one-to-many relationship to parent album
relationships = {
"artists": {"data": [{"type": "artist", "id": track.artist}]}
}
# Only add album relationship if not singleton
if not track.singleton:
relationships["albums"] = {
"data": [{"type": "album", "id": str(track.album_id)}]
}
return {
"type": "track",
"id": str(track.id),
"attributes": attributes,
"relationships": relationships,
}
def single_resource(self, track_id):
"""Get track from the library and build a document.
Args:
track_id: The beets id of the track (integer).
"""
track = current_app.config["lib"].get_item(track_id)
if not track:
return self.error(
"404 Not Found",
"No track with the requested id.",
"There is no track with an id of {} in the library.".format(
track_id
),
)
return self.single_resource_document(self.resource_object(track))
class AlbumDocument(AURADocument):
"""Class for building documents for /albums endpoints."""
attribute_map = ALBUM_ATTR_MAP
def get_collection(self, query=None, sort=None):
"""Get Album objects from the library.
Args:
query: A beets Query object or a beets query string.
sort: A beets Sort object.
"""
return current_app.config["lib"].albums(query, sort)
def get_attribute_converter(self, beets_attr):
"""Work out what data type an attribute should be for beets.
Args:
beets_attr: The name of the beets attribute, e.g. "title".
"""
try:
# Look for field in list of Album fields
# and get python type of database type.
# See beets.library.Album and beets.dbcore.types
converter = Album._fields[beets_attr].model_type
except KeyError:
# Fall back to string (NOTE: probably not good)
converter = str
return converter
@staticmethod
def resource_object(album):
"""Construct a JSON:API resource object from a beets Album.
Args:
album: A beets Album object.
"""
attributes = {}
# Use aura => beets attribute name map
for aura_attr, beets_attr in ALBUM_ATTR_MAP.items():
a = getattr(album, beets_attr)
# Only set attribute if it's not None, 0, "", etc.
# NOTE: This could mean required attributes are not set
if a:
attributes[aura_attr] = a
# Get beets Item objects for all tracks in the album sorted by
# track number. Sorting is not required but it's nice.
query = MatchQuery("album_id", album.id)
sort = FixedFieldSort("track", ascending=True)
tracks = current_app.config["lib"].items(query, sort)
# JSON:API one-to-many relationship to tracks on the album
relationships = {
"tracks": {
"data": [{"type": "track", "id": str(t.id)} for t in tracks]
}
}
# Add images relationship if album has associated images
if album.artpath:
path = py3_path(album.artpath)
filename = path.split("/")[-1]
image_id = f"album-{album.id}-{filename}"
relationships["images"] = {
"data": [{"type": "image", "id": image_id}]
}
# Add artist relationship if artist name is same on tracks
# Tracks are used to define artists so don't albumartist
# Check for all tracks in case some have featured artists
if album.albumartist in [t.artist for t in tracks]:
relationships["artists"] = {
"data": [{"type": "artist", "id": album.albumartist}]
}
return {
"type": "album",
"id": str(album.id),
"attributes": attributes,
"relationships": relationships,
}
def single_resource(self, album_id):
"""Get album from the library and build a document.
Args:
album_id: The beets id of the album (integer).
"""
album = current_app.config["lib"].get_album(album_id)
if not album:
return self.error(
"404 Not Found",
"No album with the requested id.",
"There is no album with an id of {} in the library.".format(
album_id
),
)
return self.single_resource_document(self.resource_object(album))
class ArtistDocument(AURADocument):
"""Class for building documents for /artists endpoints."""
attribute_map = ARTIST_ATTR_MAP
def get_collection(self, query=None, sort=None):
"""Get a list of artist names from the library.
Args:
query: A beets Query object or a beets query string.
sort: A beets Sort object.
"""
# Gets only tracks with matching artist information
tracks = current_app.config["lib"].items(query, sort)
collection = []
for track in tracks:
# Do not add duplicates
if track.artist not in collection:
collection.append(track.artist)
return collection
def get_attribute_converter(self, beets_attr):
"""Work out what data type an attribute should be for beets.
Args:
beets_attr: The name of the beets attribute, e.g. "artist".
"""
try:
# Look for field in list of Item fields
# and get python type of database type.
# See beets.library.Item and beets.dbcore.types
converter = Item._fields[beets_attr].model_type
except KeyError:
# Fall back to string (NOTE: probably not good)
converter = str
return converter
@staticmethod
def resource_object(artist_id):
"""Construct a JSON:API resource object for the given artist.
Args:
artist_id: A string which is the artist's name.
"""
# Get tracks where artist field exactly matches artist_id
query = MatchQuery("artist", artist_id)
tracks = current_app.config["lib"].items(query)
if not tracks:
return None
# Get artist information from the first track
# NOTE: It could be that the first track doesn't have a
# MusicBrainz id but later tracks do, which isn't ideal.
attributes = {}
# Use aura => beets attribute map, e.g. artist => name
for aura_attr, beets_attr in ARTIST_ATTR_MAP.items():
a = getattr(tracks[0], beets_attr)
# Only set attribute if it's not None, 0, "", etc.
# NOTE: This could mean required attributes are not set
if a:
attributes[aura_attr] = a
relationships = {
"tracks": {
"data": [{"type": "track", "id": str(t.id)} for t in tracks]
}
}
album_query = MatchQuery("albumartist", artist_id)
albums = current_app.config["lib"].albums(query=album_query)
if len(albums) != 0:
relationships["albums"] = {
"data": [{"type": "album", "id": str(a.id)} for a in albums]
}
return {
"type": "artist",
"id": artist_id,
"attributes": attributes,
"relationships": relationships,
}
def single_resource(self, artist_id):
"""Get info for the requested artist and build a document.
Args:
artist_id: A string which is the artist's name.
"""
artist_resource = self.resource_object(artist_id)
if not artist_resource:
return self.error(
"404 Not Found",
"No artist with the requested id.",
"There is no artist with an id of {} in the library.".format(
artist_id
),
)
return self.single_resource_document(artist_resource)
def safe_filename(fn):
"""Check whether a string is a simple (non-path) filename.
For example, `foo.txt` is safe because it is a "plain" filename. But
`foo/bar.txt` and `../foo.txt` and `.` are all non-safe because they
can traverse to other directories other than the current one.
"""
# Rule out any directories.
if os.path.basename(fn) != fn:
return False
# In single names, rule out Unix directory traversal names.
if fn in ('.', '..'):
return False
return True
class ImageDocument(AURADocument):
"""Class for building documents for /images/(id) endpoints."""
@staticmethod
def get_image_path(image_id):
"""Works out the full path to the image with the given id.
Returns None if there is no such image.
Args:
image_id: A string in the form
"<parent_type>-<parent_id>-<img_filename>".
"""
# Split image_id into its constituent parts
id_split = image_id.split("-")
if len(id_split) < 3:
# image_id is not in the required format
return None
parent_type = id_split[0]
parent_id = id_split[1]
img_filename = "-".join(id_split[2:])
if not safe_filename(img_filename):
return None
# Get the path to the directory parent's images are in
if parent_type == "album":
album = current_app.config["lib"].get_album(int(parent_id))
if not album or not album.artpath:
return None
# Cut the filename off of artpath
# This is in preparation for supporting images in the same
# directory that are not tracked by beets.
artpath = py3_path(album.artpath)
dir_path = "/".join(artpath.split("/")[:-1])
else:
# Images for other resource types are not supported
return None
img_path = os.path.join(dir_path, img_filename)
# Check the image actually exists
if isfile(img_path):
return img_path
else:
return None
@staticmethod
def resource_object(image_id):
"""Construct a JSON:API resource object for the given image.
Args:
image_id: A string in the form
"<parent_type>-<parent_id>-<img_filename>".
"""
# Could be called as a static method, so can't use
# self.get_image_path()
image_path = ImageDocument.get_image_path(image_id)
if not image_path:
return None
attributes = {
"role": "cover",
"mimetype": guess_type(image_path)[0],
"size": getsize(image_path),
}
try:
from PIL import Image
except ImportError:
pass
else:
im = Image.open(image_path)
attributes["width"] = im.width
attributes["height"] = im.height
relationships = {}
# Split id into [parent_type, parent_id, filename]
id_split = image_id.split("-")
relationships[id_split[0] + "s"] = {
"data": [{"type": id_split[0], "id": id_split[1]}]
}
return {
"id": image_id,
"type": "image",
# Remove attributes that are None, 0, "", etc.
"attributes": {k: v for k, v in attributes.items() if v},
"relationships": relationships,
}
def single_resource(self, image_id):
"""Get info for the requested image and build a document.
Args:
image_id: A string in the form
"<parent_type>-<parent_id>-<img_filename>".
"""
image_resource = self.resource_object(image_id)
if not image_resource:
return self.error(
"404 Not Found",
"No image with the requested id.",
"There is no image with an id of {} in the library.".format(
image_id
),
)
return self.single_resource_document(image_resource)
# Initialise flask blueprint
aura_bp = Blueprint("aura_bp", __name__)
@aura_bp.route("/server")
def server_info():
"""Respond with info about the server."""
return {"data": {"type": "server", "id": "0", "attributes": SERVER_INFO}}
# Track endpoints
@aura_bp.route("/tracks")
def all_tracks():
"""Respond with a list of all tracks and related information."""
doc = TrackDocument()
return doc.all_resources()
@aura_bp.route("/tracks/<int:track_id>")
def single_track(track_id):
"""Respond with info about the specified track.
Args:
track_id: The id of the track provided in the URL (integer).
"""
doc = TrackDocument()
return doc.single_resource(track_id)
@aura_bp.route("/tracks/<int:track_id>/audio")
def audio_file(track_id):
"""Supply an audio file for the specified track.
Args:
track_id: The id of the track provided in the URL (integer).
"""
track = current_app.config["lib"].get_item(track_id)
if not track:
return AURADocument.error(
"404 Not Found",
"No track with the requested id.",
"There is no track with an id of {} in the library.".format(
track_id
),
)
path = py3_path(track.path)
if not isfile(path):
return AURADocument.error(
"404 Not Found",
"No audio file for the requested track.",
(
"There is no audio file for track {} at the expected location"
).format(track_id),
)
file_mimetype = guess_type(path)[0]
if not file_mimetype:
return AURADocument.error(
"500 Internal Server Error",
"Requested audio file has an unknown mimetype.",
(
"The audio file for track {} has an unknown mimetype. "
"Its file extension is {}."
).format(track_id, path.split(".")[-1]),
)
# Check that the Accept header contains the file's mimetype
# Takes into account */* and audio/*
# Adding support for the bitrate parameter would require some effort so I
# left it out. This means the client could be sent an error even if the
# audio doesn't need transcoding.
if not request.accept_mimetypes.best_match([file_mimetype]):
return AURADocument.error(
"406 Not Acceptable",
"Unsupported MIME type or bitrate parameter in Accept header.",
(
"The audio file for track {} is only available as {} and "
"bitrate parameters are not supported."
).format(track_id, file_mimetype),
)
return send_file(
path,
mimetype=file_mimetype,
# Handles filename in Content-Disposition header
as_attachment=True,
# Tries to upgrade the stream to support range requests
conditional=True,
)
# Album endpoints
@aura_bp.route("/albums")
def all_albums():
"""Respond with a list of all albums and related information."""
doc = AlbumDocument()
return doc.all_resources()
@aura_bp.route("/albums/<int:album_id>")
def single_album(album_id):
"""Respond with info about the specified album.
Args:
album_id: The id of the album provided in the URL (integer).
"""
doc = AlbumDocument()
return doc.single_resource(album_id)
# Artist endpoints
# Artist ids are their names
@aura_bp.route("/artists")
def all_artists():
"""Respond with a list of all artists and related information."""
doc = ArtistDocument()
return doc.all_resources()
# Using the path converter allows slashes in artist_id
@aura_bp.route("/artists/<path:artist_id>")
def single_artist(artist_id):
"""Respond with info about the specified artist.
Args:
artist_id: The id of the artist provided in the URL. A string
which is the artist's name.
"""
doc = ArtistDocument()
return doc.single_resource(artist_id)
# Image endpoints
# Image ids are in the form <parent_type>-<parent_id>-<img_filename>
# For example: album-13-cover.jpg
@aura_bp.route("/images/<string:image_id>")
def single_image(image_id):
"""Respond with info about the specified image.
Args:
image_id: The id of the image provided in the URL. A string in
the form "<parent_type>-<parent_id>-<img_filename>".
"""
doc = ImageDocument()
return doc.single_resource(image_id)
@aura_bp.route("/images/<string:image_id>/file")
def image_file(image_id):
"""Supply an image file for the specified image.
Args:
image_id: The id of the image provided in the URL. A string in
the form "<parent_type>-<parent_id>-<img_filename>".
"""
img_path = ImageDocument.get_image_path(image_id)
if not img_path:
return AURADocument.error(
"404 Not Found",
"No image with the requested id.",
"There is no image with an id of {} in the library".format(
image_id
),
)
return send_file(img_path)
# WSGI app
def create_app():
"""An application factory for use by a WSGI server."""
config["aura"].add(
{
"host": "127.0.0.1",
"port": 8337,
"cors": [],
"cors_supports_credentials": False,
"page_limit": 500,
}
)
app = Flask(__name__)
# Register AURA blueprint view functions under a URL prefix
app.register_blueprint(aura_bp, url_prefix="/aura")
# AURA specifies mimetype MUST be this
app.config["JSONIFY_MIMETYPE"] = "application/vnd.api+json"
# Disable auto-sorting of JSON keys
app.config["JSON_SORT_KEYS"] = False
# Provide a way to access the beets library
# The normal method of using the Library and config provided in the
# command function is not used because create_app() could be called
# by an external WSGI server.
# NOTE: this uses a 'private' function from beets.ui.__init__
app.config["lib"] = _open_library(config)
# Enable CORS if required
cors = config["aura"]["cors"].as_str_seq(list)
if cors:
from flask_cors import CORS
# "Accept" is the only header clients use
app.config["CORS_ALLOW_HEADERS"] = "Accept"
app.config["CORS_RESOURCES"] = {r"/aura/*": {"origins": cors}}
app.config["CORS_SUPPORTS_CREDENTIALS"] = config["aura"][
"cors_supports_credentials"
].get(bool)
CORS(app)
return app
# Beets Plugin Hook
class AURAPlugin(BeetsPlugin):
"""The BeetsPlugin subclass for the AURA server plugin."""
def __init__(self):
"""Add configuration options for the AURA plugin."""
super().__init__()
def commands(self):
"""Add subcommand used to run the AURA server."""
def run_aura(lib, opts, args):
"""Run the application using Flask's built in-server.
Args:
lib: A beets Library object (not used).
opts: Command line options. An optparse.Values object.
args: The list of arguments to process (not used).
"""
app = create_app()
# Start the built-in server (not intended for production)
app.run(
host=self.config["host"].get(str),
port=self.config["port"].get(int),
debug=opts.debug,
threaded=True,
)
run_aura_cmd = Subcommand("aura", help="run an AURA server")
run_aura_cmd.parser.add_option(
"-d",
"--debug",
action="store_true",
default=False,
help="use Flask debug mode",
)
run_aura_cmd.func = run_aura
return [run_aura_cmd]

181
libs/common/beetsplug/badfiles.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, François-Xavier Thomas. # Copyright 2016, François-Xavier Thomas.
# #
@ -16,18 +15,19 @@
"""Use command-line tools to check for audio file corruption. """Use command-line tools to check for audio file corruption.
""" """
from __future__ import division, absolute_import, print_function
from beets.plugins import BeetsPlugin
from beets.ui import Subcommand
from beets.util import displayable_path, confit
from beets import ui
from subprocess import check_output, CalledProcessError, list2cmdline, STDOUT from subprocess import check_output, CalledProcessError, list2cmdline, STDOUT
import shlex import shlex
import os import os
import errno import errno
import sys import sys
import six import confuse
from beets.plugins import BeetsPlugin
from beets.ui import Subcommand
from beets.util import displayable_path, par_map
from beets import ui
from beets import importer
class CheckerCommandException(Exception): class CheckerCommandException(Exception):
@ -48,8 +48,17 @@ class CheckerCommandException(Exception):
class BadFiles(BeetsPlugin): class BadFiles(BeetsPlugin):
def __init__(self):
super().__init__()
self.verbose = False
self.register_listener('import_task_start',
self.on_import_task_start)
self.register_listener('import_task_before_choice',
self.on_import_task_before_choice)
def run_command(self, cmd): def run_command(self, cmd):
self._log.debug(u"running command: {}", self._log.debug("running command: {}",
displayable_path(list2cmdline(cmd))) displayable_path(list2cmdline(cmd)))
try: try:
output = check_output(cmd, stderr=STDOUT) output = check_output(cmd, stderr=STDOUT)
@ -61,7 +70,7 @@ class BadFiles(BeetsPlugin):
status = e.returncode status = e.returncode
except OSError as e: except OSError as e:
raise CheckerCommandException(cmd, e) raise CheckerCommandException(cmd, e)
output = output.decode(sys.getfilesystemencoding()) output = output.decode(sys.getdefaultencoding(), 'replace')
return status, errors, [line for line in output.split("\n") if line] return status, errors, [line for line in output.split("\n") if line]
def check_mp3val(self, path): def check_mp3val(self, path):
@ -85,68 +94,122 @@ class BadFiles(BeetsPlugin):
ext = ext.lower() ext = ext.lower()
try: try:
command = self.config['commands'].get(dict).get(ext) command = self.config['commands'].get(dict).get(ext)
except confit.NotFoundError: except confuse.NotFoundError:
command = None command = None
if command: if command:
return self.check_custom(command) return self.check_custom(command)
elif ext == "mp3": if ext == "mp3":
return self.check_mp3val return self.check_mp3val
elif ext == "flac": if ext == "flac":
return self.check_flac return self.check_flac
def check_bad(self, lib, opts, args): def check_item(self, item):
for item in lib.items(ui.decargs(args)): # First, check whether the path exists. If not, the user
# should probably run `beet update` to cleanup your library.
dpath = displayable_path(item.path)
self._log.debug("checking path: {}", dpath)
if not os.path.exists(item.path):
ui.print_("{}: file does not exist".format(
ui.colorize('text_error', dpath)))
# First, check whether the path exists. If not, the user # Run the checker against the file if one is found
# should probably run `beet update` to cleanup your library. ext = os.path.splitext(item.path)[1][1:].decode('utf8', 'ignore')
dpath = displayable_path(item.path) checker = self.get_checker(ext)
self._log.debug(u"checking path: {}", dpath) if not checker:
if not os.path.exists(item.path): self._log.error("no checker specified in the config for {}",
ui.print_(u"{}: file does not exist".format( ext)
ui.colorize('text_error', dpath))) return []
path = item.path
if not isinstance(path, str):
path = item.path.decode(sys.getfilesystemencoding())
try:
status, errors, output = checker(path)
except CheckerCommandException as e:
if e.errno == errno.ENOENT:
self._log.error(
"command not found: {} when validating file: {}",
e.checker,
e.path
)
else:
self._log.error("error invoking {}: {}", e.checker, e.msg)
return []
# Run the checker against the file if one is found error_lines = []
ext = os.path.splitext(item.path)[1][1:].decode('utf8', 'ignore')
checker = self.get_checker(ext) if status > 0:
if not checker: error_lines.append(
self._log.error(u"no checker specified in the config for {}", "{}: checker exited with status {}"
ext) .format(ui.colorize('text_error', dpath), status))
continue for line in output:
path = item.path error_lines.append(f" {line}")
if not isinstance(path, six.text_type):
path = item.path.decode(sys.getfilesystemencoding()) elif errors > 0:
try: error_lines.append(
status, errors, output = checker(path) "{}: checker found {} errors or warnings"
except CheckerCommandException as e: .format(ui.colorize('text_warning', dpath), errors))
if e.errno == errno.ENOENT: for line in output:
self._log.error( error_lines.append(f" {line}")
u"command not found: {} when validating file: {}", elif self.verbose:
e.checker, error_lines.append(
e.path "{}: ok".format(ui.colorize('text_success', dpath)))
)
else: return error_lines
self._log.error(u"error invoking {}: {}", e.checker, e.msg)
continue def on_import_task_start(self, task, session):
if status > 0: if not self.config['check_on_import'].get(False):
ui.print_(u"{}: checker exited with status {}" return
.format(ui.colorize('text_error', dpath), status))
for line in output: checks_failed = []
ui.print_(u" {}".format(displayable_path(line)))
elif errors > 0: for item in task.items:
ui.print_(u"{}: checker found {} errors or warnings" error_lines = self.check_item(item)
.format(ui.colorize('text_warning', dpath), errors)) if error_lines:
for line in output: checks_failed.append(error_lines)
ui.print_(u" {}".format(displayable_path(line)))
elif opts.verbose: if checks_failed:
ui.print_(u"{}: ok".format(ui.colorize('text_success', dpath))) task._badfiles_checks_failed = checks_failed
def on_import_task_before_choice(self, task, session):
if hasattr(task, '_badfiles_checks_failed'):
ui.print_('{} one or more files failed checks:'
.format(ui.colorize('text_warning', 'BAD')))
for error in task._badfiles_checks_failed:
for error_line in error:
ui.print_(error_line)
ui.print_()
ui.print_('What would you like to do?')
sel = ui.input_options(['aBort', 'skip', 'continue'])
if sel == 's':
return importer.action.SKIP
elif sel == 'c':
return None
elif sel == 'b':
raise importer.ImportAbort()
else:
raise Exception(f'Unexpected selection: {sel}')
def command(self, lib, opts, args):
# Get items from arguments
items = lib.items(ui.decargs(args))
self.verbose = opts.verbose
def check_and_print(item):
for error_line in self.check_item(item):
ui.print_(error_line)
par_map(check_and_print, items)
def commands(self): def commands(self):
bad_command = Subcommand('bad', bad_command = Subcommand('bad',
help=u'check for corrupt or missing files') help='check for corrupt or missing files')
bad_command.parser.add_option( bad_command.parser.add_option(
u'-v', u'--verbose', '-v', '--verbose',
action='store_true', default=False, dest='verbose', action='store_true', default=False, dest='verbose',
help=u'view results for both the bad and uncorrupted files' help='view results for both the bad and uncorrupted files'
) )
bad_command.func = self.check_bad bad_command.func = self.command
return [bad_command] return [bad_command]

82
libs/common/beetsplug/bareasc.py Normal file
View file

@ -0,0 +1,82 @@
# This file is part of beets.
# Copyright 2016, Philippe Mongeau.
# Copyright 2021, Graham R. Cobb.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and ascociated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# This module is adapted from Fuzzy in accordance to the licence of
# that module
"""Provides a bare-ASCII matching query."""
from beets import ui
from beets.ui import print_, decargs
from beets.plugins import BeetsPlugin
from beets.dbcore.query import StringFieldQuery
from unidecode import unidecode
class BareascQuery(StringFieldQuery):
"""Compare items using bare ASCII, without accents etc."""
@classmethod
def string_match(cls, pattern, val):
"""Convert both pattern and string to plain ASCII before matching.
If pattern is all lower case, also convert string to lower case so
match is also case insensitive
"""
# smartcase
if pattern.islower():
val = val.lower()
pattern = unidecode(pattern)
val = unidecode(val)
return pattern in val
class BareascPlugin(BeetsPlugin):
"""Plugin to provide bare-ASCII option for beets matching."""
def __init__(self):
"""Default prefix for selecting bare-ASCII matching is #."""
super().__init__()
self.config.add({
'prefix': '#',
})
def queries(self):
"""Register bare-ASCII matching."""
prefix = self.config['prefix'].as_str()
return {prefix: BareascQuery}
def commands(self):
"""Add bareasc command as unidecode version of 'list'."""
cmd = ui.Subcommand('bareasc',
help='unidecode version of beet list command')
cmd.parser.usage += "\n" \
'Example: %prog -f \'$album: $title\' artist:beatles'
cmd.parser.add_all_common_options()
cmd.func = self.unidecode_list
return [cmd]
def unidecode_list(self, lib, opts, args):
"""Emulate normal 'list' command but with unidecode output."""
query = decargs(args)
album = opts.album
# Copied from commands.py - list_items
if album:
for album in lib.albums(query):
bare = unidecode(str(album))
print_(bare)
else:
for item in lib.items(query):
bare = unidecode(str(item))
print_(bare)

171
libs/common/beetsplug/beatport.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson. # Copyright 2016, Adrian Sampson.
# #
@ -15,11 +14,9 @@
"""Adds Beatport release and track search support to the autotagger """Adds Beatport release and track search support to the autotagger
""" """
from __future__ import division, absolute_import, print_function
import json import json
import re import re
import six
from datetime import datetime, timedelta from datetime import datetime, timedelta
from requests_oauthlib import OAuth1Session from requests_oauthlib import OAuth1Session
@ -28,35 +25,35 @@ from requests_oauthlib.oauth1_session import (TokenRequestDenied, TokenMissing,
import beets import beets
import beets.ui import beets.ui
from beets.autotag.hooks import AlbumInfo, TrackInfo, Distance from beets.autotag.hooks import AlbumInfo, TrackInfo
from beets.plugins import BeetsPlugin from beets.plugins import BeetsPlugin, MetadataSourcePlugin, get_distance
from beets.util import confit import confuse
AUTH_ERRORS = (TokenRequestDenied, TokenMissing, VerifierMissing) AUTH_ERRORS = (TokenRequestDenied, TokenMissing, VerifierMissing)
USER_AGENT = u'beets/{0} +http://beets.io/'.format(beets.__version__) USER_AGENT = f'beets/{beets.__version__} +https://beets.io/'
class BeatportAPIError(Exception): class BeatportAPIError(Exception):
pass pass
class BeatportObject(object): class BeatportObject:
def __init__(self, data): def __init__(self, data):
self.beatport_id = data['id'] self.beatport_id = data['id']
self.name = six.text_type(data['name']) self.name = str(data['name'])
if 'releaseDate' in data: if 'releaseDate' in data:
self.release_date = datetime.strptime(data['releaseDate'], self.release_date = datetime.strptime(data['releaseDate'],
'%Y-%m-%d') '%Y-%m-%d')
if 'artists' in data: if 'artists' in data:
self.artists = [(x['id'], six.text_type(x['name'])) self.artists = [(x['id'], str(x['name']))
for x in data['artists']] for x in data['artists']]
if 'genres' in data: if 'genres' in data:
self.genres = [six.text_type(x['name']) self.genres = [str(x['name'])
for x in data['genres']] for x in data['genres']]
class BeatportClient(object): class BeatportClient:
_api_base = 'https://oauth-api.beatport.com' _api_base = 'https://oauth-api.beatport.com'
def __init__(self, c_key, c_secret, auth_key=None, auth_secret=None): def __init__(self, c_key, c_secret, auth_key=None, auth_secret=None):
@ -109,7 +106,7 @@ class BeatportClient(object):
:rtype: (unicode, unicode) tuple :rtype: (unicode, unicode) tuple
""" """
self.api.parse_authorization_response( self.api.parse_authorization_response(
"http://beets.io/auth?" + auth_data) "https://beets.io/auth?" + auth_data)
access_data = self.api.fetch_access_token( access_data = self.api.fetch_access_token(
self._make_url('/identity/1/oauth/access-token')) self._make_url('/identity/1/oauth/access-token'))
return access_data['oauth_token'], access_data['oauth_token_secret'] return access_data['oauth_token'], access_data['oauth_token_secret']
@ -131,7 +128,7 @@ class BeatportClient(object):
""" """
response = self._get('catalog/3/search', response = self._get('catalog/3/search',
query=query, perPage=5, query=query, perPage=5,
facets=['fieldType:{0}'.format(release_type)]) facets=[f'fieldType:{release_type}'])
for item in response: for item in response:
if release_type == 'release': if release_type == 'release':
if details: if details:
@ -150,9 +147,11 @@ class BeatportClient(object):
:rtype: :py:class:`BeatportRelease` :rtype: :py:class:`BeatportRelease`
""" """
response = self._get('/catalog/3/releases', id=beatport_id) response = self._get('/catalog/3/releases', id=beatport_id)
release = BeatportRelease(response[0]) if response:
release.tracks = self.get_release_tracks(beatport_id) release = BeatportRelease(response[0])
return release release.tracks = self.get_release_tracks(beatport_id)
return release
return None
def get_release_tracks(self, beatport_id): def get_release_tracks(self, beatport_id):
""" Get all tracks for a given release. """ Get all tracks for a given release.
@ -191,7 +190,7 @@ class BeatportClient(object):
response = self.api.get(self._make_url(endpoint), params=kwargs) response = self.api.get(self._make_url(endpoint), params=kwargs)
except Exception as e: except Exception as e:
raise BeatportAPIError("Error connecting to Beatport API: {}" raise BeatportAPIError("Error connecting to Beatport API: {}"
.format(e.message)) .format(e))
if not response: if not response:
raise BeatportAPIError( raise BeatportAPIError(
"Error {0.status_code} for '{0.request.path_url}" "Error {0.status_code} for '{0.request.path_url}"
@ -199,21 +198,20 @@ class BeatportClient(object):
return response.json()['results'] return response.json()['results']
@six.python_2_unicode_compatible
class BeatportRelease(BeatportObject): class BeatportRelease(BeatportObject):
def __str__(self): def __str__(self):
if len(self.artists) < 4: if len(self.artists) < 4:
artist_str = ", ".join(x[1] for x in self.artists) artist_str = ", ".join(x[1] for x in self.artists)
else: else:
artist_str = "Various Artists" artist_str = "Various Artists"
return u"<BeatportRelease: {0} - {1} ({2})>".format( return "<BeatportRelease: {} - {} ({})>".format(
artist_str, artist_str,
self.name, self.name,
self.catalog_number, self.catalog_number,
) )
def __repr__(self): def __repr__(self):
return six.text_type(self).encode('utf-8') return str(self).encode('utf-8')
def __init__(self, data): def __init__(self, data):
BeatportObject.__init__(self, data) BeatportObject.__init__(self, data)
@ -224,26 +222,26 @@ class BeatportRelease(BeatportObject):
if 'category' in data: if 'category' in data:
self.category = data['category'] self.category = data['category']
if 'slug' in data: if 'slug' in data:
self.url = "http://beatport.com/release/{0}/{1}".format( self.url = "https://beatport.com/release/{}/{}".format(
data['slug'], data['id']) data['slug'], data['id'])
self.genre = data.get('genre')
@six.python_2_unicode_compatible
class BeatportTrack(BeatportObject): class BeatportTrack(BeatportObject):
def __str__(self): def __str__(self):
artist_str = ", ".join(x[1] for x in self.artists) artist_str = ", ".join(x[1] for x in self.artists)
return (u"<BeatportTrack: {0} - {1} ({2})>" return ("<BeatportTrack: {} - {} ({})>"
.format(artist_str, self.name, self.mix_name)) .format(artist_str, self.name, self.mix_name))
def __repr__(self): def __repr__(self):
return six.text_type(self).encode('utf-8') return str(self).encode('utf-8')
def __init__(self, data): def __init__(self, data):
BeatportObject.__init__(self, data) BeatportObject.__init__(self, data)
if 'title' in data: if 'title' in data:
self.title = six.text_type(data['title']) self.title = str(data['title'])
if 'mixName' in data: if 'mixName' in data:
self.mix_name = six.text_type(data['mixName']) self.mix_name = str(data['mixName'])
self.length = timedelta(milliseconds=data.get('lengthMs', 0) or 0) self.length = timedelta(milliseconds=data.get('lengthMs', 0) or 0)
if not self.length: if not self.length:
try: try:
@ -252,14 +250,26 @@ class BeatportTrack(BeatportObject):
except ValueError: except ValueError:
pass pass
if 'slug' in data: if 'slug' in data:
self.url = "http://beatport.com/track/{0}/{1}".format(data['slug'], self.url = "https://beatport.com/track/{}/{}" \
data['id']) .format(data['slug'], data['id'])
self.track_number = data.get('trackNumber') self.track_number = data.get('trackNumber')
self.bpm = data.get('bpm')
self.initial_key = str(
(data.get('key') or {}).get('shortName')
)
# Use 'subgenre' and if not present, 'genre' as a fallback.
if data.get('subGenres'):
self.genre = str(data['subGenres'][0].get('name'))
elif data.get('genres'):
self.genre = str(data['genres'][0].get('name'))
class BeatportPlugin(BeetsPlugin): class BeatportPlugin(BeetsPlugin):
data_source = 'Beatport'
def __init__(self): def __init__(self):
super(BeatportPlugin, self).__init__() super().__init__()
self.config.add({ self.config.add({
'apikey': '57713c3906af6f5def151b33601389176b37b429', 'apikey': '57713c3906af6f5def151b33601389176b37b429',
'apisecret': 'b3fe08c93c80aefd749fe871a16cd2bb32e2b954', 'apisecret': 'b3fe08c93c80aefd749fe871a16cd2bb32e2b954',
@ -279,7 +289,7 @@ class BeatportPlugin(BeetsPlugin):
try: try:
with open(self._tokenfile()) as f: with open(self._tokenfile()) as f:
tokendata = json.load(f) tokendata = json.load(f)
except IOError: except OSError:
# No token yet. Generate one. # No token yet. Generate one.
token, secret = self.authenticate(c_key, c_secret) token, secret = self.authenticate(c_key, c_secret)
else: else:
@ -294,22 +304,22 @@ class BeatportPlugin(BeetsPlugin):
try: try:
url = auth_client.get_authorize_url() url = auth_client.get_authorize_url()
except AUTH_ERRORS as e: except AUTH_ERRORS as e:
self._log.debug(u'authentication error: {0}', e) self._log.debug('authentication error: {0}', e)
raise beets.ui.UserError(u'communication with Beatport failed') raise beets.ui.UserError('communication with Beatport failed')
beets.ui.print_(u"To authenticate with Beatport, visit:") beets.ui.print_("To authenticate with Beatport, visit:")
beets.ui.print_(url) beets.ui.print_(url)
# Ask for the verifier data and validate it. # Ask for the verifier data and validate it.
data = beets.ui.input_(u"Enter the string displayed in your browser:") data = beets.ui.input_("Enter the string displayed in your browser:")
try: try:
token, secret = auth_client.get_access_token(data) token, secret = auth_client.get_access_token(data)
except AUTH_ERRORS as e: except AUTH_ERRORS as e:
self._log.debug(u'authentication error: {0}', e) self._log.debug('authentication error: {0}', e)
raise beets.ui.UserError(u'Beatport token request failed') raise beets.ui.UserError('Beatport token request failed')
# Save the token for later use. # Save the token for later use.
self._log.debug(u'Beatport token {0}, secret {1}', token, secret) self._log.debug('Beatport token {0}, secret {1}', token, secret)
with open(self._tokenfile(), 'w') as f: with open(self._tokenfile(), 'w') as f:
json.dump({'token': token, 'secret': secret}, f) json.dump({'token': token, 'secret': secret}, f)
@ -318,74 +328,80 @@ class BeatportPlugin(BeetsPlugin):
def _tokenfile(self): def _tokenfile(self):
"""Get the path to the JSON file for storing the OAuth token. """Get the path to the JSON file for storing the OAuth token.
""" """
return self.config['tokenfile'].get(confit.Filename(in_app_dir=True)) return self.config['tokenfile'].get(confuse.Filename(in_app_dir=True))
def album_distance(self, items, album_info, mapping): def album_distance(self, items, album_info, mapping):
"""Returns the beatport source weight and the maximum source weight """Returns the Beatport source weight and the maximum source weight
for albums. for albums.
""" """
dist = Distance() return get_distance(
if album_info.data_source == 'Beatport': data_source=self.data_source,
dist.add('source', self.config['source_weight'].as_number()) info=album_info,
return dist config=self.config
)
def track_distance(self, item, track_info): def track_distance(self, item, track_info):
"""Returns the beatport source weight and the maximum source weight """Returns the Beatport source weight and the maximum source weight
for individual tracks. for individual tracks.
""" """
dist = Distance() return get_distance(
if track_info.data_source == 'Beatport': data_source=self.data_source,
dist.add('source', self.config['source_weight'].as_number()) info=track_info,
return dist config=self.config
)
def candidates(self, items, artist, release, va_likely): def candidates(self, items, artist, release, va_likely, extra_tags=None):
"""Returns a list of AlbumInfo objects for beatport search results """Returns a list of AlbumInfo objects for beatport search results
matching release and artist (if not various). matching release and artist (if not various).
""" """
if va_likely: if va_likely:
query = release query = release
else: else:
query = '%s %s' % (artist, release) query = f'{artist} {release}'
try: try:
return self._get_releases(query) return self._get_releases(query)
except BeatportAPIError as e: except BeatportAPIError as e:
self._log.debug(u'API Error: {0} (query: {1})', e, query) self._log.debug('API Error: {0} (query: {1})', e, query)
return [] return []
def item_candidates(self, item, artist, title): def item_candidates(self, item, artist, title):
"""Returns a list of TrackInfo objects for beatport search results """Returns a list of TrackInfo objects for beatport search results
matching title and artist. matching title and artist.
""" """
query = '%s %s' % (artist, title) query = f'{artist} {title}'
try: try:
return self._get_tracks(query) return self._get_tracks(query)
except BeatportAPIError as e: except BeatportAPIError as e:
self._log.debug(u'API Error: {0} (query: {1})', e, query) self._log.debug('API Error: {0} (query: {1})', e, query)
return [] return []
def album_for_id(self, release_id): def album_for_id(self, release_id):
"""Fetches a release by its Beatport ID and returns an AlbumInfo object """Fetches a release by its Beatport ID and returns an AlbumInfo object
or None if the release is not found. or None if the query is not a valid ID or release is not found.
""" """
self._log.debug(u'Searching for release {0}', release_id) self._log.debug('Searching for release {0}', release_id)
match = re.search(r'(^|beatport\.com/release/.+/)(\d+)$', release_id) match = re.search(r'(^|beatport\.com/release/.+/)(\d+)$', release_id)
if not match: if not match:
self._log.debug('Not a valid Beatport release ID.')
return None return None
release = self.client.get_release(match.group(2)) release = self.client.get_release(match.group(2))
album = self._get_album_info(release) if release:
return album return self._get_album_info(release)
return None
def track_for_id(self, track_id): def track_for_id(self, track_id):
"""Fetches a track by its Beatport ID and returns a TrackInfo object """Fetches a track by its Beatport ID and returns a TrackInfo object
or None if the track is not found. or None if the track is not a valid Beatport ID or track is not found.
""" """
self._log.debug(u'Searching for track {0}', track_id) self._log.debug('Searching for track {0}', track_id)
match = re.search(r'(^|beatport\.com/track/.+/)(\d+)$', track_id) match = re.search(r'(^|beatport\.com/track/.+/)(\d+)$', track_id)
if not match: if not match:
self._log.debug('Not a valid Beatport track ID.')
return None return None
bp_track = self.client.get_track(match.group(2)) bp_track = self.client.get_track(match.group(2))
track = self._get_track_info(bp_track) if bp_track is not None:
return track return self._get_track_info(bp_track)
return None
def _get_releases(self, query): def _get_releases(self, query):
"""Returns a list of AlbumInfo objects for a beatport search query. """Returns a list of AlbumInfo objects for a beatport search query.
@ -408,7 +424,7 @@ class BeatportPlugin(BeetsPlugin):
va = len(release.artists) > 3 va = len(release.artists) > 3
artist, artist_id = self._get_artist(release.artists) artist, artist_id = self._get_artist(release.artists)
if va: if va:
artist = u"Various Artists" artist = "Various Artists"
tracks = [self._get_track_info(x) for x in release.tracks] tracks = [self._get_track_info(x) for x in release.tracks]
return AlbumInfo(album=release.name, album_id=release.beatport_id, return AlbumInfo(album=release.name, album_id=release.beatport_id,
@ -418,40 +434,33 @@ class BeatportPlugin(BeetsPlugin):
month=release.release_date.month, month=release.release_date.month,
day=release.release_date.day, day=release.release_date.day,
label=release.label_name, label=release.label_name,
catalognum=release.catalog_number, media=u'Digital', catalognum=release.catalog_number, media='Digital',
data_source=u'Beatport', data_url=release.url) data_source=self.data_source, data_url=release.url,
genre=release.genre)
def _get_track_info(self, track): def _get_track_info(self, track):
"""Returns a TrackInfo object for a Beatport Track object. """Returns a TrackInfo object for a Beatport Track object.
""" """
title = track.name title = track.name
if track.mix_name != u"Original Mix": if track.mix_name != "Original Mix":
title += u" ({0})".format(track.mix_name) title += f" ({track.mix_name})"
artist, artist_id = self._get_artist(track.artists) artist, artist_id = self._get_artist(track.artists)
length = track.length.total_seconds() length = track.length.total_seconds()
return TrackInfo(title=title, track_id=track.beatport_id, return TrackInfo(title=title, track_id=track.beatport_id,
artist=artist, artist_id=artist_id, artist=artist, artist_id=artist_id,
length=length, index=track.track_number, length=length, index=track.track_number,
medium_index=track.track_number, medium_index=track.track_number,
data_source=u'Beatport', data_url=track.url) data_source=self.data_source, data_url=track.url,
bpm=track.bpm, initial_key=track.initial_key,
genre=track.genre)
def _get_artist(self, artists): def _get_artist(self, artists):
"""Returns an artist string (all artists) and an artist_id (the main """Returns an artist string (all artists) and an artist_id (the main
artist) for a list of Beatport release or track artists. artist) for a list of Beatport release or track artists.
""" """
artist_id = None return MetadataSourcePlugin.get_artist(
bits = [] artists=artists, id_key=0, name_key=1
for artist in artists: )
if not artist_id:
artist_id = artist[0]
name = artist[1]
# Strip disambiguation number.
name = re.sub(r' \(\d+\)$', '', name)
# Move articles to the front.
name = re.sub(r'^(.*?), (a|an|the)$', r'\2 \1', name, flags=re.I)
bits.append(name)
artist = ', '.join(bits).replace(' ,', ',') or None
return artist, artist_id
def _get_tracks(self, query): def _get_tracks(self, query):
"""Returns a list of TrackInfo objects for a Beatport query. """Returns a list of TrackInfo objects for a Beatport query.

2
libs/common/beetsplug/bench.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson. # Copyright 2016, Adrian Sampson.
# #
@ -16,7 +15,6 @@
"""Some simple performance benchmarks for beets. """Some simple performance benchmarks for beets.
""" """
from __future__ import division, absolute_import, print_function
from beets.plugins import BeetsPlugin from beets.plugins import BeetsPlugin
from beets import ui from beets import ui

833
libs/common/beetsplug/bpd/__init__.py
View file

File diff suppressed because it is too large Load diff

73
libs/common/beetsplug/bpd/gstplayer.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson. # Copyright 2016, Adrian Sampson.
# #
@ -17,15 +16,13 @@
music player. music player.
""" """
from __future__ import division, absolute_import, print_function
import six
import sys import sys
import time import time
from six.moves import _thread import _thread
import os import os
import copy import copy
from six.moves import urllib import urllib
from beets import ui from beets import ui
import gi import gi
@ -40,7 +37,7 @@ class QueryError(Exception):
pass pass
class GstPlayer(object): class GstPlayer:
"""A music player abstracting GStreamer's Playbin element. """A music player abstracting GStreamer's Playbin element.
Create a player object, then call run() to start a thread with a Create a player object, then call run() to start a thread with a
@ -64,7 +61,8 @@ class GstPlayer(object):
""" """
# Set up the Gstreamer player. From the pygst tutorial: # Set up the Gstreamer player. From the pygst tutorial:
# http://pygstdocs.berlios.de/pygst-tutorial/playbin.html # https://pygstdocs.berlios.de/pygst-tutorial/playbin.html (gone)
# https://brettviren.github.io/pygst-tutorial-org/pygst-tutorial.html
#### ####
# Updated to GStreamer 1.0 with: # Updated to GStreamer 1.0 with:
# https://wiki.ubuntu.com/Novacut/GStreamer1.0 # https://wiki.ubuntu.com/Novacut/GStreamer1.0
@ -109,7 +107,7 @@ class GstPlayer(object):
# error # error
self.player.set_state(Gst.State.NULL) self.player.set_state(Gst.State.NULL)
err, debug = message.parse_error() err, debug = message.parse_error()
print(u"Error: {0}".format(err)) print(f"Error: {err}")
self.playing = False self.playing = False
def _set_volume(self, volume): def _set_volume(self, volume):
@ -129,7 +127,7 @@ class GstPlayer(object):
path. path.
""" """
self.player.set_state(Gst.State.NULL) self.player.set_state(Gst.State.NULL)
if isinstance(path, six.text_type): if isinstance(path, str):
path = path.encode('utf-8') path = path.encode('utf-8')
uri = 'file://' + urllib.parse.quote(path) uri = 'file://' + urllib.parse.quote(path)
self.player.set_property("uri", uri) self.player.set_property("uri", uri)
@ -177,12 +175,12 @@ class GstPlayer(object):
posq = self.player.query_position(fmt) posq = self.player.query_position(fmt)
if not posq[0]: if not posq[0]:
raise QueryError("query_position failed") raise QueryError("query_position failed")
pos = posq[1] // (10 ** 9) pos = posq[1] / (10 ** 9)
lengthq = self.player.query_duration(fmt) lengthq = self.player.query_duration(fmt)
if not lengthq[0]: if not lengthq[0]:
raise QueryError("query_duration failed") raise QueryError("query_duration failed")
length = lengthq[1] // (10 ** 9) length = lengthq[1] / (10 ** 9)
self.cached_time = (pos, length) self.cached_time = (pos, length)
return (pos, length) return (pos, length)
@ -215,6 +213,59 @@ class GstPlayer(object):
while self.playing: while self.playing:
time.sleep(1) time.sleep(1)
def get_decoders(self):
return get_decoders()
def get_decoders():
"""Get supported audio decoders from GStreamer.
Returns a dict mapping decoder element names to the associated media types
and file extensions.
"""
# We only care about audio decoder elements.
filt = (Gst.ELEMENT_FACTORY_TYPE_DEPAYLOADER |
Gst.ELEMENT_FACTORY_TYPE_DEMUXER |
Gst.ELEMENT_FACTORY_TYPE_PARSER |
Gst.ELEMENT_FACTORY_TYPE_DECODER |
Gst.ELEMENT_FACTORY_TYPE_MEDIA_AUDIO)
decoders = {}
mime_types = set()
for f in Gst.ElementFactory.list_get_elements(filt, Gst.Rank.NONE):
for pad in f.get_static_pad_templates():
if pad.direction == Gst.PadDirection.SINK:
caps = pad.static_caps.get()
mimes = set()
for i in range(caps.get_size()):
struct = caps.get_structure(i)
mime = struct.get_name()
if mime == 'unknown/unknown':
continue
mimes.add(mime)
mime_types.add(mime)
if mimes:
decoders[f.get_name()] = (mimes, set())
# Check all the TypeFindFactory plugin features form the registry. If they
# are associated with an audio media type that we found above, get the list
# of corresponding file extensions.
mime_extensions = {mime: set() for mime in mime_types}
for feat in Gst.Registry.get().get_feature_list(Gst.TypeFindFactory):
caps = feat.get_caps()
if caps:
for i in range(caps.get_size()):
struct = caps.get_structure(i)
mime = struct.get_name()
if mime in mime_types:
mime_extensions[mime].update(feat.get_extensions())
# Fill in the slot we left for file extensions.
for name, (mimes, exts) in decoders.items():
for mime in mimes:
exts.update(mime_extensions[mime])
return decoders
def play_simple(paths): def play_simple(paths):
"""Play the files in paths in a straightforward way, without """Play the files in paths in a straightforward way, without

23
libs/common/beetsplug/bpm.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, aroquen # Copyright 2016, aroquen
# #
@ -15,10 +14,8 @@
"""Determine BPM by pressing a key to the rhythm.""" """Determine BPM by pressing a key to the rhythm."""
from __future__ import division, absolute_import, print_function
import time import time
from six.moves import input
from beets import ui from beets import ui
from beets.plugins import BeetsPlugin from beets.plugins import BeetsPlugin
@ -51,16 +48,16 @@ def bpm(max_strokes):
class BPMPlugin(BeetsPlugin): class BPMPlugin(BeetsPlugin):
def __init__(self): def __init__(self):
super(BPMPlugin, self).__init__() super().__init__()
self.config.add({ self.config.add({
u'max_strokes': 3, 'max_strokes': 3,
u'overwrite': True, 'overwrite': True,
}) })
def commands(self): def commands(self):
cmd = ui.Subcommand('bpm', cmd = ui.Subcommand('bpm',
help=u'determine bpm of a song by pressing ' help='determine bpm of a song by pressing '
u'a key to the rhythm') 'a key to the rhythm')
cmd.func = self.command cmd.func = self.command
return [cmd] return [cmd]
@ -72,19 +69,19 @@ class BPMPlugin(BeetsPlugin):
def get_bpm(self, items, write=False): def get_bpm(self, items, write=False):
overwrite = self.config['overwrite'].get(bool) overwrite = self.config['overwrite'].get(bool)
if len(items) > 1: if len(items) > 1:
raise ValueError(u'Can only get bpm of one song at time') raise ValueError('Can only get bpm of one song at time')
item = items[0] item = items[0]
if item['bpm']: if item['bpm']:
self._log.info(u'Found bpm {0}', item['bpm']) self._log.info('Found bpm {0}', item['bpm'])
if not overwrite: if not overwrite:
return return
self._log.info(u'Press Enter {0} times to the rhythm or Ctrl-D ' self._log.info('Press Enter {0} times to the rhythm or Ctrl-D '
u'to exit', self.config['max_strokes'].get(int)) 'to exit', self.config['max_strokes'].get(int))
new_bpm = bpm(self.config['max_strokes'].get(int)) new_bpm = bpm(self.config['max_strokes'].get(int))
item['bpm'] = int(new_bpm) item['bpm'] = int(new_bpm)
if write: if write:
item.try_write() item.try_write()
item.store() item.store()
self._log.info(u'Added new bpm {0}', item['bpm']) self._log.info('Added new bpm {0}', item['bpm'])

186
libs/common/beetsplug/bpsync.py Normal file
View file

@ -0,0 +1,186 @@
# This file is part of beets.
# Copyright 2019, Rahul Ahuja.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Update library's tags using Beatport.
"""
from beets.plugins import BeetsPlugin, apply_item_changes
from beets import autotag, library, ui, util
from .beatport import BeatportPlugin
class BPSyncPlugin(BeetsPlugin):
def __init__(self):
super().__init__()
self.beatport_plugin = BeatportPlugin()
self.beatport_plugin.setup()
def commands(self):
cmd = ui.Subcommand('bpsync', help='update metadata from Beatport')
cmd.parser.add_option(
'-p',
'--pretend',
action='store_true',
help='show all changes but do nothing',
)
cmd.parser.add_option(
'-m',
'--move',
action='store_true',
dest='move',
help="move files in the library directory",
)
cmd.parser.add_option(
'-M',
'--nomove',
action='store_false',
dest='move',
help="don't move files in library",
)
cmd.parser.add_option(
'-W',
'--nowrite',
action='store_false',
default=None,
dest='write',
help="don't write updated metadata to files",
)
cmd.parser.add_format_option()
cmd.func = self.func
return [cmd]
def func(self, lib, opts, args):
"""Command handler for the bpsync function.
"""
move = ui.should_move(opts.move)
pretend = opts.pretend
write = ui.should_write(opts.write)
query = ui.decargs(args)
self.singletons(lib, query, move, pretend, write)
self.albums(lib, query, move, pretend, write)
def singletons(self, lib, query, move, pretend, write):
"""Retrieve and apply info from the autotagger for items matched by
query.
"""
for item in lib.items(query + ['singleton:true']):
if not item.mb_trackid:
self._log.info(
'Skipping singleton with no mb_trackid: {}', item
)
continue
if not self.is_beatport_track(item):
self._log.info(
'Skipping non-{} singleton: {}',
self.beatport_plugin.data_source,
item,
)
continue
# Apply.
trackinfo = self.beatport_plugin.track_for_id(item.mb_trackid)
with lib.transaction():
autotag.apply_item_metadata(item, trackinfo)
apply_item_changes(lib, item, move, pretend, write)
@staticmethod
def is_beatport_track(item):
return (
item.get('data_source') == BeatportPlugin.data_source
and item.mb_trackid.isnumeric()
)
def get_album_tracks(self, album):
if not album.mb_albumid:
self._log.info('Skipping album with no mb_albumid: {}', album)
return False
if not album.mb_albumid.isnumeric():
self._log.info(
'Skipping album with invalid {} ID: {}',
self.beatport_plugin.data_source,
album,
)
return False
items = list(album.items())
if album.get('data_source') == self.beatport_plugin.data_source:
return items
if not all(self.is_beatport_track(item) for item in items):
self._log.info(
'Skipping non-{} release: {}',
self.beatport_plugin.data_source,
album,
)
return False
return items
def albums(self, lib, query, move, pretend, write):
"""Retrieve and apply info from the autotagger for albums matched by
query and their items.
"""
# Process matching albums.
for album in lib.albums(query):
# Do we have a valid Beatport album?
items = self.get_album_tracks(album)
if not items:
continue
# Get the Beatport album information.
albuminfo = self.beatport_plugin.album_for_id(album.mb_albumid)
if not albuminfo:
self._log.info(
'Release ID {} not found for album {}',
album.mb_albumid,
album,
)
continue
beatport_trackid_to_trackinfo = {
track.track_id: track for track in albuminfo.tracks
}
library_trackid_to_item = {
int(item.mb_trackid): item for item in items
}
item_to_trackinfo = {
item: beatport_trackid_to_trackinfo[track_id]
for track_id, item in library_trackid_to_item.items()
}
self._log.info('applying changes to {}', album)
with lib.transaction():
autotag.apply_metadata(albuminfo, item_to_trackinfo)
changed = False
# Find any changed item to apply Beatport changes to album.
any_changed_item = items[0]
for item in items:
item_changed = ui.show_model_changes(item)
changed |= item_changed
if item_changed:
any_changed_item = item
apply_item_changes(lib, item, move, pretend, write)
if pretend or not changed:
continue
# Update album structure to reflect an item in it.
for key in library.Album.item_keys:
album[key] = any_changed_item[key]
album.store()
# Move album art (and any inconsistent items).
if move and lib.directory in util.ancestry(items[0].path):
self._log.debug('moving album {}', album)
album.move()

27
libs/common/beetsplug/bucket.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Fabrice Laporte. # Copyright 2016, Fabrice Laporte.
# #
@ -16,12 +15,10 @@
"""Provides the %bucket{} function for path formatting. """Provides the %bucket{} function for path formatting.
""" """
from __future__ import division, absolute_import, print_function
from datetime import datetime from datetime import datetime
import re import re
import string import string
from six.moves import zip
from itertools import tee from itertools import tee
from beets import plugins, ui from beets import plugins, ui
@ -49,7 +46,7 @@ def span_from_str(span_str):
"""Convert string to a 4 digits year """Convert string to a 4 digits year
""" """
if yearfrom < 100: if yearfrom < 100:
raise BucketError(u"%d must be expressed on 4 digits" % yearfrom) raise BucketError("%d must be expressed on 4 digits" % yearfrom)
# if two digits only, pick closest year that ends by these two # if two digits only, pick closest year that ends by these two
# digits starting from yearfrom # digits starting from yearfrom
@ -60,14 +57,14 @@ def span_from_str(span_str):
d = (yearfrom - yearfrom % 100) + d d = (yearfrom - yearfrom % 100) + d
return d return d
years = [int(x) for x in re.findall('\d+', span_str)] years = [int(x) for x in re.findall(r'\d+', span_str)]
if not years: if not years:
raise ui.UserError(u"invalid range defined for year bucket '%s': no " raise ui.UserError("invalid range defined for year bucket '%s': no "
u"year found" % span_str) "year found" % span_str)
try: try:
years = [normalize_year(x, years[0]) for x in years] years = [normalize_year(x, years[0]) for x in years]
except BucketError as exc: except BucketError as exc:
raise ui.UserError(u"invalid range defined for year bucket '%s': %s" % raise ui.UserError("invalid range defined for year bucket '%s': %s" %
(span_str, exc)) (span_str, exc))
res = {'from': years[0], 'str': span_str} res = {'from': years[0], 'str': span_str}
@ -128,10 +125,10 @@ def str2fmt(s):
res = {'fromnchars': len(m.group('fromyear')), res = {'fromnchars': len(m.group('fromyear')),
'tonchars': len(m.group('toyear'))} 'tonchars': len(m.group('toyear'))}
res['fmt'] = "%s%%s%s%s%s" % (m.group('bef'), res['fmt'] = "{}%s{}{}{}".format(m.group('bef'),
m.group('sep'), m.group('sep'),
'%s' if res['tonchars'] else '', '%s' if res['tonchars'] else '',
m.group('after')) m.group('after'))
return res return res
@ -170,8 +167,8 @@ def build_alpha_spans(alpha_spans_str, alpha_regexs):
begin_index = ASCII_DIGITS.index(bucket[0]) begin_index = ASCII_DIGITS.index(bucket[0])
end_index = ASCII_DIGITS.index(bucket[-1]) end_index = ASCII_DIGITS.index(bucket[-1])
else: else:
raise ui.UserError(u"invalid range defined for alpha bucket " raise ui.UserError("invalid range defined for alpha bucket "
u"'%s': no alphanumeric character found" % "'%s': no alphanumeric character found" %
elem) elem)
spans.append( spans.append(
re.compile( re.compile(
@ -184,7 +181,7 @@ def build_alpha_spans(alpha_spans_str, alpha_regexs):
class BucketPlugin(plugins.BeetsPlugin): class BucketPlugin(plugins.BeetsPlugin):
def __init__(self): def __init__(self):
super(BucketPlugin, self).__init__() super().__init__()
self.template_funcs['bucket'] = self._tmpl_bucket self.template_funcs['bucket'] = self._tmpl_bucket
self.config.add({ self.config.add({

105
libs/common/beetsplug/chroma.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson. # Copyright 2016, Adrian Sampson.
# #
@ -16,16 +15,17 @@
"""Adds Chromaprint/Acoustid acoustic fingerprinting support to the """Adds Chromaprint/Acoustid acoustic fingerprinting support to the
autotagger. Requires the pyacoustid library. autotagger. Requires the pyacoustid library.
""" """
from __future__ import division, absolute_import, print_function
from beets import plugins from beets import plugins
from beets import ui from beets import ui
from beets import util from beets import util
from beets import config from beets import config
from beets.util import confit
from beets.autotag import hooks from beets.autotag import hooks
import confuse
import acoustid import acoustid
from collections import defaultdict from collections import defaultdict
from functools import partial
import re
API_KEY = '1vOwZtEn' API_KEY = '1vOwZtEn'
SCORE_THRESH = 0.5 SCORE_THRESH = 0.5
@ -57,6 +57,30 @@ def prefix(it, count):
yield v yield v
def releases_key(release, countries, original_year):
"""Used as a key to sort releases by date then preferred country
"""
date = release.get('date')
if date and original_year:
year = date.get('year', 9999)
month = date.get('month', 99)
day = date.get('day', 99)
else:
year = 9999
month = 99
day = 99
# Uses index of preferred countries to sort
country_key = 99
if release.get('country'):
for i, country in enumerate(countries):
if country.match(release['country']):
country_key = i
break
return (year, month, day, country_key)
def acoustid_match(log, path): def acoustid_match(log, path):
"""Gets metadata for a file from Acoustid and populates the """Gets metadata for a file from Acoustid and populates the
_matches, _fingerprints, and _acoustids dictionaries accordingly. _matches, _fingerprints, and _acoustids dictionaries accordingly.
@ -64,42 +88,55 @@ def acoustid_match(log, path):
try: try:
duration, fp = acoustid.fingerprint_file(util.syspath(path)) duration, fp = acoustid.fingerprint_file(util.syspath(path))
except acoustid.FingerprintGenerationError as exc: except acoustid.FingerprintGenerationError as exc:
log.error(u'fingerprinting of {0} failed: {1}', log.error('fingerprinting of {0} failed: {1}',
util.displayable_path(repr(path)), exc) util.displayable_path(repr(path)), exc)
return None return None
fp = fp.decode()
_fingerprints[path] = fp _fingerprints[path] = fp
try: try:
res = acoustid.lookup(API_KEY, fp, duration, res = acoustid.lookup(API_KEY, fp, duration,
meta='recordings releases') meta='recordings releases')
except acoustid.AcoustidError as exc: except acoustid.AcoustidError as exc:
log.debug(u'fingerprint matching {0} failed: {1}', log.debug('fingerprint matching {0} failed: {1}',
util.displayable_path(repr(path)), exc) util.displayable_path(repr(path)), exc)
return None return None
log.debug(u'chroma: fingerprinted {0}', log.debug('chroma: fingerprinted {0}',
util.displayable_path(repr(path))) util.displayable_path(repr(path)))
# Ensure the response is usable and parse it. # Ensure the response is usable and parse it.
if res['status'] != 'ok' or not res.get('results'): if res['status'] != 'ok' or not res.get('results'):
log.debug(u'no match found') log.debug('no match found')
return None return None
result = res['results'][0] # Best match. result = res['results'][0] # Best match.
if result['score'] < SCORE_THRESH: if result['score'] < SCORE_THRESH:
log.debug(u'no results above threshold') log.debug('no results above threshold')
return None return None
_acoustids[path] = result['id'] _acoustids[path] = result['id']
# Get recording and releases from the result. # Get recording and releases from the result
if not result.get('recordings'): if not result.get('recordings'):
log.debug(u'no recordings found') log.debug('no recordings found')
return None return None
recording_ids = [] recording_ids = []
release_ids = [] releases = []
for recording in result['recordings']: for recording in result['recordings']:
recording_ids.append(recording['id']) recording_ids.append(recording['id'])
if 'releases' in recording: if 'releases' in recording:
release_ids += [rel['id'] for rel in recording['releases']] releases.extend(recording['releases'])
log.debug(u'matched recordings {0} on releases {1}', # The releases list is essentially in random order from the Acoustid lookup
# so we optionally sort it using the match.preferred configuration options.
# 'original_year' to sort the earliest first and
# 'countries' to then sort preferred countries first.
country_patterns = config['match']['preferred']['countries'].as_str_seq()
countries = [re.compile(pat, re.I) for pat in country_patterns]
original_year = config['match']['preferred']['original_year']
releases.sort(key=partial(releases_key,
countries=countries,
original_year=original_year))
release_ids = [rel['id'] for rel in releases]
log.debug('matched recordings {0} on releases {1}',
recording_ids, release_ids) recording_ids, release_ids)
_matches[path] = recording_ids, release_ids _matches[path] = recording_ids, release_ids
@ -128,7 +165,7 @@ def _all_releases(items):
class AcoustidPlugin(plugins.BeetsPlugin): class AcoustidPlugin(plugins.BeetsPlugin):
def __init__(self): def __init__(self):
super(AcoustidPlugin, self).__init__() super().__init__()
self.config.add({ self.config.add({
'auto': True, 'auto': True,
@ -152,14 +189,14 @@ class AcoustidPlugin(plugins.BeetsPlugin):
dist.add_expr('track_id', info.track_id not in recording_ids) dist.add_expr('track_id', info.track_id not in recording_ids)
return dist return dist
def candidates(self, items, artist, album, va_likely): def candidates(self, items, artist, album, va_likely, extra_tags=None):
albums = [] albums = []
for relid in prefix(_all_releases(items), MAX_RELEASES): for relid in prefix(_all_releases(items), MAX_RELEASES):
album = hooks.album_for_mbid(relid) album = hooks.album_for_mbid(relid)
if album: if album:
albums.append(album) albums.append(album)
self._log.debug(u'acoustid album candidates: {0}', len(albums)) self._log.debug('acoustid album candidates: {0}', len(albums))
return albums return albums
def item_candidates(self, item, artist, title): def item_candidates(self, item, artist, title):
@ -172,24 +209,24 @@ class AcoustidPlugin(plugins.BeetsPlugin):
track = hooks.track_for_mbid(recording_id) track = hooks.track_for_mbid(recording_id)
if track: if track:
tracks.append(track) tracks.append(track)
self._log.debug(u'acoustid item candidates: {0}', len(tracks)) self._log.debug('acoustid item candidates: {0}', len(tracks))
return tracks return tracks
def commands(self): def commands(self):
submit_cmd = ui.Subcommand('submit', submit_cmd = ui.Subcommand('submit',
help=u'submit Acoustid fingerprints') help='submit Acoustid fingerprints')
def submit_cmd_func(lib, opts, args): def submit_cmd_func(lib, opts, args):
try: try:
apikey = config['acoustid']['apikey'].as_str() apikey = config['acoustid']['apikey'].as_str()
except confit.NotFoundError: except confuse.NotFoundError:
raise ui.UserError(u'no Acoustid user API key provided') raise ui.UserError('no Acoustid user API key provided')
submit_items(self._log, apikey, lib.items(ui.decargs(args))) submit_items(self._log, apikey, lib.items(ui.decargs(args)))
submit_cmd.func = submit_cmd_func submit_cmd.func = submit_cmd_func
fingerprint_cmd = ui.Subcommand( fingerprint_cmd = ui.Subcommand(
'fingerprint', 'fingerprint',
help=u'generate fingerprints for items without them' help='generate fingerprints for items without them'
) )
def fingerprint_cmd_func(lib, opts, args): def fingerprint_cmd_func(lib, opts, args):
@ -232,15 +269,15 @@ def submit_items(log, userkey, items, chunksize=64):
def submit_chunk(): def submit_chunk():
"""Submit the current accumulated fingerprint data.""" """Submit the current accumulated fingerprint data."""
log.info(u'submitting {0} fingerprints', len(data)) log.info('submitting {0} fingerprints', len(data))
try: try:
acoustid.submit(API_KEY, userkey, data) acoustid.submit(API_KEY, userkey, data)
except acoustid.AcoustidError as exc: except acoustid.AcoustidError as exc:
log.warning(u'acoustid submission error: {0}', exc) log.warning('acoustid submission error: {0}', exc)
del data[:] del data[:]
for item in items: for item in items:
fp = fingerprint_item(log, item) fp = fingerprint_item(log, item, write=ui.should_write())
# Construct a submission dictionary for this item. # Construct a submission dictionary for this item.
item_data = { item_data = {
@ -249,7 +286,7 @@ def submit_items(log, userkey, items, chunksize=64):
} }
if item.mb_trackid: if item.mb_trackid:
item_data['mbid'] = item.mb_trackid item_data['mbid'] = item.mb_trackid
log.debug(u'submitting MBID') log.debug('submitting MBID')
else: else:
item_data.update({ item_data.update({
'track': item.title, 'track': item.title,
@ -260,7 +297,7 @@ def submit_items(log, userkey, items, chunksize=64):
'trackno': item.track, 'trackno': item.track,
'discno': item.disc, 'discno': item.disc,
}) })
log.debug(u'submitting textual metadata') log.debug('submitting textual metadata')
data.append(item_data) data.append(item_data)
# If we have enough data, submit a chunk. # If we have enough data, submit a chunk.
@ -281,28 +318,28 @@ def fingerprint_item(log, item, write=False):
""" """
# Get a fingerprint and length for this track. # Get a fingerprint and length for this track.
if not item.length: if not item.length:
log.info(u'{0}: no duration available', log.info('{0}: no duration available',
util.displayable_path(item.path)) util.displayable_path(item.path))
elif item.acoustid_fingerprint: elif item.acoustid_fingerprint:
if write: if write:
log.info(u'{0}: fingerprint exists, skipping', log.info('{0}: fingerprint exists, skipping',
util.displayable_path(item.path)) util.displayable_path(item.path))
else: else:
log.info(u'{0}: using existing fingerprint', log.info('{0}: using existing fingerprint',
util.displayable_path(item.path)) util.displayable_path(item.path))
return item.acoustid_fingerprint return item.acoustid_fingerprint
else: else:
log.info(u'{0}: fingerprinting', log.info('{0}: fingerprinting',
util.displayable_path(item.path)) util.displayable_path(item.path))
try: try:
_, fp = acoustid.fingerprint_file(util.syspath(item.path)) _, fp = acoustid.fingerprint_file(util.syspath(item.path))
item.acoustid_fingerprint = fp item.acoustid_fingerprint = fp.decode()
if write: if write:
log.info(u'{0}: writing fingerprint', log.info('{0}: writing fingerprint',
util.displayable_path(item.path)) util.displayable_path(item.path))
item.try_write() item.try_write()
if item._db: if item._db:
item.store() item.store()
return item.acoustid_fingerprint return item.acoustid_fingerprint
except acoustid.FingerprintGenerationError as exc: except acoustid.FingerprintGenerationError as exc:
log.info(u'fingerprint generation failed: {0}', exc) log.info('fingerprint generation failed: {0}', exc)

254
libs/common/beetsplug/convert.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Jakob Schnitzer. # Copyright 2016, Jakob Schnitzer.
# #
@ -15,20 +14,18 @@
"""Converts tracks or albums to external directory """Converts tracks or albums to external directory
""" """
from __future__ import division, absolute_import, print_function from beets.util import par_map, decode_commandline_path, arg_encoding
import os import os
import threading import threading
import subprocess import subprocess
import tempfile import tempfile
import shlex import shlex
import six
from string import Template from string import Template
import platform
from beets import ui, util, plugins, config from beets import ui, util, plugins, config
from beets.plugins import BeetsPlugin from beets.plugins import BeetsPlugin
from beets.util.confit import ConfigTypeError from confuse import ConfigTypeError
from beets import art from beets import art
from beets.util.artresizer import ArtResizer from beets.util.artresizer import ArtResizer
from beets.library import parse_query_string from beets.library import parse_query_string
@ -39,8 +36,8 @@ _temp_files = [] # Keep track of temporary transcoded files for deletion.
# Some convenient alternate names for formats. # Some convenient alternate names for formats.
ALIASES = { ALIASES = {
u'wma': u'windows media', 'wma': 'windows media',
u'vorbis': u'ogg', 'vorbis': 'ogg',
} }
LOSSLESS_FORMATS = ['ape', 'flac', 'alac', 'wav', 'aiff'] LOSSLESS_FORMATS = ['ape', 'flac', 'alac', 'wav', 'aiff']
@ -68,7 +65,7 @@ def get_format(fmt=None):
extension = format_info.get('extension', fmt) extension = format_info.get('extension', fmt)
except KeyError: except KeyError:
raise ui.UserError( raise ui.UserError(
u'convert: format {0} needs the "command" field' 'convert: format {} needs the "command" field'
.format(fmt) .format(fmt)
) )
except ConfigTypeError: except ConfigTypeError:
@ -81,7 +78,7 @@ def get_format(fmt=None):
command = config['convert']['command'].as_str() command = config['convert']['command'].as_str()
elif 'opts' in keys: elif 'opts' in keys:
# Undocumented option for backwards compatibility with < 1.3.1. # Undocumented option for backwards compatibility with < 1.3.1.
command = u'ffmpeg -i $source -y {0} $dest'.format( command = 'ffmpeg -i $source -y {} $dest'.format(
config['convert']['opts'].as_str() config['convert']['opts'].as_str()
) )
if 'extension' in keys: if 'extension' in keys:
@ -110,70 +107,81 @@ def should_transcode(item, fmt):
class ConvertPlugin(BeetsPlugin): class ConvertPlugin(BeetsPlugin):
def __init__(self): def __init__(self):
super(ConvertPlugin, self).__init__() super().__init__()
self.config.add({ self.config.add({
u'dest': None, 'dest': None,
u'pretend': False, 'pretend': False,
u'threads': util.cpu_count(), 'link': False,
u'format': u'mp3', 'hardlink': False,
u'formats': { 'threads': util.cpu_count(),
u'aac': { 'format': 'mp3',
u'command': u'ffmpeg -i $source -y -vn -acodec aac ' 'id3v23': 'inherit',
u'-aq 1 $dest', 'formats': {
u'extension': u'm4a', 'aac': {
'command': 'ffmpeg -i $source -y -vn -acodec aac '
'-aq 1 $dest',
'extension': 'm4a',
}, },
u'alac': { 'alac': {
u'command': u'ffmpeg -i $source -y -vn -acodec alac $dest', 'command': 'ffmpeg -i $source -y -vn -acodec alac $dest',
u'extension': u'm4a', 'extension': 'm4a',
}, },
u'flac': u'ffmpeg -i $source -y -vn -acodec flac $dest', 'flac': 'ffmpeg -i $source -y -vn -acodec flac $dest',
u'mp3': u'ffmpeg -i $source -y -vn -aq 2 $dest', 'mp3': 'ffmpeg -i $source -y -vn -aq 2 $dest',
u'opus': 'opus':
u'ffmpeg -i $source -y -vn -acodec libopus -ab 96k $dest', 'ffmpeg -i $source -y -vn -acodec libopus -ab 96k $dest',
u'ogg': 'ogg':
u'ffmpeg -i $source -y -vn -acodec libvorbis -aq 3 $dest', 'ffmpeg -i $source -y -vn -acodec libvorbis -aq 3 $dest',
u'wma': 'wma':
u'ffmpeg -i $source -y -vn -acodec wmav2 -vn $dest', 'ffmpeg -i $source -y -vn -acodec wmav2 -vn $dest',
}, },
u'max_bitrate': 500, 'max_bitrate': 500,
u'auto': False, 'auto': False,
u'tmpdir': None, 'tmpdir': None,
u'quiet': False, 'quiet': False,
u'embed': True, 'embed': True,
u'paths': {}, 'paths': {},
u'no_convert': u'', 'no_convert': '',
u'never_convert_lossy_files': False, 'never_convert_lossy_files': False,
u'copy_album_art': False, 'copy_album_art': False,
u'album_art_maxwidth': 0, 'album_art_maxwidth': 0,
'delete_originals': False,
}) })
self.early_import_stages = [self.auto_convert] self.early_import_stages = [self.auto_convert]
self.register_listener('import_task_files', self._cleanup) self.register_listener('import_task_files', self._cleanup)
def commands(self): def commands(self):
cmd = ui.Subcommand('convert', help=u'convert to external location') cmd = ui.Subcommand('convert', help='convert to external location')
cmd.parser.add_option('-p', '--pretend', action='store_true', cmd.parser.add_option('-p', '--pretend', action='store_true',
help=u'show actions but do nothing') help='show actions but do nothing')
cmd.parser.add_option('-t', '--threads', action='store', type='int', cmd.parser.add_option('-t', '--threads', action='store', type='int',
help=u'change the number of threads, \ help='change the number of threads, \
defaults to maximum available processors') defaults to maximum available processors')
cmd.parser.add_option('-k', '--keep-new', action='store_true', cmd.parser.add_option('-k', '--keep-new', action='store_true',
dest='keep_new', help=u'keep only the converted \ dest='keep_new', help='keep only the converted \
and move the old files') and move the old files')
cmd.parser.add_option('-d', '--dest', action='store', cmd.parser.add_option('-d', '--dest', action='store',
help=u'set the destination directory') help='set the destination directory')
cmd.parser.add_option('-f', '--format', action='store', dest='format', cmd.parser.add_option('-f', '--format', action='store', dest='format',
help=u'set the target format of the tracks') help='set the target format of the tracks')
cmd.parser.add_option('-y', '--yes', action='store_true', dest='yes', cmd.parser.add_option('-y', '--yes', action='store_true', dest='yes',
help=u'do not ask for confirmation') help='do not ask for confirmation')
cmd.parser.add_option('-l', '--link', action='store_true', dest='link',
help='symlink files that do not \
need transcoding.')
cmd.parser.add_option('-H', '--hardlink', action='store_true',
dest='hardlink',
help='hardlink files that do not \
need transcoding. Overrides --link.')
cmd.parser.add_album_option() cmd.parser.add_album_option()
cmd.func = self.convert_func cmd.func = self.convert_func
return [cmd] return [cmd]
def auto_convert(self, config, task): def auto_convert(self, config, task):
if self.config['auto']: if self.config['auto']:
for item in task.imported_items(): par_map(lambda item: self.convert_on_import(config.lib, item),
self.convert_on_import(config.lib, item) task.imported_items())
# Utilities converted from functions to methods on logging overhaul # Utilities converted from functions to methods on logging overhaul
@ -191,22 +199,11 @@ class ConvertPlugin(BeetsPlugin):
quiet = self.config['quiet'].get(bool) quiet = self.config['quiet'].get(bool)
if not quiet and not pretend: if not quiet and not pretend:
self._log.info(u'Encoding {0}', util.displayable_path(source)) self._log.info('Encoding {0}', util.displayable_path(source))
# On Python 3, we need to construct the command to invoke as a command = command.decode(arg_encoding(), 'surrogateescape')
# Unicode string. On Unix, this is a little unfortunate---the OS is source = decode_commandline_path(source)
# expecting bytes---so we use surrogate escaping and decode with the dest = decode_commandline_path(dest)
# argument encoding, which is the same encoding that will then be
# *reversed* to recover the same bytes before invoking the OS. On
# Windows, we want to preserve the Unicode filename "as is."
if not six.PY2:
command = command.decode(util.arg_encoding(), 'surrogateescape')
if platform.system() == 'Windows':
source = source.decode(util._fsencoding())
dest = dest.decode(util._fsencoding())
else:
source = source.decode(util.arg_encoding(), 'surrogateescape')
dest = dest.decode(util.arg_encoding(), 'surrogateescape')
# Substitute $source and $dest in the argument list. # Substitute $source and $dest in the argument list.
args = shlex.split(command) args = shlex.split(command)
@ -216,22 +213,19 @@ class ConvertPlugin(BeetsPlugin):
'source': source, 'source': source,
'dest': dest, 'dest': dest,
}) })
if six.PY2: encode_cmd.append(args[i].encode(util.arg_encoding()))
encode_cmd.append(args[i])
else:
encode_cmd.append(args[i].encode(util.arg_encoding()))
if pretend: if pretend:
self._log.info(u'{0}', u' '.join(ui.decargs(args))) self._log.info('{0}', ' '.join(ui.decargs(args)))
return return
try: try:
util.command_output(encode_cmd) util.command_output(encode_cmd)
except subprocess.CalledProcessError as exc: except subprocess.CalledProcessError as exc:
# Something went wrong (probably Ctrl+C), remove temporary files # Something went wrong (probably Ctrl+C), remove temporary files
self._log.info(u'Encoding {0} failed. Cleaning up...', self._log.info('Encoding {0} failed. Cleaning up...',
util.displayable_path(source)) util.displayable_path(source))
self._log.debug(u'Command {0} exited with status {1}: {2}', self._log.debug('Command {0} exited with status {1}: {2}',
args, args,
exc.returncode, exc.returncode,
exc.output) exc.output)
@ -240,17 +234,17 @@ class ConvertPlugin(BeetsPlugin):
raise raise
except OSError as exc: except OSError as exc:
raise ui.UserError( raise ui.UserError(
u"convert: couldn't invoke '{0}': {1}".format( "convert: couldn't invoke '{}': {}".format(
u' '.join(ui.decargs(args)), exc ' '.join(ui.decargs(args)), exc
) )
) )
if not quiet and not pretend: if not quiet and not pretend:
self._log.info(u'Finished encoding {0}', self._log.info('Finished encoding {0}',
util.displayable_path(source)) util.displayable_path(source))
def convert_item(self, dest_dir, keep_new, path_formats, fmt, def convert_item(self, dest_dir, keep_new, path_formats, fmt,
pretend=False): pretend=False, link=False, hardlink=False):
"""A pipeline thread that converts `Item` objects from a """A pipeline thread that converts `Item` objects from a
library. library.
""" """
@ -283,41 +277,60 @@ class ConvertPlugin(BeetsPlugin):
util.mkdirall(dest) util.mkdirall(dest)
if os.path.exists(util.syspath(dest)): if os.path.exists(util.syspath(dest)):
self._log.info(u'Skipping {0} (target file exists)', self._log.info('Skipping {0} (target file exists)',
util.displayable_path(item.path)) util.displayable_path(item.path))
continue continue
if keep_new: if keep_new:
if pretend: if pretend:
self._log.info(u'mv {0} {1}', self._log.info('mv {0} {1}',
util.displayable_path(item.path), util.displayable_path(item.path),
util.displayable_path(original)) util.displayable_path(original))
else: else:
self._log.info(u'Moving to {0}', self._log.info('Moving to {0}',
util.displayable_path(original)) util.displayable_path(original))
util.move(item.path, original) util.move(item.path, original)
if should_transcode(item, fmt): if should_transcode(item, fmt):
linked = False
try: try:
self.encode(command, original, converted, pretend) self.encode(command, original, converted, pretend)
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
continue continue
else: else:
linked = link or hardlink
if pretend: if pretend:
self._log.info(u'cp {0} {1}', msg = 'ln' if hardlink else ('ln -s' if link else 'cp')
self._log.info('{2} {0} {1}',
util.displayable_path(original), util.displayable_path(original),
util.displayable_path(converted)) util.displayable_path(converted),
msg)
else: else:
# No transcoding necessary. # No transcoding necessary.
self._log.info(u'Copying {0}', msg = 'Hardlinking' if hardlink \
util.displayable_path(item.path)) else ('Linking' if link else 'Copying')
util.copy(original, converted)
self._log.info('{1} {0}',
util.displayable_path(item.path),
msg)
if hardlink:
util.hardlink(original, converted)
elif link:
util.link(original, converted)
else:
util.copy(original, converted)
if pretend: if pretend:
continue continue
id3v23 = self.config['id3v23'].as_choice([True, False, 'inherit'])
if id3v23 == 'inherit':
id3v23 = None
# Write tags from the database to the converted file. # Write tags from the database to the converted file.
item.try_write(path=converted) item.try_write(path=converted, id3v23=id3v23)
if keep_new: if keep_new:
# If we're keeping the transcoded file, read it again (after # If we're keeping the transcoded file, read it again (after
@ -326,13 +339,13 @@ class ConvertPlugin(BeetsPlugin):
item.read() item.read()
item.store() # Store new path and audio data. item.store() # Store new path and audio data.
if self.config['embed']: if self.config['embed'] and not linked:
album = item.get_album() album = item._cached_album
if album and album.artpath: if album and album.artpath:
self._log.debug(u'embedding album art from {}', self._log.debug('embedding album art from {}',
util.displayable_path(album.artpath)) util.displayable_path(album.artpath))
art.embed_item(self._log, item, album.artpath, art.embed_item(self._log, item, album.artpath,
itempath=converted) itempath=converted, id3v23=id3v23)
if keep_new: if keep_new:
plugins.send('after_convert', item=item, plugins.send('after_convert', item=item,
@ -341,7 +354,8 @@ class ConvertPlugin(BeetsPlugin):
plugins.send('after_convert', item=item, plugins.send('after_convert', item=item,
dest=converted, keepnew=False) dest=converted, keepnew=False)
def copy_album_art(self, album, dest_dir, path_formats, pretend=False): def copy_album_art(self, album, dest_dir, path_formats, pretend=False,
link=False, hardlink=False):
"""Copies or converts the associated cover art of the album. Album must """Copies or converts the associated cover art of the album. Album must
have at least one track. have at least one track.
""" """
@ -369,7 +383,7 @@ class ConvertPlugin(BeetsPlugin):
util.mkdirall(dest) util.mkdirall(dest)
if os.path.exists(util.syspath(dest)): if os.path.exists(util.syspath(dest)):
self._log.info(u'Skipping {0} (target file exists)', self._log.info('Skipping {0} (target file exists)',
util.displayable_path(album.artpath)) util.displayable_path(album.artpath))
return return
@ -383,31 +397,43 @@ class ConvertPlugin(BeetsPlugin):
if size: if size:
resize = size[0] > maxwidth resize = size[0] > maxwidth
else: else:
self._log.warning(u'Could not get size of image (please see ' self._log.warning('Could not get size of image (please see '
u'documentation for dependencies).') 'documentation for dependencies).')
# Either copy or resize (while copying) the image. # Either copy or resize (while copying) the image.
if resize: if resize:
self._log.info(u'Resizing cover art from {0} to {1}', self._log.info('Resizing cover art from {0} to {1}',
util.displayable_path(album.artpath), util.displayable_path(album.artpath),
util.displayable_path(dest)) util.displayable_path(dest))
if not pretend: if not pretend:
ArtResizer.shared.resize(maxwidth, album.artpath, dest) ArtResizer.shared.resize(maxwidth, album.artpath, dest)
else: else:
if pretend: if pretend:
self._log.info(u'cp {0} {1}', msg = 'ln' if hardlink else ('ln -s' if link else 'cp')
self._log.info('{2} {0} {1}',
util.displayable_path(album.artpath), util.displayable_path(album.artpath),
util.displayable_path(dest)) util.displayable_path(dest),
msg)
else: else:
self._log.info(u'Copying cover art to {0}', msg = 'Hardlinking' if hardlink \
else ('Linking' if link else 'Copying')
self._log.info('{2} cover art from {0} to {1}',
util.displayable_path(album.artpath), util.displayable_path(album.artpath),
util.displayable_path(dest)) util.displayable_path(dest),
util.copy(album.artpath, dest) msg)
if hardlink:
util.hardlink(album.artpath, dest)
elif link:
util.link(album.artpath, dest)
else:
util.copy(album.artpath, dest)
def convert_func(self, lib, opts, args): def convert_func(self, lib, opts, args):
dest = opts.dest or self.config['dest'].get() dest = opts.dest or self.config['dest'].get()
if not dest: if not dest:
raise ui.UserError(u'no convert destination set') raise ui.UserError('no convert destination set')
dest = util.bytestring_path(dest) dest = util.bytestring_path(dest)
threads = opts.threads or self.config['threads'].get(int) threads = opts.threads or self.config['threads'].get(int)
@ -421,33 +447,46 @@ class ConvertPlugin(BeetsPlugin):
else: else:
pretend = self.config['pretend'].get(bool) pretend = self.config['pretend'].get(bool)
if opts.hardlink is not None:
hardlink = opts.hardlink
link = False
elif opts.link is not None:
hardlink = False
link = opts.link
else:
hardlink = self.config['hardlink'].get(bool)
link = self.config['link'].get(bool)
if opts.album: if opts.album:
albums = lib.albums(ui.decargs(args)) albums = lib.albums(ui.decargs(args))
items = [i for a in albums for i in a.items()] items = [i for a in albums for i in a.items()]
if not pretend: if not pretend:
for a in albums: for a in albums:
ui.print_(format(a, u'')) ui.print_(format(a, ''))
else: else:
items = list(lib.items(ui.decargs(args))) items = list(lib.items(ui.decargs(args)))
if not pretend: if not pretend:
for i in items: for i in items:
ui.print_(format(i, u'')) ui.print_(format(i, ''))
if not items: if not items:
self._log.error(u'Empty query result.') self._log.error('Empty query result.')
return return
if not (pretend or opts.yes or ui.input_yn(u"Convert? (Y/n)")): if not (pretend or opts.yes or ui.input_yn("Convert? (Y/n)")):
return return
if opts.album and self.config['copy_album_art']: if opts.album and self.config['copy_album_art']:
for album in albums: for album in albums:
self.copy_album_art(album, dest, path_formats, pretend) self.copy_album_art(album, dest, path_formats, pretend,
link, hardlink)
convert = [self.convert_item(dest, convert = [self.convert_item(dest,
opts.keep_new, opts.keep_new,
path_formats, path_formats,
fmt, fmt,
pretend) pretend,
link,
hardlink)
for _ in range(threads)] for _ in range(threads)]
pipe = util.pipeline.Pipeline([iter(items), convert]) pipe = util.pipeline.Pipeline([iter(items), convert])
pipe.run_parallel() pipe.run_parallel()
@ -477,11 +516,16 @@ class ConvertPlugin(BeetsPlugin):
# Change the newly-imported database entry to point to the # Change the newly-imported database entry to point to the
# converted file. # converted file.
source_path = item.path
item.path = dest item.path = dest
item.write() item.write()
item.read() # Load new audio information data. item.read() # Load new audio information data.
item.store() item.store()
if self.config['delete_originals']:
self._log.info('Removing original file {0}', source_path)
util.remove(source_path, False)
def _cleanup(self, task, session): def _cleanup(self, task, session):
for path in task.old_paths: for path in task.old_paths:
if path in _temp_files: if path in _temp_files:

57
libs/common/beetsplug/cue.py
View file

@ -1,57 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright 2016 Bruno Cauet
# Split an album-file in tracks thanks a cue file
from __future__ import division, absolute_import, print_function
import subprocess
from os import path
from glob import glob
from beets.util import command_output, displayable_path
from beets.plugins import BeetsPlugin
from beets.autotag import TrackInfo
class CuePlugin(BeetsPlugin):
def __init__(self):
super(CuePlugin, self).__init__()
# this does not seem supported by shnsplit
self.config.add({
'keep_before': .1,
'keep_after': .9,
})
# self.register_listener('import_task_start', self.look_for_cues)
def candidates(self, items, artist, album, va_likely):
import pdb
pdb.set_trace()
def item_candidates(self, item, artist, album):
dir = path.dirname(item.path)
cues = glob.glob(path.join(dir, "*.cue"))
if not cues:
return
if len(cues) > 1:
self._log.info(u"Found multiple cue files doing nothing: {0}",
list(map(displayable_path, cues)))
cue_file = cues[0]
self._log.info("Found {} for {}", displayable_path(cue_file), item)
try:
# careful: will ask for input in case of conflicts
command_output(['shnsplit', '-f', cue_file, item.path])
except (subprocess.CalledProcessError, OSError):
self._log.exception(u'shnsplit execution failed')
return
tracks = glob(path.join(dir, "*.wav"))
self._log.info("Generated {0} tracks", len(tracks))
for t in tracks:
title = "dunno lol"
track_id = "wtf"
index = int(path.basename(t)[len("split-track"):-len(".wav")])
yield TrackInfo(title, track_id, index=index, artist=artist)
# generate TrackInfo instances

230
libs/common/beetsplug/deezer.py Normal file
View file

@ -0,0 +1,230 @@
# This file is part of beets.
# Copyright 2019, Rahul Ahuja.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Adds Deezer release and track search support to the autotagger
"""
import collections
import unidecode
import requests
from beets import ui
from beets.autotag import AlbumInfo, TrackInfo
from beets.plugins import MetadataSourcePlugin, BeetsPlugin
class DeezerPlugin(MetadataSourcePlugin, BeetsPlugin):
data_source = 'Deezer'
# Base URLs for the Deezer API
# Documentation: https://developers.deezer.com/api/
search_url = 'https://api.deezer.com/search/'
album_url = 'https://api.deezer.com/album/'
track_url = 'https://api.deezer.com/track/'
id_regex = {
'pattern': r'(^|deezer\.com/)([a-z]*/)?({}/)?(\d+)',
'match_group': 4,
}
def __init__(self):
super().__init__()
def album_for_id(self, album_id):
"""Fetch an album by its Deezer ID or URL and return an
AlbumInfo object or None if the album is not found.
:param album_id: Deezer ID or URL for the album.
:type album_id: str
:return: AlbumInfo object for album.
:rtype: beets.autotag.hooks.AlbumInfo or None
"""
deezer_id = self._get_id('album', album_id)
if deezer_id is None:
return None
album_data = requests.get(self.album_url + deezer_id).json()
artist, artist_id = self.get_artist(album_data['contributors'])
release_date = album_data['release_date']
date_parts = [int(part) for part in release_date.split('-')]
num_date_parts = len(date_parts)
if num_date_parts == 3:
year, month, day = date_parts
elif num_date_parts == 2:
year, month = date_parts
day = None
elif num_date_parts == 1:
year = date_parts[0]
month = None
day = None
else:
raise ui.UserError(
"Invalid `release_date` returned "
"by {} API: '{}'".format(self.data_source, release_date)
)
tracks_data = requests.get(
self.album_url + deezer_id + '/tracks'
).json()['data']
if not tracks_data:
return None
tracks = []
medium_totals = collections.defaultdict(int)
for i, track_data in enumerate(tracks_data, start=1):
track = self._get_track(track_data)
track.index = i
medium_totals[track.medium] += 1
tracks.append(track)
for track in tracks:
track.medium_total = medium_totals[track.medium]
return AlbumInfo(
album=album_data['title'],
album_id=deezer_id,
artist=artist,
artist_credit=self.get_artist([album_data['artist']])[0],
artist_id=artist_id,
tracks=tracks,
albumtype=album_data['record_type'],
va=len(album_data['contributors']) == 1
and artist.lower() == 'various artists',
year=year,
month=month,
day=day,
label=album_data['label'],
mediums=max(medium_totals.keys()),
data_source=self.data_source,
data_url=album_data['link'],
)
def _get_track(self, track_data):
"""Convert a Deezer track object dict to a TrackInfo object.
:param track_data: Deezer Track object dict
:type track_data: dict
:return: TrackInfo object for track
:rtype: beets.autotag.hooks.TrackInfo
"""
artist, artist_id = self.get_artist(
track_data.get('contributors', [track_data['artist']])
)
return TrackInfo(
title=track_data['title'],
track_id=track_data['id'],
artist=artist,
artist_id=artist_id,
length=track_data['duration'],
index=track_data['track_position'],
medium=track_data['disk_number'],
medium_index=track_data['track_position'],
data_source=self.data_source,
data_url=track_data['link'],
)
def track_for_id(self, track_id=None, track_data=None):
"""Fetch a track by its Deezer ID or URL and return a
TrackInfo object or None if the track is not found.
:param track_id: (Optional) Deezer ID or URL for the track. Either
``track_id`` or ``track_data`` must be provided.
:type track_id: str
:param track_data: (Optional) Simplified track object dict. May be
provided instead of ``track_id`` to avoid unnecessary API calls.
:type track_data: dict
:return: TrackInfo object for track
:rtype: beets.autotag.hooks.TrackInfo or None
"""
if track_data is None:
deezer_id = self._get_id('track', track_id)
if deezer_id is None:
return None
track_data = requests.get(self.track_url + deezer_id).json()
track = self._get_track(track_data)
# Get album's tracks to set `track.index` (position on the entire
# release) and `track.medium_total` (total number of tracks on
# the track's disc).
album_tracks_data = requests.get(
self.album_url + str(track_data['album']['id']) + '/tracks'
).json()['data']
medium_total = 0
for i, track_data in enumerate(album_tracks_data, start=1):
if track_data['disk_number'] == track.medium:
medium_total += 1
if track_data['id'] == track.track_id:
track.index = i
track.medium_total = medium_total
return track
@staticmethod
def _construct_search_query(filters=None, keywords=''):
"""Construct a query string with the specified filters and keywords to
be provided to the Deezer Search API
(https://developers.deezer.com/api/search).
:param filters: (Optional) Field filters to apply.
:type filters: dict
:param keywords: (Optional) Query keywords to use.
:type keywords: str
:return: Query string to be provided to the Search API.
:rtype: str
"""
query_components = [
keywords,
' '.join(f'{k}:"{v}"' for k, v in filters.items()),
]
query = ' '.join([q for q in query_components if q])
if not isinstance(query, str):
query = query.decode('utf8')
return unidecode.unidecode(query)
def _search_api(self, query_type, filters=None, keywords=''):
"""Query the Deezer Search API for the specified ``keywords``, applying
the provided ``filters``.
:param query_type: The Deezer Search API method to use. Valid types
are: 'album', 'artist', 'history', 'playlist', 'podcast',
'radio', 'track', 'user', and 'track'.
:type query_type: str
:param filters: (Optional) Field filters to apply.
:type filters: dict
:param keywords: (Optional) Query keywords to use.
:type keywords: str
:return: JSON data for the class:`Response <Response>` object or None
if no search results are returned.
:rtype: dict or None
"""
query = self._construct_search_query(
keywords=keywords, filters=filters
)
if not query:
return None
self._log.debug(
f"Searching {self.data_source} for '{query}'"
)
response = requests.get(
self.search_url + query_type, params={'q': query}
)
response.raise_for_status()
response_data = response.json().get('data', [])
self._log.debug(
"Found {} result(s) from {} for '{}'",
len(response_data),
self.data_source,
query,
)
return response_data

254
libs/common/beetsplug/discogs.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson. # Copyright 2016, Adrian Sampson.
# #
@ -14,19 +13,18 @@
# included in all copies or substantial portions of the Software. # included in all copies or substantial portions of the Software.
"""Adds Discogs album search support to the autotagger. Requires the """Adds Discogs album search support to the autotagger. Requires the
discogs-client library. python3-discogs-client library.
""" """
from __future__ import division, absolute_import, print_function
import beets.ui import beets.ui
from beets import config from beets import config
from beets.autotag.hooks import AlbumInfo, TrackInfo, Distance from beets.autotag.hooks import AlbumInfo, TrackInfo
from beets.plugins import BeetsPlugin from beets.plugins import MetadataSourcePlugin, BeetsPlugin, get_distance
from beets.util import confit import confuse
from discogs_client import Release, Master, Client from discogs_client import Release, Master, Client
from discogs_client.exceptions import DiscogsAPIError from discogs_client.exceptions import DiscogsAPIError
from requests.exceptions import ConnectionError from requests.exceptions import ConnectionError
from six.moves import http_client import http.client
import beets import beets
import re import re
import time import time
@ -37,10 +35,12 @@ import traceback
from string import ascii_lowercase from string import ascii_lowercase
USER_AGENT = u'beets/{0} +http://beets.io/'.format(beets.__version__) USER_AGENT = f'beets/{beets.__version__} +https://beets.io/'
API_KEY = 'rAzVUQYRaoFjeBjyWuWZ'
API_SECRET = 'plxtUTqoCzwxZpqdPysCwGuBSmZNdZVy'
# Exceptions that discogs_client should really handle but does not. # Exceptions that discogs_client should really handle but does not.
CONNECTION_ERRORS = (ConnectionError, socket.error, http_client.HTTPException, CONNECTION_ERRORS = (ConnectionError, socket.error, http.client.HTTPException,
ValueError, # JSON decoding raises a ValueError. ValueError, # JSON decoding raises a ValueError.
DiscogsAPIError) DiscogsAPIError)
@ -48,13 +48,15 @@ CONNECTION_ERRORS = (ConnectionError, socket.error, http_client.HTTPException,
class DiscogsPlugin(BeetsPlugin): class DiscogsPlugin(BeetsPlugin):
def __init__(self): def __init__(self):
super(DiscogsPlugin, self).__init__() super().__init__()
self.config.add({ self.config.add({
'apikey': 'rAzVUQYRaoFjeBjyWuWZ', 'apikey': API_KEY,
'apisecret': 'plxtUTqoCzwxZpqdPysCwGuBSmZNdZVy', 'apisecret': API_SECRET,
'tokenfile': 'discogs_token.json', 'tokenfile': 'discogs_token.json',
'source_weight': 0.5, 'source_weight': 0.5,
'user_token': '', 'user_token': '',
'separator': ', ',
'index_tracks': False,
}) })
self.config['apikey'].redact = True self.config['apikey'].redact = True
self.config['apisecret'].redact = True self.config['apisecret'].redact = True
@ -71,6 +73,8 @@ class DiscogsPlugin(BeetsPlugin):
# Try using a configured user token (bypassing OAuth login). # Try using a configured user token (bypassing OAuth login).
user_token = self.config['user_token'].as_str() user_token = self.config['user_token'].as_str()
if user_token: if user_token:
# The rate limit for authenticated users goes up to 60
# requests per minute.
self.discogs_client = Client(USER_AGENT, user_token=user_token) self.discogs_client = Client(USER_AGENT, user_token=user_token)
return return
@ -78,7 +82,7 @@ class DiscogsPlugin(BeetsPlugin):
try: try:
with open(self._tokenfile()) as f: with open(self._tokenfile()) as f:
tokendata = json.load(f) tokendata = json.load(f)
except IOError: except OSError:
# No token yet. Generate one. # No token yet. Generate one.
token, secret = self.authenticate(c_key, c_secret) token, secret = self.authenticate(c_key, c_secret)
else: else:
@ -97,7 +101,7 @@ class DiscogsPlugin(BeetsPlugin):
def _tokenfile(self): def _tokenfile(self):
"""Get the path to the JSON file for storing the OAuth token. """Get the path to the JSON file for storing the OAuth token.
""" """
return self.config['tokenfile'].get(confit.Filename(in_app_dir=True)) return self.config['tokenfile'].get(confuse.Filename(in_app_dir=True))
def authenticate(self, c_key, c_secret): def authenticate(self, c_key, c_secret):
# Get the link for the OAuth page. # Get the link for the OAuth page.
@ -105,24 +109,24 @@ class DiscogsPlugin(BeetsPlugin):
try: try:
_, _, url = auth_client.get_authorize_url() _, _, url = auth_client.get_authorize_url()
except CONNECTION_ERRORS as e: except CONNECTION_ERRORS as e:
self._log.debug(u'connection error: {0}', e) self._log.debug('connection error: {0}', e)
raise beets.ui.UserError(u'communication with Discogs failed') raise beets.ui.UserError('communication with Discogs failed')
beets.ui.print_(u"To authenticate with Discogs, visit:") beets.ui.print_("To authenticate with Discogs, visit:")
beets.ui.print_(url) beets.ui.print_(url)
# Ask for the code and validate it. # Ask for the code and validate it.
code = beets.ui.input_(u"Enter the code:") code = beets.ui.input_("Enter the code:")
try: try:
token, secret = auth_client.get_access_token(code) token, secret = auth_client.get_access_token(code)
except DiscogsAPIError: except DiscogsAPIError:
raise beets.ui.UserError(u'Discogs authorization failed') raise beets.ui.UserError('Discogs authorization failed')
except CONNECTION_ERRORS as e: except CONNECTION_ERRORS as e:
self._log.debug(u'connection error: {0}', e) self._log.debug('connection error: {0}', e)
raise beets.ui.UserError(u'Discogs token request failed') raise beets.ui.UserError('Discogs token request failed')
# Save the token for later use. # Save the token for later use.
self._log.debug(u'Discogs token {0}, secret {1}', token, secret) self._log.debug('Discogs token {0}, secret {1}', token, secret)
with open(self._tokenfile(), 'w') as f: with open(self._tokenfile(), 'w') as f:
json.dump({'token': token, 'secret': secret}, f) json.dump({'token': token, 'secret': secret}, f)
@ -131,12 +135,22 @@ class DiscogsPlugin(BeetsPlugin):
def album_distance(self, items, album_info, mapping): def album_distance(self, items, album_info, mapping):
"""Returns the album distance. """Returns the album distance.
""" """
dist = Distance() return get_distance(
if album_info.data_source == 'Discogs': data_source='Discogs',
dist.add('source', self.config['source_weight'].as_number()) info=album_info,
return dist config=self.config
)
def candidates(self, items, artist, album, va_likely): def track_distance(self, item, track_info):
"""Returns the track distance.
"""
return get_distance(
data_source='Discogs',
info=track_info,
config=self.config
)
def candidates(self, items, artist, album, va_likely, extra_tags=None):
"""Returns a list of AlbumInfo objects for discogs search results """Returns a list of AlbumInfo objects for discogs search results
matching an album and artist (if not various). matching an album and artist (if not various).
""" """
@ -146,20 +160,45 @@ class DiscogsPlugin(BeetsPlugin):
if va_likely: if va_likely:
query = album query = album
else: else:
query = '%s %s' % (artist, album) query = f'{artist} {album}'
try: try:
return self.get_albums(query) return self.get_albums(query)
except DiscogsAPIError as e: except DiscogsAPIError as e:
self._log.debug(u'API Error: {0} (query: {1})', e, query) self._log.debug('API Error: {0} (query: {1})', e, query)
if e.status_code == 401: if e.status_code == 401:
self.reset_auth() self.reset_auth()
return self.candidates(items, artist, album, va_likely) return self.candidates(items, artist, album, va_likely)
else: else:
return [] return []
except CONNECTION_ERRORS: except CONNECTION_ERRORS:
self._log.debug(u'Connection error in album search', exc_info=True) self._log.debug('Connection error in album search', exc_info=True)
return [] return []
@staticmethod
def extract_release_id_regex(album_id):
"""Returns the Discogs_id or None."""
# Discogs-IDs are simple integers. In order to avoid confusion with
# other metadata plugins, we only look for very specific formats of the
# input string:
# - plain integer, optionally wrapped in brackets and prefixed by an
# 'r', as this is how discogs displays the release ID on its webpage.
# - legacy url format: discogs.com/<name of release>/release/<id>
# - current url format: discogs.com/release/<id>-<name of release>
# See #291, #4080 and #4085 for the discussions leading up to these
# patterns.
# Regex has been tested here https://regex101.com/r/wyLdB4/2
for pattern in [
r'^\[?r?(?P<id>\d+)\]?$',
r'discogs\.com/release/(?P<id>\d+)-',
r'discogs\.com/[^/]+/release/(?P<id>\d+)',
]:
match = re.search(pattern, album_id)
if match:
return int(match.group('id'))
return None
def album_for_id(self, album_id): def album_for_id(self, album_id):
"""Fetches an album by its Discogs ID and returns an AlbumInfo object """Fetches an album by its Discogs ID and returns an AlbumInfo object
or None if the album is not found. or None if the album is not found.
@ -167,28 +206,28 @@ class DiscogsPlugin(BeetsPlugin):
if not self.discogs_client: if not self.discogs_client:
return return
self._log.debug(u'Searching for release {0}', album_id) self._log.debug('Searching for release {0}', album_id)
# Discogs-IDs are simple integers. We only look for those at the end
# of an input string as to avoid confusion with other metadata plugins. discogs_id = self.extract_release_id_regex(album_id)
# An optional bracket can follow the integer, as this is how discogs
# displays the release ID on its webpage. if not discogs_id:
match = re.search(r'(^|\[*r|discogs\.com/.+/release/)(\d+)($|\])',
album_id)
if not match:
return None return None
result = Release(self.discogs_client, {'id': int(match.group(2))})
result = Release(self.discogs_client, {'id': discogs_id})
# Try to obtain title to verify that we indeed have a valid Release # Try to obtain title to verify that we indeed have a valid Release
try: try:
getattr(result, 'title') getattr(result, 'title')
except DiscogsAPIError as e: except DiscogsAPIError as e:
if e.status_code != 404: if e.status_code != 404:
self._log.debug(u'API Error: {0} (query: {1})', e, result._uri) self._log.debug('API Error: {0} (query: {1})', e,
result.data['resource_url'])
if e.status_code == 401: if e.status_code == 401:
self.reset_auth() self.reset_auth()
return self.album_for_id(album_id) return self.album_for_id(album_id)
return None return None
except CONNECTION_ERRORS: except CONNECTION_ERRORS:
self._log.debug(u'Connection error in album lookup', exc_info=True) self._log.debug('Connection error in album lookup',
exc_info=True)
return None return None
return self.get_album_info(result) return self.get_album_info(result)
@ -199,18 +238,17 @@ class DiscogsPlugin(BeetsPlugin):
# cause a query to return no results, even if they match the artist or # cause a query to return no results, even if they match the artist or
# album title. Use `re.UNICODE` flag to avoid stripping non-english # album title. Use `re.UNICODE` flag to avoid stripping non-english
# word characters. # word characters.
# FIXME: Encode as ASCII to work around a bug: query = re.sub(r'(?u)\W+', ' ', query)
# https://github.com/beetbox/beets/issues/1051
# When the library is fixed, we should encode as UTF-8.
query = re.sub(r'(?u)\W+', ' ', query).encode('ascii', "replace")
# Strip medium information from query, Things like "CD1" and "disk 1" # Strip medium information from query, Things like "CD1" and "disk 1"
# can also negate an otherwise positive result. # can also negate an otherwise positive result.
query = re.sub(br'(?i)\b(CD|disc)\s*\d+', b'', query) query = re.sub(r'(?i)\b(CD|disc)\s*\d+', '', query)
try: try:
releases = self.discogs_client.search(query, releases = self.discogs_client.search(query,
type='release').page(1) type='release').page(1)
except CONNECTION_ERRORS: except CONNECTION_ERRORS:
self._log.debug(u"Communication error while searching for {0!r}", self._log.debug("Communication error while searching for {0!r}",
query, exc_info=True) query, exc_info=True)
return [] return []
return [album for album in map(self.get_album_info, releases[:5]) return [album for album in map(self.get_album_info, releases[:5])
@ -220,20 +258,22 @@ class DiscogsPlugin(BeetsPlugin):
"""Fetches a master release given its Discogs ID and returns its year """Fetches a master release given its Discogs ID and returns its year
or None if the master release is not found. or None if the master release is not found.
""" """
self._log.debug(u'Searching for master release {0}', master_id) self._log.debug('Searching for master release {0}', master_id)
result = Master(self.discogs_client, {'id': master_id}) result = Master(self.discogs_client, {'id': master_id})
try: try:
year = result.fetch('year') year = result.fetch('year')
return year return year
except DiscogsAPIError as e: except DiscogsAPIError as e:
if e.status_code != 404: if e.status_code != 404:
self._log.debug(u'API Error: {0} (query: {1})', e, result._uri) self._log.debug('API Error: {0} (query: {1})', e,
result.data['resource_url'])
if e.status_code == 401: if e.status_code == 401:
self.reset_auth() self.reset_auth()
return self.get_master_year(master_id) return self.get_master_year(master_id)
return None return None
except CONNECTION_ERRORS: except CONNECTION_ERRORS:
self._log.debug(u'Connection error in master release lookup', self._log.debug('Connection error in master release lookup',
exc_info=True) exc_info=True)
return None return None
@ -252,10 +292,12 @@ class DiscogsPlugin(BeetsPlugin):
# https://www.discogs.com/help/doc/submission-guidelines-general-rules # https://www.discogs.com/help/doc/submission-guidelines-general-rules
if not all([result.data.get(k) for k in ['artists', 'title', 'id', if not all([result.data.get(k) for k in ['artists', 'title', 'id',
'tracklist']]): 'tracklist']]):
self._log.warn(u"Release does not contain the required fields") self._log.warning("Release does not contain the required fields")
return None return None
artist, artist_id = self.get_artist([a.data for a in result.artists]) artist, artist_id = MetadataSourcePlugin.get_artist(
[a.data for a in result.artists]
)
album = re.sub(r' +', ' ', result.title) album = re.sub(r' +', ' ', result.title)
album_id = result.data['id'] album_id = result.data['id']
# Use `.data` to access the tracklist directly instead of the # Use `.data` to access the tracklist directly instead of the
@ -270,10 +312,13 @@ class DiscogsPlugin(BeetsPlugin):
mediums = [t.medium for t in tracks] mediums = [t.medium for t in tracks]
country = result.data.get('country') country = result.data.get('country')
data_url = result.data.get('uri') data_url = result.data.get('uri')
style = self.format(result.data.get('styles'))
genre = self.format(result.data.get('genres'))
discogs_albumid = self.extract_release_id(result.data.get('uri'))
# Extract information for the optional AlbumInfo fields that are # Extract information for the optional AlbumInfo fields that are
# contained on nested discogs fields. # contained on nested discogs fields.
albumtype = media = label = catalogno = None albumtype = media = label = catalogno = labelid = None
if result.data.get('formats'): if result.data.get('formats'):
albumtype = ', '.join( albumtype = ', '.join(
result.data['formats'][0].get('descriptions', [])) or None result.data['formats'][0].get('descriptions', [])) or None
@ -281,12 +326,13 @@ class DiscogsPlugin(BeetsPlugin):
if result.data.get('labels'): if result.data.get('labels'):
label = result.data['labels'][0].get('name') label = result.data['labels'][0].get('name')
catalogno = result.data['labels'][0].get('catno') catalogno = result.data['labels'][0].get('catno')
labelid = result.data['labels'][0].get('id')
# Additional cleanups (various artists name, catalog number, media). # Additional cleanups (various artists name, catalog number, media).
if va: if va:
artist = config['va_name'].as_str() artist = config['va_name'].as_str()
if catalogno == 'none': if catalogno == 'none':
catalogno = None catalogno = None
# Explicitly set the `media` for the tracks, since it is expected by # Explicitly set the `media` for the tracks, since it is expected by
# `autotag.apply_metadata`, and set `medium_total`. # `autotag.apply_metadata`, and set `medium_total`.
for track in tracks: for track in tracks:
@ -302,36 +348,29 @@ class DiscogsPlugin(BeetsPlugin):
# a master release, otherwise fetch the master release. # a master release, otherwise fetch the master release.
original_year = self.get_master_year(master_id) if master_id else year original_year = self.get_master_year(master_id) if master_id else year
return AlbumInfo(album, album_id, artist, artist_id, tracks, asin=None, return AlbumInfo(album=album, album_id=album_id, artist=artist,
albumtype=albumtype, va=va, year=year, month=None, artist_id=artist_id, tracks=tracks,
day=None, label=label, mediums=len(set(mediums)), albumtype=albumtype, va=va, year=year,
artist_sort=None, releasegroup_id=master_id, label=label, mediums=len(set(mediums)),
catalognum=catalogno, script=None, language=None, releasegroup_id=master_id, catalognum=catalogno,
country=country, albumstatus=None, media=media, country=country, style=style, genre=genre,
albumdisambig=None, artist_credit=None, media=media, original_year=original_year,
original_year=original_year, original_month=None, data_source='Discogs', data_url=data_url,
original_day=None, data_source='Discogs', discogs_albumid=discogs_albumid,
data_url=data_url) discogs_labelid=labelid, discogs_artistid=artist_id)
def get_artist(self, artists): def format(self, classification):
"""Returns an artist string (all artists) and an artist_id (the main if classification:
artist) for a list of discogs album or track artists. return self.config['separator'].as_str() \
""" .join(sorted(classification))
artist_id = None else:
bits = [] return None
for i, artist in enumerate(artists):
if not artist_id: def extract_release_id(self, uri):
artist_id = artist['id'] if uri:
name = artist['name'] return uri.split("/")[-1]
# Strip disambiguation number. else:
name = re.sub(r' \(\d+\)$', '', name) return None
# Move articles to the front.
name = re.sub(r'(?i)^(.*?), (a|an|the)$', r'\2 \1', name)
bits.append(name)
if artist['join'] and i < len(artists) - 1:
bits.append(artist['join'])
artist = ' '.join(bits).replace(' ,', ',') or None
return artist, artist_id
def get_tracks(self, tracklist): def get_tracks(self, tracklist):
"""Returns a list of TrackInfo objects for a discogs tracklist. """Returns a list of TrackInfo objects for a discogs tracklist.
@ -342,20 +381,34 @@ class DiscogsPlugin(BeetsPlugin):
# FIXME: this is an extra precaution for making sure there are no # FIXME: this is an extra precaution for making sure there are no
# side effects after #2222. It should be removed after further # side effects after #2222. It should be removed after further
# testing. # testing.
self._log.debug(u'{}', traceback.format_exc()) self._log.debug('{}', traceback.format_exc())
self._log.error(u'uncaught exception in coalesce_tracks: {}', exc) self._log.error('uncaught exception in coalesce_tracks: {}', exc)
clean_tracklist = tracklist clean_tracklist = tracklist
tracks = [] tracks = []
index_tracks = {} index_tracks = {}
index = 0 index = 0
# Distinct works and intra-work divisions, as defined by index tracks.
divisions, next_divisions = [], []
for track in clean_tracklist: for track in clean_tracklist:
# Only real tracks have `position`. Otherwise, it's an index track. # Only real tracks have `position`. Otherwise, it's an index track.
if track['position']: if track['position']:
index += 1 index += 1
track_info = self.get_track_info(track, index) if next_divisions:
# End of a block of index tracks: update the current
# divisions.
divisions += next_divisions
del next_divisions[:]
track_info = self.get_track_info(track, index, divisions)
track_info.track_alt = track['position'] track_info.track_alt = track['position']
tracks.append(track_info) tracks.append(track_info)
else: else:
next_divisions.append(track['title'])
# We expect new levels of division at the beginning of the
# tracklist (and possibly elsewhere).
try:
divisions.pop()
except IndexError:
pass
index_tracks[index + 1] = track['title'] index_tracks[index + 1] = track['title']
# Fix up medium and medium_index for each track. Discogs position is # Fix up medium and medium_index for each track. Discogs position is
@ -367,7 +420,7 @@ class DiscogsPlugin(BeetsPlugin):
# If a medium has two sides (ie. vinyl or cassette), each pair of # If a medium has two sides (ie. vinyl or cassette), each pair of
# consecutive sides should belong to the same medium. # consecutive sides should belong to the same medium.
if all([track.medium is not None for track in tracks]): if all([track.medium is not None for track in tracks]):
m = sorted(set([track.medium.lower() for track in tracks])) m = sorted({track.medium.lower() for track in tracks})
# If all track.medium are single consecutive letters, assume it is # If all track.medium are single consecutive letters, assume it is
# a 2-sided medium. # a 2-sided medium.
if ''.join(m) in ascii_lowercase: if ''.join(m) in ascii_lowercase:
@ -426,7 +479,7 @@ class DiscogsPlugin(BeetsPlugin):
# Calculate position based on first subtrack, without subindex. # Calculate position based on first subtrack, without subindex.
idx, medium_idx, sub_idx = \ idx, medium_idx, sub_idx = \
self.get_track_index(subtracks[0]['position']) self.get_track_index(subtracks[0]['position'])
position = '%s%s' % (idx or '', medium_idx or '') position = '{}{}'.format(idx or '', medium_idx or '')
if tracklist and not tracklist[-1]['position']: if tracklist and not tracklist[-1]['position']:
# Assume the previous index track contains the track title. # Assume the previous index track contains the track title.
@ -444,6 +497,12 @@ class DiscogsPlugin(BeetsPlugin):
for subtrack in subtracks: for subtrack in subtracks:
if not subtrack.get('artists'): if not subtrack.get('artists'):
subtrack['artists'] = index_track['artists'] subtrack['artists'] = index_track['artists']
# Concatenate index with track title when index_tracks
# option is set
if self.config['index_tracks']:
for subtrack in subtracks:
subtrack['title'] = '{}: {}'.format(
index_track['title'], subtrack['title'])
tracklist.extend(subtracks) tracklist.extend(subtracks)
else: else:
# Merge the subtracks, pick a title, and append the new track. # Merge the subtracks, pick a title, and append the new track.
@ -490,18 +549,23 @@ class DiscogsPlugin(BeetsPlugin):
return tracklist return tracklist
def get_track_info(self, track, index): def get_track_info(self, track, index, divisions):
"""Returns a TrackInfo object for a discogs track. """Returns a TrackInfo object for a discogs track.
""" """
title = track['title'] title = track['title']
if self.config['index_tracks']:
prefix = ', '.join(divisions)
if prefix:
title = f'{prefix}: {title}'
track_id = None track_id = None
medium, medium_index, _ = self.get_track_index(track['position']) medium, medium_index, _ = self.get_track_index(track['position'])
artist, artist_id = self.get_artist(track.get('artists', [])) artist, artist_id = MetadataSourcePlugin.get_artist(
track.get('artists', [])
)
length = self.get_track_length(track['duration']) length = self.get_track_length(track['duration'])
return TrackInfo(title, track_id, artist=artist, artist_id=artist_id, return TrackInfo(title=title, track_id=track_id, artist=artist,
length=length, index=index, artist_id=artist_id, length=length, index=index,
medium=medium, medium_index=medium_index, medium=medium, medium_index=medium_index)
artist_sort=None, disctitle=None, artist_credit=None)
def get_track_index(self, position): def get_track_index(self, position):
"""Returns the medium, medium index and subtrack index for a discogs """Returns the medium, medium index and subtrack index for a discogs
@ -528,7 +592,7 @@ class DiscogsPlugin(BeetsPlugin):
if subindex and subindex.startswith('.'): if subindex and subindex.startswith('.'):
subindex = subindex[1:] subindex = subindex[1:]
else: else:
self._log.debug(u'Invalid position: {0}', position) self._log.debug('Invalid position: {0}', position)
medium = index = subindex = None medium = index = subindex = None
return medium or None, index or None, subindex or None return medium or None, index or None, subindex or None

100
libs/common/beetsplug/duplicates.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Pedro Silva. # Copyright 2016, Pedro Silva.
# #
@ -15,16 +14,15 @@
"""List duplicate tracks or albums. """List duplicate tracks or albums.
""" """
from __future__ import division, absolute_import, print_function
import shlex import shlex
from beets.plugins import BeetsPlugin from beets.plugins import BeetsPlugin
from beets.ui import decargs, print_, Subcommand, UserError from beets.ui import decargs, print_, Subcommand, UserError
from beets.util import command_output, displayable_path, subprocess, \ from beets.util import command_output, displayable_path, subprocess, \
bytestring_path, MoveOperation bytestring_path, MoveOperation, decode_commandline_path
from beets.library import Item, Album from beets.library import Item, Album
import six
PLUGIN = 'duplicates' PLUGIN = 'duplicates'
@ -33,7 +31,7 @@ class DuplicatesPlugin(BeetsPlugin):
"""List duplicate tracks or albums """List duplicate tracks or albums
""" """
def __init__(self): def __init__(self):
super(DuplicatesPlugin, self).__init__() super().__init__()
self.config.add({ self.config.add({
'album': False, 'album': False,
@ -56,54 +54,54 @@ class DuplicatesPlugin(BeetsPlugin):
help=__doc__, help=__doc__,
aliases=['dup']) aliases=['dup'])
self._command.parser.add_option( self._command.parser.add_option(
u'-c', u'--count', dest='count', '-c', '--count', dest='count',
action='store_true', action='store_true',
help=u'show duplicate counts', help='show duplicate counts',
) )
self._command.parser.add_option( self._command.parser.add_option(
u'-C', u'--checksum', dest='checksum', '-C', '--checksum', dest='checksum',
action='store', metavar='PROG', action='store', metavar='PROG',
help=u'report duplicates based on arbitrary command', help='report duplicates based on arbitrary command',
) )
self._command.parser.add_option( self._command.parser.add_option(
u'-d', u'--delete', dest='delete', '-d', '--delete', dest='delete',
action='store_true', action='store_true',
help=u'delete items from library and disk', help='delete items from library and disk',
) )
self._command.parser.add_option( self._command.parser.add_option(
u'-F', u'--full', dest='full', '-F', '--full', dest='full',
action='store_true', action='store_true',
help=u'show all versions of duplicate tracks or albums', help='show all versions of duplicate tracks or albums',
) )
self._command.parser.add_option( self._command.parser.add_option(
u'-s', u'--strict', dest='strict', '-s', '--strict', dest='strict',
action='store_true', action='store_true',
help=u'report duplicates only if all attributes are set', help='report duplicates only if all attributes are set',
) )
self._command.parser.add_option( self._command.parser.add_option(
u'-k', u'--key', dest='keys', '-k', '--key', dest='keys',
action='append', metavar='KEY', action='append', metavar='KEY',
help=u'report duplicates based on keys (use multiple times)', help='report duplicates based on keys (use multiple times)',
) )
self._command.parser.add_option( self._command.parser.add_option(
u'-M', u'--merge', dest='merge', '-M', '--merge', dest='merge',
action='store_true', action='store_true',
help=u'merge duplicate items', help='merge duplicate items',
) )
self._command.parser.add_option( self._command.parser.add_option(
u'-m', u'--move', dest='move', '-m', '--move', dest='move',
action='store', metavar='DEST', action='store', metavar='DEST',
help=u'move items to dest', help='move items to dest',
) )
self._command.parser.add_option( self._command.parser.add_option(
u'-o', u'--copy', dest='copy', '-o', '--copy', dest='copy',
action='store', metavar='DEST', action='store', metavar='DEST',
help=u'copy items to dest', help='copy items to dest',
) )
self._command.parser.add_option( self._command.parser.add_option(
u'-t', u'--tag', dest='tag', '-t', '--tag', dest='tag',
action='store', action='store',
help=u'tag matched items with \'k=v\' attribute', help='tag matched items with \'k=v\' attribute',
) )
self._command.parser.add_all_common_options() self._command.parser.add_all_common_options()
@ -135,16 +133,21 @@ class DuplicatesPlugin(BeetsPlugin):
keys = ['mb_trackid', 'mb_albumid'] keys = ['mb_trackid', 'mb_albumid']
items = lib.items(decargs(args)) items = lib.items(decargs(args))
# If there's nothing to do, return early. The code below assumes
# `items` to be non-empty.
if not items:
return
if path: if path:
fmt = u'$path' fmt = '$path'
# Default format string for count mode. # Default format string for count mode.
if count and not fmt: if count and not fmt:
if album: if album:
fmt = u'$albumartist - $album' fmt = '$albumartist - $album'
else: else:
fmt = u'$albumartist - $album - $title' fmt = '$albumartist - $album - $title'
fmt += u': {0}' fmt += ': {0}'
if checksum: if checksum:
for i in items: for i in items:
@ -170,7 +173,7 @@ class DuplicatesPlugin(BeetsPlugin):
return [self._command] return [self._command]
def _process_item(self, item, copy=False, move=False, delete=False, def _process_item(self, item, copy=False, move=False, delete=False,
tag=False, fmt=u''): tag=False, fmt=''):
"""Process Item `item`. """Process Item `item`.
""" """
print_(format(item, fmt)) print_(format(item, fmt))
@ -187,7 +190,7 @@ class DuplicatesPlugin(BeetsPlugin):
k, v = tag.split('=') k, v = tag.split('=')
except Exception: except Exception:
raise UserError( raise UserError(
u"{}: can't parse k=v tag: {}".format(PLUGIN, tag) f"{PLUGIN}: can't parse k=v tag: {tag}"
) )
setattr(item, k, v) setattr(item, k, v)
item.store() item.store()
@ -197,25 +200,26 @@ class DuplicatesPlugin(BeetsPlugin):
output as flexattr on a key that is the name of the program, and output as flexattr on a key that is the name of the program, and
return the key, checksum tuple. return the key, checksum tuple.
""" """
args = [p.format(file=item.path) for p in shlex.split(prog)] args = [p.format(file=decode_commandline_path(item.path))
for p in shlex.split(prog)]
key = args[0] key = args[0]
checksum = getattr(item, key, False) checksum = getattr(item, key, False)
if not checksum: if not checksum:
self._log.debug(u'key {0} on item {1} not cached:' self._log.debug('key {0} on item {1} not cached:'
u'computing checksum', 'computing checksum',
key, displayable_path(item.path)) key, displayable_path(item.path))
try: try:
checksum = command_output(args) checksum = command_output(args).stdout
setattr(item, key, checksum) setattr(item, key, checksum)
item.store() item.store()
self._log.debug(u'computed checksum for {0} using {1}', self._log.debug('computed checksum for {0} using {1}',
item.title, key) item.title, key)
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
self._log.debug(u'failed to checksum {0}: {1}', self._log.debug('failed to checksum {0}: {1}',
displayable_path(item.path), e) displayable_path(item.path), e)
else: else:
self._log.debug(u'key {0} on item {1} cached:' self._log.debug('key {0} on item {1} cached:'
u'not computing checksum', 'not computing checksum',
key, displayable_path(item.path)) key, displayable_path(item.path))
return key, checksum return key, checksum
@ -231,12 +235,12 @@ class DuplicatesPlugin(BeetsPlugin):
values = [getattr(obj, k, None) for k in keys] values = [getattr(obj, k, None) for k in keys]
values = [v for v in values if v not in (None, '')] values = [v for v in values if v not in (None, '')]
if strict and len(values) < len(keys): if strict and len(values) < len(keys):
self._log.debug(u'some keys {0} on item {1} are null or empty:' self._log.debug('some keys {0} on item {1} are null or empty:'
u' skipping', ' skipping',
keys, displayable_path(obj.path)) keys, displayable_path(obj.path))
elif (not strict and not len(values)): elif (not strict and not len(values)):
self._log.debug(u'all keys {0} on item {1} are null or empty:' self._log.debug('all keys {0} on item {1} are null or empty:'
u' skipping', ' skipping',
keys, displayable_path(obj.path)) keys, displayable_path(obj.path))
else: else:
key = tuple(values) key = tuple(values)
@ -264,7 +268,7 @@ class DuplicatesPlugin(BeetsPlugin):
# between a bytes object and the empty Unicode # between a bytes object and the empty Unicode
# string ''. # string ''.
return v is not None and \ return v is not None and \
(v != '' if isinstance(v, six.text_type) else True) (v != '' if isinstance(v, str) else True)
fields = Item.all_keys() fields = Item.all_keys()
key = lambda x: sum(1 for f in fields if truthy(getattr(x, f))) key = lambda x: sum(1 for f in fields if truthy(getattr(x, f)))
else: else:
@ -284,8 +288,8 @@ class DuplicatesPlugin(BeetsPlugin):
if getattr(objs[0], f, None) in (None, ''): if getattr(objs[0], f, None) in (None, ''):
value = getattr(o, f, None) value = getattr(o, f, None)
if value: if value:
self._log.debug(u'key {0} on item {1} is null ' self._log.debug('key {0} on item {1} is null '
u'or empty: setting from item {2}', 'or empty: setting from item {2}',
f, displayable_path(objs[0].path), f, displayable_path(objs[0].path),
displayable_path(o.path)) displayable_path(o.path))
setattr(objs[0], f, value) setattr(objs[0], f, value)
@ -305,8 +309,8 @@ class DuplicatesPlugin(BeetsPlugin):
missing = Item.from_path(i.path) missing = Item.from_path(i.path)
missing.album_id = objs[0].id missing.album_id = objs[0].id
missing.add(i._db) missing.add(i._db)
self._log.debug(u'item {0} missing from album {1}:' self._log.debug('item {0} missing from album {1}:'
u' merging from {2} into {3}', ' merging from {2} into {3}',
missing, missing,
objs[0], objs[0],
displayable_path(o.path), displayable_path(o.path),

65
libs/common/beetsplug/edit.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016 # Copyright 2016
# #
@ -15,7 +14,6 @@
"""Open metadata information in a text editor to let the user edit it. """Open metadata information in a text editor to let the user edit it.
""" """
from __future__ import division, absolute_import, print_function
from beets import plugins from beets import plugins
from beets import util from beets import util
@ -28,7 +26,7 @@ import subprocess
import yaml import yaml
from tempfile import NamedTemporaryFile from tempfile import NamedTemporaryFile
import os import os
import six import shlex
# These "safe" types can avoid the format/parse cycle that most fields go # These "safe" types can avoid the format/parse cycle that most fields go
@ -45,13 +43,13 @@ class ParseError(Exception):
def edit(filename, log): def edit(filename, log):
"""Open `filename` in a text editor. """Open `filename` in a text editor.
""" """
cmd = util.shlex_split(util.editor_command()) cmd = shlex.split(util.editor_command())
cmd.append(filename) cmd.append(filename)
log.debug(u'invoking editor command: {!r}', cmd) log.debug('invoking editor command: {!r}', cmd)
try: try:
subprocess.call(cmd) subprocess.call(cmd)
except OSError as exc: except OSError as exc:
raise ui.UserError(u'could not run editor command {!r}: {}'.format( raise ui.UserError('could not run editor command {!r}: {}'.format(
cmd[0], exc cmd[0], exc
)) ))
@ -74,20 +72,20 @@ def load(s):
""" """
try: try:
out = [] out = []
for d in yaml.load_all(s): for d in yaml.safe_load_all(s):
if not isinstance(d, dict): if not isinstance(d, dict):
raise ParseError( raise ParseError(
u'each entry must be a dictionary; found {}'.format( 'each entry must be a dictionary; found {}'.format(
type(d).__name__ type(d).__name__
) )
) )
# Convert all keys to strings. They started out as strings, # Convert all keys to strings. They started out as strings,
# but the user may have inadvertently messed this up. # but the user may have inadvertently messed this up.
out.append({six.text_type(k): v for k, v in d.items()}) out.append({str(k): v for k, v in d.items()})
except yaml.YAMLError as e: except yaml.YAMLError as e:
raise ParseError(u'invalid YAML: {}'.format(e)) raise ParseError(f'invalid YAML: {e}')
return out return out
@ -143,13 +141,13 @@ def apply_(obj, data):
else: else:
# Either the field was stringified originally or the user changed # Either the field was stringified originally or the user changed
# it from a safe type to an unsafe one. Parse it as a string. # it from a safe type to an unsafe one. Parse it as a string.
obj.set_parse(key, six.text_type(value)) obj.set_parse(key, str(value))
class EditPlugin(plugins.BeetsPlugin): class EditPlugin(plugins.BeetsPlugin):
def __init__(self): def __init__(self):
super(EditPlugin, self).__init__() super().__init__()
self.config.add({ self.config.add({
# The default fields to edit. # The default fields to edit.
@ -166,18 +164,18 @@ class EditPlugin(plugins.BeetsPlugin):
def commands(self): def commands(self):
edit_command = ui.Subcommand( edit_command = ui.Subcommand(
'edit', 'edit',
help=u'interactively edit metadata' help='interactively edit metadata'
) )
edit_command.parser.add_option( edit_command.parser.add_option(
u'-f', u'--field', '-f', '--field',
metavar='FIELD', metavar='FIELD',
action='append', action='append',
help=u'edit this field also', help='edit this field also',
) )
edit_command.parser.add_option( edit_command.parser.add_option(
u'--all', '--all',
action='store_true', dest='all', action='store_true', dest='all',
help=u'edit all fields', help='edit all fields',
) )
edit_command.parser.add_album_option() edit_command.parser.add_album_option()
edit_command.func = self._edit_command edit_command.func = self._edit_command
@ -191,7 +189,7 @@ class EditPlugin(plugins.BeetsPlugin):
items, albums = _do_query(lib, query, opts.album, False) items, albums = _do_query(lib, query, opts.album, False)
objs = albums if opts.album else items objs = albums if opts.album else items
if not objs: if not objs:
ui.print_(u'Nothing to edit.') ui.print_('Nothing to edit.')
return return
# Get the fields to edit. # Get the fields to edit.
@ -244,15 +242,10 @@ class EditPlugin(plugins.BeetsPlugin):
old_data = [flatten(o, fields) for o in objs] old_data = [flatten(o, fields) for o in objs]
# Set up a temporary file with the initial data for editing. # Set up a temporary file with the initial data for editing.
if six.PY2: new = NamedTemporaryFile(mode='w', suffix='.yaml', delete=False,
new = NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) encoding='utf-8')
else:
new = NamedTemporaryFile(mode='w', suffix='.yaml', delete=False,
encoding='utf-8')
old_str = dump(old_data) old_str = dump(old_data)
new.write(old_str) new.write(old_str)
if six.PY2:
old_str = old_str.decode('utf-8')
new.close() new.close()
# Loop until we have parseable data and the user confirms. # Loop until we have parseable data and the user confirms.
@ -266,15 +259,15 @@ class EditPlugin(plugins.BeetsPlugin):
with codecs.open(new.name, encoding='utf-8') as f: with codecs.open(new.name, encoding='utf-8') as f:
new_str = f.read() new_str = f.read()
if new_str == old_str: if new_str == old_str:
ui.print_(u"No changes; aborting.") ui.print_("No changes; aborting.")
return False return False
# Parse the updated data. # Parse the updated data.
try: try:
new_data = load(new_str) new_data = load(new_str)
except ParseError as e: except ParseError as e:
ui.print_(u"Could not read data: {}".format(e)) ui.print_(f"Could not read data: {e}")
if ui.input_yn(u"Edit again to fix? (Y/n)", True): if ui.input_yn("Edit again to fix? (Y/n)", True):
continue continue
else: else:
return False return False
@ -289,18 +282,18 @@ class EditPlugin(plugins.BeetsPlugin):
for obj, obj_old in zip(objs, objs_old): for obj, obj_old in zip(objs, objs_old):
changed |= ui.show_model_changes(obj, obj_old) changed |= ui.show_model_changes(obj, obj_old)
if not changed: if not changed:
ui.print_(u'No changes to apply.') ui.print_('No changes to apply.')
return False return False
# Confirm the changes. # Confirm the changes.
choice = ui.input_options( choice = ui.input_options(
(u'continue Editing', u'apply', u'cancel') ('continue Editing', 'apply', 'cancel')
) )
if choice == u'a': # Apply. if choice == 'a': # Apply.
return True return True
elif choice == u'c': # Cancel. elif choice == 'c': # Cancel.
return False return False
elif choice == u'e': # Keep editing. elif choice == 'e': # Keep editing.
# Reset the temporary changes to the objects. I we have a # Reset the temporary changes to the objects. I we have a
# copy from above, use that, else reload from the database. # copy from above, use that, else reload from the database.
objs = [(old_obj or obj) objs = [(old_obj or obj)
@ -322,7 +315,7 @@ class EditPlugin(plugins.BeetsPlugin):
are temporary. are temporary.
""" """
if len(old_data) != len(new_data): if len(old_data) != len(new_data):
self._log.warning(u'number of objects changed from {} to {}', self._log.warning('number of objects changed from {} to {}',
len(old_data), len(new_data)) len(old_data), len(new_data))
obj_by_id = {o.id: o for o in objs} obj_by_id = {o.id: o for o in objs}
@ -333,7 +326,7 @@ class EditPlugin(plugins.BeetsPlugin):
forbidden = False forbidden = False
for key in ignore_fields: for key in ignore_fields:
if old_dict.get(key) != new_dict.get(key): if old_dict.get(key) != new_dict.get(key):
self._log.warning(u'ignoring object whose {} changed', key) self._log.warning('ignoring object whose {} changed', key)
forbidden = True forbidden = True
break break
if forbidden: if forbidden:
@ -348,7 +341,7 @@ class EditPlugin(plugins.BeetsPlugin):
# Save to the database and possibly write tags. # Save to the database and possibly write tags.
for ob in objs: for ob in objs:
if ob._dirty: if ob._dirty:
self._log.debug(u'saving changes to {}', ob) self._log.debug('saving changes to {}', ob)
ob.try_sync(ui.should_write(), ui.should_move()) ob.try_sync(ui.should_write(), ui.should_move())
# Methods for interactive importer execution. # Methods for interactive importer execution.

60
libs/common/beetsplug/embedart.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson. # Copyright 2016, Adrian Sampson.
# #
@ -14,7 +13,6 @@
# included in all copies or substantial portions of the Software. # included in all copies or substantial portions of the Software.
"""Allows beets to embed album art into file metadata.""" """Allows beets to embed album art into file metadata."""
from __future__ import division, absolute_import, print_function
import os.path import os.path
@ -34,11 +32,11 @@ def _confirm(objs, album):
`album` is a Boolean indicating whether these are albums (as opposed `album` is a Boolean indicating whether these are albums (as opposed
to items). to items).
""" """
noun = u'album' if album else u'file' noun = 'album' if album else 'file'
prompt = u'Modify artwork for {} {}{} (Y/n)?'.format( prompt = 'Modify artwork for {} {}{} (Y/n)?'.format(
len(objs), len(objs),
noun, noun,
u's' if len(objs) > 1 else u'' 's' if len(objs) > 1 else ''
) )
# Show all the items or albums. # Show all the items or albums.
@ -53,39 +51,41 @@ class EmbedCoverArtPlugin(BeetsPlugin):
"""Allows albumart to be embedded into the actual files. """Allows albumart to be embedded into the actual files.
""" """
def __init__(self): def __init__(self):
super(EmbedCoverArtPlugin, self).__init__() super().__init__()
self.config.add({ self.config.add({
'maxwidth': 0, 'maxwidth': 0,
'auto': True, 'auto': True,
'compare_threshold': 0, 'compare_threshold': 0,
'ifempty': False, 'ifempty': False,
'remove_art_file': False 'remove_art_file': False,
'quality': 0,
}) })
if self.config['maxwidth'].get(int) and not ArtResizer.shared.local: if self.config['maxwidth'].get(int) and not ArtResizer.shared.local:
self.config['maxwidth'] = 0 self.config['maxwidth'] = 0
self._log.warning(u"ImageMagick or PIL not found; " self._log.warning("ImageMagick or PIL not found; "
u"'maxwidth' option ignored") "'maxwidth' option ignored")
if self.config['compare_threshold'].get(int) and not \ if self.config['compare_threshold'].get(int) and not \
ArtResizer.shared.can_compare: ArtResizer.shared.can_compare:
self.config['compare_threshold'] = 0 self.config['compare_threshold'] = 0
self._log.warning(u"ImageMagick 6.8.7 or higher not installed; " self._log.warning("ImageMagick 6.8.7 or higher not installed; "
u"'compare_threshold' option ignored") "'compare_threshold' option ignored")
self.register_listener('art_set', self.process_album) self.register_listener('art_set', self.process_album)
def commands(self): def commands(self):
# Embed command. # Embed command.
embed_cmd = ui.Subcommand( embed_cmd = ui.Subcommand(
'embedart', help=u'embed image files into file metadata' 'embedart', help='embed image files into file metadata'
) )
embed_cmd.parser.add_option( embed_cmd.parser.add_option(
u'-f', u'--file', metavar='PATH', help=u'the image file to embed' '-f', '--file', metavar='PATH', help='the image file to embed'
) )
embed_cmd.parser.add_option( embed_cmd.parser.add_option(
u"-y", u"--yes", action="store_true", help=u"skip confirmation" "-y", "--yes", action="store_true", help="skip confirmation"
) )
maxwidth = self.config['maxwidth'].get(int) maxwidth = self.config['maxwidth'].get(int)
quality = self.config['quality'].get(int)
compare_threshold = self.config['compare_threshold'].get(int) compare_threshold = self.config['compare_threshold'].get(int)
ifempty = self.config['ifempty'].get(bool) ifempty = self.config['ifempty'].get(bool)
@ -93,7 +93,7 @@ class EmbedCoverArtPlugin(BeetsPlugin):
if opts.file: if opts.file:
imagepath = normpath(opts.file) imagepath = normpath(opts.file)
if not os.path.isfile(syspath(imagepath)): if not os.path.isfile(syspath(imagepath)):
raise ui.UserError(u'image file {0} not found'.format( raise ui.UserError('image file {} not found'.format(
displayable_path(imagepath) displayable_path(imagepath)
)) ))
@ -104,8 +104,9 @@ class EmbedCoverArtPlugin(BeetsPlugin):
return return
for item in items: for item in items:
art.embed_item(self._log, item, imagepath, maxwidth, None, art.embed_item(self._log, item, imagepath, maxwidth,
compare_threshold, ifempty) None, compare_threshold, ifempty,
quality=quality)
else: else:
albums = lib.albums(decargs(args)) albums = lib.albums(decargs(args))
@ -114,8 +115,9 @@ class EmbedCoverArtPlugin(BeetsPlugin):
return return
for album in albums: for album in albums:
art.embed_album(self._log, album, maxwidth, False, art.embed_album(self._log, album, maxwidth,
compare_threshold, ifempty) False, compare_threshold, ifempty,
quality=quality)
self.remove_artfile(album) self.remove_artfile(album)
embed_cmd.func = embed_func embed_cmd.func = embed_func
@ -123,15 +125,15 @@ class EmbedCoverArtPlugin(BeetsPlugin):
# Extract command. # Extract command.
extract_cmd = ui.Subcommand( extract_cmd = ui.Subcommand(
'extractart', 'extractart',
help=u'extract an image from file metadata', help='extract an image from file metadata',
) )
extract_cmd.parser.add_option( extract_cmd.parser.add_option(
u'-o', dest='outpath', '-o', dest='outpath',
help=u'image output file', help='image output file',
) )
extract_cmd.parser.add_option( extract_cmd.parser.add_option(
u'-n', dest='filename', '-n', dest='filename',
help=u'image filename to create for all matched albums', help='image filename to create for all matched albums',
) )
extract_cmd.parser.add_option( extract_cmd.parser.add_option(
'-a', dest='associate', action='store_true', '-a', dest='associate', action='store_true',
@ -147,7 +149,7 @@ class EmbedCoverArtPlugin(BeetsPlugin):
config['art_filename'].get()) config['art_filename'].get())
if os.path.dirname(filename) != b'': if os.path.dirname(filename) != b'':
self._log.error( self._log.error(
u"Only specify a name rather than a path for -n") "Only specify a name rather than a path for -n")
return return
for album in lib.albums(decargs(args)): for album in lib.albums(decargs(args)):
artpath = normpath(os.path.join(album.path, filename)) artpath = normpath(os.path.join(album.path, filename))
@ -161,10 +163,10 @@ class EmbedCoverArtPlugin(BeetsPlugin):
# Clear command. # Clear command.
clear_cmd = ui.Subcommand( clear_cmd = ui.Subcommand(
'clearart', 'clearart',
help=u'remove images from file metadata', help='remove images from file metadata',
) )
clear_cmd.parser.add_option( clear_cmd.parser.add_option(
u"-y", u"--yes", action="store_true", help=u"skip confirmation" "-y", "--yes", action="store_true", help="skip confirmation"
) )
def clear_func(lib, opts, args): def clear_func(lib, opts, args):
@ -189,11 +191,11 @@ class EmbedCoverArtPlugin(BeetsPlugin):
def remove_artfile(self, album): def remove_artfile(self, album):
"""Possibly delete the album art file for an album (if the """Possibly delete the album art file for an album (if the
appropriate configuration option is enabled. appropriate configuration option is enabled).
""" """
if self.config['remove_art_file'] and album.artpath: if self.config['remove_art_file'] and album.artpath:
if os.path.isfile(album.artpath): if os.path.isfile(album.artpath):
self._log.debug(u'Removing album art file for {0}', album) self._log.debug('Removing album art file for {0}', album)
os.remove(album.artpath) os.remove(album.artpath)
album.artpath = None album.artpath = None
album.store() album.store()

29
libs/common/beetsplug/embyupdate.py
View file

@ -1,5 +1,3 @@
# -*- coding: utf-8 -*-
"""Updates the Emby Library whenever the beets library is changed. """Updates the Emby Library whenever the beets library is changed.
emby: emby:
@ -9,14 +7,11 @@
apikey: apikey apikey: apikey
password: password password: password
""" """
from __future__ import division, absolute_import, print_function
import hashlib import hashlib
import requests import requests
from six.moves.urllib.parse import urlencode from urllib.parse import urlencode, urljoin, parse_qs, urlsplit, urlunsplit
from six.moves.urllib.parse import urljoin, parse_qs, urlsplit, urlunsplit
from beets import config from beets import config
from beets.plugins import BeetsPlugin from beets.plugins import BeetsPlugin
@ -146,14 +141,14 @@ def get_user(host, port, username):
class EmbyUpdate(BeetsPlugin): class EmbyUpdate(BeetsPlugin):
def __init__(self): def __init__(self):
super(EmbyUpdate, self).__init__() super().__init__()
# Adding defaults. # Adding defaults.
config['emby'].add({ config['emby'].add({
u'host': u'http://localhost', 'host': 'http://localhost',
u'port': 8096, 'port': 8096,
u'apikey': None, 'apikey': None,
u'password': None, 'password': None,
}) })
self.register_listener('database_change', self.listen_for_db_change) self.register_listener('database_change', self.listen_for_db_change)
@ -166,7 +161,7 @@ class EmbyUpdate(BeetsPlugin):
def update(self, lib): def update(self, lib):
"""When the client exists try to send refresh request to Emby. """When the client exists try to send refresh request to Emby.
""" """
self._log.info(u'Updating Emby library...') self._log.info('Updating Emby library...')
host = config['emby']['host'].get() host = config['emby']['host'].get()
port = config['emby']['port'].get() port = config['emby']['port'].get()
@ -176,13 +171,13 @@ class EmbyUpdate(BeetsPlugin):
# Check if at least a apikey or password is given. # Check if at least a apikey or password is given.
if not any([password, token]): if not any([password, token]):
self._log.warning(u'Provide at least Emby password or apikey.') self._log.warning('Provide at least Emby password or apikey.')
return return
# Get user information from the Emby API. # Get user information from the Emby API.
user = get_user(host, port, username) user = get_user(host, port, username)
if not user: if not user:
self._log.warning(u'User {0} could not be found.'.format(username)) self._log.warning(f'User {username} could not be found.')
return return
if not token: if not token:
@ -194,7 +189,7 @@ class EmbyUpdate(BeetsPlugin):
token = get_token(host, port, headers, auth_data) token = get_token(host, port, headers, auth_data)
if not token: if not token:
self._log.warning( self._log.warning(
u'Could not get token for user {0}', username 'Could not get token for user {0}', username
) )
return return
@ -205,6 +200,6 @@ class EmbyUpdate(BeetsPlugin):
url = api_url(host, port, '/Library/Refresh') url = api_url(host, port, '/Library/Refresh')
r = requests.post(url, headers=headers) r = requests.post(url, headers=headers)
if r.status_code != 204: if r.status_code != 204:
self._log.warning(u'Update could not be triggered') self._log.warning('Update could not be triggered')
else: else:
self._log.info(u'Update triggered.') self._log.info('Update triggered.')

194
libs/common/beetsplug/export.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# #
# Permission is hereby granted, free of charge, to any person obtaining # Permission is hereby granted, free of charge, to any person obtaining
@ -15,23 +14,25 @@
"""Exports data from beets """Exports data from beets
""" """
from __future__ import division, absolute_import, print_function
import sys import sys
import json
import codecs import codecs
import json
import csv
from xml.etree import ElementTree
from datetime import datetime, date from datetime import datetime, date
from beets.plugins import BeetsPlugin from beets.plugins import BeetsPlugin
from beets import ui from beets import ui
from beets import mediafile from beets import util
from beetsplug.info import make_key_filter, library_data, tag_data import mediafile
from beetsplug.info import library_data, tag_data
class ExportEncoder(json.JSONEncoder): class ExportEncoder(json.JSONEncoder):
"""Deals with dates because JSON doesn't have a standard""" """Deals with dates because JSON doesn't have a standard"""
def default(self, o): def default(self, o):
if isinstance(o, datetime) or isinstance(o, date): if isinstance(o, (datetime, date)):
return o.isoformat() return o.isoformat()
return json.JSONEncoder.default(self, o) return json.JSONEncoder.default(self, o)
@ -39,12 +40,12 @@ class ExportEncoder(json.JSONEncoder):
class ExportPlugin(BeetsPlugin): class ExportPlugin(BeetsPlugin):
def __init__(self): def __init__(self):
super(ExportPlugin, self).__init__() super().__init__()
self.config.add({ self.config.add({
'default_format': 'json', 'default_format': 'json',
'json': { 'json': {
# json module formatting options # JSON module formatting options.
'formatting': { 'formatting': {
'ensure_ascii': False, 'ensure_ascii': False,
'indent': 4, 'indent': 4,
@ -52,100 +53,175 @@ class ExportPlugin(BeetsPlugin):
'sort_keys': True 'sort_keys': True
} }
}, },
'jsonlines': {
# JSON Lines formatting options.
'formatting': {
'ensure_ascii': False,
'separators': (',', ': '),
'sort_keys': True
}
},
'csv': {
# CSV module formatting options.
'formatting': {
# The delimiter used to seperate columns.
'delimiter': ',',
# The dialect to use when formating the file output.
'dialect': 'excel'
}
},
'xml': {
# XML module formatting options.
'formatting': {}
}
# TODO: Use something like the edit plugin # TODO: Use something like the edit plugin
# 'item_fields': [] # 'item_fields': []
}) })
def commands(self): def commands(self):
# TODO: Add option to use albums cmd = ui.Subcommand('export', help='export data from beets')
cmd = ui.Subcommand('export', help=u'export data from beets')
cmd.func = self.run cmd.func = self.run
cmd.parser.add_option( cmd.parser.add_option(
u'-l', u'--library', action='store_true', '-l', '--library', action='store_true',
help=u'show library fields instead of tags', help='show library fields instead of tags',
) )
cmd.parser.add_option( cmd.parser.add_option(
u'--append', action='store_true', default=False, '-a', '--album', action='store_true',
help=u'if should append data to the file', help='show album fields instead of tracks (implies "--library")',
) )
cmd.parser.add_option( cmd.parser.add_option(
u'-i', u'--include-keys', default=[], '--append', action='store_true', default=False,
help='if should append data to the file',
)
cmd.parser.add_option(
'-i', '--include-keys', default=[],
action='append', dest='included_keys', action='append', dest='included_keys',
help=u'comma separated list of keys to show', help='comma separated list of keys to show',
) )
cmd.parser.add_option( cmd.parser.add_option(
u'-o', u'--output', '-o', '--output',
help=u'path for the output file. If not given, will print the data' help='path for the output file. If not given, will print the data'
)
cmd.parser.add_option(
'-f', '--format', default='json',
help="the output format: json (default), jsonlines, csv, or xml"
) )
return [cmd] return [cmd]
def run(self, lib, opts, args): def run(self, lib, opts, args):
file_path = opts.output file_path = opts.output
file_format = self.config['default_format'].get(str)
file_mode = 'a' if opts.append else 'w' file_mode = 'a' if opts.append else 'w'
file_format = opts.format or self.config['default_format'].get(str)
file_format_is_line_based = (file_format == 'jsonlines')
format_options = self.config[file_format]['formatting'].get(dict) format_options = self.config[file_format]['formatting'].get(dict)
export_format = ExportFormat.factory( export_format = ExportFormat.factory(
file_format, **{ file_type=file_format,
**{
'file_path': file_path, 'file_path': file_path,
'file_mode': file_mode 'file_mode': file_mode
} }
) )
items = [] if opts.library or opts.album:
data_collector = library_data if opts.library else tag_data data_collector = library_data
else:
data_collector = tag_data
included_keys = [] included_keys = []
for keys in opts.included_keys: for keys in opts.included_keys:
included_keys.extend(keys.split(',')) included_keys.extend(keys.split(','))
key_filter = make_key_filter(included_keys)
for data_emitter in data_collector(lib, ui.decargs(args)): items = []
for data_emitter in data_collector(
lib, ui.decargs(args),
album=opts.album,
):
try: try:
data, item = data_emitter() data, item = data_emitter(included_keys or '*')
except (mediafile.UnreadableFileError, IOError) as ex: except (mediafile.UnreadableFileError, OSError) as ex:
self._log.error(u'cannot read file: {0}', ex) self._log.error('cannot read file: {0}', ex)
continue continue
data = key_filter(data) for key, value in data.items():
items += [data] if isinstance(value, bytes):
data[key] = util.displayable_path(value)
export_format.export(items, **format_options) if file_format_is_line_based:
export_format.export(data, **format_options)
class ExportFormat(object):
"""The output format type"""
@classmethod
def factory(cls, type, **kwargs):
if type == "json":
if kwargs['file_path']:
return JsonFileFormat(**kwargs)
else: else:
return JsonPrintFormat() items += [data]
raise NotImplementedError()
def export(self, data, **kwargs): if not file_format_is_line_based:
raise NotImplementedError() export_format.export(items, **format_options)
class JsonPrintFormat(ExportFormat): class ExportFormat:
"""Outputs to the console""" """The output format type"""
def __init__(self, file_path, file_mode='w', encoding='utf-8'):
def export(self, data, **kwargs):
json.dump(data, sys.stdout, cls=ExportEncoder, **kwargs)
class JsonFileFormat(ExportFormat):
"""Saves in a json file"""
def __init__(self, file_path, file_mode=u'w', encoding=u'utf-8'):
self.path = file_path self.path = file_path
self.mode = file_mode self.mode = file_mode
self.encoding = encoding self.encoding = encoding
# creates a file object to write/append or sets to stdout
self.out_stream = codecs.open(self.path, self.mode, self.encoding) \
if self.path else sys.stdout
@classmethod
def factory(cls, file_type, **kwargs):
if file_type in ["json", "jsonlines"]:
return JsonFormat(**kwargs)
elif file_type == "csv":
return CSVFormat(**kwargs)
elif file_type == "xml":
return XMLFormat(**kwargs)
else:
raise NotImplementedError()
def export(self, data, **kwargs): def export(self, data, **kwargs):
with codecs.open(self.path, self.mode, self.encoding) as f: raise NotImplementedError()
json.dump(data, f, cls=ExportEncoder, **kwargs)
class JsonFormat(ExportFormat):
"""Saves in a json file"""
def __init__(self, file_path, file_mode='w', encoding='utf-8'):
super().__init__(file_path, file_mode, encoding)
def export(self, data, **kwargs):
json.dump(data, self.out_stream, cls=ExportEncoder, **kwargs)
self.out_stream.write('\n')
class CSVFormat(ExportFormat):
"""Saves in a csv file"""
def __init__(self, file_path, file_mode='w', encoding='utf-8'):
super().__init__(file_path, file_mode, encoding)
def export(self, data, **kwargs):
header = list(data[0].keys()) if data else []
writer = csv.DictWriter(self.out_stream, fieldnames=header, **kwargs)
writer.writeheader()
writer.writerows(data)
class XMLFormat(ExportFormat):
"""Saves in a xml file"""
def __init__(self, file_path, file_mode='w', encoding='utf-8'):
super().__init__(file_path, file_mode, encoding)
def export(self, data, **kwargs):
# Creates the XML file structure.
library = ElementTree.Element('library')
tracks = ElementTree.SubElement(library, 'tracks')
if data and isinstance(data[0], dict):
for index, item in enumerate(data):
track = ElementTree.SubElement(tracks, 'track')
for key, value in item.items():
track_details = ElementTree.SubElement(track, key)
track_details.text = value
# Depending on the version of python the encoding needs to change
try:
data = ElementTree.tostring(library, encoding='unicode', **kwargs)
except LookupError:
data = ElementTree.tostring(library, encoding='utf-8', **kwargs)
self.out_stream.write(data)

664
libs/common/beetsplug/fetchart.py
View file

File diff suppressed because it is too large Load diff

8
libs/common/beetsplug/filefilter.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Malte Ried. # Copyright 2016, Malte Ried.
# #
@ -16,7 +15,6 @@
"""Filter imported files using a regular expression. """Filter imported files using a regular expression.
""" """
from __future__ import division, absolute_import, print_function
import re import re
from beets import config from beets import config
@ -27,7 +25,7 @@ from beets.importer import SingletonImportTask
class FileFilterPlugin(BeetsPlugin): class FileFilterPlugin(BeetsPlugin):
def __init__(self): def __init__(self):
super(FileFilterPlugin, self).__init__() super().__init__()
self.register_listener('import_task_created', self.register_listener('import_task_created',
self.import_task_created_event) self.import_task_created_event)
self.config.add({ self.config.add({
@ -43,8 +41,8 @@ class FileFilterPlugin(BeetsPlugin):
bytestring_path(self.config['album_path'].get())) bytestring_path(self.config['album_path'].get()))
if 'singleton_path' in self.config: if 'singleton_path' in self.config:
self.path_singleton_regex = re.compile( self.path_singleton_regex = re.compile(
bytestring_path(self.config['singleton_path'].get())) bytestring_path(self.config['singleton_path'].get()))
def import_task_created_event(self, session, task): def import_task_created_event(self, session, task):
if task.items and len(task.items) > 0: if task.items and len(task.items) > 0:

285
libs/common/beetsplug/fish.py Normal file
View file

@ -0,0 +1,285 @@
# This file is part of beets.
# Copyright 2015, winters jean-marie.
# Copyright 2020, Justin Mayer <https://justinmayer.com>
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""This plugin generates tab completions for Beets commands for the Fish shell
<https://fishshell.com/>, including completions for Beets commands, plugin
commands, and option flags. Also generated are completions for all the album
and track fields, suggesting for example `genre:` or `album:` when querying the
Beets database. Completions for the *values* of those fields are not generated
by default but can be added via the `-e` / `--extravalues` flag. For example:
`beet fish -e genre -e albumartist`
"""
from beets.plugins import BeetsPlugin
from beets import library, ui
from beets.ui import commands
from operator import attrgetter
import os
BL_NEED2 = """complete -c beet -n '__fish_beet_needs_command' {} {}\n"""
BL_USE3 = """complete -c beet -n '__fish_beet_using_command {}' {} {}\n"""
BL_SUBS = """complete -c beet -n '__fish_at_level {} ""' {} {}\n"""
BL_EXTRA3 = """complete -c beet -n '__fish_beet_use_extra {}' {} {}\n"""
HEAD = '''
function __fish_beet_needs_command
set cmd (commandline -opc)
if test (count $cmd) -eq 1
return 0
end
return 1
end
function __fish_beet_using_command
set cmd (commandline -opc)
set needle (count $cmd)
if test $needle -gt 1
if begin test $argv[1] = $cmd[2];
and not contains -- $cmd[$needle] $FIELDS; end
return 0
end
end
return 1
end
function __fish_beet_use_extra
set cmd (commandline -opc)
set needle (count $cmd)
if test $argv[2] = $cmd[$needle]
return 0
end
return 1
end
'''
class FishPlugin(BeetsPlugin):
def commands(self):
cmd = ui.Subcommand('fish', help='generate Fish shell tab completions')
cmd.func = self.run
cmd.parser.add_option('-f', '--noFields', action='store_true',
default=False,
help='omit album/track field completions')
cmd.parser.add_option(
'-e',
'--extravalues',
action='append',
type='choice',
choices=library.Item.all_keys() +
library.Album.all_keys(),
help='include specified field *values* in completions')
return [cmd]
def run(self, lib, opts, args):
# Gather the commands from Beets core and its plugins.
# Collect the album and track fields.
# If specified, also collect the values for these fields.
# Make a giant string of all the above, formatted in a way that
# allows Fish to do tab completion for the `beet` command.
home_dir = os.path.expanduser("~")
completion_dir = os.path.join(home_dir, '.config/fish/completions')
try:
os.makedirs(completion_dir)
except OSError:
if not os.path.isdir(completion_dir):
raise
completion_file_path = os.path.join(completion_dir, 'beet.fish')
nobasicfields = opts.noFields # Do not complete for album/track fields
extravalues = opts.extravalues # e.g., Also complete artists names
beetcmds = sorted(
(commands.default_commands +
commands.plugins.commands()),
key=attrgetter('name'))
fields = sorted(set(
library.Album.all_keys() + library.Item.all_keys()))
# Collect commands, their aliases, and their help text
cmd_names_help = []
for cmd in beetcmds:
names = list(cmd.aliases)
names.append(cmd.name)
for name in names:
cmd_names_help.append((name, cmd.help))
# Concatenate the string
totstring = HEAD + "\n"
totstring += get_cmds_list([name[0] for name in cmd_names_help])
totstring += '' if nobasicfields else get_standard_fields(fields)
totstring += get_extravalues(lib, extravalues) if extravalues else ''
totstring += "\n" + "# ====== {} =====".format(
"setup basic beet completion") + "\n" * 2
totstring += get_basic_beet_options()
totstring += "\n" + "# ====== {} =====".format(
"setup field completion for subcommands") + "\n"
totstring += get_subcommands(
cmd_names_help, nobasicfields, extravalues)
# Set up completion for all the command options
totstring += get_all_commands(beetcmds)
with open(completion_file_path, 'w') as fish_file:
fish_file.write(totstring)
def _escape(name):
# Escape ? in fish
if name == "?":
name = "\\" + name
return name
def get_cmds_list(cmds_names):
# Make a list of all Beets core & plugin commands
substr = ''
substr += (
"set CMDS " + " ".join(cmds_names) + ("\n" * 2)
)
return substr
def get_standard_fields(fields):
# Make a list of album/track fields and append with ':'
fields = (field + ":" for field in fields)
substr = ''
substr += (
"set FIELDS " + " ".join(fields) + ("\n" * 2)
)
return substr
def get_extravalues(lib, extravalues):
# Make a list of all values from an album/track field.
# 'beet ls albumartist: <TAB>' yields completions for ABBA, Beatles, etc.
word = ''
values_set = get_set_of_values_for_field(lib, extravalues)
for fld in extravalues:
extraname = fld.upper() + 'S'
word += (
"set " + extraname + " " + " ".join(sorted(values_set[fld]))
+ ("\n" * 2)
)
return word
def get_set_of_values_for_field(lib, fields):
# Get unique values from a specified album/track field
fields_dict = {}
for each in fields:
fields_dict[each] = set()
for item in lib.items():
for field in fields:
fields_dict[field].add(wrap(item[field]))
return fields_dict
def get_basic_beet_options():
word = (
BL_NEED2.format("-l format-item",
"-f -d 'print with custom format'") +
BL_NEED2.format("-l format-album",
"-f -d 'print with custom format'") +
BL_NEED2.format("-s l -l library",
"-f -r -d 'library database file to use'") +
BL_NEED2.format("-s d -l directory",
"-f -r -d 'destination music directory'") +
BL_NEED2.format("-s v -l verbose",
"-f -d 'print debugging information'") +
BL_NEED2.format("-s c -l config",
"-f -r -d 'path to configuration file'") +
BL_NEED2.format("-s h -l help",
"-f -d 'print this help message and exit'"))
return word
def get_subcommands(cmd_name_and_help, nobasicfields, extravalues):
# Formatting for Fish to complete our fields/values
word = ""
for cmdname, cmdhelp in cmd_name_and_help:
cmdname = _escape(cmdname)
word += "\n" + "# ------ {} -------".format(
"fieldsetups for " + cmdname) + "\n"
word += (
BL_NEED2.format(
("-a " + cmdname),
("-f " + "-d " + wrap(clean_whitespace(cmdhelp)))))
if nobasicfields is False:
word += (
BL_USE3.format(
cmdname,
("-a " + wrap("$FIELDS")),
("-f " + "-d " + wrap("fieldname"))))
if extravalues:
for f in extravalues:
setvar = wrap("$" + f.upper() + "S")
word += " ".join(BL_EXTRA3.format(
(cmdname + " " + f + ":"),
('-f ' + '-A ' + '-a ' + setvar),
('-d ' + wrap(f))).split()) + "\n"
return word
def get_all_commands(beetcmds):
# Formatting for Fish to complete command options
word = ""
for cmd in beetcmds:
names = list(cmd.aliases)
names.append(cmd.name)
for name in names:
name = _escape(name)
word += "\n"
word += ("\n" * 2) + "# ====== {} =====".format(
"completions for " + name) + "\n"
for option in cmd.parser._get_all_options()[1:]:
cmd_l = (" -l " + option._long_opts[0].replace('--', '')
)if option._long_opts else ''
cmd_s = (" -s " + option._short_opts[0].replace('-', '')
) if option._short_opts else ''
cmd_need_arg = ' -r ' if option.nargs in [1] else ''
cmd_helpstr = (" -d " + wrap(' '.join(option.help.split()))
) if option.help else ''
cmd_arglist = (' -a ' + wrap(" ".join(option.choices))
) if option.choices else ''
word += " ".join(BL_USE3.format(
name,
(cmd_need_arg + cmd_s + cmd_l + " -f " + cmd_arglist),
cmd_helpstr).split()) + "\n"
word = (word + " ".join(BL_USE3.format(
name,
("-s " + "h " + "-l " + "help" + " -f "),
('-d ' + wrap("print help") + "\n")
).split()))
return word
def clean_whitespace(word):
# Remove excess whitespace and tabs in a string
return " ".join(word.split())
def wrap(word):
# Need " or ' around strings but watch out if they're in the string
sptoken = '\"'
if ('"') in word and ("'") in word:
word.replace('"', sptoken)
return '"' + word + '"'
tok = '"' if "'" in word else "'"
return tok + word + tok

12
libs/common/beetsplug/freedesktop.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Matt Lichtenberg. # Copyright 2016, Matt Lichtenberg.
# #
@ -16,7 +15,6 @@
"""Creates freedesktop.org-compliant .directory files on an album level. """Creates freedesktop.org-compliant .directory files on an album level.
""" """
from __future__ import division, absolute_import, print_function
from beets.plugins import BeetsPlugin from beets.plugins import BeetsPlugin
from beets import ui from beets import ui
@ -26,12 +24,12 @@ class FreedesktopPlugin(BeetsPlugin):
def commands(self): def commands(self):
deprecated = ui.Subcommand( deprecated = ui.Subcommand(
"freedesktop", "freedesktop",
help=u"Print a message to redirect to thumbnails --dolphin") help="Print a message to redirect to thumbnails --dolphin")
deprecated.func = self.deprecation_message deprecated.func = self.deprecation_message
return [deprecated] return [deprecated]
def deprecation_message(self, lib, opts, args): def deprecation_message(self, lib, opts, args):
ui.print_(u"This plugin is deprecated. Its functionality is " ui.print_("This plugin is deprecated. Its functionality is "
u"superseded by the 'thumbnails' plugin") "superseded by the 'thumbnails' plugin")
ui.print_(u"'thumbnails --dolphin' replaces freedesktop. See doc & " ui.print_("'thumbnails --dolphin' replaces freedesktop. See doc & "
u"changelog for more information") "changelog for more information")

7
libs/common/beetsplug/fromfilename.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Jan-Erik Dahlin # Copyright 2016, Jan-Erik Dahlin
# #
@ -16,13 +15,11 @@
"""If the title is empty, try to extract track and title from the """If the title is empty, try to extract track and title from the
filename. filename.
""" """
from __future__ import division, absolute_import, print_function
from beets import plugins from beets import plugins
from beets.util import displayable_path from beets.util import displayable_path
import os import os
import re import re
import six
# Filename field extraction patterns. # Filename field extraction patterns.
@ -124,7 +121,7 @@ def apply_matches(d):
# Apply the title and track. # Apply the title and track.
for item in d: for item in d:
if bad_title(item.title): if bad_title(item.title):
item.title = six.text_type(d[item][title_field]) item.title = str(d[item][title_field])
if 'track' in d[item] and item.track == 0: if 'track' in d[item] and item.track == 0:
item.track = int(d[item]['track']) item.track = int(d[item]['track'])
@ -133,7 +130,7 @@ def apply_matches(d):
class FromFilenamePlugin(plugins.BeetsPlugin): class FromFilenamePlugin(plugins.BeetsPlugin):
def __init__(self): def __init__(self):
super(FromFilenamePlugin, self).__init__() super().__init__()
self.register_listener('import_task_start', filename_task) self.register_listener('import_task_start', filename_task)

20
libs/common/beetsplug/ftintitle.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Verrus, <github.com/Verrus/beets-plugin-featInTitle> # Copyright 2016, Verrus, <github.com/Verrus/beets-plugin-featInTitle>
# #
@ -15,7 +14,6 @@
"""Moves "featured" artists to the title from the artist field. """Moves "featured" artists to the title from the artist field.
""" """
from __future__ import division, absolute_import, print_function
import re import re
@ -75,22 +73,22 @@ def find_feat_part(artist, albumartist):
class FtInTitlePlugin(plugins.BeetsPlugin): class FtInTitlePlugin(plugins.BeetsPlugin):
def __init__(self): def __init__(self):
super(FtInTitlePlugin, self).__init__() super().__init__()
self.config.add({ self.config.add({
'auto': True, 'auto': True,
'drop': False, 'drop': False,
'format': u'feat. {0}', 'format': 'feat. {0}',
}) })
self._command = ui.Subcommand( self._command = ui.Subcommand(
'ftintitle', 'ftintitle',
help=u'move featured artists to the title field') help='move featured artists to the title field')
self._command.parser.add_option( self._command.parser.add_option(
u'-d', u'--drop', dest='drop', '-d', '--drop', dest='drop',
action='store_true', default=None, action='store_true', default=None,
help=u'drop featuring from artists and ignore title update') help='drop featuring from artists and ignore title update')
if self.config['auto']: if self.config['auto']:
self.import_stages = [self.imported] self.import_stages = [self.imported]
@ -127,7 +125,7 @@ class FtInTitlePlugin(plugins.BeetsPlugin):
remove it from the artist field. remove it from the artist field.
""" """
# In all cases, update the artist fields. # In all cases, update the artist fields.
self._log.info(u'artist: {0} -> {1}', item.artist, item.albumartist) self._log.info('artist: {0} -> {1}', item.artist, item.albumartist)
item.artist = item.albumartist item.artist = item.albumartist
if item.artist_sort: if item.artist_sort:
# Just strip the featured artist from the sort name. # Just strip the featured artist from the sort name.
@ -138,8 +136,8 @@ class FtInTitlePlugin(plugins.BeetsPlugin):
if not drop_feat and not contains_feat(item.title): if not drop_feat and not contains_feat(item.title):
feat_format = self.config['format'].as_str() feat_format = self.config['format'].as_str()
new_format = feat_format.format(feat_part) new_format = feat_format.format(feat_part)
new_title = u"{0} {1}".format(item.title, new_format) new_title = f"{item.title} {new_format}"
self._log.info(u'title: {0} -> {1}', item.title, new_title) self._log.info('title: {0} -> {1}', item.title, new_title)
item.title = new_title item.title = new_title
def ft_in_title(self, item, drop_feat): def ft_in_title(self, item, drop_feat):
@ -165,4 +163,4 @@ class FtInTitlePlugin(plugins.BeetsPlugin):
if feat_part: if feat_part:
self.update_metadata(item, feat_part, drop_feat) self.update_metadata(item, feat_part, drop_feat)
else: else:
self._log.info(u'no featuring artists found') self._log.info('no featuring artists found')

4
libs/common/beetsplug/fuzzy.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Philippe Mongeau. # Copyright 2016, Philippe Mongeau.
# #
@ -16,7 +15,6 @@
"""Provides a fuzzy matching query. """Provides a fuzzy matching query.
""" """
from __future__ import division, absolute_import, print_function
from beets.plugins import BeetsPlugin from beets.plugins import BeetsPlugin
from beets.dbcore.query import StringFieldQuery from beets.dbcore.query import StringFieldQuery
@ -37,7 +35,7 @@ class FuzzyQuery(StringFieldQuery):
class FuzzyPlugin(BeetsPlugin): class FuzzyPlugin(BeetsPlugin):
def __init__(self): def __init__(self):
super(FuzzyPlugin, self).__init__() super().__init__()
self.config.add({ self.config.add({
'prefix': '~', 'prefix': '~',
'threshold': 0.7, 'threshold': 0.7,

81
libs/common/beetsplug/gmusic.py
View file

@ -1,6 +1,4 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2017, Tigran Kostandyan.
# #
# Permission is hereby granted, free of charge, to any person obtaining # Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the # a copy of this software and associated documentation files (the
@ -13,84 +11,15 @@
# The above copyright notice and this permission notice shall be # The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software. # included in all copies or substantial portions of the Software.
"""Upload files to Google Play Music and list songs in its library.""" """Deprecation warning for the removed gmusic plugin."""
from __future__ import absolute_import, division, print_function
import os.path
from beets.plugins import BeetsPlugin from beets.plugins import BeetsPlugin
from beets import ui
from beets import config
from beets.ui import Subcommand
from gmusicapi import Musicmanager, Mobileclient
from gmusicapi.exceptions import NotLoggedIn
import gmusicapi.clients
class Gmusic(BeetsPlugin): class Gmusic(BeetsPlugin):
def __init__(self): def __init__(self):
super(Gmusic, self).__init__() super().__init__()
# Checks for OAuth2 credentials,
# if they don't exist - performs authorization
self.m = Musicmanager()
if os.path.isfile(gmusicapi.clients.OAUTH_FILEPATH):
self.m.login()
else:
self.m.perform_oauth()
def commands(self): self._log.warning("The 'gmusic' plugin has been removed following the"
gupload = Subcommand('gmusic-upload', " shutdown of Google Play Music. Remove the plugin"
help=u'upload your tracks to Google Play Music') " from your configuration to silence this warning.")
gupload.func = self.upload
search = Subcommand('gmusic-songs',
help=u'list of songs in Google Play Music library'
)
search.parser.add_option('-t', '--track', dest='track',
action='store_true',
help='Search by track name')
search.parser.add_option('-a', '--artist', dest='artist',
action='store_true',
help='Search by artist')
search.func = self.search
return [gupload, search]
def upload(self, lib, opts, args):
items = lib.items(ui.decargs(args))
files = [x.path.decode('utf-8') for x in items]
ui.print_(u'Uploading your files...')
self.m.upload(filepaths=files)
ui.print_(u'Your files were successfully added to library')
def search(self, lib, opts, args):
password = config['gmusic']['password']
email = config['gmusic']['email']
password.redact = True
email.redact = True
# Since Musicmanager doesn't support library management
# we need to use mobileclient interface
mobile = Mobileclient()
try:
mobile.login(email.as_str(), password.as_str(),
Mobileclient.FROM_MAC_ADDRESS)
files = mobile.get_all_songs()
except NotLoggedIn:
ui.print_(
u'Authentication error. Please check your email and password.'
)
return
if not args:
for i, file in enumerate(files, start=1):
print(i, ui.colorize('blue', file['artist']),
file['title'], ui.colorize('red', file['album']))
else:
if opts.track:
self.match(files, args, 'title')
else:
self.match(files, args, 'artist')
@staticmethod
def match(files, args, search_by):
for file in files:
if ' '.join(ui.decargs(args)) in file[search_by]:
print(file['artist'], file['title'], file['album'])

57
libs/common/beetsplug/hook.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2015, Adrian Sampson. # Copyright 2015, Adrian Sampson.
# #
@ -14,14 +13,13 @@
# included in all copies or substantial portions of the Software. # included in all copies or substantial portions of the Software.
"""Allows custom commands to be run when an event is emitted by beets""" """Allows custom commands to be run when an event is emitted by beets"""
from __future__ import division, absolute_import, print_function
import string import string
import subprocess import subprocess
import six import shlex
from beets.plugins import BeetsPlugin from beets.plugins import BeetsPlugin
from beets.util import shlex_split, arg_encoding from beets.util import arg_encoding
class CodingFormatter(string.Formatter): class CodingFormatter(string.Formatter):
@ -46,13 +44,11 @@ class CodingFormatter(string.Formatter):
See str.format and string.Formatter.format. See str.format and string.Formatter.format.
""" """
try: if isinstance(format_string, bytes):
format_string = format_string.decode(self._coding) format_string = format_string.decode(self._coding)
except UnicodeEncodeError:
pass
return super(CodingFormatter, self).format(format_string, *args, return super().format(format_string, *args,
**kwargs) **kwargs)
def convert_field(self, value, conversion): def convert_field(self, value, conversion):
"""Converts the provided value given a conversion type. """Converts the provided value given a conversion type.
@ -61,8 +57,8 @@ class CodingFormatter(string.Formatter):
See string.Formatter.convert_field. See string.Formatter.convert_field.
""" """
converted = super(CodingFormatter, self).convert_field(value, converted = super().convert_field(value,
conversion) conversion)
if isinstance(converted, bytes): if isinstance(converted, bytes):
return converted.decode(self._coding) return converted.decode(self._coding)
@ -72,8 +68,9 @@ class CodingFormatter(string.Formatter):
class HookPlugin(BeetsPlugin): class HookPlugin(BeetsPlugin):
"""Allows custom commands to be run when an event is emitted by beets""" """Allows custom commands to be run when an event is emitted by beets"""
def __init__(self): def __init__(self):
super(HookPlugin, self).__init__() super().__init__()
self.config.add({ self.config.add({
'hooks': [] 'hooks': []
@ -91,28 +88,28 @@ class HookPlugin(BeetsPlugin):
def create_and_register_hook(self, event, command): def create_and_register_hook(self, event, command):
def hook_function(**kwargs): def hook_function(**kwargs):
if command is None or len(command) == 0: if command is None or len(command) == 0:
self._log.error('invalid command "{0}"', command) self._log.error('invalid command "{0}"', command)
return return
# Use a string formatter that works on Unicode strings. # Use a string formatter that works on Unicode strings.
if six.PY2: formatter = CodingFormatter(arg_encoding())
formatter = CodingFormatter(arg_encoding())
else:
formatter = string.Formatter()
command_pieces = shlex_split(command) command_pieces = shlex.split(command)
for i, piece in enumerate(command_pieces): for i, piece in enumerate(command_pieces):
command_pieces[i] = formatter.format(piece, event=event, command_pieces[i] = formatter.format(piece, event=event,
**kwargs) **kwargs)
self._log.debug(u'running command "{0}" for event {1}', self._log.debug('running command "{0}" for event {1}',
u' '.join(command_pieces), event) ' '.join(command_pieces), event)
try: try:
subprocess.Popen(command_pieces).wait() subprocess.check_call(command_pieces)
except OSError as exc: except subprocess.CalledProcessError as exc:
self._log.error(u'hook for {0} failed: {1}', event, exc) self._log.error('hook for {0} exited with status {1}',
event, exc.returncode)
except OSError as exc:
self._log.error('hook for {0} failed: {1}', event, exc)
self.register_listener(event, hook_function) self.register_listener(event, hook_function)

18
libs/common/beetsplug/ihate.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Blemjhoo Tezoulbr <baobab@heresiarch.info>. # Copyright 2016, Blemjhoo Tezoulbr <baobab@heresiarch.info>.
# #
@ -13,7 +12,6 @@
# The above copyright notice and this permission notice shall be # The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software. # included in all copies or substantial portions of the Software.
from __future__ import division, absolute_import, print_function
"""Warns you about things you hate (or even blocks import).""" """Warns you about things you hate (or even blocks import)."""
@ -33,14 +31,14 @@ def summary(task):
object. object.
""" """
if task.is_album: if task.is_album:
return u'{0} - {1}'.format(task.cur_artist, task.cur_album) return f'{task.cur_artist} - {task.cur_album}'
else: else:
return u'{0} - {1}'.format(task.item.artist, task.item.title) return f'{task.item.artist} - {task.item.title}'
class IHatePlugin(BeetsPlugin): class IHatePlugin(BeetsPlugin):
def __init__(self): def __init__(self):
super(IHatePlugin, self).__init__() super().__init__()
self.register_listener('import_task_choice', self.register_listener('import_task_choice',
self.import_task_choice_event) self.import_task_choice_event)
self.config.add({ self.config.add({
@ -69,14 +67,14 @@ class IHatePlugin(BeetsPlugin):
if task.choice_flag == action.APPLY: if task.choice_flag == action.APPLY:
if skip_queries or warn_queries: if skip_queries or warn_queries:
self._log.debug(u'processing your hate') self._log.debug('processing your hate')
if self.do_i_hate_this(task, skip_queries): if self.do_i_hate_this(task, skip_queries):
task.choice_flag = action.SKIP task.choice_flag = action.SKIP
self._log.info(u'skipped: {0}', summary(task)) self._log.info('skipped: {0}', summary(task))
return return
if self.do_i_hate_this(task, warn_queries): if self.do_i_hate_this(task, warn_queries):
self._log.info(u'you may hate this: {0}', summary(task)) self._log.info('you may hate this: {0}', summary(task))
else: else:
self._log.debug(u'nothing to do') self._log.debug('nothing to do')
else: else:
self._log.debug(u'user made a decision, nothing to do') self._log.debug('user made a decision, nothing to do')

39
libs/common/beetsplug/importadded.py
View file

@ -1,11 +1,8 @@
# -*- coding: utf-8 -*-
"""Populate an item's `added` and `mtime` fields by using the file """Populate an item's `added` and `mtime` fields by using the file
modification time (mtime) of the item's source file before import. modification time (mtime) of the item's source file before import.
Reimported albums and items are skipped. Reimported albums and items are skipped.
""" """
from __future__ import division, absolute_import, print_function
import os import os
@ -16,7 +13,7 @@ from beets.plugins import BeetsPlugin
class ImportAddedPlugin(BeetsPlugin): class ImportAddedPlugin(BeetsPlugin):
def __init__(self): def __init__(self):
super(ImportAddedPlugin, self).__init__() super().__init__()
self.config.add({ self.config.add({
'preserve_mtimes': False, 'preserve_mtimes': False,
'preserve_write_mtimes': False, 'preserve_write_mtimes': False,
@ -27,7 +24,7 @@ class ImportAddedPlugin(BeetsPlugin):
# album.path for old albums that were replaced by a reimported album # album.path for old albums that were replaced by a reimported album
self.replaced_album_paths = None self.replaced_album_paths = None
# item path in the library to the mtime of the source file # item path in the library to the mtime of the source file
self.item_mtime = dict() self.item_mtime = {}
register = self.register_listener register = self.register_listener
register('import_task_created', self.check_config) register('import_task_created', self.check_config)
@ -53,8 +50,8 @@ class ImportAddedPlugin(BeetsPlugin):
def record_if_inplace(self, task, session): def record_if_inplace(self, task, session):
if not (session.config['copy'] or session.config['move'] or if not (session.config['copy'] or session.config['move'] or
session.config['link'] or session.config['hardlink']): session.config['link'] or session.config['hardlink']):
self._log.debug(u"In place import detected, recording mtimes from " self._log.debug("In place import detected, recording mtimes from "
u"source paths") "source paths")
items = [task.item] \ items = [task.item] \
if isinstance(task, importer.SingletonImportTask) \ if isinstance(task, importer.SingletonImportTask) \
else task.items else task.items
@ -62,9 +59,9 @@ class ImportAddedPlugin(BeetsPlugin):
self.record_import_mtime(item, item.path, item.path) self.record_import_mtime(item, item.path, item.path)
def record_reimported(self, task, session): def record_reimported(self, task, session):
self.reimported_item_ids = set(item.id for item, replaced_items self.reimported_item_ids = {item.id for item, replaced_items
in task.replaced_items.items() in task.replaced_items.items()
if replaced_items) if replaced_items}
self.replaced_album_paths = set(task.replaced_albums.keys()) self.replaced_album_paths = set(task.replaced_albums.keys())
def write_file_mtime(self, path, mtime): def write_file_mtime(self, path, mtime):
@ -86,14 +83,14 @@ class ImportAddedPlugin(BeetsPlugin):
""" """
mtime = os.stat(util.syspath(source)).st_mtime mtime = os.stat(util.syspath(source)).st_mtime
self.item_mtime[destination] = mtime self.item_mtime[destination] = mtime
self._log.debug(u"Recorded mtime {0} for item '{1}' imported from " self._log.debug("Recorded mtime {0} for item '{1}' imported from "
u"'{2}'", mtime, util.displayable_path(destination), "'{2}'", mtime, util.displayable_path(destination),
util.displayable_path(source)) util.displayable_path(source))
def update_album_times(self, lib, album): def update_album_times(self, lib, album):
if self.reimported_album(album): if self.reimported_album(album):
self._log.debug(u"Album '{0}' is reimported, skipping import of " self._log.debug("Album '{0}' is reimported, skipping import of "
u"added dates for the album and its items.", "added dates for the album and its items.",
util.displayable_path(album.path)) util.displayable_path(album.path))
return return
@ -106,30 +103,30 @@ class ImportAddedPlugin(BeetsPlugin):
self.write_item_mtime(item, mtime) self.write_item_mtime(item, mtime)
item.store() item.store()
album.added = min(album_mtimes) album.added = min(album_mtimes)
self._log.debug(u"Import of album '{0}', selected album.added={1} " self._log.debug("Import of album '{0}', selected album.added={1} "
u"from item file mtimes.", album.album, album.added) "from item file mtimes.", album.album, album.added)
album.store() album.store()
def update_item_times(self, lib, item): def update_item_times(self, lib, item):
if self.reimported_item(item): if self.reimported_item(item):
self._log.debug(u"Item '{0}' is reimported, skipping import of " self._log.debug("Item '{0}' is reimported, skipping import of "
u"added date.", util.displayable_path(item.path)) "added date.", util.displayable_path(item.path))
return return
mtime = self.item_mtime.pop(item.path, None) mtime = self.item_mtime.pop(item.path, None)
if mtime: if mtime:
item.added = mtime item.added = mtime
if self.config['preserve_mtimes'].get(bool): if self.config['preserve_mtimes'].get(bool):
self.write_item_mtime(item, mtime) self.write_item_mtime(item, mtime)
self._log.debug(u"Import of item '{0}', selected item.added={1}", self._log.debug("Import of item '{0}', selected item.added={1}",
util.displayable_path(item.path), item.added) util.displayable_path(item.path), item.added)
item.store() item.store()
def update_after_write_time(self, item): def update_after_write_time(self, item, path):
"""Update the mtime of the item's file with the item.added value """Update the mtime of the item's file with the item.added value
after each write of the item if `preserve_write_mtimes` is enabled. after each write of the item if `preserve_write_mtimes` is enabled.
""" """
if item.added: if item.added:
if self.config['preserve_write_mtimes'].get(bool): if self.config['preserve_write_mtimes'].get(bool):
self.write_item_mtime(item, item.added) self.write_item_mtime(item, item.added)
self._log.debug(u"Write of item '{0}', selected item.added={1}", self._log.debug("Write of item '{0}', selected item.added={1}",
util.displayable_path(item.path), item.added) util.displayable_path(item.path), item.added)

10
libs/common/beetsplug/importfeeds.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Fabrice Laporte. # Copyright 2016, Fabrice Laporte.
# #
@ -13,7 +12,6 @@
# The above copyright notice and this permission notice shall be # The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software. # included in all copies or substantial portions of the Software.
from __future__ import division, absolute_import, print_function
"""Write paths of imported files in various formats to ease later import in a """Write paths of imported files in various formats to ease later import in a
music player. Also allow printing the new file locations to stdout in case music player. Also allow printing the new file locations to stdout in case
@ -54,11 +52,11 @@ def _write_m3u(m3u_path, items_paths):
class ImportFeedsPlugin(BeetsPlugin): class ImportFeedsPlugin(BeetsPlugin):
def __init__(self): def __init__(self):
super(ImportFeedsPlugin, self).__init__() super().__init__()
self.config.add({ self.config.add({
'formats': [], 'formats': [],
'm3u_name': u'imported.m3u', 'm3u_name': 'imported.m3u',
'dir': None, 'dir': None,
'relative_to': None, 'relative_to': None,
'absolute_path': False, 'absolute_path': False,
@ -118,9 +116,9 @@ class ImportFeedsPlugin(BeetsPlugin):
link(path, dest) link(path, dest)
if 'echo' in formats: if 'echo' in formats:
self._log.info(u"Location of imported music:") self._log.info("Location of imported music:")
for path in paths: for path in paths:
self._log.info(u" {0}", path) self._log.info(" {0}", path)
def album_imported(self, lib, album): def album_imported(self, lib, album):
self._record_items(lib, album.album, album.items()) self._record_items(lib, album.album, album.items())

117
libs/common/beetsplug/info.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson. # Copyright 2016, Adrian Sampson.
# #
@ -16,19 +15,17 @@
"""Shows file metadata. """Shows file metadata.
""" """
from __future__ import division, absolute_import, print_function
import os import os
import re
from beets.plugins import BeetsPlugin from beets.plugins import BeetsPlugin
from beets import ui from beets import ui
from beets import mediafile import mediafile
from beets.library import Item from beets.library import Item
from beets.util import displayable_path, normpath, syspath from beets.util import displayable_path, normpath, syspath
def tag_data(lib, args): def tag_data(lib, args, album=False):
query = [] query = []
for arg in args: for arg in args:
path = normpath(arg) path = normpath(arg)
@ -42,15 +39,29 @@ def tag_data(lib, args):
yield tag_data_emitter(item.path) yield tag_data_emitter(item.path)
def tag_fields():
fields = set(mediafile.MediaFile.readable_fields())
fields.add('art')
return fields
def tag_data_emitter(path): def tag_data_emitter(path):
def emitter(): def emitter(included_keys):
fields = list(mediafile.MediaFile.readable_fields()) if included_keys == '*':
fields.remove('images') fields = tag_fields()
else:
fields = included_keys
if 'images' in fields:
# We can't serialize the image data.
fields.remove('images')
mf = mediafile.MediaFile(syspath(path)) mf = mediafile.MediaFile(syspath(path))
tags = {} tags = {}
for field in fields: for field in fields:
tags[field] = getattr(mf, field) if field == 'art':
tags['art'] = mf.art is not None tags[field] = mf.art is not None
else:
tags[field] = getattr(mf, field, None)
# create a temporary Item to take advantage of __format__ # create a temporary Item to take advantage of __format__
item = Item.from_path(syspath(path)) item = Item.from_path(syspath(path))
@ -58,15 +69,14 @@ def tag_data_emitter(path):
return emitter return emitter
def library_data(lib, args): def library_data(lib, args, album=False):
for item in lib.items(args): for item in lib.albums(args) if album else lib.items(args):
yield library_data_emitter(item) yield library_data_emitter(item)
def library_data_emitter(item): def library_data_emitter(item):
def emitter(): def emitter(included_keys):
data = dict(item.formatted()) data = dict(item.formatted(included_keys=included_keys))
data.pop('path', None) # path is fetched from item
return data, item return data, item
return emitter return emitter
@ -98,7 +108,7 @@ def print_data(data, item=None, fmt=None):
formatted = {} formatted = {}
for key, value in data.items(): for key, value in data.items():
if isinstance(value, list): if isinstance(value, list):
formatted[key] = u'; '.join(value) formatted[key] = '; '.join(value)
if value is not None: if value is not None:
formatted[key] = value formatted[key] = value
@ -106,7 +116,7 @@ def print_data(data, item=None, fmt=None):
return return
maxwidth = max(len(key) for key in formatted) maxwidth = max(len(key) for key in formatted)
lineformat = u'{{0:>{0}}}: {{1}}'.format(maxwidth) lineformat = f'{{0:>{maxwidth}}}: {{1}}'
if path: if path:
ui.print_(displayable_path(path)) ui.print_(displayable_path(path))
@ -114,7 +124,7 @@ def print_data(data, item=None, fmt=None):
for field in sorted(formatted): for field in sorted(formatted):
value = formatted[field] value = formatted[field]
if isinstance(value, list): if isinstance(value, list):
value = u'; '.join(value) value = '; '.join(value)
ui.print_(lineformat.format(field, value)) ui.print_(lineformat.format(field, value))
@ -129,7 +139,7 @@ def print_data_keys(data, item=None):
if len(formatted) == 0: if len(formatted) == 0:
return return
line_format = u'{0}{{0}}'.format(u' ' * 4) line_format = '{0}{{0}}'.format(' ' * 4)
if path: if path:
ui.print_(displayable_path(path)) ui.print_(displayable_path(path))
@ -140,24 +150,28 @@ def print_data_keys(data, item=None):
class InfoPlugin(BeetsPlugin): class InfoPlugin(BeetsPlugin):
def commands(self): def commands(self):
cmd = ui.Subcommand('info', help=u'show file metadata') cmd = ui.Subcommand('info', help='show file metadata')
cmd.func = self.run cmd.func = self.run
cmd.parser.add_option( cmd.parser.add_option(
u'-l', u'--library', action='store_true', '-l', '--library', action='store_true',
help=u'show library fields instead of tags', help='show library fields instead of tags',
) )
cmd.parser.add_option( cmd.parser.add_option(
u'-s', u'--summarize', action='store_true', '-a', '--album', action='store_true',
help=u'summarize the tags of all files', help='show album fields instead of tracks (implies "--library")',
) )
cmd.parser.add_option( cmd.parser.add_option(
u'-i', u'--include-keys', default=[], '-s', '--summarize', action='store_true',
help='summarize the tags of all files',
)
cmd.parser.add_option(
'-i', '--include-keys', default=[],
action='append', dest='included_keys', action='append', dest='included_keys',
help=u'comma separated list of keys to show', help='comma separated list of keys to show',
) )
cmd.parser.add_option( cmd.parser.add_option(
u'-k', u'--keys-only', action='store_true', '-k', '--keys-only', action='store_true',
help=u'show only the keys', help='show only the keys',
) )
cmd.parser.add_format_option(target='item') cmd.parser.add_format_option(target='item')
return [cmd] return [cmd]
@ -176,7 +190,7 @@ class InfoPlugin(BeetsPlugin):
dictionary and only prints that. If two files have different values dictionary and only prints that. If two files have different values
for the same tag, the value is set to '[various]' for the same tag, the value is set to '[various]'
""" """
if opts.library: if opts.library or opts.album:
data_collector = library_data data_collector = library_data
else: else:
data_collector = tag_data data_collector = tag_data
@ -184,18 +198,21 @@ class InfoPlugin(BeetsPlugin):
included_keys = [] included_keys = []
for keys in opts.included_keys: for keys in opts.included_keys:
included_keys.extend(keys.split(',')) included_keys.extend(keys.split(','))
key_filter = make_key_filter(included_keys) # Drop path even if user provides it multiple times
included_keys = [k for k in included_keys if k != 'path']
first = True first = True
summary = {} summary = {}
for data_emitter in data_collector(lib, ui.decargs(args)): for data_emitter in data_collector(
lib, ui.decargs(args),
album=opts.album,
):
try: try:
data, item = data_emitter() data, item = data_emitter(included_keys or '*')
except (mediafile.UnreadableFileError, IOError) as ex: except (mediafile.UnreadableFileError, OSError) as ex:
self._log.error(u'cannot read file: {0}', ex) self._log.error('cannot read file: {0}', ex)
continue continue
data = key_filter(data)
if opts.summarize: if opts.summarize:
update_summary(summary, data) update_summary(summary, data)
else: else:
@ -210,33 +227,3 @@ class InfoPlugin(BeetsPlugin):
if opts.summarize: if opts.summarize:
print_data(summary) print_data(summary)
def make_key_filter(include):
"""Return a function that filters a dictionary.
The returned filter takes a dictionary and returns another
dictionary that only includes the key-value pairs where the key
glob-matches one of the keys in `include`.
"""
if not include:
return identity
matchers = []
for key in include:
key = re.escape(key)
key = key.replace(r'\*', '.*')
matchers.append(re.compile(key + '$'))
def filter_(data):
filtered = dict()
for key, value in data.items():
if any([m.match(key) for m in matchers]):
filtered[key] = value
return filtered
return filter_
def identity(val):
return val

29
libs/common/beetsplug/inline.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson. # Copyright 2016, Adrian Sampson.
# #
@ -15,25 +14,23 @@
"""Allows inline path template customization code in the config file. """Allows inline path template customization code in the config file.
""" """
from __future__ import division, absolute_import, print_function
import traceback import traceback
import itertools import itertools
from beets.plugins import BeetsPlugin from beets.plugins import BeetsPlugin
from beets import config from beets import config
import six
FUNC_NAME = u'__INLINE_FUNC__' FUNC_NAME = '__INLINE_FUNC__'
class InlineError(Exception): class InlineError(Exception):
"""Raised when a runtime error occurs in an inline expression. """Raised when a runtime error occurs in an inline expression.
""" """
def __init__(self, code, exc): def __init__(self, code, exc):
super(InlineError, self).__init__( super().__init__(
(u"error in inline path field code:\n" ("error in inline path field code:\n"
u"%s\n%s: %s") % (code, type(exc).__name__, six.text_type(exc)) "%s\n%s: %s") % (code, type(exc).__name__, str(exc))
) )
@ -41,7 +38,7 @@ def _compile_func(body):
"""Given Python code for a function body, return a compiled """Given Python code for a function body, return a compiled
callable that invokes that code. callable that invokes that code.
""" """
body = u'def {0}():\n {1}'.format( body = 'def {}():\n {}'.format(
FUNC_NAME, FUNC_NAME,
body.replace('\n', '\n ') body.replace('\n', '\n ')
) )
@ -53,7 +50,7 @@ def _compile_func(body):
class InlinePlugin(BeetsPlugin): class InlinePlugin(BeetsPlugin):
def __init__(self): def __init__(self):
super(InlinePlugin, self).__init__() super().__init__()
config.add({ config.add({
'pathfields': {}, # Legacy name. 'pathfields': {}, # Legacy name.
@ -64,14 +61,14 @@ class InlinePlugin(BeetsPlugin):
# Item fields. # Item fields.
for key, view in itertools.chain(config['item_fields'].items(), for key, view in itertools.chain(config['item_fields'].items(),
config['pathfields'].items()): config['pathfields'].items()):
self._log.debug(u'adding item field {0}', key) self._log.debug('adding item field {0}', key)
func = self.compile_inline(view.as_str(), False) func = self.compile_inline(view.as_str(), False)
if func is not None: if func is not None:
self.template_fields[key] = func self.template_fields[key] = func
# Album fields. # Album fields.
for key, view in config['album_fields'].items(): for key, view in config['album_fields'].items():
self._log.debug(u'adding album field {0}', key) self._log.debug('adding album field {0}', key)
func = self.compile_inline(view.as_str(), True) func = self.compile_inline(view.as_str(), True)
if func is not None: if func is not None:
self.album_template_fields[key] = func self.album_template_fields[key] = func
@ -84,14 +81,14 @@ class InlinePlugin(BeetsPlugin):
""" """
# First, try compiling as a single function. # First, try compiling as a single function.
try: try:
code = compile(u'({0})'.format(python_code), 'inline', 'eval') code = compile(f'({python_code})', 'inline', 'eval')
except SyntaxError: except SyntaxError:
# Fall back to a function body. # Fall back to a function body.
try: try:
func = _compile_func(python_code) func = _compile_func(python_code)
except SyntaxError: except SyntaxError:
self._log.error(u'syntax error in inline field definition:\n' self._log.error('syntax error in inline field definition:\n'
u'{0}', traceback.format_exc()) '{0}', traceback.format_exc())
return return
else: else:
is_expr = False is_expr = False
@ -117,9 +114,13 @@ class InlinePlugin(BeetsPlugin):
# For function bodies, invoke the function with values as global # For function bodies, invoke the function with values as global
# variables. # variables.
def _func_func(obj): def _func_func(obj):
old_globals = dict(func.__globals__)
func.__globals__.update(_dict_for(obj)) func.__globals__.update(_dict_for(obj))
try: try:
return func() return func()
except Exception as exc: except Exception as exc:
raise InlineError(python_code, exc) raise InlineError(python_code, exc)
finally:
func.__globals__.clear()
func.__globals__.update(old_globals)
return _func_func return _func_func

49
libs/common/beetsplug/ipfs.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# #
# Permission is hereby granted, free of charge, to any person obtaining # Permission is hereby granted, free of charge, to any person obtaining
@ -15,7 +14,6 @@
"""Adds support for ipfs. Requires go-ipfs and a running ipfs daemon """Adds support for ipfs. Requires go-ipfs and a running ipfs daemon
""" """
from __future__ import division, absolute_import, print_function
from beets import ui, util, library, config from beets import ui, util, library, config
from beets.plugins import BeetsPlugin from beets.plugins import BeetsPlugin
@ -29,9 +27,10 @@ import tempfile
class IPFSPlugin(BeetsPlugin): class IPFSPlugin(BeetsPlugin):
def __init__(self): def __init__(self):
super(IPFSPlugin, self).__init__() super().__init__()
self.config.add({ self.config.add({
'auto': True, 'auto': True,
'nocopy': False,
}) })
if self.config['auto']: if self.config['auto']:
@ -116,12 +115,15 @@ class IPFSPlugin(BeetsPlugin):
self._log.info('Adding {0} to ipfs', album_dir) self._log.info('Adding {0} to ipfs', album_dir)
cmd = "ipfs add -q -r".split() if self.config['nocopy']:
cmd = "ipfs add --nocopy -q -r".split()
else:
cmd = "ipfs add -q -r".split()
cmd.append(album_dir) cmd.append(album_dir)
try: try:
output = util.command_output(cmd).split() output = util.command_output(cmd).stdout.split()
except (OSError, subprocess.CalledProcessError) as exc: except (OSError, subprocess.CalledProcessError) as exc:
self._log.error(u'Failed to add {0}, error: {1}', album_dir, exc) self._log.error('Failed to add {0}, error: {1}', album_dir, exc)
return False return False
length = len(output) length = len(output)
@ -147,6 +149,8 @@ class IPFSPlugin(BeetsPlugin):
def ipfs_get(self, lib, query): def ipfs_get(self, lib, query):
query = query[0] query = query[0]
# Check if query is a hash # Check if query is a hash
# TODO: generalize to other hashes; probably use a multihash
# implementation
if query.startswith("Qm") and len(query) == 46: if query.startswith("Qm") and len(query) == 46:
self.ipfs_get_from_hash(lib, query) self.ipfs_get_from_hash(lib, query)
else: else:
@ -174,11 +178,14 @@ class IPFSPlugin(BeetsPlugin):
with tempfile.NamedTemporaryFile() as tmp: with tempfile.NamedTemporaryFile() as tmp:
self.ipfs_added_albums(lib, tmp.name) self.ipfs_added_albums(lib, tmp.name)
try: try:
cmd = "ipfs add -q ".split() if self.config['nocopy']:
cmd = "ipfs add --nocopy -q ".split()
else:
cmd = "ipfs add -q ".split()
cmd.append(tmp.name) cmd.append(tmp.name)
output = util.command_output(cmd) output = util.command_output(cmd).stdout
except (OSError, subprocess.CalledProcessError) as err: except (OSError, subprocess.CalledProcessError) as err:
msg = "Failed to publish library. Error: {0}".format(err) msg = f"Failed to publish library. Error: {err}"
self._log.error(msg) self._log.error(msg)
return False return False
self._log.info("hash of library: {0}", output) self._log.info("hash of library: {0}", output)
@ -190,26 +197,26 @@ class IPFSPlugin(BeetsPlugin):
else: else:
lib_name = _hash lib_name = _hash
lib_root = os.path.dirname(lib.path) lib_root = os.path.dirname(lib.path)
remote_libs = lib_root + "/remotes" remote_libs = os.path.join(lib_root, b"remotes")
if not os.path.exists(remote_libs): if not os.path.exists(remote_libs):
try: try:
os.makedirs(remote_libs) os.makedirs(remote_libs)
except OSError as e: except OSError as e:
msg = "Could not create {0}. Error: {1}".format(remote_libs, e) msg = f"Could not create {remote_libs}. Error: {e}"
self._log.error(msg) self._log.error(msg)
return False return False
path = remote_libs + "/" + lib_name + ".db" path = os.path.join(remote_libs, lib_name.encode() + b".db")
if not os.path.exists(path): if not os.path.exists(path):
cmd = "ipfs get {0} -o".format(_hash).split() cmd = f"ipfs get {_hash} -o".split()
cmd.append(path) cmd.append(path)
try: try:
util.command_output(cmd) util.command_output(cmd)
except (OSError, subprocess.CalledProcessError): except (OSError, subprocess.CalledProcessError):
self._log.error("Could not import {0}".format(_hash)) self._log.error(f"Could not import {_hash}")
return False return False
# add all albums from remotes into a combined library # add all albums from remotes into a combined library
jpath = remote_libs + "/joined.db" jpath = os.path.join(remote_libs, b"joined.db")
jlib = library.Library(jpath) jlib = library.Library(jpath)
nlib = library.Library(path) nlib = library.Library(path)
for album in nlib.albums(): for album in nlib.albums():
@ -232,12 +239,12 @@ class IPFSPlugin(BeetsPlugin):
fmt = config['format_album'].get() fmt = config['format_album'].get()
try: try:
albums = self.query(lib, args) albums = self.query(lib, args)
except IOError: except OSError:
ui.print_("No imported libraries yet.") ui.print_("No imported libraries yet.")
return return
for album in albums: for album in albums:
ui.print_(format(album, fmt), " : ", album.ipfs) ui.print_(format(album, fmt), " : ", album.ipfs.decode())
def query(self, lib, args): def query(self, lib, args):
rlib = self.get_remote_lib(lib) rlib = self.get_remote_lib(lib)
@ -246,10 +253,10 @@ class IPFSPlugin(BeetsPlugin):
def get_remote_lib(self, lib): def get_remote_lib(self, lib):
lib_root = os.path.dirname(lib.path) lib_root = os.path.dirname(lib.path)
remote_libs = lib_root + "/remotes" remote_libs = os.path.join(lib_root, b"remotes")
path = remote_libs + "/joined.db" path = os.path.join(remote_libs, b"joined.db")
if not os.path.isfile(path): if not os.path.isfile(path):
raise IOError raise OSError
return library.Library(path) return library.Library(path)
def ipfs_added_albums(self, rlib, tmpname): def ipfs_added_albums(self, rlib, tmpname):
@ -276,7 +283,7 @@ class IPFSPlugin(BeetsPlugin):
util._fsencoding(), 'ignore' util._fsencoding(), 'ignore'
) )
# Clear current path from item # Clear current path from item
item.path = '/ipfs/{0}/{1}'.format(album.ipfs, item_path) item.path = f'/ipfs/{album.ipfs}/{item_path}'
item.id = None item.id = None
items.append(item) items.append(item)

42
libs/common/beetsplug/keyfinder.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Thomas Scholtes. # Copyright 2016, Thomas Scholtes.
# #
@ -16,8 +15,8 @@
"""Uses the `KeyFinder` program to add the `initial_key` field. """Uses the `KeyFinder` program to add the `initial_key` field.
""" """
from __future__ import division, absolute_import, print_function
import os.path
import subprocess import subprocess
from beets import ui from beets import ui
@ -28,11 +27,11 @@ from beets.plugins import BeetsPlugin
class KeyFinderPlugin(BeetsPlugin): class KeyFinderPlugin(BeetsPlugin):
def __init__(self): def __init__(self):
super(KeyFinderPlugin, self).__init__() super().__init__()
self.config.add({ self.config.add({
u'bin': u'KeyFinder', 'bin': 'KeyFinder',
u'auto': True, 'auto': True,
u'overwrite': False, 'overwrite': False,
}) })
if self.config['auto'].get(bool): if self.config['auto'].get(bool):
@ -40,7 +39,7 @@ class KeyFinderPlugin(BeetsPlugin):
def commands(self): def commands(self):
cmd = ui.Subcommand('keyfinder', cmd = ui.Subcommand('keyfinder',
help=u'detect and add initial key from audio') help='detect and add initial key from audio')
cmd.func = self.command cmd.func = self.command
return [cmd] return [cmd]
@ -52,34 +51,45 @@ class KeyFinderPlugin(BeetsPlugin):
def find_key(self, items, write=False): def find_key(self, items, write=False):
overwrite = self.config['overwrite'].get(bool) overwrite = self.config['overwrite'].get(bool)
bin = self.config['bin'].as_str() command = [self.config['bin'].as_str()]
# The KeyFinder GUI program needs the -f flag before the path.
# keyfinder-cli is similar, but just wants the path with no flag.
if 'keyfinder-cli' not in os.path.basename(command[0]).lower():
command.append('-f')
for item in items: for item in items:
if item['initial_key'] and not overwrite: if item['initial_key'] and not overwrite:
continue continue
try: try:
output = util.command_output([bin, '-f', output = util.command_output(command + [util.syspath(
util.syspath(item.path)]) item.path)]).stdout
except (subprocess.CalledProcessError, OSError) as exc: except (subprocess.CalledProcessError, OSError) as exc:
self._log.error(u'execution failed: {0}', exc) self._log.error('execution failed: {0}', exc)
continue continue
except UnicodeEncodeError: except UnicodeEncodeError:
# Workaround for Python 2 Windows bug. # Workaround for Python 2 Windows bug.
# http://bugs.python.org/issue1759845 # https://bugs.python.org/issue1759845
self._log.error(u'execution failed for Unicode path: {0!r}', self._log.error('execution failed for Unicode path: {0!r}',
item.path) item.path)
continue continue
key_raw = output.rsplit(None, 1)[-1] try:
key_raw = output.rsplit(None, 1)[-1]
except IndexError:
# Sometimes keyfinder-cli returns 0 but with no key, usually
# when the file is silent or corrupt, so we log and skip.
self._log.error('no key returned for path: {0}', item.path)
continue
try: try:
key = util.text_string(key_raw) key = util.text_string(key_raw)
except UnicodeDecodeError: except UnicodeDecodeError:
self._log.error(u'output is invalid UTF-8') self._log.error('output is invalid UTF-8')
continue continue
item['initial_key'] = key item['initial_key'] = key
self._log.info(u'added computed initial key {0} for {1}', self._log.info('added computed initial key {0} for {1}',
key, util.displayable_path(item.path)) key, util.displayable_path(item.path))
if write: if write:

25
libs/common/beetsplug/kodiupdate.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2017, Pauli Kettunen. # Copyright 2017, Pauli Kettunen.
# #
@ -23,18 +22,16 @@ Put something like the following in your config.yaml to configure:
user: user user: user
pwd: secret pwd: secret
""" """
from __future__ import division, absolute_import, print_function
import requests import requests
from beets import config from beets import config
from beets.plugins import BeetsPlugin from beets.plugins import BeetsPlugin
import six
def update_kodi(host, port, user, password): def update_kodi(host, port, user, password):
"""Sends request to the Kodi api to start a library refresh. """Sends request to the Kodi api to start a library refresh.
""" """
url = "http://{0}:{1}/jsonrpc".format(host, port) url = f"http://{host}:{port}/jsonrpc"
"""Content-Type: application/json is mandatory """Content-Type: application/json is mandatory
according to the kodi jsonrpc documentation""" according to the kodi jsonrpc documentation"""
@ -54,14 +51,14 @@ def update_kodi(host, port, user, password):
class KodiUpdate(BeetsPlugin): class KodiUpdate(BeetsPlugin):
def __init__(self): def __init__(self):
super(KodiUpdate, self).__init__() super().__init__()
# Adding defaults. # Adding defaults.
config['kodi'].add({ config['kodi'].add({
u'host': u'localhost', 'host': 'localhost',
u'port': 8080, 'port': 8080,
u'user': u'kodi', 'user': 'kodi',
u'pwd': u'kodi'}) 'pwd': 'kodi'})
config['kodi']['pwd'].redact = True config['kodi']['pwd'].redact = True
self.register_listener('database_change', self.listen_for_db_change) self.register_listener('database_change', self.listen_for_db_change)
@ -73,7 +70,7 @@ class KodiUpdate(BeetsPlugin):
def update(self, lib): def update(self, lib):
"""When the client exists try to send refresh request to Kodi server. """When the client exists try to send refresh request to Kodi server.
""" """
self._log.info(u'Requesting a Kodi library update...') self._log.info('Requesting a Kodi library update...')
# Try to send update request. # Try to send update request.
try: try:
@ -85,14 +82,14 @@ class KodiUpdate(BeetsPlugin):
r.raise_for_status() r.raise_for_status()
except requests.exceptions.RequestException as e: except requests.exceptions.RequestException as e:
self._log.warning(u'Kodi update failed: {0}', self._log.warning('Kodi update failed: {0}',
six.text_type(e)) str(e))
return return
json = r.json() json = r.json()
if json.get('result') != 'OK': if json.get('result') != 'OK':
self._log.warning(u'Kodi update failed: JSON response was {0!r}', self._log.warning('Kodi update failed: JSON response was {0!r}',
json) json)
return return
self._log.info(u'Kodi update triggered') self._log.info('Kodi update triggered')

120
libs/common/beetsplug/lastgenre/__init__.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson. # Copyright 2016, Adrian Sampson.
# #
@ -13,8 +12,6 @@
# The above copyright notice and this permission notice shall be # The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software. # included in all copies or substantial portions of the Software.
from __future__ import division, absolute_import, print_function
import six
"""Gets genres for imported music based on Last.fm tags. """Gets genres for imported music based on Last.fm tags.
@ -46,7 +43,7 @@ PYLAST_EXCEPTIONS = (
) )
REPLACE = { REPLACE = {
u'\u2010': '-', '\u2010': '-',
} }
@ -73,7 +70,7 @@ def flatten_tree(elem, path, branches):
for sub in elem: for sub in elem:
flatten_tree(sub, path, branches) flatten_tree(sub, path, branches)
else: else:
branches.append(path + [six.text_type(elem)]) branches.append(path + [str(elem)])
def find_parents(candidate, branches): def find_parents(candidate, branches):
@ -97,7 +94,7 @@ C14N_TREE = os.path.join(os.path.dirname(__file__), 'genres-tree.yaml')
class LastGenrePlugin(plugins.BeetsPlugin): class LastGenrePlugin(plugins.BeetsPlugin):
def __init__(self): def __init__(self):
super(LastGenrePlugin, self).__init__() super().__init__()
self.config.add({ self.config.add({
'whitelist': True, 'whitelist': True,
@ -108,8 +105,9 @@ class LastGenrePlugin(plugins.BeetsPlugin):
'source': 'album', 'source': 'album',
'force': True, 'force': True,
'auto': True, 'auto': True,
'separator': u', ', 'separator': ', ',
'prefer_specific': False, 'prefer_specific': False,
'title_case': True,
}) })
self.setup() self.setup()
@ -132,18 +130,27 @@ class LastGenrePlugin(plugins.BeetsPlugin):
with open(wl_filename, 'rb') as f: with open(wl_filename, 'rb') as f:
for line in f: for line in f:
line = line.decode('utf-8').strip().lower() line = line.decode('utf-8').strip().lower()
if line and not line.startswith(u'#'): if line and not line.startswith('#'):
self.whitelist.add(line) self.whitelist.add(line)
# Read the genres tree for canonicalization if enabled. # Read the genres tree for canonicalization if enabled.
self.c14n_branches = [] self.c14n_branches = []
c14n_filename = self.config['canonical'].get() c14n_filename = self.config['canonical'].get()
if c14n_filename in (True, ''): # Default tree. self.canonicalize = c14n_filename is not False
# Default tree
if c14n_filename in (True, ''):
c14n_filename = C14N_TREE c14n_filename = C14N_TREE
elif not self.canonicalize and self.config['prefer_specific'].get():
# prefer_specific requires a tree, load default tree
c14n_filename = C14N_TREE
# Read the tree
if c14n_filename: if c14n_filename:
self._log.debug('Loading canonicalization tree {0}', c14n_filename)
c14n_filename = normpath(c14n_filename) c14n_filename = normpath(c14n_filename)
with codecs.open(c14n_filename, 'r', encoding='utf-8') as f: with codecs.open(c14n_filename, 'r', encoding='utf-8') as f:
genres_tree = yaml.load(f) genres_tree = yaml.safe_load(f)
flatten_tree(genres_tree, [], self.c14n_branches) flatten_tree(genres_tree, [], self.c14n_branches)
@property @property
@ -186,7 +193,7 @@ class LastGenrePlugin(plugins.BeetsPlugin):
return None return None
count = self.config['count'].get(int) count = self.config['count'].get(int)
if self.c14n_branches: if self.canonicalize:
# Extend the list to consider tags parents in the c14n tree # Extend the list to consider tags parents in the c14n tree
tags_all = [] tags_all = []
for tag in tags: for tag in tags:
@ -214,12 +221,17 @@ class LastGenrePlugin(plugins.BeetsPlugin):
# c14n only adds allowed genres but we may have had forbidden genres in # c14n only adds allowed genres but we may have had forbidden genres in
# the original tags list # the original tags list
tags = [x.title() for x in tags if self._is_allowed(x)] tags = [self._format_tag(x) for x in tags if self._is_allowed(x)]
return self.config['separator'].as_str().join( return self.config['separator'].as_str().join(
tags[:self.config['count'].get(int)] tags[:self.config['count'].get(int)]
) )
def _format_tag(self, tag):
if self.config["title_case"]:
return tag.title()
return tag
def fetch_genre(self, lastfm_obj): def fetch_genre(self, lastfm_obj):
"""Return the genre for a pylast entity or None if no suitable genre """Return the genre for a pylast entity or None if no suitable genre
can be found. Ex. 'Electronic, House, Dance' can be found. Ex. 'Electronic, House, Dance'
@ -251,8 +263,8 @@ class LastGenrePlugin(plugins.BeetsPlugin):
if any(not s for s in args): if any(not s for s in args):
return None return None
key = u'{0}.{1}'.format(entity, key = '{}.{}'.format(entity,
u'-'.join(six.text_type(a) for a in args)) '-'.join(str(a) for a in args))
if key in self._genre_cache: if key in self._genre_cache:
return self._genre_cache[key] return self._genre_cache[key]
else: else:
@ -270,28 +282,28 @@ class LastGenrePlugin(plugins.BeetsPlugin):
"""Return the album genre for this Item or Album. """Return the album genre for this Item or Album.
""" """
return self._last_lookup( return self._last_lookup(
u'album', LASTFM.get_album, obj.albumartist, obj.album 'album', LASTFM.get_album, obj.albumartist, obj.album
) )
def fetch_album_artist_genre(self, obj): def fetch_album_artist_genre(self, obj):
"""Return the album artist genre for this Item or Album. """Return the album artist genre for this Item or Album.
""" """
return self._last_lookup( return self._last_lookup(
u'artist', LASTFM.get_artist, obj.albumartist 'artist', LASTFM.get_artist, obj.albumartist
) )
def fetch_artist_genre(self, item): def fetch_artist_genre(self, item):
"""Returns the track artist genre for this Item. """Returns the track artist genre for this Item.
""" """
return self._last_lookup( return self._last_lookup(
u'artist', LASTFM.get_artist, item.artist 'artist', LASTFM.get_artist, item.artist
) )
def fetch_track_genre(self, obj): def fetch_track_genre(self, obj):
"""Returns the track genre for this Item. """Returns the track genre for this Item.
""" """
return self._last_lookup( return self._last_lookup(
u'track', LASTFM.get_track, obj.artist, obj.title 'track', LASTFM.get_track, obj.artist, obj.title
) )
def _get_genre(self, obj): def _get_genre(self, obj):
@ -361,38 +373,56 @@ class LastGenrePlugin(plugins.BeetsPlugin):
return None, None return None, None
def commands(self): def commands(self):
lastgenre_cmd = ui.Subcommand('lastgenre', help=u'fetch genres') lastgenre_cmd = ui.Subcommand('lastgenre', help='fetch genres')
lastgenre_cmd.parser.add_option( lastgenre_cmd.parser.add_option(
u'-f', u'--force', dest='force', '-f', '--force', dest='force',
action='store_true', default=False, action='store_true',
help=u're-download genre when already present' help='re-download genre when already present'
) )
lastgenre_cmd.parser.add_option( lastgenre_cmd.parser.add_option(
u'-s', u'--source', dest='source', type='string', '-s', '--source', dest='source', type='string',
help=u'genre source: artist, album, or track' help='genre source: artist, album, or track'
) )
lastgenre_cmd.parser.add_option(
'-A', '--items', action='store_false', dest='album',
help='match items instead of albums')
lastgenre_cmd.parser.add_option(
'-a', '--albums', action='store_true', dest='album',
help='match albums instead of items')
lastgenre_cmd.parser.set_defaults(album=True)
def lastgenre_func(lib, opts, args): def lastgenre_func(lib, opts, args):
write = ui.should_write() write = ui.should_write()
self.config.set_args(opts) self.config.set_args(opts)
for album in lib.albums(ui.decargs(args)): if opts.album:
album.genre, src = self._get_genre(album) # Fetch genres for whole albums
self._log.info(u'genre for album {0} ({1}): {0.genre}', for album in lib.albums(ui.decargs(args)):
album, src) album.genre, src = self._get_genre(album)
album.store() self._log.info('genre for album {0} ({1}): {0.genre}',
album, src)
album.store()
for item in album.items(): for item in album.items():
# If we're using track-level sources, also look up each # If we're using track-level sources, also look up each
# track on the album. # track on the album.
if 'track' in self.sources: if 'track' in self.sources:
item.genre, src = self._get_genre(item) item.genre, src = self._get_genre(item)
item.store() item.store()
self._log.info(u'genre for track {0} ({1}): {0.genre}', self._log.info(
item, src) 'genre for track {0} ({1}): {0.genre}',
item, src)
if write: if write:
item.try_write() item.try_write()
else:
# Just query singletons, i.e. items that are not part of
# an album
for item in lib.items(ui.decargs(args)):
item.genre, src = self._get_genre(item)
self._log.debug('added last.fm item genre ({0}): {1}',
src, item.genre)
item.store()
lastgenre_cmd.func = lastgenre_func lastgenre_cmd.func = lastgenre_func
return [lastgenre_cmd] return [lastgenre_cmd]
@ -402,21 +432,21 @@ class LastGenrePlugin(plugins.BeetsPlugin):
if task.is_album: if task.is_album:
album = task.album album = task.album
album.genre, src = self._get_genre(album) album.genre, src = self._get_genre(album)
self._log.debug(u'added last.fm album genre ({0}): {1}', self._log.debug('added last.fm album genre ({0}): {1}',
src, album.genre) src, album.genre)
album.store() album.store()
if 'track' in self.sources: if 'track' in self.sources:
for item in album.items(): for item in album.items():
item.genre, src = self._get_genre(item) item.genre, src = self._get_genre(item)
self._log.debug(u'added last.fm item genre ({0}): {1}', self._log.debug('added last.fm item genre ({0}): {1}',
src, item.genre) src, item.genre)
item.store() item.store()
else: else:
item = task.item item = task.item
item.genre, src = self._get_genre(item) item.genre, src = self._get_genre(item)
self._log.debug(u'added last.fm item genre ({0}): {1}', self._log.debug('added last.fm item genre ({0}): {1}',
src, item.genre) src, item.genre)
item.store() item.store()
@ -438,12 +468,12 @@ class LastGenrePlugin(plugins.BeetsPlugin):
try: try:
res = obj.get_top_tags() res = obj.get_top_tags()
except PYLAST_EXCEPTIONS as exc: except PYLAST_EXCEPTIONS as exc:
self._log.debug(u'last.fm error: {0}', exc) self._log.debug('last.fm error: {0}', exc)
return [] return []
except Exception as exc: except Exception as exc:
# Isolate bugs in pylast. # Isolate bugs in pylast.
self._log.debug(u'{}', traceback.format_exc()) self._log.debug('{}', traceback.format_exc())
self._log.error(u'error in pylast library: {0}', exc) self._log.error('error in pylast library: {0}', exc)
return [] return []
# Filter by weight (optionally). # Filter by weight (optionally).

25
libs/common/beetsplug/lastgenre/genres-tree.yaml
View file

@ -648,35 +648,51 @@
- glam rock - glam rock
- hard rock - hard rock
- heavy metal: - heavy metal:
- alternative metal - alternative metal:
- funk metal
- black metal: - black metal:
- viking metal - viking metal
- christian metal - christian metal
- death metal: - death metal:
- death/doom
- goregrind - goregrind
- melodic death metal - melodic death metal
- technical death metal - technical death metal
- doom metal - doom metal:
- epic doom metal
- funeral doom
- drone metal - drone metal
- epic metal
- folk metal: - folk metal:
- celtic metal - celtic metal
- medieval metal - medieval metal
- pagan metal
- funk metal - funk metal
- glam metal - glam metal
- gothic metal - gothic metal
- industrial metal:
- industrial death metal
- metalcore: - metalcore:
- deathcore - deathcore
- mathcore: - mathcore:
- djent - djent
- power metal - synthcore
- neoclassical metal
- post-metal
- power metal:
- progressive power metal
- progressive metal - progressive metal
- sludge metal - sludge metal
- speed metal - speed metal
- stoner rock - stoner rock:
- stoner metal
- symphonic metal - symphonic metal
- thrash metal: - thrash metal:
- crossover thrash - crossover thrash
- groove metal - groove metal
- progressive thrash metal
- teutonic thrash metal
- traditional heavy metal
- math rock - math rock
- new wave: - new wave:
- world fusion - world fusion
@ -719,6 +735,7 @@
- street punk - street punk
- thrashcore - thrashcore
- horror punk - horror punk
- oi!
- pop punk - pop punk
- psychobilly - psychobilly
- riot grrrl - riot grrrl

8
libs/common/beetsplug/lastgenre/genres.txt
View file

@ -450,6 +450,8 @@ emo rap
emocore emocore
emotronic emotronic
enka enka
epic doom metal
epic metal
eremwu eu eremwu eu
ethereal pop ethereal pop
ethereal wave ethereal wave
@ -1024,6 +1026,7 @@ neo-medieval
neo-prog neo-prog
neo-psychedelia neo-psychedelia
neoclassical neoclassical
neoclassical metal
neoclassical music neoclassical music
neofolk neofolk
neotraditional country neotraditional country
@ -1176,8 +1179,10 @@ progressive folk
progressive folk music progressive folk music
progressive house progressive house
progressive metal progressive metal
progressive power metal
progressive rock progressive rock
progressive trance progressive trance
progressive thrash metal
protopunk protopunk
psych folk psych folk
psychedelic music psychedelic music
@ -1396,6 +1401,7 @@ symphonic metal
symphonic poem symphonic poem
symphonic rock symphonic rock
symphony symphony
synthcore
synthpop synthpop
synthpunk synthpunk
t'ong guitar t'ong guitar
@ -1428,6 +1434,7 @@ tejano
tejano music tejano music
tekno tekno
tembang sunda tembang sunda
teutonic thrash metal
texas blues texas blues
thai pop thai pop
thillana thillana
@ -1444,6 +1451,7 @@ toeshey
togaku togaku
trad jazz trad jazz
traditional bluegrass traditional bluegrass
traditional heavy metal
traditional pop music traditional pop music
trallalero trallalero
trance trance

54
libs/common/beetsplug/lastimport.py
View file

@ -1,6 +1,5 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Rafael Bodill http://github.com/rafi # Copyright 2016, Rafael Bodill https://github.com/rafi
# #
# Permission is hereby granted, free of charge, to any person obtaining # Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the # a copy of this software and associated documentation files (the
@ -13,7 +12,6 @@
# The above copyright notice and this permission notice shall be # The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software. # included in all copies or substantial portions of the Software.
from __future__ import division, absolute_import, print_function
import pylast import pylast
from pylast import TopItem, _extract, _number from pylast import TopItem, _extract, _number
@ -28,7 +26,7 @@ API_URL = 'https://ws.audioscrobbler.com/2.0/'
class LastImportPlugin(plugins.BeetsPlugin): class LastImportPlugin(plugins.BeetsPlugin):
def __init__(self): def __init__(self):
super(LastImportPlugin, self).__init__() super().__init__()
config['lastfm'].add({ config['lastfm'].add({
'user': '', 'user': '',
'api_key': plugins.LASTFM_KEY, 'api_key': plugins.LASTFM_KEY,
@ -43,7 +41,7 @@ class LastImportPlugin(plugins.BeetsPlugin):
} }
def commands(self): def commands(self):
cmd = ui.Subcommand('lastimport', help=u'import last.fm play-count') cmd = ui.Subcommand('lastimport', help='import last.fm play-count')
def func(lib, opts, args): def func(lib, opts, args):
import_lastfm(lib, self._log) import_lastfm(lib, self._log)
@ -59,7 +57,7 @@ class CustomUser(pylast.User):
tracks. tracks.
""" """
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
super(CustomUser, self).__init__(*args, **kwargs) super().__init__(*args, **kwargs)
def _get_things(self, method, thing, thing_type, params=None, def _get_things(self, method, thing, thing_type, params=None,
cacheable=True): cacheable=True):
@ -114,9 +112,9 @@ def import_lastfm(lib, log):
per_page = config['lastimport']['per_page'].get(int) per_page = config['lastimport']['per_page'].get(int)
if not user: if not user:
raise ui.UserError(u'You must specify a user name for lastimport') raise ui.UserError('You must specify a user name for lastimport')
log.info(u'Fetching last.fm library for @{0}', user) log.info('Fetching last.fm library for @{0}', user)
page_total = 1 page_total = 1
page_current = 0 page_current = 0
@ -125,15 +123,15 @@ def import_lastfm(lib, log):
retry_limit = config['lastimport']['retry_limit'].get(int) retry_limit = config['lastimport']['retry_limit'].get(int)
# Iterate through a yet to be known page total count # Iterate through a yet to be known page total count
while page_current < page_total: while page_current < page_total:
log.info(u'Querying page #{0}{1}...', log.info('Querying page #{0}{1}...',
page_current + 1, page_current + 1,
'/{}'.format(page_total) if page_total > 1 else '') f'/{page_total}' if page_total > 1 else '')
for retry in range(0, retry_limit): for retry in range(0, retry_limit):
tracks, page_total = fetch_tracks(user, page_current + 1, per_page) tracks, page_total = fetch_tracks(user, page_current + 1, per_page)
if page_total < 1: if page_total < 1:
# It means nothing to us! # It means nothing to us!
raise ui.UserError(u'Last.fm reported no data.') raise ui.UserError('Last.fm reported no data.')
if tracks: if tracks:
found, unknown = process_tracks(lib, tracks, log) found, unknown = process_tracks(lib, tracks, log)
@ -141,22 +139,22 @@ def import_lastfm(lib, log):
unknown_total += unknown unknown_total += unknown
break break
else: else:
log.error(u'ERROR: unable to read page #{0}', log.error('ERROR: unable to read page #{0}',
page_current + 1) page_current + 1)
if retry < retry_limit: if retry < retry_limit:
log.info( log.info(
u'Retrying page #{0}... ({1}/{2} retry)', 'Retrying page #{0}... ({1}/{2} retry)',
page_current + 1, retry + 1, retry_limit page_current + 1, retry + 1, retry_limit
) )
else: else:
log.error(u'FAIL: unable to fetch page #{0}, ', log.error('FAIL: unable to fetch page #{0}, ',
u'tried {1} times', page_current, retry + 1) 'tried {1} times', page_current, retry + 1)
page_current += 1 page_current += 1
log.info(u'... done!') log.info('... done!')
log.info(u'finished processing {0} song pages', page_total) log.info('finished processing {0} song pages', page_total)
log.info(u'{0} unknown play-counts', unknown_total) log.info('{0} unknown play-counts', unknown_total)
log.info(u'{0} play-counts imported', found_total) log.info('{0} play-counts imported', found_total)
def fetch_tracks(user, page, limit): def fetch_tracks(user, page, limit):
@ -190,7 +188,7 @@ def process_tracks(lib, tracks, log):
total = len(tracks) total = len(tracks)
total_found = 0 total_found = 0
total_fails = 0 total_fails = 0
log.info(u'Received {0} tracks in this page, processing...', total) log.info('Received {0} tracks in this page, processing...', total)
for num in range(0, total): for num in range(0, total):
song = None song = None
@ -201,7 +199,7 @@ def process_tracks(lib, tracks, log):
if 'album' in tracks[num]: if 'album' in tracks[num]:
album = tracks[num]['album'].get('name', '').strip() album = tracks[num]['album'].get('name', '').strip()
log.debug(u'query: {0} - {1} ({2})', artist, title, album) log.debug('query: {0} - {1} ({2})', artist, title, album)
# First try to query by musicbrainz's trackid # First try to query by musicbrainz's trackid
if trackid: if trackid:
@ -211,7 +209,7 @@ def process_tracks(lib, tracks, log):
# If not, try just artist/title # If not, try just artist/title
if song is None: if song is None:
log.debug(u'no album match, trying by artist/title') log.debug('no album match, trying by artist/title')
query = dbcore.AndQuery([ query = dbcore.AndQuery([
dbcore.query.SubstringQuery('artist', artist), dbcore.query.SubstringQuery('artist', artist),
dbcore.query.SubstringQuery('title', title) dbcore.query.SubstringQuery('title', title)
@ -220,8 +218,8 @@ def process_tracks(lib, tracks, log):
# Last resort, try just replacing to utf-8 quote # Last resort, try just replacing to utf-8 quote
if song is None: if song is None:
title = title.replace("'", u'\u2019') title = title.replace("'", '\u2019')
log.debug(u'no title match, trying utf-8 single quote') log.debug('no title match, trying utf-8 single quote')
query = dbcore.AndQuery([ query = dbcore.AndQuery([
dbcore.query.SubstringQuery('artist', artist), dbcore.query.SubstringQuery('artist', artist),
dbcore.query.SubstringQuery('title', title) dbcore.query.SubstringQuery('title', title)
@ -231,19 +229,19 @@ def process_tracks(lib, tracks, log):
if song is not None: if song is not None:
count = int(song.get('play_count', 0)) count = int(song.get('play_count', 0))
new_count = int(tracks[num]['playcount']) new_count = int(tracks[num]['playcount'])
log.debug(u'match: {0} - {1} ({2}) ' log.debug('match: {0} - {1} ({2}) '
u'updating: play_count {3} => {4}', 'updating: play_count {3} => {4}',
song.artist, song.title, song.album, count, new_count) song.artist, song.title, song.album, count, new_count)
song['play_count'] = new_count song['play_count'] = new_count
song.store() song.store()
total_found += 1 total_found += 1
else: else:
total_fails += 1 total_fails += 1
log.info(u' - No match: {0} - {1} ({2})', log.info(' - No match: {0} - {1} ({2})',
artist, title, album) artist, title, album)
if total_fails > 0: if total_fails > 0:
log.info(u'Acquired {0}/{1} play-counts ({2} unknown)', log.info('Acquired {0}/{1} play-counts ({2} unknown)',
total_found, total, total_fails) total_found, total, total_fails)
return total_found, total_fails return total_found, total_fails

44
libs/common/beetsplug/loadext.py Normal file
View file

@ -0,0 +1,44 @@
# This file is part of beets.
# Copyright 2019, Jack Wilsdon <jack.wilsdon@gmail.com>
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Load SQLite extensions.
"""
from beets.dbcore import Database
from beets.plugins import BeetsPlugin
import sqlite3
class LoadExtPlugin(BeetsPlugin):
def __init__(self):
super().__init__()
if not Database.supports_extensions:
self._log.warn('loadext is enabled but the current SQLite '
'installation does not support extensions')
return
self.register_listener('library_opened', self.library_opened)
def library_opened(self, lib):
for v in self.config:
ext = v.as_filename()
self._log.debug('loading extension {}', ext)
try:
lib.load_extension(ext)
except sqlite3.OperationalError as e:
self._log.error('failed to load extension {}: {}', ext, e)

566
libs/common/beetsplug/lyrics.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson. # Copyright 2016, Adrian Sampson.
# #
@ -16,7 +15,6 @@
"""Fetches, embeds, and displays lyrics. """Fetches, embeds, and displays lyrics.
""" """
from __future__ import absolute_import, division, print_function
import difflib import difflib
import errno import errno
@ -29,11 +27,11 @@ import requests
import unicodedata import unicodedata
from unidecode import unidecode from unidecode import unidecode
import warnings import warnings
import six import urllib
from six.moves import urllib
try: try:
from bs4 import SoupStrainer, BeautifulSoup import bs4
from bs4 import SoupStrainer
HAS_BEAUTIFUL_SOUP = True HAS_BEAUTIFUL_SOUP = True
except ImportError: except ImportError:
HAS_BEAUTIFUL_SOUP = False HAS_BEAUTIFUL_SOUP = False
@ -48,7 +46,7 @@ try:
# PY3: HTMLParseError was removed in 3.5 as strict mode # PY3: HTMLParseError was removed in 3.5 as strict mode
# was deprecated in 3.3. # was deprecated in 3.3.
# https://docs.python.org/3.3/library/html.parser.html # https://docs.python.org/3.3/library/html.parser.html
from six.moves.html_parser import HTMLParseError from html.parser import HTMLParseError
except ImportError: except ImportError:
class HTMLParseError(Exception): class HTMLParseError(Exception):
pass pass
@ -62,23 +60,23 @@ COMMENT_RE = re.compile(r'<!--.*-->', re.S)
TAG_RE = re.compile(r'<[^>]*>') TAG_RE = re.compile(r'<[^>]*>')
BREAK_RE = re.compile(r'\n?\s*<br([\s|/][^>]*)*>\s*\n?', re.I) BREAK_RE = re.compile(r'\n?\s*<br([\s|/][^>]*)*>\s*\n?', re.I)
URL_CHARACTERS = { URL_CHARACTERS = {
u'\u2018': u"'", '\u2018': "'",
u'\u2019': u"'", '\u2019': "'",
u'\u201c': u'"', '\u201c': '"',
u'\u201d': u'"', '\u201d': '"',
u'\u2010': u'-', '\u2010': '-',
u'\u2011': u'-', '\u2011': '-',
u'\u2012': u'-', '\u2012': '-',
u'\u2013': u'-', '\u2013': '-',
u'\u2014': u'-', '\u2014': '-',
u'\u2015': u'-', '\u2015': '-',
u'\u2016': u'-', '\u2016': '-',
u'\u2026': u'...', '\u2026': '...',
} }
USER_AGENT = 'beets/{}'.format(beets.__version__) USER_AGENT = f'beets/{beets.__version__}'
# The content for the base index.rst generated in ReST mode. # The content for the base index.rst generated in ReST mode.
REST_INDEX_TEMPLATE = u'''Lyrics REST_INDEX_TEMPLATE = '''Lyrics
====== ======
* :ref:`Song index <genindex>` * :ref:`Song index <genindex>`
@ -94,11 +92,11 @@ Artist index:
''' '''
# The content for the base conf.py generated. # The content for the base conf.py generated.
REST_CONF_TEMPLATE = u'''# -*- coding: utf-8 -*- REST_CONF_TEMPLATE = '''# -*- coding: utf-8 -*-
master_doc = 'index' master_doc = 'index'
project = u'Lyrics' project = 'Lyrics'
copyright = u'none' copyright = 'none'
author = u'Various Authors' author = 'Various Authors'
latex_documents = [ latex_documents = [
(master_doc, 'Lyrics.tex', project, (master_doc, 'Lyrics.tex', project,
author, 'manual'), author, 'manual'),
@ -117,7 +115,7 @@ epub_tocdup = False
def unichar(i): def unichar(i):
try: try:
return six.unichr(i) return chr(i)
except ValueError: except ValueError:
return struct.pack('i', i).decode('utf-32') return struct.pack('i', i).decode('utf-32')
@ -126,12 +124,12 @@ def unescape(text):
"""Resolve &#xxx; HTML entities (and some others).""" """Resolve &#xxx; HTML entities (and some others)."""
if isinstance(text, bytes): if isinstance(text, bytes):
text = text.decode('utf-8', 'ignore') text = text.decode('utf-8', 'ignore')
out = text.replace(u'&nbsp;', u' ') out = text.replace('&nbsp;', ' ')
def replchar(m): def replchar(m):
num = m.group(1) num = m.group(1)
return unichar(int(num)) return unichar(int(num))
out = re.sub(u"&#(\d+);", replchar, out) out = re.sub("&#(\\d+);", replchar, out)
return out return out
@ -140,43 +138,10 @@ def extract_text_between(html, start_marker, end_marker):
_, html = html.split(start_marker, 1) _, html = html.split(start_marker, 1)
html, _ = html.split(end_marker, 1) html, _ = html.split(end_marker, 1)
except ValueError: except ValueError:
return u'' return ''
return html return html
def extract_text_in(html, starttag):
"""Extract the text from a <DIV> tag in the HTML starting with
``starttag``. Returns None if parsing fails.
"""
# Strip off the leading text before opening tag.
try:
_, html = html.split(starttag, 1)
except ValueError:
return
# Walk through balanced DIV tags.
level = 0
parts = []
pos = 0
for match in DIV_RE.finditer(html):
if match.group(1): # Closing tag.
level -= 1
if level == 0:
pos = match.end()
else: # Opening tag.
if level == 0:
parts.append(html[pos:match.start()])
level += 1
if level == -1:
parts.append(html[pos:match.start()])
break
else:
print(u'no closing tag found!')
return
return u''.join(parts)
def search_pairs(item): def search_pairs(item):
"""Yield a pairs of artists and titles to search for. """Yield a pairs of artists and titles to search for.
@ -186,6 +151,9 @@ def search_pairs(item):
In addition to the artist and title obtained from the `item` the In addition to the artist and title obtained from the `item` the
method tries to strip extra information like paranthesized suffixes method tries to strip extra information like paranthesized suffixes
and featured artists from the strings and add them as candidates. and featured artists from the strings and add them as candidates.
The artist sort name is added as a fallback candidate to help in
cases where artist name includes special characters or is in a
non-latin script.
The method also tries to split multiple titles separated with `/`. The method also tries to split multiple titles separated with `/`.
""" """
def generate_alternatives(string, patterns): def generate_alternatives(string, patterns):
@ -199,19 +167,23 @@ def search_pairs(item):
alternatives.append(match.group(1)) alternatives.append(match.group(1))
return alternatives return alternatives
title, artist = item.title, item.artist title, artist, artist_sort = item.title, item.artist, item.artist_sort
patterns = [ patterns = [
# Remove any featuring artists from the artists name # Remove any featuring artists from the artists name
r"(.*?) {0}".format(plugins.feat_tokens())] fr"(.*?) {plugins.feat_tokens()}"]
artists = generate_alternatives(artist, patterns) artists = generate_alternatives(artist, patterns)
# Use the artist_sort as fallback only if it differs from artist to avoid
# repeated remote requests with the same search terms
if artist != artist_sort:
artists.append(artist_sort)
patterns = [ patterns = [
# Remove a parenthesized suffix from a title string. Common # Remove a parenthesized suffix from a title string. Common
# examples include (live), (remix), and (acoustic). # examples include (live), (remix), and (acoustic).
r"(.+?)\s+[(].*[)]$", r"(.+?)\s+[(].*[)]$",
# Remove any featuring artists from the title # Remove any featuring artists from the title
r"(.*?) {0}".format(plugins.feat_tokens(for_artist=False)), r"(.*?) {}".format(plugins.feat_tokens(for_artist=False)),
# Remove part of title after colon ':' for songs with subtitles # Remove part of title after colon ':' for songs with subtitles
r"(.+?)\s*:.*"] r"(.+?)\s*:.*"]
titles = generate_alternatives(title, patterns) titles = generate_alternatives(title, patterns)
@ -245,14 +217,27 @@ def slug(text):
return re.sub(r'\W+', '-', unidecode(text).lower().strip()).strip('-') return re.sub(r'\W+', '-', unidecode(text).lower().strip()).strip('-')
class Backend(object): if HAS_BEAUTIFUL_SOUP:
def try_parse_html(html, **kwargs):
try:
return bs4.BeautifulSoup(html, 'html.parser', **kwargs)
except HTMLParseError:
return None
else:
def try_parse_html(html, **kwargs):
return None
class Backend:
REQUIRES_BS = False
def __init__(self, config, log): def __init__(self, config, log):
self._log = log self._log = log
@staticmethod @staticmethod
def _encode(s): def _encode(s):
"""Encode the string for inclusion in a URL""" """Encode the string for inclusion in a URL"""
if isinstance(s, six.text_type): if isinstance(s, str):
for char, repl in URL_CHARACTERS.items(): for char, repl in URL_CHARACTERS.items():
s = s.replace(char, repl) s = s.replace(char, repl)
s = s.encode('utf-8', 'ignore') s = s.encode('utf-8', 'ignore')
@ -277,20 +262,21 @@ class Backend(object):
'User-Agent': USER_AGENT, 'User-Agent': USER_AGENT,
}) })
except requests.RequestException as exc: except requests.RequestException as exc:
self._log.debug(u'lyrics request failed: {0}', exc) self._log.debug('lyrics request failed: {0}', exc)
return return
if r.status_code == requests.codes.ok: if r.status_code == requests.codes.ok:
return r.text return r.text
else: else:
self._log.debug(u'failed to fetch: {0} ({1})', url, r.status_code) self._log.debug('failed to fetch: {0} ({1})', url, r.status_code)
return None
def fetch(self, artist, title): def fetch(self, artist, title):
raise NotImplementedError() raise NotImplementedError()
class SymbolsReplaced(Backend): class MusiXmatch(Backend):
REPLACEMENTS = { REPLACEMENTS = {
r'\s+': '_', r'\s+': '-',
'<': 'Less_Than', '<': 'Less_Than',
'>': 'Greater_Than', '>': 'Greater_Than',
'#': 'Number_', '#': 'Number_',
@ -298,39 +284,40 @@ class SymbolsReplaced(Backend):
r'[\]\}]': ')', r'[\]\}]': ')',
} }
URL_PATTERN = 'https://www.musixmatch.com/lyrics/%s/%s'
@classmethod @classmethod
def _encode(cls, s): def _encode(cls, s):
for old, new in cls.REPLACEMENTS.items(): for old, new in cls.REPLACEMENTS.items():
s = re.sub(old, new, s) s = re.sub(old, new, s)
return super(SymbolsReplaced, cls)._encode(s) return super()._encode(s)
class MusiXmatch(SymbolsReplaced):
REPLACEMENTS = dict(SymbolsReplaced.REPLACEMENTS, **{
r'\s+': '-'
})
URL_PATTERN = 'https://www.musixmatch.com/lyrics/%s/%s'
def fetch(self, artist, title): def fetch(self, artist, title):
url = self.build_url(artist, title) url = self.build_url(artist, title)
html = self.fetch_url(url) html = self.fetch_url(url)
if not html: if not html:
return return None
if "We detected that your IP is blocked" in html: if "We detected that your IP is blocked" in html:
self._log.warning(u'we are blocked at MusixMatch: url %s failed' self._log.warning('we are blocked at MusixMatch: url %s failed'
% url) % url)
return return None
html_part = html.split('<p class="mxm-lyrics__content')[-1] html_parts = html.split('<p class="mxm-lyrics__content')
lyrics = extract_text_between(html_part, '>', '</p>') # Sometimes lyrics come in 2 or more parts
lyrics_parts = []
for html_part in html_parts:
lyrics_parts.append(extract_text_between(html_part, '>', '</p>'))
lyrics = '\n'.join(lyrics_parts)
lyrics = lyrics.strip(',"').replace('\\n', '\n') lyrics = lyrics.strip(',"').replace('\\n', '\n')
# another odd case: sometimes only that string remains, for # another odd case: sometimes only that string remains, for
# missing songs. this seems to happen after being blocked # missing songs. this seems to happen after being blocked
# above, when filling in the CAPTCHA. # above, when filling in the CAPTCHA.
if "Instant lyrics for all your music." in lyrics: if "Instant lyrics for all your music." in lyrics:
return return None
# sometimes there are non-existent lyrics with some content
if 'Lyrics | Musixmatch' in lyrics:
return None
return lyrics return lyrics
@ -341,87 +328,171 @@ class Genius(Backend):
bigishdata.com/2016/09/27/getting-song-lyrics-from-geniuss-api-scraping/ bigishdata.com/2016/09/27/getting-song-lyrics-from-geniuss-api-scraping/
""" """
REQUIRES_BS = True
base_url = "https://api.genius.com" base_url = "https://api.genius.com"
def __init__(self, config, log): def __init__(self, config, log):
super(Genius, self).__init__(config, log) super().__init__(config, log)
self.api_key = config['genius_api_key'].as_str() self.api_key = config['genius_api_key'].as_str()
self.headers = { self.headers = {
'Authorization': "Bearer %s" % self.api_key, 'Authorization': "Bearer %s" % self.api_key,
'User-Agent': USER_AGENT, 'User-Agent': USER_AGENT,
} }
def lyrics_from_song_api_path(self, song_api_path):
song_url = self.base_url + song_api_path
response = requests.get(song_url, headers=self.headers)
json = response.json()
path = json["response"]["song"]["path"]
# Gotta go regular html scraping... come on Genius.
page_url = "https://genius.com" + path
try:
page = requests.get(page_url)
except requests.RequestException as exc:
self._log.debug(u'Genius page request for {0} failed: {1}',
page_url, exc)
return None
html = BeautifulSoup(page.text, "html.parser")
# Remove script tags that they put in the middle of the lyrics.
[h.extract() for h in html('script')]
# At least Genius is nice and has a tag called 'lyrics'!
# Updated css where the lyrics are based in HTML.
lyrics = html.find("div", class_="lyrics").get_text()
return lyrics
def fetch(self, artist, title): def fetch(self, artist, title):
search_url = self.base_url + "/search" """Fetch lyrics from genius.com
data = {'q': title}
try: Because genius doesn't allow accesssing lyrics via the api,
response = requests.get(search_url, data=data, we first query the api for a url matching our artist & title,
headers=self.headers) then attempt to scrape that url for the lyrics.
except requests.RequestException as exc: """
self._log.debug(u'Genius API request failed: {0}', exc) json = self._search(artist, title)
if not json:
self._log.debug('Genius API request returned invalid JSON')
return None return None
try: # find a matching artist in the json
json = response.json()
except ValueError:
self._log.debug(u'Genius API request returned invalid JSON')
return None
song_info = None
for hit in json["response"]["hits"]: for hit in json["response"]["hits"]:
if hit["result"]["primary_artist"]["name"] == artist: hit_artist = hit["result"]["primary_artist"]["name"]
song_info = hit
break
if song_info: if slug(hit_artist) == slug(artist):
song_api_path = song_info["result"]["api_path"] html = self.fetch_url(hit["result"]["url"])
return self.lyrics_from_song_api_path(song_api_path) if not html:
return None
return self._scrape_lyrics_from_html(html)
self._log.debug('Genius failed to find a matching artist for \'{0}\'',
artist)
return None
class LyricsWiki(SymbolsReplaced): def _search(self, artist, title):
"""Fetch lyrics from LyricsWiki.""" """Searches the genius api for a given artist and title
URL_PATTERN = 'http://lyrics.wikia.com/%s:%s' https://docs.genius.com/#search-h2
def fetch(self, artist, title): :returns: json response
url = self.build_url(artist, title) """
html = self.fetch_url(url) search_url = self.base_url + "/search"
if not html: data = {'q': title + " " + artist.lower()}
try:
response = requests.get(
search_url, data=data, headers=self.headers)
except requests.RequestException as exc:
self._log.debug('Genius API request failed: {0}', exc)
return None
try:
return response.json()
except ValueError:
return None
def _scrape_lyrics_from_html(self, html):
"""Scrape lyrics from a given genius.com html"""
soup = try_parse_html(html)
if not soup:
return return
# Get the HTML fragment inside the appropriate HTML element and then # Remove script tags that they put in the middle of the lyrics.
# extract the text from it. [h.extract() for h in soup('script')]
html_frag = extract_text_in(html, u"<div class='lyricbox'>")
if html_frag:
lyrics = _scrape_strip_cruft(html_frag, True)
if lyrics and 'Unfortunately, we are not licensed' not in lyrics: # Most of the time, the page contains a div with class="lyrics" where
return lyrics # all of the lyrics can be found already correctly formatted
# Sometimes, though, it packages the lyrics into separate divs, most
# likely for easier ad placement
lyrics_div = soup.find("div", class_="lyrics")
if not lyrics_div:
self._log.debug('Received unusual song page html')
verse_div = soup.find("div",
class_=re.compile("Lyrics__Container"))
if not verse_div:
if soup.find("div",
class_=re.compile("LyricsPlaceholder__Message"),
string="This song is an instrumental"):
self._log.debug('Detected instrumental')
return "[Instrumental]"
else:
self._log.debug("Couldn't scrape page using known layouts")
return None
lyrics_div = verse_div.parent
for br in lyrics_div.find_all("br"):
br.replace_with("\n")
ads = lyrics_div.find_all("div",
class_=re.compile("InreadAd__Container"))
for ad in ads:
ad.replace_with("\n")
return lyrics_div.get_text()
class Tekstowo(Backend):
# Fetch lyrics from Tekstowo.pl.
REQUIRES_BS = True
BASE_URL = 'http://www.tekstowo.pl'
URL_PATTERN = BASE_URL + '/wyszukaj.html?search-title=%s&search-artist=%s'
def fetch(self, artist, title):
url = self.build_url(title, artist)
search_results = self.fetch_url(url)
if not search_results:
return None
song_page_url = self.parse_search_results(search_results)
if not song_page_url:
return None
song_page_html = self.fetch_url(song_page_url)
if not song_page_html:
return None
return self.extract_lyrics(song_page_html)
def parse_search_results(self, html):
html = _scrape_strip_cruft(html)
html = _scrape_merge_paragraphs(html)
soup = try_parse_html(html)
if not soup:
return None
content_div = soup.find("div", class_="content")
if not content_div:
return None
card_div = content_div.find("div", class_="card")
if not card_div:
return None
song_rows = card_div.find_all("div", class_="box-przeboje")
if not song_rows:
return None
song_row = song_rows[0]
if not song_row:
return None
link = song_row.find('a')
if not link:
return None
return self.BASE_URL + link.get('href')
def extract_lyrics(self, html):
html = _scrape_strip_cruft(html)
html = _scrape_merge_paragraphs(html)
soup = try_parse_html(html)
if not soup:
return None
lyrics_div = soup.find("div", class_="song-text")
if not lyrics_div:
return None
return lyrics_div.get_text()
def remove_credits(text): def remove_credits(text):
@ -446,7 +517,8 @@ def _scrape_strip_cruft(html, plain_text_out=False):
html = html.replace('\r', '\n') # Normalize EOL. html = html.replace('\r', '\n') # Normalize EOL.
html = re.sub(r' +', ' ', html) # Whitespaces collapse. html = re.sub(r' +', ' ', html) # Whitespaces collapse.
html = BREAK_RE.sub('\n', html) # <br> eats up surrounding '\n'. html = BREAK_RE.sub('\n', html) # <br> eats up surrounding '\n'.
html = re.sub(r'<(script).*?</\1>(?s)', '', html) # Strip script tags. html = re.sub(r'(?s)<(script).*?</\1>', '', html) # Strip script tags.
html = re.sub('\u2005', " ", html) # replace unicode with regular space
if plain_text_out: # Strip remaining HTML tags if plain_text_out: # Strip remaining HTML tags
html = COMMENT_RE.sub('', html) html = COMMENT_RE.sub('', html)
@ -466,12 +538,6 @@ def scrape_lyrics_from_html(html):
"""Scrape lyrics from a URL. If no lyrics can be found, return None """Scrape lyrics from a URL. If no lyrics can be found, return None
instead. instead.
""" """
if not HAS_BEAUTIFUL_SOUP:
return None
if not html:
return None
def is_text_notcode(text): def is_text_notcode(text):
length = len(text) length = len(text)
return (length > 20 and return (length > 20 and
@ -481,10 +547,8 @@ def scrape_lyrics_from_html(html):
html = _scrape_merge_paragraphs(html) html = _scrape_merge_paragraphs(html)
# extract all long text blocks that are not code # extract all long text blocks that are not code
try: soup = try_parse_html(html, parse_only=SoupStrainer(text=is_text_notcode))
soup = BeautifulSoup(html, "html.parser", if not soup:
parse_only=SoupStrainer(text=is_text_notcode))
except HTMLParseError:
return None return None
# Get the longest text element (if any). # Get the longest text element (if any).
@ -498,8 +562,10 @@ def scrape_lyrics_from_html(html):
class Google(Backend): class Google(Backend):
"""Fetch lyrics from Google search results.""" """Fetch lyrics from Google search results."""
REQUIRES_BS = True
def __init__(self, config, log): def __init__(self, config, log):
super(Google, self).__init__(config, log) super().__init__(config, log)
self.api_key = config['google_API_key'].as_str() self.api_key = config['google_API_key'].as_str()
self.engine_id = config['google_engine_ID'].as_str() self.engine_id = config['google_engine_ID'].as_str()
@ -511,7 +577,7 @@ class Google(Backend):
bad_triggers_occ = [] bad_triggers_occ = []
nb_lines = text.count('\n') nb_lines = text.count('\n')
if nb_lines <= 1: if nb_lines <= 1:
self._log.debug(u"Ignoring too short lyrics '{0}'", text) self._log.debug("Ignoring too short lyrics '{0}'", text)
return False return False
elif nb_lines < 5: elif nb_lines < 5:
bad_triggers_occ.append('too_short') bad_triggers_occ.append('too_short')
@ -522,14 +588,14 @@ class Google(Backend):
bad_triggers = ['lyrics', 'copyright', 'property', 'links'] bad_triggers = ['lyrics', 'copyright', 'property', 'links']
if artist: if artist:
bad_triggers_occ += [artist] bad_triggers += [artist]
for item in bad_triggers: for item in bad_triggers:
bad_triggers_occ += [item] * len(re.findall(r'\W%s\W' % item, bad_triggers_occ += [item] * len(re.findall(r'\W%s\W' % item,
text, re.I)) text, re.I))
if bad_triggers_occ: if bad_triggers_occ:
self._log.debug(u'Bad triggers detected: {0}', bad_triggers_occ) self._log.debug('Bad triggers detected: {0}', bad_triggers_occ)
return len(bad_triggers_occ) < 2 return len(bad_triggers_occ) < 2
def slugify(self, text): def slugify(self, text):
@ -537,14 +603,14 @@ class Google(Backend):
""" """
text = re.sub(r"[-'_\s]", '_', text) text = re.sub(r"[-'_\s]", '_', text)
text = re.sub(r"_+", '_', text).strip('_') text = re.sub(r"_+", '_', text).strip('_')
pat = "([^,\(]*)\((.*?)\)" # Remove content within parentheses pat = r"([^,\(]*)\((.*?)\)" # Remove content within parentheses
text = re.sub(pat, '\g<1>', text).strip() text = re.sub(pat, r'\g<1>', text).strip()
try: try:
text = unicodedata.normalize('NFKD', text).encode('ascii', text = unicodedata.normalize('NFKD', text).encode('ascii',
'ignore') 'ignore')
text = six.text_type(re.sub('[-\s]+', ' ', text.decode('utf-8'))) text = str(re.sub(r'[-\s]+', ' ', text.decode('utf-8')))
except UnicodeDecodeError: except UnicodeDecodeError:
self._log.exception(u"Failing to normalize '{0}'", text) self._log.exception("Failing to normalize '{0}'", text)
return text return text
BY_TRANS = ['by', 'par', 'de', 'von'] BY_TRANS = ['by', 'par', 'de', 'von']
@ -556,7 +622,7 @@ class Google(Backend):
""" """
title = self.slugify(title.lower()) title = self.slugify(title.lower())
artist = self.slugify(artist.lower()) artist = self.slugify(artist.lower())
sitename = re.search(u"//([^/]+)/.*", sitename = re.search("//([^/]+)/.*",
self.slugify(url_link.lower())).group(1) self.slugify(url_link.lower())).group(1)
url_title = self.slugify(url_title.lower()) url_title = self.slugify(url_title.lower())
@ -570,7 +636,7 @@ class Google(Backend):
[artist, sitename, sitename.replace('www.', '')] + \ [artist, sitename, sitename.replace('www.', '')] + \
self.LYRICS_TRANS self.LYRICS_TRANS
tokens = [re.escape(t) for t in tokens] tokens = [re.escape(t) for t in tokens]
song_title = re.sub(u'(%s)' % u'|'.join(tokens), u'', url_title) song_title = re.sub('(%s)' % '|'.join(tokens), '', url_title)
song_title = song_title.strip('_|') song_title = song_title.strip('_|')
typo_ratio = .9 typo_ratio = .9
@ -578,53 +644,57 @@ class Google(Backend):
return ratio >= typo_ratio return ratio >= typo_ratio
def fetch(self, artist, title): def fetch(self, artist, title):
query = u"%s %s" % (artist, title) query = f"{artist} {title}"
url = u'https://www.googleapis.com/customsearch/v1?key=%s&cx=%s&q=%s' \ url = 'https://www.googleapis.com/customsearch/v1?key=%s&cx=%s&q=%s' \
% (self.api_key, self.engine_id, % (self.api_key, self.engine_id,
urllib.parse.quote(query.encode('utf-8'))) urllib.parse.quote(query.encode('utf-8')))
data = self.fetch_url(url) data = self.fetch_url(url)
if not data: if not data:
self._log.debug(u'google backend returned no data') self._log.debug('google backend returned no data')
return None return None
try: try:
data = json.loads(data) data = json.loads(data)
except ValueError as exc: except ValueError as exc:
self._log.debug(u'google backend returned malformed JSON: {}', exc) self._log.debug('google backend returned malformed JSON: {}', exc)
if 'error' in data: if 'error' in data:
reason = data['error']['errors'][0]['reason'] reason = data['error']['errors'][0]['reason']
self._log.debug(u'google backend error: {0}', reason) self._log.debug('google backend error: {0}', reason)
return None return None
if 'items' in data.keys(): if 'items' in data.keys():
for item in data['items']: for item in data['items']:
url_link = item['link'] url_link = item['link']
url_title = item.get('title', u'') url_title = item.get('title', '')
if not self.is_page_candidate(url_link, url_title, if not self.is_page_candidate(url_link, url_title,
title, artist): title, artist):
continue continue
html = self.fetch_url(url_link) html = self.fetch_url(url_link)
if not html:
continue
lyrics = scrape_lyrics_from_html(html) lyrics = scrape_lyrics_from_html(html)
if not lyrics: if not lyrics:
continue continue
if self.is_lyrics(lyrics, artist): if self.is_lyrics(lyrics, artist):
self._log.debug(u'got lyrics from {0}', self._log.debug('got lyrics from {0}',
item['displayLink']) item['displayLink'])
return lyrics return lyrics
return None
class LyricsPlugin(plugins.BeetsPlugin): class LyricsPlugin(plugins.BeetsPlugin):
SOURCES = ['google', 'lyricwiki', 'musixmatch', 'genius'] SOURCES = ['google', 'musixmatch', 'genius', 'tekstowo']
SOURCE_BACKENDS = { SOURCE_BACKENDS = {
'google': Google, 'google': Google,
'lyricwiki': LyricsWiki,
'musixmatch': MusiXmatch, 'musixmatch': MusiXmatch,
'genius': Genius, 'genius': Genius,
'tekstowo': Tekstowo,
} }
def __init__(self): def __init__(self):
super(LyricsPlugin, self).__init__() super().__init__()
self.import_stages = [self.imported] self.import_stages = [self.imported]
self.config.add({ self.config.add({
'auto': True, 'auto': True,
@ -632,7 +702,7 @@ class LyricsPlugin(plugins.BeetsPlugin):
'bing_lang_from': [], 'bing_lang_from': [],
'bing_lang_to': None, 'bing_lang_to': None,
'google_API_key': None, 'google_API_key': None,
'google_engine_ID': u'009217259823014548361:lndtuqkycfu', 'google_engine_ID': '009217259823014548361:lndtuqkycfu',
'genius_api_key': 'genius_api_key':
"Ryq93pUGm8bM6eUWwD_M3NOFFDAtp2yEE7W" "Ryq93pUGm8bM6eUWwD_M3NOFFDAtp2yEE7W"
"76V-uFL5jks5dNvcGCdarqFjDhP9c", "76V-uFL5jks5dNvcGCdarqFjDhP9c",
@ -648,7 +718,7 @@ class LyricsPlugin(plugins.BeetsPlugin):
# State information for the ReST writer. # State information for the ReST writer.
# First, the current artist we're writing. # First, the current artist we're writing.
self.artist = u'Unknown artist' self.artist = 'Unknown artist'
# The current album: False means no album yet. # The current album: False means no album yet.
self.album = False self.album = False
# The current rest file content. None means the file is not # The current rest file content. None means the file is not
@ -659,40 +729,44 @@ class LyricsPlugin(plugins.BeetsPlugin):
sources = plugins.sanitize_choices( sources = plugins.sanitize_choices(
self.config['sources'].as_str_seq(), available_sources) self.config['sources'].as_str_seq(), available_sources)
if not HAS_BEAUTIFUL_SOUP:
sources = self.sanitize_bs_sources(sources)
if 'google' in sources: if 'google' in sources:
if not self.config['google_API_key'].get(): if not self.config['google_API_key'].get():
# We log a *debug* message here because the default # We log a *debug* message here because the default
# configuration includes `google`. This way, the source # configuration includes `google`. This way, the source
# is silent by default but can be enabled just by # is silent by default but can be enabled just by
# setting an API key. # setting an API key.
self._log.debug(u'Disabling google source: ' self._log.debug('Disabling google source: '
u'no API key configured.') 'no API key configured.')
sources.remove('google') sources.remove('google')
elif not HAS_BEAUTIFUL_SOUP:
self._log.warning(u'To use the google lyrics source, you must '
u'install the beautifulsoup4 module. See '
u'the documentation for further details.')
sources.remove('google')
if 'genius' in sources and not HAS_BEAUTIFUL_SOUP:
self._log.debug(
u'The Genius backend requires BeautifulSoup, which is not '
u'installed, so the source is disabled.'
)
sources.remove('genius')
self.config['bing_lang_from'] = [ self.config['bing_lang_from'] = [
x.lower() for x in self.config['bing_lang_from'].as_str_seq()] x.lower() for x in self.config['bing_lang_from'].as_str_seq()]
self.bing_auth_token = None self.bing_auth_token = None
if not HAS_LANGDETECT and self.config['bing_client_secret'].get(): if not HAS_LANGDETECT and self.config['bing_client_secret'].get():
self._log.warning(u'To use bing translations, you need to ' self._log.warning('To use bing translations, you need to '
u'install the langdetect module. See the ' 'install the langdetect module. See the '
u'documentation for further details.') 'documentation for further details.')
self.backends = [self.SOURCE_BACKENDS[source](self.config, self._log) self.backends = [self.SOURCE_BACKENDS[source](self.config, self._log)
for source in sources] for source in sources]
def sanitize_bs_sources(self, sources):
enabled_sources = []
for source in sources:
if self.SOURCE_BACKENDS[source].REQUIRES_BS:
self._log.debug('To use the %s lyrics source, you must '
'install the beautifulsoup4 module. See '
'the documentation for further details.'
% source)
else:
enabled_sources.append(source)
return enabled_sources
def get_bing_access_token(self): def get_bing_access_token(self):
params = { params = {
'client_id': 'beets', 'client_id': 'beets',
@ -708,30 +782,30 @@ class LyricsPlugin(plugins.BeetsPlugin):
if 'access_token' in oauth_token: if 'access_token' in oauth_token:
return "Bearer " + oauth_token['access_token'] return "Bearer " + oauth_token['access_token']
else: else:
self._log.warning(u'Could not get Bing Translate API access token.' self._log.warning('Could not get Bing Translate API access token.'
u' Check your "bing_client_secret" password') ' Check your "bing_client_secret" password')
def commands(self): def commands(self):
cmd = ui.Subcommand('lyrics', help='fetch song lyrics') cmd = ui.Subcommand('lyrics', help='fetch song lyrics')
cmd.parser.add_option( cmd.parser.add_option(
u'-p', u'--print', dest='printlyr', '-p', '--print', dest='printlyr',
action='store_true', default=False, action='store_true', default=False,
help=u'print lyrics to console', help='print lyrics to console',
) )
cmd.parser.add_option( cmd.parser.add_option(
u'-r', u'--write-rest', dest='writerest', '-r', '--write-rest', dest='writerest',
action='store', default=None, metavar='dir', action='store', default=None, metavar='dir',
help=u'write lyrics to given directory as ReST files', help='write lyrics to given directory as ReST files',
) )
cmd.parser.add_option( cmd.parser.add_option(
u'-f', u'--force', dest='force_refetch', '-f', '--force', dest='force_refetch',
action='store_true', default=False, action='store_true', default=False,
help=u'always re-download lyrics', help='always re-download lyrics',
) )
cmd.parser.add_option( cmd.parser.add_option(
u'-l', u'--local', dest='local_only', '-l', '--local', dest='local_only',
action='store_true', default=False, action='store_true', default=False,
help=u'do not fetch missing lyrics', help='do not fetch missing lyrics',
) )
def func(lib, opts, args): def func(lib, opts, args):
@ -740,7 +814,8 @@ class LyricsPlugin(plugins.BeetsPlugin):
write = ui.should_write() write = ui.should_write()
if opts.writerest: if opts.writerest:
self.writerest_indexes(opts.writerest) self.writerest_indexes(opts.writerest)
for item in lib.items(ui.decargs(args)): items = lib.items(ui.decargs(args))
for item in items:
if not opts.local_only and not self.config['local']: if not opts.local_only and not self.config['local']:
self.fetch_item_lyrics( self.fetch_item_lyrics(
lib, item, write, lib, item, write,
@ -750,51 +825,55 @@ class LyricsPlugin(plugins.BeetsPlugin):
if opts.printlyr: if opts.printlyr:
ui.print_(item.lyrics) ui.print_(item.lyrics)
if opts.writerest: if opts.writerest:
self.writerest(opts.writerest, item) self.appendrest(opts.writerest, item)
if opts.writerest: if opts.writerest and items:
# flush last artist # flush last artist & write to ReST
self.writerest(opts.writerest, None) self.writerest(opts.writerest)
ui.print_(u'ReST files generated. to build, use one of:') ui.print_('ReST files generated. to build, use one of:')
ui.print_(u' sphinx-build -b html %s _build/html' ui.print_(' sphinx-build -b html %s _build/html'
% opts.writerest) % opts.writerest)
ui.print_(u' sphinx-build -b epub %s _build/epub' ui.print_(' sphinx-build -b epub %s _build/epub'
% opts.writerest) % opts.writerest)
ui.print_((u' sphinx-build -b latex %s _build/latex ' ui.print_((' sphinx-build -b latex %s _build/latex '
u'&& make -C _build/latex all-pdf') '&& make -C _build/latex all-pdf')
% opts.writerest) % opts.writerest)
cmd.func = func cmd.func = func
return [cmd] return [cmd]
def writerest(self, directory, item): def appendrest(self, directory, item):
"""Write the item to an ReST file """Append the item to an ReST file
This will keep state (in the `rest` variable) in order to avoid This will keep state (in the `rest` variable) in order to avoid
writing continuously to the same files. writing continuously to the same files.
""" """
if item is None or slug(self.artist) != slug(item.albumartist): if slug(self.artist) != slug(item.albumartist):
if self.rest is not None: # Write current file and start a new one ~ item.albumartist
path = os.path.join(directory, 'artists', self.writerest(directory)
slug(self.artist) + u'.rst')
with open(path, 'wb') as output:
output.write(self.rest.encode('utf-8'))
self.rest = None
if item is None:
return
self.artist = item.albumartist.strip() self.artist = item.albumartist.strip()
self.rest = u"%s\n%s\n\n.. contents::\n :local:\n\n" \ self.rest = "%s\n%s\n\n.. contents::\n :local:\n\n" \
% (self.artist, % (self.artist,
u'=' * len(self.artist)) '=' * len(self.artist))
if self.album != item.album: if self.album != item.album:
tmpalbum = self.album = item.album.strip() tmpalbum = self.album = item.album.strip()
if self.album == '': if self.album == '':
tmpalbum = u'Unknown album' tmpalbum = 'Unknown album'
self.rest += u"%s\n%s\n\n" % (tmpalbum, u'-' * len(tmpalbum)) self.rest += "{}\n{}\n\n".format(tmpalbum, '-' * len(tmpalbum))
title_str = u":index:`%s`" % item.title.strip() title_str = ":index:`%s`" % item.title.strip()
block = u'| ' + item.lyrics.replace(u'\n', u'\n| ') block = '| ' + item.lyrics.replace('\n', '\n| ')
self.rest += u"%s\n%s\n\n%s\n\n" % (title_str, self.rest += "{}\n{}\n\n{}\n\n".format(title_str,
u'~' * len(title_str), '~' * len(title_str),
block) block)
def writerest(self, directory):
"""Write self.rest to a ReST file
"""
if self.rest is not None and self.artist is not None:
path = os.path.join(directory, 'artists',
slug(self.artist) + '.rst')
with open(path, 'wb') as output:
output.write(self.rest.encode('utf-8'))
def writerest_indexes(self, directory): def writerest_indexes(self, directory):
"""Write conf.py and index.rst files necessary for Sphinx """Write conf.py and index.rst files necessary for Sphinx
@ -832,7 +911,7 @@ class LyricsPlugin(plugins.BeetsPlugin):
""" """
# Skip if the item already has lyrics. # Skip if the item already has lyrics.
if not force and item.lyrics: if not force and item.lyrics:
self._log.info(u'lyrics already present: {0}', item) self._log.info('lyrics already present: {0}', item)
return return
lyrics = None lyrics = None
@ -841,10 +920,10 @@ class LyricsPlugin(plugins.BeetsPlugin):
if any(lyrics): if any(lyrics):
break break
lyrics = u"\n\n---\n\n".join([l for l in lyrics if l]) lyrics = "\n\n---\n\n".join([l for l in lyrics if l])
if lyrics: if lyrics:
self._log.info(u'fetched lyrics: {0}', item) self._log.info('fetched lyrics: {0}', item)
if HAS_LANGDETECT and self.config['bing_client_secret'].get(): if HAS_LANGDETECT and self.config['bing_client_secret'].get():
lang_from = langdetect.detect(lyrics) lang_from = langdetect.detect(lyrics)
if self.config['bing_lang_to'].get() != lang_from and ( if self.config['bing_lang_to'].get() != lang_from and (
@ -854,7 +933,7 @@ class LyricsPlugin(plugins.BeetsPlugin):
lyrics = self.append_translation( lyrics = self.append_translation(
lyrics, self.config['bing_lang_to']) lyrics, self.config['bing_lang_to'])
else: else:
self._log.info(u'lyrics not found: {0}', item) self._log.info('lyrics not found: {0}', item)
fallback = self.config['fallback'].get() fallback = self.config['fallback'].get()
if fallback: if fallback:
lyrics = fallback lyrics = fallback
@ -872,12 +951,12 @@ class LyricsPlugin(plugins.BeetsPlugin):
for backend in self.backends: for backend in self.backends:
lyrics = backend.fetch(artist, title) lyrics = backend.fetch(artist, title)
if lyrics: if lyrics:
self._log.debug(u'got lyrics from backend: {0}', self._log.debug('got lyrics from backend: {0}',
backend.__class__.__name__) backend.__class__.__name__)
return _scrape_strip_cruft(lyrics, True) return _scrape_strip_cruft(lyrics, True)
def append_translation(self, text, to_lang): def append_translation(self, text, to_lang):
import xml.etree.ElementTree as ET from xml.etree import ElementTree
if not self.bing_auth_token: if not self.bing_auth_token:
self.bing_auth_token = self.get_bing_access_token() self.bing_auth_token = self.get_bing_access_token()
@ -895,10 +974,11 @@ class LyricsPlugin(plugins.BeetsPlugin):
self.bing_auth_token = None self.bing_auth_token = None
return self.append_translation(text, to_lang) return self.append_translation(text, to_lang)
return text return text
lines_translated = ET.fromstring(r.text.encode('utf-8')).text lines_translated = ElementTree.fromstring(
r.text.encode('utf-8')).text
# Use a translation mapping dict to build resulting lyrics # Use a translation mapping dict to build resulting lyrics
translations = dict(zip(text_lines, lines_translated.split('|'))) translations = dict(zip(text_lines, lines_translated.split('|')))
result = '' result = ''
for line in text.split('\n'): for line in text.split('\n'):
result += '%s / %s\n' % (line, translations[line]) result += '{} / {}\n'.format(line, translations[line])
return result return result

28
libs/common/beetsplug/mbcollection.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright (c) 2011, Jeffrey Aylesworth <mail@jeffrey.red> # Copyright (c) 2011, Jeffrey Aylesworth <mail@jeffrey.red>
# #
@ -13,7 +12,6 @@
# The above copyright notice and this permission notice shall be # The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software. # included in all copies or substantial portions of the Software.
from __future__ import division, absolute_import, print_function
from beets.plugins import BeetsPlugin from beets.plugins import BeetsPlugin
from beets.ui import Subcommand from beets.ui import Subcommand
@ -34,11 +32,11 @@ def mb_call(func, *args, **kwargs):
try: try:
return func(*args, **kwargs) return func(*args, **kwargs)
except musicbrainzngs.AuthenticationError: except musicbrainzngs.AuthenticationError:
raise ui.UserError(u'authentication with MusicBrainz failed') raise ui.UserError('authentication with MusicBrainz failed')
except (musicbrainzngs.ResponseError, musicbrainzngs.NetworkError) as exc: except (musicbrainzngs.ResponseError, musicbrainzngs.NetworkError) as exc:
raise ui.UserError(u'MusicBrainz API error: {0}'.format(exc)) raise ui.UserError(f'MusicBrainz API error: {exc}')
except musicbrainzngs.UsageError: except musicbrainzngs.UsageError:
raise ui.UserError(u'MusicBrainz credentials missing') raise ui.UserError('MusicBrainz credentials missing')
def submit_albums(collection_id, release_ids): def submit_albums(collection_id, release_ids):
@ -55,7 +53,7 @@ def submit_albums(collection_id, release_ids):
class MusicBrainzCollectionPlugin(BeetsPlugin): class MusicBrainzCollectionPlugin(BeetsPlugin):
def __init__(self): def __init__(self):
super(MusicBrainzCollectionPlugin, self).__init__() super().__init__()
config['musicbrainz']['pass'].redact = True config['musicbrainz']['pass'].redact = True
musicbrainzngs.auth( musicbrainzngs.auth(
config['musicbrainz']['user'].as_str(), config['musicbrainz']['user'].as_str(),
@ -63,7 +61,7 @@ class MusicBrainzCollectionPlugin(BeetsPlugin):
) )
self.config.add({ self.config.add({
'auto': False, 'auto': False,
'collection': u'', 'collection': '',
'remove': False, 'remove': False,
}) })
if self.config['auto']: if self.config['auto']:
@ -72,18 +70,18 @@ class MusicBrainzCollectionPlugin(BeetsPlugin):
def _get_collection(self): def _get_collection(self):
collections = mb_call(musicbrainzngs.get_collections) collections = mb_call(musicbrainzngs.get_collections)
if not collections['collection-list']: if not collections['collection-list']:
raise ui.UserError(u'no collections exist for user') raise ui.UserError('no collections exist for user')
# Get all collection IDs, avoiding event collections # Get all collection IDs, avoiding event collections
collection_ids = [x['id'] for x in collections['collection-list']] collection_ids = [x['id'] for x in collections['collection-list']]
if not collection_ids: if not collection_ids:
raise ui.UserError(u'No collection found.') raise ui.UserError('No collection found.')
# Check that the collection exists so we can present a nice error # Check that the collection exists so we can present a nice error
collection = self.config['collection'].as_str() collection = self.config['collection'].as_str()
if collection: if collection:
if collection not in collection_ids: if collection not in collection_ids:
raise ui.UserError(u'invalid collection ID: {}' raise ui.UserError('invalid collection ID: {}'
.format(collection)) .format(collection))
return collection return collection
@ -110,7 +108,7 @@ class MusicBrainzCollectionPlugin(BeetsPlugin):
def commands(self): def commands(self):
mbupdate = Subcommand('mbupdate', mbupdate = Subcommand('mbupdate',
help=u'Update MusicBrainz collection') help='Update MusicBrainz collection')
mbupdate.parser.add_option('-r', '--remove', mbupdate.parser.add_option('-r', '--remove',
action='store_true', action='store_true',
default=None, default=None,
@ -120,7 +118,7 @@ class MusicBrainzCollectionPlugin(BeetsPlugin):
return [mbupdate] return [mbupdate]
def remove_missing(self, collection_id, lib_albums): def remove_missing(self, collection_id, lib_albums):
lib_ids = set([x.mb_albumid for x in lib_albums]) lib_ids = {x.mb_albumid for x in lib_albums}
albums_in_collection = self._get_albums_in_collection(collection_id) albums_in_collection = self._get_albums_in_collection(collection_id)
remove_me = list(set(albums_in_collection) - lib_ids) remove_me = list(set(albums_in_collection) - lib_ids)
for i in range(0, len(remove_me), FETCH_CHUNK_SIZE): for i in range(0, len(remove_me), FETCH_CHUNK_SIZE):
@ -154,13 +152,13 @@ class MusicBrainzCollectionPlugin(BeetsPlugin):
if re.match(UUID_REGEX, aid): if re.match(UUID_REGEX, aid):
album_ids.append(aid) album_ids.append(aid)
else: else:
self._log.info(u'skipping invalid MBID: {0}', aid) self._log.info('skipping invalid MBID: {0}', aid)
# Submit to MusicBrainz. # Submit to MusicBrainz.
self._log.info( self._log.info(
u'Updating MusicBrainz collection {0}...', collection_id 'Updating MusicBrainz collection {0}...', collection_id
) )
submit_albums(collection_id, album_ids) submit_albums(collection_id, album_ids)
if remove_missing: if remove_missing:
self.remove_missing(collection_id, lib.albums()) self.remove_missing(collection_id, lib.albums())
self._log.info(u'...MusicBrainz collection updated.') self._log.info('...MusicBrainz collection updated.')

11
libs/common/beetsplug/mbsubmit.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson and Diego Moreda. # Copyright 2016, Adrian Sampson and Diego Moreda.
# #
@ -19,11 +18,9 @@ This plugin allows the user to print track information in a format that is
parseable by the MusicBrainz track parser [1]. Programmatic submitting is not parseable by the MusicBrainz track parser [1]. Programmatic submitting is not
implemented by MusicBrainz yet. implemented by MusicBrainz yet.
[1] http://wiki.musicbrainz.org/History:How_To_Parse_Track_Listings [1] https://wiki.musicbrainz.org/History:How_To_Parse_Track_Listings
""" """
from __future__ import division, absolute_import, print_function
from beets.autotag import Recommendation from beets.autotag import Recommendation
from beets.plugins import BeetsPlugin from beets.plugins import BeetsPlugin
@ -33,10 +30,10 @@ from beetsplug.info import print_data
class MBSubmitPlugin(BeetsPlugin): class MBSubmitPlugin(BeetsPlugin):
def __init__(self): def __init__(self):
super(MBSubmitPlugin, self).__init__() super().__init__()
self.config.add({ self.config.add({
'format': u'$track. $title - $artist ($length)', 'format': '$track. $title - $artist ($length)',
'threshold': 'medium', 'threshold': 'medium',
}) })
@ -53,7 +50,7 @@ class MBSubmitPlugin(BeetsPlugin):
def before_choose_candidate_event(self, session, task): def before_choose_candidate_event(self, session, task):
if task.rec <= self.threshold: if task.rec <= self.threshold:
return [PromptChoice(u'p', u'Print tracks', self.print_tracks)] return [PromptChoice('p', 'Print tracks', self.print_tracks)]
def print_tracks(self, session, task): def print_tracks(self, session, task):
for i in sorted(task.items, key=lambda i: i.track): for i in sorted(task.items, key=lambda i: i.track):

65
libs/common/beetsplug/mbsync.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Jakob Schnitzer. # Copyright 2016, Jakob Schnitzer.
# #
@ -15,47 +14,37 @@
"""Update library's tags using MusicBrainz. """Update library's tags using MusicBrainz.
""" """
from __future__ import division, absolute_import, print_function
from beets.plugins import BeetsPlugin from beets.plugins import BeetsPlugin, apply_item_changes
from beets import autotag, library, ui, util from beets import autotag, library, ui, util
from beets.autotag import hooks from beets.autotag import hooks
from collections import defaultdict from collections import defaultdict
import re
def apply_item_changes(lib, item, move, pretend, write): MBID_REGEX = r"(\d|\w){8}-(\d|\w){4}-(\d|\w){4}-(\d|\w){4}-(\d|\w){12}"
"""Store, move and write the item according to the arguments.
"""
if not pretend:
# Move the item if it's in the library.
if move and lib.directory in util.ancestry(item.path):
item.move(with_album=False)
if write:
item.try_write()
item.store()
class MBSyncPlugin(BeetsPlugin): class MBSyncPlugin(BeetsPlugin):
def __init__(self): def __init__(self):
super(MBSyncPlugin, self).__init__() super().__init__()
def commands(self): def commands(self):
cmd = ui.Subcommand('mbsync', cmd = ui.Subcommand('mbsync',
help=u'update metadata from musicbrainz') help='update metadata from musicbrainz')
cmd.parser.add_option( cmd.parser.add_option(
u'-p', u'--pretend', action='store_true', '-p', '--pretend', action='store_true',
help=u'show all changes but do nothing') help='show all changes but do nothing')
cmd.parser.add_option( cmd.parser.add_option(
u'-m', u'--move', action='store_true', dest='move', '-m', '--move', action='store_true', dest='move',
help=u"move files in the library directory") help="move files in the library directory")
cmd.parser.add_option( cmd.parser.add_option(
u'-M', u'--nomove', action='store_false', dest='move', '-M', '--nomove', action='store_false', dest='move',
help=u"don't move files in library") help="don't move files in library")
cmd.parser.add_option( cmd.parser.add_option(
u'-W', u'--nowrite', action='store_false', '-W', '--nowrite', action='store_false',
default=None, dest='write', default=None, dest='write',
help=u"don't write updated metadata to files") help="don't write updated metadata to files")
cmd.parser.add_format_option() cmd.parser.add_format_option()
cmd.func = self.func cmd.func = self.func
return [cmd] return [cmd]
@ -75,17 +64,23 @@ class MBSyncPlugin(BeetsPlugin):
"""Retrieve and apply info from the autotagger for items matched by """Retrieve and apply info from the autotagger for items matched by
query. query.
""" """
for item in lib.items(query + [u'singleton:true']): for item in lib.items(query + ['singleton:true']):
item_formatted = format(item) item_formatted = format(item)
if not item.mb_trackid: if not item.mb_trackid:
self._log.info(u'Skipping singleton with no mb_trackid: {0}', self._log.info('Skipping singleton with no mb_trackid: {0}',
item_formatted) item_formatted)
continue continue
# Do we have a valid MusicBrainz track ID?
if not re.match(MBID_REGEX, item.mb_trackid):
self._log.info('Skipping singleton with invalid mb_trackid:' +
' {0}', item_formatted)
continue
# Get the MusicBrainz recording info. # Get the MusicBrainz recording info.
track_info = hooks.track_for_mbid(item.mb_trackid) track_info = hooks.track_for_mbid(item.mb_trackid)
if not track_info: if not track_info:
self._log.info(u'Recording ID not found: {0} for track {0}', self._log.info('Recording ID not found: {0} for track {0}',
item.mb_trackid, item.mb_trackid,
item_formatted) item_formatted)
continue continue
@ -103,16 +98,22 @@ class MBSyncPlugin(BeetsPlugin):
for a in lib.albums(query): for a in lib.albums(query):
album_formatted = format(a) album_formatted = format(a)
if not a.mb_albumid: if not a.mb_albumid:
self._log.info(u'Skipping album with no mb_albumid: {0}', self._log.info('Skipping album with no mb_albumid: {0}',
album_formatted) album_formatted)
continue continue
items = list(a.items()) items = list(a.items())
# Do we have a valid MusicBrainz album ID?
if not re.match(MBID_REGEX, a.mb_albumid):
self._log.info('Skipping album with invalid mb_albumid: {0}',
album_formatted)
continue
# Get the MusicBrainz album information. # Get the MusicBrainz album information.
album_info = hooks.album_for_mbid(a.mb_albumid) album_info = hooks.album_for_mbid(a.mb_albumid)
if not album_info: if not album_info:
self._log.info(u'Release ID {0} not found for album {1}', self._log.info('Release ID {0} not found for album {1}',
a.mb_albumid, a.mb_albumid,
album_formatted) album_formatted)
continue continue
@ -120,7 +121,7 @@ class MBSyncPlugin(BeetsPlugin):
# Map release track and recording MBIDs to their information. # Map release track and recording MBIDs to their information.
# Recordings can appear multiple times on a release, so each MBID # Recordings can appear multiple times on a release, so each MBID
# maps to a list of TrackInfo objects. # maps to a list of TrackInfo objects.
releasetrack_index = dict() releasetrack_index = {}
track_index = defaultdict(list) track_index = defaultdict(list)
for track_info in album_info.tracks: for track_info in album_info.tracks:
releasetrack_index[track_info.release_track_id] = track_info releasetrack_index[track_info.release_track_id] = track_info
@ -148,7 +149,7 @@ class MBSyncPlugin(BeetsPlugin):
break break
# Apply. # Apply.
self._log.debug(u'applying changes to {}', album_formatted) self._log.debug('applying changes to {}', album_formatted)
with lib.transaction(): with lib.transaction():
autotag.apply_metadata(album_info, mapping) autotag.apply_metadata(album_info, mapping)
changed = False changed = False
@ -173,5 +174,5 @@ class MBSyncPlugin(BeetsPlugin):
# Move album art (and any inconsistent items). # Move album art (and any inconsistent items).
if move and lib.directory in util.ancestry(items[0].path): if move and lib.directory in util.ancestry(items[0].path):
self._log.debug(u'moving album {0}', album_formatted) self._log.debug('moving album {0}', album_formatted)
a.move() a.move()

19
libs/common/beetsplug/metasync/__init__.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Heinz Wiesinger. # Copyright 2016, Heinz Wiesinger.
# #
@ -16,15 +15,13 @@
"""Synchronize information from music player libraries """Synchronize information from music player libraries
""" """
from __future__ import division, absolute_import, print_function
from abc import abstractmethod, ABCMeta from abc import abstractmethod, ABCMeta
from importlib import import_module from importlib import import_module
from beets.util.confit import ConfigValueError from confuse import ConfigValueError
from beets import ui from beets import ui
from beets.plugins import BeetsPlugin from beets.plugins import BeetsPlugin
import six
METASYNC_MODULE = 'beetsplug.metasync' METASYNC_MODULE = 'beetsplug.metasync'
@ -36,7 +33,7 @@ SOURCES = {
} }
class MetaSource(six.with_metaclass(ABCMeta, object)): class MetaSource(metaclass=ABCMeta):
def __init__(self, config, log): def __init__(self, config, log):
self.item_types = {} self.item_types = {}
self.config = config self.config = config
@ -77,7 +74,7 @@ class MetaSyncPlugin(BeetsPlugin):
item_types = load_item_types() item_types = load_item_types()
def __init__(self): def __init__(self):
super(MetaSyncPlugin, self).__init__() super().__init__()
def commands(self): def commands(self):
cmd = ui.Subcommand('metasync', cmd = ui.Subcommand('metasync',
@ -108,7 +105,7 @@ class MetaSyncPlugin(BeetsPlugin):
# Avoid needlessly instantiating meta sources (can be expensive) # Avoid needlessly instantiating meta sources (can be expensive)
if not items: if not items:
self._log.info(u'No items found matching query') self._log.info('No items found matching query')
return return
# Instantiate the meta sources # Instantiate the meta sources
@ -116,18 +113,18 @@ class MetaSyncPlugin(BeetsPlugin):
try: try:
cls = META_SOURCES[player] cls = META_SOURCES[player]
except KeyError: except KeyError:
self._log.error(u'Unknown metadata source \'{0}\''.format( self._log.error('Unknown metadata source \'{}\''.format(
player)) player))
try: try:
meta_source_instances[player] = cls(self.config, self._log) meta_source_instances[player] = cls(self.config, self._log)
except (ImportError, ConfigValueError) as e: except (ImportError, ConfigValueError) as e:
self._log.error(u'Failed to instantiate metadata source ' self._log.error('Failed to instantiate metadata source '
u'\'{0}\': {1}'.format(player, e)) '\'{}\': {}'.format(player, e))
# Avoid needlessly iterating over items # Avoid needlessly iterating over items
if not meta_source_instances: if not meta_source_instances:
self._log.error(u'No valid metadata sources found') self._log.error('No valid metadata sources found')
return return
# Sync the items with all of the meta sources # Sync the items with all of the meta sources

8
libs/common/beetsplug/metasync/amarok.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Heinz Wiesinger. # Copyright 2016, Heinz Wiesinger.
# #
@ -16,7 +15,6 @@
"""Synchronize information from amarok's library via dbus """Synchronize information from amarok's library via dbus
""" """
from __future__ import division, absolute_import, print_function
from os.path import basename from os.path import basename
from datetime import datetime from datetime import datetime
@ -49,14 +47,14 @@ class Amarok(MetaSource):
'amarok_lastplayed': DateType(), 'amarok_lastplayed': DateType(),
} }
queryXML = u'<query version="1.0"> \ query_xml = '<query version="1.0"> \
<filters> \ <filters> \
<and><include field="filename" value=%s /></and> \ <and><include field="filename" value=%s /></and> \
</filters> \ </filters> \
</query>' </query>'
def __init__(self, config, log): def __init__(self, config, log):
super(Amarok, self).__init__(config, log) super().__init__(config, log)
if not dbus: if not dbus:
raise ImportError('failed to import dbus') raise ImportError('failed to import dbus')
@ -72,7 +70,7 @@ class Amarok(MetaSource):
# of the result set. So query for the filename and then try to match # of the result set. So query for the filename and then try to match
# the correct item from the results we get back # the correct item from the results we get back
results = self.collection.Query( results = self.collection.Query(
self.queryXML % quoteattr(basename(path)) self.query_xml % quoteattr(basename(path))
) )
for result in results: for result in results:
if result['xesam:url'] != path: if result['xesam:url'] != path:

38
libs/common/beetsplug/metasync/itunes.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Tom Jaspers. # Copyright 2016, Tom Jaspers.
# #
@ -16,7 +15,6 @@
"""Synchronize information from iTunes's library """Synchronize information from iTunes's library
""" """
from __future__ import division, absolute_import, print_function
from contextlib import contextmanager from contextlib import contextmanager
import os import os
@ -24,13 +22,13 @@ import shutil
import tempfile import tempfile
import plistlib import plistlib
from six.moves.urllib.parse import urlparse, unquote from urllib.parse import urlparse, unquote
from time import mktime from time import mktime
from beets import util from beets import util
from beets.dbcore import types from beets.dbcore import types
from beets.library import DateType from beets.library import DateType
from beets.util.confit import ConfigValueError from confuse import ConfigValueError
from beetsplug.metasync import MetaSource from beetsplug.metasync import MetaSource
@ -63,15 +61,16 @@ def _norm_itunes_path(path):
class Itunes(MetaSource): class Itunes(MetaSource):
item_types = { item_types = {
'itunes_rating': types.INTEGER, # 0..100 scale 'itunes_rating': types.INTEGER, # 0..100 scale
'itunes_playcount': types.INTEGER, 'itunes_playcount': types.INTEGER,
'itunes_skipcount': types.INTEGER, 'itunes_skipcount': types.INTEGER,
'itunes_lastplayed': DateType(), 'itunes_lastplayed': DateType(),
'itunes_lastskipped': DateType(), 'itunes_lastskipped': DateType(),
'itunes_dateadded': DateType(),
} }
def __init__(self, config, log): def __init__(self, config, log):
super(Itunes, self).__init__(config, log) super().__init__(config, log)
config.add({'itunes': { config.add({'itunes': {
'library': '~/Music/iTunes/iTunes Library.xml' 'library': '~/Music/iTunes/iTunes Library.xml'
@ -82,19 +81,20 @@ class Itunes(MetaSource):
try: try:
self._log.debug( self._log.debug(
u'loading iTunes library from {0}'.format(library_path)) f'loading iTunes library from {library_path}')
with create_temporary_copy(library_path) as library_copy: with create_temporary_copy(library_path) as library_copy:
raw_library = plistlib.readPlist(library_copy) with open(library_copy, 'rb') as library_copy_f:
except IOError as e: raw_library = plistlib.load(library_copy_f)
raise ConfigValueError(u'invalid iTunes library: ' + e.strerror) except OSError as e:
raise ConfigValueError('invalid iTunes library: ' + e.strerror)
except Exception: except Exception:
# It's likely the user configured their '.itl' library (<> xml) # It's likely the user configured their '.itl' library (<> xml)
if os.path.splitext(library_path)[1].lower() != '.xml': if os.path.splitext(library_path)[1].lower() != '.xml':
hint = u': please ensure that the configured path' \ hint = ': please ensure that the configured path' \
u' points to the .XML library' ' points to the .XML library'
else: else:
hint = '' hint = ''
raise ConfigValueError(u'invalid iTunes library' + hint) raise ConfigValueError('invalid iTunes library' + hint)
# Make the iTunes library queryable using the path # Make the iTunes library queryable using the path
self.collection = {_norm_itunes_path(track['Location']): track self.collection = {_norm_itunes_path(track['Location']): track
@ -105,7 +105,7 @@ class Itunes(MetaSource):
result = self.collection.get(util.bytestring_path(item.path).lower()) result = self.collection.get(util.bytestring_path(item.path).lower())
if not result: if not result:
self._log.warning(u'no iTunes match found for {0}'.format(item)) self._log.warning(f'no iTunes match found for {item}')
return return
item.itunes_rating = result.get('Rating') item.itunes_rating = result.get('Rating')
@ -119,3 +119,7 @@ class Itunes(MetaSource):
if result.get('Skip Date'): if result.get('Skip Date'):
item.itunes_lastskipped = mktime( item.itunes_lastskipped = mktime(
result.get('Skip Date').timetuple()) result.get('Skip Date').timetuple())
if result.get('Date Added'):
item.itunes_dateadded = mktime(
result.get('Date Added').timetuple())

30
libs/common/beetsplug/missing.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Pedro Silva. # Copyright 2016, Pedro Silva.
# Copyright 2017, Quentin Young. # Copyright 2017, Quentin Young.
@ -16,7 +15,6 @@
"""List missing tracks. """List missing tracks.
""" """
from __future__ import division, absolute_import, print_function
import musicbrainzngs import musicbrainzngs
@ -93,7 +91,7 @@ class MissingPlugin(BeetsPlugin):
} }
def __init__(self): def __init__(self):
super(MissingPlugin, self).__init__() super().__init__()
self.config.add({ self.config.add({
'count': False, 'count': False,
@ -107,14 +105,14 @@ class MissingPlugin(BeetsPlugin):
help=__doc__, help=__doc__,
aliases=['miss']) aliases=['miss'])
self._command.parser.add_option( self._command.parser.add_option(
u'-c', u'--count', dest='count', action='store_true', '-c', '--count', dest='count', action='store_true',
help=u'count missing tracks per album') help='count missing tracks per album')
self._command.parser.add_option( self._command.parser.add_option(
u'-t', u'--total', dest='total', action='store_true', '-t', '--total', dest='total', action='store_true',
help=u'count total of missing tracks') help='count total of missing tracks')
self._command.parser.add_option( self._command.parser.add_option(
u'-a', u'--album', dest='album', action='store_true', '-a', '--album', dest='album', action='store_true',
help=u'show missing albums for artist instead of tracks') help='show missing albums for artist instead of tracks')
self._command.parser.add_format_option() self._command.parser.add_format_option()
def commands(self): def commands(self):
@ -173,10 +171,10 @@ class MissingPlugin(BeetsPlugin):
# build dict mapping artist to list of all albums # build dict mapping artist to list of all albums
for artist, albums in albums_by_artist.items(): for artist, albums in albums_by_artist.items():
if artist[1] is None or artist[1] == "": if artist[1] is None or artist[1] == "":
albs_no_mbid = [u"'" + a['album'] + u"'" for a in albums] albs_no_mbid = ["'" + a['album'] + "'" for a in albums]
self._log.info( self._log.info(
u"No musicbrainz ID for artist '{}' found in album(s) {}; " "No musicbrainz ID for artist '{}' found in album(s) {}; "
"skipping", artist[0], u", ".join(albs_no_mbid) "skipping", artist[0], ", ".join(albs_no_mbid)
) )
continue continue
@ -185,7 +183,7 @@ class MissingPlugin(BeetsPlugin):
release_groups = resp['release-group-list'] release_groups = resp['release-group-list']
except MusicBrainzError as err: except MusicBrainzError as err:
self._log.info( self._log.info(
u"Couldn't fetch info for artist '{}' ({}) - '{}'", "Couldn't fetch info for artist '{}' ({}) - '{}'",
artist[0], artist[1], err artist[0], artist[1], err
) )
continue continue
@ -207,7 +205,7 @@ class MissingPlugin(BeetsPlugin):
missing_titles = {rg['title'] for rg in missing} missing_titles = {rg['title'] for rg in missing}
for release_title in missing_titles: for release_title in missing_titles:
print_(u"{} - {}".format(artist[0], release_title)) print_("{} - {}".format(artist[0], release_title))
if total: if total:
print(total_missing) print(total_missing)
@ -216,13 +214,13 @@ class MissingPlugin(BeetsPlugin):
"""Query MusicBrainz to determine items missing from `album`. """Query MusicBrainz to determine items missing from `album`.
""" """
item_mbids = [x.mb_trackid for x in album.items()] item_mbids = [x.mb_trackid for x in album.items()]
if len([i for i in album.items()]) < album.albumtotal: if len(list(album.items())) < album.albumtotal:
# fetch missing items # fetch missing items
# TODO: Implement caching that without breaking other stuff # TODO: Implement caching that without breaking other stuff
album_info = hooks.album_for_mbid(album.mb_albumid) album_info = hooks.album_for_mbid(album.mb_albumid)
for track_info in getattr(album_info, 'tracks', []): for track_info in getattr(album_info, 'tracks', []):
if track_info.track_id not in item_mbids: if track_info.track_id not in item_mbids:
item = _item(track_info, album_info, album.id) item = _item(track_info, album_info, album.id)
self._log.debug(u'track {0} in album {1}', self._log.debug('track {0} in album {1}',
track_info.track_id, album_info.album_id) track_info.track_id, album_info.album_id)
yield item yield item

127
libs/common/beetsplug/mpdstats.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Peter Schnebel and Johann Klähn. # Copyright 2016, Peter Schnebel and Johann Klähn.
# #
@ -13,11 +12,8 @@
# The above copyright notice and this permission notice shall be # The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software. # included in all copies or substantial portions of the Software.
from __future__ import division, absolute_import, print_function
import mpd import mpd
import socket
import select
import time import time
import os import os
@ -45,14 +41,21 @@ def is_url(path):
return path.split('://', 1)[0] in ['http', 'https'] return path.split('://', 1)[0] in ['http', 'https']
class MPDClientWrapper(object): class MPDClientWrapper:
def __init__(self, log): def __init__(self, log):
self._log = log self._log = log
self.music_directory = ( self.music_directory = mpd_config['music_directory'].as_str()
mpd_config['music_directory'].as_str()) self.strip_path = mpd_config['strip_path'].as_str()
self.client = mpd.MPDClient(use_unicode=True) # Ensure strip_path end with '/'
if not self.strip_path.endswith('/'):
self.strip_path += '/'
self._log.debug('music_directory: {0}', self.music_directory)
self._log.debug('strip_path: {0}', self.strip_path)
self.client = mpd.MPDClient()
def connect(self): def connect(self):
"""Connect to the MPD. """Connect to the MPD.
@ -63,11 +66,11 @@ class MPDClientWrapper(object):
if host[0] in ['/', '~']: if host[0] in ['/', '~']:
host = os.path.expanduser(host) host = os.path.expanduser(host)
self._log.info(u'connecting to {0}:{1}', host, port) self._log.info('connecting to {0}:{1}', host, port)
try: try:
self.client.connect(host, port) self.client.connect(host, port)
except socket.error as e: except OSError as e:
raise ui.UserError(u'could not connect to MPD: {0}'.format(e)) raise ui.UserError(f'could not connect to MPD: {e}')
password = mpd_config['password'].as_str() password = mpd_config['password'].as_str()
if password: if password:
@ -75,7 +78,7 @@ class MPDClientWrapper(object):
self.client.password(password) self.client.password(password)
except mpd.CommandError as e: except mpd.CommandError as e:
raise ui.UserError( raise ui.UserError(
u'could not authenticate to MPD: {0}'.format(e) f'could not authenticate to MPD: {e}'
) )
def disconnect(self): def disconnect(self):
@ -90,12 +93,12 @@ class MPDClientWrapper(object):
""" """
try: try:
return getattr(self.client, command)() return getattr(self.client, command)()
except (select.error, mpd.ConnectionError) as err: except (OSError, mpd.ConnectionError) as err:
self._log.error(u'{0}', err) self._log.error('{0}', err)
if retries <= 0: if retries <= 0:
# if we exited without breaking, we couldn't reconnect in time :( # if we exited without breaking, we couldn't reconnect in time :(
raise ui.UserError(u'communication with MPD server failed') raise ui.UserError('communication with MPD server failed')
time.sleep(RETRY_INTERVAL) time.sleep(RETRY_INTERVAL)
@ -107,18 +110,26 @@ class MPDClientWrapper(object):
self.connect() self.connect()
return self.get(command, retries=retries - 1) return self.get(command, retries=retries - 1)
def playlist(self): def currentsong(self):
"""Return the currently active playlist. Prefixes paths with the """Return the path to the currently playing song, along with its
music_directory, to get the absolute path. songid. Prefixes paths with the music_directory, to get the absolute
path.
In some cases, we need to remove the local path from MPD server,
we replace 'strip_path' with ''.
`strip_path` defaults to ''.
""" """
result = {} result = None
for entry in self.get('playlistinfo'): entry = self.get('currentsong')
if 'file' in entry:
if not is_url(entry['file']): if not is_url(entry['file']):
result[entry['id']] = os.path.join( file = entry['file']
self.music_directory, entry['file']) if file.startswith(self.strip_path):
file = file[len(self.strip_path):]
result = os.path.join(self.music_directory, file)
else: else:
result[entry['id']] = entry['file'] result = entry['file']
return result self._log.debug('returning: {0}', result)
return result, entry.get('id')
def status(self): def status(self):
"""Return the current status of the MPD. """Return the current status of the MPD.
@ -132,7 +143,7 @@ class MPDClientWrapper(object):
return self.get('idle') return self.get('idle')
class MPDStats(object): class MPDStats:
def __init__(self, lib, log): def __init__(self, lib, log):
self.lib = lib self.lib = lib
self._log = log self._log = log
@ -164,7 +175,7 @@ class MPDStats(object):
if item: if item:
return item return item
else: else:
self._log.info(u'item not found: {0}', displayable_path(path)) self._log.info('item not found: {0}', displayable_path(path))
def update_item(self, item, attribute, value=None, increment=None): def update_item(self, item, attribute, value=None, increment=None):
"""Update the beets item. Set attribute to value or increment the value """Update the beets item. Set attribute to value or increment the value
@ -182,7 +193,7 @@ class MPDStats(object):
item[attribute] = value item[attribute] = value
item.store() item.store()
self._log.debug(u'updated: {0} = {1} [{2}]', self._log.debug('updated: {0} = {1} [{2}]',
attribute, attribute,
item[attribute], item[attribute],
displayable_path(item.path)) displayable_path(item.path))
@ -229,29 +240,31 @@ class MPDStats(object):
"""Updates the play count of a song. """Updates the play count of a song.
""" """
self.update_item(song['beets_item'], 'play_count', increment=1) self.update_item(song['beets_item'], 'play_count', increment=1)
self._log.info(u'played {0}', displayable_path(song['path'])) self._log.info('played {0}', displayable_path(song['path']))
def handle_skipped(self, song): def handle_skipped(self, song):
"""Updates the skip count of a song. """Updates the skip count of a song.
""" """
self.update_item(song['beets_item'], 'skip_count', increment=1) self.update_item(song['beets_item'], 'skip_count', increment=1)
self._log.info(u'skipped {0}', displayable_path(song['path'])) self._log.info('skipped {0}', displayable_path(song['path']))
def on_stop(self, status): def on_stop(self, status):
self._log.info(u'stop') self._log.info('stop')
if self.now_playing: # if the current song stays the same it means that we stopped on the
# current track and should not record a skip.
if self.now_playing and self.now_playing['id'] != status.get('songid'):
self.handle_song_change(self.now_playing) self.handle_song_change(self.now_playing)
self.now_playing = None self.now_playing = None
def on_pause(self, status): def on_pause(self, status):
self._log.info(u'pause') self._log.info('pause')
self.now_playing = None self.now_playing = None
def on_play(self, status): def on_play(self, status):
playlist = self.mpd.playlist()
path = playlist.get(status['songid']) path, songid = self.mpd.currentsong()
if not path: if not path:
return return
@ -276,16 +289,17 @@ class MPDStats(object):
self.handle_song_change(self.now_playing) self.handle_song_change(self.now_playing)
if is_url(path): if is_url(path):
self._log.info(u'playing stream {0}', displayable_path(path)) self._log.info('playing stream {0}', displayable_path(path))
self.now_playing = None self.now_playing = None
return return
self._log.info(u'playing {0}', displayable_path(path)) self._log.info('playing {0}', displayable_path(path))
self.now_playing = { self.now_playing = {
'started': time.time(), 'started': time.time(),
'remaining': remaining, 'remaining': remaining,
'path': path, 'path': path,
'id': songid,
'beets_item': self.get_item(path), 'beets_item': self.get_item(path),
} }
@ -305,7 +319,7 @@ class MPDStats(object):
if handler: if handler:
handler(status) handler(status)
else: else:
self._log.debug(u'unhandled status "{0}"', status) self._log.debug('unhandled status "{0}"', status)
events = self.mpd.events() events = self.mpd.events()
@ -313,37 +327,38 @@ class MPDStats(object):
class MPDStatsPlugin(plugins.BeetsPlugin): class MPDStatsPlugin(plugins.BeetsPlugin):
item_types = { item_types = {
'play_count': types.INTEGER, 'play_count': types.INTEGER,
'skip_count': types.INTEGER, 'skip_count': types.INTEGER,
'last_played': library.DateType(), 'last_played': library.DateType(),
'rating': types.FLOAT, 'rating': types.FLOAT,
} }
def __init__(self): def __init__(self):
super(MPDStatsPlugin, self).__init__() super().__init__()
mpd_config.add({ mpd_config.add({
'music_directory': config['directory'].as_filename(), 'music_directory': config['directory'].as_filename(),
'rating': True, 'strip_path': '',
'rating_mix': 0.75, 'rating': True,
'host': os.environ.get('MPD_HOST', u'localhost'), 'rating_mix': 0.75,
'port': 6600, 'host': os.environ.get('MPD_HOST', 'localhost'),
'password': u'', 'port': int(os.environ.get('MPD_PORT', 6600)),
'password': '',
}) })
mpd_config['password'].redact = True mpd_config['password'].redact = True
def commands(self): def commands(self):
cmd = ui.Subcommand( cmd = ui.Subcommand(
'mpdstats', 'mpdstats',
help=u'run a MPD client to gather play statistics') help='run a MPD client to gather play statistics')
cmd.parser.add_option( cmd.parser.add_option(
u'--host', dest='host', type='string', '--host', dest='host', type='string',
help=u'set the hostname of the server to connect to') help='set the hostname of the server to connect to')
cmd.parser.add_option( cmd.parser.add_option(
u'--port', dest='port', type='int', '--port', dest='port', type='int',
help=u'set the port of the MPD server to connect to') help='set the port of the MPD server to connect to')
cmd.parser.add_option( cmd.parser.add_option(
u'--password', dest='password', type='string', '--password', dest='password', type='string',
help=u'set the password of the MPD server to connect to') help='set the password of the MPD server to connect to')
def func(lib, opts, args): def func(lib, opts, args):
mpd_config.set_args(opts) mpd_config.set_args(opts)

27
libs/common/beetsplug/mpdupdate.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson. # Copyright 2016, Adrian Sampson.
# #
@ -21,19 +20,17 @@ Put something like the following in your config.yaml to configure:
port: 6600 port: 6600
password: seekrit password: seekrit
""" """
from __future__ import division, absolute_import, print_function
from beets.plugins import BeetsPlugin from beets.plugins import BeetsPlugin
import os import os
import socket import socket
from beets import config from beets import config
import six
# No need to introduce a dependency on an MPD library for such a # No need to introduce a dependency on an MPD library for such a
# simple use case. Here's a simple socket abstraction to make things # simple use case. Here's a simple socket abstraction to make things
# easier. # easier.
class BufferedSocket(object): class BufferedSocket:
"""Socket abstraction that allows reading by line.""" """Socket abstraction that allows reading by line."""
def __init__(self, host, port, sep=b'\n'): def __init__(self, host, port, sep=b'\n'):
if host[0] in ['/', '~']: if host[0] in ['/', '~']:
@ -66,11 +63,11 @@ class BufferedSocket(object):
class MPDUpdatePlugin(BeetsPlugin): class MPDUpdatePlugin(BeetsPlugin):
def __init__(self): def __init__(self):
super(MPDUpdatePlugin, self).__init__() super().__init__()
config['mpd'].add({ config['mpd'].add({
'host': os.environ.get('MPD_HOST', u'localhost'), 'host': os.environ.get('MPD_HOST', 'localhost'),
'port': 6600, 'port': int(os.environ.get('MPD_PORT', 6600)),
'password': u'', 'password': '',
}) })
config['mpd']['password'].redact = True config['mpd']['password'].redact = True
@ -100,21 +97,21 @@ class MPDUpdatePlugin(BeetsPlugin):
try: try:
s = BufferedSocket(host, port) s = BufferedSocket(host, port)
except socket.error as e: except OSError as e:
self._log.warning(u'MPD connection failed: {0}', self._log.warning('MPD connection failed: {0}',
six.text_type(e.strerror)) str(e.strerror))
return return
resp = s.readline() resp = s.readline()
if b'OK MPD' not in resp: if b'OK MPD' not in resp:
self._log.warning(u'MPD connection failed: {0!r}', resp) self._log.warning('MPD connection failed: {0!r}', resp)
return return
if password: if password:
s.send(b'password "%s"\n' % password.encode('utf8')) s.send(b'password "%s"\n' % password.encode('utf8'))
resp = s.readline() resp = s.readline()
if b'OK' not in resp: if b'OK' not in resp:
self._log.warning(u'Authentication failed: {0!r}', resp) self._log.warning('Authentication failed: {0!r}', resp)
s.send(b'close\n') s.send(b'close\n')
s.close() s.close()
return return
@ -122,8 +119,8 @@ class MPDUpdatePlugin(BeetsPlugin):
s.send(b'update\n') s.send(b'update\n')
resp = s.readline() resp = s.readline()
if b'updating_db' not in resp: if b'updating_db' not in resp:
self._log.warning(u'Update failed: {0!r}', resp) self._log.warning('Update failed: {0!r}', resp)
s.send(b'close\n') s.send(b'close\n')
s.close() s.close()
self._log.info(u'Database updated.') self._log.info('Database updated.')

211
libs/common/beetsplug/parentwork.py Normal file
View file

@ -0,0 +1,211 @@
# This file is part of beets.
# Copyright 2017, Dorian Soergel.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Gets parent work, its disambiguation and id, composer, composer sort name
and work composition date
"""
from beets import ui
from beets.plugins import BeetsPlugin
import musicbrainzngs
def direct_parent_id(mb_workid, work_date=None):
"""Given a Musicbrainz work id, find the id one of the works the work is
part of and the first composition date it encounters.
"""
work_info = musicbrainzngs.get_work_by_id(mb_workid,
includes=["work-rels",
"artist-rels"])
if 'artist-relation-list' in work_info['work'] and work_date is None:
for artist in work_info['work']['artist-relation-list']:
if artist['type'] == 'composer':
if 'end' in artist.keys():
work_date = artist['end']
if 'work-relation-list' in work_info['work']:
for direct_parent in work_info['work']['work-relation-list']:
if direct_parent['type'] == 'parts' \
and direct_parent.get('direction') == 'backward':
direct_id = direct_parent['work']['id']
return direct_id, work_date
return None, work_date
def work_parent_id(mb_workid):
"""Find the parent work id and composition date of a work given its id.
"""
work_date = None
while True:
new_mb_workid, work_date = direct_parent_id(mb_workid, work_date)
if not new_mb_workid:
return mb_workid, work_date
mb_workid = new_mb_workid
return mb_workid, work_date
def find_parentwork_info(mb_workid):
"""Get the MusicBrainz information dict about a parent work, including
the artist relations, and the composition date for a work's parent work.
"""
parent_id, work_date = work_parent_id(mb_workid)
work_info = musicbrainzngs.get_work_by_id(parent_id,
includes=["artist-rels"])
return work_info, work_date
class ParentWorkPlugin(BeetsPlugin):
def __init__(self):
super().__init__()
self.config.add({
'auto': False,
'force': False,
})
if self.config['auto']:
self.import_stages = [self.imported]
def commands(self):
def func(lib, opts, args):
self.config.set_args(opts)
force_parent = self.config['force'].get(bool)
write = ui.should_write()
for item in lib.items(ui.decargs(args)):
changed = self.find_work(item, force_parent)
if changed:
item.store()
if write:
item.try_write()
command = ui.Subcommand(
'parentwork',
help='fetch parent works, composers and dates')
command.parser.add_option(
'-f', '--force', dest='force',
action='store_true', default=None,
help='re-fetch when parent work is already present')
command.func = func
return [command]
def imported(self, session, task):
"""Import hook for fetching parent works automatically.
"""
force_parent = self.config['force'].get(bool)
for item in task.imported_items():
self.find_work(item, force_parent)
item.store()
def get_info(self, item, work_info):
"""Given the parent work info dict, fetch parent_composer,
parent_composer_sort, parentwork, parentwork_disambig, mb_workid and
composer_ids.
"""
parent_composer = []
parent_composer_sort = []
parentwork_info = {}
composer_exists = False
if 'artist-relation-list' in work_info['work']:
for artist in work_info['work']['artist-relation-list']:
if artist['type'] == 'composer':
composer_exists = True
parent_composer.append(artist['artist']['name'])
parent_composer_sort.append(artist['artist']['sort-name'])
if 'end' in artist.keys():
parentwork_info["parentwork_date"] = artist['end']
parentwork_info['parent_composer'] = ', '.join(parent_composer)
parentwork_info['parent_composer_sort'] = ', '.join(
parent_composer_sort)
if not composer_exists:
self._log.debug(
'no composer for {}; add one at '
'https://musicbrainz.org/work/{}',
item, work_info['work']['id'],
)
parentwork_info['parentwork'] = work_info['work']['title']
parentwork_info['mb_parentworkid'] = work_info['work']['id']
if 'disambiguation' in work_info['work']:
parentwork_info['parentwork_disambig'] = work_info[
'work']['disambiguation']
else:
parentwork_info['parentwork_disambig'] = None
return parentwork_info
def find_work(self, item, force):
"""Finds the parent work of a recording and populates the tags
accordingly.
The parent work is found recursively, by finding the direct parent
repeatedly until there are no more links in the chain. We return the
final, topmost work in the chain.
Namely, the tags parentwork, parentwork_disambig, mb_parentworkid,
parent_composer, parent_composer_sort and work_date are populated.
"""
if not item.mb_workid:
self._log.info('No work for {}, \
add one at https://musicbrainz.org/recording/{}', item, item.mb_trackid)
return
hasparent = hasattr(item, 'parentwork')
work_changed = True
if hasattr(item, 'parentwork_workid_current'):
work_changed = item.parentwork_workid_current != item.mb_workid
if force or not hasparent or work_changed:
try:
work_info, work_date = find_parentwork_info(item.mb_workid)
except musicbrainzngs.musicbrainz.WebServiceError as e:
self._log.debug("error fetching work: {}", e)
return
parent_info = self.get_info(item, work_info)
parent_info['parentwork_workid_current'] = item.mb_workid
if 'parent_composer' in parent_info:
self._log.debug("Work fetched: {} - {}",
parent_info['parentwork'],
parent_info['parent_composer'])
else:
self._log.debug("Work fetched: {} - no parent composer",
parent_info['parentwork'])
elif hasparent:
self._log.debug("{}: Work present, skipping", item)
return
# apply all non-null values to the item
for key, value in parent_info.items():
if value:
item[key] = value
if work_date:
item['work_date'] = work_date
return ui.show_model_changes(
item, fields=['parentwork', 'parentwork_disambig',
'mb_parentworkid', 'parent_composer',
'parent_composer_sort', 'work_date',
'parentwork_workid_current', 'parentwork_date'])

64
libs/common/beetsplug/permissions.py
View file

@ -1,7 +1,3 @@
# -*- coding: utf-8 -*-
from __future__ import division, absolute_import, print_function
"""Fixes file permissions after the file gets written on import. Put something """Fixes file permissions after the file gets written on import. Put something
like the following in your config.yaml to configure: like the following in your config.yaml to configure:
@ -13,7 +9,6 @@ import os
from beets import config, util from beets import config, util
from beets.plugins import BeetsPlugin from beets.plugins import BeetsPlugin
from beets.util import ancestry from beets.util import ancestry
import six
def convert_perm(perm): def convert_perm(perm):
@ -21,8 +16,8 @@ def convert_perm(perm):
Or, if `perm` is an integer, reinterpret it as an octal number that Or, if `perm` is an integer, reinterpret it as an octal number that
has been "misinterpreted" as decimal. has been "misinterpreted" as decimal.
""" """
if isinstance(perm, six.integer_types): if isinstance(perm, int):
perm = six.text_type(perm) perm = str(perm)
return int(perm, 8) return int(perm, 8)
@ -40,11 +35,11 @@ def assert_permissions(path, permission, log):
""" """
if not check_permissions(util.syspath(path), permission): if not check_permissions(util.syspath(path), permission):
log.warning( log.warning(
u'could not set permissions on {}', 'could not set permissions on {}',
util.displayable_path(path), util.displayable_path(path),
) )
log.debug( log.debug(
u'set permissions to {}, but permissions are now {}', 'set permissions to {}, but permissions are now {}',
permission, permission,
os.stat(util.syspath(path)).st_mode & 0o777, os.stat(util.syspath(path)).st_mode & 0o777,
) )
@ -60,20 +55,39 @@ def dirs_in_library(library, item):
class Permissions(BeetsPlugin): class Permissions(BeetsPlugin):
def __init__(self): def __init__(self):
super(Permissions, self).__init__() super().__init__()
# Adding defaults. # Adding defaults.
self.config.add({ self.config.add({
u'file': '644', 'file': '644',
u'dir': '755', 'dir': '755',
}) })
self.register_listener('item_imported', self.fix) self.register_listener('item_imported', self.fix)
self.register_listener('album_imported', self.fix) self.register_listener('album_imported', self.fix)
self.register_listener('art_set', self.fix_art)
def fix(self, lib, item=None, album=None): def fix(self, lib, item=None, album=None):
"""Fix the permissions for an imported Item or Album. """Fix the permissions for an imported Item or Album.
""" """
files = []
dirs = set()
if item:
files.append(item.path)
dirs.update(dirs_in_library(lib.directory, item.path))
elif album:
for album_item in album.items():
files.append(album_item.path)
dirs.update(dirs_in_library(lib.directory, album_item.path))
self.set_permissions(files=files, dirs=dirs)
def fix_art(self, album):
"""Fix the permission for Album art file.
"""
if album.artpath:
self.set_permissions(files=[album.artpath])
def set_permissions(self, files=[], dirs=[]):
# Get the configured permissions. The user can specify this either a # Get the configured permissions. The user can specify this either a
# string (in YAML quotes) or, for convenience, as an integer so the # string (in YAML quotes) or, for convenience, as an integer so the
# quotes can be omitted. In the latter case, we need to reinterpret the # quotes can be omitted. In the latter case, we need to reinterpret the
@ -83,21 +97,10 @@ class Permissions(BeetsPlugin):
file_perm = convert_perm(file_perm) file_perm = convert_perm(file_perm)
dir_perm = convert_perm(dir_perm) dir_perm = convert_perm(dir_perm)
# Create chmod_queue. for path in files:
file_chmod_queue = []
if item:
file_chmod_queue.append(item.path)
elif album:
for album_item in album.items():
file_chmod_queue.append(album_item.path)
# A set of directories to change permissions for.
dir_chmod_queue = set()
for path in file_chmod_queue:
# Changing permissions on the destination file. # Changing permissions on the destination file.
self._log.debug( self._log.debug(
u'setting file permissions on {}', 'setting file permissions on {}',
util.displayable_path(path), util.displayable_path(path),
) )
os.chmod(util.syspath(path), file_perm) os.chmod(util.syspath(path), file_perm)
@ -105,16 +108,11 @@ class Permissions(BeetsPlugin):
# Checks if the destination path has the permissions configured. # Checks if the destination path has the permissions configured.
assert_permissions(path, file_perm, self._log) assert_permissions(path, file_perm, self._log)
# Adding directories to the directory chmod queue.
dir_chmod_queue.update(
dirs_in_library(lib.directory,
path))
# Change permissions for the directories. # Change permissions for the directories.
for path in dir_chmod_queue: for path in dirs:
# Chaning permissions on the destination directory. # Changing permissions on the destination directory.
self._log.debug( self._log.debug(
u'setting directory permissions on {}', 'setting directory permissions on {}',
util.displayable_path(path), util.displayable_path(path),
) )
os.chmod(util.syspath(path), dir_perm) os.chmod(util.syspath(path), dir_perm)

37
libs/common/beetsplug/play.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, David Hamp-Gonsalves # Copyright 2016, David Hamp-Gonsalves
# #
@ -15,7 +14,6 @@
"""Send the results of a query to the configured music player as a playlist. """Send the results of a query to the configured music player as a playlist.
""" """
from __future__ import division, absolute_import, print_function
from beets.plugins import BeetsPlugin from beets.plugins import BeetsPlugin
from beets.ui import Subcommand from beets.ui import Subcommand
@ -26,6 +24,7 @@ from beets import util
from os.path import relpath from os.path import relpath
from tempfile import NamedTemporaryFile from tempfile import NamedTemporaryFile
import subprocess import subprocess
import shlex
# Indicate where arguments should be inserted into the command string. # Indicate where arguments should be inserted into the command string.
# If this is missing, they're placed at the end. # If this is missing, they're placed at the end.
@ -39,25 +38,25 @@ def play(command_str, selection, paths, open_args, log, item_type='track',
""" """
# Print number of tracks or albums to be played, log command to be run. # Print number of tracks or albums to be played, log command to be run.
item_type += 's' if len(selection) > 1 else '' item_type += 's' if len(selection) > 1 else ''
ui.print_(u'Playing {0} {1}.'.format(len(selection), item_type)) ui.print_('Playing {} {}.'.format(len(selection), item_type))
log.debug(u'executing command: {} {!r}', command_str, open_args) log.debug('executing command: {} {!r}', command_str, open_args)
try: try:
if keep_open: if keep_open:
command = util.shlex_split(command_str) command = shlex.split(command_str)
command = command + open_args command = command + open_args
subprocess.call(command) subprocess.call(command)
else: else:
util.interactive_open(open_args, command_str) util.interactive_open(open_args, command_str)
except OSError as exc: except OSError as exc:
raise ui.UserError( raise ui.UserError(
"Could not play the query: {0}".format(exc)) f"Could not play the query: {exc}")
class PlayPlugin(BeetsPlugin): class PlayPlugin(BeetsPlugin):
def __init__(self): def __init__(self):
super(PlayPlugin, self).__init__() super().__init__()
config['play'].add({ config['play'].add({
'command': None, 'command': None,
@ -65,6 +64,7 @@ class PlayPlugin(BeetsPlugin):
'relative_to': None, 'relative_to': None,
'raw': False, 'raw': False,
'warning_threshold': 100, 'warning_threshold': 100,
'bom': False,
}) })
self.register_listener('before_choose_candidate', self.register_listener('before_choose_candidate',
@ -73,18 +73,18 @@ class PlayPlugin(BeetsPlugin):
def commands(self): def commands(self):
play_command = Subcommand( play_command = Subcommand(
'play', 'play',
help=u'send music to a player as a playlist' help='send music to a player as a playlist'
) )
play_command.parser.add_album_option() play_command.parser.add_album_option()
play_command.parser.add_option( play_command.parser.add_option(
u'-A', u'--args', '-A', '--args',
action='store', action='store',
help=u'add additional arguments to the command', help='add additional arguments to the command',
) )
play_command.parser.add_option( play_command.parser.add_option(
u'-y', u'--yes', '-y', '--yes',
action="store_true", action="store_true",
help=u'skip the warning threshold', help='skip the warning threshold',
) )
play_command.func = self._play_command play_command.func = self._play_command
return [play_command] return [play_command]
@ -123,7 +123,7 @@ class PlayPlugin(BeetsPlugin):
if not selection: if not selection:
ui.print_(ui.colorize('text_warning', ui.print_(ui.colorize('text_warning',
u'No {0} to play.'.format(item_type))) f'No {item_type} to play.'))
return return
open_args = self._playlist_or_paths(paths) open_args = self._playlist_or_paths(paths)
@ -147,7 +147,7 @@ class PlayPlugin(BeetsPlugin):
if ARGS_MARKER in command_str: if ARGS_MARKER in command_str:
return command_str.replace(ARGS_MARKER, args) return command_str.replace(ARGS_MARKER, args)
else: else:
return u"{} {}".format(command_str, args) return f"{command_str} {args}"
else: else:
# Don't include the marker in the command. # Don't include the marker in the command.
return command_str.replace(" " + ARGS_MARKER, "") return command_str.replace(" " + ARGS_MARKER, "")
@ -174,10 +174,10 @@ class PlayPlugin(BeetsPlugin):
ui.print_(ui.colorize( ui.print_(ui.colorize(
'text_warning', 'text_warning',
u'You are about to queue {0} {1}.'.format( 'You are about to queue {} {}.'.format(
len(selection), item_type))) len(selection), item_type)))
if ui.input_options((u'Continue', u'Abort')) == 'a': if ui.input_options(('Continue', 'Abort')) == 'a':
return True return True
return False return False
@ -185,7 +185,12 @@ class PlayPlugin(BeetsPlugin):
def _create_tmp_playlist(self, paths_list): def _create_tmp_playlist(self, paths_list):
"""Create a temporary .m3u file. Return the filename. """Create a temporary .m3u file. Return the filename.
""" """
utf8_bom = config['play']['bom'].get(bool)
m3u = NamedTemporaryFile('wb', suffix='.m3u', delete=False) m3u = NamedTemporaryFile('wb', suffix='.m3u', delete=False)
if utf8_bom:
m3u.write(b'\xEF\xBB\xBF')
for item in paths_list: for item in paths_list:
m3u.write(item + b'\n') m3u.write(item + b'\n')
m3u.close() m3u.close()

185
libs/common/beetsplug/playlist.py Normal file
View file

@ -0,0 +1,185 @@
# This file is part of beets.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
import os
import fnmatch
import tempfile
import beets
from beets.util import path_as_posix
class PlaylistQuery(beets.dbcore.Query):
"""Matches files listed by a playlist file.
"""
def __init__(self, pattern):
self.pattern = pattern
config = beets.config['playlist']
# Get the full path to the playlist
playlist_paths = (
pattern,
os.path.abspath(os.path.join(
config['playlist_dir'].as_filename(),
f'{pattern}.m3u',
)),
)
self.paths = []
for playlist_path in playlist_paths:
if not fnmatch.fnmatch(playlist_path, '*.[mM]3[uU]'):
# This is not am M3U playlist, skip this candidate
continue
try:
f = open(beets.util.syspath(playlist_path), mode='rb')
except OSError:
continue
if config['relative_to'].get() == 'library':
relative_to = beets.config['directory'].as_filename()
elif config['relative_to'].get() == 'playlist':
relative_to = os.path.dirname(playlist_path)
else:
relative_to = config['relative_to'].as_filename()
relative_to = beets.util.bytestring_path(relative_to)
for line in f:
if line[0] == '#':
# ignore comments, and extm3u extension
continue
self.paths.append(beets.util.normpath(
os.path.join(relative_to, line.rstrip())
))
f.close()
break
def col_clause(self):
if not self.paths:
# Playlist is empty
return '0', ()
clause = 'path IN ({})'.format(', '.join('?' for path in self.paths))
return clause, (beets.library.BLOB_TYPE(p) for p in self.paths)
def match(self, item):
return item.path in self.paths
class PlaylistPlugin(beets.plugins.BeetsPlugin):
item_queries = {'playlist': PlaylistQuery}
def __init__(self):
super().__init__()
self.config.add({
'auto': False,
'playlist_dir': '.',
'relative_to': 'library',
'forward_slash': False,
})
self.playlist_dir = self.config['playlist_dir'].as_filename()
self.changes = {}
if self.config['relative_to'].get() == 'library':
self.relative_to = beets.util.bytestring_path(
beets.config['directory'].as_filename())
elif self.config['relative_to'].get() != 'playlist':
self.relative_to = beets.util.bytestring_path(
self.config['relative_to'].as_filename())
else:
self.relative_to = None
if self.config['auto']:
self.register_listener('item_moved', self.item_moved)
self.register_listener('item_removed', self.item_removed)
self.register_listener('cli_exit', self.cli_exit)
def item_moved(self, item, source, destination):
self.changes[source] = destination
def item_removed(self, item):
if not os.path.exists(beets.util.syspath(item.path)):
self.changes[item.path] = None
def cli_exit(self, lib):
for playlist in self.find_playlists():
self._log.info(f'Updating playlist: {playlist}')
base_dir = beets.util.bytestring_path(
self.relative_to if self.relative_to
else os.path.dirname(playlist)
)
try:
self.update_playlist(playlist, base_dir)
except beets.util.FilesystemError:
self._log.error('Failed to update playlist: {}'.format(
beets.util.displayable_path(playlist)))
def find_playlists(self):
"""Find M3U playlists in the playlist directory."""
try:
dir_contents = os.listdir(beets.util.syspath(self.playlist_dir))
except OSError:
self._log.warning('Unable to open playlist directory {}'.format(
beets.util.displayable_path(self.playlist_dir)))
return
for filename in dir_contents:
if fnmatch.fnmatch(filename, '*.[mM]3[uU]'):
yield os.path.join(self.playlist_dir, filename)
def update_playlist(self, filename, base_dir):
"""Find M3U playlists in the specified directory."""
changes = 0
deletions = 0
with tempfile.NamedTemporaryFile(mode='w+b', delete=False) as tempfp:
new_playlist = tempfp.name
with open(filename, mode='rb') as fp:
for line in fp:
original_path = line.rstrip(b'\r\n')
# Ensure that path from playlist is absolute
is_relative = not os.path.isabs(line)
if is_relative:
lookup = os.path.join(base_dir, original_path)
else:
lookup = original_path
try:
new_path = self.changes[beets.util.normpath(lookup)]
except KeyError:
if self.config['forward_slash']:
line = path_as_posix(line)
tempfp.write(line)
else:
if new_path is None:
# Item has been deleted
deletions += 1
continue
changes += 1
if is_relative:
new_path = os.path.relpath(new_path, base_dir)
line = line.replace(original_path, new_path)
if self.config['forward_slash']:
line = path_as_posix(line)
tempfp.write(line)
if changes or deletions:
self._log.info(
'Updated playlist {} ({} changes, {} deletions)'.format(
filename, changes, deletions))
beets.util.copy(new_playlist, filename, replace=True)
beets.util.remove(new_playlist)

64
libs/common/beetsplug/plexupdate.py
View file

@ -1,5 +1,3 @@
# -*- coding: utf-8 -*-
"""Updates an Plex library whenever the beets library is changed. """Updates an Plex library whenever the beets library is changed.
Plex Home users enter the Plex Token to enable updating. Plex Home users enter the Plex Token to enable updating.
@ -9,42 +7,51 @@ Put something like the following in your config.yaml to configure:
port: 32400 port: 32400
token: token token: token
""" """
from __future__ import division, absolute_import, print_function
import requests import requests
import xml.etree.ElementTree as ET from xml.etree import ElementTree
from six.moves.urllib.parse import urljoin, urlencode from urllib.parse import urljoin, urlencode
from beets import config from beets import config
from beets.plugins import BeetsPlugin from beets.plugins import BeetsPlugin
def get_music_section(host, port, token, library_name): def get_music_section(host, port, token, library_name, secure,
ignore_cert_errors):
"""Getting the section key for the music library in Plex. """Getting the section key for the music library in Plex.
""" """
api_endpoint = append_token('library/sections', token) api_endpoint = append_token('library/sections', token)
url = urljoin('http://{0}:{1}'.format(host, port), api_endpoint) url = urljoin('{}://{}:{}'.format(get_protocol(secure), host,
port), api_endpoint)
# Sends request. # Sends request.
r = requests.get(url) r = requests.get(url, verify=not ignore_cert_errors)
# Parse xml tree and extract music section key. # Parse xml tree and extract music section key.
tree = ET.fromstring(r.content) tree = ElementTree.fromstring(r.content)
for child in tree.findall('Directory'): for child in tree.findall('Directory'):
if child.get('title') == library_name: if child.get('title') == library_name:
return child.get('key') return child.get('key')
def update_plex(host, port, token, library_name): def update_plex(host, port, token, library_name, secure,
ignore_cert_errors):
"""Ignore certificate errors if configured to.
"""
if ignore_cert_errors:
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
"""Sends request to the Plex api to start a library refresh. """Sends request to the Plex api to start a library refresh.
""" """
# Getting section key and build url. # Getting section key and build url.
section_key = get_music_section(host, port, token, library_name) section_key = get_music_section(host, port, token, library_name,
api_endpoint = 'library/sections/{0}/refresh'.format(section_key) secure, ignore_cert_errors)
api_endpoint = f'library/sections/{section_key}/refresh'
api_endpoint = append_token(api_endpoint, token) api_endpoint = append_token(api_endpoint, token)
url = urljoin('http://{0}:{1}'.format(host, port), api_endpoint) url = urljoin('{}://{}:{}'.format(get_protocol(secure), host,
port), api_endpoint)
# Sends request and returns requests object. # Sends request and returns requests object.
r = requests.get(url) r = requests.get(url, verify=not ignore_cert_errors)
return r return r
@ -56,16 +63,25 @@ def append_token(url, token):
return url return url
def get_protocol(secure):
if secure:
return 'https'
else:
return 'http'
class PlexUpdate(BeetsPlugin): class PlexUpdate(BeetsPlugin):
def __init__(self): def __init__(self):
super(PlexUpdate, self).__init__() super().__init__()
# Adding defaults. # Adding defaults.
config['plex'].add({ config['plex'].add({
u'host': u'localhost', 'host': 'localhost',
u'port': 32400, 'port': 32400,
u'token': u'', 'token': '',
u'library_name': u'Music'}) 'library_name': 'Music',
'secure': False,
'ignore_cert_errors': False})
config['plex']['token'].redact = True config['plex']['token'].redact = True
self.register_listener('database_change', self.listen_for_db_change) self.register_listener('database_change', self.listen_for_db_change)
@ -77,7 +93,7 @@ class PlexUpdate(BeetsPlugin):
def update(self, lib): def update(self, lib):
"""When the client exists try to send refresh request to Plex server. """When the client exists try to send refresh request to Plex server.
""" """
self._log.info(u'Updating Plex library...') self._log.info('Updating Plex library...')
# Try to send update request. # Try to send update request.
try: try:
@ -85,8 +101,10 @@ class PlexUpdate(BeetsPlugin):
config['plex']['host'].get(), config['plex']['host'].get(),
config['plex']['port'].get(), config['plex']['port'].get(),
config['plex']['token'].get(), config['plex']['token'].get(),
config['plex']['library_name'].get()) config['plex']['library_name'].get(),
self._log.info(u'... started.') config['plex']['secure'].get(bool),
config['plex']['ignore_cert_errors'].get(bool))
self._log.info('... started.')
except requests.exceptions.RequestException: except requests.exceptions.RequestException:
self._log.warning(u'Update failed.') self._log.warning('Update failed.')

108
libs/common/beetsplug/random.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Philippe Mongeau. # Copyright 2016, Philippe Mongeau.
# #
@ -15,101 +14,10 @@
"""Get a random song or album from the library. """Get a random song or album from the library.
""" """
from __future__ import division, absolute_import, print_function
from beets.plugins import BeetsPlugin from beets.plugins import BeetsPlugin
from beets.ui import Subcommand, decargs, print_ from beets.ui import Subcommand, decargs, print_
import random from beets.random import random_objs
from operator import attrgetter
from itertools import groupby
def _length(obj, album):
"""Get the duration of an item or album.
"""
if album:
return sum(i.length for i in obj.items())
else:
return obj.length
def _equal_chance_permutation(objs, field='albumartist'):
"""Generate (lazily) a permutation of the objects where every group
with equal values for `field` have an equal chance of appearing in
any given position.
"""
# Group the objects by artist so we can sample from them.
key = attrgetter(field)
objs.sort(key=key)
objs_by_artists = {}
for artist, v in groupby(objs, key):
objs_by_artists[artist] = list(v)
# While we still have artists with music to choose from, pick one
# randomly and pick a track from that artist.
while objs_by_artists:
# Choose an artist and an object for that artist, removing
# this choice from the pool.
artist = random.choice(list(objs_by_artists.keys()))
objs_from_artist = objs_by_artists[artist]
i = random.randint(0, len(objs_from_artist) - 1)
yield objs_from_artist.pop(i)
# Remove the artist if we've used up all of its objects.
if not objs_from_artist:
del objs_by_artists[artist]
def _take(iter, num):
"""Return a list containing the first `num` values in `iter` (or
fewer, if the iterable ends early).
"""
out = []
for val in iter:
out.append(val)
num -= 1
if num <= 0:
break
return out
def _take_time(iter, secs, album):
"""Return a list containing the first values in `iter`, which should
be Item or Album objects, that add up to the given amount of time in
seconds.
"""
out = []
total_time = 0.0
for obj in iter:
length = _length(obj, album)
if total_time + length <= secs:
out.append(obj)
total_time += length
return out
def random_objs(objs, album, number=1, time=None, equal_chance=False):
"""Get a random subset of the provided `objs`.
If `number` is provided, produce that many matches. Otherwise, if
`time` is provided, instead select a list whose total time is close
to that number of minutes. If `equal_chance` is true, give each
artist an equal chance of being included so that artists with more
songs are not represented disproportionately.
"""
# Permute the objects either in a straightforward way or an
# artist-balanced way.
if equal_chance:
perm = _equal_chance_permutation(objs)
else:
perm = objs
random.shuffle(perm) # N.B. This shuffles the original list.
# Select objects by time our count.
if time:
return _take_time(perm, time * 60, album)
else:
return _take(perm, number)
def random_func(lib, opts, args): def random_func(lib, opts, args):
@ -130,16 +38,16 @@ def random_func(lib, opts, args):
random_cmd = Subcommand('random', random_cmd = Subcommand('random',
help=u'choose a random track or album') help='choose a random track or album')
random_cmd.parser.add_option( random_cmd.parser.add_option(
u'-n', u'--number', action='store', type="int", '-n', '--number', action='store', type="int",
help=u'number of objects to choose', default=1) help='number of objects to choose', default=1)
random_cmd.parser.add_option( random_cmd.parser.add_option(
u'-e', u'--equal-chance', action='store_true', '-e', '--equal-chance', action='store_true',
help=u'each artist has the same chance') help='each artist has the same chance')
random_cmd.parser.add_option( random_cmd.parser.add_option(
u'-t', u'--time', action='store', type="float", '-t', '--time', action='store', type="float",
help=u'total length in minutes of objects to choose') help='total length in minutes of objects to choose')
random_cmd.parser.add_all_common_options() random_cmd.parser.add_all_common_options()
random_cmd.func = random_func random_cmd.func = random_func

1040
libs/common/beetsplug/replaygain.py
View file

File diff suppressed because it is too large Load diff

10
libs/common/beetsplug/rewrite.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson. # Copyright 2016, Adrian Sampson.
# #
@ -16,7 +15,6 @@
"""Uses user-specified rewriting rules to canonicalize names for path """Uses user-specified rewriting rules to canonicalize names for path
formats. formats.
""" """
from __future__ import division, absolute_import, print_function
import re import re
from collections import defaultdict from collections import defaultdict
@ -44,7 +42,7 @@ def rewriter(field, rules):
class RewritePlugin(BeetsPlugin): class RewritePlugin(BeetsPlugin):
def __init__(self): def __init__(self):
super(RewritePlugin, self).__init__() super().__init__()
self.config.add({}) self.config.add({})
@ -55,11 +53,11 @@ class RewritePlugin(BeetsPlugin):
try: try:
fieldname, pattern = key.split(None, 1) fieldname, pattern = key.split(None, 1)
except ValueError: except ValueError:
raise ui.UserError(u"invalid rewrite specification") raise ui.UserError("invalid rewrite specification")
if fieldname not in library.Item._fields: if fieldname not in library.Item._fields:
raise ui.UserError(u"invalid field name (%s) in rewriter" % raise ui.UserError("invalid field name (%s) in rewriter" %
fieldname) fieldname)
self._log.debug(u'adding template field {0}', key) self._log.debug('adding template field {0}', key)
pattern = re.compile(pattern.lower()) pattern = re.compile(pattern.lower())
rules[fieldname].append((pattern, value)) rules[fieldname].append((pattern, value))
if fieldname == 'artist': if fieldname == 'artist':

30
libs/common/beetsplug/scrub.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Adrian Sampson. # Copyright 2016, Adrian Sampson.
# #
@ -17,13 +16,12 @@
automatically whenever tags are written. automatically whenever tags are written.
""" """
from __future__ import division, absolute_import, print_function
from beets.plugins import BeetsPlugin from beets.plugins import BeetsPlugin
from beets import ui from beets import ui
from beets import util from beets import util
from beets import config from beets import config
from beets import mediafile import mediafile
import mutagen import mutagen
_MUTAGEN_FORMATS = { _MUTAGEN_FORMATS = {
@ -48,7 +46,7 @@ _MUTAGEN_FORMATS = {
class ScrubPlugin(BeetsPlugin): class ScrubPlugin(BeetsPlugin):
"""Removes extraneous metadata from files' tags.""" """Removes extraneous metadata from files' tags."""
def __init__(self): def __init__(self):
super(ScrubPlugin, self).__init__() super().__init__()
self.config.add({ self.config.add({
'auto': True, 'auto': True,
}) })
@ -60,15 +58,15 @@ class ScrubPlugin(BeetsPlugin):
def scrub_func(lib, opts, args): def scrub_func(lib, opts, args):
# Walk through matching files and remove tags. # Walk through matching files and remove tags.
for item in lib.items(ui.decargs(args)): for item in lib.items(ui.decargs(args)):
self._log.info(u'scrubbing: {0}', self._log.info('scrubbing: {0}',
util.displayable_path(item.path)) util.displayable_path(item.path))
self._scrub_item(item, opts.write) self._scrub_item(item, opts.write)
scrub_cmd = ui.Subcommand('scrub', help=u'clean audio tags') scrub_cmd = ui.Subcommand('scrub', help='clean audio tags')
scrub_cmd.parser.add_option( scrub_cmd.parser.add_option(
u'-W', u'--nowrite', dest='write', '-W', '--nowrite', dest='write',
action='store_false', default=True, action='store_false', default=True,
help=u'leave tags empty') help='leave tags empty')
scrub_cmd.func = scrub_func scrub_cmd.func = scrub_func
return [scrub_cmd] return [scrub_cmd]
@ -79,7 +77,7 @@ class ScrubPlugin(BeetsPlugin):
""" """
classes = [] classes = []
for modname, clsname in _MUTAGEN_FORMATS.items(): for modname, clsname in _MUTAGEN_FORMATS.items():
mod = __import__('mutagen.{0}'.format(modname), mod = __import__(f'mutagen.{modname}',
fromlist=[clsname]) fromlist=[clsname])
classes.append(getattr(mod, clsname)) classes.append(getattr(mod, clsname))
return classes return classes
@ -107,8 +105,8 @@ class ScrubPlugin(BeetsPlugin):
for tag in f.keys(): for tag in f.keys():
del f[tag] del f[tag]
f.save() f.save()
except (IOError, mutagen.MutagenError) as exc: except (OSError, mutagen.MutagenError) as exc:
self._log.error(u'could not scrub {0}: {1}', self._log.error('could not scrub {0}: {1}',
util.displayable_path(path), exc) util.displayable_path(path), exc)
def _scrub_item(self, item, restore=True): def _scrub_item(self, item, restore=True):
@ -121,7 +119,7 @@ class ScrubPlugin(BeetsPlugin):
mf = mediafile.MediaFile(util.syspath(item.path), mf = mediafile.MediaFile(util.syspath(item.path),
config['id3v23'].get(bool)) config['id3v23'].get(bool))
except mediafile.UnreadableFileError as exc: except mediafile.UnreadableFileError as exc:
self._log.error(u'could not open file to scrub: {0}', self._log.error('could not open file to scrub: {0}',
exc) exc)
return return
images = mf.images images = mf.images
@ -131,21 +129,21 @@ class ScrubPlugin(BeetsPlugin):
# Restore tags, if enabled. # Restore tags, if enabled.
if restore: if restore:
self._log.debug(u'writing new tags after scrub') self._log.debug('writing new tags after scrub')
item.try_write() item.try_write()
if images: if images:
self._log.debug(u'restoring art') self._log.debug('restoring art')
try: try:
mf = mediafile.MediaFile(util.syspath(item.path), mf = mediafile.MediaFile(util.syspath(item.path),
config['id3v23'].get(bool)) config['id3v23'].get(bool))
mf.images = images mf.images = images
mf.save() mf.save()
except mediafile.UnreadableFileError as exc: except mediafile.UnreadableFileError as exc:
self._log.error(u'could not write tags: {0}', exc) self._log.error('could not write tags: {0}', exc)
def import_task_files(self, session, task): def import_task_files(self, session, task):
"""Automatically scrub imported files.""" """Automatically scrub imported files."""
for item in task.imported_items(): for item in task.imported_items():
self._log.debug(u'auto-scrubbing {0}', self._log.debug('auto-scrubbing {0}',
util.displayable_path(item.path)) util.displayable_path(item.path))
self._scrub_item(item) self._scrub_item(item)

67
libs/common/beetsplug/smartplaylist.py
View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# This file is part of beets. # This file is part of beets.
# Copyright 2016, Dang Mai <contact@dangmai.net>. # Copyright 2016, Dang Mai <contact@dangmai.net>.
# #
@ -16,30 +15,38 @@
"""Generates smart playlists based on beets queries. """Generates smart playlists based on beets queries.
""" """
from __future__ import division, absolute_import, print_function
from beets.plugins import BeetsPlugin from beets.plugins import BeetsPlugin
from beets import ui from beets import ui
from beets.util import (mkdirall, normpath, sanitize_path, syspath, from beets.util import (mkdirall, normpath, sanitize_path, syspath,
bytestring_path) bytestring_path, path_as_posix)
from beets.library import Item, Album, parse_query_string from beets.library import Item, Album, parse_query_string
from beets.dbcore import OrQuery from beets.dbcore import OrQuery
from beets.dbcore.query import MultipleSort, ParsingError from beets.dbcore.query import MultipleSort, ParsingError
import os import os
import six
try:
from urllib.request import pathname2url
except ImportError:
# python2 is a bit different
from urllib import pathname2url
class SmartPlaylistPlugin(BeetsPlugin): class SmartPlaylistPlugin(BeetsPlugin):
def __init__(self): def __init__(self):
super(SmartPlaylistPlugin, self).__init__() super().__init__()
self.config.add({ self.config.add({
'relative_to': None, 'relative_to': None,
'playlist_dir': u'.', 'playlist_dir': '.',
'auto': True, 'auto': True,
'playlists': [] 'playlists': [],
'forward_slash': False,
'prefix': '',
'urlencode': False,
}) })
self.config['prefix'].redact = True # May contain username/password.
self._matched_playlists = None self._matched_playlists = None
self._unmatched_playlists = None self._unmatched_playlists = None
@ -49,8 +56,8 @@ class SmartPlaylistPlugin(BeetsPlugin):
def commands(self): def commands(self):
spl_update = ui.Subcommand( spl_update = ui.Subcommand(
'splupdate', 'splupdate',
help=u'update the smart playlists. Playlist names may be ' help='update the smart playlists. Playlist names may be '
u'passed as arguments.' 'passed as arguments.'
) )
spl_update.func = self.update_cmd spl_update.func = self.update_cmd
return [spl_update] return [spl_update]
@ -61,14 +68,14 @@ class SmartPlaylistPlugin(BeetsPlugin):
args = set(ui.decargs(args)) args = set(ui.decargs(args))
for a in list(args): for a in list(args):
if not a.endswith(".m3u"): if not a.endswith(".m3u"):
args.add("{0}.m3u".format(a)) args.add(f"{a}.m3u")
playlists = set((name, q, a_q) playlists = {(name, q, a_q)
for name, q, a_q in self._unmatched_playlists for name, q, a_q in self._unmatched_playlists
if name in args) if name in args}
if not playlists: if not playlists:
raise ui.UserError( raise ui.UserError(
u'No playlist matching any of {0} found'.format( 'No playlist matching any of {} found'.format(
[name for name, _, _ in self._unmatched_playlists]) [name for name, _, _ in self._unmatched_playlists])
) )
@ -81,7 +88,7 @@ class SmartPlaylistPlugin(BeetsPlugin):
def build_queries(self): def build_queries(self):
""" """
Instanciate queries for the playlists. Instantiate queries for the playlists.
Each playlist has 2 queries: one or items one for albums, each with a Each playlist has 2 queries: one or items one for albums, each with a
sort. We must also remember its name. _unmatched_playlists is a set of sort. We must also remember its name. _unmatched_playlists is a set of
@ -99,22 +106,23 @@ class SmartPlaylistPlugin(BeetsPlugin):
for playlist in self.config['playlists'].get(list): for playlist in self.config['playlists'].get(list):
if 'name' not in playlist: if 'name' not in playlist:
self._log.warning(u"playlist configuration is missing name") self._log.warning("playlist configuration is missing name")
continue continue
playlist_data = (playlist['name'],) playlist_data = (playlist['name'],)
try: try:
for key, Model in (('query', Item), ('album_query', Album)): for key, model_cls in (('query', Item),
('album_query', Album)):
qs = playlist.get(key) qs = playlist.get(key)
if qs is None: if qs is None:
query_and_sort = None, None query_and_sort = None, None
elif isinstance(qs, six.string_types): elif isinstance(qs, str):
query_and_sort = parse_query_string(qs, Model) query_and_sort = parse_query_string(qs, model_cls)
elif len(qs) == 1: elif len(qs) == 1:
query_and_sort = parse_query_string(qs[0], Model) query_and_sort = parse_query_string(qs[0], model_cls)
else: else:
# multiple queries and sorts # multiple queries and sorts
queries, sorts = zip(*(parse_query_string(q, Model) queries, sorts = zip(*(parse_query_string(q, model_cls)
for q in qs)) for q in qs))
query = OrQuery(queries) query = OrQuery(queries)
final_sorts = [] final_sorts = []
@ -135,7 +143,7 @@ class SmartPlaylistPlugin(BeetsPlugin):
playlist_data += (query_and_sort,) playlist_data += (query_and_sort,)
except ParsingError as exc: except ParsingError as exc:
self._log.warning(u"invalid query in playlist {}: {}", self._log.warning("invalid query in playlist {}: {}",
playlist['name'], exc) playlist['name'], exc)
continue continue
@ -156,14 +164,14 @@ class SmartPlaylistPlugin(BeetsPlugin):
n, (q, _), (a_q, _) = playlist n, (q, _), (a_q, _) = playlist
if self.matches(model, q, a_q): if self.matches(model, q, a_q):
self._log.debug( self._log.debug(
u"{0} will be updated because of {1}", n, model) "{0} will be updated because of {1}", n, model)
self._matched_playlists.add(playlist) self._matched_playlists.add(playlist)
self.register_listener('cli_exit', self.update_playlists) self.register_listener('cli_exit', self.update_playlists)
self._unmatched_playlists -= self._matched_playlists self._unmatched_playlists -= self._matched_playlists
def update_playlists(self, lib): def update_playlists(self, lib):
self._log.info(u"Updating {0} smart playlists...", self._log.info("Updating {0} smart playlists...",
len(self._matched_playlists)) len(self._matched_playlists))
playlist_dir = self.config['playlist_dir'].as_filename() playlist_dir = self.config['playlist_dir'].as_filename()
@ -177,7 +185,7 @@ class SmartPlaylistPlugin(BeetsPlugin):
for playlist in self._matched_playlists: for playlist in self._matched_playlists:
name, (query, q_sort), (album_query, a_q_sort) = playlist name, (query, q_sort), (album_query, a_q_sort) = playlist
self._log.debug(u"Creating playlist {0}", name) self._log.debug("Creating playlist {0}", name)
items = [] items = []
if query: if query:
@ -199,6 +207,7 @@ class SmartPlaylistPlugin(BeetsPlugin):
if item_path not in m3us[m3u_name]: if item_path not in m3us[m3u_name]:
m3us[m3u_name].append(item_path) m3us[m3u_name].append(item_path)
prefix = bytestring_path(self.config['prefix'].as_str())
# Write all of the accumulated track lists to files. # Write all of the accumulated track lists to files.
for m3u in m3us: for m3u in m3us:
m3u_path = normpath(os.path.join(playlist_dir, m3u_path = normpath(os.path.join(playlist_dir,
@ -206,6 +215,10 @@ class SmartPlaylistPlugin(BeetsPlugin):
mkdirall(m3u_path) mkdirall(m3u_path)
with open(syspath(m3u_path), 'wb') as f: with open(syspath(m3u_path), 'wb') as f:
for path in m3us[m3u]: for path in m3us[m3u]:
f.write(path + b'\n') if self.config['forward_slash'].get():
path = path_as_posix(path)
if self.config['urlencode']:
path = bytestring_path(pathname2url(path))
f.write(prefix + path + b'\n')
self._log.info(u"{0} playlists updated", len(self._matched_playlists)) self._log.info("{0} playlists updated", len(self._matched_playlists))

Some files were not shown because too many files have changed in this diff Show more