diff --git a/libs/beets/__init__.py b/libs/beets/__init__.py index 17651f4b..830477a9 100644 --- a/libs/beets/__init__.py +++ b/libs/beets/__init__.py @@ -1,5 +1,6 @@ +# -*- coding: utf-8 -*- # This file is part of beets. -# Copyright 2013, Adrian Sampson. +# Copyright 2016, Adrian Sampson. # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -12,15 +13,30 @@ # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. -# This particular version has been slightly modified to work with headphones -# https://github.com/rembo10/headphones +from __future__ import division, absolute_import, print_function -__version__ = '1.3.4' -__author__ = 'Adrian Sampson ' +import os -import beets.library from beets.util import confit -Library = beets.library.Library +__version__ = u'1.3.18' +__author__ = u'Adrian Sampson ' -config = confit.LazyConfig('beets', __name__) + +class IncludeLazyConfig(confit.LazyConfig): + """A version of Confit's LazyConfig that also merges in data from + YAML files specified in an `include` setting. + """ + def read(self, user=True, defaults=True): + super(IncludeLazyConfig, self).read(user, defaults) + + try: + for view in self['include']: + filename = view.as_filename() + if os.path.isfile(filename): + self.set_file(filename) + except confit.NotFoundError: + pass + + +config = IncludeLazyConfig('beets', __name__) diff --git a/libs/beets/art.py b/libs/beets/art.py new file mode 100644 index 00000000..7a65a2b8 --- /dev/null +++ b/libs/beets/art.py @@ -0,0 +1,200 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Adrian Sampson. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""High-level utilities for manipulating image files associated with +music and items' embedded album art. +""" + +from __future__ import division, absolute_import, print_function + +import subprocess +import platform +from tempfile import NamedTemporaryFile +import imghdr +import os + +from beets.util import displayable_path, syspath +from beets.util.artresizer import ArtResizer +from beets import mediafile + + +def mediafile_image(image_path, maxwidth=None): + """Return a `mediafile.Image` object for the path. + """ + + with open(syspath(image_path), 'rb') as f: + data = f.read() + return mediafile.Image(data, type=mediafile.ImageType.front) + + +def get_art(log, item): + # Extract the art. + try: + mf = mediafile.MediaFile(syspath(item.path)) + except mediafile.UnreadableFileError as exc: + log.warning(u'Could not extract art from {0}: {1}', + displayable_path(item.path), exc) + return + + return mf.art + + +def embed_item(log, item, imagepath, maxwidth=None, itempath=None, + compare_threshold=0, ifempty=False, as_album=False): + """Embed an image into the item's media file. + """ + # Conditions and filters. + if compare_threshold: + if not check_art_similarity(log, item, imagepath, compare_threshold): + log.info(u'Image not similar; skipping.') + return + if ifempty and get_art(log, item): + log.info(u'media file already contained art') + return + if maxwidth and not as_album: + imagepath = resize_image(log, imagepath, maxwidth) + + # Get the `Image` object from the file. + try: + log.debug(u'embedding {0}', displayable_path(imagepath)) + image = mediafile_image(imagepath, maxwidth) + except IOError as exc: + log.warning(u'could not read image file: {0}', exc) + return + + # Make sure the image kind is safe (some formats only support PNG + # and JPEG). + if image.mime_type not in ('image/jpeg', 'image/png'): + log.info('not embedding image of unsupported type: {}', + image.mime_type) + return + + item.try_write(path=itempath, tags={'images': [image]}) + + +def embed_album(log, album, maxwidth=None, quiet=False, + compare_threshold=0, ifempty=False): + """Embed album art into all of the album's items. + """ + imagepath = album.artpath + if not imagepath: + log.info(u'No album art present for {0}', album) + return + if not os.path.isfile(syspath(imagepath)): + log.info(u'Album art not found at {0} for {1}', + displayable_path(imagepath), album) + return + if maxwidth: + imagepath = resize_image(log, imagepath, maxwidth) + + log.info(u'Embedding album art into {0}', album) + + for item in album.items(): + embed_item(log, item, imagepath, maxwidth, None, + compare_threshold, ifempty, as_album=True) + + +def resize_image(log, imagepath, maxwidth): + """Returns path to an image resized to maxwidth. + """ + log.debug(u'Resizing album art to {0} pixels wide', maxwidth) + imagepath = ArtResizer.shared.resize(maxwidth, syspath(imagepath)) + return imagepath + + +def check_art_similarity(log, item, imagepath, compare_threshold): + """A boolean indicating if an image is similar to embedded item art. + """ + with NamedTemporaryFile(delete=True) as f: + art = extract(log, f.name, item) + + if art: + is_windows = platform.system() == "Windows" + + # Converting images to grayscale tends to minimize the weight + # of colors in the diff score. + convert_proc = subprocess.Popen( + [b'convert', syspath(imagepath), syspath(art), + b'-colorspace', b'gray', b'MIFF:-'], + stdout=subprocess.PIPE, + close_fds=not is_windows, + ) + compare_proc = subprocess.Popen( + [b'compare', b'-metric', b'PHASH', b'-', b'null:'], + stdin=convert_proc.stdout, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + close_fds=not is_windows, + ) + convert_proc.stdout.close() + + stdout, stderr = compare_proc.communicate() + if compare_proc.returncode: + if compare_proc.returncode != 1: + log.debug(u'IM phashes compare failed for {0}, {1}', + displayable_path(imagepath), + displayable_path(art)) + return + out_str = stderr + else: + out_str = stdout + + try: + phash_diff = float(out_str) + except ValueError: + log.debug(u'IM output is not a number: {0!r}', out_str) + return + + log.debug(u'compare PHASH score is {0}', phash_diff) + return phash_diff <= compare_threshold + + return True + + +def extract(log, outpath, item): + art = get_art(log, item) + + if not art: + log.info(u'No album art present in {0}, skipping.', item) + return + + # Add an extension to the filename. + ext = imghdr.what(None, h=art) + if not ext: + log.warning(u'Unknown image type in {0}.', + displayable_path(item.path)) + return + outpath += b'.' + ext + + log.info(u'Extracting album art from: {0} to: {1}', + item, displayable_path(outpath)) + with open(syspath(outpath), 'wb') as f: + f.write(art) + return outpath + + +def extract_first(log, outpath, items): + for item in items: + real_path = extract(log, outpath, item) + if real_path: + return real_path + + +def clear(log, lib, query): + items = lib.items(query) + log.info(u'Clearing album art from {0} items', len(items)) + for item in items: + log.debug(u'Clearing art for {0}', item) + item.try_write(tags={'images': None}) diff --git a/libs/beets/autotag/__init__.py b/libs/beets/autotag/__init__.py index a3696354..f8233be6 100644 --- a/libs/beets/autotag/__init__.py +++ b/libs/beets/autotag/__init__.py @@ -1,5 +1,6 @@ +# -*- coding: utf-8 -*- # This file is part of beets. -# Copyright 2013, Adrian Sampson. +# Copyright 2016, Adrian Sampson. # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -14,135 +15,23 @@ """Facilities for automatically determining files' correct metadata. """ -import os -import logging -import re -from beets import library, mediafile, config -from beets.util import sorted_walk, ancestry, displayable_path +from __future__ import division, absolute_import, print_function + +from beets import logging +from beets import config # Parts of external interface. -from .hooks import AlbumInfo, TrackInfo, AlbumMatch, TrackMatch -from .match import tag_item, tag_album -from .match import recommendation +from .hooks import AlbumInfo, TrackInfo, AlbumMatch, TrackMatch # noqa +from .match import tag_item, tag_album # noqa +from .match import Recommendation # noqa # Global logger. log = logging.getLogger('beets') -# Constants for directory walker. -MULTIDISC_MARKERS = (r'dis[ck]', r'cd') -MULTIDISC_PAT_FMT = r'^(.*%s[\W_]*)\d' - # Additional utilities for the main interface. -def albums_in_dir(path): - """Recursively searches the given directory and returns an iterable - of (paths, items) where paths is a list of directories and items is - a list of Items that is probably an album. Specifically, any folder - containing any media files is an album. - """ - collapse_pat = collapse_paths = collapse_items = None - - for root, dirs, files in sorted_walk(path, - ignore=config['ignore'].as_str_seq(), - logger=log): - # Get a list of items in the directory. - items = [] - for filename in files: - try: - i = library.Item.from_path(os.path.join(root, filename)) - except mediafile.FileTypeError: - pass - except mediafile.UnreadableFileError: - log.warn(u'unreadable file: {0}'.format( - displayable_path(filename)) - ) - else: - items.append(i) - - # If we're currently collapsing the constituent directories in a - # multi-disc album, check whether we should continue collapsing - # and add the current directory. If so, just add the directory - # and move on to the next directory. If not, stop collapsing. - if collapse_paths: - if (not collapse_pat and collapse_paths[0] in ancestry(root)) or \ - (collapse_pat and - collapse_pat.match(os.path.basename(root))): - # Still collapsing. - collapse_paths.append(root) - collapse_items += items - continue - else: - # Collapse finished. Yield the collapsed directory and - # proceed to process the current one. - if collapse_items: - yield collapse_paths, collapse_items - collapse_pat = collapse_paths = collapse_items = None - - # Check whether this directory looks like the *first* directory - # in a multi-disc sequence. There are two indicators: the file - # is named like part of a multi-disc sequence (e.g., "Title Disc - # 1") or it contains no items but only directories that are - # named in this way. - start_collapsing = False - for marker in MULTIDISC_MARKERS: - marker_pat = re.compile(MULTIDISC_PAT_FMT % marker, re.I) - match = marker_pat.match(os.path.basename(root)) - - # Is this directory the root of a nested multi-disc album? - if dirs and not items: - # Check whether all subdirectories have the same prefix. - start_collapsing = True - subdir_pat = None - for subdir in dirs: - # The first directory dictates the pattern for - # the remaining directories. - if not subdir_pat: - match = marker_pat.match(subdir) - if match: - subdir_pat = re.compile(r'^%s\d' % - re.escape(match.group(1)), re.I) - else: - start_collapsing = False - break - - # Subsequent directories must match the pattern. - elif not subdir_pat.match(subdir): - start_collapsing = False - break - - # If all subdirectories match, don't check other - # markers. - if start_collapsing: - break - - # Is this directory the first in a flattened multi-disc album? - elif match: - start_collapsing = True - # Set the current pattern to match directories with the same - # prefix as this one, followed by a digit. - collapse_pat = re.compile(r'^%s\d' % - re.escape(match.group(1)), re.I) - break - - # If either of the above heuristics indicated that this is the - # beginning of a multi-disc album, initialize the collapsed - # directory and item lists and check the next directory. - if start_collapsing: - # Start collapsing; continue to the next iteration. - collapse_paths = [root] - collapse_items = items - continue - - # If it's nonempty, yield it. - if items: - yield [root], items - - # Clear out any unfinished collapse. - if collapse_paths and collapse_items: - yield collapse_paths, collapse_items - def apply_item_metadata(item, track_info): """Set an item's metadata from its matched TrackInfo object. """ @@ -153,9 +42,12 @@ def apply_item_metadata(item, track_info): item.mb_trackid = track_info.track_id if track_info.artist_id: item.mb_artistid = track_info.artist_id + if track_info.data_source: + item.data_source = track_info.data_source # At the moment, the other metadata is left intact (including album # and track number). Perhaps these should be emptied? + def apply_metadata(album_info, mapping): """Set the items' metadata to match an AlbumInfo object using a mapping from Items to TrackInfo objects. @@ -171,8 +63,8 @@ def apply_metadata(album_info, mapping): # Artist sort and credit names. item.artist_sort = track_info.artist_sort or album_info.artist_sort - item.artist_credit = track_info.artist_credit or \ - album_info.artist_credit + item.artist_credit = (track_info.artist_credit or + album_info.artist_credit) item.albumartist_sort = album_info.artist_sort item.albumartist_credit = album_info.artist_credit @@ -203,7 +95,11 @@ def apply_metadata(album_info, mapping): item.title = track_info.title if config['per_disc_numbering']: - item.track = track_info.medium_index or track_info.index + # We want to let the track number be zero, but if the medium index + # is not provided we need to fall back to the overall index. + item.track = track_info.medium_index + if item.track is None: + item.track = track_info.index item.tracktotal = track_info.medium_total or len(album_info.tracks) else: item.track = track_info.index @@ -235,13 +131,13 @@ def apply_metadata(album_info, mapping): 'language', 'country', 'albumstatus', - 'media', - 'albumdisambig'): + 'albumdisambig', + 'data_source',): value = getattr(album_info, field) if value is not None: item[field] = value if track_info.disctitle is not None: item.disctitle = track_info.disctitle - - # Headphones seal of approval - item.comments = 'tagged by headphones/beets' + + if track_info.media is not None: + item.media = track_info.media diff --git a/libs/beets/autotag/hooks.py b/libs/beets/autotag/hooks.py index 74c8cf82..3de80389 100644 --- a/libs/beets/autotag/hooks.py +++ b/libs/beets/autotag/hooks.py @@ -1,5 +1,6 @@ +# -*- coding: utf-8 -*- # This file is part of beets. -# Copyright 2013, Adrian Sampson. +# Copyright 2016, Adrian Sampson. # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -13,14 +14,16 @@ # included in all copies or substantial portions of the Software. """Glue between metadata sources and the matching logic.""" -import logging +from __future__ import division, absolute_import, print_function + from collections import namedtuple import re +from beets import logging from beets import plugins from beets import config from beets.autotag import mb -from beets.util import levenshtein +from jellyfish import levenshtein_distance from unidecode import unidecode log = logging.getLogger('beets') @@ -109,13 +112,14 @@ class AlbumInfo(object): 'catalognum', 'script', 'language', 'country', 'albumstatus', 'albumdisambig', 'artist_credit', 'media']: value = getattr(self, fld) - if isinstance(value, str): + if isinstance(value, bytes): setattr(self, fld, value.decode(codec, 'ignore')) if self.tracks: for track in self.tracks: track.decode(codec) + class TrackInfo(object): """Describes a canonical track present on a release. Appears as part of an AlbumInfo's ``tracks`` list. Consists of these data members: @@ -126,12 +130,15 @@ class TrackInfo(object): - ``artist_id`` - ``length``: float: duration of the track in seconds - ``index``: position on the entire release + - ``media``: delivery mechanism (Vinyl, etc.) - ``medium``: the disc number this track appears on in the album - ``medium_index``: the track's position on the disc - ``medium_total``: the number of tracks on the item's disc - ``artist_sort``: name of the track artist for sorting - ``disctitle``: name of the individual medium (subtitle) - ``artist_credit``: Recording-specific artist name + - ``data_source``: The original data source (MusicBrainz, Discogs, etc.) + - ``data_url``: The data source release URL. Only ``title`` and ``track_id`` are required. The rest of the fields may be None. The indices ``index``, ``medium``, and ``medium_index`` @@ -140,13 +147,15 @@ class TrackInfo(object): def __init__(self, title, track_id, artist=None, artist_id=None, length=None, index=None, medium=None, medium_index=None, medium_total=None, artist_sort=None, disctitle=None, - artist_credit=None, data_source=None, data_url=None): + artist_credit=None, data_source=None, data_url=None, + media=None): self.title = title self.track_id = track_id self.artist = artist self.artist_id = artist_id self.length = length self.index = index + self.media = media self.medium = medium self.medium_index = medium_index self.medium_total = medium_total @@ -162,9 +171,9 @@ class TrackInfo(object): to Unicode. """ for fld in ['title', 'artist', 'medium', 'artist_sort', 'disctitle', - 'artist_credit']: + 'artist_credit', 'media']: value = getattr(self, fld) - if isinstance(value, str): + if isinstance(value, bytes): setattr(self, fld, value.decode(codec, 'ignore')) @@ -187,27 +196,33 @@ SD_REPLACE = [ (r'&', 'and'), ] + def _string_dist_basic(str1, str2): """Basic edit distance between two strings, ignoring non-alphanumeric characters and case. Comparisons are based on a transliteration/lowering to ASCII characters. Normalized by string length. """ - str1 = unidecode(str1) - str2 = unidecode(str2) + assert isinstance(str1, unicode) + assert isinstance(str2, unicode) + str1 = unidecode(str1).decode('ascii') + str2 = unidecode(str2).decode('ascii') str1 = re.sub(r'[^a-z0-9]', '', str1.lower()) str2 = re.sub(r'[^a-z0-9]', '', str2.lower()) if not str1 and not str2: return 0.0 - return levenshtein(str1, str2) / float(max(len(str1), len(str2))) + return levenshtein_distance(str1, str2) / float(max(len(str1), len(str2))) + def string_dist(str1, str2): """Gives an "intuitive" edit distance between two strings. This is an edit distance, normalized by the string length, with a number of tweaks that reflect intuition about text. """ - if str1 == None and str2 == None: return 0.0 - if str1 == None or str2 == None: return 1.0 + if str1 is None and str2 is None: + return 0.0 + if str1 is None or str2 is None: + return 1.0 str1 = str1.lower() str2 = str2.lower() @@ -217,9 +232,9 @@ def string_dist(str1, str2): # "something, the". for word in SD_END_WORDS: if str1.endswith(', %s' % word): - str1 = '%s %s' % (word, str1[:-len(word)-2]) + str1 = '%s %s' % (word, str1[:-len(word) - 2]) if str2.endswith(', %s' % word): - str2 = '%s %s' % (word, str2[:-len(word)-2]) + str2 = '%s %s' % (word, str2[:-len(word) - 2]) # Perform a couple of basic normalizing substitutions. for pat, repl in SD_REPLACE: @@ -256,6 +271,23 @@ def string_dist(str1, str2): return base_dist + penalty + +class LazyClassProperty(object): + """A decorator implementing a read-only property that is *lazy* in + the sense that the getter is only invoked once. Subsequent accesses + through *any* instance use the cached result. + """ + def __init__(self, getter): + self.getter = getter + self.computed = False + + def __get__(self, obj, owner): + if not self.computed: + self.value = self.getter(owner) + self.computed = True + return self.value + + class Distance(object): """Keeps track of multiple distance penalties. Provides a single weighted distance for all penalties as well as a weighted distance @@ -264,11 +296,15 @@ class Distance(object): def __init__(self): self._penalties = {} + @LazyClassProperty + def _weights(cls): # noqa + """A dictionary from keys to floating-point weights. + """ weights_view = config['match']['distance_weights'] - self._weights = {} + weights = {} for key in weights_view.keys(): - self._weights[key] = weights_view[key].as_number() - + weights[key] = weights_view[key].as_number() + return weights # Access the components and their aggregates. @@ -313,8 +349,10 @@ class Distance(object): # Convert distance into a negative float we can sort items in # ascending order (for keys, when the penalty is equal) and # still get the items with the biggest distance first. - return sorted(list_, key=lambda (key, dist): (0-dist, key)) - + return sorted( + list_, + key=lambda key_and_dist: (-key_and_dist[1], key_and_dist[0]) + ) # Behave like a float. @@ -323,12 +361,15 @@ class Distance(object): def __float__(self): return self.distance + def __sub__(self, other): return self.distance - other def __rsub__(self, other): return other - self.distance + def __unicode__(self): + return "{0:.2f}".format(self.distance) # Behave like a dict. @@ -355,11 +396,11 @@ class Distance(object): """ if not isinstance(dist, Distance): raise ValueError( - '`dist` must be a Distance object. It is: %r' % dist) + u'`dist` must be a Distance object, not {0}'.format(type(dist)) + ) for key, penalties in dist._penalties.iteritems(): self._penalties.setdefault(key, []).extend(penalties) - # Adding components. def _eq(self, value1, value2): @@ -379,7 +420,8 @@ class Distance(object): """ if not 0.0 <= dist <= 1.0: raise ValueError( - '`dist` must be between 0.0 and 1.0. It is: %r' % dist) + u'`dist` must be between 0.0 and 1.0, not {0}'.format(dist) + ) self._penalties.setdefault(key, []).append(dist) def add_equality(self, key, value, options): @@ -472,31 +514,47 @@ def album_for_mbid(release_id): if the ID is not found. """ try: - return mb.album_for_id(release_id) + album = mb.album_for_id(release_id) + if album: + plugins.send(u'albuminfo_received', info=album) + return album except mb.MusicBrainzAPIError as exc: exc.log(log) + def track_for_mbid(recording_id): """Get a TrackInfo object for a MusicBrainz recording ID. Return None if the ID is not found. """ try: - return mb.track_for_id(recording_id) + track = mb.track_for_id(recording_id) + if track: + plugins.send(u'trackinfo_received', info=track) + return track except mb.MusicBrainzAPIError as exc: exc.log(log) + def albums_for_id(album_id): """Get a list of albums for an ID.""" candidates = [album_for_mbid(album_id)] - candidates.extend(plugins.album_for_id(album_id)) + plugin_albums = plugins.album_for_id(album_id) + for a in plugin_albums: + plugins.send(u'albuminfo_received', info=a) + candidates.extend(plugin_albums) return filter(None, candidates) + def tracks_for_id(track_id): """Get a list of tracks for an ID.""" candidates = [track_for_mbid(track_id)] - candidates.extend(plugins.track_for_id(track_id)) + plugin_tracks = plugins.track_for_id(track_id) + for t in plugin_tracks: + plugins.send(u'trackinfo_received', info=t) + candidates.extend(plugin_tracks) return filter(None, candidates) + def album_candidates(items, artist, album, va_likely): """Search for album matches. ``items`` is a list of Item objects that make up the album. ``artist`` and ``album`` are the respective @@ -523,8 +581,13 @@ def album_candidates(items, artist, album, va_likely): # Candidates from plugins. out.extend(plugins.candidates(items, artist, album, va_likely)) + # Notify subscribed plugins about fetched album info + for a in out: + plugins.send(u'albuminfo_received', info=a) + return out + def item_candidates(item, artist, title): """Search for item matches. ``item`` is the Item to be matched. ``artist`` and ``title`` are strings and either reflect the item or @@ -542,4 +605,8 @@ def item_candidates(item, artist, title): # Plugin candidates. out.extend(plugins.item_candidates(item, artist, title)) + # Notify subscribed plugins about fetched track info + for i in out: + plugins.send(u'trackinfo_received', info=i) + return out diff --git a/libs/beets/autotag/match.py b/libs/beets/autotag/match.py index a4bc47fa..cfe184e7 100644 --- a/libs/beets/autotag/match.py +++ b/libs/beets/autotag/match.py @@ -1,5 +1,6 @@ +# -*- coding: utf-8 -*- # This file is part of beets. -# Copyright 2013, Adrian Sampson. +# Copyright 2016, Adrian Sampson. # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -15,21 +16,20 @@ """Matches existing metadata with canonical information to identify releases and tracks. """ -from __future__ import division + +from __future__ import division, absolute_import, print_function import datetime -import logging import re from munkres import Munkres +from beets import logging from beets import plugins from beets import config from beets.util import plurality -from beets.util.enumeration import enum from beets.autotag import hooks - -# Recommendation enumeration. -recommendation = enum('none', 'low', 'medium', 'strong', name='recommendation') +from beets.util.enumeration import OrderedEnum +from functools import reduce # Artist signals that indicate "various artists". These are used at the # album level to determine whether a given release is likely a VA @@ -41,6 +41,18 @@ VA_ARTISTS = (u'', u'various artists', u'various', u'va', u'unknown') log = logging.getLogger('beets') +# Recommendation enumeration. + +class Recommendation(OrderedEnum): + """Indicates a qualitative suggestion to the user about what should + be done with a given match. + """ + none = 0 + low = 1 + medium = 2 + strong = 3 + + # Primary matching functionality. def current_metadata(items): @@ -56,10 +68,10 @@ def current_metadata(items): fields = ['artist', 'album', 'albumartist', 'year', 'disctotal', 'mb_albumid', 'label', 'catalognum', 'country', 'media', 'albumdisambig'] - for key in fields: - values = [getattr(item, key) for item in items if item] - likelies[key], freq = plurality(values) - consensus[key] = (freq == len(values)) + for field in fields: + values = [item[field] for item in items if item] + likelies[field], freq = plurality(values) + consensus[field] = (freq == len(values)) # If there's an album artist consensus, use this for the artist. if consensus['albumartist'] and likelies['albumartist']: @@ -67,6 +79,7 @@ def current_metadata(items): return likelies, consensus + def assign_items(items, tracks): """Given a list of Items and a list of TrackInfo objects, find the best mapping between them. Returns a mapping from Items to TrackInfo @@ -93,12 +106,14 @@ def assign_items(items, tracks): extra_tracks.sort(key=lambda t: (t.index, t.title)) return mapping, extra_items, extra_tracks + def track_index_changed(item, track_info): """Returns True if the item and track info index is different. Tolerates per disc and per release numbering. """ return item.track not in (track_info.medium_index, track_info.index) + def track_distance(item, track_info, incl_artist=False): """Determines the significance of a track metadata change. Returns a Distance object. `incl_artist` indicates that a distance component should @@ -109,7 +124,7 @@ def track_distance(item, track_info, incl_artist=False): # Length. if track_info.length: diff = abs(item.length - track_info.length) - \ - config['match']['track_length_grace'].as_number() + config['match']['track_length_grace'].as_number() dist.add_ratio('track_length', diff, config['match']['track_length_max'].as_number()) @@ -134,6 +149,7 @@ def track_distance(item, track_info, incl_artist=False): return dist + def distance(items, album_info, mapping): """Determines how "significant" an album metadata change would be. Returns a Distance object. `album_info` is an AlbumInfo object @@ -239,6 +255,7 @@ def distance(items, album_info, mapping): return dist + def match_by_id(items): """If the items are tagged with a MusicBrainz album ID, returns an AlbumInfo object for the corresponding album. Otherwise, returns @@ -247,16 +264,17 @@ def match_by_id(items): # Is there a consensus on the MB album ID? albumids = [item.mb_albumid for item in items if item.mb_albumid] if not albumids: - log.debug('No album IDs found.') + log.debug(u'No album IDs found.') return None # If all album IDs are equal, look up the album. - if bool(reduce(lambda x,y: x if x==y else (), albumids)): + if bool(reduce(lambda x, y: x if x == y else (), albumids)): albumid = albumids[0] - log.debug('Searching for discovered album ID: ' + albumid) + log.debug(u'Searching for discovered album ID: {0}', albumid) return hooks.album_for_mbid(albumid) else: - log.debug('No album ID consensus.') + log.debug(u'No album ID consensus.') + def _recommendation(results): """Given a sorted list of AlbumMatch or TrackMatch objects, return a @@ -268,26 +286,26 @@ def _recommendation(results): """ if not results: # No candidates: no recommendation. - return recommendation.none + return Recommendation.none # Basic distance thresholding. min_dist = results[0].distance if min_dist < config['match']['strong_rec_thresh'].as_number(): # Strong recommendation level. - rec = recommendation.strong + rec = Recommendation.strong elif min_dist <= config['match']['medium_rec_thresh'].as_number(): # Medium recommendation level. - rec = recommendation.medium + rec = Recommendation.medium elif len(results) == 1: # Only a single candidate. - rec = recommendation.low + rec = Recommendation.low elif results[1].distance - min_dist >= \ config['match']['rec_gap_thresh'].as_number(): # Gap between first two candidates is large. - rec = recommendation.low + rec = Recommendation.low else: # No conclusion. Return immediately. Can't be downgraded any further. - return recommendation.none + return Recommendation.none # Downgrade to the max rec if it is lower than the current rec for an # applied penalty. @@ -299,28 +317,40 @@ def _recommendation(results): for key in keys: if key in max_rec_view.keys(): max_rec = max_rec_view[key].as_choice({ - 'strong': recommendation.strong, - 'medium': recommendation.medium, - 'low': recommendation.low, - 'none': recommendation.none, + 'strong': Recommendation.strong, + 'medium': Recommendation.medium, + 'low': Recommendation.low, + 'none': Recommendation.none, }) rec = min(rec, max_rec) return rec + def _add_candidate(items, results, info): """Given a candidate AlbumInfo object, attempt to add the candidate to the output dictionary of AlbumMatch objects. This involves checking the track count, ordering the items, checking for duplicates, and calculating the distance. """ - log.debug('Candidate: %s - %s' % (info.artist, info.album)) + log.debug(u'Candidate: {0} - {1}', info.artist, info.album) + + # Discard albums with zero tracks. + if not info.tracks: + log.debug(u'No tracks.') + return # Don't duplicate. if info.album_id in results: - log.debug('Duplicate.') + log.debug(u'Duplicate.') return + # Discard matches without required tags. + for req_tag in config['match']['required'].as_str_seq(): + if getattr(info, req_tag) is None: + log.debug(u'Ignored. Missing required tag: {0}', req_tag) + return + # Find mapping between the items and the track info. mapping, extra_items, extra_tracks = assign_items(items, info.tracks) @@ -328,42 +358,53 @@ def _add_candidate(items, results, info): dist = distance(items, info, mapping) # Skip matches with ignored penalties. - penalties = [key for _, key in dist] + penalties = [key for key, _ in dist] for penalty in config['match']['ignored'].as_str_seq(): if penalty in penalties: - log.debug('Ignored. Penalty: %s' % penalty) + log.debug(u'Ignored. Penalty: {0}', penalty) return - log.debug('Success. Distance: %f' % dist) + log.debug(u'Success. Distance: {0}', dist) results[info.album_id] = hooks.AlbumMatch(dist, info, mapping, extra_items, extra_tracks) + def tag_album(items, search_artist=None, search_album=None, - search_id=None): - """Bundles together the functionality used to infer tags for a - set of items comprised by an album. Returns everything relevant: - - The current artist. - - The current album. - - A list of AlbumMatch objects. The candidates are sorted by - distance (i.e., best match first). - - A recommendation. - If search_artist and search_album or search_id are provided, then - they are used as search terms in place of the current metadata. + search_ids=[]): + """Return a tuple of a artist name, an album name, a list of + `AlbumMatch` candidates from the metadata backend, and a + `Recommendation`. + + The artist and album are the most common values of these fields + among `items`. + + The `AlbumMatch` objects are generated by searching the metadata + backends. By default, the metadata of the items is used for the + search. This can be customized by setting the parameters. + `search_ids` is a list of metadata backend IDs: if specified, + it will restrict the candidates to those IDs, ignoring + `search_artist` and `search album`. The `mapping` field of the + album has the matched `items` as keys. + + The recommendation is calculated from the match quality of the + candidates. """ # Get current metadata. likelies, consensus = current_metadata(items) cur_artist = likelies['artist'] cur_album = likelies['album'] - log.debug('Tagging %s - %s' % (cur_artist, cur_album)) + log.debug(u'Tagging {0} - {1}', cur_artist, cur_album) # The output result (distance, AlbumInfo) tuples (keyed by MB album # ID). candidates = {} # Search by explicit ID. - if search_id is not None: - log.debug('Searching for album ID: ' + search_id) - search_cands = hooks.albums_for_id(search_id) + if search_ids: + search_cands = [] + for search_id in search_ids: + log.debug(u'Searching for album ID: {0}', search_id) + search_cands.extend(hooks.albums_for_id(search_id)) # Use existing metadata or text search. else: @@ -372,32 +413,32 @@ def tag_album(items, search_artist=None, search_album=None, if id_info: _add_candidate(items, candidates, id_info) rec = _recommendation(candidates.values()) - log.debug('Album ID match recommendation is ' + str(rec)) + log.debug(u'Album ID match recommendation is {0}', rec) if candidates and not config['import']['timid']: # If we have a very good MBID match, return immediately. # Otherwise, this match will compete against metadata-based # matches. - if rec == recommendation.strong: - log.debug('ID match.') + if rec == Recommendation.strong: + log.debug(u'ID match.') return cur_artist, cur_album, candidates.values(), rec # Search terms. if not (search_artist and search_album): # No explicit search terms -- use current metadata. search_artist, search_album = cur_artist, cur_album - log.debug(u'Search terms: %s - %s' % (search_artist, search_album)) + log.debug(u'Search terms: {0} - {1}', search_artist, search_album) # Is this album likely to be a "various artist" release? va_likely = ((not consensus['artist']) or - (search_artist.lower() in VA_ARTISTS) or - any(item.comp for item in items)) - log.debug(u'Album might be VA: %s' % str(va_likely)) + (search_artist.lower() in VA_ARTISTS) or + any(item.comp for item in items)) + log.debug(u'Album might be VA: {0}', va_likely) # Get the results from the data sources. search_cands = hooks.album_candidates(items, search_artist, search_album, va_likely) - log.debug(u'Evaluating %i candidates.' % len(search_cands)) + log.debug(u'Evaluating {0} candidates.', len(search_cands)) for info in search_cands: _add_candidate(items, candidates, info) @@ -406,43 +447,47 @@ def tag_album(items, search_artist=None, search_album=None, rec = _recommendation(candidates) return cur_artist, cur_album, candidates, rec + def tag_item(item, search_artist=None, search_title=None, - search_id=None): + search_ids=[]): """Attempts to find metadata for a single track. Returns a `(candidates, recommendation)` pair where `candidates` is a list of TrackMatch objects. `search_artist` and `search_title` may be used to override the current metadata for the purposes of the MusicBrainz - title; likewise `search_id`. + title. `search_ids` may be used for restricting the search to a list + of metadata backend IDs. """ # Holds candidates found so far: keys are MBIDs; values are # (distance, TrackInfo) pairs. candidates = {} # First, try matching by MusicBrainz ID. - trackid = search_id or item.mb_trackid - if trackid: - log.debug('Searching for track ID: ' + trackid) - for track_info in hooks.tracks_for_id(trackid): - dist = track_distance(item, track_info, incl_artist=True) - candidates[track_info.track_id] = \ + trackids = search_ids or filter(None, [item.mb_trackid]) + if trackids: + for trackid in trackids: + log.debug(u'Searching for track ID: {0}', trackid) + for track_info in hooks.tracks_for_id(trackid): + dist = track_distance(item, track_info, incl_artist=True) + candidates[track_info.track_id] = \ hooks.TrackMatch(dist, track_info) - # If this is a good match, then don't keep searching. - rec = _recommendation(candidates.values()) - if rec == recommendation.strong and not config['import']['timid']: - log.debug('Track ID match.') - return candidates.values(), rec + # If this is a good match, then don't keep searching. + rec = _recommendation(sorted(candidates.itervalues())) + if rec == Recommendation.strong and \ + not config['import']['timid']: + log.debug(u'Track ID match.') + return sorted(candidates.itervalues()), rec # If we're searching by ID, don't proceed. - if search_id is not None: + if search_ids: if candidates: - return candidates.values(), rec + return sorted(candidates.itervalues()), rec else: - return [], recommendation.none + return [], Recommendation.none # Search terms. if not (search_artist and search_title): search_artist, search_title = item.artist, item.title - log.debug(u'Item search terms: %s - %s' % (search_artist, search_title)) + log.debug(u'Item search terms: {0} - {1}', search_artist, search_title) # Get and evaluate candidate metadata. for track_info in hooks.item_candidates(item, search_artist, search_title): @@ -450,7 +495,7 @@ def tag_item(item, search_artist=None, search_title=None, candidates[track_info.track_id] = hooks.TrackMatch(dist, track_info) # Sort by distance and return with recommendation. - log.debug('Found %i candidates.' % len(candidates)) + log.debug(u'Found {0} candidates.', len(candidates)) candidates = sorted(candidates.itervalues()) rec = _recommendation(candidates) return candidates, rec diff --git a/libs/beets/autotag/mb.py b/libs/beets/autotag/mb.py index 779ec4b3..e64da8d5 100644 --- a/libs/beets/autotag/mb.py +++ b/libs/beets/autotag/mb.py @@ -1,5 +1,6 @@ +# -*- coding: utf-8 -*- # This file is part of beets. -# Copyright 2013, Adrian Sampson. +# Copyright 2016, Adrian Sampson. # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -14,23 +15,25 @@ """Searches for albums in the MusicBrainz database. """ -import logging +from __future__ import division, absolute_import, print_function + import musicbrainzngs import re import traceback from urlparse import urljoin +from beets import logging import beets.autotag.hooks import beets from beets import util from beets import config -SEARCH_LIMIT = 5 VARIOUS_ARTISTS_ID = '89ad4ac3-39f7-470e-963a-56509c546377' BASE_URL = 'http://musicbrainz.org/' musicbrainzngs.set_useragent('beets', beets.__version__, - 'http://beets.radbox.org/') + 'http://beets.io/') + class MusicBrainzAPIError(util.HumanReadableException): """An error while talking to MusicBrainz. The `query` field is the @@ -38,10 +41,12 @@ class MusicBrainzAPIError(util.HumanReadableException): """ def __init__(self, reason, verb, query, tb=None): self.query = query + if isinstance(reason, musicbrainzngs.WebServiceError): + reason = u'MusicBrainz not reachable' super(MusicBrainzAPIError, self).__init__(reason, verb, tb) def get_message(self): - return u'"{0}" in {1} with query {2}'.format( + return u'{0} in {1} with query {2}'.format( self._reasonstr(), self.verb, repr(self.query) ) @@ -51,12 +56,15 @@ RELEASE_INCLUDES = ['artists', 'media', 'recordings', 'release-groups', 'labels', 'artist-credits', 'aliases'] TRACK_INCLUDES = ['artists', 'aliases'] + def track_url(trackid): return urljoin(BASE_URL, 'recording/' + trackid) + def album_url(albumid): return urljoin(BASE_URL, 'release/' + albumid) + def configure(): """Set up the python-musicbrainz-ngs module according to settings from the beets configuration. This should be called at startup. @@ -67,6 +75,7 @@ def configure(): config['musicbrainz']['ratelimit'].get(int), ) + def _preferred_alias(aliases): """Given an list of alias structures for an artist credit, select and return the user's preferred alias alias or None if no matching @@ -81,13 +90,15 @@ def _preferred_alias(aliases): # Search configured locales in order. for locale in config['import']['languages'].as_str_seq(): # Find matching primary aliases for this locale. - matches = [a for a in aliases if a['locale'] == locale and 'primary' in a] + matches = [a for a in aliases + if a['locale'] == locale and 'primary' in a] # Skip to the next locale if we have no matches if not matches: continue return matches[0] + def _flatten_artist_credit(credit): """Given a list representing an ``artist-credit`` block, flatten the data into a triple of joined artist name strings: canonical, sort, and @@ -133,6 +144,7 @@ def _flatten_artist_credit(credit): ''.join(artist_credit_parts), ) + def track_info(recording, index=None, medium=None, medium_index=None, medium_total=None): """Translates a MusicBrainz recording result dictionary into a beets @@ -149,6 +161,7 @@ def track_info(recording, index=None, medium=None, medium_index=None, medium=medium, medium_index=medium_index, medium_total=medium_total, + data_source=u'MusicBrainz', data_url=track_url(recording['id']), ) @@ -167,6 +180,7 @@ def track_info(recording, index=None, medium=None, medium_index=None, info.decode() return info + def _set_date_str(info, date_str, original=False): """Given a (possibly partial) YYYY-MM-DD string and an AlbumInfo object, set the object's release date fields appropriately. If @@ -186,6 +200,7 @@ def _set_date_str(info, date_str, original=False): key = 'original_' + key setattr(info, key, date_num) + def album_info(release): """Takes a MusicBrainz release result dictionary and returns a beets AlbumInfo object containing the interesting data about that release. @@ -199,7 +214,13 @@ def album_info(release): index = 0 for medium in release['medium-list']: disctitle = medium.get('title') - for track in medium['track-list']: + format = medium.get('format') + + all_tracks = medium['track-list'] + if 'pregap' in medium: + all_tracks.insert(0, medium['pregap']) + + for track in all_tracks: # Basic information from the recording. index += 1 ti = track_info( @@ -210,6 +231,7 @@ def album_info(release): len(medium['track-list']), ) ti.disctitle = disctitle + ti.media = format # Prefer track data, where present, over recording data. if track.get('title'): @@ -233,10 +255,12 @@ def album_info(release): mediums=len(release['medium-list']), artist_sort=artist_sort_name, artist_credit=artist_credit_name, - data_source='MusicBrainz', + data_source=u'MusicBrainz', data_url=album_url(release['id']), ) info.va = info.artist_id == VARIOUS_ARTISTS_ID + if info.va: + info.artist = config['va_name'].get(unicode) info.asin = release.get('asin') info.releasegroup_id = release['release-group']['id'] info.country = release.get('country') @@ -288,7 +312,8 @@ def album_info(release): info.decode() return info -def match_album(artist, album, tracks=None, limit=SEARCH_LIMIT): + +def match_album(artist, album, tracks=None): """Searches for a single album ("release" in MusicBrainz parlance) and returns an iterator over AlbumInfo objects. May raise a MusicBrainzAPIError. @@ -297,21 +322,22 @@ def match_album(artist, album, tracks=None, limit=SEARCH_LIMIT): optionally, a number of tracks on the album. """ # Build search criteria. - criteria = {'release': album.lower()} + criteria = {'release': album.lower().strip()} if artist is not None: - criteria['artist'] = artist.lower() + criteria['artist'] = artist.lower().strip() else: # Various Artists search. criteria['arid'] = VARIOUS_ARTISTS_ID if tracks is not None: - criteria['tracks'] = str(tracks) + criteria['tracks'] = unicode(tracks) # Abort if we have no search terms. if not any(criteria.itervalues()): return try: - res = musicbrainzngs.search_releases(limit=limit, **criteria) + res = musicbrainzngs.search_releases( + limit=config['musicbrainz']['searchlimit'].get(int), **criteria) except musicbrainzngs.MusicBrainzError as exc: raise MusicBrainzAPIError(exc, 'release search', criteria, traceback.format_exc()) @@ -322,69 +348,74 @@ def match_album(artist, album, tracks=None, limit=SEARCH_LIMIT): if albuminfo is not None: yield albuminfo -def match_track(artist, title, limit=SEARCH_LIMIT): + +def match_track(artist, title): """Searches for a single track and returns an iterable of TrackInfo objects. May raise a MusicBrainzAPIError. """ criteria = { - 'artist': artist.lower(), - 'recording': title.lower(), + 'artist': artist.lower().strip(), + 'recording': title.lower().strip(), } if not any(criteria.itervalues()): return try: - res = musicbrainzngs.search_recordings(limit=limit, **criteria) + res = musicbrainzngs.search_recordings( + limit=config['musicbrainz']['searchlimit'].get(int), **criteria) except musicbrainzngs.MusicBrainzError as exc: raise MusicBrainzAPIError(exc, 'recording search', criteria, traceback.format_exc()) for recording in res['recording-list']: yield track_info(recording) + def _parse_id(s): """Search for a MusicBrainz ID in the given string and return it. If no ID can be found, return None. """ # Find the first thing that looks like a UUID/MBID. - match = re.search('[a-f0-9]{8}(-[a-f0-9]{4}){3}-[a-f0-9]{12}', s) + match = re.search(ur'[a-f0-9]{8}(-[a-f0-9]{4}){3}-[a-f0-9]{12}', s) if match: return match.group() -def album_for_id(albumid): + +def album_for_id(releaseid): """Fetches an album by its MusicBrainz ID and returns an AlbumInfo object or None if the album is not found. May raise a MusicBrainzAPIError. """ - albumid = _parse_id(albumid) + albumid = _parse_id(releaseid) if not albumid: - log.error('Invalid MBID.') + log.debug(u'Invalid MBID ({0}).', releaseid) return try: res = musicbrainzngs.get_release_by_id(albumid, RELEASE_INCLUDES) except musicbrainzngs.ResponseError: - log.debug('Album ID match failed.') + log.debug(u'Album ID match failed.') return None except musicbrainzngs.MusicBrainzError as exc: - raise MusicBrainzAPIError(exc, 'get release by ID', albumid, + raise MusicBrainzAPIError(exc, u'get release by ID', albumid, traceback.format_exc()) return album_info(res['release']) -def track_for_id(trackid): + +def track_for_id(releaseid): """Fetches a track by its MusicBrainz ID. Returns a TrackInfo object or None if no track is found. May raise a MusicBrainzAPIError. """ - trackid = _parse_id(trackid) + trackid = _parse_id(releaseid) if not trackid: - log.error('Invalid MBID.') + log.debug(u'Invalid MBID ({0}).', releaseid) return try: res = musicbrainzngs.get_recording_by_id(trackid, TRACK_INCLUDES) except musicbrainzngs.ResponseError: - log.debug('Track ID match failed.') + log.debug(u'Track ID match failed.') return None except musicbrainzngs.MusicBrainzError as exc: - raise MusicBrainzAPIError(exc, 'get recording by ID', trackid, + raise MusicBrainzAPIError(exc, u'get recording by ID', trackid, traceback.format_exc()) return track_info(res['recording']) diff --git a/libs/beets/config_default.yaml b/libs/beets/config_default.yaml index d35368ea..4c12c3df 100644 --- a/libs/beets/config_default.yaml +++ b/libs/beets/config_default.yaml @@ -5,6 +5,7 @@ import: write: yes copy: yes move: no + link: no delete: no resume: ask incremental: no @@ -20,9 +21,13 @@ import: detail: no flat: no group_albums: no + pretend: false + search_ids: [] clutter: ["Thumbs.DB", ".DS_Store"] -ignore: [".*", "*~", "System Volume Information"] +ignore: [".*", "*~", "System Volume Information", "lost+found"] +ignore_hidden: yes + replace: '[\\/]': _ '^\.': _ @@ -32,27 +37,42 @@ replace: '\s+$': '' '^\s+': '' path_sep_replace: _ +asciify_paths: false art_filename: cover max_filename_length: 0 plugins: [] pluginpath: [] threaded: yes -color: yes timeout: 5.0 per_disc_numbering: no -verbose: no -terminal_encoding: utf8 +verbose: 0 +terminal_encoding: original_date: no id3v23: no +va_name: "Various Artists" ui: terminal_width: 80 length_diff_thresh: 10.0 + color: yes + colors: + text_success: green + text_warning: yellow + text_error: red + text_highlight: red + text_highlight_minor: lightgray + action_default: turquoise + action: blue -list_format_item: $artist - $album - $title -list_format_album: $albumartist - $album +format_item: $artist - $album - $title +format_album: $albumartist - $album time_format: '%Y-%m-%d %H:%M:%S' +format_raw_length: no + +sort_album: albumartist+ album+ +sort_item: artist+ album+ disc+ track+ +sort_case_insensitive: yes paths: default: $albumartist/$album%aunique{}/$track $title @@ -65,6 +85,7 @@ musicbrainz: host: musicbrainz.org ratelimit: 1 ratelimit_interval: 1.0 + searchlimit: 5 match: strong_rec_thresh: 0.04 @@ -98,5 +119,6 @@ match: media: [] original_year: no ignored: [] + required: [] track_length_grace: 10 track_length_max: 30 diff --git a/libs/beets/dbcore/__init__.py b/libs/beets/dbcore/__init__.py index b4f80fb9..689e7202 100644 --- a/libs/beets/dbcore/__init__.py +++ b/libs/beets/dbcore/__init__.py @@ -1,5 +1,6 @@ +# -*- coding: utf-8 -*- # This file is part of beets. -# Copyright 2014, Adrian Sampson. +# Copyright 2016, Adrian Sampson. # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -15,6 +16,14 @@ """DBCore is an abstract database package that forms the basis for beets' Library. """ +from __future__ import division, absolute_import, print_function + from .db import Model, Database from .query import Query, FieldQuery, MatchQuery, AndQuery, OrQuery from .types import Type +from .queryparse import query_from_strings +from .queryparse import sort_from_strings +from .queryparse import parse_sorted_query +from .query import InvalidQueryError + +# flake8: noqa diff --git a/libs/beets/dbcore/db.py b/libs/beets/dbcore/db.py index cbdaf5a7..3f701be5 100644 --- a/libs/beets/dbcore/db.py +++ b/libs/beets/dbcore/db.py @@ -1,5 +1,6 @@ +# -*- coding: utf-8 -*- # This file is part of beets. -# Copyright 2014, Adrian Sampson. +# Copyright 2016, Adrian Sampson. # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -14,22 +15,70 @@ """The central Model and Database constructs for DBCore. """ +from __future__ import division, absolute_import, print_function + import time import os from collections import defaultdict import threading import sqlite3 import contextlib +import collections import beets from beets.util.functemplate import Template -from .query import MatchQuery +from beets.dbcore import types +from .query import MatchQuery, NullSort, TrueQuery +class FormattedMapping(collections.Mapping): + """A `dict`-like formatted view of a model. + + The accessor `mapping[key]` returns the formatted version of + `model[key]` as a unicode string. + + If `for_path` is true, all path separators in the formatted values + are replaced. + """ + + def __init__(self, model, for_path=False): + self.for_path = for_path + self.model = model + self.model_keys = model.keys(True) + + def __getitem__(self, key): + if key in self.model_keys: + return self._get_formatted(self.model, key) + else: + raise KeyError(key) + + def __iter__(self): + return iter(self.model_keys) + + def __len__(self): + return len(self.model_keys) + + def get(self, key, default=None): + if default is None: + default = self.model._type(key).format(None) + return super(FormattedMapping, self).get(key, default) + + def _get_formatted(self, model, key): + value = model._type(key).format(model.get(key)) + if isinstance(value, bytes): + value = value.decode('utf8', 'ignore') + + if self.for_path: + sep_repl = beets.config['path_sep_replace'].get(unicode) + for sep in (os.path.sep, os.path.altsep): + if sep: + value = value.replace(sep, sep_repl) + + return value + # Abstract base for model classes. - class Model(object): """An abstract object representing an object in the database. Model objects act like dictionaries (i.e., the allow subscript access like @@ -66,12 +115,7 @@ class Model(object): _fields = {} """A mapping indicating available "fixed" fields on this type. The - keys are field names and the values are Type objects. - """ - - _bytes_keys = () - """Keys whose values should be stored as raw bytes blobs rather than - strings. + keys are field names and the values are `Type` objects. """ _search_fields = () @@ -79,6 +123,21 @@ class Model(object): terms. """ + _types = {} + """Optional Types for non-fixed (i.e., flexible and computed) fields. + """ + + _sorts = {} + """Optional named sort criteria. The keys are strings and the values + are subclasses of `Sort`. + """ + + _always_dirty = False + """By default, fields only become "dirty" when their value actually + changes. Enabling this flag marks fields as dirty even when the new + value is the same as the old value (e.g., `o.f = o.f`). + """ + @classmethod def _getters(cls): """Return a mapping from field names to getter functions. @@ -94,7 +153,6 @@ class Model(object): # As above: we could consider caching this result. raise NotImplementedError() - # Basic operation. def __init__(self, db=None, **values): @@ -110,6 +168,20 @@ class Model(object): self.update(values) self.clear_dirty() + @classmethod + def _awaken(cls, db=None, fixed_values={}, flex_values={}): + """Create an object with values drawn from the database. + + This is a performance optimization: the checks involved with + ordinary construction are bypassed. + """ + obj = cls(db) + for key, value in fixed_values.iteritems(): + obj._values_fixed[key] = cls._type(key).from_sql(value) + for key, value in flex_values.iteritems(): + obj._values_flex[key] = cls._type(key).from_sql(value) + return obj + def __repr__(self): return '{0}({1})'.format( type(self).__name__, @@ -128,13 +200,23 @@ class Model(object): exception is raised otherwise. """ if not self._db: - raise ValueError('{0} has no database'.format(type(self).__name__)) + raise ValueError( + u'{0} has no database'.format(type(self).__name__) + ) if need_id and not self.id: - raise ValueError('{0} has no id'.format(type(self).__name__)) - + raise ValueError(u'{0} has no id'.format(type(self).__name__)) # Essential field accessors. + @classmethod + def _type(cls, key): + """Get the type of a field, a `Type` instance. + + If the field has no explicit type, it is given the base `Type`, + which does no conversion. + """ + return cls._fields.get(key) or cls._types.get(key) or types.DEFAULT + def __getitem__(self, key): """Get the value for a field. Raise a KeyError if the field is not available. @@ -152,11 +234,19 @@ class Model(object): def __setitem__(self, key, value): """Assign the value for a field. """ - source = self._values_fixed if key in self._fields \ - else self._values_flex + # Choose where to place the value. + if key in self._fields: + source = self._values_fixed + else: + source = self._values_flex + + # If the field has a type, filter the value. + value = self._type(key).normalize(value) + + # Assign value and possibly mark as dirty. old_value = source.get(key) source[key] = value - if old_value != value: + if self._always_dirty or old_value != value: self._dirty.add(key) def __delitem__(self, key): @@ -166,11 +256,11 @@ class Model(object): del self._values_flex[key] self._dirty.add(key) # Mark for dropping on store. elif key in self._getters(): # Computed. - raise KeyError('computed field {0} cannot be deleted'.format(key)) + raise KeyError(u'computed field {0} cannot be deleted'.format(key)) elif key in self._fields: # Fixed. - raise KeyError('fixed field {0} cannot be deleted'.format(key)) + raise KeyError(u'fixed field {0} cannot be deleted'.format(key)) else: - raise KeyError('no such field {0}'.format(key)) + raise KeyError(u'no such field {0}'.format(key)) def keys(self, computed=False): """Get a list of available field names for this object. The @@ -183,6 +273,12 @@ class Model(object): else: return base_keys + @classmethod + def all_keys(cls): + """Get a list of available keys for objects of this type. + Includes fixed and computed fields. + """ + return list(cls._fields) + cls._getters().keys() # Act like a dictionary. @@ -219,17 +315,16 @@ class Model(object): """ return iter(self.keys()) - # Convenient attribute access. def __getattr__(self, key): if key.startswith('_'): - raise AttributeError('model has no attribute {0!r}'.format(key)) + raise AttributeError(u'model has no attribute {0!r}'.format(key)) else: try: return self[key] except KeyError: - raise AttributeError('no such field {0!r}'.format(key)) + raise AttributeError(u'no such field {0!r}'.format(key)) def __setattr__(self, key, value): if key.startswith('_'): @@ -243,7 +338,6 @@ class Model(object): else: del self[key] - # Database interaction (CRUD methods). def store(self): @@ -252,19 +346,15 @@ class Model(object): self._check_db() # Build assignments for query. - assignments = '' + assignments = [] subvars = [] for key in self._fields: if key != 'id' and key in self._dirty: self._dirty.remove(key) - assignments += key + '=?,' - value = self[key] - # Wrap path strings in buffers so they get stored - # "in the raw". - if key in self._bytes_keys and isinstance(value, str): - value = buffer(value) + assignments.append(key + '=?') + value = self._type(key).to_sql(self[key]) subvars.append(value) - assignments = assignments[:-1] # Knock off last , + assignments = ','.join(assignments) with self._db.transaction() as tx: # Main table update. @@ -301,7 +391,9 @@ class Model(object): """ self._check_db() stored_obj = self._db._get(type(self), self.id) - assert stored_obj is not None, "object {0} not in DB".format(self.id) + assert stored_obj is not None, u"object {0} not in DB".format(self.id) + self._values_fixed = {} + self._values_flex = {} self.update(dict(stored_obj)) self.clear_dirty() @@ -344,76 +436,26 @@ class Model(object): self._dirty.add(key) self.store() - # Formatting and templating. - @classmethod - def _format(cls, key, value, for_path=False): - """Format a value as the given field for this model. - """ - # Format the value as a string according to its type, if any. - if key in cls._fields: - value = cls._fields[key].format(value) - # Formatting must result in a string. To deal with - # Python2isms, implicitly convert ASCII strings. - assert isinstance(value, basestring), \ - u'field formatter must produce strings' - if isinstance(value, bytes): - value = value.decode('utf8', 'ignore') + _formatter = FormattedMapping - elif not isinstance(value, unicode): - # Fallback formatter. Convert to unicode at all cost. - if value is None: - value = u'' - elif isinstance(value, basestring): - if isinstance(value, bytes): - value = value.decode('utf8', 'ignore') - else: - value = unicode(value) - - if for_path: - sep_repl = beets.config['path_sep_replace'].get(unicode) - for sep in (os.path.sep, os.path.altsep): - if sep: - value = value.replace(sep, sep_repl) - - return value - - def _get_formatted(self, key, for_path=False): - """Get a field value formatted as a string (`unicode` object) - for display to the user. If `for_path` is true, then the value - will be sanitized for inclusion in a pathname (i.e., path - separators will be removed from the value). - """ - return self._format(key, self.get(key), for_path) - - def _formatted_mapping(self, for_path=False): + def formatted(self, for_path=False): """Get a mapping containing all values on this object formatted - as human-readable strings. + as human-readable unicode strings. """ - # In the future, this could be made "lazy" to avoid computing - # fields unnecessarily. - out = {} - for key in self.keys(True): - out[key] = self._get_formatted(key, for_path) - return out + return self._formatter(self, for_path) def evaluate_template(self, template, for_path=False): """Evaluate a template (a string or a `Template` object) using the object's fields. If `for_path` is true, then no new path separators will be added to the template. """ - # Build value mapping. - mapping = self._formatted_mapping(for_path) - - # Get template functions. - funcs = self._template_funcs() - # Perform substitution. if isinstance(template, basestring): template = Template(template) - return template.substitute(mapping, funcs) - + return template.substitute(self.formatted(for_path), + self._template_funcs()) # Parsing. @@ -422,65 +464,124 @@ class Model(object): """Parse a string as a value for the given key. """ if not isinstance(string, basestring): - raise TypeError("_parse() argument must be a string") + raise TypeError(u"_parse() argument must be a string") - typ = cls._fields.get(key) - if typ: - return typ.parse(string) - else: - # Fall back to unparsed string. - return string + return cls._type(key).parse(string) + def set_parse(self, key, string): + """Set the object's key to a value represented by a string. + """ + self[key] = self._parse(key, string) # Database controller and supporting interfaces. - class Results(object): """An item query result set. Iterating over the collection lazily constructs LibModel objects that reflect database rows. """ - def __init__(self, model_class, rows, db, query=None): + def __init__(self, model_class, rows, db, query=None, sort=None): """Create a result set that will construct objects of type - `model_class`, which should be a subclass of `LibModel`, out of - the query result mapping in `rows`. The new objects are - associated with the database `db`. If `query` is provided, it is - used as a predicate to filter the results for a "slow query" that - cannot be evaluated by the database directly. + `model_class`. + + `model_class` is a subclass of `LibModel` that will be + constructed. `rows` is a query result: a list of mappings. The + new objects will be associated with the database `db`. + + If `query` is provided, it is used as a predicate to filter the + results for a "slow query" that cannot be evaluated by the + database directly. If `sort` is provided, it is used to sort the + full list of results before returning. This means it is a "slow + sort" and all objects must be built before returning the first + one. """ self.model_class = model_class self.rows = rows self.db = db self.query = query + self.sort = sort + + # We keep a queue of rows we haven't yet consumed for + # materialization. We preserve the original total number of + # rows. + self._rows = rows + self._row_count = len(rows) + + # The materialized objects corresponding to rows that have been + # consumed. + self._objects = [] + + def _get_objects(self): + """Construct and generate Model objects for they query. The + objects are returned in the order emitted from the database; no + slow sort is applied. + + For performance, this generator caches materialized objects to + avoid constructing them more than once. This way, iterating over + a `Results` object a second time should be much faster than the + first. + """ + index = 0 # Position in the materialized objects. + while index < len(self._objects) or self._rows: + # Are there previously-materialized objects to produce? + if index < len(self._objects): + yield self._objects[index] + index += 1 + + # Otherwise, we consume another row, materialize its object + # and produce it. + else: + while self._rows: + row = self._rows.pop(0) + obj = self._make_model(row) + # If there is a slow-query predicate, ensurer that the + # object passes it. + if not self.query or self.query.match(obj): + self._objects.append(obj) + index += 1 + yield obj + break def __iter__(self): - """Construct Python objects for all rows that pass the query - predicate. + """Construct and generate Model objects for all matching + objects, in sorted order. """ - for row in self.rows: - # Get the flexible attributes for the object. - with self.db.transaction() as tx: - flex_rows = tx.query( - 'SELECT * FROM {0} WHERE entity_id=?'.format( - self.model_class._flex_table - ), - (row['id'],) - ) - values = dict(row) - values.update( - dict((row['key'], row['value']) for row in flex_rows) + if self.sort: + # Slow sort. Must build the full list first. + objects = self.sort.sort(list(self._get_objects())) + return iter(objects) + + else: + # Objects are pre-sorted (i.e., by the database). + return self._get_objects() + + def _make_model(self, row): + # Get the flexible attributes for the object. + with self.db.transaction() as tx: + flex_rows = tx.query( + 'SELECT * FROM {0} WHERE entity_id=?'.format( + self.model_class._flex_table + ), + (row['id'],) ) - # Construct the Python object and yield it if it passes the - # predicate. - obj = self.model_class(self.db, **values) - if not self.query or self.query.match(obj): - yield obj + cols = dict(row) + values = dict((k, v) for (k, v) in cols.items() + if not k[:4] == 'flex') + flex_values = dict((row['key'], row['value']) for row in flex_rows) + + # Construct the Python object + obj = self.model_class._awaken(self.db, values, flex_values) + return obj def __len__(self): """Get the number of matching objects. """ - if self.query: + if not self._rows: + # Fully materialized. Just count the objects. + return len(self._objects) + + elif self.query: # A slow query. Fall back to testing every object. count = 0 for obj in self: @@ -489,7 +590,7 @@ class Results(object): else: # A fast query. Just count the rows. - return len(self.rows) + return self._row_count def __nonzero__(self): """Does this result contain any objects? @@ -500,13 +601,18 @@ class Results(object): """Get the nth item in this result set. This is inefficient: all items up to n are materialized and thrown away. """ + if not self._rows and not self.sort: + # Fully materialized and already in order. Just look up the + # object. + return self._objects[n] + it = iter(self) try: for i in range(n): - it.next() - return it.next() + next(it) + return next(it) except StopIteration: - raise IndexError('result index {0} out of range'.format(n)) + raise IndexError(u'result index {0} out of range'.format(n)) def get(self): """Return the first matching object, or None if no objects @@ -514,7 +620,7 @@ class Results(object): """ it = iter(self) try: - return it.next() + return next(it) except StopIteration: return None @@ -604,7 +710,6 @@ class Database(object): self._make_table(model_cls._table, model_cls._fields) self._make_attribute_table(model_cls._flex_table) - # Primitive access control: connections and transactions. def _connection(self): @@ -644,7 +749,6 @@ class Database(object): """ return Transaction(self) - # Schema setup and migration. def _make_table(self, table, fields): @@ -698,27 +802,33 @@ class Database(object): ON {0} (entity_id); """.format(flex_table)) - # Querying. - def _fetch(self, model_cls, query, order_by=None): + def _fetch(self, model_cls, query=None, sort=None): """Fetch the objects of type `model_cls` matching the given query. The query may be given as a string, string sequence, a - Query object, or None (to fetch everything). If provided, - `order_by` is a SQLite ORDER BY clause for sorting. + Query object, or None (to fetch everything). `sort` is an + `Sort` object. """ + query = query or TrueQuery() # A null query. + sort = sort or NullSort() # Unsorted. where, subvals = query.clause() + order_by = sort.order_clause() - sql = "SELECT * FROM {0} WHERE {1}".format( + sql = ("SELECT * FROM {0} WHERE {1} {2}").format( model_cls._table, where or '1', + "ORDER BY {0}".format(order_by) if order_by else '', ) - if order_by: - sql += " ORDER BY {0}".format(order_by) + with self.transaction() as tx: rows = tx.query(sql, subvals) - return Results(model_cls, rows, self, None if where else query) + return Results( + model_cls, rows, self, + None if where else query, # Slow query component. + sort if sort.is_slow() else None, # Slow sort component. + ) def _get(self, model_cls, id): """Get a Model object by its id or None if the id does not diff --git a/libs/beets/dbcore/query.py b/libs/beets/dbcore/query.py index 4c888302..caf38026 100644 --- a/libs/beets/dbcore/query.py +++ b/libs/beets/dbcore/query.py @@ -1,5 +1,6 @@ +# -*- coding: utf-8 -*- # This file is part of beets. -# Copyright 2014, Adrian Sampson. +# Copyright 2016, Adrian Sampson. # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -14,9 +15,45 @@ """The Query type hierarchy for DBCore. """ +from __future__ import division, absolute_import, print_function + import re +from operator import mul from beets import util from datetime import datetime, timedelta +import unicodedata +from functools import reduce + + +class ParsingError(ValueError): + """Abstract class for any unparseable user-requested album/query + specification. + """ + + +class InvalidQueryError(ParsingError): + """Represent any kind of invalid query. + + The query should be a unicode string or a list, which will be space-joined. + """ + def __init__(self, query, explanation): + if isinstance(query, list): + query = " ".join(query) + message = u"'{0}': {1}".format(query, explanation) + super(InvalidQueryError, self).__init__(message) + + +class InvalidQueryArgumentTypeError(ParsingError): + """Represent a query argument that could not be converted as expected. + + It exists to be caught in upper stack levels so a meaningful (i.e. with the + query) InvalidQueryError can be raised. + """ + def __init__(self, what, expected, detail=None): + message = u"'{0}' is not {1}".format(what, expected) + if detail: + message = u"{0}: {1}".format(message, detail) + super(InvalidQueryArgumentTypeError, self).__init__(message) class Query(object): @@ -24,9 +61,8 @@ class Query(object): """ def clause(self): """Generate an SQLite expression implementing the query. - Return a clause string, a sequence of substitution values for - the clause, and a Query object representing the "remainder" - Returns (clause, subvals) where clause is a valid sqlite + + Return (clause, subvals) where clause is a valid sqlite WHERE clause implementing the query and subvals is a list of items to be substituted for ?s in the clause. """ @@ -38,6 +74,15 @@ class Query(object): """ raise NotImplementedError + def __repr__(self): + return "{0.__class__.__name__}()".format(self) + + def __eq__(self, other): + return type(self) == type(other) + + def __hash__(self): + return 0 + class FieldQuery(Query): """An abstract query that searches in a specific field for a @@ -71,6 +116,17 @@ class FieldQuery(Query): def match(self, item): return self.value_match(self.pattern, item.get(self.field)) + def __repr__(self): + return ("{0.__class__.__name__}({0.field!r}, {0.pattern!r}, " + "{0.fast})".format(self)) + + def __eq__(self, other): + return super(FieldQuery, self).__eq__(other) and \ + self.field == other.field and self.pattern == other.pattern + + def __hash__(self): + return hash((self.field, hash(self.pattern))) + class MatchQuery(FieldQuery): """A query that looks for exact matches in an item field.""" @@ -82,6 +138,25 @@ class MatchQuery(FieldQuery): return pattern == value +class NoneQuery(FieldQuery): + + def __init__(self, field, fast=True): + super(NoneQuery, self).__init__(field, None, fast) + + def col_clause(self): + return self.field + " IS NULL", () + + @classmethod + def match(cls, item): + try: + return item[cls.field] is None + except KeyError: + return True + + def __repr__(self): + return "{0.__class__.__name__}({0.field!r}, {0.fast})".format(self) + + class StringFieldQuery(FieldQuery): """A FieldQuery that converts values to strings before matching them. @@ -104,8 +179,11 @@ class StringFieldQuery(FieldQuery): class SubstringQuery(StringFieldQuery): """A query that matches a substring in a specific item field.""" def col_clause(self): - search = '%' + (self.pattern.replace('\\','\\\\').replace('%','\\%') - .replace('_','\\_')) + '%' + pattern = (self.pattern + .replace('\\', '\\\\') + .replace('%', '\\%') + .replace('_', '\\_')) + search = '%' + pattern + '%' clause = self.field + " like ? escape '\\'" subvals = [search] return clause, subvals @@ -118,15 +196,31 @@ class SubstringQuery(StringFieldQuery): class RegexpQuery(StringFieldQuery): """A query that matches a regular expression in a specific item field. + + Raises InvalidQueryError when the pattern is not a valid regular + expression. """ + def __init__(self, field, pattern, fast=True): + super(RegexpQuery, self).__init__(field, pattern, fast) + pattern = self._normalize(pattern) + try: + self.pattern = re.compile(self.pattern) + except re.error as exc: + # Invalid regular expression. + raise InvalidQueryArgumentTypeError(pattern, + u"a regular expression", + format(exc)) + + @staticmethod + def _normalize(s): + """Normalize a Unicode string's representation (used on both + patterns and matched values). + """ + return unicodedata.normalize('NFC', s) + @classmethod def string_match(cls, pattern, value): - try: - res = re.search(pattern, value) - except re.error: - # Invalid regular expression. - return False - return res is not None + return pattern.search(cls._normalize(value)) is not None class BooleanQuery(MatchQuery): @@ -142,7 +236,7 @@ class BooleanQuery(MatchQuery): class BytesQuery(MatchQuery): """Match a raw bytes field (i.e., a path). This is a necessary hack - to work around the `sqlite3` module's desire to treat `str` and + to work around the `sqlite3` module's desire to treat `bytes` and `unicode` equivalently in Python 2. Always use this query instead of `MatchQuery` when matching on BLOB values. """ @@ -170,19 +264,26 @@ class NumericQuery(FieldQuery): """Matches numeric fields. A syntax using Ruby-style range ellipses (``..``) lets users specify one- or two-sided ranges. For example, ``year:2001..`` finds music released since the turn of the century. + + Raises InvalidQueryError when the pattern does not represent an int or + a float. """ def _convert(self, s): - """Convert a string to a numeric type (float or int). If the - string cannot be converted, return None. + """Convert a string to a numeric type (float or int). + + Return None if `s` is empty. + Raise an InvalidQueryError if the string cannot be converted. """ # This is really just a bit of fun premature optimization. + if not s: + return None try: return int(s) except ValueError: try: return float(s) except ValueError: - return None + raise InvalidQueryArgumentTypeError(s, u"an int or a float") def __init__(self, field, pattern, fast=True): super(NumericQuery, self).__init__(field, pattern, fast) @@ -200,7 +301,9 @@ class NumericQuery(FieldQuery): self.rangemax = self._convert(parts[1]) def match(self, item): - value = getattr(item, self.field) + if self.field not in item: + return False + value = item[self.field] if isinstance(value, basestring): value = self._convert(value) @@ -225,7 +328,7 @@ class NumericQuery(FieldQuery): elif self.rangemax is not None: return u'{0} <= ?'.format(self.field), (self.rangemax,) else: - return '1', () + return u'1', () class CollectionQuery(Query): @@ -236,17 +339,21 @@ class CollectionQuery(Query): self.subqueries = subqueries # Act like a sequence. + def __len__(self): return len(self.subqueries) + def __getitem__(self, key): return self.subqueries[key] + def __iter__(self): return iter(self.subqueries) + def __contains__(self, item): return item in self.subqueries def clause_with_joiner(self, joiner): - """Returns a clause created by joining together the clauses of + """Return a clause created by joining together the clauses of all subqueries with the string joiner (padded by spaces). """ clause_parts = [] @@ -261,6 +368,19 @@ class CollectionQuery(Query): clause = (' ' + joiner + ' ').join(clause_parts) return clause, subvals + def __repr__(self): + return "{0.__class__.__name__}({0.subqueries!r})".format(self) + + def __eq__(self, other): + return super(CollectionQuery, self).__eq__(other) and \ + self.subqueries == other.subqueries + + def __hash__(self): + """Since subqueries are mutable, this object should not be hashable. + However and for conveniences purposes, it can be hashed. + """ + return reduce(mul, map(hash, self.subqueries), 1) + class AnyFieldQuery(CollectionQuery): """A query that matches if a given FieldQuery subclass matches in @@ -286,6 +406,17 @@ class AnyFieldQuery(CollectionQuery): return True return False + def __repr__(self): + return ("{0.__class__.__name__}({0.pattern!r}, {0.fields!r}, " + "{0.query_class.__name__})".format(self)) + + def __eq__(self, other): + return super(AnyFieldQuery, self).__eq__(other) and \ + self.query_class == other.query_class + + def __hash__(self): + return hash((self.pattern, tuple(self.fields), self.query_class)) + class MutableCollectionQuery(CollectionQuery): """A collection query whose subqueries may be modified after the @@ -316,6 +447,36 @@ class OrQuery(MutableCollectionQuery): return any([q.match(item) for q in self.subqueries]) +class NotQuery(Query): + """A query that matches the negation of its `subquery`, as a shorcut for + performing `not(subquery)` without using regular expressions. + """ + def __init__(self, subquery): + self.subquery = subquery + + def clause(self): + clause, subvals = self.subquery.clause() + if clause: + return 'not ({0})'.format(clause), subvals + else: + # If there is no clause, there is nothing to negate. All the logic + # is handled by match() for slow queries. + return clause, subvals + + def match(self, item): + return not self.subquery.match(item) + + def __repr__(self): + return "{0.__class__.__name__}({0.subquery!r})".format(self) + + def __eq__(self, other): + return super(NotQuery, self).__eq__(other) and \ + self.subquery == other.subquery + + def __hash__(self): + return hash(('not', hash(self.subquery))) + + class TrueQuery(Query): """A query that always matches.""" def clause(self): @@ -334,21 +495,15 @@ class FalseQuery(Query): return False - # Time/date queries. - def _to_epoch_time(date): """Convert a `datetime` object to an integer number of seconds since the (local) Unix epoch. """ epoch = datetime.fromtimestamp(0) delta = date - epoch - try: - return int(delta.total_seconds()) - except AttributeError: - # datetime.timedelta.total_seconds() is not available on Python 2.6 - return delta.seconds + delta.days * 24 * 3600 + return int(delta.total_seconds()) def _parse_periods(pattern): @@ -380,7 +535,7 @@ class Period(object): precision (a string, one of "year", "month", or "day"). """ if precision not in Period.precisions: - raise ValueError('Invalid precision ' + str(precision)) + raise ValueError(u'Invalid precision {0}'.format(precision)) self.date = date self.precision = precision @@ -393,10 +548,14 @@ class Period(object): return None ordinal = string.count('-') if ordinal >= len(cls.date_formats): - raise ValueError('date is not in one of the formats ' - + ', '.join(cls.date_formats)) + # Too many components. + return None date_format = cls.date_formats[ordinal] - date = datetime.strptime(string, date_format) + try: + date = datetime.strptime(string, date_format) + except ValueError: + # Parsing failed. + return None precision = cls.precisions[ordinal] return cls(date, precision) @@ -416,7 +575,7 @@ class Period(object): elif 'day' == precision: return date + timedelta(days=1) else: - raise ValueError('unhandled precision ' + str(precision)) + raise ValueError(u'unhandled precision {0}'.format(precision)) class DateInterval(object): @@ -428,7 +587,7 @@ class DateInterval(object): def __init__(self, start, end): if start is not None and end is not None and not start < end: - raise ValueError("start date {0} is not before end date {1}" + raise ValueError(u"start date {0} is not before end date {1}" .format(start, end)) self.start = start self.end = end @@ -449,7 +608,7 @@ class DateInterval(object): return True def __str__(self): - return'[{0}, {1})'.format(self.start, self.end) + return '[{0}, {1})'.format(self.start, self.end) class DateQuery(FieldQuery): @@ -492,3 +651,208 @@ class DateQuery(FieldQuery): # Match any date. clause = '1' return clause, subvals + + +class DurationQuery(NumericQuery): + """NumericQuery that allow human-friendly (M:SS) time interval formats. + + Converts the range(s) to a float value, and delegates on NumericQuery. + + Raises InvalidQueryError when the pattern does not represent an int, float + or M:SS time interval. + """ + def _convert(self, s): + """Convert a M:SS or numeric string to a float. + + Return None if `s` is empty. + Raise an InvalidQueryError if the string cannot be converted. + """ + if not s: + return None + try: + return util.raw_seconds_short(s) + except ValueError: + try: + return float(s) + except ValueError: + raise InvalidQueryArgumentTypeError( + s, + u"a M:SS string or a float") + + +# Sorting. + +class Sort(object): + """An abstract class representing a sort operation for a query into + the item database. + """ + + def order_clause(self): + """Generates a SQL fragment to be used in a ORDER BY clause, or + None if no fragment is used (i.e., this is a slow sort). + """ + return None + + def sort(self, items): + """Sort the list of objects and return a list. + """ + return sorted(items) + + def is_slow(self): + """Indicate whether this query is *slow*, meaning that it cannot + be executed in SQL and must be executed in Python. + """ + return False + + def __hash__(self): + return 0 + + def __eq__(self, other): + return type(self) == type(other) + + +class MultipleSort(Sort): + """Sort that encapsulates multiple sub-sorts. + """ + + def __init__(self, sorts=None): + self.sorts = sorts or [] + + def add_sort(self, sort): + self.sorts.append(sort) + + def _sql_sorts(self): + """Return the list of sub-sorts for which we can be (at least + partially) fast. + + A contiguous suffix of fast (SQL-capable) sub-sorts are + executable in SQL. The remaining, even if they are fast + independently, must be executed slowly. + """ + sql_sorts = [] + for sort in reversed(self.sorts): + if not sort.order_clause() is None: + sql_sorts.append(sort) + else: + break + sql_sorts.reverse() + return sql_sorts + + def order_clause(self): + order_strings = [] + for sort in self._sql_sorts(): + order = sort.order_clause() + order_strings.append(order) + + return ", ".join(order_strings) + + def is_slow(self): + for sort in self.sorts: + if sort.is_slow(): + return True + return False + + def sort(self, items): + slow_sorts = [] + switch_slow = False + for sort in reversed(self.sorts): + if switch_slow: + slow_sorts.append(sort) + elif sort.order_clause() is None: + switch_slow = True + slow_sorts.append(sort) + else: + pass + + for sort in slow_sorts: + items = sort.sort(items) + return items + + def __repr__(self): + return 'MultipleSort({!r})'.format(self.sorts) + + def __hash__(self): + return hash(tuple(self.sorts)) + + def __eq__(self, other): + return super(MultipleSort, self).__eq__(other) and \ + self.sorts == other.sorts + + +class FieldSort(Sort): + """An abstract sort criterion that orders by a specific field (of + any kind). + """ + def __init__(self, field, ascending=True, case_insensitive=True): + self.field = field + self.ascending = ascending + self.case_insensitive = case_insensitive + + def sort(self, objs): + # TODO: Conversion and null-detection here. In Python 3, + # comparisons with None fail. We should also support flexible + # attributes with different types without falling over. + + def key(item): + field_val = item.get(self.field, '') + if self.case_insensitive and isinstance(field_val, unicode): + field_val = field_val.lower() + return field_val + + return sorted(objs, key=key, reverse=not self.ascending) + + def __repr__(self): + return '<{0}: {1}{2}>'.format( + type(self).__name__, + self.field, + '+' if self.ascending else '-', + ) + + def __hash__(self): + return hash((self.field, self.ascending)) + + def __eq__(self, other): + return super(FieldSort, self).__eq__(other) and \ + self.field == other.field and \ + self.ascending == other.ascending + + +class FixedFieldSort(FieldSort): + """Sort object to sort on a fixed field. + """ + def order_clause(self): + order = "ASC" if self.ascending else "DESC" + if self.case_insensitive: + field = '(CASE ' \ + 'WHEN TYPEOF({0})="text" THEN LOWER({0}) ' \ + 'WHEN TYPEOF({0})="blob" THEN LOWER({0}) ' \ + 'ELSE {0} END)'.format(self.field) + else: + field = self.field + return "{0} {1}".format(field, order) + + +class SlowFieldSort(FieldSort): + """A sort criterion by some model field other than a fixed field: + i.e., a computed or flexible field. + """ + def is_slow(self): + return True + + +class NullSort(Sort): + """No sorting. Leave results unsorted.""" + def sort(self, items): + return items + + def __nonzero__(self): + return self.__bool__() + + def __bool__(self): + return False + + def __eq__(self, other): + return type(self) == type(other) or other is None + + def __hash__(self): + return 0 diff --git a/libs/beets/dbcore/queryparse.py b/libs/beets/dbcore/queryparse.py new file mode 100644 index 00000000..bc9cc77e --- /dev/null +++ b/libs/beets/dbcore/queryparse.py @@ -0,0 +1,250 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Adrian Sampson. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Parsing of strings into DBCore queries. +""" +from __future__ import division, absolute_import, print_function + +import re +import itertools +from . import query +import beets + +PARSE_QUERY_PART_REGEX = re.compile( + # Non-capturing optional segment for the keyword. + r'(-|\^)?' # Negation prefixes. + + r'(?:' + r'(\S+?)' # The field key. + r'(? `(None, 'stapler', SubstringQuery, False)` + - `'color:red'` -> `('color', 'red', SubstringQuery, False)` + - `':^Quiet'` -> `(None, '^Quiet', RegexpQuery, False)`, because + the `^` follows the `:` + - `'color::b..e'` -> `('color', 'b..e', RegexpQuery, False)` + - `'-color:red'` -> `('color', 'red', SubstringQuery, True)` + """ + # Apply the regular expression and extract the components. + part = part.strip() + match = PARSE_QUERY_PART_REGEX.match(part) + + assert match # Regex should always match + negate = bool(match.group(1)) + key = match.group(2) + term = match.group(3).replace('\:', ':') + + # Check whether there's a prefix in the query and use the + # corresponding query type. + for pre, query_class in prefixes.items(): + if term.startswith(pre): + return key, term[len(pre):], query_class, negate + + # No matching prefix, so use either the query class determined by + # the field or the default as a fallback. + query_class = query_classes.get(key, default_class) + return key, term, query_class, negate + + +def construct_query_part(model_cls, prefixes, query_part): + """Parse a *query part* string and return a :class:`Query` object. + + :param model_cls: The :class:`Model` class that this is a query for. + This is used to determine the appropriate query types for the + model's fields. + :param prefixes: A map from prefix strings to :class:`Query` types. + :param query_part: The string to parse. + + See the documentation for `parse_query_part` for more information on + query part syntax. + """ + # A shortcut for empty query parts. + if not query_part: + return query.TrueQuery() + + # Use `model_cls` to build up a map from field names to `Query` + # classes. + query_classes = {} + for k, t in itertools.chain(model_cls._fields.items(), + model_cls._types.items()): + query_classes[k] = t.query + + # Parse the string. + key, pattern, query_class, negate = \ + parse_query_part(query_part, query_classes, prefixes) + + # If there's no key (field name) specified, this is a "match + # anything" query. + if key is None: + if issubclass(query_class, query.FieldQuery): + # The query type matches a specific field, but none was + # specified. So we use a version of the query that matches + # any field. + q = query.AnyFieldQuery(pattern, model_cls._search_fields, + query_class) + if negate: + return query.NotQuery(q) + else: + return q + else: + # Non-field query type. + if negate: + return query.NotQuery(query_class(pattern)) + else: + return query_class(pattern) + + # Otherwise, this must be a `FieldQuery`. Use the field name to + # construct the query object. + key = key.lower() + q = query_class(key.lower(), pattern, key in model_cls._fields) + if negate: + return query.NotQuery(q) + return q + + +def query_from_strings(query_cls, model_cls, prefixes, query_parts): + """Creates a collection query of type `query_cls` from a list of + strings in the format used by parse_query_part. `model_cls` + determines how queries are constructed from strings. + """ + subqueries = [] + for part in query_parts: + subqueries.append(construct_query_part(model_cls, prefixes, part)) + if not subqueries: # No terms in query. + subqueries = [query.TrueQuery()] + return query_cls(subqueries) + + +def construct_sort_part(model_cls, part): + """Create a `Sort` from a single string criterion. + + `model_cls` is the `Model` being queried. `part` is a single string + ending in ``+`` or ``-`` indicating the sort. + """ + assert part, "part must be a field name and + or -" + field = part[:-1] + assert field, "field is missing" + direction = part[-1] + assert direction in ('+', '-'), "part must end with + or -" + is_ascending = direction == '+' + + case_insensitive = beets.config['sort_case_insensitive'].get(bool) + if field in model_cls._sorts: + sort = model_cls._sorts[field](model_cls, is_ascending, + case_insensitive) + elif field in model_cls._fields: + sort = query.FixedFieldSort(field, is_ascending, case_insensitive) + else: + # Flexible or computed. + sort = query.SlowFieldSort(field, is_ascending, case_insensitive) + return sort + + +def sort_from_strings(model_cls, sort_parts): + """Create a `Sort` from a list of sort criteria (strings). + """ + if not sort_parts: + sort = query.NullSort() + elif len(sort_parts) == 1: + sort = construct_sort_part(model_cls, sort_parts[0]) + else: + sort = query.MultipleSort() + for part in sort_parts: + sort.add_sort(construct_sort_part(model_cls, part)) + return sort + + +def parse_sorted_query(model_cls, parts, prefixes={}): + """Given a list of strings, create the `Query` and `Sort` that they + represent. + """ + # Separate query token and sort token. + query_parts = [] + sort_parts = [] + + # Split up query in to comma-separated subqueries, each representing + # an AndQuery, which need to be joined together in one OrQuery + subquery_parts = [] + for part in parts + [u',']: + if part.endswith(u','): + # Ensure we can catch "foo, bar" as well as "foo , bar" + last_subquery_part = part[:-1] + if last_subquery_part: + subquery_parts.append(last_subquery_part) + # Parse the subquery in to a single AndQuery + # TODO: Avoid needlessly wrapping AndQueries containing 1 subquery? + query_parts.append(query_from_strings( + query.AndQuery, model_cls, prefixes, subquery_parts + )) + del subquery_parts[:] + else: + # Sort parts (1) end in + or -, (2) don't have a field, and + # (3) consist of more than just the + or -. + if part.endswith((u'+', u'-')) \ + and u':' not in part \ + and len(part) > 1: + sort_parts.append(part) + else: + subquery_parts.append(part) + + # Avoid needlessly wrapping single statements in an OR + q = query.OrQuery(query_parts) if len(query_parts) > 1 else query_parts[0] + s = sort_from_strings(model_cls, sort_parts) + return q, s diff --git a/libs/beets/dbcore/types.py b/libs/beets/dbcore/types.py index 165c0b60..2726969d 100644 --- a/libs/beets/dbcore/types.py +++ b/libs/beets/dbcore/types.py @@ -1,5 +1,6 @@ +# -*- coding: utf-8 -*- # This file is part of beets. -# Copyright 2014, Adrian Sampson. +# Copyright 2016, Adrian Sampson. # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -14,59 +15,117 @@ """Representation of type information for DBCore model fields. """ +from __future__ import division, absolute_import, print_function + from . import query from beets.util import str2bool - # Abstract base. - class Type(object): """An object encapsulating the type of a model field. Includes - information about how to store the value in the database, query, - format, and parse a given field. + information about how to store, query, format, and parse a given + field. """ - sql = None + sql = u'TEXT' """The SQLite column type for the value. """ - query = None + query = query.SubstringQuery """The `Query` subclass to be used when querying the field. """ + model_type = unicode + """The Python type that is used to represent the value in the model. + + The model is guaranteed to return a value of this type if the field + is accessed. To this end, the constructor is used by the `normalize` + and `from_sql` methods and the `default` property. + """ + + @property + def null(self): + """The value to be exposed when the underlying value is None. + """ + return self.model_type() + def format(self, value): """Given a value of this type, produce a Unicode string representing the value. This is used in template evaluation. """ - raise NotImplementedError() + if value is None: + value = self.null + # `self.null` might be `None` + if value is None: + value = u'' + if isinstance(value, bytes): + value = value.decode('utf8', 'ignore') + + return unicode(value) def parse(self, string): """Parse a (possibly human-written) string and return the indicated value of this type. """ - raise NotImplementedError() + try: + return self.model_type(string) + except ValueError: + return self.null + def normalize(self, value): + """Given a value that will be assigned into a field of this + type, normalize the value to have the appropriate type. This + base implementation only reinterprets `None`. + """ + if value is None: + return self.null + else: + # TODO This should eventually be replaced by + # `self.model_type(value)` + return value + + def from_sql(self, sql_value): + """Receives the value stored in the SQL backend and return the + value to be stored in the model. + + For fixed fields the type of `value` is determined by the column + type affinity given in the `sql` property and the SQL to Python + mapping of the database adapter. For more information see: + http://www.sqlite.org/datatype3.html + https://docs.python.org/2/library/sqlite3.html#sqlite-and-python-types + + Flexible fields have the type affinity `TEXT`. This means the + `sql_value` is either a `buffer` or a `unicode` object` and the + method must handle these in addition. + """ + if isinstance(sql_value, buffer): + sql_value = bytes(sql_value).decode('utf8', 'ignore') + if isinstance(sql_value, unicode): + return self.parse(sql_value) + else: + return self.normalize(sql_value) + + def to_sql(self, model_value): + """Convert a value as stored in the model object to a value used + by the database adapter. + """ + return model_value # Reusable types. +class Default(Type): + null = None + class Integer(Type): """A basic integer type. """ sql = u'INTEGER' query = query.NumericQuery - - def format(self, value): - return unicode(value or 0) - - def parse(self, string): - try: - return int(string) - except ValueError: - return 0 + model_type = int class PaddedInt(Integer): @@ -93,9 +152,14 @@ class ScaledInt(Integer): class Id(Integer): - """An integer used as the row key for a SQLite table. + """An integer used as the row id or a foreign key in a SQLite table. + This type is nullable: None values are not translated to zero. """ - sql = u'INTEGER PRIMARY KEY' + null = None + + def __init__(self, primary=True): + if primary: + self.sql = u'INTEGER PRIMARY KEY' class Float(Type): @@ -103,15 +167,16 @@ class Float(Type): """ sql = u'REAL' query = query.NumericQuery + model_type = float def format(self, value): return u'{0:.1f}'.format(value or 0.0) - def parse(self, string): - try: - return float(string) - except ValueError: - return 0.0 + +class NullFloat(Float): + """Same as `Float`, but does not normalize `None` to `0.0`. + """ + null = None class String(Type): @@ -120,21 +185,27 @@ class String(Type): sql = u'TEXT' query = query.SubstringQuery - def format(self, value): - return unicode(value) if value else u'' - - def parse(self, string): - return string - class Boolean(Type): """A boolean type. """ sql = u'INTEGER' query = query.BooleanQuery + model_type = bool def format(self, value): return unicode(bool(value)) def parse(self, string): return str2bool(string) + + +# Shared instances of common types. +DEFAULT = Default() +INTEGER = Integer() +PRIMARY_ID = Id(True) +FOREIGN_ID = Id(False) +FLOAT = Float() +NULL_FLOAT = NullFloat() +STRING = String() +BOOLEAN = Boolean() diff --git a/libs/beets/importer.py b/libs/beets/importer.py index f997770c..bfaa21a0 100644 --- a/libs/beets/importer.py +++ b/libs/beets/importer.py @@ -1,5 +1,6 @@ +# -*- coding: utf-8 -*- # This file is part of beets. -# Copyright 2013, Adrian Sampson. +# Copyright 2016, Adrian Sampson. # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -12,40 +13,51 @@ # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. +from __future__ import division, absolute_import, print_function + """Provides the basic, interface-agnostic workflow for importing and autotagging music files. """ -from __future__ import print_function import os -import logging +import re import pickle import itertools from collections import defaultdict +from tempfile import mkdtemp +from bisect import insort, bisect_left +from contextlib import contextmanager +import shutil +import time +from beets import logging from beets import autotag from beets import library from beets import dbcore from beets import plugins from beets import util from beets import config -from beets.util import pipeline +from beets.util import pipeline, sorted_walk, ancestry from beets.util import syspath, normpath, displayable_path -from beets.util.enumeration import enum +from enum import Enum from beets import mediafile -action = enum( - 'SKIP', 'ASIS', 'TRACKS', 'MANUAL', 'APPLY', 'MANUAL_ID', - 'ALBUMS', name='action' -) +action = Enum('action', + ['SKIP', 'ASIS', 'TRACKS', 'MANUAL', 'APPLY', 'MANUAL_ID', + 'ALBUMS', 'RETAG']) +# The RETAG action represents "don't apply any match, but do record +# new metadata". It's not reachable via the standard command prompt but +# can be used by plugins. QUEUE_SIZE = 128 SINGLE_ARTIST_THRESH = 0.25 -VARIOUS_ARTISTS = u'Various Artists' +PROGRESS_KEY = 'tagprogress' +HISTORY_KEY = 'taghistory' # Global logger. log = logging.getLogger('beets') + class ImportAbort(Exception): """Raised when the user aborts the tagging operation. """ @@ -54,151 +66,91 @@ class ImportAbort(Exception): # Utilities. -def _duplicate_check(lib, task): - """Check whether an album already exists in the library. Returns a - list of Album objects (empty if no duplicates are found). - """ - assert task.choice_flag in (action.ASIS, action.APPLY) - artist, album = task.chosen_ident() - - if artist is None: - # As-is import with no artist. Skip check. - return [] - - found_albums = [] - cur_paths = set(i.path for i in task.items if i) - for album_cand in lib.albums(dbcore.MatchQuery('albumartist', artist)): - if album_cand.album == album: - # Check whether the album is identical in contents, in which - # case it is not a duplicate (will be replaced). - other_paths = set(i.path for i in album_cand.items()) - if other_paths == cur_paths: - continue - found_albums.append(album_cand) - return found_albums - -def _item_duplicate_check(lib, task): - """Check whether an item already exists in the library. Returns a - list of Item objects. - """ - assert task.choice_flag in (action.ASIS, action.APPLY) - artist, title = task.chosen_ident() - - found_items = [] - query = dbcore.AndQuery(( - dbcore.MatchQuery('artist', artist), - dbcore.MatchQuery('title', title), - )) - for other_item in lib.items(query): - # Existing items not considered duplicates. - if other_item.path == task.item.path: - continue - found_items.append(other_item) - return found_items - -def _infer_album_fields(task): - """Given an album and an associated import task, massage the - album-level metadata. This ensures that the album artist is set - and that the "compilation" flag is set automatically. - """ - assert task.is_album - assert task.items - - changes = {} - - if task.choice_flag == action.ASIS: - # Taking metadata "as-is". Guess whether this album is VA. - plur_albumartist, freq = util.plurality( - [i.albumartist or i.artist for i in task.items]) - if freq == len(task.items) or (freq > 1 and - float(freq) / len(task.items) >= SINGLE_ARTIST_THRESH): - # Single-artist album. - changes['albumartist'] = plur_albumartist - changes['comp'] = False - else: - # VA. - changes['albumartist'] = VARIOUS_ARTISTS - changes['comp'] = True - - elif task.choice_flag == action.APPLY: - # Applying autotagged metadata. Just get AA from the first - # item. - for item in task.items: - if item is not None: - first_item = item - break - else: - assert False, "all items are None" - if not first_item.albumartist: - changes['albumartist'] = first_item.artist - if not first_item.mb_albumartistid: - changes['mb_albumartistid'] = first_item.mb_artistid - - else: - assert False - - # Apply new metadata. - for item in task.items: - if item is not None: - for k, v in changes.iteritems(): - setattr(item, k, v) - -def _resume(): - """Check whether an import should resume and return a boolean or the - string 'ask' indicating that the user should be queried. - """ - return config['import']['resume'].as_choice([True, False, 'ask']) - def _open_state(): """Reads the state file, returning a dictionary.""" try: with open(config['statefile'].as_filename()) as f: return pickle.load(f) - except (IOError, EOFError): + except Exception as exc: + # The `pickle` module can emit all sorts of exceptions during + # unpickling, including ImportError. We use a catch-all + # exception to avoid enumerating them all (the docs don't even have a + # full list!). + log.debug(u'state file could not be read: {0}', exc) return {} + + def _save_state(state): """Writes the state dictionary out to disk.""" try: with open(config['statefile'].as_filename(), 'w') as f: pickle.dump(state, f) except IOError as exc: - log.error(u'state file could not be written: %s' % unicode(exc)) + log.error(u'state file could not be written: {0}', exc) # Utilities for reading and writing the beets progress file, which # allows long tagging tasks to be resumed when they pause (or crash). -PROGRESS_KEY = 'tagprogress' -def progress_set(toppath, paths): - """Record that tagging for the given `toppath` was successful up to - `paths`. If paths is None, then clear the progress value (indicating - that the tagging completed). - """ + +def progress_read(): state = _open_state() - if PROGRESS_KEY not in state: - state[PROGRESS_KEY] = {} + return state.setdefault(PROGRESS_KEY, {}) - if paths is None: - # Remove progress from file. - if toppath in state[PROGRESS_KEY]: - del state[PROGRESS_KEY][toppath] - else: - state[PROGRESS_KEY][toppath] = paths +@contextmanager +def progress_write(): + state = _open_state() + progress = state.setdefault(PROGRESS_KEY, {}) + yield progress _save_state(state) -def progress_get(toppath): - """Get the last successfully tagged subpath of toppath. If toppath - has no progress information, returns None. + + +def progress_add(toppath, *paths): + """Record that the files under all of the `paths` have been imported + under `toppath`. """ - state = _open_state() - if PROGRESS_KEY not in state: - return None - return state[PROGRESS_KEY].get(toppath) + with progress_write() as state: + imported = state.setdefault(toppath, []) + for path in paths: + # Normally `progress_add` will be called with the path + # argument increasing. This is because of the ordering in + # `albums_in_dir`. We take advantage of that to make the + # code faster + if imported and imported[len(imported) - 1] <= path: + imported.append(path) + else: + insort(imported, path) + + +def progress_element(toppath, path): + """Return whether `path` has been imported in `toppath`. + """ + state = progress_read() + if toppath not in state: + return False + imported = state[toppath] + i = bisect_left(imported, path) + return i != len(imported) and imported[i] == path + + +def has_progress(toppath): + """Return `True` if there exist paths that have already been + imported under `toppath`. + """ + state = progress_read() + return toppath in state + + +def progress_reset(toppath): + with progress_write() as state: + if toppath in state: + del state[toppath] # Similarly, utilities for manipulating the "incremental" import log. # This keeps track of all directories that were ever imported, which # allows the importer to only import new stuff. -HISTORY_KEY = 'taghistory' + def history_add(paths): """Indicate that the import of the album in `paths` is completed and should not be repeated in incremental imports. @@ -210,6 +162,8 @@ def history_add(paths): state[HISTORY_KEY].add(tuple(paths)) _save_state(state) + + def history_get(): """Get the set of completed path tuples in incremental imports. """ @@ -225,27 +179,37 @@ class ImportSession(object): """Controls an import action. Subclasses should implement methods to communicate with the user or otherwise make decisions. """ - def __init__(self, lib, logfile, paths, query): - """Create a session. `lib` is a Library object. `logfile` is a - file-like object open for writing or None if no logging is to be - performed. Either `paths` or `query` is non-null and indicates + def __init__(self, lib, loghandler, paths, query): + """Create a session. `lib` is a Library object. `loghandler` is a + logging.Handler. Either `paths` or `query` is non-null and indicates the source of files to be imported. """ self.lib = lib - self.logfile = logfile + self.logger = self._setup_logging(loghandler) self.paths = paths self.query = query + self._is_resuming = dict() # Normalize the paths. if self.paths: - self.paths = map(normpath, self.paths) + self.paths = list(map(normpath, self.paths)) - def _amend_config(self): - """Make implied changes the importer configuration. + def _setup_logging(self, loghandler): + logger = logging.getLogger(__name__) + logger.propagate = False + if not loghandler: + loghandler = logging.NullHandler() + logger.handlers = [loghandler] + return logger + + def set_config(self, config): + """Set `config` property from global import config and make + implied changes. """ # FIXME: Maybe this function should not exist and should instead # provide "decision wrappers" like "should_resume()", etc. - iconfig = config['import'] + iconfig = dict(config) + self.config = iconfig # Incremental and progress are mutually exclusive. if iconfig['incremental']: @@ -257,43 +221,46 @@ class ImportSession(object): iconfig['resume'] = False iconfig['incremental'] = False - # Copy and move are mutually exclusive. + # Copy, move, and link are mutually exclusive. if iconfig['move']: iconfig['copy'] = False + iconfig['link'] = False + elif iconfig['link']: + iconfig['copy'] = False + iconfig['move'] = False # Only delete when copying. if not iconfig['copy']: iconfig['delete'] = False + self.want_resume = config['resume'].as_choice([True, False, 'ask']) + def tag_log(self, status, paths): - """Log a message about a given album to logfile. The status should - reflect the reason the album couldn't be tagged. + """Log a message about a given album to the importer log. The status + should reflect the reason the album couldn't be tagged. """ - if self.logfile: - print(u'{0} {1}'.format(status, displayable_path(paths)), - file=self.logfile) - self.logfile.flush() + self.logger.info(u'{0} {1}', status, displayable_path(paths)) def log_choice(self, task, duplicate=False): """Logs the task's current choice if it should be logged. If ``duplicate``, then this is a secondary choice after a duplicate was detected and a decision was made. """ - paths = task.paths if task.is_album else [task.item.path] + paths = task.paths if duplicate: # Duplicate: log all three choices (skip, keep both, and trump). - if task.remove_duplicates: - self.tag_log('duplicate-replace', paths) + if task.should_remove_duplicates: + self.tag_log(u'duplicate-replace', paths) elif task.choice_flag in (action.ASIS, action.APPLY): - self.tag_log('duplicate-keep', paths) + self.tag_log(u'duplicate-keep', paths) elif task.choice_flag is (action.SKIP): - self.tag_log('duplicate-skip', paths) + self.tag_log(u'duplicate-skip', paths) else: # Non-duplicate: log "skip" and "asis" choices. if task.choice_flag is action.ASIS: - self.tag_log('asis', paths) + self.tag_log(u'asis', paths) elif task.choice_flag is action.SKIP: - self.tag_log('skip', paths) + self.tag_log(u'skip', paths) def should_resume(self, path): raise NotImplementedError @@ -301,7 +268,7 @@ class ImportSession(object): def choose_match(self, task): raise NotImplementedError - def resolve_duplicate(self, task): + def resolve_duplicate(self, task, found_duplicates): raise NotImplementedError def choose_item(self, task): @@ -310,38 +277,44 @@ class ImportSession(object): def run(self): """Run the import task. """ - self._amend_config() + self.logger.info(u'import started {0}', time.asctime()) + self.set_config(config['import']) # Set up the pipeline. if self.query is None: stages = [read_tasks(self)] else: stages = [query_tasks(self)] - if config['import']['singletons']: - # Singleton importer. - if config['import']['autotag']: - stages += [item_lookup(self), item_query(self)] - else: - stages += [item_progress(self)] + + # In pretend mode, just log what would otherwise be imported. + if self.config['pretend']: + stages += [log_files(self)] else: - # Whole-album importer. - if config['import']['group_albums']: - # Split directory tasks into one task for each album + if self.config['group_albums'] and \ + not self.config['singletons']: + # Split directory tasks into one task for each album. stages += [group_albums(self)] - if config['import']['autotag']: - # Only look up and query the user when autotagging. - stages += [initial_lookup(self), user_query(self)] + + # These stages either talk to the user to get a decision or, + # in the case of a non-autotagged import, just choose to + # import everything as-is. In *both* cases, these stages + # also add the music to the library database, so later + # stages need to read and write data from there. + if self.config['autotag']: + stages += [lookup_candidates(self), user_query(self)] else: - # When not autotagging, just display progress. - stages += [show_progress(self)] - stages += [apply_choices(self)] - for stage_func in plugins.import_stages(): - stages.append(plugin_stage(self, stage_func)) - stages += [manipulate_files(self)] - stages += [finalize(self)] + stages += [import_asis(self)] + + # Plugin stages. + for stage_func in plugins.import_stages(): + stages.append(plugin_stage(self, stage_func)) + + stages += [manipulate_files(self)] + pl = pipeline.Pipeline(stages) # Run the pipeline. + plugins.send('import_begin', session=self) try: if config['threaded']: pl.run_parallel(QUEUE_SIZE) @@ -351,91 +324,132 @@ class ImportSession(object): # User aborted operation. Silently stop. pass + # Incremental and resumed imports + + def already_imported(self, toppath, paths): + """Returns true if the files belonging to this task have already + been imported in a previous session. + """ + if self.is_resuming(toppath) \ + and all(map(lambda p: progress_element(toppath, p), paths)): + return True + if self.config['incremental'] \ + and tuple(paths) in self.history_dirs: + return True + + return False + + @property + def history_dirs(self): + if not hasattr(self, '_history_dirs'): + self._history_dirs = history_get() + return self._history_dirs + + def is_resuming(self, toppath): + """Return `True` if user wants to resume import of this path. + + You have to call `ask_resume` first to determine the return value. + """ + return self._is_resuming.get(toppath, False) + + def ask_resume(self, toppath): + """If import of `toppath` was aborted in an earlier session, ask + user if she wants to resume the import. + + Determines the return value of `is_resuming(toppath)`. + """ + if self.want_resume and has_progress(toppath): + # Either accept immediately or prompt for input to decide. + if self.want_resume is True or \ + self.should_resume(toppath): + log.warn(u'Resuming interrupted import of {0}', + util.displayable_path(toppath)) + self._is_resuming[toppath] = True + else: + # Clear progress; we're starting from the top. + progress_reset(toppath) + # The importer task class. -class ImportTask(object): - """Represents a single set of items to be imported along with its - intermediate state. May represent an album or a single item. - """ - def __init__(self, toppath=None, paths=None, items=None): +class BaseImportTask(object): + """An abstract base class for importer tasks. + + Tasks flow through the importer pipeline. Each stage can update + them. """ + def __init__(self, toppath, paths, items): + """Create a task. The primary fields that define a task are: + + * `toppath`: The user-specified base directory that contains the + music for this task. If the task has *no* user-specified base + (for example, when importing based on an -L query), this can + be None. This is used for tracking progress and history. + * `paths`: A list of *specific* paths where the music for this task + came from. These paths can be directories, when their entire + contents are being imported, or files, when the task comprises + individual tracks. This is used for progress/history tracking and + for displaying the task to the user. + * `items`: A list of `Item` objects representing the music being + imported. + + These fields should not change after initialization. + """ self.toppath = toppath self.paths = paths self.items = items - self.sentinel = False - self.remove_duplicates = False - self.is_album = True + + +class ImportTask(BaseImportTask): + """Represents a single set of items to be imported along with its + intermediate state. May represent an album or a single item. + + The import session and stages call the following methods in the + given order. + + * `lookup_candidates()` Sets the `common_artist`, `common_album`, + `candidates`, and `rec` attributes. `candidates` is a list of + `AlbumMatch` objects. + + * `choose_match()` Uses the session to set the `match` attribute + from the `candidates` list. + + * `find_duplicates()` Returns a list of albums from `lib` with the + same artist and album name as the task. + + * `apply_metadata()` Sets the attributes of the items from the + task's `match` attribute. + + * `add()` Add the imported items and album to the database. + + * `manipulate_files()` Copy, move, and write files depending on the + session configuration. + + * `finalize()` Update the import progress and cleanup the file + system. + """ + def __init__(self, toppath, paths, items): + super(ImportTask, self).__init__(toppath, paths, items) self.choice_flag = None - - @classmethod - def done_sentinel(cls, toppath): - """Create an ImportTask that indicates the end of a top-level - directory import. - """ - obj = cls(toppath) - obj.sentinel = True - return obj - - @classmethod - def progress_sentinel(cls, toppath, paths): - """Create a task indicating that a single directory in a larger - import has finished. This is only required for singleton - imports; progress is implied for album imports. - """ - obj = cls(toppath, paths) - obj.sentinel = True - return obj - - @classmethod - def item_task(cls, item): - """Creates an ImportTask for a single item.""" - obj = cls() - obj.item = item - obj.is_album = False - return obj - - def set_candidates(self, cur_artist, cur_album, candidates, rec): - """Sets the candidates for this album matched by the - `autotag.tag_album` method. - """ - assert self.is_album - assert not self.sentinel - self.cur_artist = cur_artist - self.cur_album = cur_album - self.candidates = candidates - self.rec = rec - - def set_null_candidates(self): - """Set the candidates to indicate no album match was found. - """ - self.cur_artist = None self.cur_album = None - self.candidates = None + self.cur_artist = None + self.candidates = [] self.rec = None - - def set_item_candidates(self, candidates, rec): - """Set the match for a single-item task.""" - assert not self.is_album - assert self.item is not None - self.candidates = candidates - self.rec = rec + self.should_remove_duplicates = False + self.is_album = True + self.search_ids = [] # user-supplied candidate IDs. def set_choice(self, choice): """Given an AlbumMatch or TrackMatch object or an action constant, indicates that an action has been selected for this task. """ - assert not self.sentinel # Not part of the task structure: assert choice not in (action.MANUAL, action.MANUAL_ID) assert choice != action.APPLY # Only used internally. - if choice in (action.SKIP, action.ASIS, action.TRACKS, action.ALBUMS): + if choice in (action.SKIP, action.ASIS, action.TRACKS, action.ALBUMS, + action.RETAG): self.choice_flag = choice self.match = None else: - if self.is_album: - assert isinstance(choice, autotag.AlbumMatch) - else: - assert isinstance(choice, autotag.TrackMatch) self.choice_flag = action.APPLY # Implicit choice. self.match = choice @@ -443,38 +457,24 @@ class ImportTask(object): """Updates the progress state to indicate that this album has finished. """ - if self.sentinel and self.paths is None: - # "Done" sentinel. - progress_set(self.toppath, None) - elif self.sentinel or self.is_album: - # "Directory progress" sentinel for singletons or a real - # album task, which implies the same. - progress_set(self.toppath, self.paths) + if self.toppath: + progress_add(self.toppath, *self.paths) def save_history(self): """Save the directory in the history for incremental imports. """ - if self.is_album and self.paths and not self.sentinel: + if self.paths: history_add(self.paths) - # Logical decisions. - def should_write_tags(self): - """Should new info be written to the files' metadata?""" - if self.choice_flag == action.APPLY: - return True - elif self.choice_flag in (action.ASIS, action.TRACKS, action.SKIP): - return False - else: - assert False - - def should_skip(self): - """After a choice has been made, returns True if this is a - sentinel or it has been marked for skipping. - """ - return self.sentinel or self.choice_flag == action.SKIP + @property + def apply(self): + return self.choice_flag == action.APPLY + @property + def skip(self): + return self.choice_flag == action.SKIP # Convenient data. @@ -482,37 +482,310 @@ class ImportTask(object): """Returns identifying metadata about the current choice. For albums, this is an (artist, album) pair. For items, this is (artist, title). May only be called when the choice flag is ASIS - (in which case the data comes from the files' current metadata) - or APPLY (data comes from the choice). + or RETAG (in which case the data comes from the files' current + metadata) or APPLY (data comes from the choice). """ - assert self.choice_flag in (action.ASIS, action.APPLY) - if self.is_album: - if self.choice_flag is action.ASIS: - return (self.cur_artist, self.cur_album) - elif self.choice_flag is action.APPLY: - return (self.match.info.artist, self.match.info.album) - else: - if self.choice_flag is action.ASIS: - return (self.item.artist, self.item.title) - elif self.choice_flag is action.APPLY: - return (self.match.info.artist, self.match.info.title) + if self.choice_flag in (action.ASIS, action.RETAG): + return (self.cur_artist, self.cur_album) + elif self.choice_flag is action.APPLY: + return (self.match.info.artist, self.match.info.album) def imported_items(self): """Return a list of Items that should be added to the library. - If this is an album task, return the list of items in the - selected match or everything if the choice is ASIS. If this is a - singleton task, return a list containing the item. + + If the tasks applies an album match the method only returns the + matched items. + """ + if self.choice_flag in (action.ASIS, action.RETAG): + return list(self.items) + elif self.choice_flag == action.APPLY: + return self.match.mapping.keys() + else: + assert False + + def apply_metadata(self): + """Copy metadata from match info to the items. + """ + autotag.apply_metadata(self.match.info, self.match.mapping) + + def duplicate_items(self, lib): + duplicate_items = [] + for album in self.find_duplicates(lib): + duplicate_items += album.items() + return duplicate_items + + def remove_duplicates(self, lib): + duplicate_items = self.duplicate_items(lib) + log.debug(u'removing {0} old duplicated items', len(duplicate_items)) + for item in duplicate_items: + item.remove() + if lib.directory in util.ancestry(item.path): + log.debug(u'deleting duplicate {0}', + util.displayable_path(item.path)) + util.remove(item.path) + util.prune_dirs(os.path.dirname(item.path), + lib.directory) + + def finalize(self, session): + """Save progress, clean up files, and emit plugin event. + """ + # Update progress. + if session.want_resume: + self.save_progress() + if session.config['incremental']: + self.save_history() + + self.cleanup(copy=session.config['copy'], + delete=session.config['delete'], + move=session.config['move']) + + if not self.skip: + self._emit_imported(session.lib) + + def cleanup(self, copy=False, delete=False, move=False): + """Remove and prune imported paths. + """ + # Do not delete any files or prune directories when skipping. + if self.skip: + return + + items = self.imported_items() + + # When copying and deleting originals, delete old files. + if copy and delete: + new_paths = [os.path.realpath(item.path) for item in items] + for old_path in self.old_paths: + # Only delete files that were actually copied. + if old_path not in new_paths: + util.remove(syspath(old_path), False) + self.prune(old_path) + + # When moving, prune empty directories containing the original files. + elif move: + for old_path in self.old_paths: + self.prune(old_path) + + def _emit_imported(self, lib): + plugins.send('album_imported', lib=lib, album=self.album) + + def handle_created(self, session): + """Send the `import_task_created` event for this task. Return a list of + tasks that should continue through the pipeline. By default, this is a + list containing only the task itself, but plugins can replace the task + with new ones. + """ + tasks = plugins.send('import_task_created', session=session, task=self) + if not tasks: + tasks = [self] + else: + # The plugins gave us a list of lists of tasks. Flatten it. + tasks = [t for inner in tasks for t in inner] + return tasks + + def lookup_candidates(self): + """Retrieve and store candidates for this album. User-specified + candidate IDs are stored in self.search_ids: if present, the + initial lookup is restricted to only those IDs. + """ + artist, album, candidates, recommendation = \ + autotag.tag_album(self.items, search_ids=self.search_ids) + self.cur_artist = artist + self.cur_album = album + self.candidates = candidates + self.rec = recommendation + + def find_duplicates(self, lib): + """Return a list of albums from `lib` with the same artist and + album name as the task. + """ + artist, album = self.chosen_ident() + + if artist is None: + # As-is import with no artist. Skip check. + return [] + + duplicates = [] + task_paths = set(i.path for i in self.items if i) + duplicate_query = dbcore.AndQuery(( + dbcore.MatchQuery('albumartist', artist), + dbcore.MatchQuery('album', album), + )) + + for album in lib.albums(duplicate_query): + # Check whether the album is identical in contents, in which + # case it is not a duplicate (will be replaced). + album_paths = set(i.path for i in album.items()) + if album_paths != task_paths: + duplicates.append(album) + return duplicates + + def align_album_level_fields(self): + """Make some album fields equal across `self.items`. For the + RETAG action, we assume that the responsible for returning it + (ie. a plugin) always ensures that the first item contains + valid data on the relevant fields. + """ + changes = {} + + if self.choice_flag == action.ASIS: + # Taking metadata "as-is". Guess whether this album is VA. + plur_albumartist, freq = util.plurality( + [i.albumartist or i.artist for i in self.items] + ) + if freq == len(self.items) or \ + (freq > 1 and + float(freq) / len(self.items) >= SINGLE_ARTIST_THRESH): + # Single-artist album. + changes['albumartist'] = plur_albumartist + changes['comp'] = False + else: + # VA. + changes['albumartist'] = config['va_name'].get(unicode) + changes['comp'] = True + + elif self.choice_flag in (action.APPLY, action.RETAG): + # Applying autotagged metadata. Just get AA from the first + # item. + if not self.items[0].albumartist: + changes['albumartist'] = self.items[0].artist + if not self.items[0].mb_albumartistid: + changes['mb_albumartistid'] = self.items[0].mb_artistid + + # Apply new metadata. + for item in self.items: + item.update(changes) + + def manipulate_files(self, move=False, copy=False, write=False, + link=False, session=None): + items = self.imported_items() + # Save the original paths of all items for deletion and pruning + # in the next step (finalization). + self.old_paths = [item.path for item in items] + for item in items: + if move or copy or link: + # In copy and link modes, treat re-imports specially: + # move in-library files. (Out-of-library files are + # copied/moved as usual). + old_path = item.path + if (copy or link) and self.replaced_items[item] and \ + session.lib.directory in util.ancestry(old_path): + item.move() + # We moved the item, so remove the + # now-nonexistent file from old_paths. + self.old_paths.remove(old_path) + else: + # A normal import. Just copy files and keep track of + # old paths. + item.move(copy, link) + + if write and (self.apply or self.choice_flag == action.RETAG): + item.try_write() + + with session.lib.transaction(): + for item in self.imported_items(): + item.store() + + plugins.send('import_task_files', session=session, task=self) + + def add(self, lib): + """Add the items as an album to the library and remove replaced items. + """ + self.align_album_level_fields() + with lib.transaction(): + self.record_replaced(lib) + self.remove_replaced(lib) + self.album = lib.add_album(self.imported_items()) + self.reimport_metadata(lib) + + def record_replaced(self, lib): + """Records the replaced items and albums in the `replaced_items` + and `replaced_albums` dictionaries. + """ + self.replaced_items = defaultdict(list) + self.replaced_albums = defaultdict(list) + replaced_album_ids = set() + for item in self.imported_items(): + dup_items = list(lib.items( + dbcore.query.BytesQuery('path', item.path) + )) + self.replaced_items[item] = dup_items + for dup_item in dup_items: + if (not dup_item.album_id or + dup_item.album_id in replaced_album_ids): + continue + replaced_album = dup_item.get_album() + if replaced_album: + replaced_album_ids.add(dup_item.album_id) + self.replaced_albums[replaced_album.path] = replaced_album + + def reimport_metadata(self, lib): + """For reimports, preserves metadata for reimported items and + albums. """ if self.is_album: - if self.choice_flag == action.ASIS: - return list(self.items) - elif self.choice_flag == action.APPLY: - return self.match.mapping.keys() - else: - assert False - else: - return [self.item] + replaced_album = self.replaced_albums.get(self.album.path) + if replaced_album: + self.album.added = replaced_album.added + self.album.update(replaced_album._values_flex) + self.album.artpath = replaced_album.artpath + self.album.store() + log.debug( + u'Reimported album: added {0}, flexible ' + u'attributes {1} from album {2} for {3}', + self.album.added, + replaced_album._values_flex.keys(), + replaced_album.id, + displayable_path(self.album.path) + ) + for item in self.imported_items(): + dup_items = self.replaced_items[item] + for dup_item in dup_items: + if dup_item.added and dup_item.added != item.added: + item.added = dup_item.added + log.debug( + u'Reimported item added {0} ' + u'from item {1} for {2}', + item.added, + dup_item.id, + displayable_path(item.path) + ) + item.update(dup_item._values_flex) + log.debug( + u'Reimported item flexible attributes {0} ' + u'from item {1} for {2}', + dup_item._values_flex.keys(), + dup_item.id, + displayable_path(item.path) + ) + item.store() + + def remove_replaced(self, lib): + """Removes all the items from the library that have the same + path as an item from this task. + """ + for item in self.imported_items(): + for dup_item in self.replaced_items[item]: + log.debug(u'Replacing item {0}: {1}', + dup_item.id, displayable_path(item.path)) + dup_item.remove() + log.debug(u'{0} of {1} items replaced', + sum(bool(l) for l in self.replaced_items.values()), + len(self.imported_items())) + + def choose_match(self, session): + """Ask the session which match should apply and apply it. + """ + choice = session.choose_match(self) + self.set_choice(choice) + session.log_choice(self) + + def reload(self): + """Reload albums and items from the database. + """ + for item in self.imported_items(): + item.load() + self.album.load() # Utilities. @@ -529,6 +802,389 @@ class ImportTask(object): clutter=config['clutter'].as_str_seq()) +class SingletonImportTask(ImportTask): + """ImportTask for a single track that is not associated to an album. + """ + + def __init__(self, toppath, item): + super(SingletonImportTask, self).__init__(toppath, [item.path], [item]) + self.item = item + self.is_album = False + self.paths = [item.path] + + def chosen_ident(self): + assert self.choice_flag in (action.ASIS, action.APPLY, action.RETAG) + if self.choice_flag in (action.ASIS, action.RETAG): + return (self.item.artist, self.item.title) + elif self.choice_flag is action.APPLY: + return (self.match.info.artist, self.match.info.title) + + def imported_items(self): + return [self.item] + + def apply_metadata(self): + autotag.apply_item_metadata(self.item, self.match.info) + + def _emit_imported(self, lib): + for item in self.imported_items(): + plugins.send('item_imported', lib=lib, item=item) + + def lookup_candidates(self): + candidates, recommendation = autotag.tag_item( + self.item, search_ids=self.search_ids) + self.candidates = candidates + self.rec = recommendation + + def find_duplicates(self, lib): + """Return a list of items from `lib` that have the same artist + and title as the task. + """ + artist, title = self.chosen_ident() + + found_items = [] + query = dbcore.AndQuery(( + dbcore.MatchQuery('artist', artist), + dbcore.MatchQuery('title', title), + )) + for other_item in lib.items(query): + # Existing items not considered duplicates. + if other_item.path != self.item.path: + found_items.append(other_item) + return found_items + + duplicate_items = find_duplicates + + def add(self, lib): + with lib.transaction(): + self.record_replaced(lib) + self.remove_replaced(lib) + lib.add(self.item) + self.reimport_metadata(lib) + + def infer_album_fields(self): + raise NotImplementedError + + def choose_match(self, session): + """Ask the session which match should apply and apply it. + """ + choice = session.choose_item(self) + self.set_choice(choice) + session.log_choice(self) + + def reload(self): + self.item.load() + + +# FIXME The inheritance relationships are inverted. This is why there +# are so many methods which pass. More responsibility should be delegated to +# the BaseImportTask class. +class SentinelImportTask(ImportTask): + """A sentinel task marks the progress of an import and does not + import any items itself. + + If only `toppath` is set the task indicates the end of a top-level + directory import. If the `paths` argument is also given, the task + indicates the progress in the `toppath` import. + """ + + def __init__(self, toppath, paths): + super(SentinelImportTask, self).__init__(toppath, paths, ()) + # TODO Remove the remaining attributes eventually + self.should_remove_duplicates = False + self.is_album = True + self.choice_flag = None + + def save_history(self): + pass + + def save_progress(self): + if self.paths is None: + # "Done" sentinel. + progress_reset(self.toppath) + else: + # "Directory progress" sentinel for singletons + progress_add(self.toppath, *self.paths) + + def skip(self): + return True + + def set_choice(self, choice): + raise NotImplementedError + + def cleanup(self, **kwargs): + pass + + def _emit_imported(self, session): + pass + + +class ArchiveImportTask(SentinelImportTask): + """An import task that represents the processing of an archive. + + `toppath` must be a `zip`, `tar`, or `rar` archive. Archive tasks + serve two purposes: + - First, it will unarchive the files to a temporary directory and + return it. The client should read tasks from the resulting + directory and send them through the pipeline. + - Second, it will clean up the temporary directory when it proceeds + through the pipeline. The client should send the archive task + after sending the rest of the music tasks to make this work. + """ + + def __init__(self, toppath): + super(ArchiveImportTask, self).__init__(toppath, ()) + self.extracted = False + + @classmethod + def is_archive(cls, path): + """Returns true if the given path points to an archive that can + be handled. + """ + if not os.path.isfile(path): + return False + + for path_test, _ in cls.handlers(): + if path_test(path): + return True + return False + + @classmethod + def handlers(cls): + """Returns a list of archive handlers. + + Each handler is a `(path_test, ArchiveClass)` tuple. `path_test` + is a function that returns `True` if the given path can be + handled by `ArchiveClass`. `ArchiveClass` is a class that + implements the same interface as `tarfile.TarFile`. + """ + if not hasattr(cls, '_handlers'): + cls._handlers = [] + from zipfile import is_zipfile, ZipFile + cls._handlers.append((is_zipfile, ZipFile)) + from tarfile import is_tarfile, TarFile + cls._handlers.append((is_tarfile, TarFile)) + try: + from rarfile import is_rarfile, RarFile + except ImportError: + pass + else: + cls._handlers.append((is_rarfile, RarFile)) + + return cls._handlers + + def cleanup(self, **kwargs): + """Removes the temporary directory the archive was extracted to. + """ + if self.extracted: + log.debug(u'Removing extracted directory: {0}', + displayable_path(self.toppath)) + shutil.rmtree(self.toppath) + + def extract(self): + """Extracts the archive to a temporary directory and sets + `toppath` to that directory. + """ + for path_test, handler_class in self.handlers(): + if path_test(self.toppath): + break + + try: + extract_to = mkdtemp() + archive = handler_class(self.toppath, mode='r') + archive.extractall(extract_to) + finally: + archive.close() + self.extracted = True + self.toppath = extract_to + + +class ImportTaskFactory(object): + """Generate album and singleton import tasks for all media files + indicated by a path. + """ + def __init__(self, toppath, session): + """Create a new task factory. + + `toppath` is the user-specified path to search for music to + import. `session` is the `ImportSession`, which controls how + tasks are read from the directory. + """ + self.toppath = toppath + self.session = session + self.skipped = 0 # Skipped due to incremental/resume. + self.imported = 0 # "Real" tasks created. + self.is_archive = ArchiveImportTask.is_archive(syspath(toppath)) + + def tasks(self): + """Yield all import tasks for music found in the user-specified + path `self.toppath`. Any necessary sentinel tasks are also + produced. + + During generation, update `self.skipped` and `self.imported` + with the number of tasks that were not produced (due to + incremental mode or resumed imports) and the number of concrete + tasks actually produced, respectively. + + If `self.toppath` is an archive, it is adjusted to point to the + extracted data. + """ + # Check whether this is an archive. + if self.is_archive: + archive_task = self.unarchive() + if not archive_task: + return + + # Search for music in the directory. + for dirs, paths in self.paths(): + if self.session.config['singletons']: + for path in paths: + tasks = self._create(self.singleton(path)) + for task in tasks: + yield task + yield self.sentinel(dirs) + + else: + tasks = self._create(self.album(paths, dirs)) + for task in tasks: + yield task + + # Produce the final sentinel for this toppath to indicate that + # it is finished. This is usually just a SentinelImportTask, but + # for archive imports, send the archive task instead (to remove + # the extracted directory). + if self.is_archive: + yield archive_task + else: + yield self.sentinel() + + def _create(self, task): + """Handle a new task to be emitted by the factory. + + Emit the `import_task_created` event and increment the + `imported` count if the task is not skipped. Return the same + task. If `task` is None, do nothing. + """ + if task: + tasks = task.handle_created(self.session) + self.imported += len(tasks) + return tasks + return [] + + def paths(self): + """Walk `self.toppath` and yield `(dirs, files)` pairs where + `files` are individual music files and `dirs` the set of + containing directories where the music was found. + + This can either be a recursive search in the ordinary case, a + single track when `toppath` is a file, a single directory in + `flat` mode. + """ + if not os.path.isdir(syspath(self.toppath)): + yield [self.toppath], [self.toppath] + elif self.session.config['flat']: + paths = [] + for dirs, paths_in_dir in albums_in_dir(self.toppath): + paths += paths_in_dir + yield [self.toppath], paths + else: + for dirs, paths in albums_in_dir(self.toppath): + yield dirs, paths + + def singleton(self, path): + """Return a `SingletonImportTask` for the music file. + """ + if self.session.already_imported(self.toppath, [path]): + log.debug(u'Skipping previously-imported path: {0}', + displayable_path(path)) + self.skipped += 1 + return None + + item = self.read_item(path) + if item: + return SingletonImportTask(self.toppath, item) + else: + return None + + def album(self, paths, dirs=None): + """Return a `ImportTask` with all media files from paths. + + `dirs` is a list of parent directories used to record already + imported albums. + """ + if not paths: + return None + + if dirs is None: + dirs = list(set(os.path.dirname(p) for p in paths)) + + if self.session.already_imported(self.toppath, dirs): + log.debug(u'Skipping previously-imported path: {0}', + displayable_path(dirs)) + self.skipped += 1 + return None + + items = map(self.read_item, paths) + items = [item for item in items if item] + + if items: + return ImportTask(self.toppath, dirs, items) + else: + return None + + def sentinel(self, paths=None): + """Return a `SentinelImportTask` indicating the end of a + top-level directory import. + """ + return SentinelImportTask(self.toppath, paths) + + def unarchive(self): + """Extract the archive for this `toppath`. + + Extract the archive to a new directory, adjust `toppath` to + point to the extracted directory, and return an + `ArchiveImportTask`. If extraction fails, return None. + """ + assert self.is_archive + + if not (self.session.config['move'] or + self.session.config['copy']): + log.warn(u"Archive importing requires either " + u"'copy' or 'move' to be enabled.") + return + + log.debug(u'Extracting archive: {0}', + displayable_path(self.toppath)) + archive_task = ArchiveImportTask(self.toppath) + try: + archive_task.extract() + except Exception as exc: + log.error(u'extraction failed: {0}', exc) + return + + # Now read albums from the extracted directory. + self.toppath = archive_task.toppath + log.debug(u'Archive extracted to: {0}', self.toppath) + return archive_task + + def read_item(self, path): + """Return an `Item` read from the path. + + If an item cannot be read, return `None` instead and log an + error. + """ + try: + return library.Item.from_path(path) + except library.ReadError as exc: + if isinstance(exc.reason, mediafile.FileTypeError): + # Silently ignore non-music files. + pass + elif isinstance(exc.reason, mediafile.UnreadableFileError): + log.warn(u'unreadable file: {0}', displayable_path(path)) + else: + log.error(u'error reading {0}: {1}', + displayable_path(path), exc) + + # Full-album pipeline stages. def read_tasks(session): @@ -536,483 +1192,232 @@ def read_tasks(session): in the user-specified list of paths. In the case of a singleton import, yields single-item tasks instead. """ - # Look for saved progress. - if _resume(): - resume_dirs = {} - for path in session.paths: - resume_dir = progress_get(path) - if resume_dir: - - # Either accept immediately or prompt for input to decide. - if _resume() is True: - do_resume = True - log.warn('Resuming interrupted import of %s' % path) - else: - do_resume = session.should_resume(path) - - if do_resume: - resume_dirs[path] = resume_dir - else: - # Clear progress; we're starting from the top. - progress_set(path, None) - - # Look for saved incremental directories. - if config['import']['incremental']: - incremental_skipped = 0 - history_dirs = history_get() - + skipped = 0 for toppath in session.paths: - # Check whether the path is to a file. - if config['import']['singletons'] and \ - not os.path.isdir(syspath(toppath)): - try: - item = library.Item.from_path(toppath) - except mediafile.UnreadableFileError: - log.warn(u'unreadable file: {0}'.format( - util.displayable_path(toppath) - )) - continue - yield ImportTask.item_task(item) - continue + # Check whether we need to resume the import. + session.ask_resume(toppath) - # A flat album import merges all items into one album. - if config['import']['flat'] and not config['import']['singletons']: - all_items = [] - for _, items in autotag.albums_in_dir(toppath): - all_items += items - yield ImportTask(toppath, toppath, all_items) - yield ImportTask.done_sentinel(toppath) - continue + # Generate tasks. + task_factory = ImportTaskFactory(toppath, session) + for t in task_factory.tasks(): + yield t + skipped += task_factory.skipped - # Produce paths under this directory. - if _resume(): - resume_dir = resume_dirs.get(toppath) - for path, items in autotag.albums_in_dir(toppath): - # Skip according to progress. - if _resume() and resume_dir: - # We're fast-forwarding to resume a previous tagging. - if path == resume_dir: - # We've hit the last good path! Turn off the - # fast-forwarding. - resume_dir = None - continue + if not task_factory.imported: + log.warn(u'No files imported from {0}', + displayable_path(toppath)) - # When incremental, skip paths in the history. - if config['import']['incremental'] and tuple(path) in history_dirs: - log.debug(u'Skipping previously-imported path: %s' % - displayable_path(path)) - incremental_skipped += 1 - continue + # Show skipped directories (due to incremental/resume). + if skipped: + log.info(u'Skipped {0} paths.', skipped) - # Yield all the necessary tasks. - if config['import']['singletons']: - for item in items: - yield ImportTask.item_task(item) - yield ImportTask.progress_sentinel(toppath, path) - else: - yield ImportTask(toppath, path, items) - - # Indicate the directory is finished. - yield ImportTask.done_sentinel(toppath) - - # Show skipped directories. - if config['import']['incremental'] and incremental_skipped: - log.info(u'Incremental import: skipped %i directories.' % - incremental_skipped) def query_tasks(session): """A generator that works as a drop-in-replacement for read_tasks. Instead of finding files from the filesystem, a query is used to match items from the library. """ - if config['import']['singletons']: + if session.config['singletons']: # Search for items. for item in session.lib.items(session.query): - yield ImportTask.item_task(item) + task = SingletonImportTask(None, item) + for task in task.handle_created(session): + yield task else: # Search for albums. for album in session.lib.albums(session.query): - log.debug('yielding album %i: %s - %s' % - (album.id, album.albumartist, album.album)) + log.debug(u'yielding album {0}: {1} - {2}', + album.id, album.albumartist, album.album) items = list(album.items()) - yield ImportTask(None, [album.item_dir()], items) -def initial_lookup(session): + # Clear IDs from re-tagged items so they appear "fresh" when + # we add them back to the library. + for item in items: + item.id = None + item.album_id = None + + task = ImportTask(None, [album.item_dir()], items) + for task in task.handle_created(session): + yield task + + +@pipeline.mutator_stage +def lookup_candidates(session, task): """A coroutine for performing the initial MusicBrainz lookup for an album. It accepts lists of Items and yields (items, cur_artist, cur_album, candidates, rec) tuples. If no match is found, all of the yielded parameters (except items) are None. """ - task = None - while True: - task = yield task - if task.should_skip(): - continue + if task.skip: + # FIXME This gets duplicated a lot. We need a better + # abstraction. + return - plugins.send('import_task_start', session=session, task=task) + plugins.send('import_task_start', session=session, task=task) + log.debug(u'Looking up: {0}', displayable_path(task.paths)) - log.debug('Looking up: %s' % displayable_path(task.paths)) - task.set_candidates( - *autotag.tag_album(task.items) - ) + # Restrict the initial lookup to IDs specified by the user via the -m + # option. Currently all the IDs are passed onto the tasks directly. + task.search_ids = session.config['search_ids'].as_str_seq() -def user_query(session): + task.lookup_candidates() + + +@pipeline.stage +def user_query(session, task): """A coroutine for interfacing with the user about the tagging process. The coroutine accepts an ImportTask objects. It uses the - session's ``choose_match`` method to determine the ``action`` for - this task. Depending on the action additional stages are exectuted + session's `choose_match` method to determine the `action` for + this task. Depending on the action additional stages are executed and the processed task is yielded. It emits the ``import_task_choice`` event for plugins. Plugins have acces to the choice via the ``taks.choice_flag`` property and may choose to change it. """ - recent = set() - task = None - while True: - task = yield task - if task.should_skip(): - continue + if task.skip: + return task - # Ask the user for a choice. - choice = session.choose_match(task) - task.set_choice(choice) - session.log_choice(task) - plugins.send('import_task_choice', session=session, task=task) + # Ask the user for a choice. + task.choose_match(session) + plugins.send('import_task_choice', session=session, task=task) - # As-tracks: transition to singleton workflow. - if task.choice_flag is action.TRACKS: - # Set up a little pipeline for dealing with the singletons. - def emitter(task): - for item in task.items: - yield ImportTask.item_task(item) - yield ImportTask.progress_sentinel(task.toppath, task.paths) + # As-tracks: transition to singleton workflow. + if task.choice_flag is action.TRACKS: + # Set up a little pipeline for dealing with the singletons. + def emitter(task): + for item in task.items: + task = SingletonImportTask(task.toppath, item) + for new_task in task.handle_created(session): + yield new_task + yield SentinelImportTask(task.toppath, task.paths) - ipl = pipeline.Pipeline([ - emitter(task), - item_lookup(session), - item_query(session), - ]) - task = pipeline.multiple(ipl.pull()) - continue + ipl = pipeline.Pipeline([ + emitter(task), + lookup_candidates(session), + user_query(session), + ]) + return pipeline.multiple(ipl.pull()) - # As albums: group items by albums and create task for each album - if task.choice_flag is action.ALBUMS: - def emitter(task): - yield task + # As albums: group items by albums and create task for each album + if task.choice_flag is action.ALBUMS: + ipl = pipeline.Pipeline([ + iter([task]), + group_albums(session), + lookup_candidates(session), + user_query(session) + ]) + return pipeline.multiple(ipl.pull()) - ipl = pipeline.Pipeline([ - emitter(task), - group_albums(session), - initial_lookup(session), - user_query(session) - ]) - task = pipeline.multiple(ipl.pull()) - continue + resolve_duplicates(session, task) + apply_choice(session, task) + return task - # Check for duplicates if we have a match (or ASIS). - if task.choice_flag in (action.ASIS, action.APPLY): - ident = task.chosen_ident() - # The "recent" set keeps track of identifiers for recently - # imported albums -- those that haven't reached the database - # yet. - if ident in recent or _duplicate_check(session.lib, task): - session.resolve_duplicate(task) - session.log_choice(task, True) - recent.add(ident) -def show_progress(session): - """This stage replaces the initial_lookup and user_query stages - when the importer is run without autotagging. It displays the album - name and artist as the files are added. +def resolve_duplicates(session, task): + """Check if a task conflicts with items or albums already imported + and ask the session to resolve this. """ - task = None - while True: - task = yield task - if task.should_skip(): - continue + if task.choice_flag in (action.ASIS, action.APPLY, action.RETAG): + found_duplicates = task.find_duplicates(session.lib) + if found_duplicates: + log.debug(u'found duplicates: {}'.format( + [o.id for o in found_duplicates] + )) + session.resolve_duplicate(task, found_duplicates) + session.log_choice(task, True) - log.info(displayable_path(task.paths)) - # Behave as if ASIS were selected. - task.set_null_candidates() - task.set_choice(action.ASIS) +@pipeline.mutator_stage +def import_asis(session, task): + """Select the `action.ASIS` choice for all tasks. -def apply_choices(session): - """A coroutine for applying changes to albums and singletons during - the autotag process. + This stage replaces the initial_lookup and user_query stages + when the importer is run without autotagging. """ - task = None - while True: - task = yield task - if task.should_skip(): - continue + if task.skip: + return - items = task.imported_items() - # Clear IDs in case the items are being re-tagged. - for item in items: - item.id = None - item.album_id = None + log.info(u'{}', displayable_path(task.paths)) + task.set_choice(action.ASIS) + apply_choice(session, task) - # Change metadata. - if task.should_write_tags(): - if task.is_album: - autotag.apply_metadata( - task.match.info, task.match.mapping - ) - else: - autotag.apply_item_metadata(task.item, task.match.info) - plugins.send('import_task_apply', session=session, task=task) - # Infer album-level fields. - if task.is_album: - _infer_album_fields(task) +def apply_choice(session, task): + """Apply the task's choice to the Album or Item it contains and add + it to the library. + """ + if task.skip: + return - # Find existing item entries that these are replacing (for - # re-imports). Old album structures are automatically cleaned up - # when the last item is removed. - task.replaced_items = defaultdict(list) - for item in items: - dup_items = session.lib.items( - dbcore.query.BytesQuery('path', item.path) - ) - for dup_item in dup_items: - task.replaced_items[item].append(dup_item) - log.debug('replacing item %i: %s' % - (dup_item.id, displayable_path(item.path))) - log.debug('%i of %i items replaced' % (len(task.replaced_items), - len(items))) + # Change metadata. + if task.apply: + task.apply_metadata() + plugins.send('import_task_apply', session=session, task=task) - # Find old items that should be replaced as part of a duplicate - # resolution. - duplicate_items = [] - if task.remove_duplicates: - if task.is_album: - for album in _duplicate_check(session.lib, task): - duplicate_items += album.items() - else: - duplicate_items = _item_duplicate_check(session.lib, task) - log.debug('removing %i old duplicated items' % - len(duplicate_items)) + task.add(session.lib) - # Delete duplicate files that are located inside the library - # directory. - task.duplicate_paths = [] - for duplicate_path in [i.path for i in duplicate_items]: - if session.lib.directory in util.ancestry(duplicate_path): - # Mark the path for deletion in the manipulate_files - # stage. - task.duplicate_paths.append(duplicate_path) - # Add items -- before path changes -- to the library. We add the - # items now (rather than at the end) so that album structures - # are in place before calls to destination(). - with session.lib.transaction(): - # Remove old items. - for replaced in task.replaced_items.itervalues(): - for item in replaced: - item.remove() - for item in duplicate_items: - item.remove() - - # Add new ones. - if task.is_album: - # Add an album. - album = session.lib.add_album(items) - task.album_id = album.id - else: - # Add tracks. - for item in items: - session.lib.add(item) - -def plugin_stage(session, func): +@pipeline.mutator_stage +def plugin_stage(session, func, task): """A coroutine (pipeline stage) that calls the given function with each non-skipped import task. These stages occur between applying metadata changes and moving/copying/writing files. """ - task = None - while True: - task = yield task - if task.should_skip(): - continue - func(session, task) + if task.skip: + return - # Stage may modify DB, so re-load cached item data. - for item in task.imported_items(): - item.load() + func(session, task) -def manipulate_files(session): + # Stage may modify DB, so re-load cached item data. + # FIXME Importer plugins should not modify the database but instead + # the albums and items attached to tasks. + task.reload() + + +@pipeline.stage +def manipulate_files(session, task): """A coroutine (pipeline stage) that performs necessary file - manipulations *after* items have been added to the library. + manipulations *after* items have been added to the library and + finalizes each task. """ - task = None - while True: - task = yield task - if task.should_skip(): - continue + if not task.skip: + if task.should_remove_duplicates: + task.remove_duplicates(session.lib) - # Remove duplicate files marked for deletion. - if task.remove_duplicates: - for duplicate_path in task.duplicate_paths: - log.debug(u'deleting replaced duplicate %s' % - util.displayable_path(duplicate_path)) - util.remove(duplicate_path) - util.prune_dirs(os.path.dirname(duplicate_path), - session.lib.directory) + task.manipulate_files( + move=session.config['move'], + copy=session.config['copy'], + write=session.config['write'], + link=session.config['link'], + session=session, + ) - # Move/copy/write files. - items = task.imported_items() - # Save the original paths of all items for deletion and pruning - # in the next step (finalization). - task.old_paths = [item.path for item in items] - for item in items: - if config['import']['move']: - # Just move the file. - item.move(False) - elif config['import']['copy']: - # If it's a reimport, move in-library files and copy - # out-of-library files. Otherwise, copy and keep track - # of the old path. - old_path = item.path - if task.replaced_items[item]: - # This is a reimport. Move in-library files and copy - # out-of-library files. - if session.lib.directory in util.ancestry(old_path): - item.move(False) - # We moved the item, so remove the - # now-nonexistent file from old_paths. - task.old_paths.remove(old_path) - else: - item.move(True) - else: - # A normal import. Just copy files and keep track of - # old paths. - item.move(True) + # Progress, cleanup, and event. + task.finalize(session) - if config['import']['write'] and task.should_write_tags(): - try: - item.write() - except library.FileOperationError as exc: - log.error(exc) - # Save new paths. - with session.lib.transaction(): - for item in items: - item.store() - - # Plugin event. - plugins.send('import_task_files', session=session, task=task) - -def finalize(session): - """A coroutine that finishes up importer tasks. In particular, the - coroutine sends plugin events, deletes old files, and saves - progress. This is a "terminal" coroutine (it yields None). +@pipeline.stage +def log_files(session, task): + """A coroutine (pipeline stage) to log each file to be imported. """ - while True: - task = yield - if task.should_skip(): - if _resume(): - task.save_progress() - if config['import']['incremental']: - task.save_history() - continue - - items = task.imported_items() - - # Announce that we've added an album. - if task.is_album: - album = session.lib.get_album(task.album_id) - plugins.send('album_imported', - lib=session.lib, album=album) - else: - for item in items: - plugins.send('item_imported', - lib=session.lib, item=item) - - # When copying and deleting originals, delete old files. - if config['import']['copy'] and config['import']['delete']: - new_paths = [os.path.realpath(item.path) for item in items] - for old_path in task.old_paths: - # Only delete files that were actually copied. - if old_path not in new_paths: - util.remove(syspath(old_path), False) - task.prune(old_path) - - # When moving, prune empty directories containing the original - # files. - elif config['import']['move']: - for old_path in task.old_paths: - task.prune(old_path) - - # Update progress. - if _resume(): - task.save_progress() - if config['import']['incremental']: - task.save_history() - - -# Singleton pipeline stages. - -def item_lookup(session): - """A coroutine used to perform the initial MusicBrainz lookup for - an item task. - """ - task = None - while True: - task = yield task - if task.should_skip(): - continue - - plugins.send('import_task_start', session=session, task=task) - - task.set_item_candidates(*autotag.tag_item(task.item)) - -def item_query(session): - """A coroutine that queries the user for input on single-item - lookups. - """ - task = None - recent = set() - while True: - task = yield task - if task.should_skip(): - continue - - choice = session.choose_item(task) - task.set_choice(choice) - session.log_choice(task) - plugins.send('import_task_choice', session=session, task=task) - - # Duplicate check. - if task.choice_flag in (action.ASIS, action.APPLY): - ident = task.chosen_ident() - if ident in recent or _item_duplicate_check(session.lib, task): - session.resolve_duplicate(task) - session.log_choice(task, True) - recent.add(ident) - -def item_progress(session): - """Skips the lookup and query stages in a non-autotagged singleton - import. Just shows progress. - """ - task = None - log.info('Importing items:') - while True: - task = yield task - if task.should_skip(): - continue - - log.info(displayable_path(task.item.path)) - task.set_null_candidates() - task.set_choice(action.ASIS) + if isinstance(task, SingletonImportTask): + log.info(u'Singleton: {0}', displayable_path(task.item['path'])) + elif task.items: + log.info(u'Album: {0}', displayable_path(task.paths[0])) + for item in task.items: + log.info(u' {0}', displayable_path(item['path'])) def group_albums(session): - """Group the items of a task by albumartist and album name and create a new - task for each album. Yield the tasks as a multi message. + """A pipeline stage that groups the items of each task into albums + using their metadata. + + Groups are identified using their artist and album fields. The + pipeline stage emits new album tasks for each discovered group. """ def group(item): return (item.albumartist or item.artist, item.album) @@ -1020,11 +1425,118 @@ def group_albums(session): task = None while True: task = yield task - if task.should_skip(): + if task.skip: continue tasks = [] - for _, items in itertools.groupby(task.items, group): - tasks.append(ImportTask(items=list(items))) - tasks.append(ImportTask.progress_sentinel(task.toppath, task.paths)) + sorted_items = sorted(task.items, key=group) + for _, items in itertools.groupby(sorted_items, group): + items = list(items) + task = ImportTask(task.toppath, [i.path for i in items], + items) + tasks += task.handle_created(session) + tasks.append(SentinelImportTask(task.toppath, task.paths)) task = pipeline.multiple(tasks) + + +MULTIDISC_MARKERS = (r'dis[ck]', r'cd') +MULTIDISC_PAT_FMT = r'^(.*%s[\W_]*)\d' + + +def albums_in_dir(path): + """Recursively searches the given directory and returns an iterable + of (paths, items) where paths is a list of directories and items is + a list of Items that is probably an album. Specifically, any folder + containing any media files is an album. + """ + collapse_pat = collapse_paths = collapse_items = None + ignore = config['ignore'].as_str_seq() + ignore_hidden = config['ignore_hidden'].get(bool) + + for root, dirs, files in sorted_walk(path, ignore=ignore, + ignore_hidden=ignore_hidden, + logger=log): + items = [os.path.join(root, f) for f in files] + # If we're currently collapsing the constituent directories in a + # multi-disc album, check whether we should continue collapsing + # and add the current directory. If so, just add the directory + # and move on to the next directory. If not, stop collapsing. + if collapse_paths: + if (not collapse_pat and collapse_paths[0] in ancestry(root)) or \ + (collapse_pat and + collapse_pat.match(os.path.basename(root))): + # Still collapsing. + collapse_paths.append(root) + collapse_items += items + continue + else: + # Collapse finished. Yield the collapsed directory and + # proceed to process the current one. + if collapse_items: + yield collapse_paths, collapse_items + collapse_pat = collapse_paths = collapse_items = None + + # Check whether this directory looks like the *first* directory + # in a multi-disc sequence. There are two indicators: the file + # is named like part of a multi-disc sequence (e.g., "Title Disc + # 1") or it contains no items but only directories that are + # named in this way. + start_collapsing = False + for marker in MULTIDISC_MARKERS: + marker_pat = re.compile(MULTIDISC_PAT_FMT % marker, re.I) + match = marker_pat.match(os.path.basename(root)) + + # Is this directory the root of a nested multi-disc album? + if dirs and not items: + # Check whether all subdirectories have the same prefix. + start_collapsing = True + subdir_pat = None + for subdir in dirs: + # The first directory dictates the pattern for + # the remaining directories. + if not subdir_pat: + match = marker_pat.match(subdir) + if match: + subdir_pat = re.compile( + br'^%s\d' % re.escape(match.group(1)), re.I + ) + else: + start_collapsing = False + break + + # Subsequent directories must match the pattern. + elif not subdir_pat.match(subdir): + start_collapsing = False + break + + # If all subdirectories match, don't check other + # markers. + if start_collapsing: + break + + # Is this directory the first in a flattened multi-disc album? + elif match: + start_collapsing = True + # Set the current pattern to match directories with the same + # prefix as this one, followed by a digit. + collapse_pat = re.compile( + br'^%s\d' % re.escape(match.group(1)), re.I + ) + break + + # If either of the above heuristics indicated that this is the + # beginning of a multi-disc album, initialize the collapsed + # directory and item lists and check the next directory. + if start_collapsing: + # Start collapsing; continue to the next iteration. + collapse_paths = [root] + collapse_items = items + continue + + # If it's nonempty, yield it. + if items: + yield [root], items + + # Clear out any unfinished collapse. + if collapse_paths and collapse_items: + yield collapse_paths, collapse_items diff --git a/libs/beets/library.py b/libs/beets/library.py index 94559430..99397013 100644 --- a/libs/beets/library.py +++ b/libs/beets/library.py @@ -1,5 +1,6 @@ +# -*- coding: utf-8 -*- # This file is part of beets. -# Copyright 2013, Adrian Sampson. +# Copyright 2016, Adrian Sampson. # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -14,15 +15,17 @@ """The core data store and collection logic for beets. """ +from __future__ import division, absolute_import, print_function + import os -import re import sys -import logging -import shlex import unicodedata import time +import re from unidecode import unidecode -from beets.mediafile import MediaFile, MutagenError + +from beets import logging +from beets.mediafile import MediaFile, MutagenError, UnreadableFileError from beets import plugins from beets import util from beets.util import bytestring_path, syspath, normpath, samefile @@ -32,52 +35,85 @@ from beets.dbcore import types import beets +log = logging.getLogger('beets') + # Library-specific query types. - class PathQuery(dbcore.FieldQuery): - """A query that matches all items under a given path.""" - def __init__(self, field, pattern, fast=True): + """A query that matches all items under a given path. + + Matching can either be case-insensitive or case-sensitive. By + default, the behavior depends on the OS: case-insensitive on Windows + and case-sensitive otherwise. + """ + + escape_re = re.compile(r'[\\_%]') + escape_char = b'\\' + + def __init__(self, field, pattern, fast=True, case_sensitive=None): + """Create a path query. `pattern` must be a path, either to a + file or a directory. + + `case_sensitive` can be a bool or `None`, indicating that the + behavior should depend on the filesystem. + """ super(PathQuery, self).__init__(field, pattern, fast) + # By default, the case sensitivity depends on the filesystem + # that the query path is located on. + if case_sensitive is None: + path = util.bytestring_path(util.normpath(pattern)) + case_sensitive = beets.util.case_sensitive(path) + self.case_sensitive = case_sensitive + + # Use a normalized-case pattern for case-insensitive matches. + if not case_sensitive: + pattern = pattern.lower() + # Match the path as a single file. self.file_path = util.bytestring_path(util.normpath(pattern)) # As a directory (prefix). - self.dir_path = util.bytestring_path(os.path.join(self.file_path, '')) + self.dir_path = util.bytestring_path(os.path.join(self.file_path, b'')) + + @classmethod + def is_path_query(cls, query_part): + """Try to guess whether a unicode query part is a path query. + + Condition: separator precedes colon and the file exists. + """ + colon = query_part.find(':') + if colon != -1: + query_part = query_part[:colon] + return (os.sep in query_part and + os.path.exists(syspath(normpath(query_part)))) def match(self, item): - return (item.path == self.file_path) or \ - item.path.startswith(self.dir_path) + path = item.path if self.case_sensitive else item.path.lower() + return (path == self.file_path) or path.startswith(self.dir_path) - def clause(self): - dir_pat = buffer(self.dir_path + '%') - file_blob = buffer(self.file_path) - return '({0} = ?) || ({0} LIKE ?)'.format(self.field), \ - (file_blob, dir_pat) - - -class SingletonQuery(dbcore.Query): - """Matches either singleton or non-singleton items.""" - def __init__(self, sense): - self.sense = sense - - def clause(self): - if self.sense: - return "album_id ISNULL", () - else: - return "NOT album_id ISNULL", () - - def match(self, item): - return (not item.album_id) == self.sense + def col_clause(self): + if self.case_sensitive: + file_blob = buffer(self.file_path) + dir_blob = buffer(self.dir_path) + return '({0} = ?) || (substr({0}, 1, ?) = ?)'.format(self.field), \ + (file_blob, len(dir_blob), dir_blob) + escape = lambda m: self.escape_char + m.group(0) + dir_pattern = self.escape_re.sub(escape, self.dir_path) + dir_blob = buffer(dir_pattern + b'%') + file_pattern = self.escape_re.sub(escape, self.file_path) + file_blob = buffer(file_pattern) + return '({0} LIKE ? ESCAPE ?) || ({0} LIKE ? ESCAPE ?)'.format( + self.field), (file_blob, self.escape_char, dir_blob, + self.escape_char) # Library-specific field types. - -class DateType(types.Type): - sql = u'REAL' +class DateType(types.Float): + # TODO representation should be `datetime` object + # TODO distinguish between date and time types query = dbcore.query.DateQuery def format(self, value): @@ -95,12 +131,13 @@ class DateType(types.Type): try: return float(string) except ValueError: - return 0.0 + return self.null class PathType(types.Type): sql = u'BLOB' query = PathQuery + model_type = bytes def format(self, value): return util.displayable_path(value) @@ -108,158 +145,116 @@ class PathType(types.Type): def parse(self, string): return normpath(bytestring_path(string)) + def normalize(self, value): + if isinstance(value, unicode): + # Paths stored internally as encoded bytes. + return bytestring_path(value) + + elif isinstance(value, buffer): + # SQLite must store bytestings as buffers to avoid decoding. + # We unwrap buffers to bytes. + return bytes(value) + + else: + return value + + def from_sql(self, sql_value): + return self.normalize(sql_value) + + def to_sql(self, value): + if isinstance(value, bytes): + value = buffer(value) + return value -# Model field lists. +class MusicalKey(types.String): + """String representing the musical key of a song. + + The standard format is C, Cm, C#, C#m, etc. + """ + ENHARMONIC = { + r'db': 'c#', + r'eb': 'd#', + r'gb': 'f#', + r'ab': 'g#', + r'bb': 'a#', + } + + def parse(self, key): + key = key.lower() + for flat, sharp in self.ENHARMONIC.items(): + key = re.sub(flat, sharp, key) + key = re.sub(r'[\W\s]+minor', 'm', key) + key = re.sub(r'[\W\s]+major', '', key) + return key.capitalize() + + def normalize(self, key): + if key is None: + return None + else: + return self.parse(key) -# Fields in the "items" database table; all the metadata available for -# items in the library. These are used directly in SQL; they are -# vulnerable to injection if accessible to the user. -# Each tuple has the following values: -# - The name of the field. -# - The (Python) type of the field. -# - Is the field writable? -# - Does the field reflect an attribute of a MediaFile? -ITEM_FIELDS = [ - ('id', types.Id(), False, False), - ('path', PathType(), False, False), - ('album_id', types.Integer(), False, False), +class DurationType(types.Float): + """Human-friendly (M:SS) representation of a time interval.""" + query = dbcore.query.DurationQuery - ('title', types.String(), True, True), - ('artist', types.String(), True, True), - ('artist_sort', types.String(), True, True), - ('artist_credit', types.String(), True, True), - ('album', types.String(), True, True), - ('albumartist', types.String(), True, True), - ('albumartist_sort', types.String(), True, True), - ('albumartist_credit', types.String(), True, True), - ('genre', types.String(), True, True), - ('composer', types.String(), True, True), - ('grouping', types.String(), True, True), - ('year', types.PaddedInt(4), True, True), - ('month', types.PaddedInt(2), True, True), - ('day', types.PaddedInt(2), True, True), - ('track', types.PaddedInt(2), True, True), - ('tracktotal', types.PaddedInt(2), True, True), - ('disc', types.PaddedInt(2), True, True), - ('disctotal', types.PaddedInt(2), True, True), - ('lyrics', types.String(), True, True), - ('comments', types.String(), True, True), - ('bpm', types.Integer(), True, True), - ('comp', types.Boolean(), True, True), - ('mb_trackid', types.String(), True, True), - ('mb_albumid', types.String(), True, True), - ('mb_artistid', types.String(), True, True), - ('mb_albumartistid', types.String(), True, True), - ('albumtype', types.String(), True, True), - ('label', types.String(), True, True), - ('acoustid_fingerprint', types.String(), True, True), - ('acoustid_id', types.String(), True, True), - ('mb_releasegroupid', types.String(), True, True), - ('asin', types.String(), True, True), - ('catalognum', types.String(), True, True), - ('script', types.String(), True, True), - ('language', types.String(), True, True), - ('country', types.String(), True, True), - ('albumstatus', types.String(), True, True), - ('media', types.String(), True, True), - ('albumdisambig', types.String(), True, True), - ('disctitle', types.String(), True, True), - ('encoder', types.String(), True, True), - ('rg_track_gain', types.Float(), True, True), - ('rg_track_peak', types.Float(), True, True), - ('rg_album_gain', types.Float(), True, True), - ('rg_album_peak', types.Float(), True, True), - ('original_year', types.PaddedInt(4), True, True), - ('original_month', types.PaddedInt(2), True, True), - ('original_day', types.PaddedInt(2), True, True), + def format(self, value): + if not beets.config['format_raw_length'].get(bool): + return beets.ui.human_seconds_short(value or 0.0) + else: + return value - ('length', types.Float(), False, True), - ('bitrate', types.ScaledInt(1000, u'kbps'), False, True), - ('format', types.String(), False, True), - ('samplerate', types.ScaledInt(1000, u'kHz'), False, True), - ('bitdepth', types.Integer(), False, True), - ('channels', types.Integer(), False, True), - ('mtime', DateType(), False, False), - ('added', DateType(), False, False), -] -ITEM_KEYS_WRITABLE = [f[0] for f in ITEM_FIELDS if f[3] and f[2]] -ITEM_KEYS_META = [f[0] for f in ITEM_FIELDS if f[3]] -ITEM_KEYS = [f[0] for f in ITEM_FIELDS] - -# Database fields for the "albums" table. -# The third entry in each tuple indicates whether the field reflects an -# identically-named field in the items table. -ALBUM_FIELDS = [ - ('id', types.Id(), False), - ('artpath', PathType(), False), - ('added', DateType(), True), - - ('albumartist', types.String(), True), - ('albumartist_sort', types.String(), True), - ('albumartist_credit', types.String(), True), - ('album', types.String(), True), - ('genre', types.String(), True), - ('year', types.PaddedInt(4), True), - ('month', types.PaddedInt(2), True), - ('day', types.PaddedInt(2), True), - ('tracktotal', types.PaddedInt(2), True), - ('disctotal', types.PaddedInt(2), True), - ('comp', types.Boolean(), True), - ('mb_albumid', types.String(), True), - ('mb_albumartistid', types.String(), True), - ('albumtype', types.String(), True), - ('label', types.String(), True), - ('mb_releasegroupid', types.String(), True), - ('asin', types.String(), True), - ('catalognum', types.String(), True), - ('script', types.String(), True), - ('language', types.String(), True), - ('country', types.String(), True), - ('albumstatus', types.String(), True), - ('media', types.String(), True), - ('albumdisambig', types.String(), True), - ('rg_album_gain', types.Float(), True), - ('rg_album_peak', types.Float(), True), - ('original_year', types.PaddedInt(4), True), - ('original_month', types.PaddedInt(2), True), - ('original_day', types.PaddedInt(2), True), -] -ALBUM_KEYS = [f[0] for f in ALBUM_FIELDS] -ALBUM_KEYS_ITEM = [f[0] for f in ALBUM_FIELDS if f[2]] + def parse(self, string): + try: + # Try to format back hh:ss to seconds. + return util.raw_seconds_short(string) + except ValueError: + # Fall back to a plain float. + try: + return float(string) + except ValueError: + return self.null -# Default search fields for each model. -ALBUM_DEFAULT_FIELDS = ('album', 'albumartist', 'genre') -ITEM_DEFAULT_FIELDS = ALBUM_DEFAULT_FIELDS + ('artist', 'title', 'comments') +# Library-specific sort types. + +class SmartArtistSort(dbcore.query.Sort): + """Sort by artist (either album artist or track artist), + prioritizing the sort field over the raw field. + """ + def __init__(self, model_cls, ascending=True, case_insensitive=True): + self.album = model_cls is Album + self.ascending = ascending + self.case_insensitive = case_insensitive + + def order_clause(self): + order = "ASC" if self.ascending else "DESC" + field = 'albumartist' if self.album else 'artist' + collate = 'COLLATE NOCASE' if self.case_insensitive else '' + return ('(CASE {0}_sort WHEN NULL THEN {0} ' + 'WHEN "" THEN {0} ' + 'ELSE {0}_sort END) {1} {2}').format(field, collate, order) + + def sort(self, objs): + if self.album: + field = lambda a: a.albumartist_sort or a.albumartist + else: + field = lambda i: i.artist_sort or i.artist + + if self.case_insensitive: + key = lambda x: field(x).lower() + else: + key = field + return sorted(objs, key=key, reverse=not self.ascending) # Special path format key. PF_KEY_DEFAULT = 'default' -# Logger. -log = logging.getLogger('beets') -if not log.handlers: - log.addHandler(logging.StreamHandler()) -log.propagate = False # Don't propagate to root handler. - - -# A little SQL utility. -def _orelse(exp1, exp2): - """Generates an SQLite expression that evaluates to exp1 if exp1 is - non-null and non-empty or exp2 otherwise. - """ - return ('(CASE {0} WHEN NULL THEN {1} ' - 'WHEN "" THEN {1} ' - 'ELSE {0} END)').format(exp1, exp2) - - - # Exceptions. - class FileOperationError(Exception): """Indicates an error when interacting with a file on disk. Possibilities include an unsupported media type, a permissions @@ -300,14 +295,15 @@ class WriteError(FileOperationError): return u'error writing ' + super(WriteError, self).__unicode__() - # Item and Album model classes. - class LibModel(dbcore.Model): """Shared concrete functionality for Items and Albums. """ - _bytes_keys = ('path', 'artpath') + + _format_config_key = None + """Config key that specifies how an instance should be formatted. + """ def _template_funcs(self): funcs = DefaultTemplateFunctions(self, self._db).functions() @@ -316,26 +312,188 @@ class LibModel(dbcore.Model): def store(self): super(LibModel, self).store() - plugins.send('database_change', lib=self._db) + plugins.send('database_change', lib=self._db, model=self) def remove(self): super(LibModel, self).remove() - plugins.send('database_change', lib=self._db) + plugins.send('database_change', lib=self._db, model=self) def add(self, lib=None): super(LibModel, self).add(lib) - plugins.send('database_change', lib=self._db) + plugins.send('database_change', lib=self._db, model=self) + + def __format__(self, spec): + if not spec: + spec = beets.config[self._format_config_key].get(unicode) + result = self.evaluate_template(spec) + if isinstance(spec, bytes): + # if spec is a byte string then we must return a one as well + return result.encode('utf8') + else: + return result + + def __str__(self): + return format(self).encode('utf8') + + def __unicode__(self): + return format(self) + + +class FormattedItemMapping(dbcore.db.FormattedMapping): + """Add lookup for album-level fields. + + Album-level fields take precedence if `for_path` is true. + """ + + def __init__(self, item, for_path=False): + super(FormattedItemMapping, self).__init__(item, for_path) + self.album = item.get_album() + self.album_keys = [] + if self.album: + for key in self.album.keys(True): + if key in Album.item_keys or key not in item._fields.keys(): + self.album_keys.append(key) + self.all_keys = set(self.model_keys).union(self.album_keys) + + def _get(self, key): + """Get the value for a key, either from the album or the item. + Raise a KeyError for invalid keys. + """ + if self.for_path and key in self.album_keys: + return self._get_formatted(self.album, key) + elif key in self.model_keys: + return self._get_formatted(self.model, key) + elif key in self.album_keys: + return self._get_formatted(self.album, key) + else: + raise KeyError(key) + + def __getitem__(self, key): + """Get the value for a key. Certain unset values are remapped. + """ + value = self._get(key) + + # `artist` and `albumartist` fields fall back to one another. + # This is helpful in path formats when the album artist is unset + # on as-is imports. + if key == 'artist' and not value: + return self._get('albumartist') + elif key == 'albumartist' and not value: + return self._get('artist') + else: + return value + + def __iter__(self): + return iter(self.all_keys) + + def __len__(self): + return len(self.all_keys) class Item(LibModel): - _fields = dict((name, typ) for (name, typ, _, _) in ITEM_FIELDS) _table = 'items' _flex_table = 'item_attributes' - _search_fields = ITEM_DEFAULT_FIELDS + _fields = { + 'id': types.PRIMARY_ID, + 'path': PathType(), + 'album_id': types.FOREIGN_ID, + + 'title': types.STRING, + 'artist': types.STRING, + 'artist_sort': types.STRING, + 'artist_credit': types.STRING, + 'album': types.STRING, + 'albumartist': types.STRING, + 'albumartist_sort': types.STRING, + 'albumartist_credit': types.STRING, + 'genre': types.STRING, + 'composer': types.STRING, + 'grouping': types.STRING, + 'year': types.PaddedInt(4), + 'month': types.PaddedInt(2), + 'day': types.PaddedInt(2), + 'track': types.PaddedInt(2), + 'tracktotal': types.PaddedInt(2), + 'disc': types.PaddedInt(2), + 'disctotal': types.PaddedInt(2), + 'lyrics': types.STRING, + 'comments': types.STRING, + 'bpm': types.INTEGER, + 'comp': types.BOOLEAN, + 'mb_trackid': types.STRING, + 'mb_albumid': types.STRING, + 'mb_artistid': types.STRING, + 'mb_albumartistid': types.STRING, + 'albumtype': types.STRING, + 'label': types.STRING, + 'acoustid_fingerprint': types.STRING, + 'acoustid_id': types.STRING, + 'mb_releasegroupid': types.STRING, + 'asin': types.STRING, + 'catalognum': types.STRING, + 'script': types.STRING, + 'language': types.STRING, + 'country': types.STRING, + 'albumstatus': types.STRING, + 'media': types.STRING, + 'albumdisambig': types.STRING, + 'disctitle': types.STRING, + 'encoder': types.STRING, + 'rg_track_gain': types.NULL_FLOAT, + 'rg_track_peak': types.NULL_FLOAT, + 'rg_album_gain': types.NULL_FLOAT, + 'rg_album_peak': types.NULL_FLOAT, + 'original_year': types.PaddedInt(4), + 'original_month': types.PaddedInt(2), + 'original_day': types.PaddedInt(2), + 'initial_key': MusicalKey(), + + 'length': DurationType(), + 'bitrate': types.ScaledInt(1000, u'kbps'), + 'format': types.STRING, + 'samplerate': types.ScaledInt(1000, u'kHz'), + 'bitdepth': types.INTEGER, + 'channels': types.INTEGER, + 'mtime': DateType(), + 'added': DateType(), + } + + _search_fields = ('artist', 'title', 'comments', + 'album', 'albumartist', 'genre') + + _types = { + 'data_source': types.STRING, + } + + _media_fields = set(MediaFile.readable_fields()) \ + .intersection(_fields.keys()) + """Set of item fields that are backed by `MediaFile` fields. + + Any kind of field (fixed, flexible, and computed) may be a media + field. Only these fields are read from disk in `read` and written in + `write`. + """ + + _media_tag_fields = set(MediaFile.fields()).intersection(_fields.keys()) + """Set of item fields that are backed by *writable* `MediaFile` tag + fields. + + This excludes fields that represent audio data, such as `bitrate` or + `length`. + """ + + _formatter = FormattedItemMapping + + _sorts = {'artist': SmartArtistSort} + + _format_config_key = 'format_item' @classmethod def _getters(cls): - return plugins.item_field_getters() + getters = plugins.item_field_getters() + getters['singleton'] = lambda i: i.album_id is None + getters['filesize'] = Item.try_filesize # In bytes. + return getters @classmethod def from_path(cls, path): @@ -355,15 +513,15 @@ class Item(LibModel): if isinstance(value, unicode): value = bytestring_path(value) elif isinstance(value, buffer): - value = str(value) + value = bytes(value) - if key in ITEM_KEYS_WRITABLE: + if key in MediaFile.fields(): self.mtime = 0 # Reset mtime on dirty. super(Item, self).__setitem__(key, value) def update(self, values): - """Sett all key/value pairs in the mapping. If mtime is + """Set all key/value pairs in the mapping. If mtime is specified, it is not reset (as it might otherwise be). """ super(Item, self).update(values) @@ -379,12 +537,14 @@ class Item(LibModel): return None return self._db.get_album(self) - # Interaction with file metadata. def read(self, read_path=None): - """Read the metadata from the associated file. If read_path is - specified, read metadata from that file instead. + """Read the metadata from the associated file. + + If `read_path` is specified, read metadata from that file + instead. Updates all the properties in `_media_fields` + from the media file. Raises a `ReadError` if the file could not be read. """ @@ -393,20 +553,16 @@ class Item(LibModel): else: read_path = normpath(read_path) try: - f = MediaFile(syspath(read_path)) - except (OSError, IOError) as exc: + mediafile = MediaFile(syspath(read_path)) + except (OSError, IOError, UnreadableFileError) as exc: raise ReadError(read_path, exc) - for key in ITEM_KEYS_META: - value = getattr(f, key) + for key in self._media_fields: + value = getattr(mediafile, key) if isinstance(value, (int, long)): - # Filter values wider than 64 bits (in signed - # representation). SQLite cannot store them. - # py26: Post transition, we can use: - # value.bit_length() > 63 - if abs(value) >= 2 ** 63: + if value.bit_length() > 63: value = 0 - setattr(self, key, value) + self[key] = value # Database's mtime should now reflect the on-disk value. if read_path == self.path: @@ -414,33 +570,90 @@ class Item(LibModel): self.path = read_path - def write(self): - """Write the item's metadata to the associated file. + def write(self, path=None, tags=None): + """Write the item's metadata to a media file. + + All fields in `_media_fields` are written to disk according to + the values on this object. + + `path` is the path of the mediafile to write the data to. It + defaults to the item's path. + + `tags` is a dictionary of additional metadata the should be + written to the file. (These tags need not be in `_media_fields`.) Can raise either a `ReadError` or a `WriteError`. """ + if path is None: + path = self.path + else: + path = normpath(path) + + # Get the data to write to the file. + item_tags = dict(self) + item_tags = {k: v for k, v in item_tags.items() + if k in self._media_fields} # Only write media fields. + if tags is not None: + item_tags.update(tags) + plugins.send('write', item=self, path=path, tags=item_tags) + + # Open the file. try: - f = MediaFile(syspath(self.path)) - except (OSError, IOError) as exc: + mediafile = MediaFile(syspath(path), + id3v23=beets.config['id3v23'].get(bool)) + except (OSError, IOError, UnreadableFileError) as exc: raise ReadError(self.path, exc) - plugins.send('write', item=self) - - for key in ITEM_KEYS_WRITABLE: - setattr(f, key, self[key]) + # Write the tags to the file. + mediafile.update(item_tags) try: - f.save(id3v23=beets.config['id3v23'].get(bool)) + mediafile.save() except (OSError, IOError, MutagenError) as exc: raise WriteError(self.path, exc) # The file has a new mtime. - self.mtime = self.current_mtime() - plugins.send('after_write', item=self) + if path == self.path: + self.mtime = self.current_mtime() + plugins.send('after_write', item=self, path=path) + def try_write(self, path=None, tags=None): + """Calls `write()` but catches and logs `FileOperationError` + exceptions. + + Returns `False` an exception was caught and `True` otherwise. + """ + try: + self.write(path, tags) + return True + except FileOperationError as exc: + log.error(u"{0}", exc) + return False + + def try_sync(self, write, move, with_album=True): + """Synchronize the item with the database and, possibly, updates its + tags on disk and its path (by moving the file). + + `write` indicates whether to write new tags into the file. Similarly, + `move` controls whether the path should be updated. In the + latter case, files are *only* moved when they are inside their + library's directory (if any). + + Similar to calling :meth:`write`, :meth:`move`, and :meth:`store` + (conditionally). + """ + if write: + self.try_write() + if move: + # Check whether this file is inside the library directory. + if self._db and self._db.directory in util.ancestry(self.path): + log.debug(u'moving {0} to synchronize path', + util.displayable_path(self.path)) + self.move(with_album=with_album) + self.store() # Files themselves. - def move_file(self, dest, copy=False): + def move_file(self, dest, copy=False, link=False): """Moves or copies the item's file, updating the path value if the move succeeds. If a file exists at ``dest``, then it is slightly modified to be unique. @@ -451,7 +664,13 @@ class Item(LibModel): util.copy(self.path, dest) plugins.send("item_copied", item=self, source=self.path, destination=dest) + elif link: + util.link(self.path, dest) + plugins.send("item_linked", item=self, source=self.path, + destination=dest) else: + plugins.send("before_item_moved", item=self, source=self.path, + destination=dest) util.move(self.path, dest) plugins.send("item_moved", item=self, source=self.path, destination=dest) @@ -465,6 +684,16 @@ class Item(LibModel): """ return int(os.path.getmtime(syspath(self.path))) + def try_filesize(self): + """Get the size of the underlying file in bytes. + + If the file is missing, return 0 (and log a warning). + """ + try: + return os.path.getsize(syspath(self.path)) + except (OSError, Exception) as exc: + log.warning(u'could not get filesize: {0}', exc) + return 0 # Model methods. @@ -491,13 +720,14 @@ class Item(LibModel): self._db._memotable = {} - def move(self, copy=False, basedir=None, with_album=True): + def move(self, copy=False, link=False, basedir=None, with_album=True): """Move the item to its designated location within the library directory (provided by destination()). Subdirectories are created as needed. If the operation succeeds, the item's path field is updated to reflect the new location. - If copy is True, moving the file is copied rather than moved. + If `copy` is true, moving the file is copied rather than moved. + Similarly, `link` creates a symlink instead. basedir overrides the library base directory for the destination. @@ -519,7 +749,7 @@ class Item(LibModel): # Perform the move and store the change. old_path = self.path - self.move_file(dest, copy) + self.move_file(dest, copy, link) self.store() # If this item is in an album, move its art. @@ -533,31 +763,8 @@ class Item(LibModel): if not copy: util.prune_dirs(os.path.dirname(old_path), self._db.directory) - # Templating. - def _formatted_mapping(self, for_path=False): - """Get a mapping containing string-formatted values from either - this item or the associated album, if any. - """ - mapping = super(Item, self)._formatted_mapping(for_path) - - # Merge in album-level fields. - album = self.get_album() - if album: - for key in album.keys(True): - if key in ALBUM_KEYS_ITEM or key not in ITEM_KEYS: - mapping[key] = album._get_formatted(key, for_path) - - # Use the album artist if the track artist is not set and - # vice-versa. - if not mapping['artist']: - mapping['artist'] = mapping['albumartist'] - if not mapping['albumartist']: - mapping['albumartist'] = mapping['artist'] - - return mapping - def destination(self, fragment=False, basedir=None, platform=None, path_formats=None): """Returns the path in the library directory designated for the @@ -577,7 +784,7 @@ class Item(LibModel): for query, path_format in path_formats: if query == PF_KEY_DEFAULT: continue - query = get_query(query, type(self)) + query, _ = parse_query_string(query, type(self)) if query.match(self): # The query matches the item! Use the corresponding path # format. @@ -588,7 +795,7 @@ class Item(LibModel): if query == PF_KEY_DEFAULT: break else: - assert False, "no default path format" + assert False, u"no default path format" if isinstance(path_format, Template): subpath_tmpl = path_format else: @@ -602,25 +809,28 @@ class Item(LibModel): subpath = unicodedata.normalize('NFD', subpath) else: subpath = unicodedata.normalize('NFC', subpath) - # Truncate components and remove forbidden characters. - subpath = util.sanitize_path(subpath, self._db.replacements) - # Encode for the filesystem. - if not fragment: - subpath = bytestring_path(subpath) - # Preserve extension. - _, extension = os.path.splitext(self.path) - if fragment: - # Outputting Unicode. - extension = extension.decode('utf8', 'ignore') - subpath += extension.lower() + if beets.config['asciify_paths']: + subpath = unidecode(subpath) - # Truncate too-long components. maxlen = beets.config['max_filename_length'].get(int) if not maxlen: # When zero, try to determine from filesystem. maxlen = util.max_filename_length(self._db.directory) - subpath = util.truncate_path(subpath, maxlen) + + subpath, fellback = util.legalize_path( + subpath, self._db.replacements, maxlen, + os.path.splitext(self.path)[1], fragment + ) + if fellback: + # Print an error message if legalization fell back to + # default replacements because of the maximum length. + log.warning( + u'Fell back to default replacements when naming ' + u'file {}. Configure replacements to avoid lengthening ' + u'the filename.', + subpath + ) if fragment: return subpath @@ -633,10 +843,89 @@ class Album(LibModel): library. Reflects the library's "albums" table, including album art. """ - _fields = dict((name, typ) for (name, typ, _) in ALBUM_FIELDS) _table = 'albums' _flex_table = 'album_attributes' - _search_fields = ALBUM_DEFAULT_FIELDS + _always_dirty = True + _fields = { + 'id': types.PRIMARY_ID, + 'artpath': PathType(), + 'added': DateType(), + + 'albumartist': types.STRING, + 'albumartist_sort': types.STRING, + 'albumartist_credit': types.STRING, + 'album': types.STRING, + 'genre': types.STRING, + 'year': types.PaddedInt(4), + 'month': types.PaddedInt(2), + 'day': types.PaddedInt(2), + 'disctotal': types.PaddedInt(2), + 'comp': types.BOOLEAN, + 'mb_albumid': types.STRING, + 'mb_albumartistid': types.STRING, + 'albumtype': types.STRING, + 'label': types.STRING, + 'mb_releasegroupid': types.STRING, + 'asin': types.STRING, + 'catalognum': types.STRING, + 'script': types.STRING, + 'language': types.STRING, + 'country': types.STRING, + 'albumstatus': types.STRING, + 'albumdisambig': types.STRING, + 'rg_album_gain': types.NULL_FLOAT, + 'rg_album_peak': types.NULL_FLOAT, + 'original_year': types.PaddedInt(4), + 'original_month': types.PaddedInt(2), + 'original_day': types.PaddedInt(2), + } + + _search_fields = ('album', 'albumartist', 'genre') + + _types = { + 'path': PathType(), + 'data_source': types.STRING, + } + + _sorts = { + 'albumartist': SmartArtistSort, + 'artist': SmartArtistSort, + } + + item_keys = [ + 'added', + 'albumartist', + 'albumartist_sort', + 'albumartist_credit', + 'album', + 'genre', + 'year', + 'month', + 'day', + 'disctotal', + 'comp', + 'mb_albumid', + 'mb_albumartistid', + 'albumtype', + 'label', + 'mb_releasegroupid', + 'asin', + 'catalognum', + 'script', + 'language', + 'country', + 'albumstatus', + 'albumdisambig', + 'rg_album_gain', + 'rg_album_peak', + 'original_year', + 'original_month', + 'original_day', + ] + """List of keys that are set on an album's items. + """ + + _format_config_key = 'format_album' @classmethod def _getters(cls): @@ -644,17 +933,9 @@ class Album(LibModel): # the album's directory as `path`. getters = plugins.album_field_getters() getters['path'] = Album.item_dir + getters['albumtotal'] = Album._albumtotal return getters - def __setitem__(self, key, value): - """Set the value of an album attribute.""" - if key == 'artpath': - if isinstance(value, unicode): - value = bytestring_path(value) - elif isinstance(value, buffer): - value = bytes(value) - super(Album, self).__setitem__(key, value) - def items(self): """Returns an iterable over the items associated with this album. @@ -681,7 +962,7 @@ class Album(LibModel): for item in self.items(): item.remove(delete, False) - def move_art(self, copy=False): + def move_art(self, copy=False, link=False): """Move or copy any existing album art so that it remains in the same directory as the items. """ @@ -694,9 +975,13 @@ class Album(LibModel): return new_art = util.unique_path(new_art) - log.debug('moving album art %s to %s' % (old_art, new_art)) + log.debug(u'moving album art {0} to {1}', + util.displayable_path(old_art), + util.displayable_path(new_art)) if copy: util.copy(old_art, new_art) + elif link: + util.link(old_art, new_art) else: util.move(old_art, new_art) self.artpath = new_art @@ -706,7 +991,7 @@ class Album(LibModel): util.prune_dirs(os.path.dirname(old_art), self._db.directory) - def move(self, copy=False, basedir=None): + def move(self, copy=False, link=False, basedir=None): """Moves (or copies) all items to their destination. Any album art moves along with them. basedir overrides the library base directory for the destination. The album is stored to the @@ -721,10 +1006,10 @@ class Album(LibModel): # Move items. items = list(self.items()) for item in items: - item.move(copy, basedir=basedir, with_album=False) + item.move(copy, link, basedir=basedir, with_album=False) # Move art. - self.move_art(copy) + self.move_art(copy, link) self.store() def item_dir(self): @@ -733,9 +1018,30 @@ class Album(LibModel): """ item = self.items().get() if not item: - raise ValueError('empty album') + raise ValueError(u'empty album') return os.path.dirname(item.path) + def _albumtotal(self): + """Return the total number of tracks on all discs on the album + """ + if self.disctotal == 1 or not beets.config['per_disc_numbering']: + return self.items()[0].tracktotal + + counted = [] + total = 0 + + for item in self.items(): + if item.disc in counted: + continue + + total += item.tracktotal + counted.append(item.disc) + + if len(counted) == self.disctotal: + break + + return total + def art_destination(self, image, item_dir=None): """Returns a path to the destination for the album art image for the album. `image` is the path of the image that will be @@ -750,6 +1056,8 @@ class Album(LibModel): filename_tmpl = Template(beets.config['art_filename'].get(unicode)) subpath = self.evaluate_template(filename_tmpl, True) + if beets.config['asciify_paths']: + subpath = unidecode(subpath) subpath = util.sanitize_path(subpath, replacements=self._db.replacements) subpath = bytestring_path(subpath) @@ -763,6 +1071,8 @@ class Album(LibModel): """Sets the album's cover art to the image at the given path. The image is copied (or moved) into place, replacing any existing art. + + Sends an 'art_set' event with `self` as the sole argument. """ path = bytestring_path(path) oldart = self.artpath @@ -786,13 +1096,15 @@ class Album(LibModel): util.move(path, artdest) self.artpath = artdest + plugins.send('art_set', album=self) + def store(self): """Update the database with the album information. The album's tracks are also updated. """ # Get modified track fields. track_updates = {} - for key in ALBUM_KEYS_ITEM: + for key in self.item_keys: if key in self._dirty: track_updates[key] = self[key] @@ -804,163 +1116,82 @@ class Album(LibModel): item[key] = value item.store() + def try_sync(self, write, move): + """Synchronize the album and its items with the database. + Optionally, also write any new tags into the files and update + their paths. + + `write` indicates whether to write tags to the item files, and + `move` controls whether files (both audio and album art) are + moved. + """ + self.store() + for item in self.items(): + item.try_sync(write, move) -# Query construction and parsing helpers. +# Query construction helpers. +def parse_query_parts(parts, model_cls): + """Given a beets query string as a list of components, return the + `Query` and `Sort` they represent. -PARSE_QUERY_PART_REGEX = re.compile( - # Non-capturing optional segment for the keyword. - r'(?:' - r'(\S+?)' # The field key. - r'(? (None, 'stapler', SubstringQuery) - 'color:red' -> ('color', 'red', SubstringQuery) - ':^Quiet' -> (None, '^Quiet', RegexpQuery) - 'color::b..e' -> ('color', 'b..e', RegexpQuery) - - Prefixes may be "escaped" with a backslash to disable the keying - behavior. + Like `dbcore.parse_sorted_query`, with beets query prefixes and + special path query detection. """ - part = part.strip() - match = PARSE_QUERY_PART_REGEX.match(part) - - assert match # Regex should always match. - key = match.group(1) - term = match.group(2).replace('\:', ':') - - # Match the search term against the list of prefixes. - for pre, query_class in prefixes.items(): - if term.startswith(pre): - return key, term[len(pre):], query_class - - # No matching prefix: use type-based or fallback/default query. - query_class = query_classes.get(key, default_class) - return key, term, query_class - - -def construct_query_part(query_part, model_cls): - """Create a query from a single query component, `query_part`, for - querying instances of `model_cls`. Return a `Query` instance. - """ - # Shortcut for empty query parts. - if not query_part: - return dbcore.query.TrueQuery() - - # Set up and parse the string. - query_classes = dict((k, t.query) for (k, t) in model_cls._fields.items()) + # Get query types and their prefix characters. prefixes = {':': dbcore.query.RegexpQuery} prefixes.update(plugins.queries()) - key, pattern, query_class = \ - parse_query_part(query_part, query_classes, prefixes) - # No key specified. - if key is None: - if os.sep in pattern and 'path' in model_cls._fields: - # This looks like a path. - return PathQuery('path', pattern) - elif issubclass(query_class, dbcore.FieldQuery): - # The query type matches a specific field, but none was - # specified. So we use a version of the query that matches - # any field. - return dbcore.query.AnyFieldQuery(pattern, - model_cls._search_fields, - query_class) + # Special-case path-like queries, which are non-field queries + # containing path separators (/). + path_parts = [] + non_path_parts = [] + for s in parts: + if PathQuery.is_path_query(s): + path_parts.append(s) else: - # Other query type. - return query_class(pattern) + non_path_parts.append(s) - key = key.lower() + query, sort = dbcore.parse_sorted_query( + model_cls, non_path_parts, prefixes + ) - # Singleton query (not a real field). - if key == 'singleton': - return SingletonQuery(util.str2bool(pattern)) + # Add path queries to aggregate query. + # Match field / flexattr depending on whether the model has the path field + fast_path_query = 'path' in model_cls._fields + query.subqueries += [PathQuery('path', s, fast_path_query) + for s in path_parts] - # Other field. - else: - return query_class(key.lower(), pattern, key in model_cls._fields) + return query, sort -def query_from_strings(query_cls, model_cls, query_parts): - """Creates a collection query of type `query_cls` from a list of - strings in the format used by parse_query_part. `model_cls` - determines how queries are constructed from strings. +def parse_query_string(s, model_cls): + """Given a beets query string, return the `Query` and `Sort` they + represent. + + The string is split into components using shell-like syntax. """ - subqueries = [] - for part in query_parts: - subqueries.append(construct_query_part(part, model_cls)) - if not subqueries: # No terms in query. - subqueries = [dbcore.query.TrueQuery()] - return query_cls(subqueries) - - -def get_query(val, model_cls): - """Takes a value which may be None, a query string, a query string - list, or a Query object, and returns a suitable Query object. - `model_cls` is the subclass of Model indicating which entity this - is a query for (i.e., Album or Item) and is used to determine which - fields are searched. - """ - # Convert a single string into a list of space-separated - # criteria. - if isinstance(val, basestring): - # A bug in Python < 2.7.3 prevents correct shlex splitting of - # Unicode strings. - # http://bugs.python.org/issue6988 - if isinstance(val, unicode): - val = val.encode('utf8') - val = [s.decode('utf8') for s in shlex.split(val)] - - if val is None: - return dbcore.query.TrueQuery() - elif isinstance(val, list) or isinstance(val, tuple): - return query_from_strings(dbcore.AndQuery, model_cls, val) - elif isinstance(val, dbcore.Query): - return val - else: - raise ValueError('query must be None or have type Query or str') - + assert isinstance(s, unicode), u"Query is not unicode: {0!r}".format(s) + try: + parts = util.shlex_split(s) + except ValueError as exc: + raise dbcore.InvalidQueryError(s, exc) + return parse_query_parts(parts, model_cls) # The Library: interface to the database. - class Library(dbcore.Database): """A database of music containing songs and albums. """ _models = (Item, Album) def __init__(self, path='library.blb', - directory='~/Music', - path_formats=((PF_KEY_DEFAULT, - '$artist/$album/$track $title'),), - replacements=None): + directory='~/Music', + path_formats=((PF_KEY_DEFAULT, + '$artist/$album/$track $title'),), + replacements=None): if path != ':memory:': self.path = bytestring_path(normpath(path)) super(Library, self).__init__(path) @@ -971,7 +1202,6 @@ class Library(dbcore.Database): self._memotable = {} # Used for template substitution performance. - # Adding objects to the database. def add(self, obj): @@ -983,12 +1213,17 @@ class Library(dbcore.Database): return obj.id def add_album(self, items): - """Create a new album consisting of a list of items. The items - are added to the database if they don't yet have an ID. Return a - new :class:`Album` object. + """Create a new album consisting of a list of items. + + The items are added to the database if they don't yet have an + ID. Return a new :class:`Album` object. The list items must not + be empty. """ + if not items: + raise ValueError(u'need at least one item') + # Create the album structure using metadata from the first item. - values = dict((key, items[0][key]) for key in ALBUM_KEYS_ITEM) + values = dict((key, items[0][key]) for key in Album.item_keys) album = Album(self, **values) # Add the album structure and set the items' album_id fields. @@ -1004,34 +1239,54 @@ class Library(dbcore.Database): return album - # Querying. - def _fetch(self, model_cls, query, order_by=None): - """Parse a query and fetch. + def _fetch(self, model_cls, query, sort=None): + """Parse a query and fetch. If a order specification is present + in the query string the `sort` argument is ignored. """ + # Parse the query, if necessary. + try: + parsed_sort = None + if isinstance(query, basestring): + query, parsed_sort = parse_query_string(query, model_cls) + elif isinstance(query, (list, tuple)): + query, parsed_sort = parse_query_parts(query, model_cls) + except dbcore.query.InvalidQueryArgumentTypeError as exc: + raise dbcore.InvalidQueryError(query, exc) + + # Any non-null sort specified by the parsed query overrides the + # provided sort. + if parsed_sort and not isinstance(parsed_sort, dbcore.query.NullSort): + sort = parsed_sort + return super(Library, self)._fetch( - model_cls, get_query(query, model_cls), order_by + model_cls, query, sort ) - def albums(self, query=None): - """Get a sorted list of :class:`Album` objects matching the - given query. + @staticmethod + def get_default_album_sort(): + """Get a :class:`Sort` object for albums from the config option. """ - order = '{0}, album'.format( - _orelse("albumartist_sort", "albumartist") - ) - return self._fetch(Album, query, order) + return dbcore.sort_from_strings( + Album, beets.config['sort_album'].as_str_seq()) - def items(self, query=None): - """Get a sorted list of :class:`Item` objects matching the given - query. + @staticmethod + def get_default_item_sort(): + """Get a :class:`Sort` object for items from the config option. """ - order = '{0}, album, disc, track'.format( - _orelse("artist_sort", "artist") - ) - return self._fetch(Item, query, order) + return dbcore.sort_from_strings( + Item, beets.config['sort_item'].as_str_seq()) + def albums(self, query=None, sort=None): + """Get :class:`Album` objects matching the query. + """ + return self._fetch(Album, query, sort or self.get_default_album_sort()) + + def items(self, query=None, sort=None): + """Get :class:`Item` objects matching the query. + """ + return self._fetch(Item, query, sort or self.get_default_item_sort()) # Convenience accessors. @@ -1055,10 +1310,8 @@ class Library(dbcore.Database): return self._get(Album, album_id) - # Default path template resources. - def _int_arg(s): """Convert a string argument to an integer for use in a template function. May raise a ValueError. @@ -1075,7 +1328,7 @@ class DefaultTemplateFunctions(object): _prefix = 'tmpl_' def __init__(self, item=None, lib=None): - """Paramaterize the functions. If `item` or `lib` is None, then + """Parametrize the functions. If `item` or `lib` is None, then some functions (namely, ``aunique``) will always evaluate to the empty string. """ @@ -1123,9 +1376,13 @@ class DefaultTemplateFunctions(object): otherwise, emit ``falseval`` (if provided). """ try: - condition = _int_arg(condition) + int_condition = _int_arg(condition) except ValueError: - condition = condition.strip() + if condition.lower() == "false": + return falseval + else: + condition = int_condition + if condition: return trueval else: @@ -1138,11 +1395,11 @@ class DefaultTemplateFunctions(object): return unidecode(s) @staticmethod - def tmpl_time(s, format): + def tmpl_time(s, fmt): """Format a time value using `strftime`. """ cur_fmt = beets.config['time_format'].get(unicode) - return time.strftime(format, time.strptime(s, cur_fmt)) + return time.strftime(fmt, time.strptime(s, cur_fmt)) def tmpl_aunique(self, keys=None, disam=None): """Generate a string that is guaranteed to be unique among all @@ -1176,7 +1433,7 @@ class DefaultTemplateFunctions(object): # Find matching albums to disambiguate with. subqueries = [] for key in keys: - value = getattr(album, key) + value = album.get(key, '') subqueries.append(dbcore.MatchQuery(key, value)) albums = self.lib.albums(dbcore.AndQuery(subqueries)) @@ -1189,7 +1446,7 @@ class DefaultTemplateFunctions(object): # Find the first disambiguator that distinguishes the albums. for disambiguator in disam: # Get the value for each album for the current field. - disam_values = set([getattr(a, disambiguator) for a in albums]) + disam_values = set([a.get(disambiguator, '') for a in albums]) # If the set of unique values is equal to the number of # albums in the disambiguation set, we're done -- this is @@ -1204,11 +1461,40 @@ class DefaultTemplateFunctions(object): return res # Flatten disambiguation value into a string. - disam_value = album._get_formatted(disambiguator, True) + disam_value = album.formatted(True).get(disambiguator) res = u' [{0}]'.format(disam_value) self.lib._memotable[memokey] = res return res + @staticmethod + def tmpl_first(s, count=1, skip=0, sep=u'; ', join_str=u'; '): + """ Gets the item(s) from x to y in a string separated by something + and join then with something + + :param s: the string + :param count: The number of items included + :param skip: The number of items skipped + :param sep: the separator. Usually is '; ' (default) or '/ ' + :param join_str: the string which will join the items, default '; '. + """ + skip = int(skip) + count = skip + int(count) + return join_str.join(s.split(sep)[skip:count]) + + def tmpl_ifdef(self, field, trueval=u'', falseval=u''): + """ If field exists return trueval or the field (default) + otherwise, emit return falseval (if provided). + + :param field: The name of the field + :param trueval: The string if the condition is true + :param falseval: The string if the condition is false + :return: The string, based on condition + """ + if self.item.formatted().get(field): + return trueval if trueval else self.item.formatted().get(field) + else: + return falseval + # Get the name of tmpl_* functions in the above class. DefaultTemplateFunctions._func_names = \ diff --git a/libs/beets/logging.py b/libs/beets/logging.py new file mode 100644 index 00000000..a94da1c6 --- /dev/null +++ b/libs/beets/logging.py @@ -0,0 +1,133 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Adrian Sampson. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""A drop-in replacement for the standard-library `logging` module that +allows {}-style log formatting on Python 2 and 3. + +Provides everything the "logging" module does. The only difference is +that when getLogger(name) instantiates a logger that logger uses +{}-style formatting. +""" + +from __future__ import division, absolute_import, print_function + +from copy import copy +from logging import * # noqa +import subprocess +import threading + + +def logsafe(val): + """Coerce a potentially "problematic" value so it can be formatted + in a Unicode log string. + + This works around a number of pitfalls when logging objects in + Python 2: + - Logging path names, which must be byte strings, requires + conversion for output. + - Some objects, including some exceptions, will crash when you call + `unicode(v)` while `str(v)` works fine. CalledProcessError is an + example. + """ + # Already Unicode. + if isinstance(val, unicode): + return val + + # Bytestring: needs decoding. + elif isinstance(val, bytes): + # Blindly convert with UTF-8. Eventually, it would be nice to + # (a) only do this for paths, if they can be given a distinct + # type, and (b) warn the developer if they do this for other + # bytestrings. + return val.decode('utf8', 'replace') + + # A "problem" object: needs a workaround. + elif isinstance(val, subprocess.CalledProcessError): + try: + return unicode(val) + except UnicodeDecodeError: + # An object with a broken __unicode__ formatter. Use __str__ + # instead. + return str(val).decode('utf8', 'replace') + + # Other objects are used as-is so field access, etc., still works in + # the format string. + else: + return val + + +class StrFormatLogger(Logger): + """A version of `Logger` that uses `str.format`-style formatting + instead of %-style formatting. + """ + + class _LogMessage(object): + def __init__(self, msg, args, kwargs): + self.msg = msg + self.args = args + self.kwargs = kwargs + + def __str__(self): + args = [logsafe(a) for a in self.args] + kwargs = dict((k, logsafe(v)) for (k, v) in self.kwargs.items()) + return self.msg.format(*args, **kwargs) + + def _log(self, level, msg, args, exc_info=None, extra=None, **kwargs): + """Log msg.format(*args, **kwargs)""" + m = self._LogMessage(msg, args, kwargs) + return super(StrFormatLogger, self)._log(level, m, (), exc_info, extra) + + +class ThreadLocalLevelLogger(Logger): + """A version of `Logger` whose level is thread-local instead of shared. + """ + def __init__(self, name, level=NOTSET): + self._thread_level = threading.local() + self.default_level = NOTSET + super(ThreadLocalLevelLogger, self).__init__(name, level) + + @property + def level(self): + try: + return self._thread_level.level + except AttributeError: + self._thread_level.level = self.default_level + return self.level + + @level.setter + def level(self, value): + self._thread_level.level = value + + def set_global_level(self, level): + """Set the level on the current thread + the default value for all + threads. + """ + self.default_level = level + self.setLevel(level) + + +class BeetsLogger(ThreadLocalLevelLogger, StrFormatLogger): + pass + + +my_manager = copy(Logger.manager) +my_manager.loggerClass = BeetsLogger + + +def getLogger(name=None): # noqa + if name: + return my_manager.getLogger(name) + else: + return Logger.root diff --git a/libs/beets/mediafile.py b/libs/beets/mediafile.py index 301e0f37..a359a5b4 100644 --- a/libs/beets/mediafile.py +++ b/libs/beets/mediafile.py @@ -1,5 +1,6 @@ +# -*- coding: utf-8 -*- # This file is part of beets. -# Copyright 2014, Adrian Sampson. +# Copyright 2016, Adrian Sampson. # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -32,14 +33,18 @@ Internally ``MediaFile`` uses ``MediaField`` descriptors to access the data from the tags. In turn ``MediaField`` uses a number of ``StorageStyle`` strategies to handle format specific logic. """ +from __future__ import division, absolute_import, print_function + import mutagen import mutagen.mp3 +import mutagen.id3 import mutagen.oggopus import mutagen.oggvorbis import mutagen.mp4 import mutagen.flac import mutagen.monkeysaudio import mutagen.asf +import mutagen.aiff import datetime import re import base64 @@ -47,40 +52,17 @@ import math import struct import imghdr import os -import logging import traceback -from beets.util.enumeration import enum +import enum + +from beets import logging +from beets.util import displayable_path, syspath + __all__ = ['UnreadableFileError', 'FileTypeError', 'MediaFile'] - - -# Logger. log = logging.getLogger('beets') - - -# Exceptions. - -class UnreadableFileError(Exception): - """Indicates a file that MediaFile can't read. - """ - pass - -class FileTypeError(UnreadableFileError): - """Raised for files that don't seem to have a type MediaFile - supports. - """ - pass - -class MutagenError(UnreadableFileError): - """Raised when Mutagen fails unexpectedly---probably due to a bug. - """ - - - -# Constants. - # Human-readable type names. TYPES = { 'mp3': 'MP3', @@ -93,9 +75,41 @@ TYPES = { 'wv': 'WavPack', 'mpc': 'Musepack', 'asf': 'Windows Media', + 'aiff': 'AIFF', } +# Exceptions. + +class UnreadableFileError(Exception): + """Mutagen is not able to extract information from the file. + """ + def __init__(self, path): + Exception.__init__(self, displayable_path(path)) + + +class FileTypeError(UnreadableFileError): + """Reading this type of file is not supported. + + If passed the `mutagen_type` argument this indicates that the + mutagen type is not supported by `Mediafile`. + """ + def __init__(self, path, mutagen_type=None): + path = displayable_path(path) + if mutagen_type is None: + msg = path + else: + msg = u'{0}: of mutagen type {1}'.format(path, mutagen_type) + Exception.__init__(self, msg) + + +class MutagenError(UnreadableFileError): + """Raised when Mutagen fails unexpectedly---probably due to a bug. + """ + def __init__(self, path, mutagen_exc): + msg = u'{0}: {1}'.format(displayable_path(path), mutagen_exc) + Exception.__init__(self, msg) + # Utility. @@ -105,10 +119,11 @@ def _safe_cast(out_type, val): returned. out_type should be bool, int, or unicode; otherwise, the value is just passed through. """ + if val is None: + return None + if out_type == int: - if val is None: - return 0 - elif isinstance(val, int) or isinstance(val, float): + if isinstance(val, int) or isinstance(val, float): # Just a number. return int(val) else: @@ -123,45 +138,38 @@ def _safe_cast(out_type, val): return int(val) elif out_type == bool: - if val is None: + try: + # Should work for strings, bools, ints: + return bool(int(val)) + except ValueError: return False - else: - try: - # Should work for strings, bools, ints: - return bool(int(val)) - except ValueError: - return False elif out_type == unicode: - if val is None: - return u'' + if isinstance(val, bytes): + return val.decode('utf8', 'ignore') + elif isinstance(val, unicode): + return val else: - if isinstance(val, str): - return val.decode('utf8', 'ignore') - elif isinstance(val, unicode): - return val - else: - return unicode(val) + return unicode(val) elif out_type == float: - if val is None: - return 0.0 - elif isinstance(val, int) or isinstance(val, float): + if isinstance(val, int) or isinstance(val, float): return float(val) else: if not isinstance(val, basestring): val = unicode(val) - val = re.match(r'[\+-]?[0-9\.]*', val.strip()).group(0) - if not val: - return 0.0 - else: - return float(val) + match = re.match(r'[\+-]?([0-9]+\.?[0-9]*|[0-9]*\.[0-9]+)', + val.strip()) + if match: + val = match.group(0) + if val: + return float(val) + return 0.0 else: return val - # Image coding for ASF/WMA. def _unpack_asf_image(data): @@ -173,15 +181,15 @@ def _unpack_asf_image(data): of exceptions (out-of-bounds, etc.). We should clean this up sometime so that the failure modes are well-defined. """ - type, size = struct.unpack_from(" 0 else None + image_data = frame.value[text_delimiter_index + 1:] + images.append(Image(data=image_data, type=cover_type, + desc=comment)) + except KeyError: + pass + + return images + + def set_list(self, mutagen_file, values): + self.delete(mutagen_file) + + for image in values: + image_type = image.type or ImageType.other + comment = image.desc or '' + image_data = comment.encode('utf8') + b'\x00' + image.data + cover_tag = self.TAG_NAMES[image_type] + mutagen_file[cover_tag] = image_data + + def delete(self, mutagen_file): + """Remove all images from the file. + """ + for cover_tag in self.TAG_NAMES.values(): + try: + del mutagen_file[cover_tag] + except KeyError: + pass # MediaField is a descriptor that represents a single logical field. It @@ -913,12 +1061,14 @@ class MediaField(object): def __init__(self, *styles, **kwargs): """Creates a new MediaField. - - `styles`: `StorageStyle` instances that describe the strategy - for reading and writing the field in particular formats. - There must be at least one style for each possible file - format. - - `out_type`: the type of the value that should be returned when - getting this property. + :param styles: `StorageStyle` instances that describe the strategy + for reading and writing the field in particular + formats. There must be at least one style for + each possible file format. + + :param out_type: the type of the value that should be returned when + getting this property. + """ self.out_type = kwargs.get('out_type', unicode) self._styles = styles @@ -945,6 +1095,10 @@ class MediaField(object): for style in self.styles(mediafile.mgfile): style.set(mediafile.mgfile, value) + def __delete__(self, mediafile): + for style in self.styles(mediafile.mgfile): + style.delete(mediafile.mgfile) + def _none_value(self): """Get an appropriate "null" value for this field's type. This is used internally when setting the field to None. @@ -1007,27 +1161,40 @@ class DateField(MediaField): def __get__(self, mediafile, owner=None): year, month, day = self._get_date_tuple(mediafile) + if not year: + return None try: return datetime.date( - year or datetime.MINYEAR, + year, month or 1, day or 1 ) except ValueError: # Out of range values. - return datetime.date.min + return None def __set__(self, mediafile, date): - self._set_date_tuple(mediafile, date.year, date.month, date.day) + if date is None: + self._set_date_tuple(mediafile, None, None, None) + else: + self._set_date_tuple(mediafile, date.year, date.month, date.day) + + def __delete__(self, mediafile): + super(DateField, self).__delete__(mediafile) + if hasattr(self, '_year_field'): + self._year_field.__delete__(mediafile) def _get_date_tuple(self, mediafile): """Get a 3-item sequence representing the date consisting of a year, month, and day number. Each number is either an integer or None. """ - # Get the underlying data and split on hyphens. + # Get the underlying data and split on hyphens and slashes. datestring = super(DateField, self).__get__(mediafile, None) - datestring = re.sub(r'[Tt ].*$', '', unicode(datestring)) - items = unicode(datestring).split('-') + if isinstance(datestring, basestring): + datestring = re.sub(r'[Tt ].*$', '', unicode(datestring)) + items = re.split('[-/]', unicode(datestring)) + else: + items = [] # Ensure that we have exactly 3 components, possibly by # truncating or padding. @@ -1040,20 +1207,30 @@ class DateField(MediaField): items[0] = self._year_field.__get__(mediafile) # Convert each component to an integer if possible. - return [_safe_cast(int, item) for item in items] + items_ = [] + for item in items: + try: + items_.append(int(item)) + except: + items_.append(None) + return items_ def _set_date_tuple(self, mediafile, year, month=None, day=None): """Set the value of the field given a year, month, and day number. Each number can be an integer or None to indicate an unset component. """ - date = [year or 0] + if year is None: + self.__delete__(mediafile) + return + + date = [u'{0:04d}'.format(int(year))] if month: - date.append(month) + date.append(u'{0:02d}'.format(int(month))) if month and day: - date.append(day) + date.append(u'{0:02d}'.format(int(day))) date = map(unicode, date) - super(DateField, self).__set__(mediafile, '-'.join(date)) + super(DateField, self).__set__(mediafile, u'-'.join(date)) if hasattr(self, '_year_field'): self._year_field.__set__(mediafile, year) @@ -1084,29 +1261,48 @@ class DateItemField(MediaField): items[self.item_pos] = value self.date_field._set_date_tuple(mediafile, *items) + def __delete__(self, mediafile): + self.__set__(mediafile, None) + class CoverArtField(MediaField): """A descriptor that provides access to the *raw image data* for the - first image on a file. This is used for backwards compatibility: the + cover image on a file. This is used for backwards compatibility: the full `ImageListField` provides richer `Image` objects. + + When there are multiple images we try to pick the most likely to be a front + cover. """ def __init__(self): pass def __get__(self, mediafile, _): - try: - return mediafile.images[0].data - except IndexError: + candidates = mediafile.images + if candidates: + return self.guess_cover_image(candidates).data + else: return None + @staticmethod + def guess_cover_image(candidates): + if len(candidates) == 1: + return candidates[0] + try: + return next(c for c in candidates if c.type == ImageType.front) + except StopIteration: + return candidates[0] + def __set__(self, mediafile, data): if data: mediafile.images = [Image(data=data)] else: mediafile.images = [] + def __delete__(self, mediafile): + delattr(mediafile, 'images') -class ImageListField(MediaField): + +class ImageListField(ListMediaField): """Descriptor to access the list of images embedded in tags. The getter returns a list of `Image` instances obtained from @@ -1123,31 +1319,25 @@ class ImageListField(MediaField): ASFImageStorageStyle(), VorbisImageStorageStyle(), FlacImageStorageStyle(), + APEv2ImageStorageStyle(), + out_type=Image, ) - def __get__(self, mediafile, _): - images = [] - for style in self.styles(mediafile.mgfile): - images.extend(style.get_list(mediafile.mgfile)) - return images - - def __set__(self, mediafile, images): - for style in self.styles(mediafile.mgfile): - style.set_list(mediafile.mgfile, images) - - # MediaFile is a collection of fields. - class MediaFile(object): """Represents a multimedia file on disk and provides access to its metadata. """ - def __init__(self, path): - """Constructs a new MediaFile reflecting the file at path. May - throw UnreadableFileError. + def __init__(self, path, id3v23=False): + """Constructs a new `MediaFile` reflecting the file at path. May + throw `UnreadableFileError`. + + By default, MP3 files are saved with ID3v2.4 tags. You can use + the older ID3v2.3 standard by specifying the `id3v23` option. """ + path = syspath(path) self.path = path unreadable_exc = ( @@ -1161,41 +1351,49 @@ class MediaFile(object): mutagen.ogg.error, mutagen.asf.error, mutagen.apev2.error, + mutagen.aiff.error, ) try: self.mgfile = mutagen.File(path) except unreadable_exc as exc: - log.debug(u'header parsing failed: {0}'.format(unicode(exc))) - raise UnreadableFileError('Mutagen could not read file') + log.debug(u'header parsing failed: {0}', unicode(exc)) + raise UnreadableFileError(path) except IOError as exc: if type(exc) == IOError: # This is a base IOError, not a subclass from Mutagen or # anywhere else. raise else: - log.debug(traceback.format_exc()) - raise MutagenError('Mutagen raised an exception') + log.debug(u'{}', traceback.format_exc()) + raise MutagenError(path, exc) except Exception as exc: # Isolate bugs in Mutagen. - log.debug(traceback.format_exc()) - log.error('uncaught Mutagen exception in open: {0}'.format(exc)) - raise MutagenError('Mutagen raised an exception') + log.debug(u'{}', traceback.format_exc()) + log.error(u'uncaught Mutagen exception in open: {0}', exc) + raise MutagenError(path, exc) - if self.mgfile is None: # Mutagen couldn't guess the type - raise FileTypeError('file type unsupported by Mutagen') - elif type(self.mgfile).__name__ == 'M4A' or \ - type(self.mgfile).__name__ == 'MP4': - # This hack differentiates AAC and ALAC until we find a more - # deterministic approach. Mutagen only sets the sample rate - # for AAC files. See: - # https://github.com/sampsyo/beets/pull/295 - if hasattr(self.mgfile.info, 'sample_rate') and \ - self.mgfile.info.sample_rate > 0: - self.type = 'aac' + if self.mgfile is None: + # Mutagen couldn't guess the type + raise FileTypeError(path) + elif (type(self.mgfile).__name__ == 'M4A' or + type(self.mgfile).__name__ == 'MP4'): + info = self.mgfile.info + if hasattr(info, 'codec'): + if info.codec and info.codec.startswith('alac'): + self.type = 'alac' + else: + self.type = 'aac' else: - self.type = 'alac' - elif type(self.mgfile).__name__ == 'ID3' or \ - type(self.mgfile).__name__ == 'MP3': + # This hack differentiates AAC and ALAC on versions of + # Mutagen < 1.26. Once Mutagen > 1.26 is out and + # required by beets, we can remove this. + if hasattr(self.mgfile.info, 'bitrate') and \ + self.mgfile.info.bitrate > 0: + self.type = 'aac' + else: + self.type = 'alac' + elif (type(self.mgfile).__name__ == 'ID3' or + type(self.mgfile).__name__ == 'MP3'): self.type = 'mp3' elif type(self.mgfile).__name__ == 'FLAC': self.type = 'flac' @@ -1211,22 +1409,24 @@ class MediaFile(object): self.type = 'mpc' elif type(self.mgfile).__name__ == 'ASF': self.type = 'asf' + elif type(self.mgfile).__name__ == 'AIFF': + self.type = 'aiff' else: - raise FileTypeError('file type %s unsupported by MediaFile' % - type(self.mgfile).__name__) + raise FileTypeError(path, type(self.mgfile).__name__) - # add a set of tags if it's missing + # Add a set of tags if it's missing. if self.mgfile.tags is None: self.mgfile.add_tags() - def save(self, id3v23=False): - """Write the object's tags back to the file. + # Set the ID3v2.3 flag only for MP3s. + self.id3v23 = id3v23 and self.type == 'mp3' - By default, MP3 files are saved with ID3v2.4 tags. You can use - the older ID3v2.3 standard by specifying the `id3v23` option. + def save(self): + """Write the object's tags back to the file. """ + # Possibly save the tags to ID3v2.3. kwargs = {} - if id3v23 and self.type == 'mp3': + if self.id3v23: id3 = self.mgfile if hasattr(id3, 'tags'): # In case this is an MP3 object, not an ID3 object. @@ -1241,9 +1441,9 @@ class MediaFile(object): # Propagate these through: they don't represent Mutagen bugs. raise except Exception as exc: - log.debug(traceback.format_exc()) - log.error('uncaught Mutagen exception in save: {0}'.format(exc)) - raise MutagenError('Mutagen raised an exception') + log.debug(u'{}', traceback.format_exc()) + log.error(u'uncaught Mutagen exception in save: {0}', exc) + raise MutagenError(self.path, exc) def delete(self): """Remove the current metadata tag from the file. @@ -1256,30 +1456,112 @@ class MediaFile(object): for tag in self.mgfile.keys(): del self.mgfile[tag] + # Convenient access to the set of available fields. + + @classmethod + def fields(cls): + """Get the names of all writable properties that reflect + metadata tags (i.e., those that are instances of + :class:`MediaField`). + """ + for property, descriptor in cls.__dict__.items(): + if isinstance(descriptor, MediaField): + yield property.decode('utf8') + + @classmethod + def _field_sort_name(cls, name): + """Get a sort key for a field name that determines the order + fields should be written in. + + Fields names are kept unchanged, unless they are instances of + :class:`DateItemField`, in which case `year`, `month`, and `day` + are replaced by `date0`, `date1`, and `date2`, respectively, to + make them appear in that order. + """ + if isinstance(cls.__dict__[name], DateItemField): + name = re.sub('year', 'date0', name) + name = re.sub('month', 'date1', name) + name = re.sub('day', 'date2', name) + return name + + @classmethod + def sorted_fields(cls): + """Get the names of all writable metadata fields, sorted in the + order that they should be written. + + This is a lexicographic order, except for instances of + :class:`DateItemField`, which are sorted in year-month-day + order. + """ + for property in sorted(cls.fields(), key=cls._field_sort_name): + yield property + + @classmethod + def readable_fields(cls): + """Get all metadata fields: the writable ones from + :meth:`fields` and also other audio properties. + """ + for property in cls.fields(): + yield property + for property in ('length', 'samplerate', 'bitdepth', 'bitrate', + 'channels', 'format'): + yield property + + @classmethod + def add_field(cls, name, descriptor): + """Add a field to store custom tags. + + :param name: the name of the property the field is accessed + through. It must not already exist on this class. + + :param descriptor: an instance of :class:`MediaField`. + """ + if not isinstance(descriptor, MediaField): + raise ValueError( + u'{0} must be an instance of MediaField'.format(descriptor)) + if name in cls.__dict__: + raise ValueError( + u'property "{0}" already exists on MediaField'.format(name)) + setattr(cls, name, descriptor) + + def update(self, dict): + """Set all field values from a dictionary. + + For any key in `dict` that is also a field to store tags the + method retrieves the corresponding value from `dict` and updates + the `MediaFile`. If a key has the value `None`, the + corresponding property is deleted from the `MediaFile`. + """ + for field in self.sorted_fields(): + if field in dict: + if dict[field] is None: + delattr(self, field) + else: + setattr(self, field, dict[field]) # Field definitions. title = MediaField( MP3StorageStyle('TIT2'), - MP4StorageStyle("\xa9nam"), + MP4StorageStyle(b"\xa9nam"), StorageStyle('TITLE'), ASFStorageStyle('Title'), ) artist = MediaField( MP3StorageStyle('TPE1'), - MP4StorageStyle("\xa9ART"), + MP4StorageStyle(b"\xa9ART"), StorageStyle('ARTIST'), ASFStorageStyle('Author'), ) album = MediaField( MP3StorageStyle('TALB'), - MP4StorageStyle("\xa9alb"), + MP4StorageStyle(b"\xa9alb"), StorageStyle('ALBUM'), ASFStorageStyle('WM/AlbumTitle'), ) genres = ListMediaField( MP3ListStorageStyle('TCON'), - MP4ListStorageStyle("\xa9gen"), + MP4ListStorageStyle(b"\xa9gen"), ListStorageStyle('GENRE'), ASFStorageStyle('WM/Genre'), ) @@ -1287,19 +1569,19 @@ class MediaFile(object): composer = MediaField( MP3StorageStyle('TCOM'), - MP4StorageStyle("\xa9wrt"), + MP4StorageStyle(b"\xa9wrt"), StorageStyle('COMPOSER'), ASFStorageStyle('WM/Composer'), ) grouping = MediaField( MP3StorageStyle('TIT1'), - MP4StorageStyle("\xa9grp"), + MP4StorageStyle(b"\xa9grp"), StorageStyle('GROUPING'), ASFStorageStyle('WM/ContentGroupDescription'), ) track = MediaField( MP3SlashPackStorageStyle('TRCK', pack_pos=0), - MP4TupleStorageStyle('trkn', index=0), + MP4TupleStorageStyle(b'trkn', index=0), StorageStyle('TRACK'), StorageStyle('TRACKNUMBER'), ASFStorageStyle('WM/TrackNumber'), @@ -1307,7 +1589,7 @@ class MediaFile(object): ) tracktotal = MediaField( MP3SlashPackStorageStyle('TRCK', pack_pos=1), - MP4TupleStorageStyle('trkn', index=1), + MP4TupleStorageStyle(b'trkn', index=1), StorageStyle('TRACKTOTAL'), StorageStyle('TRACKC'), StorageStyle('TOTALTRACKS'), @@ -1316,7 +1598,7 @@ class MediaFile(object): ) disc = MediaField( MP3SlashPackStorageStyle('TPOS', pack_pos=0), - MP4TupleStorageStyle('disk', index=0), + MP4TupleStorageStyle(b'disk', index=0), StorageStyle('DISC'), StorageStyle('DISCNUMBER'), ASFStorageStyle('WM/PartOfSet'), @@ -1324,7 +1606,7 @@ class MediaFile(object): ) disctotal = MediaField( MP3SlashPackStorageStyle('TPOS', pack_pos=1), - MP4TupleStorageStyle('disk', index=1), + MP4TupleStorageStyle(b'disk', index=1), StorageStyle('DISCTOTAL'), StorageStyle('DISCC'), StorageStyle('TOTALDISCS'), @@ -1333,124 +1615,125 @@ class MediaFile(object): ) lyrics = MediaField( MP3DescStorageStyle(key='USLT'), - MP4StorageStyle("\xa9lyr"), + MP4StorageStyle(b"\xa9lyr"), StorageStyle('LYRICS'), ASFStorageStyle('WM/Lyrics'), ) comments = MediaField( MP3DescStorageStyle(key='COMM'), - MP4StorageStyle("\xa9cmt"), + MP4StorageStyle(b"\xa9cmt"), StorageStyle('DESCRIPTION'), StorageStyle('COMMENT'), ASFStorageStyle('WM/Comments'), + ASFStorageStyle('Description') ) bpm = MediaField( MP3StorageStyle('TBPM'), - MP4StorageStyle('tmpo', as_type=int), + MP4StorageStyle(b'tmpo', as_type=int), StorageStyle('BPM'), ASFStorageStyle('WM/BeatsPerMinute'), out_type=int, ) comp = MediaField( MP3StorageStyle('TCMP'), - MP4BoolStorageStyle('cpil'), + MP4BoolStorageStyle(b'cpil'), StorageStyle('COMPILATION'), ASFStorageStyle('WM/IsCompilation', as_type=bool), out_type=bool, ) albumartist = MediaField( MP3StorageStyle('TPE2'), - MP4StorageStyle('aART'), + MP4StorageStyle(b'aART'), StorageStyle('ALBUM ARTIST'), StorageStyle('ALBUMARTIST'), ASFStorageStyle('WM/AlbumArtist'), ) albumtype = MediaField( MP3DescStorageStyle(u'MusicBrainz Album Type'), - MP4StorageStyle('----:com.apple.iTunes:MusicBrainz Album Type'), + MP4StorageStyle(b'----:com.apple.iTunes:MusicBrainz Album Type'), StorageStyle('MUSICBRAINZ_ALBUMTYPE'), ASFStorageStyle('MusicBrainz/Album Type'), ) label = MediaField( MP3StorageStyle('TPUB'), - MP4StorageStyle('----:com.apple.iTunes:Label'), - MP4StorageStyle('----:com.apple.iTunes:publisher'), + MP4StorageStyle(b'----:com.apple.iTunes:Label'), + MP4StorageStyle(b'----:com.apple.iTunes:publisher'), StorageStyle('LABEL'), StorageStyle('PUBLISHER'), # Traktor ASFStorageStyle('WM/Publisher'), ) artist_sort = MediaField( MP3StorageStyle('TSOP'), - MP4StorageStyle("soar"), + MP4StorageStyle(b"soar"), StorageStyle('ARTISTSORT'), ASFStorageStyle('WM/ArtistSortOrder'), ) albumartist_sort = MediaField( MP3DescStorageStyle(u'ALBUMARTISTSORT'), - MP4StorageStyle("soaa"), + MP4StorageStyle(b"soaa"), StorageStyle('ALBUMARTISTSORT'), ASFStorageStyle('WM/AlbumArtistSortOrder'), ) asin = MediaField( MP3DescStorageStyle(u'ASIN'), - MP4StorageStyle("----:com.apple.iTunes:ASIN"), + MP4StorageStyle(b"----:com.apple.iTunes:ASIN"), StorageStyle('ASIN'), ASFStorageStyle('MusicBrainz/ASIN'), ) catalognum = MediaField( MP3DescStorageStyle(u'CATALOGNUMBER'), - MP4StorageStyle("----:com.apple.iTunes:CATALOGNUMBER"), + MP4StorageStyle(b"----:com.apple.iTunes:CATALOGNUMBER"), StorageStyle('CATALOGNUMBER'), ASFStorageStyle('WM/CatalogNo'), ) disctitle = MediaField( MP3StorageStyle('TSST'), - MP4StorageStyle("----:com.apple.iTunes:DISCSUBTITLE"), + MP4StorageStyle(b"----:com.apple.iTunes:DISCSUBTITLE"), StorageStyle('DISCSUBTITLE'), ASFStorageStyle('WM/SetSubTitle'), ) encoder = MediaField( MP3StorageStyle('TENC'), - MP4StorageStyle("\xa9too"), + MP4StorageStyle(b"\xa9too"), StorageStyle('ENCODEDBY'), StorageStyle('ENCODER'), ASFStorageStyle('WM/EncodedBy'), ) script = MediaField( MP3DescStorageStyle(u'Script'), - MP4StorageStyle("----:com.apple.iTunes:SCRIPT"), + MP4StorageStyle(b"----:com.apple.iTunes:SCRIPT"), StorageStyle('SCRIPT'), ASFStorageStyle('WM/Script'), ) language = MediaField( MP3StorageStyle('TLAN'), - MP4StorageStyle("----:com.apple.iTunes:LANGUAGE"), + MP4StorageStyle(b"----:com.apple.iTunes:LANGUAGE"), StorageStyle('LANGUAGE'), ASFStorageStyle('WM/Language'), ) country = MediaField( MP3DescStorageStyle('MusicBrainz Album Release Country'), - MP4StorageStyle("----:com.apple.iTunes:MusicBrainz Album " - "Release Country"), + MP4StorageStyle(b"----:com.apple.iTunes:MusicBrainz " + b"Album Release Country"), StorageStyle('RELEASECOUNTRY'), ASFStorageStyle('MusicBrainz/Album Release Country'), ) albumstatus = MediaField( MP3DescStorageStyle(u'MusicBrainz Album Status'), - MP4StorageStyle("----:com.apple.iTunes:MusicBrainz Album Status"), + MP4StorageStyle(b"----:com.apple.iTunes:MusicBrainz Album Status"), StorageStyle('MUSICBRAINZ_ALBUMSTATUS'), ASFStorageStyle('MusicBrainz/Album Status'), ) media = MediaField( MP3StorageStyle('TMED'), - MP4StorageStyle("----:com.apple.iTunes:MEDIA"), + MP4StorageStyle(b"----:com.apple.iTunes:MEDIA"), StorageStyle('MEDIA'), ASFStorageStyle('WM/Media'), ) albumdisambig = MediaField( # This tag mapping was invented for beets (not used by Picard, etc). MP3DescStorageStyle(u'MusicBrainz Album Comment'), - MP4StorageStyle("----:com.apple.iTunes:MusicBrainz Album Comment"), + MP4StorageStyle(b"----:com.apple.iTunes:MusicBrainz Album Comment"), StorageStyle('MUSICBRAINZ_ALBUMCOMMENT'), ASFStorageStyle('MusicBrainz/Album Comment'), ) @@ -1458,7 +1741,7 @@ class MediaFile(object): # Release date. date = DateField( MP3StorageStyle('TDRC'), - MP4StorageStyle("\xa9day"), + MP4StorageStyle(b"\xa9day"), StorageStyle('DATE'), ASFStorageStyle('WM/Year'), year=(StorageStyle('YEAR'),)) @@ -1470,7 +1753,7 @@ class MediaFile(object): # *Original* release date. original_date = DateField( MP3StorageStyle('TDOR'), - MP4StorageStyle('----:com.apple.iTunes:ORIGINAL YEAR'), + MP4StorageStyle(b'----:com.apple.iTunes:ORIGINAL YEAR'), StorageStyle('ORIGINALDATE'), ASFStorageStyle('WM/OriginalReleaseYear')) @@ -1481,13 +1764,13 @@ class MediaFile(object): # Nonstandard metadata. artist_credit = MediaField( MP3DescStorageStyle(u'Artist Credit'), - MP4StorageStyle("----:com.apple.iTunes:Artist Credit"), + MP4StorageStyle(b"----:com.apple.iTunes:Artist Credit"), StorageStyle('ARTIST_CREDIT'), ASFStorageStyle('beets/Artist Credit'), ) albumartist_credit = MediaField( MP3DescStorageStyle(u'Album Artist Credit'), - MP4StorageStyle("----:com.apple.iTunes:Album Artist Credit"), + MP4StorageStyle(b"----:com.apple.iTunes:Album Artist Credit"), StorageStyle('ALBUMARTIST_CREDIT'), ASFStorageStyle('beets/Album Artist Credit'), ) @@ -1501,31 +1784,31 @@ class MediaFile(object): # MusicBrainz IDs. mb_trackid = MediaField( MP3UFIDStorageStyle(owner='http://musicbrainz.org'), - MP4StorageStyle('----:com.apple.iTunes:MusicBrainz Track Id'), + MP4StorageStyle(b'----:com.apple.iTunes:MusicBrainz Track Id'), StorageStyle('MUSICBRAINZ_TRACKID'), ASFStorageStyle('MusicBrainz/Track Id'), ) mb_albumid = MediaField( MP3DescStorageStyle(u'MusicBrainz Album Id'), - MP4StorageStyle('----:com.apple.iTunes:MusicBrainz Album Id'), + MP4StorageStyle(b'----:com.apple.iTunes:MusicBrainz Album Id'), StorageStyle('MUSICBRAINZ_ALBUMID'), ASFStorageStyle('MusicBrainz/Album Id'), ) mb_artistid = MediaField( MP3DescStorageStyle(u'MusicBrainz Artist Id'), - MP4StorageStyle('----:com.apple.iTunes:MusicBrainz Artist Id'), + MP4StorageStyle(b'----:com.apple.iTunes:MusicBrainz Artist Id'), StorageStyle('MUSICBRAINZ_ARTISTID'), ASFStorageStyle('MusicBrainz/Artist Id'), ) mb_albumartistid = MediaField( MP3DescStorageStyle(u'MusicBrainz Album Artist Id'), - MP4StorageStyle('----:com.apple.iTunes:MusicBrainz Album Artist Id'), + MP4StorageStyle(b'----:com.apple.iTunes:MusicBrainz Album Artist Id'), StorageStyle('MUSICBRAINZ_ALBUMARTISTID'), ASFStorageStyle('MusicBrainz/Album Artist Id'), ) mb_releasegroupid = MediaField( MP3DescStorageStyle(u'MusicBrainz Release Group Id'), - MP4StorageStyle('----:com.apple.iTunes:MusicBrainz Release Group Id'), + MP4StorageStyle(b'----:com.apple.iTunes:MusicBrainz Release Group Id'), StorageStyle('MUSICBRAINZ_RELEASEGROUPID'), ASFStorageStyle('MusicBrainz/Release Group Id'), ) @@ -1533,79 +1816,124 @@ class MediaFile(object): # Acoustid fields. acoustid_fingerprint = MediaField( MP3DescStorageStyle(u'Acoustid Fingerprint'), - MP4StorageStyle('----:com.apple.iTunes:Acoustid Fingerprint'), + MP4StorageStyle(b'----:com.apple.iTunes:Acoustid Fingerprint'), StorageStyle('ACOUSTID_FINGERPRINT'), ASFStorageStyle('Acoustid/Fingerprint'), ) acoustid_id = MediaField( MP3DescStorageStyle(u'Acoustid Id'), - MP4StorageStyle('----:com.apple.iTunes:Acoustid Id'), + MP4StorageStyle(b'----:com.apple.iTunes:Acoustid Id'), StorageStyle('ACOUSTID_ID'), ASFStorageStyle('Acoustid/Id'), ) # ReplayGain fields. rg_track_gain = MediaField( - MP3DescStorageStyle(u'REPLAYGAIN_TRACK_GAIN', - float_places=2, suffix=u' dB'), - MP3DescStorageStyle(u'replaygain_track_gain', - float_places=2, suffix=u' dB'), - MP3SoundCheckStorageStyle(key='COMM', index=0, desc=u'iTunNORM', - id3_lang='eng'), - MP4StorageStyle(key='----:com.apple.iTunes:replaygain_track_gain', - float_places=2, suffix=b' dB'), - MP4SoundCheckStorageStyle(key='----:com.apple.iTunes:iTunNORM', - index=0), - StorageStyle(u'REPLAYGAIN_TRACK_GAIN', - float_places=2, suffix=u' dB'), - ASFStorageStyle(u'replaygain_track_gain', - float_places=2, suffix=u' dB'), + MP3DescStorageStyle( + u'REPLAYGAIN_TRACK_GAIN', + float_places=2, suffix=u' dB' + ), + MP3DescStorageStyle( + u'replaygain_track_gain', + float_places=2, suffix=u' dB' + ), + MP3SoundCheckStorageStyle( + key='COMM', + index=0, desc=u'iTunNORM', + id3_lang='eng' + ), + MP4StorageStyle( + b'----:com.apple.iTunes:replaygain_track_gain', + float_places=2, suffix=b' dB' + ), + MP4SoundCheckStorageStyle( + b'----:com.apple.iTunes:iTunNORM', + index=0 + ), + StorageStyle( + u'REPLAYGAIN_TRACK_GAIN', + float_places=2, suffix=u' dB' + ), + ASFStorageStyle( + u'replaygain_track_gain', + float_places=2, suffix=u' dB' + ), out_type=float ) rg_album_gain = MediaField( - MP3DescStorageStyle(u'REPLAYGAIN_ALBUM_GAIN', - float_places=2, suffix=u' dB'), - MP3DescStorageStyle(u'replaygain_album_gain', - float_places=2, suffix=u' dB'), - MP4SoundCheckStorageStyle(key='----:com.apple.iTunes:iTunNORM', - index=1), - StorageStyle(u'REPLAYGAIN_ALBUM_GAIN', - float_places=2, suffix=u' dB'), - ASFStorageStyle(u'replaygain_album_gain', - float_places=2, suffix=u' dB'), + MP3DescStorageStyle( + u'REPLAYGAIN_ALBUM_GAIN', + float_places=2, suffix=u' dB' + ), + MP3DescStorageStyle( + u'replaygain_album_gain', + float_places=2, suffix=u' dB' + ), + MP4SoundCheckStorageStyle( + b'----:com.apple.iTunes:iTunNORM', + index=1 + ), + StorageStyle( + u'REPLAYGAIN_ALBUM_GAIN', + float_places=2, suffix=u' dB' + ), + ASFStorageStyle( + u'replaygain_album_gain', + float_places=2, suffix=u' dB' + ), out_type=float ) rg_track_peak = MediaField( - MP3DescStorageStyle(u'REPLAYGAIN_TRACK_PEAK', - float_places=6), - MP3DescStorageStyle(u'replaygain_track_peak', - float_places=6), - MP3SoundCheckStorageStyle(key='COMM', index=1, desc=u'iTunNORM', - id3_lang='eng'), - MP4StorageStyle('----:com.apple.iTunes:replaygain_track_peak', - float_places=6), - MP4SoundCheckStorageStyle(key='----:com.apple.iTunes:iTunNORM', - index=1), - StorageStyle(u'REPLAYGAIN_TRACK_PEAK', - float_places=6), - ASFStorageStyle(u'replaygain_track_peak', - float_places=6), + MP3DescStorageStyle( + u'REPLAYGAIN_TRACK_PEAK', + float_places=6 + ), + MP3DescStorageStyle( + u'replaygain_track_peak', + float_places=6 + ), + MP3SoundCheckStorageStyle( + key=u'COMM', + index=1, desc=u'iTunNORM', + id3_lang='eng' + ), + MP4StorageStyle( + b'----:com.apple.iTunes:replaygain_track_peak', + float_places=6 + ), + MP4SoundCheckStorageStyle( + b'----:com.apple.iTunes:iTunNORM', + index=1 + ), + StorageStyle(u'REPLAYGAIN_TRACK_PEAK', float_places=6), + ASFStorageStyle(u'replaygain_track_peak', float_places=6), out_type=float, ) rg_album_peak = MediaField( - MP3DescStorageStyle(u'REPLAYGAIN_ALBUM_PEAK', - float_places=6), - MP3DescStorageStyle(u'replaygain_album_peak', - float_places=6), - MP4StorageStyle('----:com.apple.iTunes:replaygain_album_peak', - float_places=6), - StorageStyle(u'REPLAYGAIN_ALBUM_PEAK', - float_places=6), - ASFStorageStyle(u'replaygain_album_peak', - float_places=6), + MP3DescStorageStyle( + u'REPLAYGAIN_ALBUM_PEAK', + float_places=6 + ), + MP3DescStorageStyle( + u'replaygain_album_peak', + float_places=6 + ), + MP4StorageStyle( + b'----:com.apple.iTunes:replaygain_album_peak', + float_places=6 + ), + StorageStyle(u'REPLAYGAIN_ALBUM_PEAK', float_places=6), + ASFStorageStyle(u'replaygain_album_peak', float_places=6), out_type=float, ) + initial_key = MediaField( + MP3StorageStyle('TKEY'), + MP4StorageStyle(b'----:com.apple.iTunes:initialkey'), + StorageStyle('INITIALKEY'), + ASFStorageStyle('INITIALKEY'), + ) + @property def length(self): """The duration of the audio in seconds (a float).""" diff --git a/libs/beets/plugins.py b/libs/beets/plugins.py index 6a58777c..239f64fb 100644 --- a/libs/beets/plugins.py +++ b/libs/beets/plugins.py @@ -1,5 +1,6 @@ +# -*- coding: utf-8 -*- # This file is part of beets. -# Copyright 2013, Adrian Sampson. +# Copyright 2016, Adrian Sampson. # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -14,11 +15,17 @@ """Support for beets plugins.""" -import logging +from __future__ import division, absolute_import, print_function + +import inspect import traceback +import re from collections import defaultdict +from functools import wraps + import beets +from beets import logging from beets import mediafile PLUGIN_NAMESPACE = 'beetsplug' @@ -30,6 +37,31 @@ LASTFM_KEY = '2dc3914abf35f0d9c92d97d8f8e42b43' log = logging.getLogger('beets') +class PluginConflictException(Exception): + """Indicates that the services provided by one plugin conflict with + those of another. + + For example two plugins may define different types for flexible fields. + """ + + +class PluginLogFilter(logging.Filter): + """A logging filter that identifies the plugin that emitted a log + message. + """ + def __init__(self, plugin): + self.prefix = u'{0}: '.format(plugin.name) + + def filter(self, record): + if hasattr(record.msg, 'msg') and isinstance(record.msg.msg, + basestring): + # A _LogMessage from our hacked-up Logging replacement. + record.msg.msg = self.prefix + record.msg.msg + elif isinstance(record.msg, basestring): + record.msg = self.prefix + record.msg + return True + + # Managing the plugins themselves. class BeetsPlugin(object): @@ -40,8 +72,6 @@ class BeetsPlugin(object): def __init__(self, name=None): """Perform one-time plugin setup. """ - _add_media_fields(self.item_fields()) - self.import_stages = [] self.name = name or self.__module__.split('.')[-1] self.config = beets.config[self.name] if not self.template_funcs: @@ -50,6 +80,12 @@ class BeetsPlugin(object): self.template_fields = {} if not self.album_template_fields: self.album_template_fields = {} + self.import_stages = [] + + self._log = log.getChild(self.name) + self._log.setLevel(logging.NOTSET) # Use `beets` logger level. + if not any(isinstance(f, PluginLogFilter) for f in self._log.filters): + self._log.addFilter(PluginLogFilter(self)) def commands(self): """Should return a list of beets.ui.Subcommand objects for @@ -57,6 +93,46 @@ class BeetsPlugin(object): """ return () + def get_import_stages(self): + """Return a list of functions that should be called as importer + pipelines stages. + + The callables are wrapped versions of the functions in + `self.import_stages`. Wrapping provides some bookkeeping for the + plugin: specifically, the logging level is adjusted to WARNING. + """ + return [self._set_log_level_and_params(logging.WARNING, import_stage) + for import_stage in self.import_stages] + + def _set_log_level_and_params(self, base_log_level, func): + """Wrap `func` to temporarily set this plugin's logger level to + `base_log_level` + config options (and restore it to its previous + value after the function returns). Also determines which params may not + be sent for backwards-compatibility. + """ + argspec = inspect.getargspec(func) + + @wraps(func) + def wrapper(*args, **kwargs): + assert self._log.level == logging.NOTSET + verbosity = beets.config['verbose'].get(int) + log_level = max(logging.DEBUG, base_log_level - 10 * verbosity) + self._log.setLevel(log_level) + try: + try: + return func(*args, **kwargs) + except TypeError as exc: + if exc.args[0].startswith(func.__name__): + # caused by 'func' and not stuff internal to 'func' + kwargs = dict((arg, val) for arg, val in kwargs.items() + if arg in argspec.args) + return func(*args, **kwargs) + else: + raise + finally: + self._log.setLevel(logging.NOTSET) + return wrapper + def queries(self): """Should return a dict mapping prefixes to Query subclasses. """ @@ -86,14 +162,6 @@ class BeetsPlugin(object): """ return () - def item_fields(self): - """Returns field descriptors to be added to the MediaFile class, - in the form of a dictionary whose keys are field names and whose - values are descriptor (e.g., MediaField) instances. The Library - database schema is not (currently) extended. - """ - return {} - def album_for_id(self, album_id): """Return an AlbumInfo object or None if no matching release was found. @@ -106,38 +174,36 @@ class BeetsPlugin(object): """ return None + def add_media_field(self, name, descriptor): + """Add a field that is synchronized between media files and items. + When a media field is added ``item.write()`` will set the name + property of the item's MediaFile to ``item[name]`` and save the + changes. Similarly ``item.read()`` will set ``item[name]`` to + the value of the name property of the media file. + + ``descriptor`` must be an instance of ``mediafile.MediaField``. + """ + # Defer impor to prevent circular dependency + from beets import library + mediafile.MediaFile.add_field(name, descriptor) + library.Item._media_fields.add(name) + + _raw_listeners = None listeners = None - @classmethod - def register_listener(cls, event, func): - """Add a function as a listener for the specified event. (An - imperative alternative to the @listen decorator.) + def register_listener(self, event, func): + """Add a function as a listener for the specified event. """ - if cls.listeners is None: + wrapped_func = self._set_log_level_and_params(logging.WARNING, func) + + cls = self.__class__ + if cls.listeners is None or cls._raw_listeners is None: + cls._raw_listeners = defaultdict(list) cls.listeners = defaultdict(list) - cls.listeners[event].append(func) - - @classmethod - def listen(cls, event): - """Decorator that adds a function as an event handler for the - specified event (as a string). The parameters passed to function - will vary depending on what event occurred. - - The function should respond to named parameters. - function(**kwargs) will trap all arguments in a dictionary. - Example: - - >>> @MyPlugin.listen("imported") - >>> def importListener(**kwargs): - >>> pass - """ - def helper(func): - if cls.listeners is None: - cls.listeners = defaultdict(list) - cls.listeners[event].append(func) - return func - return helper + if func not in cls._raw_listeners[event]: + cls._raw_listeners[event].append(func) + cls.listeners[event].append(wrapped_func) template_funcs = None template_fields = None @@ -170,7 +236,10 @@ class BeetsPlugin(object): return func return helper + _classes = set() + + def load_plugins(names=()): """Imports the modules for a sequence of plugin names. Each name must be the name of a Python module under the "beetsplug" namespace @@ -178,14 +247,14 @@ def load_plugins(names=()): BeetsPlugin subclasses desired. """ for name in names: - modname = '%s.%s' % (PLUGIN_NAMESPACE, name) + modname = '{0}.{1}'.format(PLUGIN_NAMESPACE, name) try: try: namespace = __import__(modname, None, None) except ImportError as exc: # Again, this is hacky: if exc.args[0].endswith(' ' + name): - log.warn('** plugin %s not found' % name) + log.warn(u'** plugin {0} not found', name) else: raise else: @@ -195,10 +264,16 @@ def load_plugins(names=()): _classes.add(obj) except: - log.warn('** error loading plugin %s' % name) - log.warn(traceback.format_exc()) + log.warn( + u'** error loading plugin {}:\n{}', + name, + traceback.format_exc(), + ) + _instances = {} + + def find_plugins(): """Returns a list of BeetsPlugin subclass instances from all currently loaded beets plugins. Loads the default plugin set @@ -224,6 +299,7 @@ def commands(): out += plugin.commands() return out + def queries(): """Returns a dict mapping prefix strings to Query subclasses all loaded plugins. @@ -233,6 +309,24 @@ def queries(): out.update(plugin.queries()) return out + +def types(model_cls): + # Gives us `item_types` and `album_types` + attr_name = '{0}_types'.format(model_cls.__name__.lower()) + types = {} + for plugin in find_plugins(): + plugin_types = getattr(plugin, attr_name, {}) + for field in plugin_types: + if field in types and plugin_types[field] != types[field]: + raise PluginConflictException( + u'Plugin {0} defines flexible field {1} ' + u'which has already been defined with ' + u'another type.'.format(plugin.name, field) + ) + types.update(plugin_types) + return types + + def track_distance(item, info): """Gets the track distance calculated by all loaded plugins. Returns a Distance object. @@ -243,6 +337,7 @@ def track_distance(item, info): dist.update(plugin.track_distance(item, info)) return dist + def album_distance(items, album_info, mapping): """Returns the album distance calculated by plugins.""" from beets.autotag.hooks import Distance @@ -251,6 +346,7 @@ def album_distance(items, album_info, mapping): dist.update(plugin.album_distance(items, album_info, mapping)) return dist + def candidates(items, artist, album, va_likely): """Gets MusicBrainz candidates for an album from each plugin. """ @@ -259,6 +355,7 @@ def candidates(items, artist, album, va_likely): out.extend(plugin.candidates(items, artist, album, va_likely)) return out + def item_candidates(item, artist, title): """Gets MusicBrainz candidates for an item from the plugins. """ @@ -267,6 +364,7 @@ def item_candidates(item, artist, title): out.extend(plugin.item_candidates(item, artist, title)) return out + def album_for_id(album_id): """Get AlbumInfo objects for a given ID string. """ @@ -277,6 +375,7 @@ def album_for_id(album_id): out.append(res) return out + def track_for_id(track_id): """Get TrackInfo objects for a given ID string. """ @@ -287,6 +386,7 @@ def track_for_id(track_id): out.append(res) return out + def template_funcs(): """Get all the template functions declared by plugins as a dictionary. @@ -297,19 +397,12 @@ def template_funcs(): funcs.update(plugin.template_funcs) return funcs -def _add_media_fields(fields): - """Adds a {name: descriptor} dictionary of fields to the MediaFile - class. Called during the plugin initialization. - """ - for key, value in fields.iteritems(): - setattr(mediafile.MediaFile, key, value) def import_stages(): """Get a list of import stage functions defined by plugins.""" stages = [] for plugin in find_plugins(): - if hasattr(plugin, 'import_stages'): - stages += plugin.import_stages + stages += plugin.get_import_stages() return stages @@ -325,6 +418,7 @@ def item_field_getters(): funcs.update(plugin.template_fields) return funcs + def album_field_getters(): """As above, for album fields. """ @@ -348,12 +442,48 @@ def event_handlers(): all_handlers[event] += handlers return all_handlers -def send(event, **arguments): - """Sends an event to all assigned event listeners. Event is the - name of the event to send, all other named arguments go to the - event handler(s). - Returns a list of return values from the handlers. +def send(event, **arguments): + """Send an event to all assigned event listeners. + + `event` is the name of the event to send, all other named arguments + are passed along to the handlers. + + Return a list of non-None values returned from the handlers. """ - log.debug('Sending event: %s' % event) - return [handler(**arguments) for handler in event_handlers()[event]] + log.debug(u'Sending event: {0}', event) + results = [] + for handler in event_handlers()[event]: + result = handler(**arguments) + if result is not None: + results.append(result) + return results + + +def feat_tokens(for_artist=True): + """Return a regular expression that matches phrases like "featuring" + that separate a main artist or a song title from secondary artists. + The `for_artist` option determines whether the regex should be + suitable for matching artist fields (the default) or title fields. + """ + feat_words = ['ft', 'featuring', 'feat', 'feat.', 'ft.'] + if for_artist: + feat_words += ['with', 'vs', 'and', 'con', '&'] + return '(?<=\s)(?:{0})(?=\s)'.format( + '|'.join(re.escape(x) for x in feat_words) + ) + + +def sanitize_choices(choices, choices_all): + """Clean up a stringlist configuration attribute: keep only choices + elements present in choices_all, remove duplicate elements, expand '*' + wildcard while keeping original stringlist order. + """ + seen = set() + others = [x for x in choices_all if x not in choices] + res = [] + for s in choices: + if s in list(choices_all) + ['*']: + if not (s in seen or seen.add(s)): + res.extend(list(others) if s == '*' else [s]) + return res diff --git a/libs/beets/ui/__init__.py b/libs/beets/ui/__init__.py index 2df74ea7..797df44d 100644 --- a/libs/beets/ui/__init__.py +++ b/libs/beets/ui/__init__.py @@ -1,5 +1,6 @@ +# -*- coding: utf-8 -*- # This file is part of beets. -# Copyright 2014, Adrian Sampson. +# Copyright 2016, Adrian Sampson. # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -16,20 +17,22 @@ interface. To invoke the CLI, just call beets.ui.main(). The actual CLI commands are implemented in the ui.commands module. """ -from __future__ import print_function + +from __future__ import division, absolute_import, print_function import locale import optparse import textwrap import sys from difflib import SequenceMatcher -import logging import sqlite3 import errno import re import struct import traceback +import os.path +from beets import logging from beets import library from beets import plugins from beets import util @@ -37,10 +40,9 @@ from beets.util.functemplate import Template from beets import config from beets.util import confit from beets.autotag import mb - +from beets.dbcore import query as db_query # On Windows platforms, use colorama to support "ANSI" terminal colors. - if sys.platform == 'win32': try: import colorama @@ -50,36 +52,59 @@ if sys.platform == 'win32': colorama.init() - -# Constants. +log = logging.getLogger('beets') +if not log.handlers: + log.addHandler(logging.StreamHandler()) +log.propagate = False # Don't propagate to root handler. PF_KEY_QUERIES = { - 'comp': 'comp:true', - 'singleton': 'singleton:true', + 'comp': u'comp:true', + 'singleton': u'singleton:true', } -# UI exception. Commands should throw this in order to display -# nonrecoverable errors to the user. + class UserError(Exception): - pass - -# Main logger. -log = logging.getLogger('beets') + """UI exception. Commands should throw this in order to display + nonrecoverable errors to the user. + """ - -# Utilities. +# Encoding utilities. -def _encoding(): - """Tries to guess the encoding used by the terminal.""" +def _in_encoding(default=u'utf-8'): + """Get the encoding to use for *inputting* strings from the console. + + :param default: the fallback sys.stdin encoding + """ + + return config['terminal_encoding'].get() or getattr(sys.stdin, 'encoding', + default) + + +def _out_encoding(): + """Get the encoding to use for *outputting* strings to the console. + """ # Configured override? encoding = config['terminal_encoding'].get() if encoding: return encoding - # Determine from locale settings. + # For testing: When sys.stdout is a StringIO under the test harness, + # it doesn't have an `encoding` attribute. Just use UTF-8. + if not hasattr(sys.stdout, 'encoding'): + return 'utf8' + + # Python's guessed output stream encoding, or UTF-8 as a fallback + # (e.g., when piped to a file). + return sys.stdout.encoding or 'utf8' + + +def _arg_encoding(): + """Get the encoding for command-line arguments (and other OS + locale-sensitive strings). + """ try: return locale.getdefaultlocale()[1] or 'utf8' except ValueError: @@ -92,25 +117,75 @@ def decargs(arglist): """Given a list of command-line argument bytestrings, attempts to decode them to Unicode strings. """ - return [s.decode(_encoding()) for s in arglist] + return [s.decode(_arg_encoding()) for s in arglist] -def print_(*strings): +def print_(*strings, **kwargs): """Like print, but rather than raising an error when a character is not in the terminal's encoding's character set, just silently replaces it. - """ - if strings: - if isinstance(strings[0], unicode): - txt = u' '.join(strings) - else: - txt = ' '.join(strings) - else: - txt = u'' - if isinstance(txt, unicode): - txt = txt.encode(_encoding(), 'replace') - print(txt) + If the arguments are strings then they're expected to share the same + type: either bytes or unicode. + + The `end` keyword argument behaves similarly to the built-in `print` + (it defaults to a newline). The value should have the same string + type as the arguments. + """ + end = kwargs.get('end') + + if not strings or isinstance(strings[0], unicode): + txt = u' '.join(strings) + txt += u'\n' if end is None else end + else: + txt = b' '.join(strings) + txt += b'\n' if end is None else end + + # Always send bytes to the stdout stream. + if isinstance(txt, unicode): + txt = txt.encode(_out_encoding(), 'replace') + + sys.stdout.write(txt) + + +# Configuration wrappers. + +def _bool_fallback(a, b): + """Given a boolean or None, return the original value or a fallback. + """ + if a is None: + assert isinstance(b, bool) + return b + else: + assert isinstance(a, bool) + return a + + +def should_write(write_opt=None): + """Decide whether a command that updates metadata should also write + tags, using the importer configuration as the default. + """ + return _bool_fallback(write_opt, config['import']['write'].get(bool)) + + +def should_move(move_opt=None): + """Decide whether a command that updates metadata should also move + files when they're inside the library, using the importer + configuration as the default. + + Specifically, commands should move files after metadata updates only + when the importer is configured *either* to move *or* to copy files. + They should avoid moving files when the importer is configured not + to touch any filenames. + """ + return _bool_fallback( + move_opt, + config['import']['move'].get(bool) or + config['import']['copy'].get(bool) + ) + + +# Input prompts. def input_(prompt=None): """Like `raw_input`, but decodes the result to a Unicode string. @@ -122,16 +197,14 @@ def input_(prompt=None): # use print() explicitly to display prompts. # http://bugs.python.org/issue1927 if prompt: - if isinstance(prompt, unicode): - prompt = prompt.encode(_encoding(), 'replace') - print(prompt, end=' ') + print_(prompt, end=' ') try: resp = raw_input() except EOFError: - raise UserError('stdin stream ended while input required') + raise UserError(u'stdin stream ended while input required') - return resp.decode(sys.stdin.encoding or 'utf8', 'ignore') + return resp.decode(_in_encoding(), 'ignore') def input_options(options, require=False, prompt=None, fallback_prompt=None, @@ -170,20 +243,21 @@ def input_options(options, require=False, prompt=None, fallback_prompt=None, # Infer a letter. for letter in option: if not letter.isalpha(): - continue # Don't use punctuation. + continue # Don't use punctuation. if letter not in letters: found_letter = letter break else: - raise ValueError('no unambiguous lettering found') + raise ValueError(u'no unambiguous lettering found') letters[found_letter.lower()] = option index = option.index(found_letter) # Mark the option's shortcut letter for display. - if not require and ((default is None and not numrange and first) or - (isinstance(default, basestring) and - found_letter.lower() == default.lower())): + if not require and ( + (default is None and not numrange and first) or + (isinstance(default, basestring) and + found_letter.lower() == default.lower())): # The first option is the default; mark it. show_letter = '[%s]' % found_letter.upper() is_default = True @@ -192,7 +266,7 @@ def input_options(options, require=False, prompt=None, fallback_prompt=None, is_default = False # Colorize the letter shortcut. - show_letter = colorize('turquoise' if is_default else 'blue', + show_letter = colorize('action_default' if is_default else 'action', show_letter) # Insert the highlighted letter back into the word. @@ -218,11 +292,11 @@ def input_options(options, require=False, prompt=None, fallback_prompt=None, prompt_part_lengths = [] if numrange: if isinstance(default, int): - default_name = str(default) - default_name = colorize('turquoise', default_name) + default_name = unicode(default) + default_name = colorize('action_default', default_name) tmpl = '# selection (default %s)' prompt_parts.append(tmpl % default_name) - prompt_part_lengths.append(len(tmpl % str(default))) + prompt_part_lengths.append(len(tmpl % unicode(default))) else: prompt_parts.append('# selection') prompt_part_lengths.append(len(prompt_parts[-1])) @@ -257,9 +331,9 @@ def input_options(options, require=False, prompt=None, fallback_prompt=None, # Make a fallback prompt too. This is displayed if the user enters # something that is not recognized. if not fallback_prompt: - fallback_prompt = 'Enter one of ' + fallback_prompt = u'Enter one of ' if numrange: - fallback_prompt += '%i-%i, ' % numrange + fallback_prompt += u'%i-%i, ' % numrange fallback_prompt += ', '.join(display_letters) + ':' resp = input_(prompt) @@ -298,19 +372,52 @@ def input_yn(prompt, require=False): "yes" unless `require` is `True`, in which case there is no default. """ sel = input_options( - ('y', 'n'), require, prompt, 'Enter Y or N:' + ('y', 'n'), require, prompt, u'Enter Y or N:' ) - return sel == 'y' + return sel == u'y' +def input_select_objects(prompt, objs, rep): + """Prompt to user to choose all, none, or some of the given objects. + Return the list of selected objects. + + `prompt` is the prompt string to use for each question (it should be + phrased as an imperative verb). `rep` is a function to call on each + object to print it out when confirming objects individually. + """ + choice = input_options( + (u'y', u'n', u's'), False, + u'%s? (Yes/no/select)' % prompt) + print() # Blank line. + + if choice == u'y': # Yes. + return objs + + elif choice == u's': # Select. + out = [] + for obj in objs: + rep(obj) + if input_yn(u'%s? (yes/no)' % prompt, True): + out.append(obj) + print() # go to a new line + return out + + else: # No. + return [] + + +# Human output formatting. + def human_bytes(size): """Formats size, a number of bytes, in a human-readable way.""" - suffices = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB', 'HB'] - for suffix in suffices: + powers = [u'', u'K', u'M', u'G', u'T', u'P', u'E', u'Z', u'Y', u'H'] + unit = 'B' + for power in powers: if size < 1024: - return "%3.1f %s" % (size, suffix) + return u"%3.1f %s%s" % (size, power, unit) size /= 1024.0 - return "big" + unit = u'iB' + return u"big" def human_seconds(interval): @@ -318,13 +425,13 @@ def human_seconds(interval): interval using English words. """ units = [ - (1, 'second'), - (60, 'minute'), - (60, 'hour'), - (24, 'day'), - (7, 'week'), - (52, 'year'), - (10, 'decade'), + (1, u'second'), + (60, u'minute'), + (60, u'hour'), + (24, u'day'), + (7, u'week'), + (52, u'year'), + (10, u'decade'), ] for i in range(len(units) - 1): increment, suffix = units[i] @@ -337,7 +444,7 @@ def human_seconds(interval): increment, suffix = units[-1] interval /= float(increment) - return "%3.1f %ss" % (interval, suffix) + return u"%3.1f %ss" % (interval, suffix) def human_seconds_short(interval): @@ -348,40 +455,82 @@ def human_seconds_short(interval): return u'%i:%02i' % (interval // 60, interval % 60) +# Colorization. + # ANSI terminal colorization code heavily inspired by pygments: # http://dev.pocoo.org/hg/pygments-main/file/b2deea5b5030/pygments/console.py # (pygments is by Tim Hatch, Armin Ronacher, et al.) COLOR_ESCAPE = "\x1b[" -DARK_COLORS = ["black", "darkred", "darkgreen", "brown", "darkblue", - "purple", "teal", "lightgray"] -LIGHT_COLORS = ["darkgray", "red", "green", "yellow", "blue", - "fuchsia", "turquoise", "white"] +DARK_COLORS = { + "black": 0, + "darkred": 1, + "darkgreen": 2, + "brown": 3, + "darkyellow": 3, + "darkblue": 4, + "purple": 5, + "darkmagenta": 5, + "teal": 6, + "darkcyan": 6, + "lightgray": 7 +} +LIGHT_COLORS = { + "darkgray": 0, + "red": 1, + "green": 2, + "yellow": 3, + "blue": 4, + "fuchsia": 5, + "magenta": 5, + "turquoise": 6, + "cyan": 6, + "white": 7 +} RESET_COLOR = COLOR_ESCAPE + "39;49;00m" + +# These abstract COLOR_NAMES are lazily mapped on to the actual color in COLORS +# as they are defined in the configuration files, see function: colorize +COLOR_NAMES = ['text_success', 'text_warning', 'text_error', 'text_highlight', + 'text_highlight_minor', 'action_default', 'action'] +COLORS = None + + def _colorize(color, text): """Returns a string that prints the given text in the given color in a terminal that is ANSI color-aware. The color must be something in DARK_COLORS or LIGHT_COLORS. """ if color in DARK_COLORS: - escape = COLOR_ESCAPE + "%im" % (DARK_COLORS.index(color) + 30) + escape = COLOR_ESCAPE + "%im" % (DARK_COLORS[color] + 30) elif color in LIGHT_COLORS: - escape = COLOR_ESCAPE + "%i;01m" % (LIGHT_COLORS.index(color) + 30) + escape = COLOR_ESCAPE + "%i;01m" % (LIGHT_COLORS[color] + 30) else: - raise ValueError('no such color %s', color) + raise ValueError(u'no such color %s', color) return escape + text + RESET_COLOR -def colorize(color, text): +def colorize(color_name, text): """Colorize text if colored output is enabled. (Like _colorize but conditional.) """ - if config['color']: + if config['ui']['color']: + global COLORS + if not COLORS: + COLORS = dict((name, config['ui']['colors'][name].get(unicode)) + for name in COLOR_NAMES) + # In case a 3rd party plugin is still passing the actual color ('red') + # instead of the abstract color name ('text_error') + color = COLORS.get(color_name) + if not color: + log.debug(u'Invalid color_name: {0}', color_name) + color = color_name return _colorize(color, text) else: return text -def _colordiff(a, b, highlight='red', minor_highlight='lightgray'): +def _colordiff(a, b, highlight='text_highlight', + minor_highlight='text_highlight_minor'): """Given two values, return the same pair of strings except with their differences highlighted in the specified color. Strings are highlighted intelligently to show differences; other values are @@ -431,40 +580,16 @@ def _colordiff(a, b, highlight='red', minor_highlight='lightgray'): return u''.join(a_out), u''.join(b_out) -def colordiff(a, b, highlight='red'): +def colordiff(a, b, highlight='text_highlight'): """Colorize differences between two values if color is enabled. (Like _colordiff but conditional.) """ - if config['color']: + if config['ui']['color']: return _colordiff(a, b, highlight) else: return unicode(a), unicode(b) -def color_diff_suffix(a, b, highlight='red'): - """Colorize the differing suffix between two strings.""" - a, b = unicode(a), unicode(b) - if not config['color']: - return a, b - - # Fast path. - if a == b: - return a, b - - # Find the longest common prefix. - first_diff = None - for i in range(min(len(a), len(b))): - if a[i] != b[i]: - first_diff = i - break - else: - first_diff = min(len(a), len(b)) - - # Colorize from the first difference on. - return a[:first_diff] + colorize(highlight, a[first_diff:]), \ - b[:first_diff] + colorize(highlight, b[first_diff:]) - - def get_path_formats(subview=None): """Get the configuration's path formats as a list of query/template pairs. @@ -494,46 +619,6 @@ def get_replacements(): return replacements -def get_plugin_paths(): - """Get the list of search paths for plugins from the config file. - The value for "pluginpath" may be a single string or a list of - strings. - """ - pluginpaths = config['pluginpath'].get() - if isinstance(pluginpaths, basestring): - pluginpaths = [pluginpaths] - if not isinstance(pluginpaths, list): - raise confit.ConfigTypeError( - u'pluginpath must be string or a list of strings' - ) - return map(util.normpath, pluginpaths) - - -def _pick_format(album, fmt=None): - """Pick a format string for printing Album or Item objects, - falling back to config options and defaults. - """ - if fmt: - return fmt - if album: - return config['list_format_album'].get(unicode) - else: - return config['list_format_item'].get(unicode) - - -def print_obj(obj, lib, fmt=None): - """Print an Album or Item object. If `fmt` is specified, use that - format string. Otherwise, use the configured template. - """ - album = isinstance(obj, library.Album) - fmt = _pick_format(album, fmt) - if isinstance(fmt, Template): - template = fmt - else: - template = Template(fmt) - print_(obj.evaluate_template(template)) - - def term_width(): """Get the width (columns) of the terminal.""" fallback = config['ui']['terminal_width'].get(int) @@ -558,6 +643,8 @@ def term_width(): FLOAT_EPSILON = 0.01 + + def _field_diff(field, old, new): """Given two Model objects, format their values for `field` and highlight changes among them. Return a human-readable string. If the @@ -574,15 +661,16 @@ def _field_diff(field, old, new): return None # Get formatted values for output. - oldstr = old._get_formatted(field) - newstr = new._get_formatted(field) + oldstr = old.formatted().get(field, u'') + newstr = new.formatted().get(field, u'') # For strings, highlight changes. For others, colorize the whole # thing. if isinstance(oldval, basestring): - oldstr, newstr = colordiff(oldval, newval) + oldstr, newstr = colordiff(oldval, newstr) else: - oldstr, newstr = colorize('red', oldstr), colorize('red', newstr) + oldstr = colorize('text_error', oldstr) + newstr = colorize('text_error', newstr) return u'{0} -> {1}'.format(oldstr, newstr) @@ -613,24 +701,178 @@ def show_model_changes(new, old=None, fields=None, always=False): # New fields. for field in set(new) - set(old): + if fields and field not in fields: + continue + changes.append(u' {0}: {1}'.format( field, - colorize('red', new._get_formatted(field)) + colorize('text_highlight', new.formatted()[field]) )) # Print changes. if changes or always: - print_obj(old, old._db) + print_(format(old)) if changes: print_(u'\n'.join(changes)) return bool(changes) +def show_path_changes(path_changes): + """Given a list of tuples (source, destination) that indicate the + path changes, log the changes as INFO-level output to the beets log. + The output is guaranteed to be unicode. + + Every pair is shown on a single line if the terminal width permits it, + else it is split over two lines. E.g., + + Source -> Destination + + vs. + + Source + -> Destination + """ + sources, destinations = zip(*path_changes) + + # Ensure unicode output + sources = list(map(util.displayable_path, sources)) + destinations = list(map(util.displayable_path, destinations)) + + # Calculate widths for terminal split + col_width = (term_width() - len(' -> ')) // 2 + max_width = len(max(sources + destinations, key=len)) + + if max_width > col_width: + # Print every change over two lines + for source, dest in zip(sources, destinations): + log.info(u'{0} \n -> {1}', source, dest) + else: + # Print every change on a single line, and add a header + title_pad = max_width - len('Source ') + len(' -> ') + + log.info(u'Source {0} Destination', ' ' * title_pad) + for source, dest in zip(sources, destinations): + pad = max_width - len(source) + log.info(u'{0} {1} -> {2}', source, ' ' * pad, dest) + + +class CommonOptionsParser(optparse.OptionParser, object): + """Offers a simple way to add common formatting options. + + Options available include: + - matching albums instead of tracks: add_album_option() + - showing paths instead of items/albums: add_path_option() + - changing the format of displayed items/albums: add_format_option() + + The last one can have several behaviors: + - against a special target + - with a certain format + - autodetected target with the album option + + Each method is fully documented in the related method. + """ + def __init__(self, *args, **kwargs): + super(CommonOptionsParser, self).__init__(*args, **kwargs) + self._album_flags = False + # this serves both as an indicator that we offer the feature AND allows + # us to check whether it has been specified on the CLI - bypassing the + # fact that arguments may be in any order + + def add_album_option(self, flags=('-a', '--album')): + """Add a -a/--album option to match albums instead of tracks. + + If used then the format option can auto-detect whether we're setting + the format for items or albums. + Sets the album property on the options extracted from the CLI. + """ + album = optparse.Option(*flags, action='store_true', + help=u'match albums instead of tracks') + self.add_option(album) + self._album_flags = set(flags) + + def _set_format(self, option, opt_str, value, parser, target=None, + fmt=None, store_true=False): + """Internal callback that sets the correct format while parsing CLI + arguments. + """ + if store_true: + setattr(parser.values, option.dest, True) + + value = fmt or value and unicode(value) or '' + parser.values.format = value + if target: + config[target._format_config_key].set(value) + else: + if self._album_flags: + if parser.values.album: + target = library.Album + else: + # the option is either missing either not parsed yet + if self._album_flags & set(parser.rargs): + target = library.Album + else: + target = library.Item + config[target._format_config_key].set(value) + else: + config[library.Item._format_config_key].set(value) + config[library.Album._format_config_key].set(value) + + def add_path_option(self, flags=('-p', '--path')): + """Add a -p/--path option to display the path instead of the default + format. + + By default this affects both items and albums. If add_album_option() + is used then the target will be autodetected. + + Sets the format property to u'$path' on the options extracted from the + CLI. + """ + path = optparse.Option(*flags, nargs=0, action='callback', + callback=self._set_format, + callback_kwargs={'fmt': '$path', + 'store_true': True}, + help=u'print paths for matched items or albums') + self.add_option(path) + + def add_format_option(self, flags=('-f', '--format'), target=None): + """Add -f/--format option to print some LibModel instances with a + custom format. + + `target` is optional and can be one of ``library.Item``, 'item', + ``library.Album`` and 'album'. + + Several behaviors are available: + - if `target` is given then the format is only applied to that + LibModel + - if the album option is used then the target will be autodetected + - otherwise the format is applied to both items and albums. + + Sets the format property on the options extracted from the CLI. + """ + kwargs = {} + if target: + if isinstance(target, basestring): + target = {'item': library.Item, + 'album': library.Album}[target] + kwargs['target'] = target + + opt = optparse.Option(*flags, action='callback', + callback=self._set_format, + callback_kwargs=kwargs, + help=u'print with custom format') + self.add_option(opt) + + def add_all_common_options(self): + """Add album, path and format options. + """ + self.add_album_option() + self.add_path_option() + self.add_format_option() + # Subcommand parsing infrastructure. - - +# # This is a fairly generic subcommand parser for optparse. It is # maintained externally here: # http://gist.github.com/462717 @@ -646,58 +888,68 @@ class Subcommand(object): the subcommand; aliases are alternate names. parser is an OptionParser responsible for parsing the subcommand's options. help is a short description of the command. If no parser is - given, it defaults to a new, empty OptionParser. + given, it defaults to a new, empty CommonOptionsParser. """ self.name = name - self.parser = parser or optparse.OptionParser() + self.parser = parser or CommonOptionsParser() self.aliases = aliases self.help = help self.hide = hide + self._root_parser = None -class SubcommandsOptionParser(optparse.OptionParser): + def print_help(self): + self.parser.print_help() + + def parse_args(self, args): + return self.parser.parse_args(args) + + @property + def root_parser(self): + return self._root_parser + + @root_parser.setter + def root_parser(self, root_parser): + self._root_parser = root_parser + self.parser.prog = '{0} {1}'.format( + root_parser.get_prog_name().decode('utf8'), self.name) + + +class SubcommandsOptionParser(CommonOptionsParser): """A variant of OptionParser that parses subcommands and their arguments. """ - # A singleton command used to give help on other subcommands. - _HelpSubcommand = Subcommand('help', optparse.OptionParser(), - help='give detailed help on a specific sub-command', - aliases=('?',)) def __init__(self, *args, **kwargs): """Create a new subcommand-aware option parser. All of the options to OptionParser.__init__ are supported in addition to subcommands, a sequence of Subcommand objects. """ - # The subcommand array, with the help command included. - self.subcommands = list(kwargs.pop('subcommands', [])) - self.subcommands.append(self._HelpSubcommand) - # A more helpful default usage. if 'usage' not in kwargs: - kwargs['usage'] = """ + kwargs['usage'] = u""" %prog COMMAND [ARGS...] %prog help COMMAND""" + kwargs['add_help_option'] = False # Super constructor. - optparse.OptionParser.__init__(self, *args, **kwargs) - - # Adjust the help-visible name of each subcommand. - for subcommand in self.subcommands: - subcommand.parser.prog = '%s %s' % \ - (self.get_prog_name(), subcommand.name) + super(SubcommandsOptionParser, self).__init__(*args, **kwargs) # Our root parser needs to stop on the first unrecognized argument. self.disable_interspersed_args() - def add_subcommand(self, cmd): + self.subcommands = [] + + def add_subcommand(self, *cmds): """Adds a Subcommand object to the parser's list of commands. """ - self.subcommands.append(cmd) + for cmd in cmds: + cmd.root_parser = self + self.subcommands.append(cmd) # Add the list of subcommands to the help message. def format_help(self, formatter=None): # Get the original help message, to which we will append. - out = optparse.OptionParser.format_help(self, formatter) + out = super(SubcommandsOptionParser, self).format_help(formatter) if formatter is None: formatter = self.formatter @@ -711,6 +963,7 @@ class SubcommandsOptionParser(optparse.OptionParser): disp_names = [] help_position = 0 subcommands = [c for c in self.subcommands if not c.hide] + subcommands.sort(key=lambda c: c.name) for subcommand in subcommands: name = subcommand.name if subcommand.aliases: @@ -736,7 +989,8 @@ class SubcommandsOptionParser(optparse.OptionParser): result.append(name) help_width = formatter.width - help_position help_lines = textwrap.wrap(subcommand.help, help_width) - result.append("%*s%s\n" % (indent_first, "", help_lines[0])) + help_line = help_lines[0] if help_lines else '' + result.append("%*s%s\n" % (indent_first, "", help_line)) result.extend(["%*s%s\n" % (help_position, "", line) for line in help_lines[1:]]) formatter.dedent() @@ -756,52 +1010,40 @@ class SubcommandsOptionParser(optparse.OptionParser): return subcommand return None - def parse_args(self, a=None, v=None): - """Like OptionParser.parse_args, but returns these four items: - - options: the options passed to the root parser - - subcommand: the Subcommand object that was invoked - - suboptions: the options passed to the subcommand parser - - subargs: the positional arguments passed to the subcommand + def parse_global_options(self, args): + """Parse options up to the subcommand argument. Returns a tuple + of the options object and the remaining arguments. """ - options, args = optparse.OptionParser.parse_args(self, a, v) - subcommand, suboptions, subargs = self._parse_sub(args) - return options, subcommand, suboptions, subargs + options, subargs = self.parse_args(args) - def _parse_sub(self, args): - """Given the `args` left unused by a typical OptionParser - `parse_args`, return the invoked subcommand, the subcommand - options, and the subcommand arguments. + # Force the help command + if options.help: + subargs = ['help'] + elif options.version: + subargs = ['version'] + return options, subargs + + def parse_subcommand(self, args): + """Given the `args` left unused by a `parse_global_options`, + return the invoked subcommand, the subcommand options, and the + subcommand arguments. """ + # Help is default command if not args: - # No command given. - self.print_help() - self.exit() - else: - cmdname = args.pop(0) - subcommand = self._subcommand_for_name(cmdname) - if not subcommand: - self.error('unknown command ' + cmdname) + args = ['help'] - suboptions, subargs = subcommand.parser.parse_args(args) - - if subcommand is self._HelpSubcommand: - if subargs: - # particular - cmdname = subargs[0] - helpcommand = self._subcommand_for_name(cmdname) - if not helpcommand: - self.error('no command named {0}'.format(cmdname)) - helpcommand.parser.print_help() - self.exit() - else: - # general - self.print_help() - self.exit() + cmdname = args.pop(0) + subcommand = self._subcommand_for_name(cmdname) + if not subcommand: + raise UserError(u"unknown command '{0}'".format(cmdname)) + suboptions, subargs = subcommand.parse_args(args) return subcommand, suboptions, subargs optparse.Option.ALWAYS_TYPED_ACTIONS += ('callback',) + + def vararg_callback(option, opt_str, value, parser): """Callback for an option with variable arguments. Manually collect arguments right of a callback-action @@ -838,53 +1080,55 @@ def vararg_callback(option, opt_str, value, parser): setattr(parser.values, option.dest, value) - # The main entry point and bootstrapping. - -def _load_plugins(): +def _load_plugins(config): """Load the plugins specified in the configuration. """ - # Add plugin paths. + paths = config['pluginpath'].get(confit.StrSeq(split=False)) + paths = map(util.normpath, paths) + log.debug(u'plugin paths: {0}', util.displayable_path(paths)) + import beetsplug - beetsplug.__path__ = get_plugin_paths() + beetsplug.__path__ - + beetsplug.__path__ = paths + beetsplug.__path__ # For backwards compatibility. - sys.path += get_plugin_paths() + sys.path += paths - # Load requested plugins. plugins.load_plugins(config['plugins'].as_str_seq()) plugins.send("pluginload") + return plugins -def _configure(args): - """Parse the command line, load configuration files (including - loading any indicated plugins), and return the invoked subcomand, - the subcommand options, and the subcommand arguments. +def _setup(options, lib=None): + """Prepare and global state and updates it with command line options. + + Returns a list of subcommands, a list of plugins, and a library instance. """ - # Temporary: Migrate from 1.0-style configuration. - from beets.ui import migrate - migrate.automigrate() + # Configure the MusicBrainz API. + mb.configure() + + config = _configure(options) + + plugins = _load_plugins(config) # Get the default subcommands. from beets.ui.commands import default_commands - # Construct the root parser. - commands = list(default_commands) - commands.append(migrate.migrate_cmd) # Temporary. - parser = SubcommandsOptionParser(subcommands=commands) - parser.add_option('-l', '--library', dest='library', - help='library database file to use') - parser.add_option('-d', '--directory', dest='directory', - help="destination music directory") - parser.add_option('-v', '--verbose', dest='verbose', action='store_true', - help='print debugging information') - parser.add_option('-c', '--config', dest='config', - help='path to configuration file') + subcommands = list(default_commands) + subcommands.extend(plugins.commands()) - # Parse the command-line! - options, args = optparse.OptionParser.parse_args(parser, args) + if lib is None: + lib = _open_library(config) + plugins.send("library_opened", lib=lib) + library.Item._types.update(plugins.types(library.Item)) + library.Album._types.update(plugins.types(library.Album)) + return subcommands, plugins, lib + + +def _configure(options): + """Amend the global configuration object with command line options. + """ # Add any additional config files specified with --config. This # special handling lets specified plugins get loaded before we # finish parsing the command line. @@ -894,22 +1138,50 @@ def _configure(args): config.set_file(config_path) config.set_args(options) - # Now add the plugin commands to the parser. - _load_plugins() - for cmd in plugins.commands(): - parser.add_subcommand(cmd) + # Configure the logger. + if config['verbose'].get(int): + log.set_global_level(logging.DEBUG) + else: + log.set_global_level(logging.INFO) - # Parse the remainder of the command line with loaded plugins. - return parser._parse_sub(args) + # Ensure compatibility with old (top-level) color configuration. + # Deprecation msg to motivate user to switch to config['ui']['color]. + if config['color'].exists(): + log.warning(u'Warning: top-level configuration of `color` ' + u'is deprecated. Configure color use under `ui`. ' + u'See documentation for more info.') + config['ui']['color'].set(config['color'].get(bool)) + + # Compatibility from list_format_{item,album} to format_{item,album} + for elem in ('item', 'album'): + old_key = 'list_format_{0}'.format(elem) + if config[old_key].exists(): + new_key = 'format_{0}'.format(elem) + log.warning( + u'Warning: configuration uses "{0}" which is deprecated' + u' in favor of "{1}" now that it affects all commands. ' + u'See changelog & documentation.', + old_key, + new_key, + ) + config[new_key].set(config[old_key]) + + config_path = config.user_config_path() + if os.path.isfile(config_path): + log.debug(u'user configuration: {0}', + util.displayable_path(config_path)) + else: + log.debug(u'no user configuration found at {0}', + util.displayable_path(config_path)) + + log.debug(u'data directory: {0}', + util.displayable_path(config.config_dir())) + return config -def _raw_main(args): - """A helper function for `main` without top-level exception - handling. +def _open_library(config): + """Create a new library instance from the configuration. """ - subcommand, suboptions, subargs = _configure(args) - - # Open library file. dbpath = config['library'].as_filename() try: lib = library.Library( @@ -918,32 +1190,55 @@ def _raw_main(args): get_path_formats(), get_replacements(), ) - except sqlite3.OperationalError: + lib.get_item(0) # Test database connection. + except (sqlite3.OperationalError, sqlite3.DatabaseError): + log.debug(u'{}', traceback.format_exc()) raise UserError(u"database file {0} could not be opened".format( util.displayable_path(dbpath) )) - plugins.send("library_opened", lib=lib) + log.debug(u'library database: {0}\n' + u'library directory: {1}', + util.displayable_path(lib.path), + util.displayable_path(lib.directory)) + return lib - # Configure the logger. - if config['verbose'].get(bool): - log.setLevel(logging.DEBUG) - else: - log.setLevel(logging.INFO) - log.debug(u'data directory: {0}\n' - u'library database: {1}\n' - u'library directory: {2}' - .format( - util.displayable_path(config.config_dir()), - util.displayable_path(lib.path), - util.displayable_path(lib.directory), - ) - ) - # Configure the MusicBrainz API. - mb.configure() +def _raw_main(args, lib=None): + """A helper function for `main` without top-level exception + handling. + """ + parser = SubcommandsOptionParser() + parser.add_format_option(flags=('--format-item',), target=library.Item) + parser.add_format_option(flags=('--format-album',), target=library.Album) + parser.add_option('-l', '--library', dest='library', + help=u'library database file to use') + parser.add_option('-d', '--directory', dest='directory', + help=u"destination music directory") + parser.add_option('-v', '--verbose', dest='verbose', action='count', + help=u'log more details (use twice for even more)') + parser.add_option('-c', '--config', dest='config', + help=u'path to configuration file') + parser.add_option('-h', '--help', dest='help', action='store_true', + help=u'show this help message and exit') + parser.add_option('--version', dest='version', action='store_true', + help=optparse.SUPPRESS_HELP) - # Invoke the subcommand. + options, subargs = parser.parse_global_options(args) + + # Special case for the `config --edit` command: bypass _setup so + # that an invalid configuration does not prevent the editor from + # starting. + if subargs and subargs[0] == 'config' \ + and ('-e' in subargs or '--edit' in subargs): + from beets.ui.commands import config_edit + return config_edit() + + subcommands, plugins, lib = _setup(options, lib) + parser.add_subcommand(*subcommands) + + subcommand, suboptions, subargs = parser.parse_subcommand(subargs) subcommand.func(lib, suboptions, subargs) + plugins.send('cli_exit', lib=lib) @@ -955,7 +1250,7 @@ def main(args=None): _raw_main(args) except UserError as exc: message = exc.args[0] if exc.args else None - log.error(u'error: {0}'.format(message)) + log.error(u'error: {0}', message) sys.exit(1) except util.HumanReadableException as exc: exc.log(log) @@ -963,11 +1258,14 @@ def main(args=None): except library.FileOperationError as exc: # These errors have reasonable human-readable descriptions, but # we still want to log their tracebacks for debugging. - log.debug(traceback.format_exc()) - log.error(exc) + log.debug('{}', traceback.format_exc()) + log.error('{}', exc) sys.exit(1) except confit.ConfigError as exc: - log.error(u'configuration error: {0}'.format(exc)) + log.error(u'configuration error: {0}', exc) + sys.exit(1) + except db_query.InvalidQueryError as exc: + log.error(u'invalid query: {0}', exc) sys.exit(1) except IOError as exc: if exc.errno == errno.EPIPE: @@ -977,4 +1275,4 @@ def main(args=None): raise except KeyboardInterrupt: # Silently ignore ^C except in verbose mode. - log.debug(traceback.format_exc()) + log.debug(u'{}', traceback.format_exc()) diff --git a/libs/beets/ui/commands.py b/libs/beets/ui/commands.py index e7e631a4..867a4737 100644 --- a/libs/beets/ui/commands.py +++ b/libs/beets/ui/commands.py @@ -1,5 +1,6 @@ +# -*- coding: utf-8 -*- # This file is part of beets. -# Copyright 2014, Adrian Sampson. +# Copyright 2016, Adrian Sampson. # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -15,30 +16,32 @@ """This module provides the default commands for beets' command-line interface. """ -from __future__ import print_function -import logging +from __future__ import division, absolute_import, print_function + import os -import time -import itertools -import codecs -import platform +import re +from collections import namedtuple, Counter +from itertools import chain import beets from beets import ui -from beets.ui import print_, input_, decargs +from beets.ui import print_, input_, decargs, show_path_changes from beets import autotag -from beets.autotag import recommendation +from beets.autotag import Recommendation from beets.autotag import hooks from beets import plugins from beets import importer from beets import util from beets.util import syspath, normpath, ancestry, displayable_path -from beets.util.functemplate import Template from beets import library from beets import config +from beets import logging from beets.util.confit import _package_path +VARIOUS_ARTISTS = u'Various Artists' +PromptChoice = namedtuple('ExtraChoice', ['short', 'long', 'callback']) + # Global logger. log = logging.getLogger('beets') @@ -47,10 +50,8 @@ log = logging.getLogger('beets') default_commands = [] - # Utilities. - def _do_query(lib, query, album, also_items=True): """For commands that operate on matched items, performs a query and returns a list of matching items and a list of matching @@ -70,49 +71,77 @@ def _do_query(lib, query, album, also_items=True): items = list(lib.items(query)) if album and not albums: - raise ui.UserError('No matching albums found.') + raise ui.UserError(u'No matching albums found.') elif not album and not items: - raise ui.UserError('No matching items found.') + raise ui.UserError(u'No matching items found.') return items, albums # fields: Shows a list of available fields for queries and format strings. -fields_cmd = ui.Subcommand('fields', - help='show fields available for queries and format strings') +def _print_keys(query): + """Given a SQLite query result, print the `key` field of each + returned row, with identation of 2 spaces. + """ + for row in query: + print_(' ' * 2 + row['key']) + def fields_func(lib, opts, args): def _print_rows(names): - print(" " + "\n ".join(names)) + names.sort() + print_(" " + "\n ".join(names)) - def _show_plugin_fields(album): - plugin_fields = [] - for plugin in plugins.find_plugins(): - if album: - fdict = plugin.album_template_fields - else: - fdict = plugin.template_fields - plugin_fields += fdict.keys() - if plugin_fields: - print("Template fields from plugins:") - _print_rows(plugin_fields) + print_(u"Item fields:") + _print_rows(library.Item.all_keys()) - print("Item fields:") - _print_rows(library.ITEM_KEYS) - _show_plugin_fields(False) + print_(u"Album fields:") + _print_rows(library.Album.all_keys()) - print("\nAlbum fields:") - _print_rows(library.ALBUM_KEYS) - _show_plugin_fields(True) + with lib.transaction() as tx: + # The SQL uses the DISTINCT to get unique values from the query + unique_fields = 'SELECT DISTINCT key FROM (%s)' + print_(u"Item flexible attributes:") + _print_keys(tx.query(unique_fields % library.Item._flex_table)) + + print_(u"Album flexible attributes:") + _print_keys(tx.query(unique_fields % library.Album._flex_table)) + +fields_cmd = ui.Subcommand( + 'fields', + help=u'show fields available for queries and format strings' +) fields_cmd.func = fields_func default_commands.append(fields_cmd) -# import: Autotagger and importer. +# help: Print help text for commands -VARIOUS_ARTISTS = u'Various Artists' +class HelpCommand(ui.Subcommand): + + def __init__(self): + super(HelpCommand, self).__init__( + 'help', aliases=('?',), + help=u'give detailed help on a specific sub-command', + ) + + def func(self, lib, opts, args): + if args: + cmdname = args[0] + helpcommand = self.root_parser._subcommand_for_name(cmdname) + if not helpcommand: + raise ui.UserError(u"unknown command '{0}'".format(cmdname)) + helpcommand.print_help() + else: + self.root_parser.print_help() + + +default_commands.append(HelpCommand()) + + +# import: Autotagger and importer. # Importer utilities and support. @@ -145,19 +174,21 @@ def disambig_string(info): if disambig: return u', '.join(disambig) + def dist_string(dist): """Formats a distance (a float) as a colorized similarity percentage string. """ - out = '%.1f%%' % ((1 - dist) * 100) + out = u'%.1f%%' % ((1 - dist) * 100) if dist <= config['match']['strong_rec_thresh'].as_number(): - out = ui.colorize('green', out) + out = ui.colorize('text_success', out) elif dist <= config['match']['medium_rec_thresh'].as_number(): - out = ui.colorize('yellow', out) + out = ui.colorize('text_warning', out) else: - out = ui.colorize('red', out) + out = ui.colorize('text_error', out) return out + def penalty_string(distance, limit=None): """Returns a colorized string that indicates all the penalties applied to a distance object. @@ -171,7 +202,8 @@ def penalty_string(distance, limit=None): if penalties: if limit and len(penalties) > limit: penalties = penalties[:limit] + ['...'] - return ui.colorize('yellow', '(%s)' % ', '.join(penalties)) + return ui.colorize('text_warning', u'(%s)' % ', '.join(penalties)) + def show_change(cur_artist, cur_album, match): """Print out a representation of the changes that will be made if an @@ -213,29 +245,29 @@ def show_change(cur_artist, cur_album, match): (cur_album != match.info.album and match.info.album != VARIOUS_ARTISTS): artist_l, artist_r = cur_artist or '', match.info.artist - album_l, album_r = cur_album or '', match.info.album + album_l, album_r = cur_album or '', match.info.album if artist_r == VARIOUS_ARTISTS: # Hide artists for VA releases. artist_l, artist_r = u'', u'' artist_l, artist_r = ui.colordiff(artist_l, artist_r) - album_l, album_r = ui.colordiff(album_l, album_r) + album_l, album_r = ui.colordiff(album_l, album_r) - print_("Correcting tags from:") + print_(u"Correcting tags from:") show_album(artist_l, album_l) - print_("To:") + print_(u"To:") show_album(artist_r, album_r) else: print_(u"Tagging:\n {0.artist} - {0.album}".format(match.info)) # Data URL. if match.info.data_url: - print_('URL:\n %s' % match.info.data_url) + print_(u'URL:\n %s' % match.info.data_url) # Info line. info = [] # Similarity. - info.append('(Similarity: %s)' % dist_string(match.distance)) + info.append(u'(Similarity: %s)' % dist_string(match.distance)) # Penalties. penalties = penalty_string(match.distance) if penalties: @@ -243,12 +275,12 @@ def show_change(cur_artist, cur_album, match): # Disambiguation. disambig = disambig_string(match.info) if disambig: - info.append(ui.colorize('lightgray', '(%s)' % disambig)) + info.append(ui.colorize('text_highlight_minor', u'(%s)' % disambig)) print_(' '.join(info)) # Tracks. pairs = match.mapping.items() - pairs.sort(key=lambda (_, track_info): track_info.index) + pairs.sort(key=lambda item_and_track_info: item_and_track_info[1].index) # Build up LHS and RHS for track difference display. The `lines` list # contains ``(lhs, rhs, width)`` tuples where `width` is the length (in @@ -261,16 +293,16 @@ def show_change(cur_artist, cur_album, match): if medium != track_info.medium or disctitle != track_info.disctitle: media = match.info.media or 'Media' if match.info.mediums > 1 and track_info.disctitle: - lhs = '%s %s: %s' % (media, track_info.medium, - track_info.disctitle) + lhs = u'%s %s: %s' % (media, track_info.medium, + track_info.disctitle) elif match.info.mediums > 1: - lhs = '%s %s' % (media, track_info.medium) + lhs = u'%s %s' % (media, track_info.medium) elif track_info.disctitle: - lhs = '%s: %s' % (media, track_info.disctitle) + lhs = u'%s: %s' % (media, track_info.disctitle) else: lhs = None if lhs: - lines.append((lhs, '', 0)) + lines.append((lhs, u'', 0)) medium, disctitle = track_info.medium, track_info.disctitle # Titles. @@ -288,20 +320,12 @@ def show_change(cur_artist, cur_album, match): cur_track, new_track = format_index(item), format_index(track_info) if cur_track != new_track: if item.track in (track_info.index, track_info.medium_index): - color = 'lightgray' + color = 'text_highlight_minor' else: - color = 'red' - if (cur_track + new_track).count('-') == 1: - lhs_track, rhs_track = ui.colorize(color, cur_track), \ - ui.colorize(color, new_track) - else: - color = 'red' - lhs_track, rhs_track = ui.color_diff_suffix(cur_track, - new_track) - templ = ui.colorize(color, u' (#') + u'{0}' + \ - ui.colorize(color, u')') - lhs += templ.format(lhs_track) - rhs += templ.format(rhs_track) + color = 'text_highlight' + templ = ui.colorize(color, u' (#{0})') + lhs += templ.format(cur_track) + rhs += templ.format(new_track) lhs_width += len(cur_track) + 4 # Length change. @@ -310,12 +334,9 @@ def show_change(cur_artist, cur_album, match): config['ui']['length_diff_thresh'].as_number(): cur_length = ui.human_seconds_short(item.length) new_length = ui.human_seconds_short(track_info.length) - lhs_length, rhs_length = ui.color_diff_suffix(cur_length, - new_length) - templ = ui.colorize('red', u' (') + u'{0}' + \ - ui.colorize('red', u')') - lhs += templ.format(lhs_length) - rhs += templ.format(rhs_length) + templ = ui.colorize('text_highlight', u' ({0})') + lhs += templ.format(cur_length) + rhs += templ.format(new_length) lhs_width += len(cur_length) + 3 # Penalties. @@ -324,9 +345,9 @@ def show_change(cur_artist, cur_album, match): rhs += ' %s' % penalties if lhs != rhs: - lines.append((' * %s' % lhs, rhs, lhs_width)) + lines.append((u' * %s' % lhs, rhs, lhs_width)) elif config['import']['detail']: - lines.append((' * %s' % lhs, '', lhs_width)) + lines.append((u' * %s' % lhs, '', lhs_width)) # Print each track in two columns, or across two lines. col_width = (ui.term_width() - len(''.join([' * ', ' -> ']))) // 2 @@ -343,19 +364,24 @@ def show_change(cur_artist, cur_album, match): # Missing and unmatched tracks. if match.extra_tracks: - print_('Missing tracks:') + print_(u'Missing tracks ({0}/{1} - {2:.1%}):'.format( + len(match.extra_tracks), + len(match.info.tracks), + len(match.extra_tracks) / len(match.info.tracks) + )) for track_info in match.extra_tracks: - line = ' ! %s (#%s)' % (track_info.title, format_index(track_info)) + line = u' ! %s (#%s)' % (track_info.title, format_index(track_info)) if track_info.length: - line += ' (%s)' % ui.human_seconds_short(track_info.length) - print_(ui.colorize('yellow', line)) + line += u' (%s)' % ui.human_seconds_short(track_info.length) + print_(ui.colorize('text_warning', line)) if match.extra_items: - print_('Unmatched tracks:') + print_(u'Unmatched tracks ({0}):'.format(len(match.extra_items))) for item in match.extra_items: - line = ' ! %s (#%s)' % (item.title, format_index(item)) + line = u' ! %s (#%s)' % (item.title, format_index(item)) if item.length: - line += ' (%s)' % ui.human_seconds_short(item.length) - print_(ui.colorize('yellow', line)) + line += u' (%s)' % ui.human_seconds_short(item.length) + print_(ui.colorize('text_warning', line)) + def show_item_change(item, match): """Print out the change that would occur by tagging `item` with the @@ -368,22 +394,22 @@ def show_item_change(item, match): cur_artist, new_artist = ui.colordiff(cur_artist, new_artist) cur_title, new_title = ui.colordiff(cur_title, new_title) - print_("Correcting track tags from:") - print_(" %s - %s" % (cur_artist, cur_title)) - print_("To:") - print_(" %s - %s" % (new_artist, new_title)) + print_(u"Correcting track tags from:") + print_(u" %s - %s" % (cur_artist, cur_title)) + print_(u"To:") + print_(u" %s - %s" % (new_artist, new_title)) else: - print_("Tagging track: %s - %s" % (cur_artist, cur_title)) + print_(u"Tagging track: %s - %s" % (cur_artist, cur_title)) # Data URL. if match.info.data_url: - print_('URL:\n %s' % match.info.data_url) + print_(u'URL:\n %s' % match.info.data_url) # Info line. info = [] # Similarity. - info.append('(Similarity: %s)' % dist_string(match.distance)) + info.append(u'(Similarity: %s)' % dist_string(match.distance)) # Penalties. penalties = penalty_string(match.distance) if penalties: @@ -391,10 +417,48 @@ def show_item_change(item, match): # Disambiguation. disambig = disambig_string(match.info) if disambig: - info.append(ui.colorize('lightgray', '(%s)' % disambig)) + info.append(ui.colorize('text_highlight_minor', u'(%s)' % disambig)) print_(' '.join(info)) -def _summary_judment(rec): + +def summarize_items(items, singleton): + """Produces a brief summary line describing a set of items. Used for + manually resolving duplicates during import. + + `items` is a list of `Item` objects. `singleton` indicates whether + this is an album or single-item import (if the latter, them `items` + should only have one element). + """ + summary_parts = [] + if not singleton: + summary_parts.append(u"{0} items".format(len(items))) + + format_counts = {} + for item in items: + format_counts[item.format] = format_counts.get(item.format, 0) + 1 + if len(format_counts) == 1: + # A single format. + summary_parts.append(items[0].format) + else: + # Enumerate all the formats by decreasing frequencies: + for fmt, count in sorted( + format_counts.items(), + key=lambda fmt_and_count: (-fmt_and_count[1], fmt_and_count[0]) + ): + summary_parts.append('{0} {1}'.format(fmt, count)) + + if items: + average_bitrate = sum([item.bitrate for item in items]) / len(items) + total_duration = sum([item.length for item in items]) + total_filesize = sum([item.filesize for item in items]) + summary_parts.append(u'{0}kbps'.format(int(average_bitrate / 1000))) + summary_parts.append(ui.human_seconds_short(total_duration)) + summary_parts.append(ui.human_bytes(total_filesize)) + + return u', '.join(summary_parts) + + +def _summary_judgment(rec): """Determines whether a decision should be made without even asking the user. This occurs in quiet mode and when an action is chosen for NONE recommendations. Return an action or None if the user should be @@ -402,7 +466,7 @@ def _summary_judment(rec): made. """ if config['import']['quiet']: - if rec == recommendation.strong: + if rec == Recommendation.strong: return importer.action.APPLY else: action = config['import']['quiet_fallback'].as_choice({ @@ -410,7 +474,7 @@ def _summary_judment(rec): 'asis': importer.action.ASIS, }) - elif rec == recommendation.none: + elif rec == Recommendation.none: action = config['import']['none_rec_action'].as_choice({ 'skip': importer.action.SKIP, 'asis': importer.action.ASIS, @@ -421,13 +485,15 @@ def _summary_judment(rec): return None if action == importer.action.SKIP: - print_('Skipping.') + print_(u'Skipping.') elif action == importer.action.ASIS: - print_('Importing as-is.') + print_(u'Importing as-is.') return action + def choose_candidate(candidates, singleton, rec, cur_artist=None, - cur_album=None, item=None, itemcount=None): + cur_album=None, item=None, itemcount=None, + extra_choices=[]): """Given a sorted list of candidates, ask the user for a selection of which candidate to use. Applies to both full albums and singletons (tracks). Candidates are either AlbumMatch or TrackMatch @@ -435,8 +501,16 @@ def choose_candidate(candidates, singleton, rec, cur_artist=None, `cur_album`, and `itemcount` must be provided. For singletons, `item` must be provided. - Returns the result of the choice, which may SKIP, ASIS, TRACKS, or - MANUAL or a candidate (an AlbumMatch/TrackMatch object). + `extra_choices` is a list of `PromptChoice`s, containg the choices + appended by the plugins after receiving the `before_choose_candidate` + event. If not empty, the choices are appended to the prompt presented + to the user. + + Returns one of the following: + * the result of the choice, which may be SKIP, ASIS, TRACKS, or MANUAL + * a candidate (an AlbumMatch/TrackMatch object) + * the short letter of a `PromptChoice` (if the user selected one of + the `extra_choices`). """ # Sanity check. if singleton: @@ -445,47 +519,53 @@ def choose_candidate(candidates, singleton, rec, cur_artist=None, assert cur_artist is not None assert cur_album is not None + # Build helper variables for extra choices. + extra_opts = tuple(c.long for c in extra_choices) + extra_actions = tuple(c.short for c in extra_choices) + # Zero candidates. if not candidates: if singleton: - print_("No matching recordings found.") - opts = ('Use as-is', 'Skip', 'Enter search', 'enter Id', - 'aBort') + print_(u"No matching recordings found.") + opts = (u'Use as-is', u'Skip', u'Enter search', u'enter Id', + u'aBort') else: - print_("No matching release found for {0} tracks." + print_(u"No matching release found for {0} tracks." .format(itemcount)) - print_('For help, see: ' - 'http://beets.readthedocs.org/en/latest/faq.html#nomatch') - opts = ('Use as-is', 'as Tracks', 'Group albums', 'Skip', - 'Enter search', 'enter Id', 'aBort') - sel = ui.input_options(opts) - if sel == 'u': + print_(u'For help, see: ' + u'http://beets.readthedocs.org/en/latest/faq.html#nomatch') + opts = (u'Use as-is', u'as Tracks', u'Group albums', u'Skip', + u'Enter search', u'enter Id', u'aBort') + sel = ui.input_options(opts + extra_opts) + if sel == u'u': return importer.action.ASIS - elif sel == 't': + elif sel == u't': assert not singleton return importer.action.TRACKS - elif sel == 'e': + elif sel == u'e': return importer.action.MANUAL - elif sel == 's': + elif sel == u's': return importer.action.SKIP - elif sel == 'b': + elif sel == u'b': raise importer.ImportAbort() - elif sel == 'i': + elif sel == u'i': return importer.action.MANUAL_ID - elif sel == 'g': + elif sel == u'g': return importer.action.ALBUMS + elif sel in extra_actions: + return sel else: assert False # Is the change good enough? bypass_candidates = False - if rec != recommendation.none: + if rec != Recommendation.none: match = candidates[0] bypass_candidates = True while True: # Display and choose from candidates. - require = rec <= recommendation.low + require = rec <= Recommendation.low if not bypass_candidates: # Display list of candidates. @@ -515,35 +595,39 @@ def choose_candidate(candidates, singleton, rec, cur_artist=None, # Disambiguation disambig = disambig_string(match.info) if disambig: - line.append(ui.colorize('lightgray', '(%s)' % disambig)) + line.append(ui.colorize('text_highlight_minor', + u'(%s)' % disambig)) - print_(' '.join(line)) + print_(u' '.join(line)) # Ask the user for a choice. if singleton: - opts = ('Skip', 'Use as-is', 'Enter search', 'enter Id', - 'aBort') + opts = (u'Skip', u'Use as-is', u'Enter search', u'enter Id', + u'aBort') else: - opts = ('Skip', 'Use as-is', 'as Tracks', 'Group albums', - 'Enter search', 'enter Id', 'aBort') - sel = ui.input_options(opts, numrange=(1, len(candidates))) - if sel == 's': + opts = (u'Skip', u'Use as-is', u'as Tracks', u'Group albums', + u'Enter search', u'enter Id', u'aBort') + sel = ui.input_options(opts + extra_opts, + numrange=(1, len(candidates))) + if sel == u's': return importer.action.SKIP - elif sel == 'u': + elif sel == u'u': return importer.action.ASIS - elif sel == 'm': + elif sel == u'm': pass - elif sel == 'e': + elif sel == u'e': return importer.action.MANUAL - elif sel == 't': + elif sel == u't': assert not singleton return importer.action.TRACKS - elif sel == 'b': + elif sel == u'b': raise importer.ImportAbort() - elif sel == 'i': + elif sel == u'i': return importer.action.MANUAL_ID - elif sel == 'g': + elif sel == u'g': return importer.action.ALBUMS + elif sel in extra_actions: + return sel else: # Numerical selection. match = candidates[sel - 1] if sel != 1: @@ -559,58 +643,64 @@ def choose_candidate(candidates, singleton, rec, cur_artist=None, show_change(cur_artist, cur_album, match) # Exact match => tag automatically if we're not in timid mode. - if rec == recommendation.strong and not config['import']['timid']: + if rec == Recommendation.strong and not config['import']['timid']: return match # Ask for confirmation. if singleton: - opts = ('Apply', 'More candidates', 'Skip', 'Use as-is', - 'Enter search', 'enter Id', 'aBort') + opts = (u'Apply', u'More candidates', u'Skip', u'Use as-is', + u'Enter search', u'enter Id', u'aBort') else: - opts = ('Apply', 'More candidates', 'Skip', 'Use as-is', - 'as Tracks', 'Group albums', 'Enter search', 'enter Id', - 'aBort') + opts = (u'Apply', u'More candidates', u'Skip', u'Use as-is', + u'as Tracks', u'Group albums', u'Enter search', + u'enter Id', u'aBort') default = config['import']['default_action'].as_choice({ - 'apply': 'a', - 'skip': 's', - 'asis': 'u', - 'none': None, + u'apply': u'a', + u'skip': u's', + u'asis': u'u', + u'none': None, }) if default is None: require = True - sel = ui.input_options(opts, require=require, default=default) - if sel == 'a': + sel = ui.input_options(opts + extra_opts, require=require, + default=default) + if sel == u'a': return match - elif sel == 'g': + elif sel == u'g': return importer.action.ALBUMS - elif sel == 's': + elif sel == u's': return importer.action.SKIP - elif sel == 'u': + elif sel == u'u': return importer.action.ASIS - elif sel == 't': + elif sel == u't': assert not singleton return importer.action.TRACKS - elif sel == 'e': + elif sel == u'e': return importer.action.MANUAL - elif sel == 'b': + elif sel == u'b': raise importer.ImportAbort() - elif sel == 'i': + elif sel == u'i': return importer.action.MANUAL_ID + elif sel in extra_actions: + return sel + def manual_search(singleton): """Input either an artist and album (for full albums) or artist and track name (for singletons) for manual search. """ - artist = input_('Artist:') - name = input_('Track:' if singleton else 'Album:') + artist = input_(u'Artist:') + name = input_(u'Track:' if singleton else u'Album:') return artist.strip(), name.strip() + def manual_id(singleton): """Input an ID, either for an album ("release") or a track ("recording"). """ - prompt = u'Enter {0} ID:'.format('recording' if singleton else 'release') + prompt = u'Enter {0} ID:'.format(u'recording' if singleton else u'release') return input_(prompt).strip() + class TerminalImportSession(importer.ImportSession): """An import session that runs in a terminal. """ @@ -625,7 +715,7 @@ class TerminalImportSession(importer.ImportSession): u' ({0} items)'.format(len(task.items))) # Take immediate action if appropriate. - action = _summary_judment(task.rec) + action = _summary_judgment(task.rec) if action == importer.action.APPLY: match = task.candidates[0] show_change(task.cur_artist, task.cur_album, match) @@ -636,13 +726,19 @@ class TerminalImportSession(importer.ImportSession): # Loop until we have a choice. candidates, rec = task.candidates, task.rec while True: + # Gather extra choices from plugins. + extra_choices = self._get_plugin_choices(task) + extra_ops = {c.short: c.callback for c in extra_choices} + # Ask for a choice from the user. - choice = choose_candidate(candidates, False, rec, task.cur_artist, - task.cur_album, itemcount=len(task.items)) + choice = choose_candidate( + candidates, False, rec, task.cur_artist, task.cur_album, + itemcount=len(task.items), extra_choices=extra_choices + ) # Choose which tags to use. if choice in (importer.action.SKIP, importer.action.ASIS, - importer.action.TRACKS, importer.action.ALBUMS): + importer.action.TRACKS, importer.action.ALBUMS): # Pass selection to main control flow. return choice elif choice is importer.action.MANUAL: @@ -656,8 +752,14 @@ class TerminalImportSession(importer.ImportSession): search_id = manual_id(False) if search_id: _, _, candidates, rec = autotag.tag_album( - task.items, search_id=search_id + task.items, search_ids=search_id.split() ) + elif choice in extra_ops.keys(): + # Allow extra ops to automatically set the post-choice. + post_choice = extra_ops[choice](self, task) + if isinstance(post_choice, importer.action): + # MANUAL and MANUAL_ID have no effect, even if returned. + return post_choice else: # We have a candidate! Finish tagging. Here, choice is an # AlbumMatch object. @@ -673,7 +775,7 @@ class TerminalImportSession(importer.ImportSession): candidates, rec = task.candidates, task.rec # Take immediate action if appropriate. - action = _summary_judment(task.rec) + action = _summary_judgment(task.rec) if action == importer.action.APPLY: match = candidates[0] show_item_change(task.item, match) @@ -682,148 +784,169 @@ class TerminalImportSession(importer.ImportSession): return action while True: + extra_choices = self._get_plugin_choices(task) + extra_ops = {c.short: c.callback for c in extra_choices} + # Ask for a choice. - choice = choose_candidate(candidates, True, rec, item=task.item) + choice = choose_candidate(candidates, True, rec, item=task.item, + extra_choices=extra_choices) if choice in (importer.action.SKIP, importer.action.ASIS): return choice elif choice == importer.action.TRACKS: - assert False # TRACKS is only legal for albums. + assert False # TRACKS is only legal for albums. elif choice == importer.action.MANUAL: # Continue in the loop with a new set of candidates. search_artist, search_title = manual_search(True) candidates, rec = autotag.tag_item(task.item, search_artist, - search_title) + search_title) elif choice == importer.action.MANUAL_ID: # Ask for a track ID. search_id = manual_id(True) if search_id: - candidates, rec = autotag.tag_item(task.item, - search_id=search_id) + candidates, rec = autotag.tag_item( + task.item, search_ids=search_id.split()) + elif choice in extra_ops.keys(): + # Allow extra ops to automatically set the post-choice. + post_choice = extra_ops[choice](self, task) + if isinstance(post_choice, importer.action): + # MANUAL and MANUAL_ID have no effect, even if returned. + return post_choice else: # Chose a candidate. assert isinstance(choice, autotag.TrackMatch) return choice - def resolve_duplicate(self, task): + def resolve_duplicate(self, task, found_duplicates): """Decide what to do when a new album or item seems similar to one that's already in the library. """ - log.warn("This %s is already in the library!" % - ("album" if task.is_album else "item")) + log.warn(u"This {0} is already in the library!", + (u"album" if task.is_album else u"item")) if config['import']['quiet']: # In quiet mode, don't prompt -- just skip. - log.info('Skipping.') - sel = 's' + log.info(u'Skipping.') + sel = u's' else: + # Print some detail about the existing and new items so the + # user can make an informed decision. + for duplicate in found_duplicates: + print_(u"Old: " + summarize_items( + list(duplicate.items()) if task.is_album else [duplicate], + not task.is_album, + )) + + print_(u"New: " + summarize_items( + task.imported_items(), + not task.is_album, + )) + sel = ui.input_options( - ('Skip new', 'Keep both', 'Remove old') + (u'Skip new', u'Keep both', u'Remove old') ) - if sel == 's': + if sel == u's': # Skip new. task.set_choice(importer.action.SKIP) - elif sel == 'k': + elif sel == u'k': # Keep both. Do nothing; leave the choice intact. pass - elif sel == 'r': + elif sel == u'r': # Remove old. - task.remove_duplicates = True + task.should_remove_duplicates = True else: assert False def should_resume(self, path): return ui.input_yn(u"Import of the directory:\n{0}\n" - "was interrupted. Resume (Y/n)?" + u"was interrupted. Resume (Y/n)?" .format(displayable_path(path))) + def _get_plugin_choices(self, task): + """Get the extra choices appended to the plugins to the ui prompt. + + The `before_choose_candidate` event is sent to the plugins, with + session and task as its parameters. Plugins are responsible for + checking the right conditions and returning a list of `PromptChoice`s, + which is flattened and checked for conflicts. + + If two or more choices have the same short letter, a warning is + emitted and all but one choices are discarded, giving preference + to the default importer choices. + + Returns a list of `PromptChoice`s. + """ + # Send the before_choose_candidate event and flatten list. + extra_choices = list(chain(*plugins.send('before_choose_candidate', + session=self, task=task))) + # Take into account default options, for duplicate checking. + all_choices = [PromptChoice(u'a', u'Apply', None), + PromptChoice(u's', u'Skip', None), + PromptChoice(u'u', u'Use as-is', None), + PromptChoice(u't', u'as Tracks', None), + PromptChoice(u'g', u'Group albums', None), + PromptChoice(u'e', u'Enter search', None), + PromptChoice(u'i', u'enter Id', None), + PromptChoice(u'b', u'aBort', None)] +\ + extra_choices + + short_letters = [c.short for c in all_choices] + if len(short_letters) != len(set(short_letters)): + # Duplicate short letter has been found. + duplicates = [i for i, count in Counter(short_letters).items() + if count > 1] + for short in duplicates: + # Keep the first of the choices, removing the rest. + dup_choices = [c for c in all_choices if c.short == short] + for c in dup_choices[1:]: + log.warn(u"Prompt choice '{0}' removed due to conflict " + u"with '{1}' (short letter: '{2}')", + c.long, dup_choices[0].long, c.short) + extra_choices.remove(c) + return extra_choices + + # The import command. + def import_files(lib, paths, query): """Import the files in the given list of paths or matching the query. """ # Check the user-specified directories. for path in paths: - fullpath = syspath(normpath(path)) - if not config['import']['singletons'] and not os.path.isdir(fullpath): - raise ui.UserError(u'not a directory: {0}'.format( - displayable_path(path))) - elif config['import']['singletons'] and not os.path.exists(fullpath): - raise ui.UserError(u'no such file: {0}'.format( + if not os.path.exists(syspath(normpath(path))): + raise ui.UserError(u'no such file or directory: {0}'.format( displayable_path(path))) # Check parameter consistency. if config['import']['quiet'] and config['import']['timid']: - raise ui.UserError("can't be both quiet and timid") + raise ui.UserError(u"can't be both quiet and timid") # Open the log. if config['import']['log'].get() is not None: - logpath = config['import']['log'].as_filename() + logpath = syspath(config['import']['log'].as_filename()) try: - logfile = codecs.open(syspath(logpath), 'a', 'utf8') + loghandler = logging.FileHandler(logpath) except IOError: - raise ui.UserError(u"could not open log file for writing: %s" % - displayable_path(logpath)) - print(u'import started', time.asctime(), file=logfile) + raise ui.UserError(u"could not open log file for writing: " + u"{0}".format(displayable_path(logpath))) else: - logfile = None + loghandler = None # Never ask for input in quiet mode. if config['import']['resume'].get() == 'ask' and \ config['import']['quiet']: config['import']['resume'] = False - session = TerminalImportSession(lib, logfile, paths, query) - try: - session.run() - finally: - # If we were logging, close the file. - if logfile: - print(u'', file=logfile) - logfile.close() + session = TerminalImportSession(lib, loghandler, paths, query) + session.run() # Emit event. plugins.send('import', lib=lib, paths=paths) -import_cmd = ui.Subcommand('import', help='import new music', - aliases=('imp', 'im')) -import_cmd.parser.add_option('-c', '--copy', action='store_true', - default=None, help="copy tracks into library directory (default)") -import_cmd.parser.add_option('-C', '--nocopy', action='store_false', - dest='copy', help="don't copy tracks (opposite of -c)") -import_cmd.parser.add_option('-w', '--write', action='store_true', - default=None, help="write new metadata to files' tags (default)") -import_cmd.parser.add_option('-W', '--nowrite', action='store_false', - dest='write', help="don't write metadata (opposite of -w)") -import_cmd.parser.add_option('-a', '--autotag', action='store_true', - dest='autotag', help="infer tags for imported files (default)") -import_cmd.parser.add_option('-A', '--noautotag', action='store_false', - dest='autotag', - help="don't infer tags for imported files (opposite of -a)") -import_cmd.parser.add_option('-p', '--resume', action='store_true', - default=None, help="resume importing if interrupted") -import_cmd.parser.add_option('-P', '--noresume', action='store_false', - dest='resume', help="do not try to resume importing") -import_cmd.parser.add_option('-q', '--quiet', action='store_true', - dest='quiet', help="never prompt for input: skip albums instead") -import_cmd.parser.add_option('-l', '--log', dest='log', - help='file to log untaggable albums for later review') -import_cmd.parser.add_option('-s', '--singletons', action='store_true', - help='import individual tracks instead of full albums') -import_cmd.parser.add_option('-t', '--timid', dest='timid', - action='store_true', help='always confirm all actions') -import_cmd.parser.add_option('-L', '--library', dest='library', - action='store_true', help='retag items matching a query') -import_cmd.parser.add_option('-i', '--incremental', dest='incremental', - action='store_true', help='skip already-imported directories') -import_cmd.parser.add_option('-I', '--noincremental', dest='incremental', - action='store_false', help='do not skip already-imported directories') -import_cmd.parser.add_option('--flat', dest='flat', - action='store_true', help='import an entire tree as a single album') -import_cmd.parser.add_option('-g', '--group-albums', dest='group_albums', - action='store_true', help='group tracks in a folder into seperate albums') + def import_func(lib, opts, args): config['import'].set_args(opts) @@ -839,40 +962,117 @@ def import_func(lib, opts, args): query = None paths = args if not paths: - raise ui.UserError('no path specified') + raise ui.UserError(u'no path specified') import_files(lib, paths, query) + + +import_cmd = ui.Subcommand( + u'import', help=u'import new music', aliases=(u'imp', u'im') +) +import_cmd.parser.add_option( + u'-c', u'--copy', action='store_true', default=None, + help=u"copy tracks into library directory (default)" +) +import_cmd.parser.add_option( + u'-C', u'--nocopy', action='store_false', dest='copy', + help=u"don't copy tracks (opposite of -c)" +) +import_cmd.parser.add_option( + u'-w', u'--write', action='store_true', default=None, + help=u"write new metadata to files' tags (default)" +) +import_cmd.parser.add_option( + u'-W', u'--nowrite', action='store_false', dest='write', + help=u"don't write metadata (opposite of -w)" +) +import_cmd.parser.add_option( + u'-a', u'--autotag', action='store_true', dest='autotag', + help=u"infer tags for imported files (default)" +) +import_cmd.parser.add_option( + u'-A', u'--noautotag', action='store_false', dest='autotag', + help=u"don't infer tags for imported files (opposite of -a)" +) +import_cmd.parser.add_option( + u'-p', u'--resume', action='store_true', default=None, + help=u"resume importing if interrupted" +) +import_cmd.parser.add_option( + u'-P', u'--noresume', action='store_false', dest='resume', + help=u"do not try to resume importing" +) +import_cmd.parser.add_option( + u'-q', u'--quiet', action='store_true', dest='quiet', + help=u"never prompt for input: skip albums instead" +) +import_cmd.parser.add_option( + u'-l', u'--log', dest='log', + help=u'file to log untaggable albums for later review' +) +import_cmd.parser.add_option( + u'-s', u'--singletons', action='store_true', + help=u'import individual tracks instead of full albums' +) +import_cmd.parser.add_option( + u'-t', u'--timid', dest='timid', action='store_true', + help=u'always confirm all actions' +) +import_cmd.parser.add_option( + u'-L', u'--library', dest='library', action='store_true', + help=u'retag items matching a query' +) +import_cmd.parser.add_option( + u'-i', u'--incremental', dest='incremental', action='store_true', + help=u'skip already-imported directories' +) +import_cmd.parser.add_option( + u'-I', u'--noincremental', dest='incremental', action='store_false', + help=u'do not skip already-imported directories' +) +import_cmd.parser.add_option( + u'--flat', dest='flat', action='store_true', + help=u'import an entire tree as a single album' +) +import_cmd.parser.add_option( + u'-g', u'--group-albums', dest='group_albums', action='store_true', + help=u'group tracks in a folder into separate albums' +) +import_cmd.parser.add_option( + u'--pretend', dest='pretend', action='store_true', + help=u'just print the files to import' +) +import_cmd.parser.add_option( + u'-S', u'--search-id', dest='search_ids', action='append', + metavar='BACKEND_ID', + help=u'restrict matching to a specific metadata backend ID' +) import_cmd.func = import_func default_commands.append(import_cmd) # list: Query and show library contents. -def list_items(lib, query, album, fmt): +def list_items(lib, query, album, fmt=''): """Print out items in lib matching query. If album, then search for albums instead of single items. """ - tmpl = Template(ui._pick_format(album, fmt)) if album: for album in lib.albums(query): - ui.print_obj(album, lib, tmpl) + ui.print_(format(album, fmt)) else: for item in lib.items(query): - ui.print_obj(item, lib, tmpl) + ui.print_(format(item, fmt)) + -list_cmd = ui.Subcommand('list', help='query the library', aliases=('ls',)) -list_cmd.parser.add_option('-a', '--album', action='store_true', - help='show matching albums instead of tracks') -list_cmd.parser.add_option('-p', '--path', action='store_true', - help='print paths for matched items or albums') -list_cmd.parser.add_option('-f', '--format', action='store', - help='print with custom format', default=None) def list_func(lib, opts, args): - if opts.path: - fmt = '$path' - else: - fmt = opts.format - list_items(lib, decargs(args), opts.album, fmt) + list_items(lib, decargs(args), opts.album) + + +list_cmd = ui.Subcommand(u'list', help=u'query the library', aliases=(u'ls',)) +list_cmd.parser.usage += u"\n" \ + u'Example: %prog -f \'$album: $title\' artist:beatles' +list_cmd.parser.add_all_common_options() list_cmd.func = list_func default_commands.append(list_cmd) @@ -891,8 +1091,8 @@ def update_items(lib, query, album, move, pretend): for item in items: # Item deleted? if not os.path.exists(syspath(item.path)): - ui.print_obj(item, lib) - ui.print_(ui.colorize('red', u' deleted')) + ui.print_(format(item)) + ui.print_(ui.colorize('text_error', u' deleted')) if not pretend: item.remove(True) affected_albums.add(item.album_id) @@ -900,16 +1100,16 @@ def update_items(lib, query, album, move, pretend): # Did the item change since last checked? if item.current_mtime() <= item.mtime: - log.debug(u'skipping %s because mtime is up to date (%i)' % - (displayable_path(item.path), item.mtime)) + log.debug(u'skipping {0} because mtime is up to date ({1})', + displayable_path(item.path), item.mtime) continue # Read new data. try: item.read() - except Exception as exc: - log.error(u'error reading {0}: {1}'.format( - displayable_path(item.path), exc)) + except library.ReadError as exc: + log.error(u'error reading {0}: {1}', + displayable_path(item.path), exc) continue # Special-case album artist when it matches track artist. (Hacky @@ -919,11 +1119,11 @@ def update_items(lib, query, album, move, pretend): old_item = lib.get_item(item.id) if old_item.albumartist == old_item.artist == item.artist: item.albumartist = old_item.albumartist - item._dirty.discard('albumartist') + item._dirty.discard(u'albumartist') # Check for and display changes. changed = ui.show_model_changes(item, - fields=library.ITEM_KEYS_META) + fields=library.Item._media_fields) # Save changes. if not pretend: @@ -951,32 +1151,43 @@ def update_items(lib, query, album, move, pretend): continue album = lib.get_album(album_id) if not album: # Empty albums have already been removed. - log.debug('emptied album %i' % album_id) + log.debug(u'emptied album {0}', album_id) continue first_item = album.items().get() # Update album structure to reflect an item in it. - for key in library.ALBUM_KEYS_ITEM: + for key in library.Album.item_keys: album[key] = first_item[key] album.store() # Move album art (and any inconsistent items). if move and lib.directory in ancestry(first_item.path): - log.debug('moving album %i' % album_id) + log.debug(u'moving album {0}', album_id) album.move() -update_cmd = ui.Subcommand('update', - help='update the library', aliases=('upd','up',)) -update_cmd.parser.add_option('-a', '--album', action='store_true', - help='match albums instead of tracks') -update_cmd.parser.add_option('-M', '--nomove', action='store_false', - default=True, dest='move', help="don't move files in library") -update_cmd.parser.add_option('-p', '--pretend', action='store_true', - help="show all changes but do nothing") -update_cmd.parser.add_option('-f', '--format', action='store', - help='print with custom format', default=None) + def update_func(lib, opts, args): - update_items(lib, decargs(args), opts.album, opts.move, opts.pretend) + update_items(lib, decargs(args), opts.album, ui.should_move(opts.move), + opts.pretend) + + +update_cmd = ui.Subcommand( + u'update', help=u'update the library', aliases=(u'upd', u'up',) +) +update_cmd.parser.add_album_option() +update_cmd.parser.add_format_option() +update_cmd.parser.add_option( + u'-m', u'--move', action='store_true', dest='move', + help=u"move files in the library directory" +) +update_cmd.parser.add_option( + u'-M', u'--nomove', action='store_false', dest='move', + help=u"don't move files in library" +) +update_cmd.parser.add_option( + u'-p', u'--pretend', action='store_true', + help=u"show all changes but do nothing" +) update_cmd.func = update_func default_commands.append(update_cmd) @@ -990,17 +1201,22 @@ def remove_items(lib, query, album, delete): # Get the matching items. items, albums = _do_query(lib, query, album) - # Show all the items. - for item in items: - ui.print_obj(item, lib) - - # Confirm with user. + # Prepare confirmation with user. print_() if delete: - prompt = 'Really DELETE %i files (y/n)?' % len(items) + fmt = u'$path - $title' + prompt = u'Really DELETE %i file%s (y/n)?' % \ + (len(items), 's' if len(items) > 1 else '') else: - prompt = 'Really remove %i items from the library (y/n)?' % \ - len(items) + fmt = '' + prompt = u'Really remove %i item%s from the library (y/n)?' % \ + (len(items), 's' if len(items) > 1 else '') + + # Show all the items. + for item in items: + ui.print_(format(item, fmt)) + + # Confirm with user. if not ui.input_yn(prompt, True): return @@ -1009,14 +1225,19 @@ def remove_items(lib, query, album, delete): for obj in (albums if album else items): obj.remove(delete) -remove_cmd = ui.Subcommand('remove', - help='remove matching items from the library', aliases=('rm',)) -remove_cmd.parser.add_option("-d", "--delete", action="store_true", - help="also remove files from disk") -remove_cmd.parser.add_option('-a', '--album', action='store_true', - help='match albums instead of tracks') + def remove_func(lib, opts, args): remove_items(lib, decargs(args), opts.album, opts.delete) + + +remove_cmd = ui.Subcommand( + u'remove', help=u'remove matching items from the library', aliases=(u'rm',) +) +remove_cmd.parser.add_option( + u"-d", u"--delete", action="store_true", + help=u"also remove files from disk" +) +remove_cmd.parser.add_album_option() remove_cmd.func = remove_func default_commands.append(remove_cmd) @@ -1032,34 +1253,55 @@ def show_stats(lib, query, exact): total_items = 0 artists = set() albums = set() + album_artists = set() for item in items: if exact: - total_size += os.path.getsize(item.path) + try: + total_size += os.path.getsize(syspath(item.path)) + except OSError as exc: + log.info(u'could not get size of {}: {}', item.path, exc) else: total_size += int(item.length * item.bitrate / 8) total_time += item.length total_items += 1 artists.add(item.artist) - albums.add(item.album) + album_artists.add(item.albumartist) + if item.album_id: + albums.add(item.album_id) - size_str = '' + ui.human_bytes(total_size) + size_str = u'' + ui.human_bytes(total_size) if exact: - size_str += ' ({0} bytes)'.format(total_size) + size_str += u' ({0} bytes)'.format(total_size) + + print_(u"""Tracks: {0} +Total time: {1}{2} +{3}: {4} +Artists: {5} +Albums: {6} +Album artists: {7}""".format( + total_items, + ui.human_seconds(total_time), + u' ({0:.2f} seconds)'.format(total_time) if exact else '', + u'Total size' if exact else u'Approximate total size', + size_str, + len(artists), + len(albums), + len(album_artists)), + ) - print_("""Tracks: {0} -Total time: {1} ({2:.2f} seconds) -Total size: {3} -Artists: {4} -Albums: {5}""".format(total_items, ui.human_seconds(total_time), total_time, - size_str, len(artists), len(albums))) -stats_cmd = ui.Subcommand('stats', - help='show statistics about the library or a query') -stats_cmd.parser.add_option('-e', '--exact', action='store_true', - help='get exact file sizes') def stats_func(lib, opts, args): show_stats(lib, decargs(args), opts.exact) + + +stats_cmd = ui.Subcommand( + u'stats', help=u'show statistics about the library or a query' +) +stats_cmd.parser.add_option( + u'-e', u'--exact', action='store_true', + help=u'exact size and time' +) stats_cmd.func = stats_func default_commands.append(stats_cmd) @@ -1067,15 +1309,18 @@ default_commands.append(stats_cmd) # version: Show current beets version. def show_version(lib, opts, args): - print_('beets version %s' % beets.__version__) + print_(u'beets version %s' % beets.__version__) # Show plugins. - names = [p.name for p in plugins.find_plugins()] + names = sorted(p.name for p in plugins.find_plugins()) if names: - print_('plugins:', ', '.join(names)) + print_(u'plugins:', ', '.join(names)) else: - print_('no plugins loaded') -version_cmd = ui.Subcommand('version', - help='output version information') + print_(u'no plugins loaded') + + +version_cmd = ui.Subcommand( + u'version', help=u'output version information' +) version_cmd.func = show_version default_commands.append(version_cmd) @@ -1083,13 +1328,17 @@ default_commands.append(version_cmd) # modify: Declaratively change metadata. def modify_items(lib, mods, dels, query, write, move, album, confirm): - """Modifies matching items according to key=value assignments.""" + """Modifies matching items according to user-specified assignments and + deletions. + + `mods` is a dictionary of field and value pairse indicating + assignments. `dels` is a list of fields to be deleted. + """ # Parse key=value specifications into a dictionary. model_cls = library.Album if album else library.Item - fsets = {} - for mod in mods: - key, value = mod.split('=', 1) - fsets[key] = model_cls._parse(key, value) + + for key, value in mods.items(): + mods[key] = model_cls._parse(key, value) # Get the items to modify. items, albums = _do_query(lib, query, album, False) @@ -1097,89 +1346,115 @@ def modify_items(lib, mods, dels, query, write, move, album, confirm): # Apply changes *temporarily*, preview them, and collect modified # objects. - print_('Modifying %i %ss.' % (len(objs), 'album' if album else 'item')) + print_(u'Modifying {0} {1}s.' + .format(len(objs), u'album' if album else u'item')) changed = set() for obj in objs: - for field, value in fsets.iteritems(): - obj[field] = value - for field in dels: - del obj[field] - if ui.show_model_changes(obj): + if print_and_modify(obj, mods, dels): changed.add(obj) # Still something to do? if not changed: - print_('No changes to make.') + print_(u'No changes to make.') return # Confirm action. if confirm: - extra = ' and write tags' if write else '' - if not ui.input_yn('Really modify%s (Y/n)?' % extra): - return + if write and move: + extra = u', move and write tags' + elif write: + extra = u' and write tags' + elif move: + extra = u' and move' + else: + extra = u'' - # Apply changes to database. + changed = ui.input_select_objects( + u'Really modify%s' % extra, changed, + lambda o: print_and_modify(o, mods, dels) + ) + + # Apply changes to database and files with lib.transaction(): for obj in changed: - if move: - cur_path = obj.path - if lib.directory in ancestry(cur_path): # In library? - log.debug('moving object %s' % cur_path) - obj.move() + obj.try_sync(write, move) - obj.store() - # Apply tags if requested. - if write: - if album: - changed_items = itertools.chain(*(a.items() for a in changed)) - else: - changed_items = changed - for item in changed_items: - try: - item.write() - except library.FileOperationError as exc: - log.error(exc) +def print_and_modify(obj, mods, dels): + """Print the modifications to an item and return a bool indicating + whether any changes were made. -modify_cmd = ui.Subcommand('modify', - help='change metadata fields', aliases=('mod',)) -modify_cmd.parser.add_option('-M', '--nomove', action='store_false', - default=True, dest='move', help="don't move files in library") -modify_cmd.parser.add_option('-w', '--write', action='store_true', - default=None, help="write new metadata to files' tags (default)") -modify_cmd.parser.add_option('-W', '--nowrite', action='store_false', - dest='write', help="don't write metadata (opposite of -w)") -modify_cmd.parser.add_option('-a', '--album', action='store_true', - help='modify whole albums instead of tracks') -modify_cmd.parser.add_option('-y', '--yes', action='store_true', - help='skip confirmation') -modify_cmd.parser.add_option('-f', '--format', action='store', - help='print with custom format', default=None) -def modify_func(lib, opts, args): - args = decargs(args) - mods = [] + `mods` is a dictionary of fields and values to update on the object; + `dels` is a sequence of fields to delete. + """ + obj.update(mods) + for field in dels: + try: + del obj[field] + except KeyError: + pass + return ui.show_model_changes(obj) + + +def modify_parse_args(args): + """Split the arguments for the modify subcommand into query parts, + assignments (field=value), and deletions (field!). Returns the result as + a three-tuple in that order. + """ + mods = {} dels = [] query = [] for arg in args: if arg.endswith('!') and '=' not in arg and ':' not in arg: - dels.append(arg[:-1]) - elif '=' in arg: - mods.append(arg) + dels.append(arg[:-1]) # Strip trailing !. + elif '=' in arg and ':' not in arg.split('=', 1)[0]: + key, val = arg.split('=', 1) + mods[key] = val else: query.append(arg) + return query, mods, dels + + +def modify_func(lib, opts, args): + query, mods, dels = modify_parse_args(decargs(args)) if not mods and not dels: - raise ui.UserError('no modifications specified') - write = opts.write if opts.write is not None else \ - config['import']['write'].get(bool) - modify_items(lib, mods, dels, query, write, opts.move, opts.album, - not opts.yes) + raise ui.UserError(u'no modifications specified') + modify_items(lib, mods, dels, query, ui.should_write(opts.write), + ui.should_move(opts.move), opts.album, not opts.yes) + + +modify_cmd = ui.Subcommand( + u'modify', help=u'change metadata fields', aliases=(u'mod',) +) +modify_cmd.parser.add_option( + u'-m', u'--move', action='store_true', dest='move', + help=u"move files in the library directory" +) +modify_cmd.parser.add_option( + u'-M', u'--nomove', action='store_false', dest='move', + help=u"don't move files in library" +) +modify_cmd.parser.add_option( + u'-w', u'--write', action='store_true', default=None, + help=u"write new metadata to files' tags (default)" +) +modify_cmd.parser.add_option( + u'-W', u'--nowrite', action='store_false', dest='write', + help=u"don't write metadata (opposite of -w)" +) +modify_cmd.parser.add_album_option() +modify_cmd.parser.add_format_option(target='item') +modify_cmd.parser.add_option( + u'-y', u'--yes', action='store_true', + help=u'skip confirmation' +) modify_cmd.func = modify_func default_commands.append(modify_cmd) # move: Move/copy files to the library or a new base directory. -def move_items(lib, dest, query, copy, album): +def move_items(lib, dest, query, copy, album, pretend, confirm=False): """Moves or copies items to a new base directory, given by dest. If dest is None, then the library's base directory is used, making the command "consolidate" files. @@ -1187,38 +1462,78 @@ def move_items(lib, dest, query, copy, album): items, albums = _do_query(lib, query, album, False) objs = albums if album else items - action = 'Copying' if copy else 'Moving' - entity = 'album' if album else 'item' - log.info('%s %i %ss.' % (action, len(objs), entity)) - for obj in objs: - log.debug('moving: %s' % obj.path) + # Filter out files that don't need to be moved. + isitemmoved = lambda item: item.path != item.destination(basedir=dest) + isalbummoved = lambda album: any(isitemmoved(i) for i in album.items()) + objs = [o for o in objs if (isalbummoved if album else isitemmoved)(o)] + + action = u'Copying' if copy else u'Moving' + act = u'copy' if copy else u'move' + entity = u'album' if album else u'item' + log.info(u'{0} {1} {2}{3}.', action, len(objs), entity, + u's' if len(objs) != 1 else u'') + if not objs: + return + + if pretend: + if album: + show_path_changes([(item.path, item.destination(basedir=dest)) + for obj in objs for item in obj.items()]) + else: + show_path_changes([(obj.path, obj.destination(basedir=dest)) + for obj in objs]) + else: + if confirm: + objs = ui.input_select_objects( + u'Really %s' % act, objs, + lambda o: show_path_changes( + [(o.path, o.destination(basedir=dest))])) + + for obj in objs: + log.debug(u'moving: {0}', util.displayable_path(obj.path)) + + obj.move(copy, basedir=dest) + obj.store() - obj.move(copy, basedir=dest) - obj.store() -move_cmd = ui.Subcommand('move', - help='move or copy items', aliases=('mv',)) -move_cmd.parser.add_option('-d', '--dest', metavar='DIR', dest='dest', - help='destination directory') -move_cmd.parser.add_option('-c', '--copy', default=False, action='store_true', - help='copy instead of moving') -move_cmd.parser.add_option('-a', '--album', default=False, action='store_true', - help='match whole albums instead of tracks') def move_func(lib, opts, args): dest = opts.dest if dest is not None: dest = normpath(dest) if not os.path.isdir(dest): - raise ui.UserError('no such directory: %s' % dest) + raise ui.UserError(u'no such directory: %s' % dest) - move_items(lib, dest, decargs(args), opts.copy, opts.album) + move_items(lib, dest, decargs(args), opts.copy, opts.album, opts.pretend, + opts.timid) + + +move_cmd = ui.Subcommand( + u'move', help=u'move or copy items', aliases=(u'mv',) +) +move_cmd.parser.add_option( + u'-d', u'--dest', metavar='DIR', dest='dest', + help=u'destination directory' +) +move_cmd.parser.add_option( + u'-c', u'--copy', default=False, action='store_true', + help=u'copy instead of moving' +) +move_cmd.parser.add_option( + u'-p', u'--pretend', default=False, action='store_true', + help=u'show how files would be moved, but don\'t touch anything' +) +move_cmd.parser.add_option( + u'-t', u'--timid', dest='timid', action='store_true', + help=u'always confirm all actions' +) +move_cmd.parser.add_album_option() move_cmd.func = move_func default_commands.append(move_cmd) # write: Write tags into files. -def write_items(lib, query, pretend): +def write_items(lib, query, pretend, force): """Write tag information from the database to the respective files in the filesystem. """ @@ -1227,48 +1542,45 @@ def write_items(lib, query, pretend): for item in items: # Item deleted? if not os.path.exists(syspath(item.path)): - log.info(u'missing file: {0}'.format( - util.displayable_path(item.path) - )) + log.info(u'missing file: {0}', util.displayable_path(item.path)) continue # Get an Item object reflecting the "clean" (on-disk) state. try: clean_item = library.Item.from_path(item.path) - except Exception as exc: - log.error(u'error reading {0}: {1}'.format( - displayable_path(item.path), exc - )) + except library.ReadError as exc: + log.error(u'error reading {0}: {1}', + displayable_path(item.path), exc) continue # Check for and display changes. changed = ui.show_model_changes(item, clean_item, - library.ITEM_KEYS_WRITABLE, always=True) - if changed and not pretend: - try: - item.write() - except library.FileOperationError as exc: - log.error(exc) + library.Item._media_tag_fields, force) + if (changed or force) and not pretend: + # We use `try_sync` here to keep the mtime up to date in the + # database. + item.try_sync(True, False) + -write_cmd = ui.Subcommand('write', help='write tag information to files') -write_cmd.parser.add_option('-p', '--pretend', action='store_true', - help="show all changes but do nothing") def write_func(lib, opts, args): - write_items(lib, decargs(args), opts.pretend) + write_items(lib, decargs(args), opts.pretend, opts.force) + + +write_cmd = ui.Subcommand(u'write', help=u'write tag information to files') +write_cmd.parser.add_option( + u'-p', u'--pretend', action='store_true', + help=u"show all changes but do nothing" +) +write_cmd.parser.add_option( + u'-f', u'--force', action='store_true', + help=u"write tags even if the existing tags match the database" +) write_cmd.func = write_func default_commands.append(write_cmd) # config: Show and edit user configuration. -config_cmd = ui.Subcommand('config', - help='show or edit the user configuration') -config_cmd.parser.add_option('-p', '--paths', action='store_true', - help='show files that configuration was loaded from') -config_cmd.parser.add_option('-e', '--edit', action='store_true', - help='edit user configuration with $EDITOR') -config_cmd.parser.add_option('-d', '--defaults', action='store_true', - help='include the default configuration') def config_func(lib, opts, args): # Make sure lazy configuration is loaded config.resolve() @@ -1289,52 +1601,74 @@ def config_func(lib, opts, args): filenames.insert(0, user_path) for filename in filenames: - print(filename) + print_(filename) # Open in editor. elif opts.edit: - path = config.user_config_path() - - if 'EDITOR' in os.environ: - editor = os.environ['EDITOR'] - args = [editor, editor, path] - elif platform.system() == 'Darwin': - args = ['open', 'open', '-n', path] - elif platform.system() == 'Windows': - # On windows we can execute arbitrary files. The os will - # take care of starting an appropriate application - args = [path, path] - else: - # Assume Unix - args = ['xdg-open', 'xdg-open', path] - - try: - os.execlp(*args) - except OSError: - raise ui.UserError("Could not edit configuration. Please" - "set the EDITOR environment variable.") + config_edit() # Dump configuration. else: - print(config.dump(full=opts.defaults)) + print_(config.dump(full=opts.defaults, redact=opts.redact)) + +def config_edit(): + """Open a program to edit the user configuration. + An empty config file is created if no existing config file exists. + """ + path = config.user_config_path() + editor = util.editor_command() + try: + if not os.path.isfile(path): + open(path, 'w+').close() + util.interactive_open([path], editor) + except OSError as exc: + message = u"Could not edit configuration: {0}".format(exc) + if not editor: + message += u". Please set the EDITOR environment variable" + raise ui.UserError(message) + +config_cmd = ui.Subcommand(u'config', + help=u'show or edit the user configuration') +config_cmd.parser.add_option( + u'-p', u'--paths', action='store_true', + help=u'show files that configuration was loaded from' +) +config_cmd.parser.add_option( + u'-e', u'--edit', action='store_true', + help=u'edit user configuration with $EDITOR' +) +config_cmd.parser.add_option( + u'-d', u'--defaults', action='store_true', + help=u'include the default configuration' +) +config_cmd.parser.add_option( + u'-c', u'--clear', action='store_false', + dest='redact', default=True, + help=u'do not redact sensitive fields' +) config_cmd.func = config_func default_commands.append(config_cmd) # completion: print completion script -completion_cmd = ui.Subcommand('completion', - help='print shell script that provides command line completion') def print_completion(*args): for line in completion_script(default_commands + plugins.commands()): - print(line, end='') - if not (os.path.isfile(u'/etc/bash_completion') or - os.path.isfile(u'/usr/share/bash-completion/bash_completion') or - os.path.isfile(u'/usr/share/local/bash-completion/bash_completion')): + print_(line, end='') + if not any(map(os.path.isfile, BASH_COMPLETION_PATHS)): log.warn(u'Warning: Unable to find the bash-completion package. ' u'Command line completion might not work.') +BASH_COMPLETION_PATHS = map(syspath, [ + u'/etc/bash_completion', + u'/usr/share/bash-completion/bash_completion', + u'/usr/share/local/bash-completion/bash_completion', + u'/opt/local/share/bash-completion/bash_completion', # SmartOS + u'/usr/local/etc/bash_completion', # Homebrew +]) + + def completion_script(commands): """Yield the full completion shell script as strings. @@ -1355,7 +1689,8 @@ def completion_script(commands): command_names.append(name) for alias in cmd.aliases: - aliases[alias] = name + if re.match(r'^\w+$', alias): + aliases[alias] = name options[name] = {'flags': [], 'opts': []} for opts in cmd.parser._get_all_options()[1:]: @@ -1370,46 +1705,50 @@ def completion_script(commands): # Add global options options['_global'] = { - 'flags': ['-v', '--verbose'], - 'opts': '-l --library -c --config -d --directory -h --help'.split(' ') + 'flags': [u'-v', u'--verbose'], + 'opts': u'-l --library -c --config -d --directory -h --help'.split( + u' ') } - # Help subcommand - command_names.append('help') - # Add flags common to all commands options['_common'] = { - 'flags': ['-h', '--help'] + 'flags': [u'-h', u'--help'] } # Start generating the script - yield "_beet() {\n" + yield u"_beet() {\n" # Command names - yield " local commands='%s'\n" % ' '.join(command_names) - yield "\n" + yield u" local commands='%s'\n" % ' '.join(command_names) + yield u"\n" # Command aliases - yield " local aliases='%s'\n" % ' '.join(aliases.keys()) + yield u" local aliases='%s'\n" % ' '.join(aliases.keys()) for alias, cmd in aliases.items(): - yield " local alias__%s=%s\n" % (alias, cmd) - yield '\n' + yield u" local alias__%s=%s\n" % (alias, cmd) + yield u'\n' # Fields - yield " fields='%s'\n" % ' '.join( - set(library.ITEM_KEYS + library.ALBUM_KEYS)) + yield u" fields='%s'\n" % ' '.join( + set(library.Item._fields.keys() + library.Album._fields.keys()) + ) # Command options for cmd, opts in options.items(): for option_type, option_list in opts.items(): if option_list: option_list = ' '.join(option_list) - yield " local %s__%s='%s'\n" % (option_type, cmd, option_list) + yield u" local %s__%s='%s'\n" % ( + option_type, cmd, option_list) - yield ' _beet_dispatch\n' - yield '}\n' + yield u' _beet_dispatch\n' + yield u'}\n' +completion_cmd = ui.Subcommand( + 'completion', + help=u'print shell script that provides command line completion' +) completion_cmd.func = print_completion completion_cmd.hide = True default_commands.append(completion_cmd) diff --git a/libs/beets/ui/migrate.py b/libs/beets/ui/migrate.py deleted file mode 100644 index 784d7c82..00000000 --- a/libs/beets/ui/migrate.py +++ /dev/null @@ -1,401 +0,0 @@ -# This file is part of beets. -# Copyright 2013, Adrian Sampson. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. - -"""Conversion from legacy (pre-1.1) configuration to Confit/YAML -configuration. -""" -import os -import ConfigParser -import codecs -import yaml -import logging -import time -import itertools -import re - -import beets -from beets import util -from beets import ui -from beets.util import confit - -CONFIG_PATH_VAR = 'BEETSCONFIG' -DEFAULT_CONFIG_FILENAME_UNIX = '.beetsconfig' -DEFAULT_CONFIG_FILENAME_WINDOWS = 'beetsconfig.ini' -DEFAULT_LIBRARY_FILENAME_UNIX = '.beetsmusic.blb' -DEFAULT_LIBRARY_FILENAME_WINDOWS = 'beetsmusic.blb' -WINDOWS_BASEDIR = os.environ.get('APPDATA') or '~' - -OLD_CONFIG_SUFFIX = '.old' -PLUGIN_NAMES = { - 'rdm': 'random', - 'fuzzy_search': 'fuzzy', -} -AUTO_KEYS = ('automatic', 'autofetch', 'autoembed', 'autoscrub') -IMPORTFEEDS_PREFIX = 'feeds_' -CONFIG_MIGRATED_MESSAGE = u""" -You appear to be upgrading from beets 1.0 (or earlier) to 1.1. Your -configuration file has been migrated automatically to: -{newconfig} -Edit this file to configure beets. You might want to remove your -old-style ".beetsconfig" file now. See the documentation for more -details on the new configuration system: -http://beets.readthedocs.org/page/reference/config.html -""".strip() -DB_MIGRATED_MESSAGE = u'Your database file has also been copied to:\n{newdb}' -YAML_COMMENT = '# Automatically migrated from legacy .beetsconfig.\n\n' - -log = logging.getLogger('beets') - -# An itertools recipe. -def grouper(n, iterable): - args = [iter(iterable)] * n - return itertools.izip_longest(*args) - -def _displace(fn): - """Move a file aside using a timestamp suffix so a new file can be - put in its place. - """ - util.move( - fn, - u'{0}.old.{1}'.format(fn, int(time.time())), - True - ) - -def default_paths(): - """Produces the appropriate default config and library database - paths for the current system. On Unix, this is always in ~. On - Windows, tries ~ first and then $APPDATA for the config and library - files (for backwards compatibility). - """ - windows = os.path.__name__ == 'ntpath' - if windows: - windata = os.environ.get('APPDATA') or '~' - - # Shorthand for joining paths. - def exp(*vals): - return os.path.expanduser(os.path.join(*vals)) - - config = exp('~', DEFAULT_CONFIG_FILENAME_UNIX) - if windows and not os.path.exists(config): - config = exp(windata, DEFAULT_CONFIG_FILENAME_WINDOWS) - - libpath = exp('~', DEFAULT_LIBRARY_FILENAME_UNIX) - if windows and not os.path.exists(libpath): - libpath = exp(windata, DEFAULT_LIBRARY_FILENAME_WINDOWS) - - return config, libpath - -def get_config(): - """Using the same logic as beets 1.0, locate and read the - .beetsconfig file. Return a ConfigParser instance or None if no - config is found. - """ - default_config, default_libpath = default_paths() - if CONFIG_PATH_VAR in os.environ: - configpath = os.path.expanduser(os.environ[CONFIG_PATH_VAR]) - else: - configpath = default_config - - config = ConfigParser.SafeConfigParser() - if os.path.exists(util.syspath(configpath)): - with codecs.open(configpath, 'r', encoding='utf-8') as f: - config.readfp(f) - return config, configpath - else: - return None, configpath - -def flatten_config(config): - """Given a ConfigParser, flatten the values into a dict-of-dicts - representation where each section gets its own dictionary of values. - """ - out = confit.OrderedDict() - for section in config.sections(): - sec_dict = out[section] = confit.OrderedDict() - for option in config.options(section): - sec_dict[option] = config.get(section, option, True) - return out - -def transform_value(value): - """Given a string read as the value of a config option, return a - massaged version of that value (possibly with a different type). - """ - # Booleans. - if value.lower() in ('false', 'no', 'off'): - return False - elif value.lower() in ('true', 'yes', 'on'): - return True - - # Integers. - try: - return int(value) - except ValueError: - pass - - # Floats. - try: - return float(value) - except ValueError: - pass - - return value - -def transform_data(data): - """Given a dict-of-dicts representation of legacy config data, tweak - the data into a new form. This new form is suitable for dumping as - YAML. - """ - out = confit.OrderedDict() - - for section, pairs in data.items(): - if section == 'beets': - # The "main" section. In the new config system, these values - # are in the "root": no section at all. - for key, value in pairs.items(): - value = transform_value(value) - - if key.startswith('import_'): - # Importer config is now under an "import:" key. - if 'import' not in out: - out['import'] = confit.OrderedDict() - out['import'][key[7:]] = value - - elif key == 'plugins': - # Renamed plugins. - plugins = value.split() - new_plugins = [PLUGIN_NAMES.get(p, p) for p in plugins] - out['plugins'] = ' '.join(new_plugins) - - elif key == 'replace': - # YAMLy representation for character replacements. - replacements = confit.OrderedDict() - for pat, repl in grouper(2, value.split()): - if repl == '': - repl = '' - replacements[pat] = repl - out['replace'] = replacements - - elif key == 'pluginpath': - # Used to be a colon-separated string. Now a list. - out['pluginpath'] = value.split(':') - - else: - out[key] = value - - elif pairs: - # Other sections (plugins, etc). - sec_out = out[section] = confit.OrderedDict() - for key, value in pairs.items(): - - # Standardized "auto" option. - if key in AUTO_KEYS: - key = 'auto' - - # Unnecessary : hack in queries. - if section == 'paths': - key = key.replace('_', ':') - - # Changed option names for importfeeds plugin. - if section == 'importfeeds': - if key.startswith(IMPORTFEEDS_PREFIX): - key = key[len(IMPORTFEEDS_PREFIX):] - - sec_out[key] = transform_value(value) - - return out - -class Dumper(yaml.SafeDumper): - """A PyYAML Dumper that represents OrderedDicts as ordinary mappings - (in order, of course). - """ - # From http://pyyaml.org/attachment/ticket/161/use_ordered_dict.py - def represent_mapping(self, tag, mapping, flow_style=None): - value = [] - node = yaml.MappingNode(tag, value, flow_style=flow_style) - if self.alias_key is not None: - self.represented_objects[self.alias_key] = node - best_style = True - if hasattr(mapping, 'items'): - mapping = list(mapping.items()) - for item_key, item_value in mapping: - node_key = self.represent_data(item_key) - node_value = self.represent_data(item_value) - if not (isinstance(node_key, yaml.ScalarNode) and \ - not node_key.style): - best_style = False - if not (isinstance(node_value, yaml.ScalarNode) and \ - not node_value.style): - best_style = False - value.append((node_key, node_value)) - if flow_style is None: - if self.default_flow_style is not None: - node.flow_style = self.default_flow_style - else: - node.flow_style = best_style - return node -Dumper.add_representer(confit.OrderedDict, Dumper.represent_dict) - -def migrate_config(replace=False): - """Migrate a legacy beetsconfig file to a new-style config.yaml file - in an appropriate place. If `replace` is enabled, then any existing - config.yaml will be moved aside. Otherwise, the process is aborted - when the file exists. - """ - - # Load legacy configuration data, if any. - config, configpath = get_config() - if not config: - log.debug(u'no config file found at {0}'.format( - util.displayable_path(configpath) - )) - return - - # Get the new configuration file path and possibly move it out of - # the way. - destfn = os.path.join(beets.config.config_dir(), confit.CONFIG_FILENAME) - if os.path.exists(destfn): - if replace: - log.debug(u'moving old config aside: {0}'.format( - util.displayable_path(destfn) - )) - _displace(destfn) - else: - # File exists and we won't replace it. We're done. - return - - log.debug(u'migrating config file {0}'.format( - util.displayable_path(configpath) - )) - - # Convert the configuration to a data structure ready to be dumped - # as the new Confit file. - data = transform_data(flatten_config(config)) - - # Encode result as YAML. - yaml_out = yaml.dump( - data, - Dumper=Dumper, - default_flow_style=False, - indent=4, - width=1000, - ) - # A ridiculous little hack to add some whitespace between "sections" - # in the YAML output. I hope this doesn't break any YAML syntax. - yaml_out = re.sub(r'(\n\w+:\n [^-\s])', '\n\\1', yaml_out) - yaml_out = YAML_COMMENT + yaml_out - - # Write the data to the new config destination. - log.debug(u'writing migrated config to {0}'.format( - util.displayable_path(destfn) - )) - with open(destfn, 'w') as f: - f.write(yaml_out) - return destfn - -def migrate_db(replace=False): - """Copy the beets library database file to the new location (e.g., - from ~/.beetsmusic.blb to ~/.config/beets/library.db). - """ - _, srcfn = default_paths() - destfn = beets.config['library'].as_filename() - - if not os.path.exists(srcfn) or srcfn == destfn: - # Old DB does not exist or we're configured to point to the same - # database. Do nothing. - return - - if os.path.exists(destfn): - if replace: - log.debug(u'moving old database aside: {0}'.format( - util.displayable_path(destfn) - )) - _displace(destfn) - else: - return - - log.debug(u'copying database from {0} to {1}'.format( - util.displayable_path(srcfn), util.displayable_path(destfn) - )) - util.copy(srcfn, destfn) - return destfn - -def migrate_state(replace=False): - """Copy the beets runtime state file from the old path (i.e., - ~/.beetsstate) to the new path (i.e., ~/.config/beets/state.pickle). - """ - srcfn = os.path.expanduser(os.path.join('~', '.beetsstate')) - if not os.path.exists(srcfn): - return - - destfn = beets.config['statefile'].as_filename() - if os.path.exists(destfn): - if replace: - _displace(destfn) - else: - return - - log.debug(u'copying state file from {0} to {1}'.format( - util.displayable_path(srcfn), util.displayable_path(destfn) - )) - util.copy(srcfn, destfn) - return destfn - - -# Automatic migration when beets starts. - -def automigrate(): - """Migrate the configuration, database, and state files. If any - migration occurs, print out a notice with some helpful next steps. - """ - config_fn = migrate_config() - db_fn = migrate_db() - migrate_state() - - if config_fn: - ui.print_(ui.colorize('fuchsia', u'MIGRATED CONFIGURATION')) - - ui.print_(CONFIG_MIGRATED_MESSAGE.format( - newconfig=util.displayable_path(config_fn)) - ) - if db_fn: - ui.print_(DB_MIGRATED_MESSAGE.format( - newdb=util.displayable_path(db_fn) - )) - - ui.input_(ui.colorize('fuchsia', u'Press ENTER to continue:')) - ui.print_() - - -# CLI command for explicit migration. - -migrate_cmd = ui.Subcommand('migrate', help='convert legacy config') -def migrate_func(lib, opts, args): - """Explicit command for migrating files. Existing files in each - destination are moved aside. - """ - config_fn = migrate_config(replace=True) - if config_fn: - log.info(u'Migrated configuration to: {0}'.format( - util.displayable_path(config_fn) - )) - db_fn = migrate_db(replace=True) - if db_fn: - log.info(u'Migrated library database to: {0}'.format( - util.displayable_path(db_fn) - )) - state_fn = migrate_state(replace=True) - if state_fn: - log.info(u'Migrated state file to: {0}'.format( - util.displayable_path(state_fn) - )) -migrate_cmd.func = migrate_func diff --git a/libs/beets/util/__init__.py b/libs/beets/util/__init__.py index f5810ff3..3cc270ae 100644 --- a/libs/beets/util/__init__.py +++ b/libs/beets/util/__init__.py @@ -1,5 +1,6 @@ +# -*- coding: utf-8 -*- # This file is part of beets. -# Copyright 2013, Adrian Sampson. +# Copyright 2016, Adrian Sampson. # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -13,20 +14,25 @@ # included in all copies or substantial portions of the Software. """Miscellaneous utility functions.""" -from __future__ import division +from __future__ import division, absolute_import, print_function import os import sys import re import shutil import fnmatch -from collections import defaultdict +from collections import Counter import traceback import subprocess +import platform +import shlex +from beets.util import hidden + MAX_FILENAME_LENGTH = 200 WINDOWS_MAGIC_PREFIX = u'\\\\?\\' + class HumanReadableException(Exception): """An Exception that can include a human-readable error message to be logged without a traceback. Can preserve a traceback for @@ -51,12 +57,12 @@ class HumanReadableException(Exception): def _gerund(self): """Generate a (likely) gerund form of the English verb. """ - if ' ' in self.verb: + if u' ' in self.verb: return self.verb - gerund = self.verb[:-1] if self.verb.endswith('e') else self.verb - gerund += 'ing' + gerund = self.verb[:-1] if self.verb.endswith(u'e') else self.verb + gerund += u'ing' return gerund - + def _reasonstr(self): """Get the reason as a string.""" if isinstance(self.reason, unicode): @@ -80,7 +86,8 @@ class HumanReadableException(Exception): """ if self.tb: logger.debug(self.tb) - logger.error(u'{0}: {1}'.format(self.error_kind, self.args[0])) + logger.error(u'{0}: {1}', self.error_kind, self.args[0]) + class FilesystemError(HumanReadableException): """An error that occurred while performing a filesystem manipulation @@ -111,6 +118,7 @@ class FilesystemError(HumanReadableException): return u'{0} {1}'.format(self._reasonstr(), clause) + def normpath(path): """Provide the canonical form of the path suitable for storing in the database. @@ -119,6 +127,7 @@ def normpath(path): path = os.path.normpath(os.path.abspath(os.path.expanduser(path))) return bytestring_path(path) + def ancestry(path): """Return a list consisting of path's parent directory, its grandparent, and so on. For instance: @@ -137,11 +146,13 @@ def ancestry(path): break last_path = path - if path: # don't yield '' + if path: + # don't yield '' out.insert(0, path) return out -def sorted_walk(path, ignore=(), logger=None): + +def sorted_walk(path, ignore=(), ignore_hidden=False, logger=None): """Like `os.walk`, but yields things in case-insensitive sorted, breadth-first order. Directory and file names matching any glob pattern in `ignore` are skipped. If `logger` is provided, then @@ -175,10 +186,11 @@ def sorted_walk(path, ignore=(), logger=None): # Add to output as either a file or a directory. cur = os.path.join(path, base) - if os.path.isdir(syspath(cur)): - dirs.append(base) - else: - files.append(base) + if (ignore_hidden and not hidden.is_hidden(cur)) or not ignore_hidden: + if os.path.isdir(syspath(cur)): + dirs.append(base) + else: + files.append(base) # Sort lists (case-insensitive) and yield the current level. dirs.sort(key=bytes.lower) @@ -189,9 +201,10 @@ def sorted_walk(path, ignore=(), logger=None): for base in dirs: cur = os.path.join(path, base) # yield from sorted_walk(...) - for res in sorted_walk(cur, ignore, logger): + for res in sorted_walk(cur, ignore, ignore_hidden, logger): yield res + def mkdirall(path): """Make all the enclosing directories of path (like mkdir -p on the parent). @@ -204,6 +217,7 @@ def mkdirall(path): raise FilesystemError(exc, 'create', (ancestor,), traceback.format_exc()) + def fnmatch_all(names, patterns): """Determine whether all strings in `names` match at least one of the `patterns`, which should be shell glob expressions. @@ -218,6 +232,7 @@ def fnmatch_all(names, patterns): return False return True + def prune_dirs(path, root=None, clutter=('.DS_Store', 'Thumbs.db')): """If path is an empty directory, then remove it. Recursively remove path's ancestry up to root (which is never removed) where there are @@ -236,7 +251,7 @@ def prune_dirs(path, root=None, clutter=('.DS_Store', 'Thumbs.db')): ancestors = [] elif root in ancestors: # Only remove directories below the root. - ancestors = ancestors[ancestors.index(root)+1:] + ancestors = ancestors[ancestors.index(root) + 1:] else: # Remove nothing. return @@ -258,6 +273,7 @@ def prune_dirs(path, root=None, clutter=('.DS_Store', 'Thumbs.db')): else: break + def components(path): """Return a list of the path components in path. For instance: @@ -281,6 +297,7 @@ def components(path): return comps + def _fsencoding(): """Get the system's filesystem encoding. On Windows, this is always UTF-8 (not MBCS). @@ -295,12 +312,13 @@ def _fsencoding(): encoding = 'utf8' return encoding + def bytestring_path(path): - """Given a path, which is either a str or a unicode, returns a str + """Given a path, which is either a bytes or a unicode, returns a str path (ensuring that we never deal with Unicode pathnames). """ # Pass through bytestrings. - if isinstance(path, str): + if isinstance(path, bytes): return path # On Windows, remove the magic prefix added by `syspath`. This makes @@ -315,6 +333,7 @@ def bytestring_path(path): except (UnicodeError, LookupError): return path.encode('utf8') + def displayable_path(path, separator=u'; '): """Attempts to decode a bytestring path to a unicode object for the purpose of displaying it to the user. If the `path` argument is a @@ -324,7 +343,7 @@ def displayable_path(path, separator=u'; '): return separator.join(displayable_path(p) for p in path) elif isinstance(path, unicode): return path - elif not isinstance(path, str): + elif not isinstance(path, bytes): # A non-string object: just get its unicode representation. return unicode(path) @@ -333,6 +352,7 @@ def displayable_path(path, separator=u'; '): except (UnicodeError, LookupError): return path.decode('utf8', 'ignore') + def syspath(path, prefix=True): """Convert a path for use by the operating system. In particular, paths on Windows must receive a magic prefix and must be converted @@ -356,16 +376,22 @@ def syspath(path, prefix=True): encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() path = path.decode(encoding, 'replace') - # Add the magic prefix if it isn't already there + # Add the magic prefix if it isn't already there. + # http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247.aspx if prefix and not path.startswith(WINDOWS_MAGIC_PREFIX): + if path.startswith(u'\\\\'): + # UNC path. Final path should look like \\?\UNC\... + path = u'UNC' + path[1:] path = WINDOWS_MAGIC_PREFIX + path return path + def samefile(p1, p2): """Safer equality for paths.""" return shutil._samefile(syspath(p1), syspath(p2)) + def remove(path, soft=True): """Remove the file. If `soft`, then no error will be raised if the file does not exist. @@ -378,6 +404,7 @@ def remove(path, soft=True): except (OSError, IOError) as exc: raise FilesystemError(exc, 'delete', (path,), traceback.format_exc()) + def copy(path, dest, replace=False): """Copy a plain file. Permissions are not copied. If `dest` already exists, raises a FilesystemError unless `replace` is True. Has no @@ -389,13 +416,14 @@ def copy(path, dest, replace=False): path = syspath(path) dest = syspath(dest) if not replace and os.path.exists(dest): - raise FilesystemError('file exists', 'copy', (path, dest)) + raise FilesystemError(u'file exists', 'copy', (path, dest)) try: shutil.copyfile(path, dest) except (OSError, IOError) as exc: raise FilesystemError(exc, 'copy', (path, dest), traceback.format_exc()) + def move(path, dest, replace=False): """Rename a file. `dest` may not be a directory. If `dest` already exists, raises an OSError unless `replace` is True. Has no effect if @@ -409,7 +437,7 @@ def move(path, dest, replace=False): path = syspath(path) dest = syspath(dest) if os.path.exists(dest) and not replace: - raise FilesystemError('file exists', 'rename', (path, dest), + raise FilesystemError(u'file exists', 'rename', (path, dest), traceback.format_exc()) # First, try renaming the file. @@ -424,6 +452,27 @@ def move(path, dest, replace=False): raise FilesystemError(exc, 'move', (path, dest), traceback.format_exc()) + +def link(path, dest, replace=False): + """Create a symbolic link from path to `dest`. Raises an OSError if + `dest` already exists, unless `replace` is True. Does nothing if + `path` == `dest`.""" + if (samefile(path, dest)): + return + + path = syspath(path) + dest = syspath(dest) + if os.path.exists(dest) and not replace: + raise FilesystemError(u'file exists', 'rename', (path, dest), + traceback.format_exc()) + try: + os.symlink(path, dest) + except OSError: + raise FilesystemError(u'Operating system does not support symbolic ' + u'links.', 'link', (path, dest), + traceback.format_exc()) + + def unique_path(path): """Returns a version of ``path`` that does not exist on the filesystem. Specifically, if ``path` itself already exists, then @@ -433,7 +482,7 @@ def unique_path(path): return path base, ext = os.path.splitext(path) - match = re.search(r'\.(\d)+$', base) + match = re.search(br'\.(\d)+$', base) if match: num = int(match.group(1)) base = base[:match.start()] @@ -441,7 +490,7 @@ def unique_path(path): num = 0 while True: num += 1 - new_path = '%s.%i%s' % (base, num, ext) + new_path = b'%s.%i%s' % (base, num, ext) if not os.path.exists(new_path): return new_path @@ -450,13 +499,15 @@ def unique_path(path): # shares, which are sufficiently common as to cause frequent problems. # http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247.aspx CHAR_REPLACE = [ - (re.compile(ur'[\\/]'), u'_'), # / and \ -- forbidden everywhere. - (re.compile(ur'^\.'), u'_'), # Leading dot (hidden files on Unix). - (re.compile(ur'[\x00-\x1f]'), u''), # Control characters. - (re.compile(ur'[<>:"\?\*\|]'), u'_'), # Windows "reserved characters". - (re.compile(ur'\.$'), u'_'), # Trailing dots. - (re.compile(ur'\s+$'), u''), # Trailing whitespace. + (re.compile(r'[\\/]'), u'_'), # / and \ -- forbidden everywhere. + (re.compile(r'^\.'), u'_'), # Leading dot (hidden files on Unix). + (re.compile(r'[\x00-\x1f]'), u''), # Control characters. + (re.compile(r'[<>:"\?\*\|]'), u'_'), # Windows "reserved characters". + (re.compile(r'\.$'), u'_'), # Trailing dots. + (re.compile(r'\s+$'), u''), # Trailing whitespace. ] + + def sanitize_path(path, replacements=None): """Takes a path (as a Unicode string) and makes sure that it is legal. Returns a new path. Only works with fragments; won't work @@ -477,6 +528,7 @@ def sanitize_path(path, replacements=None): comps[i] = comp return os.path.join(*comps) + def truncate_path(path, length=MAX_FILENAME_LENGTH): """Given a bytestring path or a Unicode path fragment, truncate the components to a legal length. In the last component, the extension @@ -493,12 +545,83 @@ def truncate_path(path, length=MAX_FILENAME_LENGTH): return os.path.join(*out) + +def _legalize_stage(path, replacements, length, extension, fragment): + """Perform a single round of path legalization steps + (sanitation/replacement, encoding from Unicode to bytes, + extension-appending, and truncation). Return the path (Unicode if + `fragment` is set, `bytes` otherwise) and whether truncation was + required. + """ + # Perform an initial sanitization including user replacements. + path = sanitize_path(path, replacements) + + # Encode for the filesystem. + if not fragment: + path = bytestring_path(path) + + # Preserve extension. + path += extension.lower() + + # Truncate too-long components. + pre_truncate_path = path + path = truncate_path(path, length) + + return path, path != pre_truncate_path + + +def legalize_path(path, replacements, length, extension, fragment): + """Given a path-like Unicode string, produce a legal path. Return + the path and a flag indicating whether some replacements had to be + ignored (see below). + + The legalization process (see `_legalize_stage`) consists of + applying the sanitation rules in `replacements`, encoding the string + to bytes (unless `fragment` is set), truncating components to + `length`, appending the `extension`. + + This function performs up to three calls to `_legalize_stage` in + case truncation conflicts with replacements (as can happen when + truncation creates whitespace at the end of the string, for + example). The limited number of iterations iterations avoids the + possibility of an infinite loop of sanitation and truncation + operations, which could be caused by replacement rules that make the + string longer. The flag returned from this function indicates that + the path has to be truncated twice (indicating that replacements + made the string longer again after it was truncated); the + application should probably log some sort of warning. + """ + + if fragment: + # Outputting Unicode. + extension = extension.decode('utf8', 'ignore') + + first_stage_path, _ = _legalize_stage( + path, replacements, length, extension, fragment + ) + + # Convert back to Unicode with extension removed. + first_stage_path, _ = os.path.splitext(displayable_path(first_stage_path)) + + # Re-sanitize following truncation (including user replacements). + second_stage_path, retruncated = _legalize_stage( + first_stage_path, replacements, length, extension, fragment + ) + + # If the path was once again truncated, discard user replacements + # and run through one last legalization stage. + if retruncated: + second_stage_path, _ = _legalize_stage( + first_stage_path, None, length, extension, fragment + ) + + return second_stage_path, retruncated + + def str2bool(value): """Returns a boolean reflecting a human-entered string.""" - if value.lower() in ('yes', '1', 'true', 't', 'y'): - return True - else: - return False + return value.lower() in (u'yes', u'1', u'true', u't', u'y') + def as_string(value): """Convert a value to a Unicode object for matching with a query. @@ -507,56 +630,23 @@ def as_string(value): if value is None: return u'' elif isinstance(value, buffer): - return str(value).decode('utf8', 'ignore') - elif isinstance(value, str): + return bytes(value).decode('utf8', 'ignore') + elif isinstance(value, bytes): return value.decode('utf8', 'ignore') else: return unicode(value) -def levenshtein(s1, s2): - """A nice DP edit distance implementation from Wikibooks: - http://en.wikibooks.org/wiki/Algorithm_implementation/Strings/ - Levenshtein_distance#Python - """ - if len(s1) < len(s2): - return levenshtein(s2, s1) - if not s1: - return len(s2) - - previous_row = xrange(len(s2) + 1) - for i, c1 in enumerate(s1): - current_row = [i + 1] - for j, c2 in enumerate(s2): - insertions = previous_row[j + 1] + 1 - deletions = current_row[j] + 1 - substitutions = previous_row[j] + (c1 != c2) - current_row.append(min(insertions, deletions, substitutions)) - previous_row = current_row - - return previous_row[-1] def plurality(objs): - """Given a sequence of comparable objects, returns the object that - is most common in the set and the frequency of that object. The + """Given a sequence of hashble objects, returns the object that + is most common in the set and the its number of appearance. The sequence must contain at least one object. """ - # Calculate frequencies. - freqs = defaultdict(int) - for obj in objs: - freqs[obj] += 1 + c = Counter(objs) + if not c: + raise ValueError(u'sequence must be non-empty') + return c.most_common(1)[0] - if not freqs: - raise ValueError('sequence must be non-empty') - - # Find object with maximum frequency. - max_freq = 0 - res = None - for obj, freq in freqs.items(): - if freq > max_freq: - max_freq = freq - res = obj - - return res, max_freq def cpu_count(): """Return the number of hardware thread contexts (cores or SMT @@ -571,8 +661,8 @@ def cpu_count(): num = 0 elif sys.platform == 'darwin': try: - num = int(os.popen('sysctl -n hw.ncpu').read()) - except ValueError: + num = int(command_output([b'/usr/sbin/sysctl', b'-n', b'hw.ncpu'])) + except (ValueError, OSError, subprocess.CalledProcessError): num = 0 else: try: @@ -584,23 +674,38 @@ def cpu_count(): else: return 1 -def command_output(cmd): - """Wraps the `subprocess` module to invoke a command (given as a - list of arguments starting with the command name) and collect - stdout. The stderr stream is ignored. May raise - `subprocess.CalledProcessError` or an `OSError`. - This replaces `subprocess.check_output`, which isn't available in - Python 2.6 and which can have problems if lots of output is sent to - stderr. +def command_output(cmd, shell=False): + """Runs the command and returns its output after it has exited. + + ``cmd`` is a list of byte string arguments starting with the command names. + If ``shell`` is true, ``cmd`` is assumed to be a string and passed to a + shell to execute. + + If the process exits with a non-zero return code + ``subprocess.CalledProcessError`` is raised. May also raise + ``OSError``. + + This replaces `subprocess.check_output` which can have problems if lots of + output is sent to stderr. """ - with open(os.devnull, 'w') as devnull: - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=devnull) - stdout, _ = proc.communicate() + proc = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + close_fds=platform.system() != 'Windows', + shell=shell + ) + stdout, stderr = proc.communicate() if proc.returncode: - raise subprocess.CalledProcessError(proc.returncode, cmd) + raise subprocess.CalledProcessError( + returncode=proc.returncode, + cmd=b' '.join(cmd), + output=stdout + stderr, + ) return stdout + def max_filename_length(path, limit=MAX_FILENAME_LENGTH): """Attempt to determine the maximum filename length for the filesystem containing `path`. If the value is greater than `limit`, @@ -616,3 +721,142 @@ def max_filename_length(path, limit=MAX_FILENAME_LENGTH): return min(res[9], limit) else: return limit + + +def open_anything(): + """Return the system command that dispatches execution to the correct + program. + """ + sys_name = platform.system() + if sys_name == 'Darwin': + base_cmd = 'open' + elif sys_name == 'Windows': + base_cmd = 'start' + else: # Assume Unix + base_cmd = 'xdg-open' + return base_cmd + + +def editor_command(): + """Get a command for opening a text file. + + Use the `EDITOR` environment variable by default. If it is not + present, fall back to `open_anything()`, the platform-specific tool + for opening files in general. + """ + editor = os.environ.get('EDITOR') + if editor: + return editor + return open_anything() + + +def shlex_split(s): + """Split a Unicode or bytes string according to shell lexing rules. + + Raise `ValueError` if the string is not a well-formed shell string. + This is a workaround for a bug in some versions of Python. + """ + if isinstance(s, bytes): + # Shlex works fine. + return shlex.split(s) + + elif isinstance(s, unicode): + # Work around a Python bug. + # http://bugs.python.org/issue6988 + bs = s.encode('utf8') + return [c.decode('utf8') for c in shlex.split(bs)] + + else: + raise TypeError(u'shlex_split called with non-string') + + +def interactive_open(targets, command): + """Open the files in `targets` by `exec`ing a new `command`, given + as a Unicode string. (The new program takes over, and Python + execution ends: this does not fork a subprocess.) + + Can raise `OSError`. + """ + assert command + + # Split the command string into its arguments. + try: + args = shlex_split(command) + except ValueError: # Malformed shell tokens. + args = [command] + + args.insert(0, args[0]) # for argv[0] + + args += targets + + return os.execlp(*args) + + +def _windows_long_path_name(short_path): + """Use Windows' `GetLongPathNameW` via ctypes to get the canonical, + long path given a short filename. + """ + if not isinstance(short_path, unicode): + short_path = unicode(short_path) + + import ctypes + buf = ctypes.create_unicode_buffer(260) + get_long_path_name_w = ctypes.windll.kernel32.GetLongPathNameW + return_value = get_long_path_name_w(short_path, buf, 260) + + if return_value == 0 or return_value > 260: + # An error occurred + return short_path + else: + long_path = buf.value + # GetLongPathNameW does not change the case of the drive + # letter. + if len(long_path) > 1 and long_path[1] == ':': + long_path = long_path[0].upper() + long_path[1:] + return long_path + + +def case_sensitive(path): + """Check whether the filesystem at the given path is case sensitive. + + To work best, the path should point to a file or a directory. If the path + does not exist, assume a case sensitive file system on every platform + except Windows. + """ + # A fallback in case the path does not exist. + if not os.path.exists(syspath(path)): + # By default, the case sensitivity depends on the platform. + return platform.system() != 'Windows' + + # If an upper-case version of the path exists but a lower-case + # version does not, then the filesystem must be case-sensitive. + # (Otherwise, we have more work to do.) + if not (os.path.exists(syspath(path.lower())) and + os.path.exists(syspath(path.upper()))): + return True + + # Both versions of the path exist on the file system. Check whether + # they refer to different files by their inodes. Alas, + # `os.path.samefile` is only available on Unix systems on Python 2. + if platform.system() != 'Windows': + return not os.path.samefile(syspath(path.lower()), + syspath(path.upper())) + + # On Windows, we check whether the canonical, long filenames for the + # files are the same. + lower = _windows_long_path_name(path.lower()) + upper = _windows_long_path_name(path.upper()) + return lower != upper + + +def raw_seconds_short(string): + """Formats a human-readable M:SS string as a float (number of seconds). + + Raises ValueError if the conversion cannot take place due to `string` not + being in the right format. + """ + match = re.match(r'^(\d+):([0-5]\d)$', string) + if not match: + raise ValueError(u'String not in M:SS format') + minutes, seconds = map(int, match.groups()) + return float(minutes * 60 + seconds) diff --git a/libs/beets/util/artresizer.py b/libs/beets/util/artresizer.py index 6e367a0a..6970a7da 100644 --- a/libs/beets/util/artresizer.py +++ b/libs/beets/util/artresizer.py @@ -1,5 +1,6 @@ +# -*- coding: utf-8 -*- # This file is part of beets. -# Copyright 2013, Fabrice Laporte +# Copyright 2016, Fabrice Laporte # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -15,11 +16,15 @@ """Abstraction layer to resize images using PIL, ImageMagick, or a public resizing proxy if neither is available. """ +from __future__ import division, absolute_import, print_function + import urllib import subprocess import os +import re from tempfile import NamedTemporaryFile -import logging + +from beets import logging from beets import util # Resizing methods @@ -37,8 +42,8 @@ def resize_url(url, maxwidth): maxwidth (preserving aspect ratio). """ return '{0}?{1}'.format(PROXY_URL, urllib.urlencode({ - 'url': url.replace('http://',''), - 'w': str(maxwidth), + 'url': url.replace('http://', ''), + 'w': bytes(maxwidth), })) @@ -57,9 +62,8 @@ def pil_resize(maxwidth, path_in, path_out=None): """ path_out = path_out or temp_file_for(path_in) from PIL import Image - log.debug(u'artresizer: PIL resizing {0} to {1}'.format( - util.displayable_path(path_in), util.displayable_path(path_out) - )) + log.debug(u'artresizer: PIL resizing {0} to {1}', + util.displayable_path(path_in), util.displayable_path(path_out)) try: im = Image.open(util.syspath(path_in)) @@ -68,20 +72,18 @@ def pil_resize(maxwidth, path_in, path_out=None): im.save(path_out) return path_out except IOError: - log.error(u"PIL cannot create thumbnail for '{0}'".format( - util.displayable_path(path_in) - )) + log.error(u"PIL cannot create thumbnail for '{0}'", + util.displayable_path(path_in)) return path_in def im_resize(maxwidth, path_in, path_out=None): """Resize using ImageMagick's ``convert`` tool. - tool. Return the output path of resized image. + Return the output path of resized image. """ path_out = path_out or temp_file_for(path_in) - log.debug(u'artresizer: ImageMagick resizing {0} to {1}'.format( - util.displayable_path(path_in), util.displayable_path(path_out) - )) + log.debug(u'artresizer: ImageMagick resizing {0} to {1}', + util.displayable_path(path_in), util.displayable_path(path_out)) # "-resize widthxheight>" shrinks images with dimension(s) larger # than the corresponding width and/or height dimension(s). The > @@ -89,13 +91,13 @@ def im_resize(maxwidth, path_in, path_out=None): # compatibility. try: util.command_output([ - 'convert', util.syspath(path_in), - '-resize', '{0}x^>'.format(maxwidth), path_out + b'convert', util.syspath(path_in, prefix=False), + b'-resize', b'{0}x^>'.format(maxwidth), + util.syspath(path_out, prefix=False), ]) except subprocess.CalledProcessError: - log.warn(u'artresizer: IM convert failed for {0}'.format( - util.displayable_path(path_in) - )) + log.warn(u'artresizer: IM convert failed for {0}', + util.displayable_path(path_in)) return path_in return path_out @@ -106,21 +108,56 @@ BACKEND_FUNCS = { } +def pil_getsize(path_in): + from PIL import Image + try: + im = Image.open(util.syspath(path_in)) + return im.size + except IOError as exc: + log.error(u"PIL could not read file {}: {}", + util.displayable_path(path_in), exc) + + +def im_getsize(path_in): + cmd = [b'identify', b'-format', b'%w %h', + util.syspath(path_in, prefix=False)] + try: + out = util.command_output(cmd) + except subprocess.CalledProcessError as exc: + log.warn(u'ImageMagick size query failed') + log.debug( + u'`convert` exited with (status {}) when ' + u'getting size with command {}:\n{}', + exc.returncode, cmd, exc.output.strip() + ) + return + try: + return tuple(map(int, out.split(b' '))) + except IndexError: + log.warn(u'Could not understand IM output: {0!r}', out) + + +BACKEND_GET_SIZE = { + PIL: pil_getsize, + IMAGEMAGICK: im_getsize, +} + + class Shareable(type): """A pseudo-singleton metaclass that allows both shared and non-shared instances. The ``MyClass.shared`` property holds a lazily-created shared instance of ``MyClass`` while calling ``MyClass()`` to construct a new object works as usual. """ - def __init__(cls, name, bases, dict): - super(Shareable, cls).__init__(name, bases, dict) - cls._instance = None + def __init__(self, name, bases, dict): + super(Shareable, self).__init__(name, bases, dict) + self._instance = None @property - def shared(cls): - if cls._instance is None: - cls._instance = cls() - return cls._instance + def shared(self): + if self._instance is None: + self._instance = self() + return self._instance class ArtResizer(object): @@ -128,12 +165,12 @@ class ArtResizer(object): """ __metaclass__ = Shareable - def __init__(self, method=None): - """Create a resizer object for the given method or, if none is - specified, with an inferred method. + def __init__(self): + """Create a resizer object with an inferred method. """ - self.method = method or self._guess_method() - log.debug(u"artresizer: method is {0}".format(self.method)) + self.method = self._check_method() + log.debug(u"artresizer: method is {0}", self.method) + self.can_compare = self._can_compare() def resize(self, maxwidth, path_in, path_out=None): """Manipulate an image file according to the method, returning a @@ -141,7 +178,7 @@ class ArtResizer(object): temporary file. For WEBPROXY, returns `path_in` unmodified. """ if self.local: - func = BACKEND_FUNCS[self.method] + func = BACKEND_FUNCS[self.method[0]] return func(maxwidth, path_in, path_out) else: return path_in @@ -159,30 +196,63 @@ class ArtResizer(object): @property def local(self): """A boolean indicating whether the resizing method is performed - locally (i.e., PIL or IMAGEMAGICK). + locally (i.e., PIL or ImageMagick). """ - return self.method in BACKEND_FUNCS + return self.method[0] in BACKEND_FUNCS + + def get_size(self, path_in): + """Return the size of an image file as an int couple (width, height) + in pixels. + + Only available locally + """ + if self.local: + func = BACKEND_GET_SIZE[self.method[0]] + return func(path_in) + + def _can_compare(self): + """A boolean indicating whether image comparison is available""" + + return self.method[0] == IMAGEMAGICK and self.method[1] > (6, 8, 7) @staticmethod - def _guess_method(): - """Determine which resizing method to use. Returns PIL, - IMAGEMAGICK, or WEBPROXY depending on available dependencies. - """ - # Try importing PIL. - try: - __import__('PIL', fromlist=['Image']) - return PIL - except ImportError: - pass + def _check_method(): + """Return a tuple indicating an available method and its version.""" + version = get_im_version() + if version: + return IMAGEMAGICK, version - # Try invoking ImageMagick's "convert". - try: - out = util.command_output(['convert', '--version']) - if 'imagemagick' in out.lower(): - # system32/convert.exe may be interfering - return IMAGEMAGICK - except (subprocess.CalledProcessError, OSError): - pass + version = get_pil_version() + if version: + return PIL, version - # Fall back to Web proxy method. - return WEBPROXY + return WEBPROXY, (0) + + +def get_im_version(): + """Return Image Magick version or None if it is unavailable + Try invoking ImageMagick's "convert".""" + try: + out = util.command_output([b'identify', b'--version']) + + if 'imagemagick' in out.lower(): + pattern = r".+ (\d+)\.(\d+)\.(\d+).*" + match = re.search(pattern, out) + if match: + return (int(match.group(1)), + int(match.group(2)), + int(match.group(3))) + return (0,) + + except (subprocess.CalledProcessError, OSError): + return None + + +def get_pil_version(): + """Return Image Magick version or None if it is unavailable + Try importing PIL.""" + try: + __import__('PIL', fromlist=[str('Image')]) + return (0,) + except ImportError: + return None diff --git a/libs/beets/util/bluelet.py b/libs/beets/util/bluelet.py index 9d9432f2..d81c2919 100644 --- a/libs/beets/util/bluelet.py +++ b/libs/beets/util/bluelet.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + """Extremely simple pure-Python implementation of coroutine-style asynchronous socket I/O. Inspired by, but inferior to, Eventlet. Bluelet can also be thought of as a less-terrible replacement for @@ -5,6 +7,8 @@ asyncore. Bluelet: easy concurrency without all the messy parallelism. """ +from __future__ import division, absolute_import, print_function + import socket import select import sys @@ -38,6 +42,7 @@ class Event(object): """ pass + class WaitableEvent(Event): """A waitable event is one encapsulating an action that can be waited for using a select() call. That is, it's an event with an @@ -57,21 +62,25 @@ class WaitableEvent(Event): """ pass + class ValueEvent(Event): """An event that does nothing but return a fixed value.""" def __init__(self, value): self.value = value + class ExceptionEvent(Event): """Raise an exception at the yield point. Used internally.""" def __init__(self, exc_info): self.exc_info = exc_info + class SpawnEvent(Event): """Add a new coroutine thread to the scheduler.""" def __init__(self, coro): self.spawned = coro + class JoinEvent(Event): """Suspend the thread until the specified child thread has completed. @@ -79,11 +88,13 @@ class JoinEvent(Event): def __init__(self, child): self.child = child + class KillEvent(Event): """Unschedule a child thread.""" def __init__(self, child): self.child = child + class DelegationEvent(Event): """Suspend execution of the current thread, start a new thread and, once the child thread finished, return control to the parent @@ -92,6 +103,7 @@ class DelegationEvent(Event): def __init__(self, coro): self.spawned = coro + class ReturnEvent(Event): """Return a value the current thread's delegator at the point of delegation. Ends the current (delegate) thread. @@ -99,6 +111,7 @@ class ReturnEvent(Event): def __init__(self, value): self.value = value + class SleepEvent(WaitableEvent): """Suspend the thread for a given duration. """ @@ -108,6 +121,7 @@ class SleepEvent(WaitableEvent): def time_left(self): return max(self.wakeup_time - time.time(), 0.0) + class ReadEvent(WaitableEvent): """Reads from a file-like object.""" def __init__(self, fd, bufsize): @@ -120,6 +134,7 @@ class ReadEvent(WaitableEvent): def fire(self): return self.fd.read(self.bufsize) + class WriteEvent(WaitableEvent): """Writes to a file-like object.""" def __init__(self, fd, data): @@ -192,15 +207,19 @@ def _event_select(events): return ready_events + class ThreadException(Exception): def __init__(self, coro, exc_info): self.coro = coro self.exc_info = exc_info + def reraise(self): _reraise(self.exc_info[0], self.exc_info[1], self.exc_info[2]) + SUSPENDED = Event() # Special sentinel placeholder for suspended threads. + class Delegated(Event): """Placeholder indicating that a thread has delegated execution to a different thread. @@ -208,6 +227,7 @@ class Delegated(Event): def __init__(self, child): self.child = child + def run(root_coro): """Schedules a coroutine, running it to completion. This encapsulates the Bluelet scheduler, which the root coroutine can @@ -329,7 +349,7 @@ def run(root_coro): break # Wait and fire. - event2coro = dict((v,k) for k,v in threads.items()) + event2coro = dict((v, k) for k, v in threads.items()) for event in _event_select(threads.values()): # Run the IO operation, but catch socket errors. try: @@ -378,6 +398,7 @@ def run(root_coro): class SocketClosedError(Exception): pass + class Listener(object): """A socket wrapper object for listening sockets. """ @@ -407,6 +428,7 @@ class Listener(object): self._closed = True self.sock.close() + class Connection(object): """A socket wrapper object for connected sockets. """ @@ -468,6 +490,7 @@ class Connection(object): yield ReturnEvent(line) break + class AcceptEvent(WaitableEvent): """An event for Listener objects (listening sockets) that suspends execution until the socket gets a connection. @@ -482,6 +505,7 @@ class AcceptEvent(WaitableEvent): sock, addr = self.listener.sock.accept() return Connection(sock, addr) + class ReceiveEvent(WaitableEvent): """An event for Connection objects (connected sockets) for asynchronously reading data. @@ -496,6 +520,7 @@ class ReceiveEvent(WaitableEvent): def fire(self): return self.conn.sock.recv(self.bufsize) + class SendEvent(WaitableEvent): """An event for Connection objects (connected sockets) for asynchronously writing data. @@ -523,29 +548,33 @@ def null(): """ return ValueEvent(None) + def spawn(coro): """Event: add another coroutine to the scheduler. Both the parent and child coroutines run concurrently. """ if not isinstance(coro, types.GeneratorType): - raise ValueError('%s is not a coroutine' % str(coro)) + raise ValueError(u'%s is not a coroutine' % coro) return SpawnEvent(coro) + def call(coro): """Event: delegate to another coroutine. The current coroutine is resumed once the sub-coroutine finishes. If the sub-coroutine returns a value using end(), then this event returns that value. """ if not isinstance(coro, types.GeneratorType): - raise ValueError('%s is not a coroutine' % str(coro)) + raise ValueError(u'%s is not a coroutine' % coro) return DelegationEvent(coro) + def end(value=None): """Event: ends the coroutine and returns a value to its delegator. """ return ReturnEvent(value) + def read(fd, bufsize=None): """Event: read from a file descriptor asynchronously.""" if bufsize is None: @@ -563,10 +592,12 @@ def read(fd, bufsize=None): else: return ReadEvent(fd, bufsize) + def write(fd, data): """Event: write to a file descriptor asynchronously.""" return WriteEvent(fd, data) + def connect(host, port): """Event: connect to a network address and return a Connection object for communicating on the socket. @@ -575,17 +606,20 @@ def connect(host, port): sock = socket.create_connection(addr) return ValueEvent(Connection(sock, addr)) + def sleep(duration): """Event: suspend the thread for ``duration`` seconds. """ return SleepEvent(duration) + def join(coro): """Suspend the thread until another, previously `spawn`ed thread completes. """ return JoinEvent(coro) + def kill(coro): """Halt the execution of a different `spawn`ed thread. """ diff --git a/libs/beets/util/confit.py b/libs/beets/util/confit.py index cf8b3629..aa49f6f1 100644 --- a/libs/beets/util/confit.py +++ b/libs/beets/util/confit.py @@ -1,5 +1,6 @@ +# -*- coding: utf-8 -*- # This file is part of Confit. -# Copyright 2014, Adrian Sampson. +# Copyright 2016, Adrian Sampson. # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -14,13 +15,15 @@ """Worry-free YAML configuration files. """ -from __future__ import unicode_literals +from __future__ import division, absolute_import, print_function + import platform import os import pkgutil import sys import yaml -import types +import collections +import re try: from collections import OrderedDict except ImportError: @@ -38,6 +41,8 @@ ROOT_NAME = 'root' YAML_TAB_PROBLEM = "found character '\\t' that cannot start any token" +REDACTED_TOMBSTONE = 'REDACTED' + # Utilities. @@ -45,7 +50,7 @@ PY3 = sys.version_info[0] == 3 STRING = str if PY3 else unicode BASESTRING = str if PY3 else basestring NUMERIC_TYPES = (int, float) if PY3 else (int, float, long) -TYPE_TYPES = (type,) if PY3 else (type, types.ClassType) + def iter_first(sequence): """Get the first element from an iterable or raise a ValueError if @@ -53,10 +58,7 @@ def iter_first(sequence): """ it = iter(sequence) try: - if PY3: - return next(it) - else: - return it.next() + return next(it) except StopIteration: raise ValueError() @@ -67,16 +69,25 @@ class ConfigError(Exception): """Base class for exceptions raised when querying a configuration. """ + class NotFoundError(ConfigError): """A requested value could not be found in the configuration trees. """ -class ConfigTypeError(ConfigError, TypeError): + +class ConfigValueError(ConfigError): + """The value in the configuration is illegal.""" + + +class ConfigTypeError(ConfigValueError): """The value in the configuration did not match the expected type. """ -class ConfigValueError(ConfigError, ValueError): - """The value in the configuration is illegal.""" + +class ConfigTemplateError(ConfigError): + """Base class for exceptions raised because of an invalid template. + """ + class ConfigReadError(ConfigError): """A configuration file could not be read.""" @@ -84,17 +95,17 @@ class ConfigReadError(ConfigError): self.filename = filename self.reason = reason - message = 'file {0} could not be read'.format(filename) + message = u'file {0} could not be read'.format(filename) if isinstance(reason, yaml.scanner.ScannerError) and \ reason.problem == YAML_TAB_PROBLEM: # Special-case error message for tab indentation in YAML markup. - message += ': found tab character at line {0}, column {1}'.format( + message += u': found tab character at line {0}, column {1}'.format( reason.problem_mark.line + 1, reason.problem_mark.column + 1, ) elif reason: # Generic error message uses exception's message. - message += ': {0}'.format(reason) + message += u': {0}'.format(reason) super(ConfigReadError, self).__init__(message) @@ -108,19 +119,19 @@ class ConfigSource(dict): def __init__(self, value, filename=None, default=False): super(ConfigSource, self).__init__(value) if filename is not None and not isinstance(filename, BASESTRING): - raise TypeError('filename must be a string or None') + raise TypeError(u'filename must be a string or None') self.filename = filename self.default = default def __repr__(self): - return 'ConfigSource({0}, {1}, {2})'.format( - super(ConfigSource, self).__repr__(), - repr(self.filename), - repr(self.default) + return 'ConfigSource({0!r}, {1!r}, {2!r})'.format( + super(ConfigSource, self), + self.filename, + self.default, ) @classmethod - def of(self, value): + def of(cls, value): """Given either a dictionary or a `ConfigSource` object, return a `ConfigSource` object. This lets a function accept either type of object as an argument. @@ -130,7 +141,8 @@ class ConfigSource(dict): elif isinstance(value, dict): return ConfigSource(value) else: - raise TypeError('source value must be a dict') + raise TypeError(u'source value must be a dict') + class ConfigView(object): """A configuration "view" is a query into a program's configuration @@ -164,7 +176,7 @@ class ConfigView(object): try: return iter_first(pairs) except ValueError: - raise NotFoundError("{0} not found".format(self.name)) + raise NotFoundError(u"{0} not found".format(self.name)) def exists(self): """Determine whether the view has a setting in any source. @@ -195,7 +207,31 @@ class ConfigView(object): raise NotImplementedError def __repr__(self): - return '' % self.name + return '<{}: {}>'.format(self.__class__.__name__, self.name) + + def __iter__(self): + """Iterate over the keys of a dictionary view or the *subviews* + of a list view. + """ + # Try getting the keys, if this is a dictionary view. + try: + keys = self.keys() + for key in keys: + yield key + + except ConfigTypeError: + # Otherwise, try iterating over a list. + collection = self.get() + if not isinstance(collection, (list, tuple)): + raise ConfigTypeError( + u'{0} must be a dictionary or a list, not {1}'.format( + self.name, type(collection).__name__ + ) + ) + + # Yield all the indices in the list. + for index in range(len(collection)): + yield self[index] def __getitem__(self, key): """Get a subview of this view.""" @@ -207,6 +243,9 @@ class ConfigView(object): """ self.set({key: value}) + def __contains__(self, key): + return self[key].exists() + def set_args(self, namespace): """Overlay parsed command-line arguments, generated by a library like argparse or optparse, onto this view's value. @@ -223,14 +262,17 @@ class ConfigView(object): # just say ``bool(view)`` or use ``view`` in a conditional. def __str__(self): - """Gets the value for this view as a byte string.""" - return str(self.get()) + """Get the value for this view as a bytestring. + """ + if PY3: + return self.__unicode__() + else: + return bytes(self.get()) def __unicode__(self): - """Gets the value for this view as a unicode string. (Python 2 - only.) + """Get the value for this view as a Unicode string. """ - return unicode(self.get()) + return STRING(self.get()) def __nonzero__(self): """Gets the value for this view as a boolean. (Python 2 only.) @@ -260,7 +302,7 @@ class ConfigView(object): cur_keys = dic.keys() except AttributeError: raise ConfigTypeError( - '{0} must be a dict, not {1}'.format( + u'{0} must be a dict, not {1}'.format( self.name, type(dic).__name__ ) ) @@ -301,7 +343,7 @@ class ConfigView(object): it = iter(collection) except TypeError: raise ConfigTypeError( - '{0} must be an iterable, not {1}'.format( + u'{0} must be an iterable, not {1}'.format( self.name, type(collection).__name__ ) ) @@ -310,111 +352,78 @@ class ConfigView(object): # Validation and conversion. - def get(self, typ=None): - """Returns the canonical value for the view, checked against the - passed-in type. If the value is not an instance of the given - type, a ConfigTypeError is raised. May also raise a - NotFoundError. - """ - value, _ = self.first() - - if typ is not None: - if not isinstance(typ, TYPE_TYPES): - raise TypeError('argument to get() must be a type') - - if not isinstance(value, typ): - raise ConfigTypeError( - "{0} must be of type {1}, not {2}".format( - self.name, typ.__name__, type(value).__name__ - ) - ) - - return value - - def as_filename(self): - """Get a string as a normalized as an absolute, tilde-free path. - - Relative paths are relative to the configuration directory (see - the `config_dir` method) if they come from a file. Otherwise, - they are relative to the current working directory. This helps - attain the expected behavior when using command-line options. - """ - path, source = self.first() - if not isinstance(path, BASESTRING): - raise ConfigTypeError('{0} must be a filename, not {1}'.format( - self.name, type(path).__name__ - )) - path = os.path.expanduser(STRING(path)) - - if not os.path.isabs(path) and source.filename: - # From defaults: relative to the app's directory. - path = os.path.join(self.root().config_dir(), path) - - return os.path.abspath(path) - - def as_choice(self, choices): - """Ensure that the value is among a collection of choices and - return it. If `choices` is a dictionary, then return the - corresponding value rather than the value itself (the key). - """ - value = self.get() - - if value not in choices: - raise ConfigValueError( - '{0} must be one of {1}, not {2}'.format( - self.name, repr(list(choices)), repr(value) - ) - ) - - if isinstance(choices, dict): - return choices[value] - else: - return value - - def as_number(self): - """Ensure that a value is of numeric type.""" - value = self.get() - if isinstance(value, NUMERIC_TYPES): - return value - raise ConfigTypeError( - '{0} must be numeric, not {1}'.format( - self.name, type(value).__name__ - ) - ) - - def as_str_seq(self): - """Get the value as a list of strings. The underlying configured - value can be a sequence or a single string. In the latter case, - the string is treated as a white-space separated list of words. - """ - value = self.get() - if isinstance(value, bytes): - value = value.decode('utf8', 'ignore') - - if isinstance(value, STRING): - return value.split() - else: - try: - return list(value) - except TypeError: - raise ConfigTypeError( - '{0} must be a whitespace-separated string or ' - 'a list'.format(self.name) - ) - - def flatten(self): + def flatten(self, redact=False): """Create a hierarchy of OrderedDicts containing the data from this view, recursively reifying all views to get their represented values. + + If `redact` is set, then sensitive values are replaced with + the string "REDACTED". """ od = OrderedDict() for key, view in self.items(): - try: - od[key] = view.flatten() - except ConfigTypeError: - od[key] = view.get() + if redact and view.redact: + od[key] = REDACTED_TOMBSTONE + else: + try: + od[key] = view.flatten(redact=redact) + except ConfigTypeError: + od[key] = view.get() return od + def get(self, template=None): + """Retrieve the value for this view according to the template. + + The `template` against which the values are checked can be + anything convertible to a `Template` using `as_template`. This + means you can pass in a default integer or string value, for + example, or a type to just check that something matches the type + you expect. + + May raise a `ConfigValueError` (or its subclass, + `ConfigTypeError`) or a `NotFoundError` when the configuration + doesn't satisfy the template. + """ + return as_template(template).value(self, template) + + # Old validation methods (deprecated). + + def as_filename(self): + return self.get(Filename()) + + def as_choice(self, choices): + return self.get(Choice(choices)) + + def as_number(self): + return self.get(Number()) + + def as_str_seq(self): + return self.get(StrSeq()) + + # Redaction. + + @property + def redact(self): + """Whether the view contains sensitive information and should be + redacted from output. + """ + return () in self.get_redactions() + + @redact.setter + def redact(self, flag): + self.set_redaction((), flag) + + def set_redaction(self, path, flag): + """Add or remove a redaction for a key path, which should be an + iterable of keys. + """ + raise NotImplementedError() + + def get_redactions(self): + """Get the set of currently-redacted sub-key-paths at this view. + """ + raise NotImplementedError() + class RootView(ConfigView): """The base of a view hierarchy. This view keeps track of the @@ -427,6 +436,7 @@ class RootView(ConfigView): """ self.sources = list(sources) self.name = ROOT_NAME + self.redactions = set() def add(self, obj): self.sources.append(ConfigSource.of(obj)) @@ -438,12 +448,24 @@ class RootView(ConfigView): return ((dict(s), s) for s in self.sources) def clear(self): - """Remove all sources from this configuration.""" + """Remove all sources (and redactions) from this + configuration. + """ del self.sources[:] + self.redactions.clear() def root(self): return self + def set_redaction(self, path, flag): + if flag: + self.redactions.add(path) + elif path in self.redactions: + self.redactions.remove(path) + + def get_redactions(self): + return self.redactions + class Subview(ConfigView): """A subview accessed via a subscript of a parent view.""" @@ -461,11 +483,14 @@ class Subview(ConfigView): if not isinstance(self.key, int): self.name += '.' if isinstance(self.key, int): - self.name += '#{0}'.format(self.key) + self.name += u'#{0}'.format(self.key) elif isinstance(self.key, BASESTRING): - self.name += '{0}'.format(self.key) + if isinstance(self.key, bytes): + self.name += self.key.decode('utf8') + else: + self.name += self.key else: - self.name += '{0}'.format(repr(self.key)) + self.name += repr(self.key) def resolve(self): for collection, source in self.parent.resolve(): @@ -480,7 +505,7 @@ class Subview(ConfigView): except TypeError: # Not subscriptable. raise ConfigTypeError( - "{0} must be a collection, not {1}".format( + u"{0} must be a collection, not {1}".format( self.parent.name, type(collection).__name__ ) ) @@ -495,6 +520,13 @@ class Subview(ConfigView): def root(self): return self.parent.root() + def set_redaction(self, path, flag): + self.parent.set_redaction((self.key,) + path, flag) + + def get_redactions(self): + return (kp[1:] for kp in self.parent.get_redactions() + if kp and kp[0] == self.key) + # Config file paths, including platform-specific paths and in-package # defaults. @@ -518,6 +550,7 @@ def _package_path(name): return os.path.dirname(os.path.abspath(filepath)) + def config_dirs(): """Return a platform-specific list of candidates for user configuration directories on the system. @@ -582,7 +615,7 @@ class Loader(yaml.SafeLoader): else: raise yaml.constructor.ConstructorError( None, None, - 'expected a mapping node, but found %s' % node.id, + u'expected a mapping node, but found %s' % node.id, node.start_mark ) @@ -593,7 +626,7 @@ class Loader(yaml.SafeLoader): hash(key) except TypeError as exc: raise yaml.constructor.ConstructorError( - 'while constructing a mapping', + u'while constructing a mapping', node.start_mark, 'found unacceptable key (%s)' % exc, key_node.start_mark ) @@ -606,10 +639,12 @@ class Loader(yaml.SafeLoader): plain = super(Loader, self).check_plain() return plain or self.peek() == '%' + Loader.add_constructor('tag:yaml.org,2002:str', Loader._construct_unicode) Loader.add_constructor('tag:yaml.org,2002:map', Loader.construct_yaml_map) Loader.add_constructor('tag:yaml.org,2002:omap', Loader.construct_yaml_map) + def load_yaml(filename): """Read a YAML document from a file. If the file cannot be read or parsed, a ConfigReadError is raised. @@ -639,11 +674,11 @@ class Dumper(yaml.SafeDumper): for item_key, item_value in mapping: node_key = self.represent_data(item_key) node_value = self.represent_data(item_value) - if not (isinstance(node_key, yaml.ScalarNode) - and not node_key.style): + if not (isinstance(node_key, yaml.ScalarNode) and + not node_key.style): best_style = False - if not (isinstance(node_value, yaml.ScalarNode) - and not node_value.style): + if not (isinstance(node_value, yaml.ScalarNode) and + not node_value.style): best_style = False value.append((node_key, node_value)) if flow_style is None: @@ -669,9 +704,9 @@ class Dumper(yaml.SafeDumper): """Represent bool as 'yes' or 'no' instead of 'true' or 'false'. """ if data: - value = 'yes' + value = u'yes' else: - value = 'no' + value = u'no' return self.represent_scalar('tag:yaml.org,2002:bool', value) def represent_none(self, data): @@ -679,11 +714,13 @@ class Dumper(yaml.SafeDumper): """ return self.represent_scalar('tag:yaml.org,2002:null', '') + Dumper.add_representer(OrderedDict, Dumper.represent_dict) Dumper.add_representer(bool, Dumper.represent_bool) Dumper.add_representer(type(None), Dumper.represent_none) Dumper.add_representer(list, Dumper.represent_list) + def restore_yaml_comments(data, default_data): """Scan default_data for comments (we include empty lines in our definition of comments) and place them before the same keys in data. @@ -794,7 +831,7 @@ class Configuration(RootView): appdir = os.environ[self._env_var] appdir = os.path.abspath(os.path.expanduser(appdir)) if os.path.isfile(appdir): - raise ConfigError('{0} must be a directory'.format( + raise ConfigError(u'{0} must be a directory'.format( self._env_var )) @@ -818,7 +855,7 @@ class Configuration(RootView): filename = os.path.abspath(filename) self.set(ConfigSource(load_yaml(filename), filename)) - def dump(self, full=True): + def dump(self, full=True, redact=False): """Dump the Configuration object to a YAML file. The order of the keys is determined from the default @@ -830,13 +867,17 @@ class Configuration(RootView): :type filename: unicode :param full: Dump settings that don't differ from the defaults as well + :param redact: Remove sensitive information (views with the `redact` + flag set) from the output """ if full: - out_dict = self.flatten() + out_dict = self.flatten(redact=redact) else: # Exclude defaults when flattening. sources = [s for s in self.sources if not s.default] - out_dict = RootView(sources).flatten() + temp_root = RootView(sources) + temp_root.redactions = self.redactions + out_dict = temp_root.flatten(redact=redact) yaml_out = yaml.dump(out_dict, Dumper=Dumper, default_flow_style=None, indent=4, @@ -848,7 +889,7 @@ class Configuration(RootView): if source.default: default_source = source break - if default_source: + if default_source and default_source.filename: with open(default_source.filename, 'r') as fp: default_data = fp.read() yaml_out = restore_yaml_comments(yaml_out, default_data) @@ -895,6 +936,506 @@ class LazyConfig(Configuration): def clear(self): """Remove all sources from this configuration.""" - del self.sources[:] + super(LazyConfig, self).clear() self._lazy_suffix = [] self._lazy_prefix = [] + + +# "Validated" configuration views: experimental! + + +REQUIRED = object() +"""A sentinel indicating that there is no default value and an exception +should be raised when the value is missing. +""" + + +class Template(object): + """A value template for configuration fields. + + The template works like a type and instructs Confit about how to + interpret a deserialized YAML value. This includes type conversions, + providing a default value, and validating for errors. For example, a + filepath type might expand tildes and check that the file exists. + """ + def __init__(self, default=REQUIRED): + """Create a template with a given default value. + + If `default` is the sentinel `REQUIRED` (as it is by default), + then an error will be raised when a value is missing. Otherwise, + missing values will instead return `default`. + """ + self.default = default + + def __call__(self, view): + """Invoking a template on a view gets the view's value according + to the template. + """ + return self.value(view, self) + + def value(self, view, template=None): + """Get the value for a `ConfigView`. + + May raise a `NotFoundError` if the value is missing (and the + template requires it) or a `ConfigValueError` for invalid values. + """ + if view.exists(): + value, _ = view.first() + return self.convert(value, view) + elif self.default is REQUIRED: + # Missing required value. This is an error. + raise NotFoundError(u"{0} not found".format(view.name)) + else: + # Missing value, but not required. + return self.default + + def convert(self, value, view): + """Convert the YAML-deserialized value to a value of the desired + type. + + Subclasses should override this to provide useful conversions. + May raise a `ConfigValueError` when the configuration is wrong. + """ + # Default implementation does no conversion. + return value + + def fail(self, message, view, type_error=False): + """Raise an exception indicating that a value cannot be + accepted. + + `type_error` indicates whether the error is due to a type + mismatch rather than a malformed value. In this case, a more + specific exception is raised. + """ + exc_class = ConfigTypeError if type_error else ConfigValueError + raise exc_class( + u'{0}: {1}'.format(view.name, message) + ) + + def __repr__(self): + return '{0}({1})'.format( + type(self).__name__, + '' if self.default is REQUIRED else repr(self.default), + ) + + +class Integer(Template): + """An integer configuration value template. + """ + def convert(self, value, view): + """Check that the value is an integer. Floats are rounded. + """ + if isinstance(value, int): + return value + elif isinstance(value, float): + return int(value) + else: + self.fail(u'must be a number', view, True) + + +class Number(Template): + """A numeric type: either an integer or a floating-point number. + """ + def convert(self, value, view): + """Check that the value is an int or a float. + """ + if isinstance(value, NUMERIC_TYPES): + return value + else: + self.fail( + u'must be numeric, not {0}'.format(type(value).__name__), + view, + True + ) + + +class MappingTemplate(Template): + """A template that uses a dictionary to specify other types for the + values for a set of keys and produce a validated `AttrDict`. + """ + def __init__(self, mapping): + """Create a template according to a dict (mapping). The + mapping's values should themselves either be Types or + convertible to Types. + """ + subtemplates = {} + for key, typ in mapping.items(): + subtemplates[key] = as_template(typ) + self.subtemplates = subtemplates + + def value(self, view, template=None): + """Get a dict with the same keys as the template and values + validated according to the value types. + """ + out = AttrDict() + for key, typ in self.subtemplates.items(): + out[key] = typ.value(view[key], self) + return out + + def __repr__(self): + return 'MappingTemplate({0})'.format(repr(self.subtemplates)) + + +class String(Template): + """A string configuration value template. + """ + def __init__(self, default=REQUIRED, pattern=None): + """Create a template with the added optional `pattern` argument, + a regular expression string that the value should match. + """ + super(String, self).__init__(default) + self.pattern = pattern + if pattern: + self.regex = re.compile(pattern) + + def __repr__(self): + args = [] + + if self.default is not REQUIRED: + args.append(repr(self.default)) + + if self.pattern is not None: + args.append('pattern=' + repr(self.pattern)) + + return 'String({0})'.format(', '.join(args)) + + def convert(self, value, view): + """Check that the value is a string and matches the pattern. + """ + if isinstance(value, BASESTRING): + if self.pattern and not self.regex.match(value): + self.fail( + u"must match the pattern {0}".format(self.pattern), + view + ) + return value + else: + self.fail(u'must be a string', view, True) + + +class Choice(Template): + """A template that permits values from a sequence of choices. + """ + def __init__(self, choices): + """Create a template that validates any of the values from the + iterable `choices`. + + If `choices` is a map, then the corresponding value is emitted. + Otherwise, the value itself is emitted. + """ + self.choices = choices + + def convert(self, value, view): + """Ensure that the value is among the choices (and remap if the + choices are a mapping). + """ + if value not in self.choices: + self.fail( + u'must be one of {0}, not {1}'.format( + repr(list(self.choices)), repr(value) + ), + view + ) + + if isinstance(self.choices, collections.Mapping): + return self.choices[value] + else: + return value + + def __repr__(self): + return 'Choice({0!r})'.format(self.choices) + + +class OneOf(Template): + """A template that permits values complying to one of the given templates. + """ + def __init__(self, allowed, default=REQUIRED): + super(OneOf, self).__init__(default) + self.allowed = list(allowed) + + def __repr__(self): + args = [] + + if self.allowed is not None: + args.append('allowed=' + repr(self.allowed)) + + if self.default is not REQUIRED: + args.append(repr(self.default)) + + return 'OneOf({0})'.format(', '.join(args)) + + def value(self, view, template): + self.template = template + return super(OneOf, self).value(view, template) + + def convert(self, value, view): + """Ensure that the value follows at least one template. + """ + is_mapping = isinstance(self.template, MappingTemplate) + + for candidate in self.allowed: + try: + if is_mapping: + if isinstance(candidate, Filename) and \ + candidate.relative_to: + next_template = candidate.template_with_relatives( + view, + self.template + ) + + next_template.subtemplates[view.key] = as_template( + candidate + ) + else: + next_template = MappingTemplate({view.key: candidate}) + + return view.parent.get(next_template)[view.key] + else: + return view.get(candidate) + except ConfigTemplateError: + raise + except ConfigError: + pass + except ValueError as exc: + raise ConfigTemplateError(exc) + + self.fail( + u'must be one of {0}, not {1}'.format( + repr(self.allowed), repr(value) + ), + view + ) + + +class StrSeq(Template): + """A template for values that are lists of strings. + + Validates both actual YAML string lists and single strings. Strings + can optionally be split on whitespace. + """ + def __init__(self, split=True): + """Create a new template. + + `split` indicates whether, when the underlying value is a single + string, it should be split on whitespace. Otherwise, the + resulting value is a list containing a single string. + """ + super(StrSeq, self).__init__() + self.split = split + + def convert(self, value, view): + if isinstance(value, bytes): + value = value.decode('utf8', 'ignore') + + if isinstance(value, STRING): + if self.split: + return value.split() + else: + return [value] + + try: + value = list(value) + except TypeError: + self.fail(u'must be a whitespace-separated string or a list', + view, True) + + def convert(x): + if isinstance(x, STRING): + return x + elif isinstance(x, bytes): + return x.decode('utf8', 'ignore') + else: + self.fail(u'must be a list of strings', view, True) + return list(map(convert, value)) + + +class Filename(Template): + """A template that validates strings as filenames. + + Filenames are returned as absolute, tilde-free paths. + + Relative paths are relative to the template's `cwd` argument + when it is specified, then the configuration directory (see + the `config_dir` method) if they come from a file. Otherwise, + they are relative to the current working directory. This helps + attain the expected behavior when using command-line options. + """ + def __init__(self, default=REQUIRED, cwd=None, relative_to=None, + in_app_dir=False): + """`relative_to` is the name of a sibling value that is + being validated at the same time. + + `in_app_dir` indicates whether the path should be resolved + inside the application's config directory (even when the setting + does not come from a file). + """ + super(Filename, self).__init__(default) + self.cwd = cwd + self.relative_to = relative_to + self.in_app_dir = in_app_dir + + def __repr__(self): + args = [] + + if self.default is not REQUIRED: + args.append(repr(self.default)) + + if self.cwd is not None: + args.append('cwd=' + repr(self.cwd)) + + if self.relative_to is not None: + args.append('relative_to=' + repr(self.relative_to)) + + if self.in_app_dir: + args.append('in_app_dir=True') + + return 'Filename({0})'.format(', '.join(args)) + + def resolve_relative_to(self, view, template): + if not isinstance(template, (collections.Mapping, MappingTemplate)): + # disallow config.get(Filename(relative_to='foo')) + raise ConfigTemplateError( + u'relative_to may only be used when getting multiple values.' + ) + + elif self.relative_to == view.key: + raise ConfigTemplateError( + u'{0} is relative to itself'.format(view.name) + ) + + elif self.relative_to not in view.parent.keys(): + # self.relative_to is not in the config + self.fail( + ( + u'needs sibling value "{0}" to expand relative path' + ).format(self.relative_to), + view + ) + + old_template = {} + old_template.update(template.subtemplates) + + # save time by skipping MappingTemplate's init loop + next_template = MappingTemplate({}) + next_relative = self.relative_to + + # gather all the needed templates and nothing else + while next_relative is not None: + try: + # pop to avoid infinite loop because of recursive + # relative paths + rel_to_template = old_template.pop(next_relative) + except KeyError: + if next_relative in template.subtemplates: + # we encountered this config key previously + raise ConfigTemplateError(( + u'{0} and {1} are recursively relative' + ).format(view.name, self.relative_to)) + else: + raise ConfigTemplateError(( + u'missing template for {0}, needed to expand {1}\'s' + + u'relative path' + ).format(self.relative_to, view.name)) + + next_template.subtemplates[next_relative] = rel_to_template + next_relative = rel_to_template.relative_to + + return view.parent.get(next_template)[self.relative_to] + + def value(self, view, template=None): + path, source = view.first() + if not isinstance(path, BASESTRING): + self.fail( + u'must be a filename, not {0}'.format(type(path).__name__), + view, + True + ) + path = os.path.expanduser(STRING(path)) + + if not os.path.isabs(path): + if self.cwd is not None: + # relative to the template's argument + path = os.path.join(self.cwd, path) + + elif self.relative_to is not None: + path = os.path.join( + self.resolve_relative_to(view, template), + path, + ) + + elif source.filename or self.in_app_dir: + # From defaults: relative to the app's directory. + path = os.path.join(view.root().config_dir(), path) + + return os.path.abspath(path) + + +class TypeTemplate(Template): + """A simple template that checks that a value is an instance of a + desired Python type. + """ + def __init__(self, typ, default=REQUIRED): + """Create a template that checks that the value is an instance + of `typ`. + """ + super(TypeTemplate, self).__init__(default) + self.typ = typ + + def convert(self, value, view): + if not isinstance(value, self.typ): + self.fail( + u'must be a {0}, not {1}'.format( + self.typ.__name__, + type(value).__name__, + ), + view, + True + ) + return value + + +class AttrDict(dict): + """A `dict` subclass that can be accessed via attributes (dot + notation) for convenience. + """ + def __getattr__(self, key): + if key in self: + return self[key] + else: + raise AttributeError(key) + + +def as_template(value): + """Convert a simple "shorthand" Python value to a `Template`. + """ + if isinstance(value, Template): + # If it's already a Template, pass it through. + return value + elif isinstance(value, collections.Mapping): + # Dictionaries work as templates. + return MappingTemplate(value) + elif value is int: + return Integer() + elif isinstance(value, int): + return Integer(value) + elif isinstance(value, type) and issubclass(value, BASESTRING): + return String() + elif isinstance(value, BASESTRING): + return String(value) + elif isinstance(value, set): + # convert to list to avoid hash related problems + return Choice(list(value)) + elif isinstance(value, list): + return OneOf(value) + elif value is float: + return Number() + elif value is None: + return Template() + elif value is dict: + return TypeTemplate(collections.Mapping) + elif value is list: + return TypeTemplate(collections.Sequence) + elif isinstance(value, type): + return TypeTemplate(value) + else: + raise ValueError(u'cannot convert to template: {0!r}'.format(value)) diff --git a/libs/beets/util/enumeration.py b/libs/beets/util/enumeration.py index e6ec0766..3e946718 100644 --- a/libs/beets/util/enumeration.py +++ b/libs/beets/util/enumeration.py @@ -1,5 +1,6 @@ +# -*- coding: utf-8 -*- # This file is part of beets. -# Copyright 2013, Adrian Sampson. +# Copyright 2016, Adrian Sampson. # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -12,167 +13,31 @@ # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. -"""A metaclass for enumerated types that really are types. +from __future__ import division, absolute_import, print_function -You can create enumerations with `enum(values, [name])` and they work -how you would expect them to. +from enum import Enum - >>> from enumeration import enum - >>> Direction = enum('north east south west', name='Direction') - >>> Direction.west - Direction.west - >>> Direction.west == Direction.west - True - >>> Direction.west == Direction.east - False - >>> isinstance(Direction.west, Direction) - True - >>> Direction[3] - Direction.west - >>> Direction['west'] - Direction.west - >>> Direction.west.name - 'west' - >>> Direction.north < Direction.west - True -Enumerations are classes; their instances represent the possible values -of the enumeration. Because Python classes must have names, you may -provide a `name` parameter to `enum`; if you don't, a meaningless one -will be chosen for you. -""" -import random - -class Enumeration(type): - """A metaclass whose classes are enumerations. - - The `values` attribute of the class is used to populate the - enumeration. Values may either be a list of enumerated names or a - string containing a space-separated list of names. When the class - is created, it is instantiated for each name value in `values`. - Each such instance is the name of the enumerated item as the sole - argument. - - The `Enumerated` class is a good choice for a superclass. +class OrderedEnum(Enum): """ - - def __init__(cls, name, bases, dic): - super(Enumeration, cls).__init__(name, bases, dic) - - if 'values' not in dic: - # Do nothing if no values are provided (i.e., with - # Enumerated itself). - return - - # May be called with a single string, in which case we split on - # whitespace for convenience. - values = dic['values'] - if isinstance(values, basestring): - values = values.split() - - # Create the Enumerated instances for each value. We have to use - # super's __setattr__ here because we disallow setattr below. - super(Enumeration, cls).__setattr__('_items_dict', {}) - super(Enumeration, cls).__setattr__('_items_list', []) - for value in values: - item = cls(value, len(cls._items_list)) - cls._items_dict[value] = item - cls._items_list.append(item) - - def __getattr__(cls, key): - try: - return cls._items_dict[key] - except KeyError: - raise AttributeError("enumeration '" + cls.__name__ + - "' has no item '" + key + "'") - - def __setattr__(cls, key, val): - raise TypeError("enumerations do not support attribute assignment") - - def __getitem__(cls, key): - if isinstance(key, int): - return cls._items_list[key] - else: - return getattr(cls, key) - - def __len__(cls): - return len(cls._items_list) - - def __iter__(cls): - return iter(cls._items_list) - - def __nonzero__(cls): - # Ensures that __len__ doesn't get called before __init__ by - # pydoc. - return True - -class Enumerated(object): - """An item in an enumeration. - - Contains instance methods inherited by enumerated objects. The - metaclass is preset to `Enumeration` for your convenience. - - Instance attributes: - name -- The name of the item. - index -- The index of the item in its enumeration. - - >>> from enumeration import Enumerated - >>> class Garment(Enumerated): - ... values = 'hat glove belt poncho lederhosen suspenders' - ... def wear(self): - ... print('now wearing a ' + self.name) - ... - >>> Garment.poncho.wear() - now wearing a poncho + An Enum subclass that allows comparison of members. """ + def __ge__(self, other): + if self.__class__ is other.__class__: + return self.value >= other.value + return NotImplemented - __metaclass__ = Enumeration + def __gt__(self, other): + if self.__class__ is other.__class__: + return self.value > other.value + return NotImplemented - def __init__(self, name, index): - self.name = name - self.index = index + def __le__(self, other): + if self.__class__ is other.__class__: + return self.value <= other.value + return NotImplemented - def __str__(self): - return type(self).__name__ + '.' + self.name - - def __repr__(self): - return str(self) - - def __cmp__(self, other): - if type(self) is type(other): - # Note that we're assuming that the items are direct - # instances of the same Enumeration (i.e., no fancy - # subclassing), which is probably okay. - return cmp(self.index, other.index) - else: - return NotImplemented - -def enum(*values, **kwargs): - """Shorthand for creating a new Enumeration class. - - Call with enumeration values as a list, a space-delimited string, or - just an argument list. To give the class a name, pass it as the - `name` keyword argument. Otherwise, a name will be chosen for you. - - The following are all equivalent: - - enum('pinkie ring middle index thumb') - enum('pinkie', 'ring', 'middle', 'index', 'thumb') - enum(['pinkie', 'ring', 'middle', 'index', 'thumb']) - """ - - if ('name' not in kwargs) or kwargs['name'] is None: - # Create a probably-unique name. It doesn't really have to be - # unique, but getting distinct names each time helps with - # identification in debugging. - name = 'Enumeration' + hex(random.randint(0,0xfffffff))[2:].upper() - else: - name = kwargs['name'] - - if len(values) == 1: - # If there's only one value, we have a couple of alternate calling - # styles. - if isinstance(values[0], basestring) or hasattr(values[0], '__iter__'): - values = values[0] - - return type(name, (Enumerated,), {'values': values}) + def __lt__(self, other): + if self.__class__ is other.__class__: + return self.value < other.value + return NotImplemented diff --git a/libs/beets/util/functemplate.py b/libs/beets/util/functemplate.py index 0fce41e5..05f0892c 100644 --- a/libs/beets/util/functemplate.py +++ b/libs/beets/util/functemplate.py @@ -1,5 +1,6 @@ +# -*- coding: utf-8 -*- # This file is part of beets. -# Copyright 2013, Adrian Sampson. +# Copyright 2016, Adrian Sampson. # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -25,13 +26,16 @@ library: unknown symbols are left intact. This is sort of like a tiny, horrible degeneration of a real templating engine like Jinja2 or Mustache. """ -from __future__ import print_function + +from __future__ import division, absolute_import, print_function import re import ast import dis import types +from .confit import NUMERIC_TYPES + SYMBOL_DELIM = u'$' FUNC_DELIM = u'%' GROUP_OPEN = u'{' @@ -42,6 +46,7 @@ ESCAPE_CHAR = u'$' VARIABLE_PREFIX = '__var_' FUNCTION_PREFIX = '__func_' + class Environment(object): """Contains the values and functions to be substituted into a template. @@ -57,23 +62,26 @@ def ex_lvalue(name): """A variable load expression.""" return ast.Name(name, ast.Store()) + def ex_rvalue(name): """A variable store expression.""" return ast.Name(name, ast.Load()) + def ex_literal(val): """An int, float, long, bool, string, or None literal with the given value. """ if val is None: return ast.Name('None', ast.Load()) - elif isinstance(val, (int, float, long)): + elif isinstance(val, NUMERIC_TYPES): return ast.Num(val) elif isinstance(val, bool): - return ast.Name(str(val), ast.Load()) + return ast.Name(bytes(val), ast.Load()) elif isinstance(val, basestring): return ast.Str(val) - raise TypeError('no literal for {0}'.format(type(val))) + raise TypeError(u'no literal for {0}'.format(type(val))) + def ex_varassign(name, expr): """Assign an expression into a single variable. The expression may @@ -83,6 +91,7 @@ def ex_varassign(name, expr): expr = ex_literal(expr) return ast.Assign([ex_lvalue(name)], expr) + def ex_call(func, args): """A function-call expression with only positional parameters. The function may be an expression or the name of a function. Each @@ -98,13 +107,14 @@ def ex_call(func, args): return ast.Call(func, args, [], None, None) + def compile_func(arg_names, statements, name='_the_func', debug=False): """Compile a list of statements as the body of a function and return the resulting Python function. If `debug`, then print out the bytecode of the compiled function. """ func_def = ast.FunctionDef( - name, + name.encode('utf8'), ast.arguments( [ast.Name(n, ast.Param()) for n in arg_names], None, None, @@ -126,7 +136,7 @@ def compile_func(arg_names, statements, name='_the_func', debug=False): dis.dis(const) the_locals = {} - exec prog in {}, the_locals + exec(prog, {}, the_locals) return the_locals[name] @@ -157,6 +167,7 @@ class Symbol(object): expr = ex_rvalue(VARIABLE_PREFIX + self.ident.encode('utf8')) return [expr], set([self.ident.encode('utf8')]), set() + class Call(object): """A function call in a template.""" def __init__(self, ident, args, original): @@ -214,6 +225,7 @@ class Call(object): ) return [subexpr_call], varnames, funcnames + class Expression(object): """Top-level template construct: contains a list of text blobs, Symbols, and Calls. @@ -259,6 +271,7 @@ class Expression(object): class ParseError(Exception): pass + class Parser(object): """Parses a template expression string. Instantiate the class with the template source and call ``parse_expression``. The ``pos`` field @@ -280,7 +293,7 @@ class Parser(object): # Common parsing resources. special_chars = (SYMBOL_DELIM, FUNC_DELIM, GROUP_OPEN, GROUP_CLOSE, ARG_SEP, ESCAPE_CHAR) - special_char_re = re.compile(ur'[%s]|$' % + special_char_re = re.compile(r'[%s]|$' % u''.join(re.escape(c) for c in special_chars)) def parse_expression(self): @@ -298,8 +311,8 @@ class Parser(object): # A non-special character. Skip to the next special # character, treating the interstice as literal text. next_pos = ( - self.special_char_re.search(self.string[self.pos:]).start() - + self.pos + self.special_char_re.search( + self.string[self.pos:]).start() + self.pos ) text_parts.append(self.string[self.pos:next_pos]) self.pos = next_pos @@ -316,13 +329,13 @@ class Parser(object): next_char = self.string[self.pos + 1] if char == ESCAPE_CHAR and next_char in \ - (SYMBOL_DELIM, FUNC_DELIM, GROUP_CLOSE, ARG_SEP): + (SYMBOL_DELIM, FUNC_DELIM, GROUP_CLOSE, ARG_SEP): # An escaped special character ($$, $}, etc.). Note that # ${ is not an escape sequence: this is ambiguous with # the start of a symbol and it's not necessary (just # using { suffices in all cases). text_parts.append(next_char) - self.pos += 2 # Skip the next character. + self.pos += 2 # Skip the next character. continue # Shift all characters collected so far into a single string. @@ -372,7 +385,7 @@ class Parser(object): if next_char == GROUP_OPEN: # A symbol like ${this}. - self.pos += 1 # Skip opening. + self.pos += 1 # Skip opening. closer = self.string.find(GROUP_CLOSE, self.pos) if closer == -1 or closer == self.pos: # No closing brace found or identifier is empty. @@ -431,7 +444,7 @@ class Parser(object): self.parts.append(self.string[start_pos:self.pos]) return - self.pos += 1 # Move past closing brace. + self.pos += 1 # Move past closing brace. self.parts.append(Call(ident, args, self.string[start_pos:self.pos])) def parse_argument_list(self): @@ -468,10 +481,11 @@ class Parser(object): Updates ``pos``. """ remainder = self.string[self.pos:] - ident = re.match(ur'\w*', remainder).group(0) + ident = re.match(r'\w*', remainder).group(0) self.pos += len(ident) return ident + def _parse(template): """Parse a top-level template string Expression. Any extraneous text is considered literal text. @@ -522,9 +536,9 @@ class Template(object): argnames = [] for varname in varnames: - argnames.append(VARIABLE_PREFIX.encode('utf8') + varname) + argnames.append(VARIABLE_PREFIX + varname) for funcname in funcnames: - argnames.append(FUNCTION_PREFIX.encode('utf8') + funcname) + argnames.append(FUNCTION_PREFIX + funcname) func = compile_func( argnames, @@ -558,4 +572,4 @@ if __name__ == '__main__': 'from __main__ import _tmpl, _vars, _funcs', number=10000) print(comp_time) - print('Speedup:', interp_time / comp_time) + print(u'Speedup:', interp_time / comp_time) diff --git a/libs/beets/util/hidden.py b/libs/beets/util/hidden.py new file mode 100644 index 00000000..262d371e --- /dev/null +++ b/libs/beets/util/hidden.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Adrian Sampson. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Simple library to work out if a file is hidden on different platforms.""" +from __future__ import division, absolute_import, print_function + +import os +import stat +import ctypes +import sys + + +def _is_hidden_osx(path): + """Return whether or not a file is hidden on OS X. + + This uses os.lstat to work out if a file has the "hidden" flag. + """ + file_stat = os.lstat(path) + + if hasattr(file_stat, 'st_flags') and hasattr(stat, 'UF_HIDDEN'): + return bool(file_stat.st_flags & stat.UF_HIDDEN) + else: + return False + + +def _is_hidden_win(path): + """Return whether or not a file is hidden on Windows. + + This uses GetFileAttributes to work out if a file has the "hidden" flag + (FILE_ATTRIBUTE_HIDDEN). + """ + # FILE_ATTRIBUTE_HIDDEN = 2 (0x2) from GetFileAttributes documentation. + hidden_mask = 2 + + # Retrieve the attributes for the file. + attrs = ctypes.windll.kernel32.GetFileAttributesW(path) + + # Ensure we have valid attribues and compare them against the mask. + return attrs >= 0 and attrs & hidden_mask + + +def _is_hidden_dot(path): + """Return whether or not a file starts with a dot. + + Files starting with a dot are seen as "hidden" files on Unix-based OSes. + """ + return os.path.basename(path).startswith('.') + + +def is_hidden(path): + """Return whether or not a file is hidden. + + This method works differently depending on the platform it is called on. + + On OS X, it uses both the result of `is_hidden_osx` and `is_hidden_dot` to + work out if a file is hidden. + + On Windows, it uses the result of `is_hidden_win` to work out if a file is + hidden. + + On any other operating systems (i.e. Linux), it uses `is_hidden_dot` to + work out if a file is hidden. + """ + # Convert the path to unicode if it is not already. + if not isinstance(path, unicode): + path = path.decode('utf-8') + + # Run platform specific functions depending on the platform + if sys.platform == 'darwin': + return _is_hidden_osx(path) or _is_hidden_dot(path) + elif sys.platform == 'win32': + return _is_hidden_win(path) + else: + return _is_hidden_dot(path) + +__all__ = ['is_hidden'] diff --git a/libs/beets/util/pipeline.py b/libs/beets/util/pipeline.py index c64454ff..b5f77733 100644 --- a/libs/beets/util/pipeline.py +++ b/libs/beets/util/pipeline.py @@ -1,5 +1,6 @@ +# -*- coding: utf-8 -*- # This file is part of beets. -# Copyright 2013, Adrian Sampson. +# Copyright 2016, Adrian Sampson. # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -30,18 +31,19 @@ up a bottleneck stage by dividing its work among multiple threads. To do so, pass an iterable of coroutines to the Pipeline constructor in place of any single coroutine. """ -from __future__ import print_function + +from __future__ import division, absolute_import, print_function import Queue from threading import Thread, Lock import sys -import types BUBBLE = '__PIPELINE_BUBBLE__' POISON = '__PIPELINE_POISON__' DEFAULT_QUEUE_SIZE = 16 + def _invalidate_queue(q, val=None, sync=True): """Breaks a Queue such that it never blocks, always has size 1, and has no maximum size. get()ing from the queue returns `val`, @@ -50,8 +52,10 @@ def _invalidate_queue(q, val=None, sync=True): """ def _qsize(len=len): return 1 + def _put(item): pass + def _get(): return val @@ -70,6 +74,7 @@ def _invalidate_queue(q, val=None, sync=True): if sync: q.mutex.release() + class CountedQueue(Queue.Queue): """A queue that keeps track of the number of threads that are still feeding into it. The queue is poisoned when all threads are @@ -104,6 +109,7 @@ class CountedQueue(Queue.Queue): # Replacement _get invalidates when no items remain. _old_get = self._get + def _get(): out = _old_get() if not self.queue: @@ -117,18 +123,67 @@ class CountedQueue(Queue.Queue): # No items. Invalidate immediately. _invalidate_queue(self, POISON, False) + class MultiMessage(object): """A message yielded by a pipeline stage encapsulating multiple values to be sent to the next stage. """ def __init__(self, messages): self.messages = messages + + def multiple(messages): """Yield multiple([message, ..]) from a pipeline stage to send multiple values to the next pipeline stage. """ return MultiMessage(messages) + +def stage(func): + """Decorate a function to become a simple stage. + + >>> @stage + ... def add(n, i): + ... return i + n + >>> pipe = Pipeline([ + ... iter([1, 2, 3]), + ... add(2), + ... ]) + >>> list(pipe.pull()) + [3, 4, 5] + """ + + def coro(*args): + task = None + while True: + task = yield task + task = func(*(args + (task,))) + return coro + + +def mutator_stage(func): + """Decorate a function that manipulates items in a coroutine to + become a simple stage. + + >>> @mutator_stage + ... def setkey(key, item): + ... item[key] = True + >>> pipe = Pipeline([ + ... iter([{'x': False}, {'a': False}]), + ... setkey('x'), + ... ]) + >>> list(pipe.pull()) + [{'x': True}, {'a': False, 'x': True}] + """ + + def coro(*args): + task = None + while True: + task = yield task + func(*(args + (task,))) + return coro + + def _allmsgs(obj): """Returns a list of all the messages encapsulated in obj. If obj is a MultiMessage, returns its enclosed messages. If obj is BUBBLE, @@ -141,6 +196,7 @@ def _allmsgs(obj): else: return [obj] + class PipelineThread(Thread): """Abstract base class for pipeline-stage threads.""" def __init__(self, all_threads): @@ -169,6 +225,7 @@ class PipelineThread(Thread): for thread in self.all_threads: thread.abort() + class FirstPipelineThread(PipelineThread): """The thread running the first stage in a parallel pipeline setup. The coroutine should just be a generator. @@ -191,7 +248,7 @@ class FirstPipelineThread(PipelineThread): # Get the value from the generator. try: - msg = self.coro.next() + msg = next(self.coro) except StopIteration: break @@ -209,6 +266,7 @@ class FirstPipelineThread(PipelineThread): # Generator finished; shut down the pipeline. self.out_queue.release() + class MiddlePipelineThread(PipelineThread): """A thread running any stage in the pipeline except the first or last. @@ -223,7 +281,7 @@ class MiddlePipelineThread(PipelineThread): def run(self): try: # Prime the coroutine. - self.coro.next() + next(self.coro) while True: with self.abort_lock: @@ -256,6 +314,7 @@ class MiddlePipelineThread(PipelineThread): # Pipeline is shutting down normally. self.out_queue.release() + class LastPipelineThread(PipelineThread): """A thread running the last stage in a pipeline. The coroutine should yield nothing. @@ -267,7 +326,7 @@ class LastPipelineThread(PipelineThread): def run(self): # Prime the coroutine. - self.coro.next() + next(self.coro) try: while True: @@ -291,6 +350,7 @@ class LastPipelineThread(PipelineThread): self.abort_all(sys.exc_info()) return + class Pipeline(object): """Represents a staged pattern of work. Each stage in the pipeline is a coroutine that receives messages from the previous stage and @@ -301,7 +361,7 @@ class Pipeline(object): be at least two stages. """ if len(stages) < 2: - raise ValueError('pipeline must have at least two stages') + raise ValueError(u'pipeline must have at least two stages') self.stages = [] for stage in stages: if isinstance(stage, (list, tuple)): @@ -322,7 +382,8 @@ class Pipeline(object): messages between the stages are stored in queues of the given size. """ - queues = [CountedQueue(queue_size) for i in range(len(self.stages)-1)] + queue_count = len(self.stages) - 1 + queues = [CountedQueue(queue_size) for i in range(queue_count)] threads = [] # Set up first stage. @@ -330,10 +391,10 @@ class Pipeline(object): threads.append(FirstPipelineThread(coro, queues[0], threads)) # Middle stages. - for i in range(1, len(self.stages)-1): + for i in range(1, queue_count): for coro in self.stages[i]: threads.append(MiddlePipelineThread( - coro, queues[i-1], queues[i], threads + coro, queues[i - 1], queues[i], threads )) # Last stage. @@ -383,7 +444,7 @@ class Pipeline(object): # "Prime" the coroutines. for coro in coros[1:]: - coro.next() + next(coro) # Begin the pipeline. for out in coros[0]: @@ -405,20 +466,23 @@ if __name__ == '__main__': # in parallel. def produce(): for i in range(5): - print('generating %i' % i) + print(u'generating %i' % i) time.sleep(1) yield i + def work(): num = yield while True: - print('processing %i' % num) + print(u'processing %i' % num) time.sleep(2) - num = yield num*2 + num = yield num * 2 + def consume(): while True: num = yield time.sleep(1) - print('received %i' % num) + print(u'received %i' % num) + ts_start = time.time() Pipeline([produce(), work(), consume()]).run_sequential() ts_seq = time.time() @@ -426,29 +490,30 @@ if __name__ == '__main__': ts_par = time.time() Pipeline([produce(), (work(), work()), consume()]).run_parallel() ts_end = time.time() - print('Sequential time:', ts_seq - ts_start) - print('Parallel time:', ts_par - ts_seq) - print('Multiply-parallel time:', ts_end - ts_par) + print(u'Sequential time:', ts_seq - ts_start) + print(u'Parallel time:', ts_par - ts_seq) + print(u'Multiply-parallel time:', ts_end - ts_par) print() # Test a pipeline that raises an exception. def exc_produce(): for i in range(10): - print('generating %i' % i) + print(u'generating %i' % i) time.sleep(1) yield i + def exc_work(): num = yield while True: - print('processing %i' % num) + print(u'processing %i' % num) time.sleep(3) if num == 3: raise Exception() num = yield num * 2 + def exc_consume(): while True: num = yield - #if num == 4: - # raise Exception() - print('received %i' % num) + print(u'received %i' % num) + Pipeline([exc_produce(), exc_work(), exc_consume()]).run_parallel(1) diff --git a/libs/beets/vfs.py b/libs/beets/vfs.py index 235f3604..7f9a049e 100644 --- a/libs/beets/vfs.py +++ b/libs/beets/vfs.py @@ -1,5 +1,6 @@ +# -*- coding: utf-8 -*- # This file is part of beets. -# Copyright 2013, Adrian Sampson. +# Copyright 2016, Adrian Sampson. # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -15,11 +16,14 @@ """A simple utility for constructing filesystem-like trees from beets libraries. """ +from __future__ import division, absolute_import, print_function + from collections import namedtuple from beets import util Node = namedtuple('Node', ['files', 'dirs']) + def _insert(node, path, itemid): """Insert an item into a virtual filesystem node.""" if len(path) == 1: @@ -33,6 +37,7 @@ def _insert(node, path, itemid): node.dirs[dirname] = Node({}, {}) _insert(node.dirs[dirname], rest, itemid) + def libtree(lib): """Generates a filesystem-like directory tree for the files contained in `lib`. Filesystem nodes are (files, dirs) named diff --git a/libs/beetsplug/__init__.py b/libs/beetsplug/__init__.py new file mode 100644 index 00000000..febeb66f --- /dev/null +++ b/libs/beetsplug/__init__.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Adrian Sampson. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""A namespace package for beets plugins.""" + +from __future__ import division, absolute_import, print_function + +# Make this a namespace package. +from pkgutil import extend_path +__path__ = extend_path(__path__, __name__) diff --git a/libs/beetsplug/acousticbrainz.py b/libs/beetsplug/acousticbrainz.py new file mode 100644 index 00000000..df790b26 --- /dev/null +++ b/libs/beetsplug/acousticbrainz.py @@ -0,0 +1,165 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2015-2016, Ohm Patel. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Fetch various AcousticBrainz metadata using MBID. +""" +from __future__ import division, absolute_import, print_function + +import requests +import operator + +from beets import plugins, ui +from functools import reduce + +ACOUSTIC_BASE = "https://acousticbrainz.org/" +LEVELS = ["/low-level", "/high-level"] + + +class AcousticPlugin(plugins.BeetsPlugin): + def __init__(self): + super(AcousticPlugin, self).__init__() + + self.config.add({'auto': True}) + if self.config['auto']: + self.register_listener('import_task_files', + self.import_task_files) + + def commands(self): + cmd = ui.Subcommand('acousticbrainz', + help=u"fetch metadata from AcousticBrainz") + + def func(lib, opts, args): + items = lib.items(ui.decargs(args)) + fetch_info(self._log, items, ui.should_write()) + + cmd.func = func + return [cmd] + + def import_task_files(self, session, task): + """Function is called upon beet import. + """ + + items = task.imported_items() + fetch_info(self._log, items, False) + + +def fetch_info(log, items, write): + """Get data from AcousticBrainz for the items. + """ + + def get_value(*map_path): + try: + return reduce(operator.getitem, map_path, data) + except KeyError: + log.debug(u'Invalid Path: {}', map_path) + + for item in items: + if item.mb_trackid: + log.info(u'getting data for: {}', item) + + # Fetch the data from the AB API. + urls = [generate_url(item.mb_trackid, path) for path in LEVELS] + log.debug(u'fetching URLs: {}', ' '.join(urls)) + try: + res = [requests.get(url) for url in urls] + except requests.RequestException as exc: + log.info(u'request error: {}', exc) + continue + + # Check for missing tracks. + if any(r.status_code == 404 for r in res): + log.info(u'recording ID {} not found', item.mb_trackid) + continue + + # Parse the JSON response. + try: + data = res[0].json() + data.update(res[1].json()) + except ValueError: + log.debug(u'Invalid Response: {} & {}', [r.text for r in res]) + + # Get each field and assign it on the item. + item.danceable = get_value( + "highlevel", "danceability", "all", "danceable", + ) + item.gender = get_value( + "highlevel", "gender", "value", + ) + item.genre_rosamerica = get_value( + "highlevel", "genre_rosamerica", "value" + ) + item.mood_acoustic = get_value( + "highlevel", "mood_acoustic", "all", "acoustic" + ) + item.mood_aggressive = get_value( + "highlevel", "mood_aggressive", "all", "aggressive" + ) + item.mood_electronic = get_value( + "highlevel", "mood_electronic", "all", "electronic" + ) + item.mood_happy = get_value( + "highlevel", "mood_happy", "all", "happy" + ) + item.mood_party = get_value( + "highlevel", "mood_party", "all", "party" + ) + item.mood_relaxed = get_value( + "highlevel", "mood_relaxed", "all", "relaxed" + ) + item.mood_sad = get_value( + "highlevel", "mood_sad", "all", "sad" + ) + item.rhythm = get_value( + "highlevel", "ismir04_rhythm", "value" + ) + item.tonal = get_value( + "highlevel", "tonal_atonal", "all", "tonal" + ) + item.voice_instrumental = get_value( + "highlevel", "voice_instrumental", "value" + ) + item.average_loudness = get_value( + "lowlevel", "average_loudness" + ) + item.chords_changes_rate = get_value( + "tonal", "chords_changes_rate" + ) + item.chords_key = get_value( + "tonal", "chords_key" + ) + item.chords_number_rate = get_value( + "tonal", "chords_number_rate" + ) + item.chords_scale = get_value( + "tonal", "chords_scale" + ) + item.initial_key = '{} {}'.format( + get_value("tonal", "key_key"), + get_value("tonal", "key_scale") + ) + item.key_strength = get_value( + "tonal", "key_strength" + ) + + # Store the data. + item.store() + if write: + item.try_write() + + +def generate_url(mbid, level): + """Generates AcousticBrainz end point url for given MBID. + """ + return ACOUSTIC_BASE + mbid + level diff --git a/libs/beetsplug/badfiles.py b/libs/beetsplug/badfiles.py new file mode 100644 index 00000000..f9704d48 --- /dev/null +++ b/libs/beetsplug/badfiles.py @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, François-Xavier Thomas. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Use command-line tools to check for audio file corruption. +""" + +from __future__ import division, absolute_import, print_function + +from beets.plugins import BeetsPlugin +from beets.ui import Subcommand +from beets.util import displayable_path, confit +from beets import ui +from subprocess import check_output, CalledProcessError, list2cmdline, STDOUT +import shlex +import os +import errno +import sys + + +class BadFiles(BeetsPlugin): + def run_command(self, cmd): + self._log.debug(u"running command: {}", + displayable_path(list2cmdline(cmd))) + try: + output = check_output(cmd, stderr=STDOUT) + errors = 0 + status = 0 + except CalledProcessError as e: + output = e.output + errors = 1 + status = e.returncode + except OSError as e: + if e.errno == errno.ENOENT: + ui.print_(u"command not found: {}".format(cmd[0])) + sys.exit(1) + else: + raise + output = output.decode(sys.getfilesystemencoding()) + return status, errors, [line for line in output.split("\n") if line] + + def check_mp3val(self, path): + status, errors, output = self.run_command(["mp3val", path]) + if status == 0: + output = [line for line in output if line.startswith("WARNING:")] + errors = len(output) + return status, errors, output + + def check_flac(self, path): + return self.run_command(["flac", "-wst", path]) + + def check_custom(self, command): + def checker(path): + cmd = shlex.split(command) + cmd.append(path) + return self.run_command(cmd) + return checker + + def get_checker(self, ext): + ext = ext.lower() + try: + command = self.config['commands'].get(dict).get(ext) + except confit.NotFoundError: + command = None + if command: + return self.check_custom(command) + elif ext == "mp3": + return self.check_mp3val + elif ext == "flac": + return self.check_flac + + def check_bad(self, lib, opts, args): + for item in lib.items(ui.decargs(args)): + + # First, check whether the path exists. If not, the user + # should probably run `beet update` to cleanup your library. + dpath = displayable_path(item.path) + self._log.debug(u"checking path: {}", dpath) + if not os.path.exists(item.path): + ui.print_(u"{}: file does not exist".format( + ui.colorize('text_error', dpath))) + + # Run the checker against the file if one is found + ext = os.path.splitext(item.path)[1][1:] + checker = self.get_checker(ext) + if not checker: + continue + path = item.path + if not isinstance(path, unicode): + path = item.path.decode(sys.getfilesystemencoding()) + status, errors, output = checker(path) + if status > 0: + ui.print_(u"{}: checker exited withs status {}" + .format(ui.colorize('text_error', dpath), status)) + for line in output: + ui.print_(" {}".format(displayable_path(line))) + elif errors > 0: + ui.print_(u"{}: checker found {} errors or warnings" + .format(ui.colorize('text_warning', dpath), errors)) + for line in output: + ui.print_(u" {}".format(displayable_path(line))) + else: + ui.print_(u"{}: ok".format(ui.colorize('text_success', dpath))) + + def commands(self): + bad_command = Subcommand('bad', + help=u'check for corrupt or missing files') + bad_command.func = self.check_bad + return [bad_command] diff --git a/libs/beetsplug/bench.py b/libs/beetsplug/bench.py new file mode 100644 index 00000000..41f575cd --- /dev/null +++ b/libs/beetsplug/bench.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Adrian Sampson. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Some simple performance benchmarks for beets. +""" + +from __future__ import division, absolute_import, print_function + +from beets.plugins import BeetsPlugin +from beets import ui +from beets import vfs +from beets import library +from beets.util.functemplate import Template +from beets.autotag import match +from beets import plugins +from beets import importer +import cProfile +import timeit + + +def aunique_benchmark(lib, prof): + def _build_tree(): + vfs.libtree(lib) + + # Measure path generation performance with %aunique{} included. + lib.path_formats = [ + (library.PF_KEY_DEFAULT, + Template('$albumartist/$album%aunique{}/$track $title')), + ] + if prof: + cProfile.runctx('_build_tree()', {}, {'_build_tree': _build_tree}, + 'paths.withaunique.prof') + else: + interval = timeit.timeit(_build_tree, number=1) + print('With %aunique:', interval) + + # And with %aunique replaceed with a "cheap" no-op function. + lib.path_formats = [ + (library.PF_KEY_DEFAULT, + Template('$albumartist/$album%lower{}/$track $title')), + ] + if prof: + cProfile.runctx('_build_tree()', {}, {'_build_tree': _build_tree}, + 'paths.withoutaunique.prof') + else: + interval = timeit.timeit(_build_tree, number=1) + print('Without %aunique:', interval) + + +def match_benchmark(lib, prof, query=None, album_id=None): + # If no album ID is provided, we'll match against a suitably huge + # album. + if not album_id: + album_id = '9c5c043e-bc69-4edb-81a4-1aaf9c81e6dc' + + # Get an album from the library to use as the source for the match. + items = lib.albums(query).get().items() + + # Ensure fingerprinting is invoked (if enabled). + plugins.send('import_task_start', + task=importer.ImportTask(None, None, items), + session=importer.ImportSession(lib, None, None, None)) + + # Run the match. + def _run_match(): + match.tag_album(items, search_ids=[album_id]) + if prof: + cProfile.runctx('_run_match()', {}, {'_run_match': _run_match}, + 'match.prof') + else: + interval = timeit.timeit(_run_match, number=1) + print('match duration:', interval) + + +class BenchmarkPlugin(BeetsPlugin): + """A plugin for performing some simple performance benchmarks. + """ + def commands(self): + aunique_bench_cmd = ui.Subcommand('bench_aunique', + help='benchmark for %aunique{}') + aunique_bench_cmd.parser.add_option('-p', '--profile', + action='store_true', default=False, + help='performance profiling') + aunique_bench_cmd.func = lambda lib, opts, args: \ + aunique_benchmark(lib, opts.profile) + + match_bench_cmd = ui.Subcommand('bench_match', + help='benchmark for track matching') + match_bench_cmd.parser.add_option('-p', '--profile', + action='store_true', default=False, + help='performance profiling') + match_bench_cmd.parser.add_option('-i', '--id', default=None, + help='album ID to match against') + match_bench_cmd.func = lambda lib, opts, args: \ + match_benchmark(lib, opts.profile, ui.decargs(args), opts.id) + + return [aunique_bench_cmd, match_bench_cmd] diff --git a/libs/beetsplug/bpd/__init__.py b/libs/beetsplug/bpd/__init__.py new file mode 100644 index 00000000..33deda02 --- /dev/null +++ b/libs/beetsplug/bpd/__init__.py @@ -0,0 +1,1193 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Adrian Sampson. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""A clone of the Music Player Daemon (MPD) that plays music from a +Beets library. Attempts to implement a compatible protocol to allow +use of the wide range of MPD clients. +""" + +from __future__ import division, absolute_import, print_function + +import re +from string import Template +import traceback +import random +import time + +import beets +from beets.plugins import BeetsPlugin +import beets.ui +from beets import logging +from beets import vfs +from beets.util import bluelet +from beets.library import Item +from beets import dbcore +from beets.mediafile import MediaFile + +PROTOCOL_VERSION = '0.13.0' +BUFSIZE = 1024 + +HELLO = 'OK MPD %s' % PROTOCOL_VERSION +CLIST_BEGIN = 'command_list_begin' +CLIST_VERBOSE_BEGIN = 'command_list_ok_begin' +CLIST_END = 'command_list_end' +RESP_OK = 'OK' +RESP_CLIST_VERBOSE = 'list_OK' +RESP_ERR = 'ACK' + +NEWLINE = u"\n" + +ERROR_NOT_LIST = 1 +ERROR_ARG = 2 +ERROR_PASSWORD = 3 +ERROR_PERMISSION = 4 +ERROR_UNKNOWN = 5 +ERROR_NO_EXIST = 50 +ERROR_PLAYLIST_MAX = 51 +ERROR_SYSTEM = 52 +ERROR_PLAYLIST_LOAD = 53 +ERROR_UPDATE_ALREADY = 54 +ERROR_PLAYER_SYNC = 55 +ERROR_EXIST = 56 + +VOLUME_MIN = 0 +VOLUME_MAX = 100 + +SAFE_COMMANDS = ( + # Commands that are available when unauthenticated. + u'close', u'commands', u'notcommands', u'password', u'ping', +) + +ITEM_KEYS_WRITABLE = set(MediaFile.fields()).intersection(Item._fields.keys()) + +# Loggers. +log = logging.getLogger('beets.bpd') +global_log = logging.getLogger('beets') + + +# Gstreamer import error. +class NoGstreamerError(Exception): + pass + + +# Error-handling, exceptions, parameter parsing. + +class BPDError(Exception): + """An error that should be exposed to the client to the BPD + server. + """ + def __init__(self, code, message, cmd_name='', index=0): + self.code = code + self.message = message + self.cmd_name = cmd_name + self.index = index + + template = Template(u'$resp [$code@$index] {$cmd_name} $message') + + def response(self): + """Returns a string to be used as the response code for the + erring command. + """ + return self.template.substitute({ + 'resp': RESP_ERR, + 'code': self.code, + 'index': self.index, + 'cmd_name': self.cmd_name, + 'message': self.message, + }) + + +def make_bpd_error(s_code, s_message): + """Create a BPDError subclass for a static code and message. + """ + + class NewBPDError(BPDError): + code = s_code + message = s_message + cmd_name = '' + index = 0 + + def __init__(self): + pass + return NewBPDError + +ArgumentTypeError = make_bpd_error(ERROR_ARG, u'invalid type for argument') +ArgumentIndexError = make_bpd_error(ERROR_ARG, u'argument out of range') +ArgumentNotFoundError = make_bpd_error(ERROR_NO_EXIST, u'argument not found') + + +def cast_arg(t, val): + """Attempts to call t on val, raising a ArgumentTypeError + on ValueError. + + If 't' is the special string 'intbool', attempts to cast first + to an int and then to a bool (i.e., 1=True, 0=False). + """ + if t == 'intbool': + return cast_arg(bool, cast_arg(int, val)) + else: + try: + return t(val) + except ValueError: + raise ArgumentTypeError() + + +class BPDClose(Exception): + """Raised by a command invocation to indicate that the connection + should be closed. + """ + +# Generic server infrastructure, implementing the basic protocol. + + +class BaseServer(object): + """A MPD-compatible music player server. + + The functions with the `cmd_` prefix are invoked in response to + client commands. For instance, if the client says `status`, + `cmd_status` will be invoked. The arguments to the client's commands + are used as function arguments following the connection issuing the + command. The functions may send data on the connection. They may + also raise BPDError exceptions to report errors. + + This is a generic superclass and doesn't support many commands. + """ + + def __init__(self, host, port, password): + """Create a new server bound to address `host` and listening + on port `port`. If `password` is given, it is required to do + anything significant on the server. + """ + self.host, self.port, self.password = host, port, password + + # Default server values. + self.random = False + self.repeat = False + self.volume = VOLUME_MAX + self.crossfade = 0 + self.playlist = [] + self.playlist_version = 0 + self.current_index = -1 + self.paused = False + self.error = None + + # Object for random numbers generation + self.random_obj = random.Random() + + def run(self): + """Block and start listening for connections from clients. An + interrupt (^C) closes the server. + """ + self.startup_time = time.time() + bluelet.run(bluelet.server(self.host, self.port, + Connection.handler(self))) + + def _item_info(self, item): + """An abstract method that should response lines containing a + single song's metadata. + """ + raise NotImplementedError + + def _item_id(self, item): + """An abstract method returning the integer id for an item. + """ + raise NotImplementedError + + def _id_to_index(self, track_id): + """Searches the playlist for a song with the given id and + returns its index in the playlist. + """ + track_id = cast_arg(int, track_id) + for index, track in enumerate(self.playlist): + if self._item_id(track) == track_id: + return index + # Loop finished with no track found. + raise ArgumentNotFoundError() + + def _random_idx(self): + """Returns a random index different from the current one. + If there are no songs in the playlist it returns -1. + If there is only one song in the playlist it returns 0. + """ + if len(self.playlist) < 2: + return len(self.playlist) - 1 + new_index = self.random_obj.randint(0, len(self.playlist) - 1) + while new_index == self.current_index: + new_index = self.random_obj.randint(0, len(self.playlist) - 1) + return new_index + + def _succ_idx(self): + """Returns the index for the next song to play. + It also considers random and repeat flags. + No boundaries are checked. + """ + if self.repeat: + return self.current_index + if self.random: + return self._random_idx() + return self.current_index + 1 + + def _prev_idx(self): + """Returns the index for the previous song to play. + It also considers random and repeat flags. + No boundaries are checked. + """ + if self.repeat: + return self.current_index + if self.random: + return self._random_idx() + return self.current_index - 1 + + def cmd_ping(self, conn): + """Succeeds.""" + pass + + def cmd_kill(self, conn): + """Exits the server process.""" + exit(0) + + def cmd_close(self, conn): + """Closes the connection.""" + raise BPDClose() + + def cmd_password(self, conn, password): + """Attempts password authentication.""" + if password == self.password: + conn.authenticated = True + else: + conn.authenticated = False + raise BPDError(ERROR_PASSWORD, u'incorrect password') + + def cmd_commands(self, conn): + """Lists the commands available to the user.""" + if self.password and not conn.authenticated: + # Not authenticated. Show limited list of commands. + for cmd in SAFE_COMMANDS: + yield u'command: ' + cmd + + else: + # Authenticated. Show all commands. + for func in dir(self): + if func.startswith('cmd_'): + yield u'command: ' + func[4:] + + def cmd_notcommands(self, conn): + """Lists all unavailable commands.""" + if self.password and not conn.authenticated: + # Not authenticated. Show privileged commands. + for func in dir(self): + if func.startswith('cmd_'): + cmd = func[4:] + if cmd not in SAFE_COMMANDS: + yield u'command: ' + cmd + + else: + # Authenticated. No commands are unavailable. + pass + + def cmd_status(self, conn): + """Returns some status information for use with an + implementation of cmd_status. + + Gives a list of response-lines for: volume, repeat, random, + playlist, playlistlength, and xfade. + """ + yield ( + u'volume: ' + unicode(self.volume), + u'repeat: ' + unicode(int(self.repeat)), + u'random: ' + unicode(int(self.random)), + u'playlist: ' + unicode(self.playlist_version), + u'playlistlength: ' + unicode(len(self.playlist)), + u'xfade: ' + unicode(self.crossfade), + ) + + if self.current_index == -1: + state = u'stop' + elif self.paused: + state = u'pause' + else: + state = u'play' + yield u'state: ' + state + + if self.current_index != -1: # i.e., paused or playing + current_id = self._item_id(self.playlist[self.current_index]) + yield u'song: ' + unicode(self.current_index) + yield u'songid: ' + unicode(current_id) + + if self.error: + yield u'error: ' + self.error + + def cmd_clearerror(self, conn): + """Removes the persistent error state of the server. This + error is set when a problem arises not in response to a + command (for instance, when playing a file). + """ + self.error = None + + def cmd_random(self, conn, state): + """Set or unset random (shuffle) mode.""" + self.random = cast_arg('intbool', state) + + def cmd_repeat(self, conn, state): + """Set or unset repeat mode.""" + self.repeat = cast_arg('intbool', state) + + def cmd_setvol(self, conn, vol): + """Set the player's volume level (0-100).""" + vol = cast_arg(int, vol) + if vol < VOLUME_MIN or vol > VOLUME_MAX: + raise BPDError(ERROR_ARG, u'volume out of range') + self.volume = vol + + def cmd_crossfade(self, conn, crossfade): + """Set the number of seconds of crossfading.""" + crossfade = cast_arg(int, crossfade) + if crossfade < 0: + raise BPDError(ERROR_ARG, u'crossfade time must be nonnegative') + + def cmd_clear(self, conn): + """Clear the playlist.""" + self.playlist = [] + self.playlist_version += 1 + self.cmd_stop(conn) + + def cmd_delete(self, conn, index): + """Remove the song at index from the playlist.""" + index = cast_arg(int, index) + try: + del(self.playlist[index]) + except IndexError: + raise ArgumentIndexError() + self.playlist_version += 1 + + if self.current_index == index: # Deleted playing song. + self.cmd_stop(conn) + elif index < self.current_index: # Deleted before playing. + # Shift playing index down. + self.current_index -= 1 + + def cmd_deleteid(self, conn, track_id): + self.cmd_delete(conn, self._id_to_index(track_id)) + + def cmd_move(self, conn, idx_from, idx_to): + """Move a track in the playlist.""" + idx_from = cast_arg(int, idx_from) + idx_to = cast_arg(int, idx_to) + try: + track = self.playlist.pop(idx_from) + self.playlist.insert(idx_to, track) + except IndexError: + raise ArgumentIndexError() + + # Update currently-playing song. + if idx_from == self.current_index: + self.current_index = idx_to + elif idx_from < self.current_index <= idx_to: + self.current_index -= 1 + elif idx_from > self.current_index >= idx_to: + self.current_index += 1 + + self.playlist_version += 1 + + def cmd_moveid(self, conn, idx_from, idx_to): + idx_from = self._id_to_index(idx_from) + return self.cmd_move(conn, idx_from, idx_to) + + def cmd_swap(self, conn, i, j): + """Swaps two tracks in the playlist.""" + i = cast_arg(int, i) + j = cast_arg(int, j) + try: + track_i = self.playlist[i] + track_j = self.playlist[j] + except IndexError: + raise ArgumentIndexError() + + self.playlist[j] = track_i + self.playlist[i] = track_j + + # Update currently-playing song. + if self.current_index == i: + self.current_index = j + elif self.current_index == j: + self.current_index = i + + self.playlist_version += 1 + + def cmd_swapid(self, conn, i_id, j_id): + i = self._id_to_index(i_id) + j = self._id_to_index(j_id) + return self.cmd_swap(conn, i, j) + + def cmd_urlhandlers(self, conn): + """Indicates supported URL schemes. None by default.""" + pass + + def cmd_playlistinfo(self, conn, index=-1): + """Gives metadata information about the entire playlist or a + single track, given by its index. + """ + index = cast_arg(int, index) + if index == -1: + for track in self.playlist: + yield self._item_info(track) + else: + try: + track = self.playlist[index] + except IndexError: + raise ArgumentIndexError() + yield self._item_info(track) + + def cmd_playlistid(self, conn, track_id=-1): + return self.cmd_playlistinfo(conn, self._id_to_index(track_id)) + + def cmd_plchanges(self, conn, version): + """Sends playlist changes since the given version. + + This is a "fake" implementation that ignores the version and + just returns the entire playlist (rather like version=0). This + seems to satisfy many clients. + """ + return self.cmd_playlistinfo(conn) + + def cmd_plchangesposid(self, conn, version): + """Like plchanges, but only sends position and id. + + Also a dummy implementation. + """ + for idx, track in enumerate(self.playlist): + yield u'cpos: ' + unicode(idx) + yield u'Id: ' + unicode(track.id) + + def cmd_currentsong(self, conn): + """Sends information about the currently-playing song. + """ + if self.current_index != -1: # -1 means stopped. + track = self.playlist[self.current_index] + yield self._item_info(track) + + def cmd_next(self, conn): + """Advance to the next song in the playlist.""" + self.current_index = self._succ_idx() + if self.current_index >= len(self.playlist): + # Fallen off the end. Just move to stopped state. + return self.cmd_stop(conn) + else: + return self.cmd_play(conn) + + def cmd_previous(self, conn): + """Step back to the last song.""" + self.current_index = self._prev_idx() + if self.current_index < 0: + return self.cmd_stop(conn) + else: + return self.cmd_play(conn) + + def cmd_pause(self, conn, state=None): + """Set the pause state playback.""" + if state is None: + self.paused = not self.paused # Toggle. + else: + self.paused = cast_arg('intbool', state) + + def cmd_play(self, conn, index=-1): + """Begin playback, possibly at a specified playlist index.""" + index = cast_arg(int, index) + + if index < -1 or index > len(self.playlist): + raise ArgumentIndexError() + + if index == -1: # No index specified: start where we are. + if not self.playlist: # Empty playlist: stop immediately. + return self.cmd_stop(conn) + if self.current_index == -1: # No current song. + self.current_index = 0 # Start at the beginning. + # If we have a current song, just stay there. + + else: # Start with the specified index. + self.current_index = index + + self.paused = False + + def cmd_playid(self, conn, track_id=0): + track_id = cast_arg(int, track_id) + if track_id == -1: + index = -1 + else: + index = self._id_to_index(track_id) + return self.cmd_play(conn, index) + + def cmd_stop(self, conn): + """Stop playback.""" + self.current_index = -1 + self.paused = False + + def cmd_seek(self, conn, index, pos): + """Seek to a specified point in a specified song.""" + index = cast_arg(int, index) + if index < 0 or index >= len(self.playlist): + raise ArgumentIndexError() + self.current_index = index + + def cmd_seekid(self, conn, track_id, pos): + index = self._id_to_index(track_id) + return self.cmd_seek(conn, index, pos) + + def cmd_profile(self, conn): + """Memory profiling for debugging.""" + from guppy import hpy + heap = hpy().heap() + print(heap) + + +class Connection(object): + """A connection between a client and the server. Handles input and + output from and to the client. + """ + def __init__(self, server, sock): + """Create a new connection for the accepted socket `client`. + """ + self.server = server + self.sock = sock + self.authenticated = False + + def send(self, lines): + """Send lines, which which is either a single string or an + iterable consisting of strings, to the client. A newline is + added after every string. Returns a Bluelet event that sends + the data. + """ + if isinstance(lines, basestring): + lines = [lines] + out = NEWLINE.join(lines) + NEWLINE + log.debug('{}', out[:-1]) # Don't log trailing newline. + if isinstance(out, unicode): + out = out.encode('utf8') + return self.sock.sendall(out) + + def do_command(self, command): + """A coroutine that runs the given command and sends an + appropriate response.""" + try: + yield bluelet.call(command.run(self)) + except BPDError as e: + # Send the error. + yield self.send(e.response()) + else: + # Send success code. + yield self.send(RESP_OK) + + def run(self): + """Send a greeting to the client and begin processing commands + as they arrive. + """ + yield self.send(HELLO) + + clist = None # Initially, no command list is being constructed. + while True: + line = yield self.sock.readline() + if not line: + break + line = line.strip() + if not line: + break + log.debug('{}', line) + + if clist is not None: + # Command list already opened. + if line == CLIST_END: + yield bluelet.call(self.do_command(clist)) + clist = None # Clear the command list. + else: + clist.append(Command(line)) + + elif line == CLIST_BEGIN or line == CLIST_VERBOSE_BEGIN: + # Begin a command list. + clist = CommandList([], line == CLIST_VERBOSE_BEGIN) + + else: + # Ordinary command. + try: + yield bluelet.call(self.do_command(Command(line))) + except BPDClose: + # Command indicates that the conn should close. + self.sock.close() + return + + @classmethod + def handler(cls, server): + def _handle(sock): + """Creates a new `Connection` and runs it. + """ + return cls(server, sock).run() + return _handle + + +class Command(object): + """A command issued by the client for processing by the server. + """ + + command_re = re.compile(br'^([^ \t]+)[ \t]*') + arg_re = re.compile(br'"((?:\\"|[^"])+)"|([^ \t"]+)') + + def __init__(self, s): + """Creates a new `Command` from the given string, `s`, parsing + the string for command name and arguments. + """ + command_match = self.command_re.match(s) + self.name = command_match.group(1) + + self.args = [] + arg_matches = self.arg_re.findall(s[command_match.end():]) + for match in arg_matches: + if match[0]: + # Quoted argument. + arg = match[0] + arg = arg.replace(b'\\"', b'"').replace(b'\\\\', b'\\') + else: + # Unquoted argument. + arg = match[1] + arg = arg.decode('utf8') + self.args.append(arg) + + def run(self, conn): + """A coroutine that executes the command on the given + connection. + """ + # Attempt to get correct command function. + func_name = 'cmd_' + self.name + if not hasattr(conn.server, func_name): + raise BPDError(ERROR_UNKNOWN, u'unknown command', self.name) + func = getattr(conn.server, func_name) + + # Ensure we have permission for this command. + if conn.server.password and \ + not conn.authenticated and \ + self.name not in SAFE_COMMANDS: + raise BPDError(ERROR_PERMISSION, u'insufficient privileges') + + try: + args = [conn] + self.args + results = func(*args) + if results: + for data in results: + yield conn.send(data) + + except BPDError as e: + # An exposed error. Set the command name and then let + # the Connection handle it. + e.cmd_name = self.name + raise e + + except BPDClose: + # An indication that the connection should close. Send + # it on the Connection. + raise + + except Exception as e: + # An "unintentional" error. Hide it from the client. + log.error('{}', traceback.format_exc(e)) + raise BPDError(ERROR_SYSTEM, u'server error', self.name) + + +class CommandList(list): + """A list of commands issued by the client for processing by the + server. May be verbose, in which case the response is delimited, or + not. Should be a list of `Command` objects. + """ + + def __init__(self, sequence=None, verbose=False): + """Create a new `CommandList` from the given sequence of + `Command`s. If `verbose`, this is a verbose command list. + """ + if sequence: + for item in sequence: + self.append(item) + self.verbose = verbose + + def run(self, conn): + """Coroutine executing all the commands in this list. + """ + for i, command in enumerate(self): + try: + yield bluelet.call(command.run(conn)) + except BPDError as e: + # If the command failed, stop executing. + e.index = i # Give the error the correct index. + raise e + + # Otherwise, possibly send the output delimeter if we're in a + # verbose ("OK") command list. + if self.verbose: + yield conn.send(RESP_CLIST_VERBOSE) + + +# A subclass of the basic, protocol-handling server that actually plays +# music. + +class Server(BaseServer): + """An MPD-compatible server using GStreamer to play audio and beets + to store its library. + """ + + def __init__(self, library, host, port, password): + try: + from beetsplug.bpd import gstplayer + except ImportError as e: + # This is a little hacky, but it's the best I know for now. + if e.args[0].endswith(' gst'): + raise NoGstreamerError() + else: + raise + super(Server, self).__init__(host, port, password) + self.lib = library + self.player = gstplayer.GstPlayer(self.play_finished) + self.cmd_update(None) + + def run(self): + self.player.run() + super(Server, self).run() + + def play_finished(self): + """A callback invoked every time our player finishes a + track. + """ + self.cmd_next(None) + + # Metadata helper functions. + + def _item_info(self, item): + info_lines = [ + u'file: ' + item.destination(fragment=True), + u'Time: ' + unicode(int(item.length)), + u'Title: ' + item.title, + u'Artist: ' + item.artist, + u'Album: ' + item.album, + u'Genre: ' + item.genre, + ] + + track = unicode(item.track) + if item.tracktotal: + track += u'/' + unicode(item.tracktotal) + info_lines.append(u'Track: ' + track) + + info_lines.append(u'Date: ' + unicode(item.year)) + + try: + pos = self._id_to_index(item.id) + info_lines.append(u'Pos: ' + unicode(pos)) + except ArgumentNotFoundError: + # Don't include position if not in playlist. + pass + + info_lines.append(u'Id: ' + unicode(item.id)) + + return info_lines + + def _item_id(self, item): + return item.id + + # Database updating. + + def cmd_update(self, conn, path=u'/'): + """Updates the catalog to reflect the current database state. + """ + # Path is ignored. Also, the real MPD does this asynchronously; + # this is done inline. + print(u'Building directory tree...') + self.tree = vfs.libtree(self.lib) + print(u'... done.') + self.updated_time = time.time() + + # Path (directory tree) browsing. + + def _resolve_path(self, path): + """Returns a VFS node or an item ID located at the path given. + If the path does not exist, raises a + """ + components = path.split(u'/') + node = self.tree + + for component in components: + if not component: + continue + + if isinstance(node, int): + # We're trying to descend into a file node. + raise ArgumentNotFoundError() + + if component in node.files: + node = node.files[component] + elif component in node.dirs: + node = node.dirs[component] + else: + raise ArgumentNotFoundError() + + return node + + def _path_join(self, p1, p2): + """Smashes together two BPD paths.""" + out = p1 + u'/' + p2 + return out.replace(u'//', u'/').replace(u'//', u'/') + + def cmd_lsinfo(self, conn, path=u"/"): + """Sends info on all the items in the path.""" + node = self._resolve_path(path) + if isinstance(node, int): + # Trying to list a track. + raise BPDError(ERROR_ARG, u'this is not a directory') + else: + for name, itemid in iter(sorted(node.files.items())): + item = self.lib.get_item(itemid) + yield self._item_info(item) + for name, _ in iter(sorted(node.dirs.iteritems())): + dirpath = self._path_join(path, name) + if dirpath.startswith(u"/"): + # Strip leading slash (libmpc rejects this). + dirpath = dirpath[1:] + yield u'directory: %s' % dirpath + + def _listall(self, basepath, node, info=False): + """Helper function for recursive listing. If info, show + tracks' complete info; otherwise, just show items' paths. + """ + if isinstance(node, int): + # List a single file. + if info: + item = self.lib.get_item(node) + yield self._item_info(item) + else: + yield u'file: ' + basepath + else: + # List a directory. Recurse into both directories and files. + for name, itemid in sorted(node.files.iteritems()): + newpath = self._path_join(basepath, name) + # "yield from" + for v in self._listall(newpath, itemid, info): + yield v + for name, subdir in sorted(node.dirs.iteritems()): + newpath = self._path_join(basepath, name) + yield u'directory: ' + newpath + for v in self._listall(newpath, subdir, info): + yield v + + def cmd_listall(self, conn, path=u"/"): + """Send the paths all items in the directory, recursively.""" + return self._listall(path, self._resolve_path(path), False) + + def cmd_listallinfo(self, conn, path=u"/"): + """Send info on all the items in the directory, recursively.""" + return self._listall(path, self._resolve_path(path), True) + + # Playlist manipulation. + + def _all_items(self, node): + """Generator yielding all items under a VFS node. + """ + if isinstance(node, int): + # Could be more efficient if we built up all the IDs and + # then issued a single SELECT. + yield self.lib.get_item(node) + else: + # Recurse into a directory. + for name, itemid in sorted(node.files.iteritems()): + # "yield from" + for v in self._all_items(itemid): + yield v + for name, subdir in sorted(node.dirs.iteritems()): + for v in self._all_items(subdir): + yield v + + def _add(self, path, send_id=False): + """Adds a track or directory to the playlist, specified by the + path. If `send_id`, write each item's id to the client. + """ + for item in self._all_items(self._resolve_path(path)): + self.playlist.append(item) + if send_id: + yield u'Id: ' + unicode(item.id) + self.playlist_version += 1 + + def cmd_add(self, conn, path): + """Adds a track or directory to the playlist, specified by a + path. + """ + return self._add(path, False) + + def cmd_addid(self, conn, path): + """Same as `cmd_add` but sends an id back to the client.""" + return self._add(path, True) + + # Server info. + + def cmd_status(self, conn): + for line in super(Server, self).cmd_status(conn): + yield line + if self.current_index > -1: + item = self.playlist[self.current_index] + + yield u'bitrate: ' + unicode(item.bitrate / 1000) + # Missing 'audio'. + + (pos, total) = self.player.time() + yield u'time: ' + unicode(pos) + u':' + unicode(total) + + # Also missing 'updating_db'. + + def cmd_stats(self, conn): + """Sends some statistics about the library.""" + with self.lib.transaction() as tx: + statement = 'SELECT COUNT(DISTINCT artist), ' \ + 'COUNT(DISTINCT album), ' \ + 'COUNT(id), ' \ + 'SUM(length) ' \ + 'FROM items' + artists, albums, songs, totaltime = tx.query(statement)[0] + + yield ( + u'artists: ' + unicode(artists), + u'albums: ' + unicode(albums), + u'songs: ' + unicode(songs), + u'uptime: ' + unicode(int(time.time() - self.startup_time)), + u'playtime: ' + u'0', # Missing. + u'db_playtime: ' + unicode(int(totaltime)), + u'db_update: ' + unicode(int(self.updated_time)), + ) + + # Searching. + + tagtype_map = { + u'Artist': u'artist', + u'Album': u'album', + u'Title': u'title', + u'Track': u'track', + u'AlbumArtist': u'albumartist', + u'AlbumArtistSort': u'albumartist_sort', + # Name? + u'Genre': u'genre', + u'Date': u'year', + u'Composer': u'composer', + # Performer? + u'Disc': u'disc', + u'filename': u'path', # Suspect. + } + + def cmd_tagtypes(self, conn): + """Returns a list of the metadata (tag) fields available for + searching. + """ + for tag in self.tagtype_map: + yield u'tagtype: ' + tag + + def _tagtype_lookup(self, tag): + """Uses `tagtype_map` to look up the beets column name for an + MPD tagtype (or throw an appropriate exception). Returns both + the canonical name of the MPD tagtype and the beets column + name. + """ + for test_tag, key in self.tagtype_map.items(): + # Match case-insensitively. + if test_tag.lower() == tag.lower(): + return test_tag, key + raise BPDError(ERROR_UNKNOWN, u'no such tagtype') + + def _metadata_query(self, query_type, any_query_type, kv): + """Helper function returns a query object that will find items + according to the library query type provided and the key-value + pairs specified. The any_query_type is used for queries of + type "any"; if None, then an error is thrown. + """ + if kv: # At least one key-value pair. + queries = [] + # Iterate pairwise over the arguments. + it = iter(kv) + for tag, value in zip(it, it): + if tag.lower() == u'any': + if any_query_type: + queries.append(any_query_type(value, + ITEM_KEYS_WRITABLE, + query_type)) + else: + raise BPDError(ERROR_UNKNOWN, u'no such tagtype') + else: + _, key = self._tagtype_lookup(tag) + queries.append(query_type(key, value)) + return dbcore.query.AndQuery(queries) + else: # No key-value pairs. + return dbcore.query.TrueQuery() + + def cmd_search(self, conn, *kv): + """Perform a substring match for items.""" + query = self._metadata_query(dbcore.query.SubstringQuery, + dbcore.query.AnyFieldQuery, + kv) + for item in self.lib.items(query): + yield self._item_info(item) + + def cmd_find(self, conn, *kv): + """Perform an exact match for items.""" + query = self._metadata_query(dbcore.query.MatchQuery, + None, + kv) + for item in self.lib.items(query): + yield self._item_info(item) + + def cmd_list(self, conn, show_tag, *kv): + """List distinct metadata values for show_tag, possibly + filtered by matching match_tag to match_term. + """ + show_tag_canon, show_key = self._tagtype_lookup(show_tag) + query = self._metadata_query(dbcore.query.MatchQuery, None, kv) + + clause, subvals = query.clause() + statement = 'SELECT DISTINCT ' + show_key + \ + ' FROM items WHERE ' + clause + \ + ' ORDER BY ' + show_key + with self.lib.transaction() as tx: + rows = tx.query(statement, subvals) + + for row in rows: + yield show_tag_canon + u': ' + unicode(row[0]) + + def cmd_count(self, conn, tag, value): + """Returns the number and total time of songs matching the + tag/value query. + """ + _, key = self._tagtype_lookup(tag) + songs = 0 + playtime = 0.0 + for item in self.lib.items(dbcore.query.MatchQuery(key, value)): + songs += 1 + playtime += item.length + yield u'songs: ' + unicode(songs) + yield u'playtime: ' + unicode(int(playtime)) + + # "Outputs." Just a dummy implementation because we don't control + # any outputs. + + def cmd_outputs(self, conn): + """List the available outputs.""" + yield ( + u'outputid: 0', + u'outputname: gstreamer', + u'outputenabled: 1', + ) + + def cmd_enableoutput(self, conn, output_id): + output_id = cast_arg(int, output_id) + if output_id != 0: + raise ArgumentIndexError() + + def cmd_disableoutput(self, conn, output_id): + output_id = cast_arg(int, output_id) + if output_id == 0: + raise BPDError(ERROR_ARG, u'cannot disable this output') + else: + raise ArgumentIndexError() + + # Playback control. The functions below hook into the + # half-implementations provided by the base class. Together, they're + # enough to implement all normal playback functionality. + + def cmd_play(self, conn, index=-1): + new_index = index != -1 and index != self.current_index + was_paused = self.paused + super(Server, self).cmd_play(conn, index) + + if self.current_index > -1: # Not stopped. + if was_paused and not new_index: + # Just unpause. + self.player.play() + else: + self.player.play_file(self.playlist[self.current_index].path) + + def cmd_pause(self, conn, state=None): + super(Server, self).cmd_pause(conn, state) + if self.paused: + self.player.pause() + elif self.player.playing: + self.player.play() + + def cmd_stop(self, conn): + super(Server, self).cmd_stop(conn) + self.player.stop() + + def cmd_seek(self, conn, index, pos): + """Seeks to the specified position in the specified song.""" + index = cast_arg(int, index) + pos = cast_arg(int, pos) + super(Server, self).cmd_seek(conn, index, pos) + self.player.seek(pos) + + # Volume control. + + def cmd_setvol(self, conn, vol): + vol = cast_arg(int, vol) + super(Server, self).cmd_setvol(conn, vol) + self.player.volume = float(vol) / 100 + + +# Beets plugin hooks. + +class BPDPlugin(BeetsPlugin): + """Provides the "beet bpd" command for running a music player + server. + """ + def __init__(self): + super(BPDPlugin, self).__init__() + self.config.add({ + 'host': u'', + 'port': 6600, + 'password': u'', + 'volume': VOLUME_MAX, + }) + self.config['password'].redact = True + + def start_bpd(self, lib, host, port, password, volume, debug): + """Starts a BPD server.""" + if debug: # FIXME this should be managed by BeetsPlugin + self._log.setLevel(logging.DEBUG) + else: + self._log.setLevel(logging.WARNING) + try: + server = Server(lib, host, port, password) + server.cmd_setvol(None, volume) + server.run() + except NoGstreamerError: + global_log.error(u'Gstreamer Python bindings not found.') + global_log.error(u'Install "python-gst0.10", "py27-gst-python", ' + u'or similar package to use BPD.') + + def commands(self): + cmd = beets.ui.Subcommand( + 'bpd', help=u'run an MPD-compatible music player server' + ) + cmd.parser.add_option( + '-d', '--debug', action='store_true', + help=u'dump all MPD traffic to stdout' + ) + + def func(lib, opts, args): + host = args.pop(0) if args else self.config['host'].get(unicode) + port = args.pop(0) if args else self.config['port'].get(int) + if args: + raise beets.ui.UserError(u'too many arguments') + password = self.config['password'].get(unicode) + volume = self.config['volume'].get(int) + debug = opts.debug or False + self.start_bpd(lib, host, int(port), password, volume, debug) + + cmd.func = func + return [cmd] diff --git a/libs/beetsplug/bpd/gstplayer.py b/libs/beetsplug/bpd/gstplayer.py new file mode 100644 index 00000000..b64cd009 --- /dev/null +++ b/libs/beetsplug/bpd/gstplayer.py @@ -0,0 +1,223 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Adrian Sampson. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""A wrapper for the GStreamer Python bindings that exposes a simple +music player. +""" + +from __future__ import division, absolute_import, print_function + +import sys +import time +import gobject +import thread +import os +import copy +import urllib + +import pygst +pygst.require('0.10') +import gst # noqa + + +class GstPlayer(object): + """A music player abstracting GStreamer's Playbin element. + + Create a player object, then call run() to start a thread with a + runloop. Then call play_file to play music. Use player.playing + to check whether music is currently playing. + + A basic play queue is also implemented (just a Python list, + player.queue, whose last element is next to play). To use it, + just call enqueue() and then play(). When a track finishes and + another is available on the queue, it is played automatically. + """ + + def __init__(self, finished_callback=None): + """Initialize a player. + + If a finished_callback is provided, it is called every time a + track started with play_file finishes. + + Once the player has been created, call run() to begin the main + runloop in a separate thread. + """ + + # Set up the Gstreamer player. From the pygst tutorial: + # http://pygstdocs.berlios.de/pygst-tutorial/playbin.html + self.player = gst.element_factory_make("playbin2", "player") + fakesink = gst.element_factory_make("fakesink", "fakesink") + self.player.set_property("video-sink", fakesink) + bus = self.player.get_bus() + bus.add_signal_watch() + bus.connect("message", self._handle_message) + + # Set up our own stuff. + self.playing = False + self.finished_callback = finished_callback + self.cached_time = None + self._volume = 1.0 + + def _get_state(self): + """Returns the current state flag of the playbin.""" + # gst's get_state function returns a 3-tuple; we just want the + # status flag in position 1. + return self.player.get_state()[1] + + def _handle_message(self, bus, message): + """Callback for status updates from GStreamer.""" + if message.type == gst.MESSAGE_EOS: + # file finished playing + self.player.set_state(gst.STATE_NULL) + self.playing = False + self.cached_time = None + if self.finished_callback: + self.finished_callback() + + elif message.type == gst.MESSAGE_ERROR: + # error + self.player.set_state(gst.STATE_NULL) + err, debug = message.parse_error() + print(u"Error: {0}".format(err)) + self.playing = False + + def _set_volume(self, volume): + """Set the volume level to a value in the range [0, 1.5].""" + # And the volume for the playbin. + self._volume = volume + self.player.set_property("volume", volume) + + def _get_volume(self): + """Get the volume as a float in the range [0, 1.5].""" + return self._volume + + volume = property(_get_volume, _set_volume) + + def play_file(self, path): + """Immediately begin playing the audio file at the given + path. + """ + self.player.set_state(gst.STATE_NULL) + if isinstance(path, unicode): + path = path.encode('utf8') + uri = 'file://' + urllib.quote(path) + self.player.set_property("uri", uri) + self.player.set_state(gst.STATE_PLAYING) + self.playing = True + + def play(self): + """If paused, resume playback.""" + if self._get_state() == gst.STATE_PAUSED: + self.player.set_state(gst.STATE_PLAYING) + self.playing = True + + def pause(self): + """Pause playback.""" + self.player.set_state(gst.STATE_PAUSED) + + def stop(self): + """Halt playback.""" + self.player.set_state(gst.STATE_NULL) + self.playing = False + self.cached_time = None + + def run(self): + """Start a new thread for the player. + + Call this function before trying to play any music with + play_file() or play(). + """ + # If we don't use the MainLoop, messages are never sent. + gobject.threads_init() + + def start(): + loop = gobject.MainLoop() + loop.run() + thread.start_new_thread(start, ()) + + def time(self): + """Returns a tuple containing (position, length) where both + values are integers in seconds. If no stream is available, + returns (0, 0). + """ + fmt = gst.Format(gst.FORMAT_TIME) + try: + pos = self.player.query_position(fmt, None)[0] / (10 ** 9) + length = self.player.query_duration(fmt, None)[0] / (10 ** 9) + self.cached_time = (pos, length) + return (pos, length) + + except gst.QueryError: + # Stream not ready. For small gaps of time, for instance + # after seeking, the time values are unavailable. For this + # reason, we cache recent. + if self.playing and self.cached_time: + return self.cached_time + else: + return (0, 0) + + def seek(self, position): + """Seeks to position (in seconds).""" + cur_pos, cur_len = self.time() + if position > cur_len: + self.stop() + return + + fmt = gst.Format(gst.FORMAT_TIME) + ns = position * 10 ** 9 # convert to nanoseconds + self.player.seek_simple(fmt, gst.SEEK_FLAG_FLUSH, ns) + + # save new cached time + self.cached_time = (position, cur_len) + + def block(self): + """Block until playing finishes.""" + while self.playing: + time.sleep(1) + + +def play_simple(paths): + """Play the files in paths in a straightforward way, without + using the player's callback function. + """ + p = GstPlayer() + p.run() + for path in paths: + p.play_file(path) + p.block() + + +def play_complicated(paths): + """Play the files in the path one after the other by using the + callback function to advance to the next song. + """ + my_paths = copy.copy(paths) + + def next_song(): + my_paths.pop(0) + p.play_file(my_paths[0]) + p = GstPlayer(next_song) + p.run() + p.play_file(my_paths[0]) + while my_paths: + time.sleep(1) + +if __name__ == '__main__': + # A very simple command-line player. Just give it names of audio + # files on the command line; these are all played in sequence. + paths = [os.path.abspath(os.path.expanduser(p)) + for p in sys.argv[1:]] + # play_simple(paths) + play_complicated(paths) diff --git a/libs/beetsplug/bpm.py b/libs/beetsplug/bpm.py new file mode 100644 index 00000000..ba284c04 --- /dev/null +++ b/libs/beetsplug/bpm.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, aroquen +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Determine BPM by pressing a key to the rhythm.""" + +from __future__ import division, absolute_import, print_function + +import time + +from beets import ui +from beets.plugins import BeetsPlugin + + +def bpm(max_strokes): + """Returns average BPM (possibly of a playing song) + listening to Enter keystrokes. + """ + t0 = None + dt = [] + for i in range(max_strokes): + # Press enter to the rhythm... + s = raw_input() + if s == '': + t1 = time.time() + # Only start measuring at the second stroke + if t0: + dt.append(t1 - t0) + t0 = t1 + else: + break + + # Return average BPM + # bpm = (max_strokes-1) / sum(dt) * 60 + ave = sum([1.0 / dti * 60 for dti in dt]) / len(dt) + return ave + + +class BPMPlugin(BeetsPlugin): + + def __init__(self): + super(BPMPlugin, self).__init__() + self.config.add({ + u'max_strokes': 3, + u'overwrite': True, + }) + + def commands(self): + cmd = ui.Subcommand('bpm', + help=u'determine bpm of a song by pressing ' + u'a key to the rhythm') + cmd.func = self.command + return [cmd] + + def command(self, lib, opts, args): + self.get_bpm(lib.items(ui.decargs(args))) + + def get_bpm(self, items, write=False): + overwrite = self.config['overwrite'].get(bool) + if len(items) > 1: + raise ValueError(u'Can only get bpm of one song at time') + + item = items[0] + if item['bpm']: + self._log.info(u'Found bpm {0}', item['bpm']) + if not overwrite: + return + + self._log.info(u'Press Enter {0} times to the rhythm or Ctrl-D ' + u'to exit', self.config['max_strokes'].get(int)) + new_bpm = bpm(self.config['max_strokes'].get(int)) + item['bpm'] = int(new_bpm) + if write: + item.try_write() + item.store() + self._log.info(u'Added new bpm {0}', item['bpm']) diff --git a/libs/beetsplug/bucket.py b/libs/beetsplug/bucket.py new file mode 100644 index 00000000..21acb1f1 --- /dev/null +++ b/libs/beetsplug/bucket.py @@ -0,0 +1,243 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Fabrice Laporte. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Provides the %bucket{} function for path formatting. +""" + +from __future__ import division, absolute_import, print_function + +from datetime import datetime +import re +import string +from itertools import tee, izip + +from beets import plugins, ui + + +ASCII_DIGITS = string.digits + string.ascii_lowercase + + +class BucketError(Exception): + pass + + +def pairwise(iterable): + "s -> (s0,s1), (s1,s2), (s2, s3), ..." + a, b = tee(iterable) + next(b, None) + return izip(a, b) + + +def span_from_str(span_str): + """Build a span dict from the span string representation. + """ + + def normalize_year(d, yearfrom): + """Convert string to a 4 digits year + """ + if yearfrom < 100: + raise BucketError(u"%d must be expressed on 4 digits" % yearfrom) + + # if two digits only, pick closest year that ends by these two + # digits starting from yearfrom + if d < 100: + if (d % 100) < (yearfrom % 100): + d = (yearfrom - yearfrom % 100) + 100 + d + else: + d = (yearfrom - yearfrom % 100) + d + return d + + years = [int(x) for x in re.findall('\d+', span_str)] + if not years: + raise ui.UserError(u"invalid range defined for year bucket '%s': no " + u"year found" % span_str) + try: + years = [normalize_year(x, years[0]) for x in years] + except BucketError as exc: + raise ui.UserError(u"invalid range defined for year bucket '%s': %s" % + (span_str, exc)) + + res = {'from': years[0], 'str': span_str} + if len(years) > 1: + res['to'] = years[-1] + return res + + +def complete_year_spans(spans): + """Set the `to` value of spans if empty and sort them chronologically. + """ + spans.sort(key=lambda x: x['from']) + for (x, y) in pairwise(spans): + if 'to' not in x: + x['to'] = y['from'] - 1 + if spans and 'to' not in spans[-1]: + spans[-1]['to'] = datetime.now().year + + +def extend_year_spans(spans, spanlen, start=1900, end=2014): + """Add new spans to given spans list so that every year of [start,end] + belongs to a span. + """ + extended_spans = spans[:] + for (x, y) in pairwise(spans): + # if a gap between two spans, fill the gap with as much spans of + # spanlen length as necessary + for span_from in range(x['to'] + 1, y['from'], spanlen): + extended_spans.append({'from': span_from}) + # Create spans prior to declared ones + for span_from in range(spans[0]['from'] - spanlen, start, -spanlen): + extended_spans.append({'from': span_from}) + # Create spans after the declared ones + for span_from in range(spans[-1]['to'] + 1, end, spanlen): + extended_spans.append({'from': span_from}) + + complete_year_spans(extended_spans) + return extended_spans + + +def build_year_spans(year_spans_str): + """Build a chronologically ordered list of spans dict from unordered spans + stringlist. + """ + spans = [] + for elem in year_spans_str: + spans.append(span_from_str(elem)) + complete_year_spans(spans) + return spans + + +def str2fmt(s): + """Deduces formatting syntax from a span string. + """ + regex = re.compile(r"(?P\D*)(?P\d+)(?P\D*)" + r"(?P\d*)(?P\D*)") + m = re.match(regex, s) + + res = {'fromnchars': len(m.group('fromyear')), + 'tonchars': len(m.group('toyear'))} + res['fmt'] = "%s%%s%s%s%s" % (m.group('bef'), + m.group('sep'), + '%s' if res['tonchars'] else '', + m.group('after')) + return res + + +def format_span(fmt, yearfrom, yearto, fromnchars, tonchars): + """Return a span string representation. + """ + args = (bytes(yearfrom)[-fromnchars:]) + if tonchars: + args = (bytes(yearfrom)[-fromnchars:], bytes(yearto)[-tonchars:]) + return fmt % args + + +def extract_modes(spans): + """Extract the most common spans lengths and representation formats + """ + rangelen = sorted([x['to'] - x['from'] + 1 for x in spans]) + deflen = sorted(rangelen, key=rangelen.count)[-1] + reprs = [str2fmt(x['str']) for x in spans] + deffmt = sorted(reprs, key=reprs.count)[-1] + return deflen, deffmt + + +def build_alpha_spans(alpha_spans_str, alpha_regexs): + """Extract alphanumerics from string and return sorted list of chars + [from...to] + """ + spans = [] + + for elem in alpha_spans_str: + if elem in alpha_regexs: + spans.append(re.compile(alpha_regexs[elem])) + else: + bucket = sorted([x for x in elem.lower() if x.isalnum()]) + if bucket: + begin_index = ASCII_DIGITS.index(bucket[0]) + end_index = ASCII_DIGITS.index(bucket[-1]) + else: + raise ui.UserError(u"invalid range defined for alpha bucket " + u"'%s': no alphanumeric character found" % + elem) + spans.append( + re.compile( + "^[" + ASCII_DIGITS[begin_index:end_index + 1] + + ASCII_DIGITS[begin_index:end_index + 1].upper() + "]" + ) + ) + return spans + + +class BucketPlugin(plugins.BeetsPlugin): + def __init__(self): + super(BucketPlugin, self).__init__() + self.template_funcs['bucket'] = self._tmpl_bucket + + self.config.add({ + 'bucket_year': [], + 'bucket_alpha': [], + 'bucket_alpha_regex': {}, + 'extrapolate': False + }) + self.setup() + + def setup(self): + """Setup plugin from config options + """ + self.year_spans = build_year_spans(self.config['bucket_year'].get()) + if self.year_spans and self.config['extrapolate']: + [self.ys_len_mode, + self.ys_repr_mode] = extract_modes(self.year_spans) + self.year_spans = extend_year_spans(self.year_spans, + self.ys_len_mode) + + self.alpha_spans = build_alpha_spans( + self.config['bucket_alpha'].get(), + self.config['bucket_alpha_regex'].get() + ) + + def find_bucket_year(self, year): + """Return bucket that matches given year or return the year + if no matching bucket. + """ + for ys in self.year_spans: + if ys['from'] <= int(year) <= ys['to']: + if 'str' in ys: + return ys['str'] + else: + return format_span(self.ys_repr_mode['fmt'], + ys['from'], ys['to'], + self.ys_repr_mode['fromnchars'], + self.ys_repr_mode['tonchars']) + return year + + def find_bucket_alpha(self, s): + """Return alpha-range bucket that matches given string or return the + string initial if no matching bucket. + """ + for (i, span) in enumerate(self.alpha_spans): + if span.match(s): + return self.config['bucket_alpha'].get()[i] + return s[0].upper() + + def _tmpl_bucket(self, text, field=None): + if not field and len(text) == 4 and text.isdigit(): + field = 'year' + + if field == 'year': + func = self.find_bucket_year + else: + func = self.find_bucket_alpha + return func(text) diff --git a/libs/beetsplug/chroma.py b/libs/beetsplug/chroma.py new file mode 100644 index 00000000..148e9c20 --- /dev/null +++ b/libs/beetsplug/chroma.py @@ -0,0 +1,308 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Adrian Sampson. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Adds Chromaprint/Acoustid acoustic fingerprinting support to the +autotagger. Requires the pyacoustid library. +""" +from __future__ import division, absolute_import, print_function + +from beets import plugins +from beets import ui +from beets import util +from beets import config +from beets.util import confit +from beets.autotag import hooks +import acoustid +from collections import defaultdict + +API_KEY = '1vOwZtEn' +SCORE_THRESH = 0.5 +TRACK_ID_WEIGHT = 10.0 +COMMON_REL_THRESH = 0.6 # How many tracks must have an album in common? +MAX_RECORDINGS = 5 +MAX_RELEASES = 5 + +# Stores the Acoustid match information for each track. This is +# populated when an import task begins and then used when searching for +# candidates. It maps audio file paths to (recording_ids, release_ids) +# pairs. If a given path is not present in the mapping, then no match +# was found. +_matches = {} + +# Stores the fingerprint and Acoustid ID for each track. This is stored +# as metadata for each track for later use but is not relevant for +# autotagging. +_fingerprints = {} +_acoustids = {} + + +def prefix(it, count): + """Truncate an iterable to at most `count` items. + """ + for i, v in enumerate(it): + if i >= count: + break + yield v + + +def acoustid_match(log, path): + """Gets metadata for a file from Acoustid and populates the + _matches, _fingerprints, and _acoustids dictionaries accordingly. + """ + try: + duration, fp = acoustid.fingerprint_file(util.syspath(path)) + except acoustid.FingerprintGenerationError as exc: + log.error(u'fingerprinting of {0} failed: {1}', + util.displayable_path(repr(path)), exc) + return None + _fingerprints[path] = fp + try: + res = acoustid.lookup(API_KEY, fp, duration, + meta='recordings releases') + except acoustid.AcoustidError as exc: + log.debug(u'fingerprint matching {0} failed: {1}', + util.displayable_path(repr(path)), exc) + return None + log.debug(u'chroma: fingerprinted {0}', + util.displayable_path(repr(path))) + + # Ensure the response is usable and parse it. + if res['status'] != 'ok' or not res.get('results'): + log.debug(u'no match found') + return None + result = res['results'][0] # Best match. + if result['score'] < SCORE_THRESH: + log.debug(u'no results above threshold') + return None + _acoustids[path] = result['id'] + + # Get recording and releases from the result. + if not result.get('recordings'): + log.debug(u'no recordings found') + return None + recording_ids = [] + release_ids = [] + for recording in result['recordings']: + recording_ids.append(recording['id']) + if 'releases' in recording: + release_ids += [rel['id'] for rel in recording['releases']] + + log.debug(u'matched recordings {0} on releases {1}', + recording_ids, release_ids) + _matches[path] = recording_ids, release_ids + + +# Plugin structure and autotagging logic. + + +def _all_releases(items): + """Given an iterable of Items, determines (according to Acoustid) + which releases the items have in common. Generates release IDs. + """ + # Count the number of "hits" for each release. + relcounts = defaultdict(int) + for item in items: + if item.path not in _matches: + continue + + _, release_ids = _matches[item.path] + for release_id in release_ids: + relcounts[release_id] += 1 + + for release_id, count in relcounts.iteritems(): + if float(count) / len(items) > COMMON_REL_THRESH: + yield release_id + + +class AcoustidPlugin(plugins.BeetsPlugin): + def __init__(self): + super(AcoustidPlugin, self).__init__() + + self.config.add({ + 'auto': True, + }) + config['acoustid']['apikey'].redact = True + + if self.config['auto']: + self.register_listener('import_task_start', self.fingerprint_task) + self.register_listener('import_task_apply', apply_acoustid_metadata) + + def fingerprint_task(self, task, session): + return fingerprint_task(self._log, task, session) + + def track_distance(self, item, info): + dist = hooks.Distance() + if item.path not in _matches or not info.track_id: + # Match failed or no track ID. + return dist + + recording_ids, _ = _matches[item.path] + dist.add_expr('track_id', info.track_id not in recording_ids) + return dist + + def candidates(self, items, artist, album, va_likely): + albums = [] + for relid in prefix(_all_releases(items), MAX_RELEASES): + album = hooks.album_for_mbid(relid) + if album: + albums.append(album) + + self._log.debug(u'acoustid album candidates: {0}', len(albums)) + return albums + + def item_candidates(self, item, artist, title): + if item.path not in _matches: + return [] + + recording_ids, _ = _matches[item.path] + tracks = [] + for recording_id in prefix(recording_ids, MAX_RECORDINGS): + track = hooks.track_for_mbid(recording_id) + if track: + tracks.append(track) + self._log.debug(u'acoustid item candidates: {0}', len(tracks)) + return tracks + + def commands(self): + submit_cmd = ui.Subcommand('submit', + help=u'submit Acoustid fingerprints') + + def submit_cmd_func(lib, opts, args): + try: + apikey = config['acoustid']['apikey'].get(unicode) + except confit.NotFoundError: + raise ui.UserError(u'no Acoustid user API key provided') + submit_items(self._log, apikey, lib.items(ui.decargs(args))) + submit_cmd.func = submit_cmd_func + + fingerprint_cmd = ui.Subcommand( + 'fingerprint', + help=u'generate fingerprints for items without them' + ) + + def fingerprint_cmd_func(lib, opts, args): + for item in lib.items(ui.decargs(args)): + fingerprint_item(self._log, item, write=ui.should_write()) + fingerprint_cmd.func = fingerprint_cmd_func + + return [submit_cmd, fingerprint_cmd] + + +# Hooks into import process. + + +def fingerprint_task(log, task, session): + """Fingerprint each item in the task for later use during the + autotagging candidate search. + """ + items = task.items if task.is_album else [task.item] + for item in items: + acoustid_match(log, item.path) + + +def apply_acoustid_metadata(task, session): + """Apply Acoustid metadata (fingerprint and ID) to the task's items. + """ + for item in task.imported_items(): + if item.path in _fingerprints: + item.acoustid_fingerprint = _fingerprints[item.path] + if item.path in _acoustids: + item.acoustid_id = _acoustids[item.path] + + +# UI commands. + + +def submit_items(log, userkey, items, chunksize=64): + """Submit fingerprints for the items to the Acoustid server. + """ + data = [] # The running list of dictionaries to submit. + + def submit_chunk(): + """Submit the current accumulated fingerprint data.""" + log.info(u'submitting {0} fingerprints', len(data)) + try: + acoustid.submit(API_KEY, userkey, data) + except acoustid.AcoustidError as exc: + log.warn(u'acoustid submission error: {0}', exc) + del data[:] + + for item in items: + fp = fingerprint_item(log, item) + + # Construct a submission dictionary for this item. + item_data = { + 'duration': int(item.length), + 'fingerprint': fp, + } + if item.mb_trackid: + item_data['mbid'] = item.mb_trackid + log.debug(u'submitting MBID') + else: + item_data.update({ + 'track': item.title, + 'artist': item.artist, + 'album': item.album, + 'albumartist': item.albumartist, + 'year': item.year, + 'trackno': item.track, + 'discno': item.disc, + }) + log.debug(u'submitting textual metadata') + data.append(item_data) + + # If we have enough data, submit a chunk. + if len(data) >= chunksize: + submit_chunk() + + # Submit remaining data in a final chunk. + if data: + submit_chunk() + + +def fingerprint_item(log, item, write=False): + """Get the fingerprint for an Item. If the item already has a + fingerprint, it is not regenerated. If fingerprint generation fails, + return None. If the items are associated with a library, they are + saved to the database. If `write` is set, then the new fingerprints + are also written to files' metadata. + """ + # Get a fingerprint and length for this track. + if not item.length: + log.info(u'{0}: no duration available', + util.displayable_path(item.path)) + elif item.acoustid_fingerprint: + if write: + log.info(u'{0}: fingerprint exists, skipping', + util.displayable_path(item.path)) + else: + log.info(u'{0}: using existing fingerprint', + util.displayable_path(item.path)) + return item.acoustid_fingerprint + else: + log.info(u'{0}: fingerprinting', + util.displayable_path(item.path)) + try: + _, fp = acoustid.fingerprint_file(item.path) + item.acoustid_fingerprint = fp + if write: + log.info(u'{0}: writing fingerprint', + util.displayable_path(item.path)) + item.try_write() + if item._db: + item.store() + return item.acoustid_fingerprint + except acoustid.FingerprintGenerationError as exc: + log.info(u'fingerprint generation failed: {0}', exc) diff --git a/libs/beetsplug/convert.py b/libs/beetsplug/convert.py new file mode 100644 index 00000000..de91604f --- /dev/null +++ b/libs/beetsplug/convert.py @@ -0,0 +1,449 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Jakob Schnitzer. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Converts tracks or albums to external directory +""" +from __future__ import division, absolute_import, print_function + +import os +import threading +import subprocess +import tempfile +import shlex +from string import Template + +from beets import ui, util, plugins, config +from beets.plugins import BeetsPlugin +from beets.util.confit import ConfigTypeError +from beets import art +from beets.util.artresizer import ArtResizer + +_fs_lock = threading.Lock() +_temp_files = [] # Keep track of temporary transcoded files for deletion. + +# Some convenient alternate names for formats. +ALIASES = { + u'wma': u'windows media', + u'vorbis': u'ogg', +} + +LOSSLESS_FORMATS = ['ape', 'flac', 'alac', 'wav', 'aiff'] + + +def replace_ext(path, ext): + """Return the path with its extension replaced by `ext`. + + The new extension must not contain a leading dot. + """ + return os.path.splitext(path)[0] + b'.' + ext + + +def get_format(fmt=None): + """Return the command template and the extension from the config. + """ + if not fmt: + fmt = config['convert']['format'].get(unicode).lower() + fmt = ALIASES.get(fmt, fmt) + + try: + format_info = config['convert']['formats'][fmt].get(dict) + command = format_info['command'] + extension = format_info.get('extension', fmt) + except KeyError: + raise ui.UserError( + u'convert: format {0} needs the "command" field' + .format(fmt) + ) + except ConfigTypeError: + command = config['convert']['formats'][fmt].get(bytes) + extension = fmt + + # Convenience and backwards-compatibility shortcuts. + keys = config['convert'].keys() + if 'command' in keys: + command = config['convert']['command'].get(unicode) + elif 'opts' in keys: + # Undocumented option for backwards compatibility with < 1.3.1. + command = u'ffmpeg -i $source -y {0} $dest'.format( + config['convert']['opts'].get(unicode) + ) + if 'extension' in keys: + extension = config['convert']['extension'].get(unicode) + + return (command.encode('utf8'), extension.encode('utf8')) + + +def should_transcode(item, fmt): + """Determine whether the item should be transcoded as part of + conversion (i.e., its bitrate is high or it has the wrong format). + """ + if config['convert']['never_convert_lossy_files'] and \ + not (item.format.lower() in LOSSLESS_FORMATS): + return False + maxbr = config['convert']['max_bitrate'].get(int) + return fmt.lower() != item.format.lower() or \ + item.bitrate >= 1000 * maxbr + + +class ConvertPlugin(BeetsPlugin): + def __init__(self): + super(ConvertPlugin, self).__init__() + self.config.add({ + u'dest': None, + u'pretend': False, + u'threads': util.cpu_count(), + u'format': u'mp3', + u'formats': { + u'aac': { + u'command': u'ffmpeg -i $source -y -vn -acodec libfaac ' + u'-aq 100 $dest', + u'extension': u'm4a', + }, + u'alac': { + u'command': u'ffmpeg -i $source -y -vn -acodec alac $dest', + u'extension': u'm4a', + }, + u'flac': u'ffmpeg -i $source -y -vn -acodec flac $dest', + u'mp3': u'ffmpeg -i $source -y -vn -aq 2 $dest', + u'opus': + u'ffmpeg -i $source -y -vn -acodec libopus -ab 96k $dest', + u'ogg': + u'ffmpeg -i $source -y -vn -acodec libvorbis -aq 3 $dest', + u'wma': + u'ffmpeg -i $source -y -vn -acodec wmav2 -vn $dest', + }, + u'max_bitrate': 500, + u'auto': False, + u'tmpdir': None, + u'quiet': False, + u'embed': True, + u'paths': {}, + u'never_convert_lossy_files': False, + u'copy_album_art': False, + u'album_art_maxwidth': 0, + }) + self.import_stages = [self.auto_convert] + + self.register_listener('import_task_files', self._cleanup) + + def commands(self): + cmd = ui.Subcommand('convert', help=u'convert to external location') + cmd.parser.add_option('-p', '--pretend', action='store_true', + help=u'show actions but do nothing') + cmd.parser.add_option('-t', '--threads', action='store', type='int', + help=u'change the number of threads, \ + defaults to maximum available processors') + cmd.parser.add_option('-k', '--keep-new', action='store_true', + dest='keep_new', help=u'keep only the converted \ + and move the old files') + cmd.parser.add_option('-d', '--dest', action='store', + help=u'set the destination directory') + cmd.parser.add_option('-f', '--format', action='store', dest='format', + help=u'set the target format of the tracks') + cmd.parser.add_option('-y', '--yes', action='store_true', dest='yes', + help=u'do not ask for confirmation') + cmd.parser.add_album_option() + cmd.func = self.convert_func + return [cmd] + + def auto_convert(self, config, task): + if self.config['auto']: + for item in task.imported_items(): + self.convert_on_import(config.lib, item) + + # Utilities converted from functions to methods on logging overhaul + + def encode(self, command, source, dest, pretend=False): + """Encode `source` to `dest` using command template `command`. + + Raises `subprocess.CalledProcessError` if the command exited with a + non-zero status code. + """ + # The paths and arguments must be bytes. + assert isinstance(command, bytes) + assert isinstance(source, bytes) + assert isinstance(dest, bytes) + + quiet = self.config['quiet'].get(bool) + + if not quiet and not pretend: + self._log.info(u'Encoding {0}', util.displayable_path(source)) + + # Substitute $source and $dest in the argument list. + args = shlex.split(command) + for i, arg in enumerate(args): + args[i] = Template(arg).safe_substitute({ + 'source': source, + 'dest': dest, + }) + + if pretend: + self._log.info(u' '.join(ui.decargs(args))) + return + + try: + util.command_output(args) + except subprocess.CalledProcessError as exc: + # Something went wrong (probably Ctrl+C), remove temporary files + self._log.info(u'Encoding {0} failed. Cleaning up...', + util.displayable_path(source)) + self._log.debug(u'Command {0} exited with status {1}', + exc.cmd.decode('utf8', 'ignore'), + exc.returncode) + util.remove(dest) + util.prune_dirs(os.path.dirname(dest)) + raise + except OSError as exc: + raise ui.UserError( + u"convert: couldn't invoke '{0}': {1}".format( + u' '.join(ui.decargs(args)), exc + ) + ) + + if not quiet and not pretend: + self._log.info(u'Finished encoding {0}', + util.displayable_path(source)) + + def convert_item(self, dest_dir, keep_new, path_formats, fmt, + pretend=False): + command, ext = get_format(fmt) + item, original, converted = None, None, None + while True: + item = yield (item, original, converted) + dest = item.destination(basedir=dest_dir, + path_formats=path_formats) + + # When keeping the new file in the library, we first move the + # current (pristine) file to the destination. We'll then copy it + # back to its old path or transcode it to a new path. + if keep_new: + original = dest + converted = item.path + if should_transcode(item, fmt): + converted = replace_ext(converted, ext) + else: + original = item.path + if should_transcode(item, fmt): + dest = replace_ext(dest, ext) + converted = dest + + # Ensure that only one thread tries to create directories at a + # time. (The existence check is not atomic with the directory + # creation inside this function.) + if not pretend: + with _fs_lock: + util.mkdirall(dest) + + if os.path.exists(util.syspath(dest)): + self._log.info(u'Skipping {0} (target file exists)', + util.displayable_path(item.path)) + continue + + if keep_new: + if pretend: + self._log.info(u'mv {0} {1}', + util.displayable_path(item.path), + util.displayable_path(original)) + else: + self._log.info(u'Moving to {0}', + util.displayable_path(original)) + util.move(item.path, original) + + if should_transcode(item, fmt): + try: + self.encode(command, original, converted, pretend) + except subprocess.CalledProcessError: + continue + else: + if pretend: + self._log.info(u'cp {0} {1}', + util.displayable_path(original), + util.displayable_path(converted)) + else: + # No transcoding necessary. + self._log.info(u'Copying {0}', + util.displayable_path(item.path)) + util.copy(original, converted) + + if pretend: + continue + + # Write tags from the database to the converted file. + item.try_write(path=converted) + + if keep_new: + # If we're keeping the transcoded file, read it again (after + # writing) to get new bitrate, duration, etc. + item.path = converted + item.read() + item.store() # Store new path and audio data. + + if self.config['embed']: + album = item.get_album() + if album and album.artpath: + self._log.debug(u'embedding album art from {}', + util.displayable_path(album.artpath)) + art.embed_item(self._log, item, album.artpath, + itempath=converted) + + if keep_new: + plugins.send('after_convert', item=item, + dest=dest, keepnew=True) + else: + plugins.send('after_convert', item=item, + dest=converted, keepnew=False) + + def copy_album_art(self, album, dest_dir, path_formats, pretend=False): + """Copies or converts the associated cover art of the album. Album must + have at least one track. + """ + if not album or not album.artpath: + return + + album_item = album.items().get() + # Album shouldn't be empty. + if not album_item: + return + + # Get the destination of the first item (track) of the album, we use + # this function to format the path accordingly to path_formats. + dest = album_item.destination(basedir=dest_dir, + path_formats=path_formats) + + # Remove item from the path. + dest = os.path.join(*util.components(dest)[:-1]) + + dest = album.art_destination(album.artpath, item_dir=dest) + if album.artpath == dest: + return + + if not pretend: + util.mkdirall(dest) + + if os.path.exists(util.syspath(dest)): + self._log.info(u'Skipping {0} (target file exists)', + util.displayable_path(album.artpath)) + return + + # Decide whether we need to resize the cover-art image. + resize = False + maxwidth = None + if self.config['album_art_maxwidth']: + maxwidth = self.config['album_art_maxwidth'].get(int) + size = ArtResizer.shared.get_size(album.artpath) + self._log.debug('image size: {}', size) + if size: + resize = size[0] > maxwidth + else: + self._log.warning(u'Could not get size of image (please see ' + u'documentation for dependencies).') + + # Either copy or resize (while copying) the image. + if resize: + self._log.info(u'Resizing cover art from {0} to {1}', + util.displayable_path(album.artpath), + util.displayable_path(dest)) + if not pretend: + ArtResizer.shared.resize(maxwidth, album.artpath, dest) + else: + if pretend: + self._log.info(u'cp {0} {1}', + util.displayable_path(album.artpath), + util.displayable_path(dest)) + else: + self._log.info(u'Copying cover art to {0}', + util.displayable_path(album.artpath), + util.displayable_path(dest)) + util.copy(album.artpath, dest) + + def convert_func(self, lib, opts, args): + if not opts.dest: + opts.dest = self.config['dest'].get() + if not opts.dest: + raise ui.UserError(u'no convert destination set') + opts.dest = util.bytestring_path(opts.dest) + + if not opts.threads: + opts.threads = self.config['threads'].get(int) + + if self.config['paths']: + path_formats = ui.get_path_formats(self.config['paths']) + else: + path_formats = ui.get_path_formats() + + if not opts.format: + opts.format = self.config['format'].get(unicode).lower() + + pretend = opts.pretend if opts.pretend is not None else \ + self.config['pretend'].get(bool) + + if not pretend: + ui.commands.list_items(lib, ui.decargs(args), opts.album) + + if not (opts.yes or ui.input_yn(u"Convert? (Y/n)")): + return + + if opts.album: + albums = lib.albums(ui.decargs(args)) + items = (i for a in albums for i in a.items()) + if self.config['copy_album_art']: + for album in albums: + self.copy_album_art(album, opts.dest, path_formats, + pretend) + else: + items = iter(lib.items(ui.decargs(args))) + convert = [self.convert_item(opts.dest, + opts.keep_new, + path_formats, + opts.format, + pretend) + for _ in range(opts.threads)] + pipe = util.pipeline.Pipeline([items, convert]) + pipe.run_parallel() + + def convert_on_import(self, lib, item): + """Transcode a file automatically after it is imported into the + library. + """ + fmt = self.config['format'].get(unicode).lower() + if should_transcode(item, fmt): + command, ext = get_format() + + # Create a temporary file for the conversion. + tmpdir = self.config['tmpdir'].get() + fd, dest = tempfile.mkstemp('.' + ext, dir=tmpdir) + os.close(fd) + dest = util.bytestring_path(dest) + _temp_files.append(dest) # Delete the transcode later. + + # Convert. + try: + self.encode(command, item.path, dest) + except subprocess.CalledProcessError: + return + + # Change the newly-imported database entry to point to the + # converted file. + item.path = dest + item.write() + item.read() # Load new audio information data. + item.store() + + def _cleanup(self, task, session): + for path in task.old_paths: + if path in _temp_files: + if os.path.isfile(path): + util.remove(path) + _temp_files.remove(path) diff --git a/libs/beetsplug/cue.py b/libs/beetsplug/cue.py new file mode 100644 index 00000000..63051bfc --- /dev/null +++ b/libs/beetsplug/cue.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 Bruno Cauet +# Split an album-file in tracks thanks a cue file + +from __future__ import division, absolute_import, print_function + +import subprocess +from os import path +from glob import glob + +from beets.util import command_output, displayable_path +from beets.plugins import BeetsPlugin +from beets.autotag import TrackInfo + + +class CuePlugin(BeetsPlugin): + def __init__(self): + super(CuePlugin, self).__init__() + # this does not seem supported by shnsplit + self.config.add({ + 'keep_before': .1, + 'keep_after': .9, + }) + + # self.register_listener('import_task_start', self.look_for_cues) + + def candidates(self, items, artist, album, va_likely): + import pdb + pdb.set_trace() + + def item_candidates(self, item, artist, album): + dir = path.dirname(item.path) + cues = glob.glob(path.join(dir, "*.cue")) + if not cues: + return + if len(cues) > 1: + self._log.info(u"Found multiple cue files doing nothing: {0}", + map(displayable_path, cues)) + + cue_file = cues[0] + self._log.info("Found {} for {}", displayable_path(cue_file), item) + + try: + # careful: will ask for input in case of conflicts + command_output(['shnsplit', '-f', cue_file, item.path]) + except (subprocess.CalledProcessError, OSError): + self._log.exception(u'shnsplit execution failed') + return + + tracks = glob(path.join(dir, "*.wav")) + self._log.info("Generated {0} tracks", len(tracks)) + for t in tracks: + title = "dunno lol" + track_id = "wtf" + index = int(path.basename(t)[len("split-track"):-len(".wav")]) + yield TrackInfo(title, track_id, index=index, artist=artist) + # generate TrackInfo instances diff --git a/libs/beetsplug/discogs.py b/libs/beetsplug/discogs.py new file mode 100644 index 00000000..62a78a5f --- /dev/null +++ b/libs/beetsplug/discogs.py @@ -0,0 +1,350 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Adrian Sampson. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Adds Discogs album search support to the autotagger. Requires the +discogs-client library. +""" +from __future__ import division, absolute_import, print_function + +import beets.ui +from beets import logging +from beets import config +from beets.autotag.hooks import AlbumInfo, TrackInfo, Distance +from beets.plugins import BeetsPlugin +from beets.util import confit +from discogs_client import Release, Client +from discogs_client.exceptions import DiscogsAPIError +from requests.exceptions import ConnectionError +import beets +import re +import time +import json +import socket +import httplib +import os + + +# Silence spurious INFO log lines generated by urllib3. +urllib3_logger = logging.getLogger('requests.packages.urllib3') +urllib3_logger.setLevel(logging.CRITICAL) + +USER_AGENT = u'beets/{0} +http://beets.io/'.format(beets.__version__) + +# Exceptions that discogs_client should really handle but does not. +CONNECTION_ERRORS = (ConnectionError, socket.error, httplib.HTTPException, + ValueError, # JSON decoding raises a ValueError. + DiscogsAPIError) + + +class DiscogsPlugin(BeetsPlugin): + + def __init__(self): + super(DiscogsPlugin, self).__init__() + self.config.add({ + 'apikey': 'rAzVUQYRaoFjeBjyWuWZ', + 'apisecret': 'plxtUTqoCzwxZpqdPysCwGuBSmZNdZVy', + 'tokenfile': 'discogs_token.json', + 'source_weight': 0.5, + }) + self.config['apikey'].redact = True + self.config['apisecret'].redact = True + self.discogs_client = None + self.register_listener('import_begin', self.setup) + + def setup(self, session=None): + """Create the `discogs_client` field. Authenticate if necessary. + """ + c_key = self.config['apikey'].get(unicode) + c_secret = self.config['apisecret'].get(unicode) + + # Get the OAuth token from a file or log in. + try: + with open(self._tokenfile()) as f: + tokendata = json.load(f) + except IOError: + # No token yet. Generate one. + token, secret = self.authenticate(c_key, c_secret) + else: + token = tokendata['token'] + secret = tokendata['secret'] + + self.discogs_client = Client(USER_AGENT, c_key, c_secret, + token, secret) + + def reset_auth(self): + """Delete toke file & redo the auth steps. + """ + os.remove(self._tokenfile()) + self.setup() + + def _tokenfile(self): + """Get the path to the JSON file for storing the OAuth token. + """ + return self.config['tokenfile'].get(confit.Filename(in_app_dir=True)) + + def authenticate(self, c_key, c_secret): + # Get the link for the OAuth page. + auth_client = Client(USER_AGENT, c_key, c_secret) + try: + _, _, url = auth_client.get_authorize_url() + except CONNECTION_ERRORS as e: + self._log.debug(u'connection error: {0}', e) + raise beets.ui.UserError(u'communication with Discogs failed') + + beets.ui.print_(u"To authenticate with Discogs, visit:") + beets.ui.print_(url) + + # Ask for the code and validate it. + code = beets.ui.input_(u"Enter the code:") + try: + token, secret = auth_client.get_access_token(code) + except DiscogsAPIError: + raise beets.ui.UserError(u'Discogs authorization failed') + except CONNECTION_ERRORS as e: + self._log.debug(u'connection error: {0}', e) + raise beets.ui.UserError(u'Discogs token request failed') + + # Save the token for later use. + self._log.debug(u'Discogs token {0}, secret {1}', token, secret) + with open(self._tokenfile(), 'w') as f: + json.dump({'token': token, 'secret': secret}, f) + + return token, secret + + def album_distance(self, items, album_info, mapping): + """Returns the album distance. + """ + dist = Distance() + if album_info.data_source == 'Discogs': + dist.add('source', self.config['source_weight'].as_number()) + return dist + + def candidates(self, items, artist, album, va_likely): + """Returns a list of AlbumInfo objects for discogs search results + matching an album and artist (if not various). + """ + if not self.discogs_client: + return + + if va_likely: + query = album + else: + query = '%s %s' % (artist, album) + try: + return self.get_albums(query) + except DiscogsAPIError as e: + self._log.debug(u'API Error: {0} (query: {1})', e, query) + if e.status_code == 401: + self.reset_auth() + return self.candidates(items, artist, album, va_likely) + else: + return [] + except CONNECTION_ERRORS: + self._log.debug(u'Connection error in album search', exc_info=True) + return [] + + def album_for_id(self, album_id): + """Fetches an album by its Discogs ID and returns an AlbumInfo object + or None if the album is not found. + """ + if not self.discogs_client: + return + + self._log.debug(u'Searching for release {0}', album_id) + # Discogs-IDs are simple integers. We only look for those at the end + # of an input string as to avoid confusion with other metadata plugins. + # An optional bracket can follow the integer, as this is how discogs + # displays the release ID on its webpage. + match = re.search(r'(^|\[*r|discogs\.com/.+/release/)(\d+)($|\])', + album_id) + if not match: + return None + result = Release(self.discogs_client, {'id': int(match.group(2))}) + # Try to obtain title to verify that we indeed have a valid Release + try: + getattr(result, 'title') + except DiscogsAPIError as e: + if e.status_code != 404: + self._log.debug(u'API Error: {0} (query: {1})', e, result._uri) + if e.status_code == 401: + self.reset_auth() + return self.album_for_id(album_id) + return None + except CONNECTION_ERRORS: + self._log.debug(u'Connection error in album lookup', exc_info=True) + return None + return self.get_album_info(result) + + def get_albums(self, query): + """Returns a list of AlbumInfo objects for a discogs search query. + """ + # Strip non-word characters from query. Things like "!" and "-" can + # cause a query to return no results, even if they match the artist or + # album title. Use `re.UNICODE` flag to avoid stripping non-english + # word characters. + # TEMPORARY: Encode as ASCII to work around a bug: + # https://github.com/beetbox/beets/issues/1051 + # When the library is fixed, we should encode as UTF-8. + query = re.sub(r'(?u)\W+', ' ', query).encode('ascii', "replace") + # Strip medium information from query, Things like "CD1" and "disk 1" + # can also negate an otherwise positive result. + query = re.sub(r'(?i)\b(CD|disc)\s*\d+', '', query) + try: + releases = self.discogs_client.search(query, + type='release').page(1) + except CONNECTION_ERRORS: + self._log.debug(u"Communication error while searching for {0!r}", + query, exc_info=True) + return [] + return [self.get_album_info(release) for release in releases[:5]] + + def get_album_info(self, result): + """Returns an AlbumInfo object for a discogs Release object. + """ + artist, artist_id = self.get_artist([a.data for a in result.artists]) + album = re.sub(r' +', ' ', result.title) + album_id = result.data['id'] + # Use `.data` to access the tracklist directly instead of the + # convenient `.tracklist` property, which will strip out useful artist + # information and leave us with skeleton `Artist` objects that will + # each make an API call just to get the same data back. + tracks = self.get_tracks(result.data['tracklist']) + albumtype = ', '.join( + result.data['formats'][0].get('descriptions', [])) or None + va = result.data['artists'][0]['name'].lower() == 'various' + if va: + artist = config['va_name'].get(unicode) + year = result.data['year'] + label = result.data['labels'][0]['name'] + mediums = len(set(t.medium for t in tracks)) + catalogno = result.data['labels'][0]['catno'] + if catalogno == 'none': + catalogno = None + country = result.data.get('country') + media = result.data['formats'][0]['name'] + data_url = result.data['uri'] + return AlbumInfo(album, album_id, artist, artist_id, tracks, asin=None, + albumtype=albumtype, va=va, year=year, month=None, + day=None, label=label, mediums=mediums, + artist_sort=None, releasegroup_id=None, + catalognum=catalogno, script=None, language=None, + country=country, albumstatus=None, media=media, + albumdisambig=None, artist_credit=None, + original_year=None, original_month=None, + original_day=None, data_source='Discogs', + data_url=data_url) + + def get_artist(self, artists): + """Returns an artist string (all artists) and an artist_id (the main + artist) for a list of discogs album or track artists. + """ + artist_id = None + bits = [] + for i, artist in enumerate(artists): + if not artist_id: + artist_id = artist['id'] + name = artist['name'] + # Strip disambiguation number. + name = re.sub(r' \(\d+\)$', '', name) + # Move articles to the front. + name = re.sub(r'(?i)^(.*?), (a|an|the)$', r'\2 \1', name) + bits.append(name) + if artist['join'] and i < len(artists) - 1: + bits.append(artist['join']) + artist = ' '.join(bits).replace(' ,', ',') or None + return artist, artist_id + + def get_tracks(self, tracklist): + """Returns a list of TrackInfo objects for a discogs tracklist. + """ + tracks = [] + index_tracks = {} + index = 0 + for track in tracklist: + # Only real tracks have `position`. Otherwise, it's an index track. + if track['position']: + index += 1 + tracks.append(self.get_track_info(track, index)) + else: + index_tracks[index + 1] = track['title'] + + # Fix up medium and medium_index for each track. Discogs position is + # unreliable, but tracks are in order. + medium = None + medium_count, index_count = 0, 0 + for track in tracks: + # Handle special case where a different medium does not indicate a + # new disc, when there is no medium_index and the ordinal of medium + # is not sequential. For example, I, II, III, IV, V. Assume these + # are the track index, not the medium. + medium_is_index = track.medium and not track.medium_index and ( + len(track.medium) != 1 or + ord(track.medium) - 64 != medium_count + 1 + ) + + if not medium_is_index and medium != track.medium: + # Increment medium_count and reset index_count when medium + # changes. + medium = track.medium + medium_count += 1 + index_count = 0 + index_count += 1 + track.medium, track.medium_index = medium_count, index_count + + # Get `disctitle` from Discogs index tracks. Assume that an index track + # before the first track of each medium is a disc title. + for track in tracks: + if track.medium_index == 1: + if track.index in index_tracks: + disctitle = index_tracks[track.index] + else: + disctitle = None + track.disctitle = disctitle + + return tracks + + def get_track_info(self, track, index): + """Returns a TrackInfo object for a discogs track. + """ + title = track['title'] + track_id = None + medium, medium_index = self.get_track_index(track['position']) + artist, artist_id = self.get_artist(track.get('artists', [])) + length = self.get_track_length(track['duration']) + return TrackInfo(title, track_id, artist, artist_id, length, index, + medium, medium_index, artist_sort=None, + disctitle=None, artist_credit=None) + + def get_track_index(self, position): + """Returns the medium and medium index for a discogs track position. + """ + # medium_index is a number at the end of position. medium is everything + # else. E.g. (A)(1), (Side A, Track )(1), (A)(), ()(1), etc. + match = re.match(r'^(.*?)(\d*)$', position.upper()) + if match: + medium, index = match.groups() + else: + self._log.debug(u'Invalid position: {0}', position) + medium = index = None + return medium or None, index or None + + def get_track_length(self, duration): + """Returns the track length in seconds for a discogs duration. + """ + try: + length = time.strptime(duration, '%M:%S') + except ValueError: + return None + return length.tm_min * 60 + length.tm_sec diff --git a/libs/beetsplug/duplicates.py b/libs/beetsplug/duplicates.py new file mode 100644 index 00000000..4f039717 --- /dev/null +++ b/libs/beetsplug/duplicates.py @@ -0,0 +1,337 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Pedro Silva. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""List duplicate tracks or albums. +""" +from __future__ import division, absolute_import, print_function + +import shlex + +from beets.plugins import BeetsPlugin +from beets.ui import decargs, print_, vararg_callback, Subcommand, UserError +from beets.util import command_output, displayable_path, subprocess +from beets.library import Item, Album + +PLUGIN = 'duplicates' + + +class DuplicatesPlugin(BeetsPlugin): + """List duplicate tracks or albums + """ + def __init__(self): + super(DuplicatesPlugin, self).__init__() + + self.config.add({ + 'album': False, + 'checksum': '', + 'copy': '', + 'count': False, + 'delete': False, + 'format': '', + 'full': False, + 'keys': [], + 'merge': False, + 'move': '', + 'path': False, + 'tiebreak': {}, + 'strict': False, + 'tag': '', + }) + + self._command = Subcommand('duplicates', + help=__doc__, + aliases=['dup']) + self._command.parser.add_option( + u'-c', u'--count', dest='count', + action='store_true', + help=u'show duplicate counts', + ) + self._command.parser.add_option( + u'-C', u'--checksum', dest='checksum', + action='store', metavar='PROG', + help=u'report duplicates based on arbitrary command', + ) + self._command.parser.add_option( + u'-d', u'--delete', dest='delete', + action='store_true', + help=u'delete items from library and disk', + ) + self._command.parser.add_option( + u'-F', u'--full', dest='full', + action='store_true', + help=u'show all versions of duplicate tracks or albums', + ) + self._command.parser.add_option( + u'-s', u'--strict', dest='strict', + action='store_true', + help=u'report duplicates only if all attributes are set', + ) + self._command.parser.add_option( + u'-k', u'--keys', dest='keys', + action='callback', metavar='KEY1 KEY2', + callback=vararg_callback, + help=u'report duplicates based on keys', + ) + self._command.parser.add_option( + u'-M', u'--merge', dest='merge', + action='store_true', + help=u'merge duplicate items', + ) + self._command.parser.add_option( + u'-m', u'--move', dest='move', + action='store', metavar='DEST', + help=u'move items to dest', + ) + self._command.parser.add_option( + u'-o', u'--copy', dest='copy', + action='store', metavar='DEST', + help=u'copy items to dest', + ) + self._command.parser.add_option( + u'-t', u'--tag', dest='tag', + action='store', + help=u'tag matched items with \'k=v\' attribute', + ) + self._command.parser.add_all_common_options() + + def commands(self): + + def _dup(lib, opts, args): + self.config.set_args(opts) + album = self.config['album'].get(bool) + checksum = self.config['checksum'].get(str) + copy = self.config['copy'].get(str) + count = self.config['count'].get(bool) + delete = self.config['delete'].get(bool) + fmt = self.config['format'].get(str) + full = self.config['full'].get(bool) + keys = self.config['keys'].get(list) + merge = self.config['merge'].get(bool) + move = self.config['move'].get(str) + path = self.config['path'].get(bool) + tiebreak = self.config['tiebreak'].get(dict) + strict = self.config['strict'].get(bool) + tag = self.config['tag'].get(str) + + if album: + if not keys: + keys = ['mb_albumid'] + items = lib.albums(decargs(args)) + else: + if not keys: + keys = ['mb_trackid', 'mb_albumid'] + items = lib.items(decargs(args)) + + if path: + fmt = '$path' + + # Default format string for count mode. + if count and not fmt: + if album: + fmt = '$albumartist - $album' + else: + fmt = '$albumartist - $album - $title' + fmt += ': {0}' + + if checksum: + for i in items: + k, _ = self._checksum(i, checksum) + keys = [k] + + for obj_id, obj_count, objs in self._duplicates(items, + keys=keys, + full=full, + strict=strict, + tiebreak=tiebreak, + merge=merge): + if obj_id: # Skip empty IDs. + for o in objs: + self._process_item(o, + copy=copy, + move=move, + delete=delete, + tag=tag, + fmt=fmt.format(obj_count)) + + self._command.func = _dup + return [self._command] + + def _process_item(self, item, copy=False, move=False, delete=False, + tag=False, fmt=''): + """Process Item `item`. + """ + print_(format(item, fmt)) + if copy: + item.move(basedir=copy, copy=True) + item.store() + if move: + item.move(basedir=move, copy=False) + item.store() + if delete: + item.remove(delete=True) + if tag: + try: + k, v = tag.split('=') + except: + raise UserError( + u"{}: can't parse k=v tag: {}".format(PLUGIN, tag) + ) + setattr(item, k, v) + item.store() + + def _checksum(self, item, prog): + """Run external `prog` on file path associated with `item`, cache + output as flexattr on a key that is the name of the program, and + return the key, checksum tuple. + """ + args = [p.format(file=item.path) for p in shlex.split(prog)] + key = args[0] + checksum = getattr(item, key, False) + if not checksum: + self._log.debug(u'key {0} on item {1} not cached:' + u'computing checksum', + key, displayable_path(item.path)) + try: + checksum = command_output(args) + setattr(item, key, checksum) + item.store() + self._log.debug(u'computed checksum for {0} using {1}', + item.title, key) + except subprocess.CalledProcessError as e: + self._log.debug(u'failed to checksum {0}: {1}', + displayable_path(item.path), e) + else: + self._log.debug(u'key {0} on item {1} cached:' + u'not computing checksum', + key, displayable_path(item.path)) + return key, checksum + + def _group_by(self, objs, keys, strict): + """Return a dictionary with keys arbitrary concatenations of attributes + and values lists of objects (Albums or Items) with those keys. + + If strict, all attributes must be defined for a duplicate match. + """ + import collections + counts = collections.defaultdict(list) + for obj in objs: + values = [getattr(obj, k, None) for k in keys] + values = [v for v in values if v not in (None, '')] + if strict and len(values) < len(keys): + self._log.debug(u'some keys {0} on item {1} are null or empty:' + u' skipping', + keys, displayable_path(obj.path)) + elif (not strict and not len(values)): + self._log.debug(u'all keys {0} on item {1} are null or empty:' + u' skipping', + keys, displayable_path(obj.path)) + else: + key = tuple(values) + counts[key].append(obj) + + return counts + + def _order(self, objs, tiebreak=None): + """Return the objects (Items or Albums) sorted by descending + order of priority. + + If provided, the `tiebreak` dict indicates the field to use to + prioritize the objects. Otherwise, Items are placed in order of + "completeness" (objects with more non-null fields come first) + and Albums are ordered by their track count. + """ + if tiebreak: + kind = 'items' if all(isinstance(o, Item) + for o in objs) else 'albums' + key = lambda x: tuple(getattr(x, k) for k in tiebreak[kind]) + else: + kind = Item if all(isinstance(o, Item) for o in objs) else Album + if kind is Item: + def truthy(v): + # Avoid a Unicode warning by avoiding comparison + # between a bytes object and the empty Unicode + # string ''. + return v is not None and \ + (v != '' if isinstance(v, unicode) else True) + fields = kind.all_keys() + key = lambda x: sum(1 for f in fields if truthy(getattr(x, f))) + else: + key = lambda x: len(x.items()) + + return sorted(objs, key=key, reverse=True) + + def _merge_items(self, objs): + """Merge Item objs by copying missing fields from items in the tail to + the head item. + + Return same number of items, with the head item modified. + """ + fields = Item.all_keys() + for f in fields: + for o in objs[1:]: + if getattr(objs[0], f, None) in (None, ''): + value = getattr(o, f, None) + if value: + self._log.debug(u'key {0} on item {1} is null ' + u'or empty: setting from item {2}', + f, displayable_path(objs[0].path), + displayable_path(o.path)) + setattr(objs[0], f, value) + objs[0].store() + break + return objs + + def _merge_albums(self, objs): + """Merge Album objs by copying missing items from albums in the tail + to the head album. + + Return same number of albums, with the head album modified.""" + ids = [i.mb_trackid for i in objs[0].items()] + for o in objs[1:]: + for i in o.items(): + if i.mb_trackid not in ids: + missing = Item.from_path(i.path) + missing.album_id = objs[0].id + missing.add(i._db) + self._log.debug(u'item {0} missing from album {1}:' + u' merging from {2} into {3}', + missing, + objs[0], + displayable_path(o.path), + displayable_path(missing.destination())) + missing.move(copy=True) + return objs + + def _merge(self, objs): + """Merge duplicate items. See ``_merge_items`` and ``_merge_albums`` + for the relevant strategies. + """ + kind = Item if all(isinstance(o, Item) for o in objs) else Album + if kind is Item: + objs = self._merge_items(objs) + else: + objs = self._merge_albums(objs) + return objs + + def _duplicates(self, objs, keys, full, strict, tiebreak, merge): + """Generate triples of keys, duplicate counts, and constituent objects. + """ + offset = 0 if full else 1 + for k, objs in self._group_by(objs, keys, strict).iteritems(): + if len(objs) > 1: + objs = self._order(objs, tiebreak) + if merge: + objs = self._merge(objs) + yield (k, len(objs) - offset, objs[offset:]) diff --git a/libs/beetsplug/edit.py b/libs/beetsplug/edit.py new file mode 100644 index 00000000..5c7796ee --- /dev/null +++ b/libs/beetsplug/edit.py @@ -0,0 +1,392 @@ +# This file is part of beets. +# Copyright 2016 +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Open metadata information in a text editor to let the user edit it. +""" +from __future__ import division, absolute_import, print_function + +from beets import plugins +from beets import util +from beets import ui +from beets.dbcore import types +from beets.importer import action +from beets.ui.commands import _do_query, PromptChoice +from copy import deepcopy +import subprocess +import yaml +from tempfile import NamedTemporaryFile +import os + + +# These "safe" types can avoid the format/parse cycle that most fields go +# through: they are safe to edit with native YAML types. +SAFE_TYPES = (types.Float, types.Integer, types.Boolean) + + +class ParseError(Exception): + """The modified file is unreadable. The user should be offered a chance to + fix the error. + """ + + +def edit(filename, log): + """Open `filename` in a text editor. + """ + cmd = util.shlex_split(util.editor_command()) + cmd.append(filename) + log.debug(u'invoking editor command: {!r}', cmd) + try: + subprocess.call(cmd) + except OSError as exc: + raise ui.UserError(u'could not run editor command {!r}: {}'.format( + cmd[0], exc + )) + + +def dump(arg): + """Dump a sequence of dictionaries as YAML for editing. + """ + return yaml.safe_dump_all( + arg, + allow_unicode=True, + default_flow_style=False, + ) + + +def load(s): + """Read a sequence of YAML documents back to a list of dictionaries + with string keys. + + Can raise a `ParseError`. + """ + try: + out = [] + for d in yaml.load_all(s): + if not isinstance(d, dict): + raise ParseError( + u'each entry must be a dictionary; found {}'.format( + type(d).__name__ + ) + ) + + # Convert all keys to strings. They started out as strings, + # but the user may have inadvertently messed this up. + out.append({unicode(k): v for k, v in d.items()}) + + except yaml.YAMLError as e: + raise ParseError(u'invalid YAML: {}'.format(e)) + return out + + +def _safe_value(obj, key, value): + """Check whether the `value` is safe to represent in YAML and trust as + returned from parsed YAML. + + This ensures that values do not change their type when the user edits their + YAML representation. + """ + typ = obj._type(key) + return isinstance(typ, SAFE_TYPES) and isinstance(value, typ.model_type) + + +def flatten(obj, fields): + """Represent `obj`, a `dbcore.Model` object, as a dictionary for + serialization. Only include the given `fields` if provided; + otherwise, include everything. + + The resulting dictionary's keys are strings and the values are + safely YAML-serializable types. + """ + # Format each value. + d = {} + for key in obj.keys(): + value = obj[key] + if _safe_value(obj, key, value): + # A safe value that is faithfully representable in YAML. + d[key] = value + else: + # A value that should be edited as a string. + d[key] = obj.formatted()[key] + + # Possibly filter field names. + if fields: + return {k: v for k, v in d.items() if k in fields} + else: + return d + + +def apply_(obj, data): + """Set the fields of a `dbcore.Model` object according to a + dictionary. + + This is the opposite of `flatten`. The `data` dictionary should have + strings as values. + """ + for key, value in data.items(): + if _safe_value(obj, key, value): + # A safe value *stayed* represented as a safe type. Assign it + # directly. + obj[key] = value + else: + # Either the field was stringified originally or the user changed + # it from a safe type to an unsafe one. Parse it as a string. + obj.set_parse(key, unicode(value)) + + +class EditPlugin(plugins.BeetsPlugin): + + def __init__(self): + super(EditPlugin, self).__init__() + + self.config.add({ + # The default fields to edit. + 'albumfields': 'album albumartist', + 'itemfields': 'track title artist album', + + # Silently ignore any changes to these fields. + 'ignore_fields': 'id path', + }) + + self.register_listener('before_choose_candidate', + self.before_choose_candidate_listener) + + def commands(self): + edit_command = ui.Subcommand( + 'edit', + help=u'interactively edit metadata' + ) + edit_command.parser.add_option( + u'-f', u'--field', + metavar='FIELD', + action='append', + help=u'edit this field also', + ) + edit_command.parser.add_option( + u'--all', + action='store_true', dest='all', + help=u'edit all fields', + ) + edit_command.parser.add_album_option() + edit_command.func = self._edit_command + return [edit_command] + + def _edit_command(self, lib, opts, args): + """The CLI command function for the `beet edit` command. + """ + # Get the objects to edit. + query = ui.decargs(args) + items, albums = _do_query(lib, query, opts.album, False) + objs = albums if opts.album else items + if not objs: + ui.print_(u'Nothing to edit.') + return + + # Get the fields to edit. + if opts.all: + fields = None + else: + fields = self._get_fields(opts.album, opts.field) + self.edit(opts.album, objs, fields) + + def _get_fields(self, album, extra): + """Get the set of fields to edit. + """ + # Start with the configured base fields. + if album: + fields = self.config['albumfields'].as_str_seq() + else: + fields = self.config['itemfields'].as_str_seq() + + # Add the requested extra fields. + if extra: + fields += extra + + # Ensure we always have the `id` field for identification. + fields.append('id') + + return set(fields) + + def edit(self, album, objs, fields): + """The core editor function. + + - `album`: A flag indicating whether we're editing Items or Albums. + - `objs`: The `Item`s or `Album`s to edit. + - `fields`: The set of field names to edit (or None to edit + everything). + """ + # Present the YAML to the user and let her change it. + success = self.edit_objects(objs, fields) + + # Save the new data. + if success: + self.save_changes(objs) + + def edit_objects(self, objs, fields): + """Dump a set of Model objects to a file as text, ask the user + to edit it, and apply any changes to the objects. + + Return a boolean indicating whether the edit succeeded. + """ + # Get the content to edit as raw data structures. + old_data = [flatten(o, fields) for o in objs] + + # Set up a temporary file with the initial data for editing. + new = NamedTemporaryFile(suffix='.yaml', delete=False) + old_str = dump(old_data) + new.write(old_str) + new.close() + + # Loop until we have parseable data and the user confirms. + try: + while True: + # Ask the user to edit the data. + edit(new.name, self._log) + + # Read the data back after editing and check whether anything + # changed. + with open(new.name) as f: + new_str = f.read() + if new_str == old_str: + ui.print_(u"No changes; aborting.") + return False + + # Parse the updated data. + try: + new_data = load(new_str) + except ParseError as e: + ui.print_(u"Could not read data: {}".format(e)) + if ui.input_yn(u"Edit again to fix? (Y/n)", True): + continue + else: + return False + + # Show the changes. + # If the objects are not on the DB yet, we need a copy of their + # original state for show_model_changes. + objs_old = [deepcopy(obj) if not obj._db else None + for obj in objs] + self.apply_data(objs, old_data, new_data) + changed = False + for obj, obj_old in zip(objs, objs_old): + changed |= ui.show_model_changes(obj, obj_old) + if not changed: + ui.print_(u'No changes to apply.') + return False + + # Confirm the changes. + choice = ui.input_options( + (u'continue Editing', u'apply', u'cancel') + ) + if choice == u'a': # Apply. + return True + elif choice == u'c': # Cancel. + return False + elif choice == u'e': # Keep editing. + # Reset the temporary changes to the objects. + for obj in objs: + obj.read() + continue + + # Remove the temporary file before returning. + finally: + os.remove(new.name) + + def apply_data(self, objs, old_data, new_data): + """Take potentially-updated data and apply it to a set of Model + objects. + + The objects are not written back to the database, so the changes + are temporary. + """ + if len(old_data) != len(new_data): + self._log.warn(u'number of objects changed from {} to {}', + len(old_data), len(new_data)) + + obj_by_id = {o.id: o for o in objs} + ignore_fields = self.config['ignore_fields'].as_str_seq() + for old_dict, new_dict in zip(old_data, new_data): + # Prohibit any changes to forbidden fields to avoid + # clobbering `id` and such by mistake. + forbidden = False + for key in ignore_fields: + if old_dict.get(key) != new_dict.get(key): + self._log.warn(u'ignoring object whose {} changed', key) + forbidden = True + break + if forbidden: + continue + + id_ = int(old_dict['id']) + apply_(obj_by_id[id_], new_dict) + + def save_changes(self, objs): + """Save a list of updated Model objects to the database. + """ + # Save to the database and possibly write tags. + for ob in objs: + if ob._dirty: + self._log.debug(u'saving changes to {}', ob) + ob.try_sync(ui.should_write(), ui.should_move()) + + # Methods for interactive importer execution. + + def before_choose_candidate_listener(self, session, task): + """Append an "Edit" choice and an "edit Candidates" choice (if + there are candidates) to the interactive importer prompt. + """ + choices = [PromptChoice('d', 'eDit', self.importer_edit)] + if task.candidates: + choices.append(PromptChoice('c', 'edit Candidates', + self.importer_edit_candidate)) + + return choices + + def importer_edit(self, session, task): + """Callback for invoking the functionality during an interactive + import session on the *original* item tags. + """ + # Assign temporary ids to the Items. + for i, obj in enumerate(task.items): + obj.id = i + 1 + + # Present the YAML to the user and let her change it. + fields = self._get_fields(album=False, extra=[]) + success = self.edit_objects(task.items, fields) + + # Remove temporary ids. + for obj in task.items: + obj.id = None + + # Save the new data. + if success: + # Return action.RETAG, which makes the importer write the tags + # to the files if needed without re-applying metadata. + return action.RETAG + else: + # Edit cancelled / no edits made. Revert changes. + for obj in task.items: + obj.read() + + def importer_edit_candidate(self, session, task): + """Callback for invoking the functionality during an interactive + import session on a *candidate*. The candidate's metadata is + applied to the original items. + """ + # Prompt the user for a candidate. + sel = ui.input_options([], numrange=(1, len(task.candidates))) + # Force applying the candidate on the items. + task.match = task.candidates[sel - 1] + task.apply_metadata() + + return self.importer_edit(session, task) diff --git a/libs/beetsplug/embedart.py b/libs/beetsplug/embedart.py new file mode 100644 index 00000000..71b05f37 --- /dev/null +++ b/libs/beetsplug/embedart.py @@ -0,0 +1,154 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Adrian Sampson. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Allows beets to embed album art into file metadata.""" +from __future__ import division, absolute_import, print_function + +import os.path + +from beets.plugins import BeetsPlugin +from beets import ui +from beets.ui import decargs +from beets.util import syspath, normpath, displayable_path, bytestring_path +from beets.util.artresizer import ArtResizer +from beets import config +from beets import art + + +class EmbedCoverArtPlugin(BeetsPlugin): + """Allows albumart to be embedded into the actual files. + """ + def __init__(self): + super(EmbedCoverArtPlugin, self).__init__() + self.config.add({ + 'maxwidth': 0, + 'auto': True, + 'compare_threshold': 0, + 'ifempty': False, + 'remove_art_file': False + }) + + if self.config['maxwidth'].get(int) and not ArtResizer.shared.local: + self.config['maxwidth'] = 0 + self._log.warning(u"ImageMagick or PIL not found; " + u"'maxwidth' option ignored") + if self.config['compare_threshold'].get(int) and not \ + ArtResizer.shared.can_compare: + self.config['compare_threshold'] = 0 + self._log.warning(u"ImageMagick 6.8.7 or higher not installed; " + u"'compare_threshold' option ignored") + + self.register_listener('art_set', self.process_album) + + def commands(self): + # Embed command. + embed_cmd = ui.Subcommand( + 'embedart', help=u'embed image files into file metadata' + ) + embed_cmd.parser.add_option( + u'-f', u'--file', metavar='PATH', help=u'the image file to embed' + ) + maxwidth = self.config['maxwidth'].get(int) + compare_threshold = self.config['compare_threshold'].get(int) + ifempty = self.config['ifempty'].get(bool) + + def embed_func(lib, opts, args): + if opts.file: + imagepath = normpath(opts.file) + if not os.path.isfile(syspath(imagepath)): + raise ui.UserError(u'image file {0} not found'.format( + displayable_path(imagepath) + )) + for item in lib.items(decargs(args)): + art.embed_item(self._log, item, imagepath, maxwidth, None, + compare_threshold, ifempty) + else: + for album in lib.albums(decargs(args)): + art.embed_album(self._log, album, maxwidth, False, + compare_threshold, ifempty) + self.remove_artfile(album) + + embed_cmd.func = embed_func + + # Extract command. + extract_cmd = ui.Subcommand( + 'extractart', + help=u'extract an image from file metadata', + ) + extract_cmd.parser.add_option( + u'-o', dest='outpath', + help=u'image output file', + ) + extract_cmd.parser.add_option( + u'-n', dest='filename', + help=u'image filename to create for all matched albums', + ) + extract_cmd.parser.add_option( + '-a', dest='associate', action='store_true', + help='associate the extracted images with the album', + ) + + def extract_func(lib, opts, args): + if opts.outpath: + art.extract_first(self._log, normpath(opts.outpath), + lib.items(decargs(args))) + else: + filename = bytestring_path(opts.filename or + config['art_filename'].get()) + if os.path.dirname(filename) != '': + self._log.error( + u"Only specify a name rather than a path for -n") + return + for album in lib.albums(decargs(args)): + artpath = normpath(os.path.join(album.path, filename)) + artpath = art.extract_first(self._log, artpath, + album.items()) + if artpath and opts.associate: + album.set_art(artpath) + album.store() + extract_cmd.func = extract_func + + # Clear command. + clear_cmd = ui.Subcommand( + 'clearart', + help=u'remove images from file metadata', + ) + + def clear_func(lib, opts, args): + art.clear(self._log, lib, decargs(args)) + clear_cmd.func = clear_func + + return [embed_cmd, extract_cmd, clear_cmd] + + def process_album(self, album): + """Automatically embed art after art has been set + """ + if self.config['auto'] and ui.should_write(): + max_width = self.config['maxwidth'].get(int) + art.embed_album(self._log, album, max_width, True, + self.config['compare_threshold'].get(int), + self.config['ifempty'].get(bool)) + self.remove_artfile(album) + + def remove_artfile(self, album): + """Possibly delete the album art file for an album (if the + appropriate configuration option is enabled. + """ + if self.config['remove_art_file'] and album.artpath: + if os.path.isfile(album.artpath): + self._log.debug(u'Removing album art file for {0}', album) + os.remove(album.artpath) + album.artpath = None + album.store() diff --git a/libs/beetsplug/embyupdate.py b/libs/beetsplug/embyupdate.py new file mode 100644 index 00000000..38f8929e --- /dev/null +++ b/libs/beetsplug/embyupdate.py @@ -0,0 +1,135 @@ +# -*- coding: utf-8 -*- + +"""Updates the Emby Library whenever the beets library is changed. + + emby: + host: localhost + port: 8096 + username: user + password: password +""" +from __future__ import division, absolute_import, print_function + +from beets import config +from beets.plugins import BeetsPlugin +from urllib import urlencode +from urlparse import urljoin, parse_qs, urlsplit, urlunsplit +import hashlib +import requests + + +def api_url(host, port, endpoint): + """Returns a joined url. + """ + joined = urljoin('http://{0}:{1}'.format(host, port), endpoint) + scheme, netloc, path, query_string, fragment = urlsplit(joined) + query_params = parse_qs(query_string) + + query_params['format'] = ['json'] + new_query_string = urlencode(query_params, doseq=True) + + return urlunsplit((scheme, netloc, path, new_query_string, fragment)) + + +def password_data(username, password): + """Returns a dict with username and its encoded password. + """ + return { + 'username': username, + 'password': hashlib.sha1(password).hexdigest(), + 'passwordMd5': hashlib.md5(password).hexdigest() + } + + +def create_headers(user_id, token=None): + """Return header dict that is needed to talk to the Emby API. + """ + headers = { + 'Authorization': 'MediaBrowser', + 'UserId': user_id, + 'Client': 'other', + 'Device': 'empy', + 'DeviceId': 'beets', + 'Version': '0.0.0' + } + + if token: + headers['X-MediaBrowser-Token'] = token + + return headers + + +def get_token(host, port, headers, auth_data): + """Return token for a user. + """ + url = api_url(host, port, '/Users/AuthenticateByName') + r = requests.post(url, headers=headers, data=auth_data) + + return r.json().get('AccessToken') + + +def get_user(host, port, username): + """Return user dict from server or None if there is no user. + """ + url = api_url(host, port, '/Users/Public') + r = requests.get(url) + user = [i for i in r.json() if i['Name'] == username] + + return user + + +class EmbyUpdate(BeetsPlugin): + def __init__(self): + super(EmbyUpdate, self).__init__() + + # Adding defaults. + config['emby'].add({ + u'host': u'localhost', + u'port': 8096 + }) + + self.register_listener('database_change', self.listen_for_db_change) + + def listen_for_db_change(self, lib, model): + """Listens for beets db change and register the update for the end. + """ + self.register_listener('cli_exit', self.update) + + def update(self, lib): + """When the client exists try to send refresh request to Emby. + """ + self._log.info(u'Updating Emby library...') + + host = config['emby']['host'].get() + port = config['emby']['port'].get() + username = config['emby']['username'].get() + password = config['emby']['password'].get() + + # Get user information from the Emby API. + user = get_user(host, port, username) + if not user: + self._log.warning(u'User {0} could not be found.'.format(username)) + return + + # Create Authentication data and headers. + auth_data = password_data(username, password) + headers = create_headers(user[0]['Id']) + + # Get authentication token. + token = get_token(host, port, headers, auth_data) + if not token: + self._log.warning( + u'Could not get token for user {0}', username + ) + return + + # Recreate headers with a token. + headers = create_headers(user[0]['Id'], token=token) + + # Trigger the Update. + url = api_url(host, port, '/Library/Refresh') + r = requests.post(url, headers=headers) + if r.status_code != 204: + self._log.warning(u'Update could not be triggered') + else: + self._log.info(u'Update triggered.') diff --git a/libs/beetsplug/export.py b/libs/beetsplug/export.py new file mode 100644 index 00000000..641b9fef --- /dev/null +++ b/libs/beetsplug/export.py @@ -0,0 +1,151 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Exports data from beets +""" + +from __future__ import division, absolute_import, print_function + +import sys +import json +import codecs + +from datetime import datetime, date +from beets.plugins import BeetsPlugin +from beets import ui +from beets import mediafile +from beetsplug.info import make_key_filter, library_data, tag_data + + +class ExportEncoder(json.JSONEncoder): + """Deals with dates because JSON doesn't have a standard""" + def default(self, o): + if isinstance(o, datetime) or isinstance(o, date): + return o.isoformat() + return json.JSONEncoder.default(self, o) + + +class ExportPlugin(BeetsPlugin): + + def __init__(self): + super(ExportPlugin, self).__init__() + + self.config.add({ + 'default_format': 'json', + 'json': { + # json module formatting options + 'formatting': { + 'ensure_ascii': False, + 'indent': 4, + 'separators': (',', ': '), + 'sort_keys': True + } + }, + # TODO: Use something like the edit plugin + # 'item_fields': [] + }) + + def commands(self): + # TODO: Add option to use albums + + cmd = ui.Subcommand('export', help=u'export data from beets') + cmd.func = self.run + cmd.parser.add_option( + u'-l', u'--library', action='store_true', + help=u'show library fields instead of tags', + ) + cmd.parser.add_option( + u'--append', action='store_true', default=False, + help=u'if should append data to the file', + ) + cmd.parser.add_option( + u'-i', u'--include-keys', default=[], + action='append', dest='included_keys', + help=u'comma separated list of keys to show', + ) + cmd.parser.add_option( + u'-o', u'--output', + help=u'path for the output file. If not given, will print the data' + ) + return [cmd] + + def run(self, lib, opts, args): + + file_path = opts.output + file_format = self.config['default_format'].get(str) + file_mode = 'a' if opts.append else 'w' + format_options = self.config[file_format]['formatting'].get(dict) + + export_format = ExportFormat.factory( + file_format, **{ + 'file_path': file_path, + 'file_mode': file_mode + } + ) + + items = [] + data_collector = library_data if opts.library else tag_data + + included_keys = [] + for keys in opts.included_keys: + included_keys.extend(keys.split(',')) + key_filter = make_key_filter(included_keys) + + for data_emitter in data_collector(lib, ui.decargs(args)): + try: + data, item = data_emitter() + except (mediafile.UnreadableFileError, IOError) as ex: + self._log.error(u'cannot read file: {0}', ex) + continue + + data = key_filter(data) + items += [data] + + export_format.export(items, **format_options) + + +class ExportFormat(object): + """The output format type""" + + @classmethod + def factory(cls, type, **kwargs): + if type == "json": + if kwargs['file_path']: + return JsonFileFormat(**kwargs) + else: + return JsonPrintFormat() + raise NotImplementedError() + + def export(self, data, **kwargs): + raise NotImplementedError() + + +class JsonPrintFormat(ExportFormat): + """Outputs to the console""" + + def export(self, data, **kwargs): + json.dump(data, sys.stdout, cls=ExportEncoder, **kwargs) + + +class JsonFileFormat(ExportFormat): + """Saves in a json file""" + + def __init__(self, file_path, file_mode=u'w', encoding=u'utf-8'): + self.path = file_path + self.mode = file_mode + self.encoding = encoding + + def export(self, data, **kwargs): + with codecs.open(self.path, self.mode, self.encoding) as f: + json.dump(data, f, cls=ExportEncoder, **kwargs) diff --git a/libs/beetsplug/fetchart.py b/libs/beetsplug/fetchart.py new file mode 100644 index 00000000..2cc362e4 --- /dev/null +++ b/libs/beetsplug/fetchart.py @@ -0,0 +1,861 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Adrian Sampson. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Fetches album art. +""" +from __future__ import division, absolute_import, print_function + +from contextlib import closing +import os +import re +from tempfile import NamedTemporaryFile + +import requests + +from beets import plugins +from beets import importer +from beets import ui +from beets import util +from beets import config +from beets.util.artresizer import ArtResizer +from beets.util import confit + +try: + import itunes + HAVE_ITUNES = True +except ImportError: + HAVE_ITUNES = False + +IMAGE_EXTENSIONS = ['png', 'jpg', 'jpeg'] +CONTENT_TYPES = ('image/jpeg', 'image/png') +DOWNLOAD_EXTENSION = '.jpg' + + +class Candidate(object): + """Holds information about a matching artwork, deals with validation of + dimension restrictions and resizing. + """ + CANDIDATE_BAD = 0 + CANDIDATE_EXACT = 1 + CANDIDATE_DOWNSCALE = 2 + + MATCH_EXACT = 0 + MATCH_FALLBACK = 1 + + def __init__(self, log, path=None, url=None, source=u'', + match=None, size=None): + self._log = log + self.path = path + self.url = url + self.source = source + self.check = None + self.match = match + self.size = size + + def _validate(self, extra): + """Determine whether the candidate artwork is valid based on + its dimensions (width and ratio). + + Return `CANDIDATE_BAD` if the file is unusable. + Return `CANDIDATE_EXACT` if the file is usable as-is. + Return `CANDIDATE_DOWNSCALE` if the file must be resized. + """ + if not self.path: + return self.CANDIDATE_BAD + + if not (extra['enforce_ratio'] or + extra['minwidth'] or + extra['maxwidth']): + return self.CANDIDATE_EXACT + + # get_size returns None if no local imaging backend is available + if not self.size: + self.size = ArtResizer.shared.get_size(self.path) + self._log.debug(u'image size: {}', self.size) + + if not self.size: + self._log.warning(u'Could not get size of image (please see ' + u'documentation for dependencies). ' + u'The configuration options `minwidth` and ' + u'`enforce_ratio` may be violated.') + return self.CANDIDATE_EXACT + + short_edge = min(self.size) + long_edge = max(self.size) + + # Check minimum size. + if extra['minwidth'] and self.size[0] < extra['minwidth']: + self._log.debug(u'image too small ({} < {})', + self.size[0], extra['minwidth']) + return self.CANDIDATE_BAD + + # Check aspect ratio. + edge_diff = long_edge - short_edge + if extra['enforce_ratio']: + if extra['margin_px']: + if edge_diff > extra['margin_px']: + self._log.debug(u'image is not close enough to being ' + u'square, ({} - {} > {})', + long_edge, short_edge, extra['margin_px']) + return self.CANDIDATE_BAD + elif extra['margin_percent']: + margin_px = extra['margin_percent'] * long_edge + if edge_diff > margin_px: + self._log.debug(u'image is not close enough to being ' + u'square, ({} - {} > {})', + long_edge, short_edge, margin_px) + return self.CANDIDATE_BAD + elif edge_diff: + # also reached for margin_px == 0 and margin_percent == 0.0 + self._log.debug(u'image is not square ({} != {})', + self.size[0], self.size[1]) + return self.CANDIDATE_BAD + + # Check maximum size. + if extra['maxwidth'] and self.size[0] > extra['maxwidth']: + self._log.debug(u'image needs resizing ({} > {})', + self.size[0], extra['maxwidth']) + return self.CANDIDATE_DOWNSCALE + + return self.CANDIDATE_EXACT + + def validate(self, extra): + self.check = self._validate(extra) + return self.check + + def resize(self, extra): + if extra['maxwidth'] and self.check == self.CANDIDATE_DOWNSCALE: + self.path = ArtResizer.shared.resize(extra['maxwidth'], self.path) + + +def _logged_get(log, *args, **kwargs): + """Like `requests.get`, but logs the effective URL to the specified + `log` at the `DEBUG` level. + + Use the optional `message` parameter to specify what to log before + the URL. By default, the string is "getting URL". + + Also sets the User-Agent header to indicate beets. + """ + # Use some arguments with the `send` call but most with the + # `Request` construction. This is a cheap, magic-filled way to + # emulate `requests.get` or, more pertinently, + # `requests.Session.request`. + req_kwargs = kwargs + send_kwargs = {} + for arg in ('stream', 'verify', 'proxies', 'cert', 'timeout'): + if arg in kwargs: + send_kwargs[arg] = req_kwargs.pop(arg) + + # Our special logging message parameter. + if 'message' in kwargs: + message = kwargs.pop('message') + else: + message = 'getting URL' + + req = requests.Request('GET', *args, **req_kwargs) + with requests.Session() as s: + s.headers = {'User-Agent': 'beets'} + prepped = s.prepare_request(req) + log.debug('{}: {}', message, prepped.url) + return s.send(prepped, **send_kwargs) + + +class RequestMixin(object): + """Adds a Requests wrapper to the class that uses the logger, which + must be named `self._log`. + """ + + def request(self, *args, **kwargs): + """Like `requests.get`, but uses the logger `self._log`. + + See also `_logged_get`. + """ + return _logged_get(self._log, *args, **kwargs) + + +# ART SOURCES ################################################################ + +class ArtSource(RequestMixin): + def __init__(self, log, config): + self._log = log + self._config = config + + def get(self, album, extra): + raise NotImplementedError() + + def _candidate(self, **kwargs): + return Candidate(source=self, log=self._log, **kwargs) + + def fetch_image(self, candidate, extra): + raise NotImplementedError() + + +class LocalArtSource(ArtSource): + IS_LOCAL = True + LOC_STR = u'local' + + def fetch_image(self, candidate, extra): + pass + + +class RemoteArtSource(ArtSource): + IS_LOCAL = False + LOC_STR = u'remote' + + def fetch_image(self, candidate, extra): + """Downloads an image from a URL and checks whether it seems to + actually be an image. If so, returns a path to the downloaded image. + Otherwise, returns None. + """ + if extra['maxwidth']: + candidate.url = ArtResizer.shared.proxy_url(extra['maxwidth'], + candidate.url) + try: + with closing(self.request(candidate.url, stream=True, + message=u'downloading image')) as resp: + if 'Content-Type' not in resp.headers \ + or resp.headers['Content-Type'] not in CONTENT_TYPES: + self._log.debug( + u'not a supported image: {}', + resp.headers.get('Content-Type') or u'no content type', + ) + candidate.path = None + return + + # Generate a temporary file with the correct extension. + with NamedTemporaryFile(suffix=DOWNLOAD_EXTENSION, + delete=False) as fh: + for chunk in resp.iter_content(chunk_size=1024): + fh.write(chunk) + self._log.debug(u'downloaded art to: {0}', + util.displayable_path(fh.name)) + candidate.path = fh.name + return + + except (IOError, requests.RequestException, TypeError) as exc: + # Handling TypeError works around a urllib3 bug: + # https://github.com/shazow/urllib3/issues/556 + self._log.debug(u'error fetching art: {}', exc) + candidate.path = None + return + + +class CoverArtArchive(RemoteArtSource): + NAME = u"Cover Art Archive" + + URL = 'http://coverartarchive.org/release/{mbid}/front' + GROUP_URL = 'http://coverartarchive.org/release-group/{mbid}/front' + + def get(self, album, extra): + """Return the Cover Art Archive and Cover Art Archive release group URLs + using album MusicBrainz release ID and release group ID. + """ + if album.mb_albumid: + yield self._candidate(url=self.URL.format(mbid=album.mb_albumid), + match=Candidate.MATCH_EXACT) + if album.mb_releasegroupid: + yield self._candidate( + url=self.GROUP_URL.format(mbid=album.mb_releasegroupid), + match=Candidate.MATCH_FALLBACK) + + +class Amazon(RemoteArtSource): + NAME = u"Amazon" + URL = 'http://images.amazon.com/images/P/%s.%02i.LZZZZZZZ.jpg' + INDICES = (1, 2) + + def get(self, album, extra): + """Generate URLs using Amazon ID (ASIN) string. + """ + if album.asin: + for index in self.INDICES: + yield self._candidate(url=self.URL % (album.asin, index), + match=Candidate.MATCH_EXACT) + + +class AlbumArtOrg(RemoteArtSource): + NAME = u"AlbumArt.org scraper" + URL = 'http://www.albumart.org/index_detail.php' + PAT = r'href\s*=\s*"([^>"]*)"[^>]*title\s*=\s*"View larger image"' + + def get(self, album, extra): + """Return art URL from AlbumArt.org using album ASIN. + """ + if not album.asin: + return + # Get the page from albumart.org. + try: + resp = self.request(self.URL, params={'asin': album.asin}) + self._log.debug(u'scraped art URL: {0}', resp.url) + except requests.RequestException: + self._log.debug(u'error scraping art page') + return + + # Search the page for the image URL. + m = re.search(self.PAT, resp.text) + if m: + image_url = m.group(1) + yield self._candidate(url=image_url, match=Candidate.MATCH_EXACT) + else: + self._log.debug(u'no image found on page') + + +class GoogleImages(RemoteArtSource): + NAME = u"Google Images" + URL = u'https://www.googleapis.com/customsearch/v1' + + def __init__(self, *args, **kwargs): + super(GoogleImages, self).__init__(*args, **kwargs) + self.key = self._config['google_key'].get(), + self.cx = self._config['google_engine'].get(), + + def get(self, album, extra): + """Return art URL from google custom search engine + given an album title and interpreter. + """ + if not (album.albumartist and album.album): + return + search_string = (album.albumartist + ',' + album.album).encode('utf-8') + response = self.request(self.URL, params={ + 'key': self.key, + 'cx': self.cx, + 'q': search_string, + 'searchType': 'image' + }) + + # Get results using JSON. + try: + data = response.json() + except ValueError: + self._log.debug(u'google: error loading response: {}' + .format(response.text)) + return + + if 'error' in data: + reason = data['error']['errors'][0]['reason'] + self._log.debug(u'google fetchart error: {0}', reason) + return + + if 'items' in data.keys(): + for item in data['items']: + yield self._candidate(url=item['link'], + match=Candidate.MATCH_EXACT) + + +class FanartTV(RemoteArtSource): + """Art from fanart.tv requested using their API""" + NAME = u"fanart.tv" + + API_URL = 'http://webservice.fanart.tv/v3/' + API_ALBUMS = API_URL + 'music/albums/' + PROJECT_KEY = '61a7d0ab4e67162b7a0c7c35915cd48e' + + def __init__(self, *args, **kwargs): + super(FanartTV, self).__init__(*args, **kwargs) + self.client_key = self._config['fanarttv_key'].get() + + def get(self, album, extra): + if not album.mb_releasegroupid: + return + + response = self.request( + self.API_ALBUMS + album.mb_releasegroupid, + headers={'api-key': self.PROJECT_KEY, + 'client-key': self.client_key}) + + try: + data = response.json() + except ValueError: + self._log.debug(u'fanart.tv: error loading response: {}', + response.text) + return + + if u'status' in data and data[u'status'] == u'error': + if u'not found' in data[u'error message'].lower(): + self._log.debug(u'fanart.tv: no image found') + elif u'api key' in data[u'error message'].lower(): + self._log.warning(u'fanart.tv: Invalid API key given, please ' + u'enter a valid one in your config file.') + else: + self._log.debug(u'fanart.tv: error on request: {}', + data[u'error message']) + return + + matches = [] + # can there be more than one releasegroupid per response? + for mbid, art in data.get(u'albums', dict()).items(): + # there might be more art referenced, e.g. cdart, and an albumcover + # might not be present, even if the request was succesful + if album.mb_releasegroupid == mbid and u'albumcover' in art: + matches.extend(art[u'albumcover']) + # can this actually occur? + else: + self._log.debug(u'fanart.tv: unexpected mb_releasegroupid in ' + u'response!') + + matches.sort(key=lambda x: x[u'likes'], reverse=True) + for item in matches: + # fanart.tv has a strict size requirement for album art to be + # uploaded + yield self._candidate(url=item[u'url'], + match=Candidate.MATCH_EXACT, + size=(1000, 1000)) + + +class ITunesStore(RemoteArtSource): + NAME = u"iTunes Store" + + def get(self, album, extra): + """Return art URL from iTunes Store given an album title. + """ + if not (album.albumartist and album.album): + return + search_string = (album.albumartist + ' ' + album.album).encode('utf-8') + try: + # Isolate bugs in the iTunes library while searching. + try: + results = itunes.search_album(search_string) + except Exception as exc: + self._log.debug(u'iTunes search failed: {0}', exc) + return + + # Get the first match. + if results: + itunes_album = results[0] + else: + self._log.debug(u'iTunes search for {:r} got no results', + search_string) + return + + if itunes_album.get_artwork()['100']: + small_url = itunes_album.get_artwork()['100'] + big_url = small_url.replace('100x100', '1200x1200') + yield self._candidate(url=big_url, match=Candidate.MATCH_EXACT) + else: + self._log.debug(u'album has no artwork in iTunes Store') + except IndexError: + self._log.debug(u'album not found in iTunes Store') + + +class Wikipedia(RemoteArtSource): + NAME = u"Wikipedia (queried through DBpedia)" + DBPEDIA_URL = 'http://dbpedia.org/sparql' + WIKIPEDIA_URL = 'http://en.wikipedia.org/w/api.php' + SPARQL_QUERY = u'''PREFIX rdf: + PREFIX dbpprop: + PREFIX owl: + PREFIX rdfs: + PREFIX foaf: + + SELECT DISTINCT ?pageId ?coverFilename WHERE {{ + ?subject owl:wikiPageID ?pageId . + ?subject dbpprop:name ?name . + ?subject rdfs:label ?label . + {{ ?subject dbpprop:artist ?artist }} + UNION + {{ ?subject owl:artist ?artist }} + {{ ?artist foaf:name "{artist}"@en }} + UNION + {{ ?artist dbpprop:name "{artist}"@en }} + ?subject rdf:type . + ?subject dbpprop:cover ?coverFilename . + FILTER ( regex(?name, "{album}", "i") ) + }} + Limit 1''' + + def get(self, album, extra): + if not (album.albumartist and album.album): + return + + # Find the name of the cover art filename on DBpedia + cover_filename, page_id = None, None + dbpedia_response = self.request( + self.DBPEDIA_URL, + params={ + 'format': 'application/sparql-results+json', + 'timeout': 2500, + 'query': self.SPARQL_QUERY.format( + artist=album.albumartist.title(), album=album.album) + }, + headers={'content-type': 'application/json'}, + ) + try: + data = dbpedia_response.json() + results = data['results']['bindings'] + if results: + cover_filename = 'File:' + results[0]['coverFilename']['value'] + page_id = results[0]['pageId']['value'] + else: + self._log.debug(u'wikipedia: album not found on dbpedia') + except (ValueError, KeyError, IndexError): + self._log.debug(u'wikipedia: error scraping dbpedia response: {}', + dbpedia_response.text) + + # Ensure we have a filename before attempting to query wikipedia + if not (cover_filename and page_id): + return + + # DBPedia sometimes provides an incomplete cover_filename, indicated + # by the filename having a space before the extension, e.g., 'foo .bar' + # An additional Wikipedia call can help to find the real filename. + # This may be removed once the DBPedia issue is resolved, see: + # https://github.com/dbpedia/extraction-framework/issues/396 + if ' .' in cover_filename and \ + '.' not in cover_filename.split(' .')[-1]: + self._log.debug( + u'wikipedia: dbpedia provided incomplete cover_filename' + ) + lpart, rpart = cover_filename.rsplit(' .', 1) + + # Query all the images in the page + wikipedia_response = self.request( + self.WIKIPEDIA_URL, + params={ + 'format': 'json', + 'action': 'query', + 'continue': '', + 'prop': 'images', + 'pageids': page_id, + }, + headers={'content-type': 'application/json'}, + ) + + # Try to see if one of the images on the pages matches our + # imcomplete cover_filename + try: + data = wikipedia_response.json() + results = data['query']['pages'][page_id]['images'] + for result in results: + if re.match(re.escape(lpart) + r'.*?\.' + re.escape(rpart), + result['title']): + cover_filename = result['title'] + break + except (ValueError, KeyError): + self._log.debug( + u'wikipedia: failed to retrieve a cover_filename' + ) + return + + # Find the absolute url of the cover art on Wikipedia + wikipedia_response = self.request( + self.WIKIPEDIA_URL, + params={ + 'format': 'json', + 'action': 'query', + 'continue': '', + 'prop': 'imageinfo', + 'iiprop': 'url', + 'titles': cover_filename.encode('utf-8'), + }, + headers={'content-type': 'application/json'}, + ) + + try: + data = wikipedia_response.json() + results = data['query']['pages'] + for _, result in results.iteritems(): + image_url = result['imageinfo'][0]['url'] + yield self._candidate(url=image_url, + match=Candidate.MATCH_EXACT) + except (ValueError, KeyError, IndexError): + self._log.debug(u'wikipedia: error scraping imageinfo') + return + + +class FileSystem(LocalArtSource): + NAME = u"Filesystem" + + @staticmethod + def filename_priority(filename, cover_names): + """Sort order for image names. + + Return indexes of cover names found in the image filename. This + means that images with lower-numbered and more keywords will have + higher priority. + """ + return [idx for (idx, x) in enumerate(cover_names) if x in filename] + + def get(self, album, extra): + """Look for album art files in the specified directories. + """ + paths = extra['paths'] + if not paths: + return + cover_names = extra['cover_names'] + cover_pat = br"(\b|_)({0})(\b|_)".format(b'|'.join(cover_names)) + cautious = extra['cautious'] + + for path in paths: + if not os.path.isdir(path): + continue + + # Find all files that look like images in the directory. + images = [] + for fn in os.listdir(path): + for ext in IMAGE_EXTENSIONS: + if fn.lower().endswith(b'.' + ext.encode('utf8')) and \ + os.path.isfile(os.path.join(path, fn)): + images.append(fn) + + # Look for "preferred" filenames. + images = sorted(images, + key=lambda x: + self.filename_priority(x, cover_names)) + remaining = [] + for fn in images: + if re.search(cover_pat, os.path.splitext(fn)[0], re.I): + self._log.debug(u'using well-named art file {0}', + util.displayable_path(fn)) + yield self._candidate(path=os.path.join(path, fn), + match=Candidate.MATCH_EXACT) + else: + remaining.append(fn) + + # Fall back to any image in the folder. + if remaining and not cautious: + self._log.debug(u'using fallback art file {0}', + util.displayable_path(remaining[0])) + yield self._candidate(path=os.path.join(path, remaining[0]), + match=Candidate.MATCH_FALLBACK) + + +# Try each source in turn. + +SOURCES_ALL = [u'filesystem', + u'coverart', u'itunes', u'amazon', u'albumart', + u'wikipedia', u'google', u'fanarttv'] + +ART_SOURCES = { + u'filesystem': FileSystem, + u'coverart': CoverArtArchive, + u'itunes': ITunesStore, + u'albumart': AlbumArtOrg, + u'amazon': Amazon, + u'wikipedia': Wikipedia, + u'google': GoogleImages, + u'fanarttv': FanartTV, +} +SOURCE_NAMES = {v: k for k, v in ART_SOURCES.items()} + +# PLUGIN LOGIC ############################################################### + + +class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): + PAT_PX = r"(0|[1-9][0-9]*)px" + PAT_PERCENT = r"(100(\.00?)?|[1-9]?[0-9](\.[0-9]{1,2})?)%" + + def __init__(self): + super(FetchArtPlugin, self).__init__() + + # Holds candidates corresponding to downloaded images between + # fetching them and placing them in the filesystem. + self.art_candidates = {} + + self.config.add({ + 'auto': True, + 'minwidth': 0, + 'maxwidth': 0, + 'enforce_ratio': False, + 'cautious': False, + 'cover_names': ['cover', 'front', 'art', 'album', 'folder'], + 'sources': ['filesystem', + 'coverart', 'itunes', 'amazon', 'albumart'], + 'google_key': None, + 'google_engine': u'001442825323518660753:hrh5ch1gjzm', + 'fanarttv_key': None, + 'store_source': False, + }) + self.config['google_key'].redact = True + self.config['fanarttv_key'].redact = True + + self.minwidth = self.config['minwidth'].get(int) + self.maxwidth = self.config['maxwidth'].get(int) + + # allow both pixel and percentage-based margin specifications + self.enforce_ratio = self.config['enforce_ratio'].get( + confit.OneOf([bool, + confit.String(pattern=self.PAT_PX), + confit.String(pattern=self.PAT_PERCENT)])) + self.margin_px = None + self.margin_percent = None + if type(self.enforce_ratio) is unicode: + if self.enforce_ratio[-1] == u'%': + self.margin_percent = float(self.enforce_ratio[:-1]) / 100 + elif self.enforce_ratio[-2:] == u'px': + self.margin_px = int(self.enforce_ratio[:-2]) + else: + # shouldn't happen + raise confit.ConfigValueError() + self.enforce_ratio = True + + cover_names = self.config['cover_names'].as_str_seq() + self.cover_names = map(util.bytestring_path, cover_names) + self.cautious = self.config['cautious'].get(bool) + self.store_source = self.config['store_source'].get(bool) + + self.src_removed = (config['import']['delete'].get(bool) or + config['import']['move'].get(bool)) + + if self.config['auto']: + # Enable two import hooks when fetching is enabled. + self.import_stages = [self.fetch_art] + self.register_listener('import_task_files', self.assign_art) + + available_sources = list(SOURCES_ALL) + if not HAVE_ITUNES and u'itunes' in available_sources: + available_sources.remove(u'itunes') + if not self.config['google_key'].get() and \ + u'google' in available_sources: + available_sources.remove(u'google') + sources_name = plugins.sanitize_choices( + self.config['sources'].as_str_seq(), available_sources) + if 'remote_priority' in self.config: + self._log.warning( + u'The `fetch_art.remote_priority` configuration option has ' + u'been deprecated, see the documentation.') + if self.config['remote_priority'].get(bool): + try: + sources_name.remove(u'filesystem') + sources_name.append(u'filesystem') + except ValueError: + pass + self.sources = [ART_SOURCES[s](self._log, self.config) + for s in sources_name] + + # Asynchronous; after music is added to the library. + def fetch_art(self, session, task): + """Find art for the album being imported.""" + if task.is_album: # Only fetch art for full albums. + if task.album.artpath and os.path.isfile(task.album.artpath): + # Album already has art (probably a re-import); skip it. + return + if task.choice_flag == importer.action.ASIS: + # For as-is imports, don't search Web sources for art. + local = True + elif task.choice_flag == importer.action.APPLY: + # Search everywhere for art. + local = False + else: + # For any other choices (e.g., TRACKS), do nothing. + return + + candidate = self.art_for_album(task.album, task.paths, local) + + if candidate: + self.art_candidates[task] = candidate + + def _set_art(self, album, candidate, delete=False): + album.set_art(candidate.path, delete) + if self.store_source: + # store the source of the chosen artwork in a flexible field + self._log.debug( + u"Storing art_source for {0.albumartist} - {0.album}", + album) + album.art_source = SOURCE_NAMES[type(candidate.source)] + album.store() + + # Synchronous; after music files are put in place. + def assign_art(self, session, task): + """Place the discovered art in the filesystem.""" + if task in self.art_candidates: + candidate = self.art_candidates.pop(task) + + self._set_art(task.album, candidate, not self.src_removed) + + if self.src_removed: + task.prune(candidate.path) + + # Manual album art fetching. + def commands(self): + cmd = ui.Subcommand('fetchart', help='download album art') + cmd.parser.add_option( + u'-f', u'--force', dest='force', + action='store_true', default=False, + help=u're-download art when already present' + ) + + def func(lib, opts, args): + self.batch_fetch_art(lib, lib.albums(ui.decargs(args)), opts.force) + cmd.func = func + return [cmd] + + # Utilities converted from functions to methods on logging overhaul + + def art_for_album(self, album, paths, local_only=False): + """Given an Album object, returns a path to downloaded art for the + album (or None if no art is found). If `maxwidth`, then images are + resized to this maximum pixel size. If `local_only`, then only local + image files from the filesystem are returned; no network requests + are made. + """ + out = None + + # all the information any of the sources might need + extra = {'paths': paths, + 'cover_names': self.cover_names, + 'cautious': self.cautious, + 'enforce_ratio': self.enforce_ratio, + 'margin_px': self.margin_px, + 'margin_percent': self.margin_percent, + 'minwidth': self.minwidth, + 'maxwidth': self.maxwidth} + + for source in self.sources: + if source.IS_LOCAL or not local_only: + self._log.debug( + u'trying source {0} for album {1.albumartist} - {1.album}', + SOURCE_NAMES[type(source)], + album, + ) + # URLs might be invalid at this point, or the image may not + # fulfill the requirements + for candidate in source.get(album, extra): + source.fetch_image(candidate, extra) + if candidate.validate(extra): + out = candidate + self._log.debug( + u'using {0.LOC_STR} image {1}'.format( + source, util.displayable_path(out.path))) + break + if out: + break + + if out: + out.resize(extra) + + return out + + def batch_fetch_art(self, lib, albums, force): + """Fetch album art for each of the albums. This implements the manual + fetchart CLI command. + """ + for album in albums: + if album.artpath and not force and os.path.isfile(album.artpath): + message = ui.colorize('text_highlight_minor', u'has album art') + else: + # In ordinary invocations, look for images on the + # filesystem. When forcing, however, always go to the Web + # sources. + local_paths = None if force else [album.path] + + candidate = self.art_for_album(album, local_paths) + if candidate: + self._set_art(album, candidate) + message = ui.colorize('text_success', u'found album art') + else: + message = ui.colorize('text_error', u'no art found') + + self._log.info(u'{0}: {1}', album, message) diff --git a/libs/beetsplug/filefilter.py b/libs/beetsplug/filefilter.py new file mode 100644 index 00000000..72b5ea9e --- /dev/null +++ b/libs/beetsplug/filefilter.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Malte Ried. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Filter imported files using a regular expression. +""" + +from __future__ import division, absolute_import, print_function + +import re +from beets import config +from beets.plugins import BeetsPlugin +from beets.importer import SingletonImportTask + + +class FileFilterPlugin(BeetsPlugin): + def __init__(self): + super(FileFilterPlugin, self).__init__() + self.register_listener('import_task_created', + self.import_task_created_event) + self.config.add({ + 'path': '.*' + }) + + self.path_album_regex = \ + self.path_singleton_regex = \ + re.compile(self.config['path'].get()) + + if 'album_path' in self.config: + self.path_album_regex = re.compile(self.config['album_path'].get()) + + if 'singleton_path' in self.config: + self.path_singleton_regex = re.compile( + self.config['singleton_path'].get()) + + def import_task_created_event(self, session, task): + if task.items and len(task.items) > 0: + items_to_import = [] + for item in task.items: + if self.file_filter(item['path']): + items_to_import.append(item) + if len(items_to_import) > 0: + task.items = items_to_import + else: + # Returning an empty list of tasks from the handler + # drops the task from the rest of the importer pipeline. + return [] + + elif isinstance(task, SingletonImportTask): + if not self.file_filter(task.item['path']): + return [] + + # If not filtered, return the original task unchanged. + return [task] + + def file_filter(self, full_path): + """Checks if the configured regular expressions allow the import + of the file given in full_path. + """ + import_config = dict(config['import']) + if 'singletons' not in import_config or not import_config[ + 'singletons']: + # Album + return self.path_album_regex.match(full_path) is not None + else: + # Singleton + return self.path_singleton_regex.match(full_path) is not None diff --git a/libs/beetsplug/freedesktop.py b/libs/beetsplug/freedesktop.py new file mode 100644 index 00000000..a768be2d --- /dev/null +++ b/libs/beetsplug/freedesktop.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Matt Lichtenberg. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Creates freedesktop.org-compliant .directory files on an album level. +""" + +from __future__ import division, absolute_import, print_function + +from beets.plugins import BeetsPlugin +from beets import ui + + +class FreedesktopPlugin(BeetsPlugin): + def commands(self): + deprecated = ui.Subcommand( + "freedesktop", + help=u"Print a message to redirect to thumbnails --dolphin") + deprecated.func = self.deprecation_message + return [deprecated] + + def deprecation_message(self, lib, opts, args): + ui.print_(u"This plugin is deprecated. Its functionality is " + u"superseded by the 'thumbnails' plugin") + ui.print_(u"'thumbnails --dolphin' replaces freedesktop. See doc & " + u"changelog for more information") diff --git a/libs/beetsplug/fromfilename.py b/libs/beetsplug/fromfilename.py new file mode 100644 index 00000000..e9c49bee --- /dev/null +++ b/libs/beetsplug/fromfilename.py @@ -0,0 +1,173 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Jan-Erik Dahlin +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""If the title is empty, try to extract track and title from the +filename. +""" +from __future__ import division, absolute_import, print_function + +from beets import plugins +from beets.util import displayable_path +import os +import re + + +# Filename field extraction patterns. +PATTERNS = [ + # "01 - Track 01" and "01": do nothing + r'^(\d+)\s*-\s*track\s*\d$', + r'^\d+$', + + # Useful patterns. + r'^(?P.+)-(?P.+)-(?P<tag>.*)$', + r'^(?P<track>\d+)\s*-(?P<artist>.+)-(?P<title>.+)-(?P<tag>.*)$', + r'^(?P<track>\d+)\s(?P<artist>.+)-(?P<title>.+)-(?P<tag>.*)$', + r'^(?P<artist>.+)-(?P<title>.+)$', + r'^(?P<track>\d+)\.\s*(?P<artist>.+)-(?P<title>.+)$', + r'^(?P<track>\d+)\s*-\s*(?P<artist>.+)-(?P<title>.+)$', + r'^(?P<track>\d+)\s*-(?P<artist>.+)-(?P<title>.+)$', + r'^(?P<track>\d+)\s(?P<artist>.+)-(?P<title>.+)$', + r'^(?P<title>.+)$', + r'^(?P<track>\d+)\.\s*(?P<title>.+)$', + r'^(?P<track>\d+)\s*-\s*(?P<title>.+)$', + r'^(?P<track>\d+)\s(?P<title>.+)$', + r'^(?P<title>.+) by (?P<artist>.+)$', +] + +# Titles considered "empty" and in need of replacement. +BAD_TITLE_PATTERNS = [ + r'^$', + r'\d+?\s?-?\s*track\s*\d+', +] + + +def equal(seq): + """Determine whether a sequence holds identical elements. + """ + return len(set(seq)) <= 1 + + +def equal_fields(matchdict, field): + """Do all items in `matchdict`, whose values are dictionaries, have + the same value for `field`? (If they do, the field is probably not + the title.) + """ + return equal(m[field] for m in matchdict.values()) + + +def all_matches(names, pattern): + """If all the filenames in the item/filename mapping match the + pattern, return a dictionary mapping the items to dictionaries + giving the value for each named subpattern in the match. Otherwise, + return None. + """ + matches = {} + for item, name in names.items(): + m = re.match(pattern, name, re.IGNORECASE) + if m and m.groupdict(): + # Only yield a match when the regex applies *and* has + # capture groups. Otherwise, no information can be extracted + # from the filename. + matches[item] = m.groupdict() + else: + return None + return matches + + +def bad_title(title): + """Determine whether a given title is "bad" (empty or otherwise + meaningless) and in need of replacement. + """ + for pat in BAD_TITLE_PATTERNS: + if re.match(pat, title, re.IGNORECASE): + return True + return False + + +def apply_matches(d): + """Given a mapping from items to field dicts, apply the fields to + the objects. + """ + some_map = d.values()[0] + keys = some_map.keys() + + # Only proceed if the "tag" field is equal across all filenames. + if 'tag' in keys and not equal_fields(d, 'tag'): + return + + # Given both an "artist" and "title" field, assume that one is + # *actually* the artist, which must be uniform, and use the other + # for the title. This, of course, won't work for VA albums. + if 'artist' in keys: + if equal_fields(d, 'artist'): + artist = some_map['artist'] + title_field = 'title' + elif equal_fields(d, 'title'): + artist = some_map['title'] + title_field = 'artist' + else: + # Both vary. Abort. + return + + for item in d: + if not item.artist: + item.artist = artist + + # No artist field: remaining field is the title. + else: + title_field = 'title' + + # Apply the title and track. + for item in d: + if bad_title(item.title): + item.title = unicode(d[item][title_field]) + if 'track' in d[item] and item.track == 0: + item.track = int(d[item]['track']) + + +# Plugin structure and hook into import process. + +class FromFilenamePlugin(plugins.BeetsPlugin): + def __init__(self): + super(FromFilenamePlugin, self).__init__() + self.register_listener('import_task_start', filename_task) + + +def filename_task(task, session): + """Examine each item in the task to see if we can extract a title + from the filename. Try to match all filenames to a number of + regexps, starting with the most complex patterns and successively + trying less complex patterns. As soon as all filenames match the + same regex we can make an educated guess of which part of the + regex that contains the title. + """ + items = task.items if task.is_album else [task.item] + + # Look for suspicious (empty or meaningless) titles. + missing_titles = sum(bad_title(i.title) for i in items) + + if missing_titles: + # Get the base filenames (no path or extension). + names = {} + for item in items: + path = displayable_path(item.path) + name, _ = os.path.splitext(os.path.basename(path)) + names[item] = name + + # Look for useful information in the filenames. + for pattern in PATTERNS: + d = all_matches(names, pattern) + if d: + apply_matches(d) diff --git a/libs/beetsplug/ftintitle.py b/libs/beetsplug/ftintitle.py new file mode 100644 index 00000000..eefdfcf1 --- /dev/null +++ b/libs/beetsplug/ftintitle.py @@ -0,0 +1,169 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Verrus, <github.com/Verrus/beets-plugin-featInTitle> +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Moves "featured" artists to the title from the artist field. +""" +from __future__ import division, absolute_import, print_function + +import re + +from beets import plugins +from beets import ui +from beets.util import displayable_path + + +def split_on_feat(artist): + """Given an artist string, split the "main" artist from any artist + on the right-hand side of a string like "feat". Return the main + artist, which is always a string, and the featuring artist, which + may be a string or None if none is present. + """ + # split on the first "feat". + regex = re.compile(plugins.feat_tokens(), re.IGNORECASE) + parts = [s.strip() for s in regex.split(artist, 1)] + if len(parts) == 1: + return parts[0], None + else: + return tuple(parts) + + +def contains_feat(title): + """Determine whether the title contains a "featured" marker. + """ + return bool(re.search(plugins.feat_tokens(), title, flags=re.IGNORECASE)) + + +def find_feat_part(artist, albumartist): + """Attempt to find featured artists in the item's artist fields and + return the results. Returns None if no featured artist found. + """ + feat_part = None + + # Look for the album artist in the artist field. If it's not + # present, give up. + albumartist_split = artist.split(albumartist, 1) + if len(albumartist_split) <= 1: + return feat_part + + # If the last element of the split (the right-hand side of the + # album artist) is nonempty, then it probably contains the + # featured artist. + elif albumartist_split[-1] != '': + # Extract the featured artist from the right-hand side. + _, feat_part = split_on_feat(albumartist_split[-1]) + + # Otherwise, if there's nothing on the right-hand side, look for a + # featuring artist on the left-hand side. + else: + lhs, rhs = split_on_feat(albumartist_split[0]) + if lhs: + feat_part = lhs + + return feat_part + + +class FtInTitlePlugin(plugins.BeetsPlugin): + def __init__(self): + super(FtInTitlePlugin, self).__init__() + + self.config.add({ + 'auto': True, + 'drop': False, + 'format': u'feat. {0}', + }) + + self._command = ui.Subcommand( + 'ftintitle', + help=u'move featured artists to the title field') + + self._command.parser.add_option( + u'-d', u'--drop', dest='drop', + action='store_true', default=False, + help=u'drop featuring from artists and ignore title update') + + if self.config['auto']: + self.import_stages = [self.imported] + + def commands(self): + + def func(lib, opts, args): + self.config.set_args(opts) + drop_feat = self.config['drop'].get(bool) + write = ui.should_write() + + for item in lib.items(ui.decargs(args)): + self.ft_in_title(item, drop_feat) + item.store() + if write: + item.try_write() + + self._command.func = func + return [self._command] + + def imported(self, session, task): + """Import hook for moving featuring artist automatically. + """ + drop_feat = self.config['drop'].get(bool) + + for item in task.imported_items(): + self.ft_in_title(item, drop_feat) + item.store() + + def update_metadata(self, item, feat_part, drop_feat): + """Choose how to add new artists to the title and set the new + metadata. Also, print out messages about any changes that are made. + If `drop_feat` is set, then do not add the artist to the title; just + remove it from the artist field. + """ + # In all cases, update the artist fields. + self._log.info(u'artist: {0} -> {1}', item.artist, item.albumartist) + item.artist = item.albumartist + if item.artist_sort: + # Just strip the featured artist from the sort name. + item.artist_sort, _ = split_on_feat(item.artist_sort) + + # Only update the title if it does not already contain a featured + # artist and if we do not drop featuring information. + if not drop_feat and not contains_feat(item.title): + feat_format = self.config['format'].get(unicode) + new_format = feat_format.format(feat_part) + new_title = u"{0} {1}".format(item.title, new_format) + self._log.info(u'title: {0} -> {1}', item.title, new_title) + item.title = new_title + + def ft_in_title(self, item, drop_feat): + """Look for featured artists in the item's artist fields and move + them to the title. + """ + artist = item.artist.strip() + albumartist = item.albumartist.strip() + + # Check whether there is a featured artist on this track and the + # artist field does not exactly match the album artist field. In + # that case, we attempt to move the featured artist to the title. + _, featured = split_on_feat(artist) + if featured and albumartist != artist and albumartist: + self._log.info('{}', displayable_path(item.path)) + + feat_part = None + + # Attempt to find the featured artist. + feat_part = find_feat_part(artist, albumartist) + + # If we have a featuring artist, move it to the title. + if feat_part: + self.update_metadata(item, feat_part, drop_feat) + else: + self._log.info(u'no featuring artists found') diff --git a/libs/beetsplug/fuzzy.py b/libs/beetsplug/fuzzy.py new file mode 100644 index 00000000..3decdc60 --- /dev/null +++ b/libs/beetsplug/fuzzy.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Philippe Mongeau. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Provides a fuzzy matching query. +""" + +from __future__ import division, absolute_import, print_function + +from beets.plugins import BeetsPlugin +from beets.dbcore.query import StringFieldQuery +from beets import config +import difflib + + +class FuzzyQuery(StringFieldQuery): + @classmethod + def string_match(cls, pattern, val): + # smartcase + if pattern.islower(): + val = val.lower() + query_matcher = difflib.SequenceMatcher(None, pattern, val) + threshold = config['fuzzy']['threshold'].as_number() + return query_matcher.quick_ratio() >= threshold + + +class FuzzyPlugin(BeetsPlugin): + def __init__(self): + super(FuzzyPlugin, self).__init__() + self.config.add({ + 'prefix': '~', + 'threshold': 0.7, + }) + + def queries(self): + prefix = self.config['prefix'].get(basestring) + return {prefix: FuzzyQuery} diff --git a/libs/beetsplug/hook.py b/libs/beetsplug/hook.py new file mode 100644 index 00000000..4f2b8f0e --- /dev/null +++ b/libs/beetsplug/hook.py @@ -0,0 +1,108 @@ +# This file is part of beets. +# Copyright 2015, Adrian Sampson. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Allows custom commands to be run when an event is emitted by beets""" +from __future__ import division, absolute_import, print_function + +import string +import subprocess + +from beets.plugins import BeetsPlugin +from beets.ui import _arg_encoding +from beets.util import shlex_split + + +class CodingFormatter(string.Formatter): + """A custom string formatter that decodes the format string and it's + fields. + """ + + def __init__(self, coding): + """Creates a new coding formatter with the provided coding.""" + self._coding = coding + + def format(self, format_string, *args, **kwargs): + """Formats the provided string using the provided arguments and keyword + arguments. + + This method decodes the format string using the formatter's coding. + + See str.format and string.Formatter.format. + """ + try: + format_string = format_string.decode(self._coding) + except UnicodeEncodeError: + pass + + return super(CodingFormatter, self).format(format_string, *args, + **kwargs) + + def convert_field(self, value, conversion): + """Converts the provided value given a conversion type. + + This method decodes the converted value using the formatter's coding. + + See string.Formatter.convert_field. + """ + converted = super(CodingFormatter, self).convert_field(value, + conversion) + try: + converted = converted.decode(self._coding) + except UnicodeEncodeError: + pass + + return converted + + +class HookPlugin(BeetsPlugin): + """Allows custom commands to be run when an event is emitted by beets""" + def __init__(self): + super(HookPlugin, self).__init__() + + self.config.add({ + 'hooks': [] + }) + + hooks = self.config['hooks'].get(list) + + for hook_index in range(len(hooks)): + hook = self.config['hooks'][hook_index] + + hook_event = hook['event'].get(unicode) + hook_command = hook['command'].get(unicode) + + self.create_and_register_hook(hook_event, hook_command) + + def create_and_register_hook(self, event, command): + def hook_function(**kwargs): + if command is None or len(command) == 0: + self._log.error('invalid command "{0}"', command) + return + + formatter = CodingFormatter(_arg_encoding()) + command_pieces = shlex_split(command) + + for i, piece in enumerate(command_pieces): + command_pieces[i] = formatter.format(piece, event=event, + **kwargs) + + self._log.debug(u'running command "{0}" for event {1}', + u' '.join(command_pieces), event) + + try: + subprocess.Popen(command_pieces).wait() + except OSError as exc: + self._log.error(u'hook for {0} failed: {1}', event, exc) + + self.register_listener(event, hook_function) diff --git a/libs/beetsplug/ihate.py b/libs/beetsplug/ihate.py new file mode 100644 index 00000000..6ed250fe --- /dev/null +++ b/libs/beetsplug/ihate.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Blemjhoo Tezoulbr <baobab@heresiarch.info>. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +from __future__ import division, absolute_import, print_function + +"""Warns you about things you hate (or even blocks import).""" + +from beets.plugins import BeetsPlugin +from beets.importer import action +from beets.library import parse_query_string +from beets.library import Item +from beets.library import Album + + +__author__ = 'baobab@heresiarch.info' +__version__ = '2.0' + + +def summary(task): + """Given an ImportTask, produce a short string identifying the + object. + """ + if task.is_album: + return u'{0} - {1}'.format(task.cur_artist, task.cur_album) + else: + return u'{0} - {1}'.format(task.item.artist, task.item.title) + + +class IHatePlugin(BeetsPlugin): + def __init__(self): + super(IHatePlugin, self).__init__() + self.register_listener('import_task_choice', + self.import_task_choice_event) + self.config.add({ + 'warn': [], + 'skip': [], + }) + + @classmethod + def do_i_hate_this(cls, task, action_patterns): + """Process group of patterns (warn or skip) and returns True if + task is hated and not whitelisted. + """ + if action_patterns: + for query_string in action_patterns: + query, _ = parse_query_string( + query_string, + Album if task.is_album else Item, + ) + if any(query.match(item) for item in task.imported_items()): + return True + return False + + def import_task_choice_event(self, session, task): + skip_queries = self.config['skip'].as_str_seq() + warn_queries = self.config['warn'].as_str_seq() + + if task.choice_flag == action.APPLY: + if skip_queries or warn_queries: + self._log.debug(u'processing your hate') + if self.do_i_hate_this(task, skip_queries): + task.choice_flag = action.SKIP + self._log.info(u'skipped: {0}', summary(task)) + return + if self.do_i_hate_this(task, warn_queries): + self._log.info(u'you may hate this: {0}', summary(task)) + else: + self._log.debug(u'nothing to do') + else: + self._log.debug(u'user made a decision, nothing to do') diff --git a/libs/beetsplug/importadded.py b/libs/beetsplug/importadded.py new file mode 100644 index 00000000..77c7e7ab --- /dev/null +++ b/libs/beetsplug/importadded.py @@ -0,0 +1,134 @@ +# -*- coding: utf-8 -*- + +"""Populate an item's `added` and `mtime` fields by using the file +modification time (mtime) of the item's source file before import. + +Reimported albums and items are skipped. +""" +from __future__ import division, absolute_import, print_function + +import os + +from beets import util +from beets import importer +from beets.plugins import BeetsPlugin + + +class ImportAddedPlugin(BeetsPlugin): + def __init__(self): + super(ImportAddedPlugin, self).__init__() + self.config.add({ + 'preserve_mtimes': False, + 'preserve_write_mtimes': False, + }) + + # item.id for new items that were reimported + self.reimported_item_ids = None + # album.path for old albums that were replaced by a reimported album + self.replaced_album_paths = None + # item path in the library to the mtime of the source file + self.item_mtime = dict() + + register = self.register_listener + register('import_task_start', self.check_config) + register('import_task_start', self.record_if_inplace) + register('import_task_files', self.record_reimported) + register('before_item_moved', self.record_import_mtime) + register('item_copied', self.record_import_mtime) + register('item_linked', self.record_import_mtime) + register('album_imported', self.update_album_times) + register('item_imported', self.update_item_times) + register('after_write', self.update_after_write_time) + + def check_config(self, task, session): + self.config['preserve_mtimes'].get(bool) + + def reimported_item(self, item): + return item.id in self.reimported_item_ids + + def reimported_album(self, album): + return album.path in self.replaced_album_paths + + def record_if_inplace(self, task, session): + if not (session.config['copy'] or session.config['move'] or + session.config['link']): + self._log.debug(u"In place import detected, recording mtimes from " + u"source paths") + items = [task.item] \ + if isinstance(task, importer.SingletonImportTask) \ + else task.items + for item in items: + self.record_import_mtime(item, item.path, item.path) + + def record_reimported(self, task, session): + self.reimported_item_ids = set(item.id for item, replaced_items + in task.replaced_items.iteritems() + if replaced_items) + self.replaced_album_paths = set(task.replaced_albums.keys()) + + def write_file_mtime(self, path, mtime): + """Write the given mtime to the destination path. + """ + stat = os.stat(util.syspath(path)) + os.utime(util.syspath(path), (stat.st_atime, mtime)) + + def write_item_mtime(self, item, mtime): + """Write the given mtime to an item's `mtime` field and to the mtime + of the item's file. + """ + # The file's mtime on disk must be in sync with the item's mtime + self.write_file_mtime(util.syspath(item.path), mtime) + item.mtime = mtime + + def record_import_mtime(self, item, source, destination): + """Record the file mtime of an item's path before its import. + """ + mtime = os.stat(util.syspath(source)).st_mtime + self.item_mtime[destination] = mtime + self._log.debug(u"Recorded mtime {0} for item '{1}' imported from " + u"'{2}'", mtime, util.displayable_path(destination), + util.displayable_path(source)) + + def update_album_times(self, lib, album): + if self.reimported_album(album): + self._log.debug(u"Album '{0}' is reimported, skipping import of " + u"added dates for the album and its items.", + util.displayable_path(album.path)) + return + + album_mtimes = [] + for item in album.items(): + mtime = self.item_mtime.pop(item.path, None) + if mtime: + album_mtimes.append(mtime) + if self.config['preserve_mtimes'].get(bool): + self.write_item_mtime(item, mtime) + item.store() + album.added = min(album_mtimes) + self._log.debug(u"Import of album '{0}', selected album.added={1} " + u"from item file mtimes.", album.album, album.added) + album.store() + + def update_item_times(self, lib, item): + if self.reimported_item(item): + self._log.debug(u"Item '{0}' is reimported, skipping import of " + u"added date.", util.displayable_path(item.path)) + return + mtime = self.item_mtime.pop(item.path, None) + if mtime: + item.added = mtime + if self.config['preserve_mtimes'].get(bool): + self.write_item_mtime(item, mtime) + self._log.debug(u"Import of item '{0}', selected item.added={1}", + util.displayable_path(item.path), item.added) + item.store() + + def update_after_write_time(self, item): + """Update the mtime of the item's file with the item.added value + after each write of the item if `preserve_write_mtimes` is enabled. + """ + if item.added: + if self.config['preserve_write_mtimes'].get(bool): + self.write_item_mtime(item, item.added) + self._log.debug(u"Write of item '{0}', selected item.added={1}", + util.displayable_path(item.path), item.added) diff --git a/libs/beetsplug/importfeeds.py b/libs/beetsplug/importfeeds.py new file mode 100644 index 00000000..d046ddc4 --- /dev/null +++ b/libs/beetsplug/importfeeds.py @@ -0,0 +1,149 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Fabrice Laporte. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +from __future__ import division, absolute_import, print_function + +"""Write paths of imported files in various formats to ease later import in a +music player. Also allow printing the new file locations to stdout in case +one wants to manually add music to a player by its path. +""" +import datetime +import os +import re + +from beets.plugins import BeetsPlugin +from beets.util import mkdirall, normpath, syspath, bytestring_path +from beets import config + +M3U_DEFAULT_NAME = 'imported.m3u' + + +def _get_feeds_dir(lib): + """Given a Library object, return the path to the feeds directory to be + used (either in the library directory or an explicitly configured + path). Ensures that the directory exists. + """ + # Inside library directory. + dirpath = lib.directory + + # Ensure directory exists. + if not os.path.exists(syspath(dirpath)): + os.makedirs(syspath(dirpath)) + return dirpath + + +def _build_m3u_filename(basename): + """Builds unique m3u filename by appending given basename to current + date.""" + + basename = re.sub(r"[\s,/\\'\"]", '_', basename) + date = datetime.datetime.now().strftime("%Y%m%d_%Hh%M") + path = normpath(os.path.join( + config['importfeeds']['dir'].as_filename(), + date + '_' + basename + '.m3u' + )) + return path + + +def _write_m3u(m3u_path, items_paths): + """Append relative paths to items into m3u file. + """ + mkdirall(m3u_path) + with open(syspath(m3u_path), 'a') as f: + for path in items_paths: + f.write(path + b'\n') + + +class ImportFeedsPlugin(BeetsPlugin): + def __init__(self): + super(ImportFeedsPlugin, self).__init__() + + self.config.add({ + 'formats': [], + 'm3u_name': u'imported.m3u', + 'dir': None, + 'relative_to': None, + 'absolute_path': False, + }) + + feeds_dir = self.config['dir'].get() + if feeds_dir: + feeds_dir = os.path.expanduser(bytestring_path(feeds_dir)) + self.config['dir'] = feeds_dir + if not os.path.exists(syspath(feeds_dir)): + os.makedirs(syspath(feeds_dir)) + + relative_to = self.config['relative_to'].get() + if relative_to: + self.config['relative_to'] = normpath(relative_to) + else: + self.config['relative_to'] = feeds_dir + + self.register_listener('library_opened', self.library_opened) + self.register_listener('album_imported', self.album_imported) + self.register_listener('item_imported', self.item_imported) + + def _record_items(self, lib, basename, items): + """Records relative paths to the given items for each feed format + """ + feedsdir = bytestring_path(self.config['dir'].as_filename()) + formats = self.config['formats'].as_str_seq() + relative_to = self.config['relative_to'].get() \ + or self.config['dir'].as_filename() + relative_to = bytestring_path(relative_to) + + paths = [] + for item in items: + if self.config['absolute_path']: + paths.append(item.path) + else: + try: + relpath = os.path.relpath(item.path, relative_to) + except ValueError: + # On Windows, it is sometimes not possible to construct a + # relative path (if the files are on different disks). + relpath = item.path + paths.append(relpath) + + if 'm3u' in formats: + m3u_basename = bytestring_path( + self.config['m3u_name'].get(unicode)) + m3u_path = os.path.join(feedsdir, m3u_basename) + _write_m3u(m3u_path, paths) + + if 'm3u_multi' in formats: + m3u_path = _build_m3u_filename(basename) + _write_m3u(m3u_path, paths) + + if 'link' in formats: + for path in paths: + dest = os.path.join(feedsdir, os.path.basename(path)) + if not os.path.exists(syspath(dest)): + os.symlink(syspath(path), syspath(dest)) + + if 'echo' in formats: + self._log.info(u"Location of imported music:") + for path in paths: + self._log.info(u" {0}", path) + + def library_opened(self, lib): + if self.config['dir'].get() is None: + self.config['dir'] = _get_feeds_dir(lib) + + def album_imported(self, lib, album): + self._record_items(lib, album.album, album.items()) + + def item_imported(self, lib, item): + self._record_items(lib, item.title, [item]) diff --git a/libs/beetsplug/info.py b/libs/beetsplug/info.py new file mode 100644 index 00000000..29bff7a2 --- /dev/null +++ b/libs/beetsplug/info.py @@ -0,0 +1,241 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Adrian Sampson. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Shows file metadata. +""" + +from __future__ import division, absolute_import, print_function + +import os +import re + +from beets.plugins import BeetsPlugin +from beets import ui +from beets import mediafile +from beets.library import Item +from beets.util import displayable_path, normpath, syspath + + +def tag_data(lib, args): + query = [] + for arg in args: + path = normpath(arg) + if os.path.isfile(syspath(path)): + yield tag_data_emitter(path) + else: + query.append(arg) + + if query: + for item in lib.items(query): + yield tag_data_emitter(item.path) + + +def tag_data_emitter(path): + def emitter(): + fields = list(mediafile.MediaFile.readable_fields()) + fields.remove('images') + mf = mediafile.MediaFile(syspath(path)) + tags = {} + for field in fields: + tags[field] = getattr(mf, field) + tags['art'] = mf.art is not None + # create a temporary Item to take advantage of __format__ + item = Item.from_path(syspath(path)) + + return tags, item + return emitter + + +def library_data(lib, args): + for item in lib.items(args): + yield library_data_emitter(item) + + +def library_data_emitter(item): + def emitter(): + data = dict(item.formatted()) + data.pop('path', None) # path is fetched from item + + return data, item + return emitter + + +def update_summary(summary, tags): + for key, value in tags.iteritems(): + if key not in summary: + summary[key] = value + elif summary[key] != value: + summary[key] = '[various]' + return summary + + +def print_data(data, item=None, fmt=None): + """Print, with optional formatting, the fields of a single element. + + If no format string `fmt` is passed, the entries on `data` are printed one + in each line, with the format 'field: value'. If `fmt` is not `None`, the + `item` is printed according to `fmt`, using the `Item.__format__` + machinery. + """ + if fmt: + # use fmt specified by the user + ui.print_(format(item, fmt)) + return + + path = displayable_path(item.path) if item else None + formatted = {} + for key, value in data.iteritems(): + if isinstance(value, list): + formatted[key] = u'; '.join(value) + if value is not None: + formatted[key] = value + + if len(formatted) == 0: + return + + maxwidth = max(len(key) for key in formatted) + lineformat = u'{{0:>{0}}}: {{1}}'.format(maxwidth) + + if path: + ui.print_(displayable_path(path)) + + for field in sorted(formatted): + value = formatted[field] + if isinstance(value, list): + value = u'; '.join(value) + ui.print_(lineformat.format(field, value)) + + +def print_data_keys(data, item=None): + """Print only the keys (field names) for an item. + """ + path = displayable_path(item.path) if item else None + formatted = [] + for key, value in data.iteritems(): + formatted.append(key) + + if len(formatted) == 0: + return + + line_format = u'{0}{{0}}'.format(u' ' * 4) + if path: + ui.print_(displayable_path(path)) + + for field in sorted(formatted): + ui.print_(line_format.format(field)) + + +class InfoPlugin(BeetsPlugin): + + def commands(self): + cmd = ui.Subcommand('info', help=u'show file metadata') + cmd.func = self.run + cmd.parser.add_option( + u'-l', u'--library', action='store_true', + help=u'show library fields instead of tags', + ) + cmd.parser.add_option( + u'-s', u'--summarize', action='store_true', + help=u'summarize the tags of all files', + ) + cmd.parser.add_option( + u'-i', u'--include-keys', default=[], + action='append', dest='included_keys', + help=u'comma separated list of keys to show', + ) + cmd.parser.add_option( + u'-k', u'--keys-only', action='store_true', + help=u'show only the keys', + ) + cmd.parser.add_format_option(target='item') + return [cmd] + + def run(self, lib, opts, args): + """Print tag info or library data for each file referenced by args. + + Main entry point for the `beet info ARGS...` command. + + If an argument is a path pointing to an existing file, then the tags + of that file are printed. All other arguments are considered + queries, and for each item matching all those queries the tags from + the file are printed. + + If `opts.summarize` is true, the function merges all tags into one + dictionary and only prints that. If two files have different values + for the same tag, the value is set to '[various]' + """ + if opts.library: + data_collector = library_data + else: + data_collector = tag_data + + included_keys = [] + for keys in opts.included_keys: + included_keys.extend(keys.split(',')) + key_filter = make_key_filter(included_keys) + + first = True + summary = {} + for data_emitter in data_collector(lib, ui.decargs(args)): + try: + data, item = data_emitter() + except (mediafile.UnreadableFileError, IOError) as ex: + self._log.error(u'cannot read file: {0}', ex) + continue + + data = key_filter(data) + if opts.summarize: + update_summary(summary, data) + else: + if not first: + ui.print_() + if opts.keys_only: + print_data_keys(data, item) + else: + print_data(data, item, opts.format) + first = False + + if opts.summarize: + print_data(summary) + + +def make_key_filter(include): + """Return a function that filters a dictionary. + + The returned filter takes a dictionary and returns another + dictionary that only includes the key-value pairs where the key + glob-matches one of the keys in `include`. + """ + if not include: + return identity + + matchers = [] + for key in include: + key = re.escape(key) + key = key.replace(r'\*', '.*') + matchers.append(re.compile(key + '$')) + + def filter_(data): + filtered = dict() + for key, value in data.items(): + if any(map(lambda m: m.match(key), matchers)): + filtered[key] = value + return filtered + + return filter_ + + +def identity(val): + return val diff --git a/libs/beetsplug/inline.py b/libs/beetsplug/inline.py new file mode 100644 index 00000000..6e3771f2 --- /dev/null +++ b/libs/beetsplug/inline.py @@ -0,0 +1,124 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Adrian Sampson. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Allows inline path template customization code in the config file. +""" +from __future__ import division, absolute_import, print_function + +import traceback +import itertools + +from beets.plugins import BeetsPlugin +from beets import config + +FUNC_NAME = u'__INLINE_FUNC__' + + +class InlineError(Exception): + """Raised when a runtime error occurs in an inline expression. + """ + def __init__(self, code, exc): + super(InlineError, self).__init__( + (u"error in inline path field code:\n" + u"%s\n%s: %s") % (code, type(exc).__name__, unicode(exc)) + ) + + +def _compile_func(body): + """Given Python code for a function body, return a compiled + callable that invokes that code. + """ + body = u'def {0}():\n {1}'.format( + FUNC_NAME, + body.replace('\n', '\n ') + ) + code = compile(body, 'inline', 'exec') + env = {} + eval(code, env) + return env[FUNC_NAME] + + +class InlinePlugin(BeetsPlugin): + def __init__(self): + super(InlinePlugin, self).__init__() + + config.add({ + 'pathfields': {}, # Legacy name. + 'item_fields': {}, + 'album_fields': {}, + }) + + # Item fields. + for key, view in itertools.chain(config['item_fields'].items(), + config['pathfields'].items()): + self._log.debug(u'adding item field {0}', key) + func = self.compile_inline(view.get(unicode), False) + if func is not None: + self.template_fields[key] = func + + # Album fields. + for key, view in config['album_fields'].items(): + self._log.debug(u'adding album field {0}', key) + func = self.compile_inline(view.get(unicode), True) + if func is not None: + self.album_template_fields[key] = func + + def compile_inline(self, python_code, album): + """Given a Python expression or function body, compile it as a path + field function. The returned function takes a single argument, an + Item, and returns a Unicode string. If the expression cannot be + compiled, then an error is logged and this function returns None. + """ + # First, try compiling as a single function. + try: + code = compile(u'({0})'.format(python_code), 'inline', 'eval') + except SyntaxError: + # Fall back to a function body. + try: + func = _compile_func(python_code) + except SyntaxError: + self._log.error(u'syntax error in inline field definition:\n' + u'{0}', traceback.format_exc()) + return + else: + is_expr = False + else: + is_expr = True + + def _dict_for(obj): + out = dict(obj) + if album: + out['items'] = list(obj.items()) + return out + + if is_expr: + # For expressions, just evaluate and return the result. + def _expr_func(obj): + values = _dict_for(obj) + try: + return eval(code, values) + except Exception as exc: + raise InlineError(python_code, exc) + return _expr_func + else: + # For function bodies, invoke the function with values as global + # variables. + def _func_func(obj): + func.__globals__.update(_dict_for(obj)) + try: + return func() + except Exception as exc: + raise InlineError(python_code, exc) + return _func_func diff --git a/libs/beetsplug/ipfs.py b/libs/beetsplug/ipfs.py new file mode 100644 index 00000000..87a100b1 --- /dev/null +++ b/libs/beetsplug/ipfs.py @@ -0,0 +1,286 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Adds support for ipfs. Requires go-ipfs and a running ipfs daemon +""" + +from __future__ import division, absolute_import, print_function + +from beets import ui, util, library, config +from beets.plugins import BeetsPlugin + +import subprocess +import shutil +import os +import tempfile + + +class IPFSPlugin(BeetsPlugin): + + def __init__(self): + super(IPFSPlugin, self).__init__() + self.config.add({ + 'auto': True, + }) + + if self.config['auto']: + self.import_stages = [self.auto_add] + + def commands(self): + cmd = ui.Subcommand('ipfs', + help='interact with ipfs') + cmd.parser.add_option('-a', '--add', dest='add', + action='store_true', + help='Add to ipfs') + cmd.parser.add_option('-g', '--get', dest='get', + action='store_true', + help='Get from ipfs') + cmd.parser.add_option('-p', '--publish', dest='publish', + action='store_true', + help='Publish local library to ipfs') + cmd.parser.add_option('-i', '--import', dest='_import', + action='store_true', + help='Import remote library from ipfs') + cmd.parser.add_option('-l', '--list', dest='_list', + action='store_true', + help='Query imported libraries') + cmd.parser.add_option('-m', '--play', dest='play', + action='store_true', + help='Play music from remote libraries') + + def func(lib, opts, args): + if opts.add: + for album in lib.albums(ui.decargs(args)): + if len(album.items()) == 0: + self._log.info('{0} does not contain items, aborting', + album) + + self.ipfs_add(album) + album.store() + + if opts.get: + self.ipfs_get(lib, ui.decargs(args)) + + if opts.publish: + self.ipfs_publish(lib) + + if opts._import: + self.ipfs_import(lib, ui.decargs(args)) + + if opts._list: + self.ipfs_list(lib, ui.decargs(args)) + + if opts.play: + self.ipfs_play(lib, opts, ui.decargs(args)) + + cmd.func = func + return [cmd] + + def auto_add(self, session, task): + if task.is_album: + if self.ipfs_add(task.album): + task.album.store() + + def ipfs_play(self, lib, opts, args): + from beetsplug.play import PlayPlugin + + jlib = self.get_remote_lib(lib) + player = PlayPlugin() + config['play']['relative_to'] = None + player.album = True + player.play_music(jlib, player, args) + + def ipfs_add(self, album): + try: + album_dir = album.item_dir() + except AttributeError: + return False + try: + if album.ipfs: + self._log.debug('{0} already added', album_dir) + # Already added to ipfs + return False + except AttributeError: + pass + + self._log.info('Adding {0} to ipfs', album_dir) + + cmd = "ipfs add -q -r".split() + cmd.append(album_dir) + try: + output = util.command_output(cmd).split() + except (OSError, subprocess.CalledProcessError) as exc: + self._log.error(u'Failed to add {0}, error: {1}', album_dir, exc) + return False + length = len(output) + + for linenr, line in enumerate(output): + line = line.strip() + if linenr == length - 1: + # last printed line is the album hash + self._log.info("album: {0}", line) + album.ipfs = line + else: + try: + item = album.items()[linenr] + self._log.info("item: {0}", line) + item.ipfs = line + item.store() + except IndexError: + # if there's non music files in the to-add folder they'll + # get ignored here + pass + + return True + + def ipfs_get(self, lib, query): + query = query[0] + # Check if query is a hash + if query.startswith("Qm") and len(query) == 46: + self.ipfs_get_from_hash(lib, query) + else: + albums = self.query(lib, query) + for album in albums: + self.ipfs_get_from_hash(lib, album.ipfs) + + def ipfs_get_from_hash(self, lib, _hash): + try: + cmd = "ipfs get".split() + cmd.append(_hash) + util.command_output(cmd) + except (OSError, subprocess.CalledProcessError) as err: + self._log.error('Failed to get {0} from ipfs.\n{1}', + _hash, err.output) + return False + + self._log.info('Getting {0} from ipfs', _hash) + imp = ui.commands.TerminalImportSession(lib, loghandler=None, + query=None, paths=[_hash]) + imp.run() + shutil.rmtree(_hash) + + def ipfs_publish(self, lib): + with tempfile.NamedTemporaryFile() as tmp: + self.ipfs_added_albums(lib, tmp.name) + try: + cmd = "ipfs add -q ".split() + cmd.append(tmp.name) + output = util.command_output(cmd) + except (OSError, subprocess.CalledProcessError) as err: + msg = "Failed to publish library. Error: {0}".format(err) + self._log.error(msg) + return False + self._log.info("hash of library: {0}", output) + + def ipfs_import(self, lib, args): + _hash = args[0] + if len(args) > 1: + lib_name = args[1] + else: + lib_name = _hash + lib_root = os.path.dirname(lib.path) + remote_libs = lib_root + "/remotes" + if not os.path.exists(remote_libs): + try: + os.makedirs(remote_libs) + except OSError as e: + msg = "Could not create {0}. Error: {1}".format(remote_libs, e) + self._log.error(msg) + return False + path = remote_libs + "/" + lib_name + ".db" + if not os.path.exists(path): + cmd = "ipfs get {0} -o".format(_hash).split() + cmd.append(path) + try: + util.command_output(cmd) + except (OSError, subprocess.CalledProcessError): + self._log.error("Could not import {0}".format(_hash)) + return False + + # add all albums from remotes into a combined library + jpath = remote_libs + "/joined.db" + jlib = library.Library(jpath) + nlib = library.Library(path) + for album in nlib.albums(): + if not self.already_added(album, jlib): + new_album = [] + for item in album.items(): + item.id = None + new_album.append(item) + added_album = jlib.add_album(new_album) + added_album.ipfs = album.ipfs + added_album.store() + + def already_added(self, check, jlib): + for jalbum in jlib.albums(): + if jalbum.mb_albumid == check.mb_albumid: + return True + return False + + def ipfs_list(self, lib, args): + fmt = config['format_album'].get() + try: + albums = self.query(lib, args) + except IOError: + ui.print_("No imported libraries yet.") + return + + for album in albums: + ui.print_(format(album, fmt), " : ", album.ipfs) + + def query(self, lib, args): + rlib = self.get_remote_lib(lib) + albums = rlib.albums(args) + return albums + + def get_remote_lib(self, lib): + lib_root = os.path.dirname(lib.path) + remote_libs = lib_root + "/remotes" + path = remote_libs + "/joined.db" + if not os.path.isfile(path): + raise IOError + return library.Library(path) + + def ipfs_added_albums(self, rlib, tmpname): + """ Returns a new library with only albums/items added to ipfs + """ + tmplib = library.Library(tmpname) + for album in rlib.albums(): + try: + if album.ipfs: + self.create_new_album(album, tmplib) + except AttributeError: + pass + return tmplib + + def create_new_album(self, album, tmplib): + items = [] + for item in album.items(): + try: + if not item.ipfs: + break + except AttributeError: + pass + # Clear current path from item + item.path = '/ipfs/{0}/{1}'.format(album.ipfs, + os.path.basename(item.path)) + + item.id = None + items.append(item) + if len(items) < 1: + return False + self._log.info("Adding '{0}' to temporary library", album) + new_album = tmplib.add_album(items) + new_album.ipfs = album.ipfs + new_album.store() diff --git a/libs/beetsplug/keyfinder.py b/libs/beetsplug/keyfinder.py new file mode 100644 index 00000000..b6131a4b --- /dev/null +++ b/libs/beetsplug/keyfinder.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Thomas Scholtes. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Uses the `KeyFinder` program to add the `initial_key` field. +""" + +from __future__ import division, absolute_import, print_function + +import subprocess + +from beets import ui +from beets import util +from beets.plugins import BeetsPlugin + + +class KeyFinderPlugin(BeetsPlugin): + + def __init__(self): + super(KeyFinderPlugin, self).__init__() + self.config.add({ + u'bin': u'KeyFinder', + u'auto': True, + u'overwrite': False, + }) + + if self.config['auto'].get(bool): + self.import_stages = [self.imported] + + def commands(self): + cmd = ui.Subcommand('keyfinder', + help=u'detect and add initial key from audio') + cmd.func = self.command + return [cmd] + + def command(self, lib, opts, args): + self.find_key(lib.items(ui.decargs(args)), write=ui.should_write()) + + def imported(self, session, task): + self.find_key(task.items) + + def find_key(self, items, write=False): + overwrite = self.config['overwrite'].get(bool) + bin = util.bytestring_path(self.config['bin'].get(unicode)) + + for item in items: + if item['initial_key'] and not overwrite: + continue + + try: + output = util.command_output([bin, b'-f', + util.syspath(item.path)]) + except (subprocess.CalledProcessError, OSError) as exc: + self._log.error(u'execution failed: {0}', exc) + continue + except UnicodeEncodeError: + # Workaround for Python 2 Windows bug. + # http://bugs.python.org/issue1759845 + self._log.error(u'execution failed for Unicode path: {0!r}', + item.path) + continue + + key_raw = output.rsplit(None, 1)[-1] + try: + key = key_raw.decode('utf8') + except UnicodeDecodeError: + self._log.error(u'output is invalid UTF-8') + continue + + item['initial_key'] = key + self._log.info(u'added computed initial key {0} for {1}', + key, util.displayable_path(item.path)) + + if write: + item.try_write() + item.store() diff --git a/libs/beetsplug/lastgenre/__init__.py b/libs/beetsplug/lastgenre/__init__.py new file mode 100644 index 00000000..a4b8f062 --- /dev/null +++ b/libs/beetsplug/lastgenre/__init__.py @@ -0,0 +1,425 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Adrian Sampson. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +from __future__ import division, absolute_import, print_function + +"""Gets genres for imported music based on Last.fm tags. + +Uses a provided whitelist file to determine which tags are valid genres. +The included (default) genre list was originally produced by scraping Wikipedia +and has been edited to remove some questionable entries. +The scraper script used is available here: +https://gist.github.com/1241307 +""" +import pylast +import os +import yaml +import traceback + +from beets import plugins +from beets import ui +from beets import config +from beets.util import normpath, plurality +from beets import library + + +LASTFM = pylast.LastFMNetwork(api_key=plugins.LASTFM_KEY) + +PYLAST_EXCEPTIONS = ( + pylast.WSError, + pylast.MalformedResponseError, + pylast.NetworkError, +) + +REPLACE = { + u'\u2010': '-', +} + + +def deduplicate(seq): + """Remove duplicates from sequence wile preserving order. + """ + seen = set() + return [x for x in seq if x not in seen and not seen.add(x)] + + +# Canonicalization tree processing. + +def flatten_tree(elem, path, branches): + """Flatten nested lists/dictionaries into lists of strings + (branches). + """ + if not path: + path = [] + + if isinstance(elem, dict): + for (k, v) in elem.items(): + flatten_tree(v, path + [k], branches) + elif isinstance(elem, list): + for sub in elem: + flatten_tree(sub, path, branches) + else: + branches.append(path + [unicode(elem)]) + + +def find_parents(candidate, branches): + """Find parents genre of a given genre, ordered from the closest to + the further parent. + """ + for branch in branches: + try: + idx = branch.index(candidate.lower()) + return list(reversed(branch[:idx + 1])) + except ValueError: + continue + return [candidate] + + +# Main plugin logic. + +WHITELIST = os.path.join(os.path.dirname(__file__), 'genres.txt') +C14N_TREE = os.path.join(os.path.dirname(__file__), 'genres-tree.yaml') + + +class LastGenrePlugin(plugins.BeetsPlugin): + def __init__(self): + super(LastGenrePlugin, self).__init__() + + self.config.add({ + 'whitelist': True, + 'min_weight': 10, + 'count': 1, + 'fallback': None, + 'canonical': False, + 'source': 'album', + 'force': True, + 'auto': True, + 'separator': u', ', + }) + + self.setup() + + def setup(self): + """Setup plugin from config options + """ + if self.config['auto']: + self.import_stages = [self.imported] + + self._genre_cache = {} + + # Read the whitelist file if enabled. + self.whitelist = set() + wl_filename = self.config['whitelist'].get() + if wl_filename in (True, ''): # Indicates the default whitelist. + wl_filename = WHITELIST + if wl_filename: + wl_filename = normpath(wl_filename) + with open(wl_filename, 'r') as f: + for line in f: + line = line.decode('utf8').strip().lower() + if line and not line.startswith(u'#'): + self.whitelist.add(line) + + # Read the genres tree for canonicalization if enabled. + self.c14n_branches = [] + c14n_filename = self.config['canonical'].get() + if c14n_filename in (True, ''): # Default tree. + c14n_filename = C14N_TREE + if c14n_filename: + c14n_filename = normpath(c14n_filename) + genres_tree = yaml.load(open(c14n_filename, 'r')) + flatten_tree(genres_tree, [], self.c14n_branches) + + @property + def sources(self): + """A tuple of allowed genre sources. May contain 'track', + 'album', or 'artist.' + """ + source = self.config['source'].as_choice(('track', 'album', 'artist')) + if source == 'track': + return 'track', 'album', 'artist' + elif source == 'album': + return 'album', 'artist' + elif source == 'artist': + return 'artist', + + def _resolve_genres(self, tags): + """Given a list of strings, return a genre by joining them into a + single string and (optionally) canonicalizing each. + """ + if not tags: + return None + + count = self.config['count'].get(int) + if self.c14n_branches: + # Extend the list to consider tags parents in the c14n tree + tags_all = [] + for tag in tags: + # Add parents that are in the whitelist, or add the oldest + # ancestor if no whitelist + if self.whitelist: + parents = [x for x in find_parents(tag, self.c14n_branches) + if self._is_allowed(x)] + else: + parents = [find_parents(tag, self.c14n_branches)[-1]] + + tags_all += parents + if len(tags_all) >= count: + break + tags = tags_all + + tags = deduplicate(tags) + + # c14n only adds allowed genres but we may have had forbidden genres in + # the original tags list + tags = [x.title() for x in tags if self._is_allowed(x)] + + return self.config['separator'].get(unicode).join( + tags[:self.config['count'].get(int)] + ) + + def fetch_genre(self, lastfm_obj): + """Return the genre for a pylast entity or None if no suitable genre + can be found. Ex. 'Electronic, House, Dance' + """ + min_weight = self.config['min_weight'].get(int) + return self._resolve_genres(self._tags_for(lastfm_obj, min_weight)) + + def _is_allowed(self, genre): + """Determine whether the genre is present in the whitelist, + returning a boolean. + """ + if genre is None: + return False + if not self.whitelist or genre in self.whitelist: + return True + return False + + # Cached entity lookups. + + def _last_lookup(self, entity, method, *args): + """Get a genre based on the named entity using the callable `method` + whose arguments are given in the sequence `args`. The genre lookup + is cached based on the entity name and the arguments. Before the + lookup, each argument is has some Unicode characters replaced with + rough ASCII equivalents in order to return better results from the + Last.fm database. + """ + # Shortcut if we're missing metadata. + if any(not s for s in args): + return None + + key = u'{0}.{1}'.format(entity, u'-'.join(unicode(a) for a in args)) + if key in self._genre_cache: + return self._genre_cache[key] + else: + args_replaced = [] + for arg in args: + for k, v in REPLACE.items(): + arg = arg.replace(k, v) + args_replaced.append(arg) + + genre = self.fetch_genre(method(*args_replaced)) + self._genre_cache[key] = genre + return genre + + def fetch_album_genre(self, obj): + """Return the album genre for this Item or Album. + """ + return self._last_lookup( + u'album', LASTFM.get_album, obj.albumartist, obj.album + ) + + def fetch_album_artist_genre(self, obj): + """Return the album artist genre for this Item or Album. + """ + return self._last_lookup( + u'artist', LASTFM.get_artist, obj.albumartist + ) + + def fetch_artist_genre(self, item): + """Returns the track artist genre for this Item. + """ + return self._last_lookup( + u'artist', LASTFM.get_artist, item.artist + ) + + def fetch_track_genre(self, obj): + """Returns the track genre for this Item. + """ + return self._last_lookup( + u'track', LASTFM.get_track, obj.artist, obj.title + ) + + def _get_genre(self, obj): + """Get the genre string for an Album or Item object based on + self.sources. Return a `(genre, source)` pair. The + prioritization order is: + - track (for Items only) + - album + - artist + - original + - fallback + - None + """ + + # Shortcut to existing genre if not forcing. + if not self.config['force'] and self._is_allowed(obj.genre): + return obj.genre, 'keep' + + # Track genre (for Items only). + if isinstance(obj, library.Item): + if 'track' in self.sources: + result = self.fetch_track_genre(obj) + if result: + return result, 'track' + + # Album genre. + if 'album' in self.sources: + result = self.fetch_album_genre(obj) + if result: + return result, 'album' + + # Artist (or album artist) genre. + if 'artist' in self.sources: + result = None + if isinstance(obj, library.Item): + result = self.fetch_artist_genre(obj) + elif obj.albumartist != config['va_name'].get(unicode): + result = self.fetch_album_artist_genre(obj) + else: + # For "Various Artists", pick the most popular track genre. + item_genres = [] + for item in obj.items(): + item_genre = None + if 'track' in self.sources: + item_genre = self.fetch_track_genre(item) + if not item_genre: + item_genre = self.fetch_artist_genre(item) + if item_genre: + item_genres.append(item_genre) + if item_genres: + result, _ = plurality(item_genres) + + if result: + return result, 'artist' + + # Filter the existing genre. + if obj.genre: + result = self._resolve_genres([obj.genre]) + if result: + return result, 'original' + + # Fallback string. + fallback = self.config['fallback'].get() + if fallback: + return fallback, 'fallback' + + return None, None + + def commands(self): + lastgenre_cmd = ui.Subcommand('lastgenre', help=u'fetch genres') + lastgenre_cmd.parser.add_option( + u'-f', u'--force', dest='force', + action='store_true', default=False, + help=u're-download genre when already present' + ) + lastgenre_cmd.parser.add_option( + u'-s', u'--source', dest='source', type='string', + help=u'genre source: artist, album, or track' + ) + + def lastgenre_func(lib, opts, args): + write = ui.should_write() + self.config.set_args(opts) + + for album in lib.albums(ui.decargs(args)): + album.genre, src = self._get_genre(album) + self._log.info(u'genre for album {0} ({1}): {0.genre}', + album, src) + album.store() + + for item in album.items(): + # If we're using track-level sources, also look up each + # track on the album. + if 'track' in self.sources: + item.genre, src = self._get_genre(item) + item.store() + self._log.info(u'genre for track {0} ({1}): {0.genre}', + item, src) + + if write: + item.try_write() + + lastgenre_cmd.func = lastgenre_func + return [lastgenre_cmd] + + def imported(self, session, task): + """Event hook called when an import task finishes.""" + if task.is_album: + album = task.album + album.genre, src = self._get_genre(album) + self._log.debug(u'added last.fm album genre ({0}): {1}', + src, album.genre) + album.store() + + if 'track' in self.sources: + for item in album.items(): + item.genre, src = self._get_genre(item) + self._log.debug(u'added last.fm item genre ({0}): {1}', + src, item.genre) + item.store() + + else: + item = task.item + item.genre, src = self._get_genre(item) + self._log.debug(u'added last.fm item genre ({0}): {1}', + src, item.genre) + item.store() + + def _tags_for(self, obj, min_weight=None): + """Core genre identification routine. + + Given a pylast entity (album or track), return a list of + tag names for that entity. Return an empty list if the entity is + not found or another error occurs. + + If `min_weight` is specified, tags are filtered by weight. + """ + # Work around an inconsistency in pylast where + # Album.get_top_tags() does not return TopItem instances. + # https://code.google.com/p/pylast/issues/detail?id=85 + if isinstance(obj, pylast.Album): + obj = super(pylast.Album, obj) + + try: + res = obj.get_top_tags() + except PYLAST_EXCEPTIONS as exc: + self._log.debug(u'last.fm error: {0}', exc) + return [] + except Exception as exc: + # Isolate bugs in pylast. + self._log.debug(u'{}', traceback.format_exc()) + self._log.error(u'error in pylast library: {0}', exc) + return [] + + # Filter by weight (optionally). + if min_weight: + res = [el for el in res if (int(el.weight or 0)) >= min_weight] + + # Get strings from tags. + res = [el.item.get_name().lower() for el in res] + + return res diff --git a/libs/beetsplug/lastgenre/genres-tree.yaml b/libs/beetsplug/lastgenre/genres-tree.yaml new file mode 100644 index 00000000..a09f7e6b --- /dev/null +++ b/libs/beetsplug/lastgenre/genres-tree.yaml @@ -0,0 +1,749 @@ +- african: + - african heavy metal + - african hip hop + - afrobeat + - apala + - benga + - bikutsi + - bongo flava + - cape jazz + - chimurenga + - coupé-décalé + - fuji music + - genge + - highlife + - hiplife + - isicathamiya + - jit + - jùjú + - kapuka + - kizomba + - kuduro + - kwaito + - kwela + - makossa + - maloya + - marrabenta + - mbalax + - mbaqanga + - mbube + - morna + - museve + - palm-wine + - raï + - sakara + - sega + - seggae + - semba + - soukous + - taarab + - zouglou +- asian: + - east asian: + - anison + - c-pop + - cantopop + - enka + - hong kong english pop + - j-pop + - k-pop + - kayōkyoku + - korean pop + - mandopop + - onkyokei + - taiwanese pop + - fann at-tanbura + - fijiri + - khaliji + - liwa + - sawt + - south and southeast asian: + - baila + - bhangra + - bhojpuri + - dangdut + - filmi + - indian pop + - lavani + - luk thung: + - luk krung + - manila sound + - morlam + - pinoy pop + - pop sunda + - ragini + - thai pop +- avant-garde: + - experimental music + - lo-fi + - musique concrète +- blues: + - african blues + - blues rock + - blues shouter + - british blues + - canadian blues + - chicago blues + - classic female blues + - contemporary r&b + - country blues + - delta blues + - detroit blues + - electric blues + - gospel blues + - hill country blues + - hokum blues + - jazz blues + - jump blues + - kansas city blues + - louisiana blues + - memphis blues + - piano blues + - piedmont blues + - punk blues + - soul blues + - st. louis blues + - swamp blues + - texas blues + - west coast blues +- caribbean and latin american: + - bachata + - baithak gana + - bolero + - brazilian: + - axé + - bossa nova + - brazilian rock + - brega + - choro + - forró + - frevo + - funk carioca + - lambada + - maracatu + - música popular brasileira + - música sertaneja + - pagode + - samba + - samba rock + - tecnobrega + - tropicalia + - zouk-lambada + - calypso + - chutney + - chutney soca + - compas + - mambo + - merengue + - méringue + - other latin: + - chicha + - criolla + - cumbia + - huayno + - mariachi + - ranchera + - tejano + - punta + - punta rock + - rasin + - reggaeton + - salsa + - soca + - son + - timba + - twoubadou + - zouk +- classical: + - ballet + - baroque: + - baroque music + - cantata + - chamber music: + - string quartet + - classical music + - concerto: + - concerto grosso + - contemporary classical + - modern classical + - opera + - oratorio + - orchestra: + - orchestral + - symphonic + - symphony + - organum + - mass: + - requiem + - sacred music: + - cantique + - gregorian chant + - sonata +- comedy: + - comedy music + - comedy rock + - humor + - parody music + - stand-up +- country: + - alternative country: + - cowpunk + - americana + - australian country music + - bakersfield sound + - bluegrass: + - progressive bluegrass + - reactionary bluegrass + - blues country + - cajun: + - cajun fiddle tunes + - christian country music + - classic country + - close harmony + - country pop + - country rap + - country rock + - country soul + - cowboy/western music + - dansband music + - franco-country + - gulf and western + - hellbilly music + - hokum + - honky tonk + - instrumental country + - lubbock sound + - nashville sound + - neotraditional country + - outlaw country + - progressive country + - psychobilly/punkabilly + - red dirt + - rockabilly + - sertanejo + - texas country + - traditional country music + - truck-driving country + - western swing + - zydeco +- easy listening: + - background music + - beautiful music + - elevator music + - furniture music + - lounge music + - middle of the road + - new-age music +- electronic: + - ambient: + - ambient dub + - ambient house + - ambient techno + - dark ambient + - drone music + - illbient + - isolationism + - lowercase + - asian underground + - breakbeat: + - 4-beat + - acid breaks + - baltimore club + - big beat + - breakbeat hardcore + - broken beat + - florida breaks + - nu skool breaks + - chiptune: + - bitpop + - game boy music + - nintendocore + - video game music + - yorkshire bleeps and bass + - disco: + - cosmic disco + - disco polo + - euro disco + - italo disco + - nu-disco + - space disco + - downtempo: + - acid jazz + - balearic beat + - chill out + - dub music + - dubtronica + - ethnic electronica + - moombahton + - nu jazz + - trip hop + - drum and bass: + - darkcore + - darkstep + - drumfunk + - drumstep + - hardstep + - intelligent drum and bass + - jump-up + - liquid funk + - neurofunk + - oldschool jungle: + - darkside jungle + - ragga jungle + - raggacore + - sambass + - techstep + - electro: + - crunk + - electro backbeat + - electro-grime + - electropop + - electroacoustic: + - acousmatic music + - computer music + - electroacoustic improvisation + - field recording + - live coding + - live electronics + - soundscape composition + - tape music + - electronic rock: + - alternative dance: + - baggy + - madchester + - dance-punk + - dance-rock + - dark wave + - electroclash + - electronicore + - electropunk + - ethereal wave + - indietronica + - new rave + - space rock + - synthpop + - synthpunk + - electronica: + - berlin school + - chillwave + - electronic art music + - electronic dance music + - folktronica + - freestyle music + - glitch + - idm + - laptronica + - skweee + - sound art + - synthcore + - eurodance: + - bubblegum dance + - italo dance + - turbofolk + - hardcore: + - bouncy house + - bouncy techno + - breakcore + - digital hardcore + - doomcore + - dubstyle + - gabber + - happy hardcore + - hardstyle + - jumpstyle + - makina + - speedcore + - terrorcore + - uk hardcore + - hi-nrg: + - eurobeat + - hard nrg + - new beat + - house: + - acid house + - chicago house + - deep house + - diva house + - dutch house + - electro house + - freestyle house + - french house + - funky house + - ghetto house + - hardbag + - hip house + - italo house + - latin house + - minimal house + - progressive house + - rave music + - swing house + - tech house + - tribal house + - uk hard house + - us garage + - vocal house + - industrial: + - aggrotech + - coldwave + - cybergrind + - dark electro + - death industrial + - electro-industrial + - electronic body music: + - futurepop + - industrial metal: + - neue deutsche härte + - industrial rock + - noise: + - japanoise + - power electronics + - power noise + - witch house + - post-disco: + - boogie + - dance-pop + - progressive: + - progressive house/trance: + - disco house + - dream house + - space house + - progressive breaks + - progressive drum & bass + - progressive techno + - techno: + - acid techno + - detroit techno + - free tekno + - ghettotech + - minimal + - nortec + - schranz + - techno-dnb + - technopop + - tecno brega + - toytown techno + - trance: + - acid trance + - classic trance + - dream trance + - goa trance: + - dark psytrance + - full on + - psybreaks + - psyprog + - suomisaundi + - hard trance + - tech trance + - uplifting trance: + - orchestral uplifting + - vocal trance + - uk garage: + - 2-step + - 4x4 + - bassline + - breakstep + - dubstep + - funky + - grime + - speed garage + - trap +- folk: + - american folk revival + - anti-folk + - british folk revival + - celtic music + - contemporary folk + - filk music + - freak folk + - indie folk + - industrial folk + - neofolk + - progressive folk + - psychedelic folk + - sung poetry + - techno-folk +- hip hop: + - alternative hip hop + - avant-garde hip hop + - chap hop + - christian hip hop + - conscious hip hop + - country-rap + - crunkcore + - cumbia rap + - east coast hip hop: + - brick city club + - hardcore hip hop + - mafioso rap + - new jersey hip hop + - electro music + - freestyle rap + - g-funk + - gangsta rap + - golden age hip hop + - hip hop soul + - hip pop + - hyphy + - industrial hip hop + - instrumental hip hop + - jazz rap + - low bap + - lyrical hip hop + - merenrap + - midwest hip hop: + - chicago hip hop + - detroit hip hop + - horrorcore + - st. louis hip hop + - twin cities hip hop + - motswako + - nerdcore + - new jack swing + - new school hip hop + - old school hip hop + - political hip hop + - rap opera + - rap rock: + - rap metal + - rapcore + - songo-salsa + - southern hip hop: + - atlanta hip hop: + - snap music + - bounce music + - houston hip hop: + - chopped and screwed + - miami bass + - turntablism + - underground hip hop + - urban pasifika + - west coast hip hop: + - chicano rap + - jerkin' +- jazz: + - asian american jazz + - avant-garde jazz + - bebop + - boogie-woogie + - british dance band + - chamber jazz + - continental jazz + - cool jazz + - crossover jazz + - cubop + - dixieland + - ethno jazz + - european free jazz + - free funk + - free improvisation + - free jazz + - gypsy jazz + - hard bop + - jazz fusion + - jazz rock + - jazz-funk + - kansas city jazz + - latin jazz + - livetronica + - m-base + - mainstream jazz + - modal jazz + - neo-bop jazz + - neo-swing + - novelty ragtime + - orchestral jazz + - post-bop + - punk jazz + - ragtime + - shibuya-kei + - ska jazz + - smooth jazz + - soul jazz + - straight-ahead jazz + - stride jazz + - swing + - third stream + - trad jazz + - vocal jazz + - west coast gypsy jazz + - west coast jazz +- other: + - worldbeat +- pop: + - adult contemporary + - arab pop + - baroque pop + - bubblegum pop + - chanson + - christian pop + - classical crossover + - europop: + - austropop + - balkan pop + - french pop + - latin pop + - laïkó + - nederpop + - russian pop + - iranian pop + - jangle pop + - latin ballad + - levenslied + - louisiana swamp pop + - mexican pop + - motorpop + - new romanticism + - pop rap + - popera + - psychedelic pop + - schlager + - soft rock + - sophisti-pop + - space age pop + - sunshine pop + - surf pop + - teen pop + - traditional pop music + - turkish pop + - vispop + - wonky pop +- rhythm and blues: + - funk: + - deep funk + - go-go + - p-funk + - soul: + - blue-eyed soul + - neo soul + - northern soul +- rock: + - alternative rock: + - britpop: + - post-britpop + - dream pop + - grunge: + - post-grunge + - indie pop: + - dunedin sound + - twee pop + - indie rock + - noise pop + - nu metal + - post-punk revival + - post-rock: + - post-metal + - sadcore + - shoegaze + - slowcore + - art rock + - beat music + - chinese rock + - christian rock + - dark cabaret + - desert rock + - experimental rock + - folk rock + - garage rock + - glam rock + - hard rock + - heavy metal: + - alternative metal + - black metal: + - viking metal + - christian metal + - death metal: + - goregrind + - melodic death metal + - technical death metal + - doom metal + - drone metal + - folk metal: + - celtic metal + - medieval metal + - funk metal + - glam metal + - gothic metal + - metalcore: + - deathcore + - mathcore: + - djent + - power metal + - progressive metal + - sludge metal + - speed metal + - stoner rock + - symphonic metal + - thrash metal: + - crossover thrash + - groove metal + - math rock + - new wave: + - world fusion + - paisley underground + - pop rock + - post-punk: + - gothic rock + - no wave + - noise rock + - power pop + - progressive rock: + - canterbury scene + - krautrock + - new prog + - rock in opposition + - psychedelic rock: + - acid rock + - freakbeat + - neo-psychedelia + - raga rock + - punk rock: + - anarcho punk: + - crust punk: + - d-beat + - art punk + - christian punk + - deathrock + - folk punk: + - celtic punk + - gypsy punk + - garage punk + - grindcore: + - crustgrind + - noisegrind + - hardcore punk: + - post-hardcore: + - emo: + - screamo + - powerviolence + - street punk + - thrashcore + - horror punk + - pop punk + - psychobilly + - riot grrrl + - ska punk: + - ska-core + - skate punk + - rock and roll + - southern rock + - sufi rock + - surf rock + - visual kei: + - nagoya kei +- reggae: + - roots reggae + - reggae fusion + - reggae en español: + - spanish reggae + - reggae 110 + - reggae bultrón + - romantic flow + - lovers rock + - raggamuffin: + - ragga + - dancehall + - ska: + - 2 tone + - dub + - rocksteady diff --git a/libs/beetsplug/lastgenre/genres.txt b/libs/beetsplug/lastgenre/genres.txt new file mode 100644 index 00000000..914ee129 --- /dev/null +++ b/libs/beetsplug/lastgenre/genres.txt @@ -0,0 +1,1534 @@ +2 tone +2-step garage +4-beat +4x4 garage +8-bit +acapella +acid +acid breaks +acid house +acid jazz +acid rock +acoustic music +acousticana +adult contemporary music +african popular music +african rumba +afrobeat +aleatoric music +alternative country +alternative dance +alternative hip hop +alternative metal +alternative rock +ambient +ambient house +ambient music +americana +anarcho punk +anti-folk +apala +ape haters +arab pop +arabesque +arabic pop +argentine rock +ars antiqua +ars nova +art punk +art rock +ashiq +asian american jazz +australian country music +australian hip hop +australian pub rock +austropop +avant-garde +avant-garde jazz +avant-garde metal +avant-garde music +axé +bac-bal +bachata +baggy +baila +baile funk +baisha xiyue +baithak gana +baião +bajourou +bakersfield sound +bakou +bakshy +bal-musette +balakadri +balinese gamelan +balkan pop +ballad +ballata +ballet +bamboo band +bambuco +banda +bangsawan +bantowbol +barbershop music +barndance +baroque +baroque music +baroque pop +bass music +batcave +batucada +batuco +batá-rumba +beach music +beat +beatboxing +beautiful music +bebop +beiguan +bel canto +bend-skin +benga +berlin school of electronic music +bhajan +bhangra +bhangra-wine +bhangragga +bhangramuffin +big band +big band music +big beat +biguine +bihu +bikutsi +biomusic +bitcore +bitpop +black metal +blackened death metal +blue-eyed soul +bluegrass +blues +blues ballad +blues-rock +boogie +boogie woogie +boogie-woogie +bossa nova +brass band +brazilian funk +brazilian jazz +breakbeat +breakbeat hardcore +breakcore +breton music +brill building pop +britfunk +british blues +british invasion +britpop +broken beat +brown-eyed soul +brukdown +brutal death metal +bubblegum dance +bubblegum pop +bulerias +bumba-meu-boi +bunraku +burger-highlife +burgundian school +byzantine chant +ca din tulnic +ca pe lunca +ca trù +cabaret +cadence +cadence rampa +cadence-lypso +café-aman +cai luong +cajun music +cakewalk +calenda +calentanos +calgia +calypso +calypso jazz +calypso-style baila +campursari +canatronic +candombe +canon +canrock +cantata +cante chico +cante jondo +canterbury scene +cantiga +cantique +cantiñas +canto livre +canto nuevo +canto popular +cantopop +canzone napoletana +cape jazz +capoeira music +caracoles +carceleras +cardas +cardiowave +carimbó +cariso +carnatic music +carol +cartageneras +cassette culture +casséy-co +cavacha +caveman +caña +celempungan +cello rock +celtic +celtic fusion +celtic metal +celtic punk +celtic reggae +celtic rock +cha-cha-cha +chakacha +chalga +chamamé +chamber jazz +chamber music +chamber pop +champeta +changuí +chanson +chant +charanga +charanga-vallenata +charikawi +chastushki +chau van +chemical breaks +chicago blues +chicago house +chicago soul +chicano rap +chicha +chicken scratch +children's music +chillout +chillwave +chimurenga +chinese music +chinese pop +chinese rock +chip music +cho-kantrum +chongak +chopera +chorinho +choro +chouval bwa +chowtal +christian alternative +christian black metal +christian electronic music +christian hardcore +christian hip hop +christian industrial +christian metal +christian music +christian punk +christian r&b +christian rock +christian ska +christmas carol +christmas music +chumba +chut-kai-pang +chutney +chutney soca +chutney-bhangra +chutney-hip hop +chutney-soca +chylandyk +chzalni +chèo +cigányzene +classic +classic country +classic female blues +classic rock +classical +classical music +classical music era +clicks n cuts +close harmony +club music +cocobale +coimbra fado +coladeira +colombianas +combined rhythm +comedy +comedy rap +comedy rock +comic opera +comparsa +compas direct +compas meringue +concert overture +concerto +concerto grosso +congo +conjunto +contemporary christian +contemporary christian music +contemporary classical +contemporary r&b +contonbley +contradanza +cool jazz +corrido +corsican polyphonic song +cothoza mfana +country +country blues +country gospel +country music +country pop +country r&b +country rock +country-rap +countrypolitan +couple de sonneurs +coupé-décalé +cowpunk +cretan music +crossover jazz +crossover music +crossover thrash +crossover thrash metal +crunk +crunk&b +crunkcore +crust punk +csárdás +cuarteto +cuban rumba +cuddlecore +cueca +cumbia +cumbia villera +cybergrind +dabka +dadra +daina +dalauna +dance +dance music +dance-pop +dance-punk +dance-rock +dancehall +dangdut +danger music +dansband +danza +danzón +dark ambient +dark cabaret +dark pop +darkcore +darkstep +darkwave +de ascultat la servici +de codru +de dragoste +de jale +de pahar +death industrial +death metal +death rock +death/doom +deathcore +deathgrind +deathrock +deep funk +deep house +deep soul +degung +delta blues +dementia +desert rock +desi +detroit blues +detroit techno +dhamar +dhimotiká +dhrupad +dhun +digital hardcore +dirge +dirty dutch +dirty rap +dirty rap/pornocore +dirty south +disco +disco house +disco polo +disney +disney hardcore +disney pop +diva house +divine rock +dixieland +dixieland jazz +djambadon +djent +dodompa +doina +dombola +dondang sayang +donegal fiddle tradition +dongjing +doo wop +doom metal +doomcore +downtempo +drag +dream pop +drone doom +drone metal +drone music +dronology +drum and bass +dub +dub house +dubanguthu +dubstep +dubtronica +dunedin sound +dunun +dutch jazz +décima +early music +east coast blues +east coast hip hop +easy listening +electric blues +electric folk +electro +electro backbeat +electro hop +electro house +electro punk +electro-industrial +electro-swing +electroclash +electrofunk +electronic +electronic art music +electronic body music +electronic dance +electronic luk thung +electronic music +electronic rock +electronica +electropop +elevator music +emo +emo pop +emo rap +emocore +emotronic +enka +eremwu eu +ethereal pop +ethereal wave +euro +euro disco +eurobeat +eurodance +europop +eurotrance +eurourban +exotica +experimental music +experimental noise +experimental pop +experimental rock +extreme metal +ezengileer +fado +falak +fandango +farruca +fife and drum blues +filk +film score +filmi +filmi-ghazal +finger-style +fjatpangarri +flamenco +flamenco rumba +flower power +foaie verde +fofa +folk hop +folk metal +folk music +folk pop +folk punk +folk rock +folktronica +forró +franco-country +freak-folk +freakbeat +free improvisation +free jazz +free music +freestyle +freestyle house +freetekno +french pop +frenchcore +frevo +fricote +fuji +fuji music +fulia +full on +funaná +funeral doom +funk +funk metal +funk rock +funkcore +funky house +furniture music +fusion jazz +g-funk +gaana +gabba +gabber +gagaku +gaikyoku +gaita +galant +gamad +gambang kromong +gamelan +gamelan angklung +gamelan bang +gamelan bebonangan +gamelan buh +gamelan degung +gamelan gede +gamelan kebyar +gamelan salendro +gamelan selunding +gamelan semar pegulingan +gamewave +gammeldans +gandrung +gangsta rap +gar +garage rock +garrotin +gavotte +gelugpa chanting +gender wayang +gending +german folk music +gharbi +gharnati +ghazal +ghazal-song +ghetto house +ghettotech +girl group +glam metal +glam punk +glam rock +glitch +gnawa +go-go +goa +goa trance +gong-chime music +goombay +goregrind +goshu ondo +gospel music +gothic metal +gothic rock +granadinas +grebo +gregorian chant +grime +grindcore +groove metal +group sounds +grunge +grupera +guaguanbo +guajira +guasca +guitarra baiana +guitarradas +gumbe +gunchei +gunka +guoyue +gwo ka +gwo ka moderne +gypsy jazz +gypsy punk +gypsybilly +gyu ke +habanera +hajnali +hakka +halling +hambo +hands up +hapa haole +happy hardcore +haqibah +hard +hard bop +hard house +hard rock +hard trance +hardcore hip hop +hardcore metal +hardcore punk +hardcore techno +hardstyle +harepa +harmonica blues +hasaposérviko +heart attack +heartland rock +heavy beat +heavy metal +hesher +hi-nrg +highlands +highlife +highlife fusion +hillybilly music +hindustani classical music +hip hop +hip hop & rap +hip hop soul +hip house +hiplife +hiragasy +hiva usu +hong kong and cantonese pop +hong kong english pop +honky tonk +honkyoku +hora lunga +hornpipe +horror punk +horrorcore +horrorcore rap +house +house music +hua'er +huasteco +huayno +hula +humor +humppa +hunguhungu +hyangak +hymn +hyphy +hát chau van +hát chèo +hát cãi luong +hát tuồng +ibiza music +icaro +idm +igbo music +ijexá +ilahije +illbient +impressionist music +improvisational +incidental music +indian pop +indie folk +indie music +indie pop +indie rock +indietronica +indo jazz +indo rock +indonesian pop +indoyíftika +industrial death metal +industrial hip-hop +industrial metal +industrial music +industrial musical +industrial rock +instrumental rock +intelligent dance music +international latin +inuit music +iranian pop +irish folk +irish rebel music +iscathamiya +isicathamiya +isikhwela jo +island +isolationist +italo dance +italo disco +italo house +itsmeños +izvorna bosanska muzika +j'ouvert +j-fusion +j-pop +j-rock +jaipongan +jaliscienses +jam band +jam rock +jamana kura +jamrieng samai +jangle pop +japanese pop +jarana +jariang +jarochos +jawaiian +jazz +jazz blues +jazz fusion +jazz metal +jazz rap +jazz-funk +jazz-rock +jegog +jenkka +jesus music +jibaro +jig +jig punk +jing ping +jingle +jit +jitterbug +jive +joged +joged bumbung +joik +jonnycore +joropo +jota +jtek +jug band +jujitsu +juju +juke joint blues +jump blues +jumpstyle +jungle +junkanoo +juré +jùjú +k-pop +kaba +kabuki +kachāshī +kadans +kagok +kagyupa chanting +kaiso +kalamatianó +kalattuut +kalinda +kamba pop +kan ha diskan +kansas city blues +kantrum +kantádhes +kargyraa +karma +kaseko +katajjaq +kawachi ondo +kayōkyoku +ke-kwe +kebyar +kecak +kecapi suling +kertok +khaleeji +khap +khelimaski djili +khene +khoomei +khorovodi +khplam wai +khrung sai +khyal +kilapanda +kinko +kirtan +kiwi rock +kizomba +klape +klasik +klezmer +kliningan +kléftiko +kochare +kolomyjka +komagaku +kompa +konpa +korean pop +koumpaneia +kpanlogo +krakowiak +krautrock +kriti +kroncong +krump +krzesany +kuduro +kulintang +kulning +kumina +kun-borrk +kundere +kundiman +kussundé +kutumba wake +kveding +kvæði +kwaito +kwassa kwassa +kwela +käng +kélé +kĩkũyũ pop +la la +latin american +latin jazz +latin pop +latin rap +lavway +laïko +laïkó +le leagan +legényes +lelio +letkajenkka +levenslied +lhamo +lieder +light music +light rock +likanos +liquid drum&bass +liquid funk +liquindi +llanera +llanto +lo-fi +lo-fi music +loki djili +long-song +louisiana blues +louisiana swamp pop +lounge music +lovers rock +lowercase +lubbock sound +lucknavi thumri +luhya omutibo +luk grung +lullaby +lundu +lundum +m-base +madchester +madrigal +mafioso rap +maglaal +magnificat +mahori +mainstream jazz +makossa +makossa-soukous +malagueñas +malawian jazz +malhun +maloya +maluf +maluka +mambo +manaschi +mandarin pop +manding swing +mango +mangue bit +mangulina +manikay +manila sound +manouche +manzuma +mapouka +mapouka-serré +marabi +maracatu +marga +mariachi +marimba +marinera +marrabenta +martial industrial +martinetes +maskanda +mass +matamuerte +math rock +mathcore +matt bello +maxixe +mazurka +mbalax +mbaqanga +mbube +mbumba +medh +medieval folk rock +medieval metal +medieval music +meditation +mejorana +melhoun +melhûn +melodic black metal +melodic death metal +melodic hardcore +melodic metalcore +melodic music +melodic trance +memphis blues +memphis rap +memphis soul +mento +merengue +merengue típico moderno +merengue-bomba +meringue +merseybeat +metal +metalcore +metallic hardcore +mexican pop +mexican rock +mexican son +meykhana +mezwed +miami bass +microhouse +middle of the road +midwest hip hop +milonga +min'yo +mineras +mini compas +mini-jazz +minimal techno +minimalist music +minimalist trance +minneapolis sound +minstrel show +minuet +mirolóyia +modal jazz +modern classical +modern classical music +modern laika +modern rock +modinha +mohabelo +montuno +monumental dance +mor lam +mor lam sing +morna +motorpop +motown +mozambique +mpb +mugam +multicultural +murga +musette +museve +mushroom jazz +music drama +music hall +musiqi-e assil +musique concrète +mutuashi +muwashshah +muzak +méringue +música campesina +música criolla +música de la interior +música llanera +música nordestina +música popular brasileira +música tropical +nagauta +nakasi +nangma +nanguan +narcocorrido +nardcore +narodna muzika +nasheed +nashville sound +nashville sound/countrypolitan +national socialist black metal +naturalismo +nederpop +neo soul +neo-classical metal +neo-medieval +neo-prog +neo-psychedelia +neoclassical +neoclassical music +neofolk +neotraditional country +nerdcore +neue deutsche härte +neue deutsche welle +new age music +new beat +new instrumental +new jack swing +new orleans blues +new orleans jazz +new pop +new prog +new rave +new romantic +new school hip hop +new taiwanese song +new wave +new wave of british heavy metal +new wave of new wave +new weird america +new york blues +new york house +newgrass +nganja +nightcore +nintendocore +nisiótika +no wave +noh +noise music +noise pop +noise rock +nongak +norae undong +nordic folk dance music +nordic folk music +nortec +norteño +northern soul +nota +nu breaks +nu jazz +nu metal +nu soul +nueva canción +nyatiti +néo kýma +obscuro +oi! +old school hip hop +old-time +oldies +olonkho +oltului +ondo +opera +operatic pop +oratorio +orchestra +orchestral +organ trio +organic ambient +organum +orgel +oriental metal +ottava rima +outlaw country +outsider music +p-funk +pagan metal +pagan rock +pagode +paisley underground +palm wine +palm-wine +pambiche +panambih +panchai baja +panchavadyam +pansori +paranda +parang +parody +parranda +partido alto +pasillo +patriotic +peace punk +pelimanni music +petenera +peyote song +philadelphia soul +piano blues +piano rock +piedmont blues +pimba +pinoy pop +pinoy rock +pinpeat orchestra +piphat +piyyutim +plainchant +plena +pleng phua cheewit +pleng thai sakorn +political hip hop +polka +polo +polonaise +pols +polska +pong lang +pop +pop folk +pop music +pop punk +pop rap +pop rock +pop sunda +pornocore +porro +post disco +post-britpop +post-disco +post-grunge +post-hardcore +post-industrial +post-metal +post-minimalism +post-punk +post-rock +post-romanticism +pow-wow +power electronics +power metal +power noise +power pop +powerviolence +ppongtchak +praise song +program symphony +progressive bluegrass +progressive country +progressive death metal +progressive electronic +progressive electronic music +progressive folk +progressive folk music +progressive house +progressive metal +progressive rock +progressive trance +protopunk +psych folk +psychedelic music +psychedelic pop +psychedelic rock +psychedelic trance +psychobilly +punk blues +punk cabaret +punk jazz +punk rock +punta +punta rock +qasidah +qasidah modern +qawwali +quadrille +quan ho +queercore +quiet storm +rada +raga +raga rock +ragga +ragga jungle +raggamuffin +ragtime +rai +rake-and-scrape +ramkbach +ramvong +ranchera +rap +rap metal +rap rock +rapcore +rara +rare groove +rasiya +rave +raw rock +raï +rebetiko +red dirt +reel +reggae +reggae 110 +reggae bultrón +reggae en español +reggae fusion +reggae highlife +reggaefusion +reggaeton +rekilaulu +relax music +religious +rembetiko +renaissance music +requiem +rhapsody +rhyming spiritual +rhythm & blues +rhythm and blues +ricercar +riot grrrl +rock +rock and roll +rock en español +rock opera +rockabilly +rocksteady +rococo +romantic flow +romantic period in music +rondeaux +ronggeng +roots reggae +roots rock +roots rock reggae +rumba +russian pop +rímur +sabar +sacred harp +sacred music +sadcore +saibara +sakara +salegy +salsa +salsa erotica +salsa romantica +saltarello +samba +samba-canção +samba-reggae +samba-rock +sambai +sanjo +sato kagura +sawt +saya +scat +schlager +schottisch +schranz +scottish baroque music +screamo +scrumpy and western +sea shanty +sean nós +second viennese school +sega music +seggae +seis +semba +sephardic music +serialism +set dance +sevdalinka +sevillana +shabab +shabad +shalako +shan'ge +shango +shape note +shibuya-kei +shidaiqu +shima uta +shock rock +shoegaze +shoegazer +shoka +shomyo +show tune +sica +siguiriyas +silat +sinawi +situational +ska +ska punk +skacore +skald +skate punk +skiffle +slack-key guitar +slide +slowcore +sludge metal +slängpolska +smooth jazz +soca +soft rock +son +son montuno +son-batá +sonata +songo +songo-salsa +sophisti-pop +soukous +soul +soul blues +soul jazz +soul music +southern gospel +southern harmony +southern hip hop +southern metal +southern rock +southern soul +space age pop +space music +space rock +spectralism +speed garage +speed metal +speedcore +spirituals +spouge +sprechgesang +square dance +squee +st. louis blues +stand-up +steelband +stoner metal +stoner rock +straight edge +strathspeys +stride +string +string quartet +sufi music +suite +sunshine pop +suomirock +super eurobeat +surf ballad +surf instrumental +surf music +surf pop +surf rock +swamp blues +swamp pop +swamp rock +swing +swing music +swingbeat +sygyt +symphonic +symphonic black metal +symphonic metal +symphonic poem +symphonic rock +symphony +synthpop +synthpunk +t'ong guitar +taarab +tai tu +taiwanese pop +tala +talempong +tambu +tamburitza +tamil christian keerthanai +tango +tanguk +tappa +tarana +tarantella +taranto +tech +tech house +tech trance +technical death metal +technical metal +techno +technoid +technopop +techstep +techtonik +teen pop +tejano +tejano music +tekno +tembang sunda +texas blues +thai pop +thillana +thrash metal +thrashcore +thumri +tibetan pop +tiento +timbila +tin pan alley +tinga +tinku +toeshey +togaku +trad jazz +traditional bluegrass +traditional pop music +trallalero +trance +tribal house +trikitixa +trip hop +trip rock +trip-hop +tropicalia +tropicalismo +tropipop +truck-driving country +tumba +turbo-folk +turkish music +turkish pop +turntablism +tuvan throat-singing +twee pop +twist +two tone +táncház +uk garage +uk pub rock +unblack metal +underground music +uplifting +uplifting trance +urban cowboy +urban folk +urban jazz +vallenato +vaudeville +venezuela +verbunkos +verismo +viking metal +villanella +virelai +vispop +visual kei +visual music +vocal +vocal house +vocal jazz +vocal music +volksmusik +waila +waltz +wangga +warabe uta +wassoulou +weld +were music +west coast hip hop +west coast jazz +western +western blues +western swing +witch house +wizard rock +women's music +wong shadow +wonky pop +wood +work song +world fusion +world fusion music +world music +worldbeat +xhosa music +xoomii +yo-pop +yodeling +yukar +yé-yé +zajal +zapin +zarzuela +zeibekiko +zeuhl +ziglibithy +zouglou +zouk +zouk chouv +zouklove +zulu music +zydeco diff --git a/libs/beetsplug/lastimport.py b/libs/beetsplug/lastimport.py new file mode 100644 index 00000000..2d8cc700 --- /dev/null +++ b/libs/beetsplug/lastimport.py @@ -0,0 +1,249 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Rafael Bodill http://github.com/rafi +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +from __future__ import division, absolute_import, print_function + +import pylast +from pylast import TopItem, _extract, _number +from beets import ui +from beets import dbcore +from beets import config +from beets import plugins +from beets.dbcore import types + +API_URL = 'http://ws.audioscrobbler.com/2.0/' + + +class LastImportPlugin(plugins.BeetsPlugin): + def __init__(self): + super(LastImportPlugin, self).__init__() + config['lastfm'].add({ + 'user': '', + 'api_key': plugins.LASTFM_KEY, + }) + config['lastfm']['api_key'].redact = True + self.config.add({ + 'per_page': 500, + 'retry_limit': 3, + }) + self.item_types = { + 'play_count': types.INTEGER, + } + + def commands(self): + cmd = ui.Subcommand('lastimport', help=u'import last.fm play-count') + + def func(lib, opts, args): + import_lastfm(lib, self._log) + + cmd.func = func + return [cmd] + + +class CustomUser(pylast.User): + """ Custom user class derived from pylast.User, and overriding the + _get_things method to return MBID and album. Also introduces new + get_top_tracks_by_page method to allow access to more than one page of top + tracks. + """ + def __init__(self, *args, **kwargs): + super(CustomUser, self).__init__(*args, **kwargs) + + def _get_things(self, method, thing, thing_type, params=None, + cacheable=True): + """Returns a list of the most played thing_types by this thing, in a + tuple with the total number of pages of results. Includes an MBID, if + found. + """ + doc = self._request( + self.ws_prefix + "." + method, cacheable, params) + + toptracks_node = doc.getElementsByTagName('toptracks')[0] + total_pages = int(toptracks_node.getAttribute('totalPages')) + + seq = [] + for node in doc.getElementsByTagName(thing): + title = _extract(node, "name") + artist = _extract(node, "name", 1) + mbid = _extract(node, "mbid") + playcount = _number(_extract(node, "playcount")) + + thing = thing_type(artist, title, self.network) + thing.mbid = mbid + seq.append(TopItem(thing, playcount)) + + return seq, total_pages + + def get_top_tracks_by_page(self, period=pylast.PERIOD_OVERALL, limit=None, + page=1, cacheable=True): + """Returns the top tracks played by a user, in a tuple with the total + number of pages of results. + * period: The period of time. Possible values: + o PERIOD_OVERALL + o PERIOD_7DAYS + o PERIOD_1MONTH + o PERIOD_3MONTHS + o PERIOD_6MONTHS + o PERIOD_12MONTHS + """ + + params = self._get_params() + params['period'] = period + params['page'] = page + if limit: + params['limit'] = limit + + return self._get_things( + "getTopTracks", "track", pylast.Track, params, cacheable) + + +def import_lastfm(lib, log): + user = config['lastfm']['user'].get(unicode) + per_page = config['lastimport']['per_page'].get(int) + + if not user: + raise ui.UserError(u'You must specify a user name for lastimport') + + log.info(u'Fetching last.fm library for @{0}', user) + + page_total = 1 + page_current = 0 + found_total = 0 + unknown_total = 0 + retry_limit = config['lastimport']['retry_limit'].get(int) + # Iterate through a yet to be known page total count + while page_current < page_total: + log.info(u'Querying page #{0}{1}...', + page_current + 1, + '/{}'.format(page_total) if page_total > 1 else '') + + for retry in range(0, retry_limit): + tracks, page_total = fetch_tracks(user, page_current + 1, per_page) + if page_total < 1: + # It means nothing to us! + raise ui.UserError(u'Last.fm reported no data.') + + if tracks: + found, unknown = process_tracks(lib, tracks, log) + found_total += found + unknown_total += unknown + break + else: + log.error(u'ERROR: unable to read page #{0}', + page_current + 1) + if retry < retry_limit: + log.info( + u'Retrying page #{0}... ({1}/{2} retry)', + page_current + 1, retry + 1, retry_limit + ) + else: + log.error(u'FAIL: unable to fetch page #{0}, ', + u'tried {1} times', page_current, retry + 1) + page_current += 1 + + log.info(u'... done!') + log.info(u'finished processing {0} song pages', page_total) + log.info(u'{0} unknown play-counts', unknown_total) + log.info(u'{0} play-counts imported', found_total) + + +def fetch_tracks(user, page, limit): + """ JSON format: + [ + { + "mbid": "...", + "artist": "...", + "title": "...", + "playcount": "..." + } + ] + """ + network = pylast.LastFMNetwork(api_key=config['lastfm']['api_key']) + user_obj = CustomUser(user, network) + results, total_pages =\ + user_obj.get_top_tracks_by_page(limit=limit, page=page) + return [ + { + "mbid": track.item.mbid if track.item.mbid else '', + "artist": { + "name": track.item.artist.name + }, + "name": track.item.title, + "playcount": track.weight + } for track in results + ], total_pages + + +def process_tracks(lib, tracks, log): + total = len(tracks) + total_found = 0 + total_fails = 0 + log.info(u'Received {0} tracks in this page, processing...', total) + + for num in xrange(0, total): + song = None + trackid = tracks[num]['mbid'].strip() + artist = tracks[num]['artist'].get('name', '').strip() + title = tracks[num]['name'].strip() + album = '' + if 'album' in tracks[num]: + album = tracks[num]['album'].get('name', '').strip() + + log.debug(u'query: {0} - {1} ({2})', artist, title, album) + + # First try to query by musicbrainz's trackid + if trackid: + song = lib.items( + dbcore.query.MatchQuery('mb_trackid', trackid) + ).get() + + # If not, try just artist/title + if song is None: + log.debug(u'no album match, trying by artist/title') + query = dbcore.AndQuery([ + dbcore.query.SubstringQuery('artist', artist), + dbcore.query.SubstringQuery('title', title) + ]) + song = lib.items(query).get() + + # Last resort, try just replacing to utf-8 quote + if song is None: + title = title.replace("'", u'\u2019') + log.debug(u'no title match, trying utf-8 single quote') + query = dbcore.AndQuery([ + dbcore.query.SubstringQuery('artist', artist), + dbcore.query.SubstringQuery('title', title) + ]) + song = lib.items(query).get() + + if song is not None: + count = int(song.get('play_count', 0)) + new_count = int(tracks[num]['playcount']) + log.debug(u'match: {0} - {1} ({2}) ' + u'updating: play_count {3} => {4}', + song.artist, song.title, song.album, count, new_count) + song['play_count'] = new_count + song.store() + total_found += 1 + else: + total_fails += 1 + log.info(u' - No match: {0} - {1} ({2})', + artist, title, album) + + if total_fails > 0: + log.info(u'Acquired {0}/{1} play-counts ({2} unknown)', + total_found, total, total_fails) + + return total_found, total_fails diff --git a/libs/beetsplug/lyrics.py b/libs/beetsplug/lyrics.py new file mode 100644 index 00000000..b6936e1b --- /dev/null +++ b/libs/beetsplug/lyrics.py @@ -0,0 +1,760 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Adrian Sampson. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Fetches, embeds, and displays lyrics. +""" + +from __future__ import absolute_import, division, print_function + +import difflib +import itertools +import json +import re +import requests +import unicodedata +import urllib +import warnings +from HTMLParser import HTMLParseError + +try: + from bs4 import SoupStrainer, BeautifulSoup + HAS_BEAUTIFUL_SOUP = True +except ImportError: + HAS_BEAUTIFUL_SOUP = False + +try: + import langdetect + HAS_LANGDETECT = True +except ImportError: + HAS_LANGDETECT = False + +from beets import plugins +from beets import ui + + +DIV_RE = re.compile(r'<(/?)div>?', re.I) +COMMENT_RE = re.compile(r'<!--.*-->', re.S) +TAG_RE = re.compile(r'<[^>]*>') +BREAK_RE = re.compile(r'\n?\s*<br([\s|/][^>]*)*>\s*\n?', re.I) +URL_CHARACTERS = { + u'\u2018': u"'", + u'\u2019': u"'", + u'\u201c': u'"', + u'\u201d': u'"', + u'\u2010': u'-', + u'\u2011': u'-', + u'\u2012': u'-', + u'\u2013': u'-', + u'\u2014': u'-', + u'\u2015': u'-', + u'\u2016': u'-', + u'\u2026': u'...', +} + + +# Utilities. + + +def unescape(text): + """Resolve &#xxx; HTML entities (and some others).""" + if isinstance(text, bytes): + text = text.decode('utf8', 'ignore') + out = text.replace(u' ', u' ') + + def replchar(m): + num = m.group(1) + return unichr(int(num)) + out = re.sub(u"&#(\d+);", replchar, out) + return out + + +def extract_text_between(html, start_marker, end_marker): + try: + _, html = html.split(start_marker, 1) + html, _ = html.split(end_marker, 1) + except ValueError: + return u'' + return html + + +def extract_text_in(html, starttag): + """Extract the text from a <DIV> tag in the HTML starting with + ``starttag``. Returns None if parsing fails. + """ + + # Strip off the leading text before opening tag. + try: + _, html = html.split(starttag, 1) + except ValueError: + return + + # Walk through balanced DIV tags. + level = 0 + parts = [] + pos = 0 + for match in DIV_RE.finditer(html): + if match.group(1): # Closing tag. + level -= 1 + if level == 0: + pos = match.end() + else: # Opening tag. + if level == 0: + parts.append(html[pos:match.start()]) + level += 1 + + if level == -1: + parts.append(html[pos:match.start()]) + break + else: + print(u'no closing tag found!') + return + return u''.join(parts) + + +def search_pairs(item): + """Yield a pairs of artists and titles to search for. + + The first item in the pair is the name of the artist, the second + item is a list of song names. + + In addition to the artist and title obtained from the `item` the + method tries to strip extra information like paranthesized suffixes + and featured artists from the strings and add them as candidates. + The method also tries to split multiple titles separated with `/`. + """ + + title, artist = item.title, item.artist + titles = [title] + artists = [artist] + + # Remove any featuring artists from the artists name + pattern = r"(.*?) {0}".format(plugins.feat_tokens()) + match = re.search(pattern, artist, re.IGNORECASE) + if match: + artists.append(match.group(1)) + + # Remove a parenthesized suffix from a title string. Common + # examples include (live), (remix), and (acoustic). + pattern = r"(.+?)\s+[(].*[)]$" + match = re.search(pattern, title, re.IGNORECASE) + if match: + titles.append(match.group(1)) + + # Remove any featuring artists from the title + pattern = r"(.*?) {0}".format(plugins.feat_tokens(for_artist=False)) + for title in titles[:]: + match = re.search(pattern, title, re.IGNORECASE) + if match: + titles.append(match.group(1)) + + # Check for a dual song (e.g. Pink Floyd - Speak to Me / Breathe) + # and each of them. + multi_titles = [] + for title in titles: + multi_titles.append([title]) + if '/' in title: + multi_titles.append([x.strip() for x in title.split('/')]) + + return itertools.product(artists, multi_titles) + + +class Backend(object): + def __init__(self, config, log): + self._log = log + + @staticmethod + def _encode(s): + """Encode the string for inclusion in a URL""" + if isinstance(s, unicode): + for char, repl in URL_CHARACTERS.items(): + s = s.replace(char, repl) + s = s.encode('utf8', 'ignore') + return urllib.quote(s) + + def build_url(self, artist, title): + return self.URL_PATTERN % (self._encode(artist.title()), + self._encode(title.title())) + + def fetch_url(self, url): + """Retrieve the content at a given URL, or return None if the source + is unreachable. + """ + try: + # Disable the InsecureRequestWarning that comes from using + # `verify=false`. + # https://github.com/kennethreitz/requests/issues/2214 + # We're not overly worried about the NSA MITMing our lyrics scraper + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + r = requests.get(url, verify=False) + except requests.RequestException as exc: + self._log.debug(u'lyrics request failed: {0}', exc) + return + if r.status_code == requests.codes.ok: + return r.text + else: + self._log.debug(u'failed to fetch: {0} ({1})', url, r.status_code) + + def fetch(self, artist, title): + raise NotImplementedError() + + +class SymbolsReplaced(Backend): + REPLACEMENTS = { + r'\s+': '_', + '<': 'Less_Than', + '>': 'Greater_Than', + '#': 'Number_', + r'[\[\{]': '(', + r'[\[\{]': ')' + } + + @classmethod + def _encode(cls, s): + for old, new in cls.REPLACEMENTS.iteritems(): + s = re.sub(old, new, s) + + return super(SymbolsReplaced, cls)._encode(s) + + +class MusiXmatch(SymbolsReplaced): + REPLACEMENTS = dict(SymbolsReplaced.REPLACEMENTS, **{ + r'\s+': '-' + }) + + URL_PATTERN = 'https://www.musixmatch.com/lyrics/%s/%s' + + def fetch(self, artist, title): + url = self.build_url(artist, title) + html = self.fetch_url(url) + if not html: + return + lyrics = extract_text_between(html, + '"body":', '"language":') + return lyrics.strip(',"').replace('\\n', '\n') + + +class Genius(Backend): + """Fetch lyrics from Genius via genius-api.""" + def __init__(self, config, log): + super(Genius, self).__init__(config, log) + self.api_key = config['genius_api_key'].get(unicode) + self.headers = {'Authorization': "Bearer %s" % self.api_key} + + def search_genius(self, artist, title): + query = u"%s %s" % (artist, title) + url = u'https://api.genius.com/search?q=%s' \ + % (urllib.quote(query.encode('utf8'))) + + self._log.debug(u'genius: requesting search {}', url) + try: + req = requests.get( + url, + headers=self.headers, + allow_redirects=True + ) + req.raise_for_status() + except requests.RequestException as exc: + self._log.debug(u'genius: request error: {}', exc) + return None + + try: + return req.json() + except ValueError: + self._log.debug(u'genius: invalid response: {}', req.text) + return None + + def get_lyrics(self, link): + url = u'http://genius-api.com/api/lyricsInfo' + + self._log.debug(u'genius: requesting lyrics for link {}', link) + try: + req = requests.post( + url, + data={'link': link}, + headers=self.headers, + allow_redirects=True + ) + req.raise_for_status() + except requests.RequestException as exc: + self._log.debug(u'genius: request error: {}', exc) + return None + + try: + return req.json() + except ValueError: + self._log.debug(u'genius: invalid response: {}', req.text) + return None + + def build_lyric_string(self, lyrics): + if 'lyrics' not in lyrics: + return + sections = lyrics['lyrics']['sections'] + + lyrics_list = [] + for section in sections: + lyrics_list.append(section['name']) + lyrics_list.append('\n') + for verse in section['verses']: + if 'content' in verse: + lyrics_list.append(verse['content']) + + return ''.join(lyrics_list) + + def fetch(self, artist, title): + search_data = self.search_genius(artist, title) + if not search_data: + return + + if not search_data['meta']['status'] == 200: + return + else: + records = search_data['response']['hits'] + if not records: + return + + record_url = records[0]['result']['url'] + lyric_data = self.get_lyrics(record_url) + if not lyric_data: + return + lyrics = self.build_lyric_string(lyric_data) + + return lyrics + + +class LyricsWiki(SymbolsReplaced): + """Fetch lyrics from LyricsWiki.""" + URL_PATTERN = 'http://lyrics.wikia.com/%s:%s' + + def fetch(self, artist, title): + url = self.build_url(artist, title) + html = self.fetch_url(url) + if not html: + return + + # Get the HTML fragment inside the appropriate HTML element and then + # extract the text from it. + html_frag = extract_text_in(html, u"<div class='lyricbox'>") + if html_frag: + lyrics = _scrape_strip_cruft(html_frag, True) + + if lyrics and 'Unfortunately, we are not licensed' not in lyrics: + return lyrics + + +class LyricsCom(Backend): + """Fetch lyrics from Lyrics.com.""" + URL_PATTERN = 'http://www.lyrics.com/%s-lyrics-%s.html' + NOT_FOUND = ( + 'Sorry, we do not have the lyric', + 'Submit Lyrics', + ) + + @classmethod + def _encode(cls, s): + s = re.sub(r'[^\w\s-]', '', s) + s = re.sub(r'\s+', '-', s) + return super(LyricsCom, cls)._encode(s).lower() + + def fetch(self, artist, title): + url = self.build_url(artist, title) + html = self.fetch_url(url) + if not html: + return + lyrics = extract_text_between(html, '<div id="lyrics" class="SCREENO' + 'NLY" itemprop="description">', '</div>') + if not lyrics: + return + for not_found_str in self.NOT_FOUND: + if not_found_str in lyrics: + return + + parts = lyrics.split('\n---\nLyrics powered by', 1) + if parts: + return parts[0] + + +def remove_credits(text): + """Remove first/last line of text if it contains the word 'lyrics' + eg 'Lyrics by songsdatabase.com' + """ + textlines = text.split('\n') + credits = None + for i in (0, -1): + if textlines and 'lyrics' in textlines[i].lower(): + credits = textlines.pop(i) + if credits: + text = '\n'.join(textlines) + return text + + +def _scrape_strip_cruft(html, plain_text_out=False): + """Clean up HTML + """ + html = unescape(html) + + html = html.replace('\r', '\n') # Normalize EOL. + html = re.sub(r' +', ' ', html) # Whitespaces collapse. + html = BREAK_RE.sub('\n', html) # <br> eats up surrounding '\n'. + html = re.sub(r'<(script).*?</\1>(?s)', '', html) # Strip script tags. + + if plain_text_out: # Strip remaining HTML tags + html = COMMENT_RE.sub('', html) + html = TAG_RE.sub('', html) + + html = '\n'.join([x.strip() for x in html.strip().split('\n')]) + html = re.sub(r'\n{3,}', r'\n\n', html) + return html + + +def _scrape_merge_paragraphs(html): + html = re.sub(r'</p>\s*<p(\s*[^>]*)>', '\n', html) + return re.sub(r'<div .*>\s*</div>', '\n', html) + + +def scrape_lyrics_from_html(html): + """Scrape lyrics from a URL. If no lyrics can be found, return None + instead. + """ + if not HAS_BEAUTIFUL_SOUP: + return None + + if not html: + return None + + def is_text_notcode(text): + length = len(text) + return (length > 20 and + text.count(' ') > length / 25 and + (text.find('{') == -1 or text.find(';') == -1)) + html = _scrape_strip_cruft(html) + html = _scrape_merge_paragraphs(html) + + # extract all long text blocks that are not code + try: + soup = BeautifulSoup(html, "html.parser", + parse_only=SoupStrainer(text=is_text_notcode)) + except HTMLParseError: + return None + + # Get the longest text element (if any). + strings = sorted(soup.stripped_strings, key=len, reverse=True) + if strings: + return strings[0] + else: + return None + + +class Google(Backend): + """Fetch lyrics from Google search results.""" + def __init__(self, config, log): + super(Google, self).__init__(config, log) + self.api_key = config['google_API_key'].get(unicode) + self.engine_id = config['google_engine_ID'].get(unicode) + + def is_lyrics(self, text, artist=None): + """Determine whether the text seems to be valid lyrics. + """ + if not text: + return False + bad_triggers_occ = [] + nb_lines = text.count('\n') + if nb_lines <= 1: + self._log.debug(u"Ignoring too short lyrics '{0}'", text) + return False + elif nb_lines < 5: + bad_triggers_occ.append('too_short') + else: + # Lyrics look legit, remove credits to avoid being penalized + # further down + text = remove_credits(text) + + bad_triggers = ['lyrics', 'copyright', 'property', 'links'] + if artist: + bad_triggers_occ += [artist] + + for item in bad_triggers: + bad_triggers_occ += [item] * len(re.findall(r'\W%s\W' % item, + text, re.I)) + + if bad_triggers_occ: + self._log.debug(u'Bad triggers detected: {0}', bad_triggers_occ) + return len(bad_triggers_occ) < 2 + + def slugify(self, text): + """Normalize a string and remove non-alphanumeric characters. + """ + text = re.sub(r"[-'_\s]", '_', text) + text = re.sub(r"_+", '_', text).strip('_') + pat = "([^,\(]*)\((.*?)\)" # Remove content within parentheses + text = re.sub(pat, '\g<1>', text).strip() + try: + text = unicodedata.normalize('NFKD', text).encode('ascii', + 'ignore') + text = unicode(re.sub('[-\s]+', ' ', text)) + except UnicodeDecodeError: + self._log.exception(u"Failing to normalize '{0}'", text) + return text + + BY_TRANS = ['by', 'par', 'de', 'von'] + LYRICS_TRANS = ['lyrics', 'paroles', 'letras', 'liedtexte'] + + def is_page_candidate(self, url_link, url_title, title, artist): + """Return True if the URL title makes it a good candidate to be a + page that contains lyrics of title by artist. + """ + title = self.slugify(title.lower()) + artist = self.slugify(artist.lower()) + sitename = re.search(u"//([^/]+)/.*", + self.slugify(url_link.lower())).group(1) + url_title = self.slugify(url_title.lower()) + + # Check if URL title contains song title (exact match) + if url_title.find(title) != -1: + return True + + # or try extracting song title from URL title and check if + # they are close enough + tokens = [by + '_' + artist for by in self.BY_TRANS] + \ + [artist, sitename, sitename.replace('www.', '')] + \ + self.LYRICS_TRANS + tokens = [re.escape(t) for t in tokens] + song_title = re.sub(u'(%s)' % u'|'.join(tokens), u'', url_title) + + song_title = song_title.strip('_|') + typo_ratio = .9 + ratio = difflib.SequenceMatcher(None, song_title, title).ratio() + return ratio >= typo_ratio + + def fetch(self, artist, title): + query = u"%s %s" % (artist, title) + url = u'https://www.googleapis.com/customsearch/v1?key=%s&cx=%s&q=%s' \ + % (self.api_key, self.engine_id, + urllib.quote(query.encode('utf8'))) + + data = urllib.urlopen(url) + data = json.load(data) + if 'error' in data: + reason = data['error']['errors'][0]['reason'] + self._log.debug(u'google lyrics backend error: {0}', reason) + return + + if 'items' in data.keys(): + for item in data['items']: + url_link = item['link'] + url_title = item.get('title', u'') + if not self.is_page_candidate(url_link, url_title, + title, artist): + continue + html = self.fetch_url(url_link) + lyrics = scrape_lyrics_from_html(html) + if not lyrics: + continue + + if self.is_lyrics(lyrics, artist): + self._log.debug(u'got lyrics from {0}', + item['displayLink']) + return lyrics + + +class LyricsPlugin(plugins.BeetsPlugin): + SOURCES = ['google', 'lyricwiki', 'lyrics.com', 'musixmatch'] + SOURCE_BACKENDS = { + 'google': Google, + 'lyricwiki': LyricsWiki, + 'lyrics.com': LyricsCom, + 'musixmatch': MusiXmatch, + 'genius': Genius, + } + + def __init__(self): + super(LyricsPlugin, self).__init__() + self.import_stages = [self.imported] + self.config.add({ + 'auto': True, + 'bing_client_secret': None, + 'bing_lang_from': [], + 'bing_lang_to': None, + 'google_API_key': None, + 'google_engine_ID': u'009217259823014548361:lndtuqkycfu', + 'genius_api_key': + "Ryq93pUGm8bM6eUWwD_M3NOFFDAtp2yEE7W" + "76V-uFL5jks5dNvcGCdarqFjDhP9c", + 'fallback': None, + 'force': False, + 'sources': self.SOURCES, + }) + self.config['bing_client_secret'].redact = True + self.config['google_API_key'].redact = True + self.config['google_engine_ID'].redact = True + self.config['genius_api_key'].redact = True + + available_sources = list(self.SOURCES) + sources = plugins.sanitize_choices( + self.config['sources'].as_str_seq(), available_sources) + + if 'google' in sources: + if not self.config['google_API_key'].get(): + self._log.warn(u'To use the google lyrics source, you must ' + u'provide an API key in the configuration. ' + u'See the documentation for further details.') + sources.remove('google') + if not HAS_BEAUTIFUL_SOUP: + self._log.warn(u'To use the google lyrics source, you must ' + u'install the beautifulsoup4 module. See the ' + u'documentation for further details.') + sources.remove('google') + + self.config['bing_lang_from'] = [ + x.lower() for x in self.config['bing_lang_from'].as_str_seq()] + self.bing_auth_token = None + + if not HAS_LANGDETECT and self.config['bing_client_secret'].get(): + self._log.warn(u'To use bing translations, you need to ' + u'install the langdetect module. See the ' + u'documentation for further details.') + + self.backends = [self.SOURCE_BACKENDS[source](self.config, self._log) + for source in sources] + + def get_bing_access_token(self): + params = { + 'client_id': 'beets', + 'client_secret': self.config['bing_client_secret'], + 'scope': 'http://api.microsofttranslator.com', + 'grant_type': 'client_credentials', + } + + oauth_url = 'https://datamarket.accesscontrol.windows.net/v2/OAuth2-13' + oauth_token = json.loads(requests.post( + oauth_url, + data=urllib.urlencode(params)).content) + if 'access_token' in oauth_token: + return "Bearer " + oauth_token['access_token'] + else: + self._log.warning(u'Could not get Bing Translate API access token.' + u' Check your "bing_client_secret" password') + + def commands(self): + cmd = ui.Subcommand('lyrics', help='fetch song lyrics') + cmd.parser.add_option( + u'-p', u'--print', dest='printlyr', + action='store_true', default=False, + help=u'print lyrics to console', + ) + cmd.parser.add_option( + u'-f', u'--force', dest='force_refetch', + action='store_true', default=False, + help=u'always re-download lyrics', + ) + + def func(lib, opts, args): + # The "write to files" option corresponds to the + # import_write config value. + write = ui.should_write() + for item in lib.items(ui.decargs(args)): + self.fetch_item_lyrics( + lib, item, write, + opts.force_refetch or self.config['force'], + ) + if opts.printlyr and item.lyrics: + ui.print_(item.lyrics) + + cmd.func = func + return [cmd] + + def imported(self, session, task): + """Import hook for fetching lyrics automatically. + """ + if self.config['auto']: + for item in task.imported_items(): + self.fetch_item_lyrics(session.lib, item, + False, self.config['force']) + + def fetch_item_lyrics(self, lib, item, write, force): + """Fetch and store lyrics for a single item. If ``write``, then the + lyrics will also be written to the file itself.""" + # Skip if the item already has lyrics. + if not force and item.lyrics: + self._log.info(u'lyrics already present: {0}', item) + return + + lyrics = None + for artist, titles in search_pairs(item): + lyrics = [self.get_lyrics(artist, title) for title in titles] + if any(lyrics): + break + + lyrics = u"\n\n---\n\n".join([l for l in lyrics if l]) + + if lyrics: + self._log.info(u'fetched lyrics: {0}', item) + if HAS_LANGDETECT and self.config['bing_client_secret'].get(): + lang_from = langdetect.detect(lyrics) + if self.config['bing_lang_to'].get() != lang_from and ( + not self.config['bing_lang_from'] or ( + lang_from in self.config[ + 'bing_lang_from'].as_str_seq())): + lyrics = self.append_translation( + lyrics, self.config['bing_lang_to']) + else: + self._log.info(u'lyrics not found: {0}', item) + fallback = self.config['fallback'].get() + if fallback: + lyrics = fallback + else: + return + item.lyrics = lyrics + if write: + item.try_write() + item.store() + + def get_lyrics(self, artist, title): + """Fetch lyrics, trying each source in turn. Return a string or + None if no lyrics were found. + """ + for backend in self.backends: + lyrics = backend.fetch(artist, title) + if lyrics: + self._log.debug(u'got lyrics from backend: {0}', + backend.__class__.__name__) + return _scrape_strip_cruft(lyrics, True) + + def append_translation(self, text, to_lang): + import xml.etree.ElementTree as ET + + if not self.bing_auth_token: + self.bing_auth_token = self.get_bing_access_token() + if self.bing_auth_token: + # Extract unique lines to limit API request size per song + text_lines = set(text.split('\n')) + url = ('http://api.microsofttranslator.com/v2/Http.svc/' + 'Translate?text=%s&to=%s' % ('|'.join(text_lines), to_lang)) + r = requests.get(url, + headers={"Authorization ": self.bing_auth_token}) + if r.status_code != 200: + self._log.debug('translation API error {}: {}', r.status_code, + r.text) + if 'token has expired' in r.text: + self.bing_auth_token = None + return self.append_translation(text, to_lang) + return text + lines_translated = ET.fromstring(r.text.encode('utf8')).text + # Use a translation mapping dict to build resulting lyrics + translations = dict(zip(text_lines, lines_translated.split('|'))) + result = '' + for line in text.split('\n'): + result += '%s / %s\n' % (line, translations[line]) + return result diff --git a/libs/beetsplug/mbcollection.py b/libs/beetsplug/mbcollection.py new file mode 100644 index 00000000..b95ba6fe --- /dev/null +++ b/libs/beetsplug/mbcollection.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2011, Jeffrey Aylesworth <jeffrey@aylesworth.ca> +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import division, absolute_import, print_function + +from beets.plugins import BeetsPlugin +from beets.ui import Subcommand +from beets import ui +from beets import config +import musicbrainzngs + +import re + +SUBMISSION_CHUNK_SIZE = 200 +UUID_REGEX = r'^[a-f0-9]{8}(-[a-f0-9]{4}){3}-[a-f0-9]{12}$' + + +def mb_call(func, *args, **kwargs): + """Call a MusicBrainz API function and catch exceptions. + """ + try: + return func(*args, **kwargs) + except musicbrainzngs.AuthenticationError: + raise ui.UserError(u'authentication with MusicBrainz failed') + except (musicbrainzngs.ResponseError, musicbrainzngs.NetworkError) as exc: + raise ui.UserError(u'MusicBrainz API error: {0}'.format(exc)) + except musicbrainzngs.UsageError: + raise ui.UserError(u'MusicBrainz credentials missing') + + +def submit_albums(collection_id, release_ids): + """Add all of the release IDs to the indicated collection. Multiple + requests are made if there are many release IDs to submit. + """ + for i in range(0, len(release_ids), SUBMISSION_CHUNK_SIZE): + chunk = release_ids[i:i + SUBMISSION_CHUNK_SIZE] + mb_call( + musicbrainzngs.add_releases_to_collection, + collection_id, chunk + ) + + +class MusicBrainzCollectionPlugin(BeetsPlugin): + def __init__(self): + super(MusicBrainzCollectionPlugin, self).__init__() + config['musicbrainz']['pass'].redact = True + musicbrainzngs.auth( + config['musicbrainz']['user'].get(unicode), + config['musicbrainz']['pass'].get(unicode), + ) + self.config.add({'auto': False}) + if self.config['auto']: + self.import_stages = [self.imported] + + def commands(self): + mbupdate = Subcommand('mbupdate', + help=u'Update MusicBrainz collection') + mbupdate.func = self.update_collection + return [mbupdate] + + def update_collection(self, lib, opts, args): + self.update_album_list(lib.albums()) + + def imported(self, session, task): + """Add each imported album to the collection. + """ + if task.is_album: + self.update_album_list([task.album]) + + def update_album_list(self, album_list): + """Update the MusicBrainz colleciton from a list of Beets albums + """ + # Get the available collections. + collections = mb_call(musicbrainzngs.get_collections) + if not collections['collection-list']: + raise ui.UserError(u'no collections exist for user') + + # Get the first release collection. MusicBrainz also has event + # collections, so we need to avoid adding to those. + for collection in collections['collection-list']: + if 'release-count' in collection: + collection_id = collection['id'] + break + else: + raise ui.UserError(u'No collection found.') + + # Get a list of all the album IDs. + album_ids = [] + for album in album_list: + aid = album.mb_albumid + if aid: + if re.match(UUID_REGEX, aid): + album_ids.append(aid) + else: + self._log.info(u'skipping invalid MBID: {0}', aid) + + # Submit to MusicBrainz. + self._log.info( + u'Updating MusicBrainz collection {0}...', collection_id + ) + submit_albums(collection_id, album_ids) + self._log.info(u'...MusicBrainz collection updated.') diff --git a/libs/beetsplug/mbsubmit.py b/libs/beetsplug/mbsubmit.py new file mode 100644 index 00000000..91de6128 --- /dev/null +++ b/libs/beetsplug/mbsubmit.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Adrian Sampson and Diego Moreda. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Aid in submitting information to MusicBrainz. + +This plugin allows the user to print track information in a format that is +parseable by the MusicBrainz track parser [1]. Programmatic submitting is not +implemented by MusicBrainz yet. + +[1] http://wiki.musicbrainz.org/History:How_To_Parse_Track_Listings +""" + +from __future__ import division, absolute_import, print_function + + +from beets.autotag import Recommendation +from beets.plugins import BeetsPlugin +from beets.ui.commands import PromptChoice +from beetsplug.info import print_data + + +class MBSubmitPlugin(BeetsPlugin): + def __init__(self): + super(MBSubmitPlugin, self).__init__() + + self.config.add({ + 'format': '$track. $title - $artist ($length)', + 'threshold': 'medium', + }) + + # Validate and store threshold. + self.threshold = self.config['threshold'].as_choice({ + 'none': Recommendation.none, + 'low': Recommendation.low, + 'medium': Recommendation.medium, + 'strong': Recommendation.strong + }) + + self.register_listener('before_choose_candidate', + self.before_choose_candidate_event) + + def before_choose_candidate_event(self, session, task): + if task.rec <= self.threshold: + return [PromptChoice(u'p', u'Print tracks', self.print_tracks)] + + def print_tracks(self, session, task): + for i in task.items: + print_data(None, i, self.config['format'].get()) diff --git a/libs/beetsplug/mbsync.py b/libs/beetsplug/mbsync.py new file mode 100644 index 00000000..cf58c82d --- /dev/null +++ b/libs/beetsplug/mbsync.py @@ -0,0 +1,167 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Jakob Schnitzer. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Update library's tags using MusicBrainz. +""" +from __future__ import division, absolute_import, print_function + +from beets.plugins import BeetsPlugin +from beets import autotag, library, ui, util +from beets.autotag import hooks +from collections import defaultdict + + +def apply_item_changes(lib, item, move, pretend, write): + """Store, move and write the item according to the arguments. + """ + if not pretend: + # Move the item if it's in the library. + if move and lib.directory in util.ancestry(item.path): + item.move(with_album=False) + + if write: + item.try_write() + item.store() + + +class MBSyncPlugin(BeetsPlugin): + def __init__(self): + super(MBSyncPlugin, self).__init__() + + def commands(self): + cmd = ui.Subcommand('mbsync', + help=u'update metadata from musicbrainz') + cmd.parser.add_option( + u'-p', u'--pretend', action='store_true', + help=u'show all changes but do nothing') + cmd.parser.add_option( + u'-m', u'--move', action='store_true', dest='move', + help=u"move files in the library directory") + cmd.parser.add_option( + u'-M', u'--nomove', action='store_false', dest='move', + help=u"don't move files in library") + cmd.parser.add_option( + u'-W', u'--nowrite', action='store_false', + default=None, dest='write', + help=u"don't write updated metadata to files") + cmd.parser.add_format_option() + cmd.func = self.func + return [cmd] + + def func(self, lib, opts, args): + """Command handler for the mbsync function. + """ + move = ui.should_move(opts.move) + pretend = opts.pretend + write = ui.should_write(opts.write) + query = ui.decargs(args) + + self.singletons(lib, query, move, pretend, write) + self.albums(lib, query, move, pretend, write) + + def singletons(self, lib, query, move, pretend, write): + """Retrieve and apply info from the autotagger for items matched by + query. + """ + for item in lib.items(query + [u'singleton:true']): + item_formatted = format(item) + if not item.mb_trackid: + self._log.info(u'Skipping singleton with no mb_trackid: {0}', + item_formatted) + continue + + # Get the MusicBrainz recording info. + track_info = hooks.track_for_mbid(item.mb_trackid) + if not track_info: + self._log.info(u'Recording ID not found: {0} for track {0}', + item.mb_trackid, + item_formatted) + continue + + # Apply. + with lib.transaction(): + autotag.apply_item_metadata(item, track_info) + apply_item_changes(lib, item, move, pretend, write) + + def albums(self, lib, query, move, pretend, write): + """Retrieve and apply info from the autotagger for albums matched by + query and their items. + """ + # Process matching albums. + for a in lib.albums(query): + album_formatted = format(a) + if not a.mb_albumid: + self._log.info(u'Skipping album with no mb_albumid: {0}', + album_formatted) + continue + + items = list(a.items()) + + # Get the MusicBrainz album information. + album_info = hooks.album_for_mbid(a.mb_albumid) + if not album_info: + self._log.info(u'Release ID {0} not found for album {1}', + a.mb_albumid, + album_formatted) + continue + + # Map recording MBIDs to their information. Recordings can appear + # multiple times on a release, so each MBID maps to a list of + # TrackInfo objects. + track_index = defaultdict(list) + for track_info in album_info.tracks: + track_index[track_info.track_id].append(track_info) + + # Construct a track mapping according to MBIDs. This should work + # for albums that have missing or extra tracks. If there are + # multiple copies of a recording, they are disambiguated using + # their disc and track number. + mapping = {} + for item in items: + candidates = track_index[item.mb_trackid] + if len(candidates) == 1: + mapping[item] = candidates[0] + else: + for c in candidates: + if (c.medium_index == item.track and + c.medium == item.disc): + mapping[item] = c + break + + # Apply. + self._log.debug(u'applying changes to {}', album_formatted) + with lib.transaction(): + autotag.apply_metadata(album_info, mapping) + changed = False + for item in items: + item_changed = ui.show_model_changes(item) + changed |= item_changed + if item_changed: + apply_item_changes(lib, item, move, pretend, write) + + if not changed: + # No change to any item. + continue + + if not pretend: + # Update album structure to reflect an item in it. + for key in library.Album.item_keys: + a[key] = items[0][key] + a.store() + + # Move album art (and any inconsistent items). + if move and lib.directory in util.ancestry(items[0].path): + self._log.debug(u'moving album {0}', album_formatted) + a.move() diff --git a/libs/beetsplug/metasync/__init__.py b/libs/beetsplug/metasync/__init__.py new file mode 100644 index 00000000..3fc0be4c --- /dev/null +++ b/libs/beetsplug/metasync/__init__.py @@ -0,0 +1,142 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Heinz Wiesinger. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Synchronize information from music player libraries +""" + +from __future__ import division, absolute_import, print_function + +from abc import abstractmethod, ABCMeta +from importlib import import_module + +from beets.util.confit import ConfigValueError +from beets import ui +from beets.plugins import BeetsPlugin + + +METASYNC_MODULE = 'beetsplug.metasync' + +# Dictionary to map the MODULE and the CLASS NAME of meta sources +SOURCES = { + 'amarok': 'Amarok', + 'itunes': 'Itunes', +} + + +class MetaSource(object): + __metaclass__ = ABCMeta + + def __init__(self, config, log): + self.item_types = {} + self.config = config + self._log = log + + @abstractmethod + def sync_from_source(self, item): + pass + + +def load_meta_sources(): + """ Returns a dictionary of all the MetaSources + E.g., {'itunes': Itunes} with isinstance(Itunes, MetaSource) true + """ + meta_sources = {} + + for module_path, class_name in SOURCES.items(): + module = import_module(METASYNC_MODULE + '.' + module_path) + meta_sources[class_name.lower()] = getattr(module, class_name) + + return meta_sources + + +META_SOURCES = load_meta_sources() + + +def load_item_types(): + """ Returns a dictionary containing the item_types of all the MetaSources + """ + item_types = {} + for meta_source in META_SOURCES.values(): + item_types.update(meta_source.item_types) + return item_types + + +class MetaSyncPlugin(BeetsPlugin): + + item_types = load_item_types() + + def __init__(self): + super(MetaSyncPlugin, self).__init__() + + def commands(self): + cmd = ui.Subcommand('metasync', + help='update metadata from music player libraries') + cmd.parser.add_option('-p', '--pretend', action='store_true', + help='show all changes but do nothing') + cmd.parser.add_option('-s', '--source', default=[], + action='append', dest='sources', + help='comma-separated list of sources to sync') + cmd.parser.add_format_option() + cmd.func = self.func + return [cmd] + + def func(self, lib, opts, args): + """Command handler for the metasync function. + """ + pretend = opts.pretend + query = ui.decargs(args) + + sources = [] + for source in opts.sources: + sources.extend(source.split(',')) + + sources = sources or self.config['source'].as_str_seq() + + meta_source_instances = {} + items = lib.items(query) + + # Avoid needlessly instantiating meta sources (can be expensive) + if not items: + self._log.info(u'No items found matching query') + return + + # Instantiate the meta sources + for player in sources: + try: + cls = META_SOURCES[player] + except KeyError: + self._log.error(u'Unknown metadata source \'{0}\''.format( + player)) + + try: + meta_source_instances[player] = cls(self.config, self._log) + except (ImportError, ConfigValueError) as e: + self._log.error(u'Failed to instantiate metadata source ' + u'\'{0}\': {1}'.format(player, e)) + + # Avoid needlessly iterating over items + if not meta_source_instances: + self._log.error(u'No valid metadata sources found') + return + + # Sync the items with all of the meta sources + for item in items: + for meta_source in meta_source_instances.values(): + meta_source.sync_from_source(item) + + changed = ui.show_model_changes(item) + + if changed and not pretend: + item.store() diff --git a/libs/beetsplug/metasync/amarok.py b/libs/beetsplug/metasync/amarok.py new file mode 100644 index 00000000..aaa1ee91 --- /dev/null +++ b/libs/beetsplug/metasync/amarok.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Heinz Wiesinger. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Synchronize information from amarok's library via dbus +""" + +from __future__ import division, absolute_import, print_function + +from os.path import basename +from datetime import datetime +from time import mktime +from xml.sax.saxutils import escape + +from beets.util import displayable_path +from beets.dbcore import types +from beets.library import DateType +from beetsplug.metasync import MetaSource + + +def import_dbus(): + try: + return __import__('dbus') + except ImportError: + return None + +dbus = import_dbus() + + +class Amarok(MetaSource): + + item_types = { + 'amarok_rating': types.INTEGER, + 'amarok_score': types.FLOAT, + 'amarok_uid': types.STRING, + 'amarok_playcount': types.INTEGER, + 'amarok_firstplayed': DateType(), + 'amarok_lastplayed': DateType(), + } + + queryXML = u'<query version="1.0"> \ + <filters> \ + <and><include field="filename" value="%s" /></and> \ + </filters> \ + </query>' + + def __init__(self, config, log): + super(Amarok, self).__init__(config, log) + + if not dbus: + raise ImportError('failed to import dbus') + + self.collection = \ + dbus.SessionBus().get_object('org.kde.amarok', '/Collection') + + def sync_from_source(self, item): + path = displayable_path(item.path) + + # amarok unfortunately doesn't allow searching for the full path, only + # for the patch relative to the mount point. But the full path is part + # of the result set. So query for the filename and then try to match + # the correct item from the results we get back + results = self.collection.Query(self.queryXML % escape(basename(path))) + for result in results: + if result['xesam:url'] != path: + continue + + item.amarok_rating = result['xesam:userRating'] + item.amarok_score = result['xesam:autoRating'] + item.amarok_playcount = result['xesam:useCount'] + item.amarok_uid = \ + result['xesam:id'].replace('amarok-sqltrackuid://', '') + + if result['xesam:firstUsed'][0][0] != 0: + # These dates are stored as timestamps in amarok's db, but + # exposed over dbus as fixed integers in the current timezone. + first_played = datetime( + result['xesam:firstUsed'][0][0], + result['xesam:firstUsed'][0][1], + result['xesam:firstUsed'][0][2], + result['xesam:firstUsed'][1][0], + result['xesam:firstUsed'][1][1], + result['xesam:firstUsed'][1][2] + ) + + if result['xesam:lastUsed'][0][0] != 0: + last_played = datetime( + result['xesam:lastUsed'][0][0], + result['xesam:lastUsed'][0][1], + result['xesam:lastUsed'][0][2], + result['xesam:lastUsed'][1][0], + result['xesam:lastUsed'][1][1], + result['xesam:lastUsed'][1][2] + ) + else: + last_played = first_played + + item.amarok_firstplayed = mktime(first_played.timetuple()) + item.amarok_lastplayed = mktime(last_played.timetuple()) diff --git a/libs/beetsplug/metasync/itunes.py b/libs/beetsplug/metasync/itunes.py new file mode 100644 index 00000000..a6274684 --- /dev/null +++ b/libs/beetsplug/metasync/itunes.py @@ -0,0 +1,121 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Tom Jaspers. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Synchronize information from iTunes's library +""" + +from __future__ import division, absolute_import, print_function + +from contextlib import contextmanager +import os +import shutil +import tempfile +import plistlib +import urllib +from urlparse import urlparse +from time import mktime + +from beets import util +from beets.dbcore import types +from beets.library import DateType +from beets.util.confit import ConfigValueError +from beetsplug.metasync import MetaSource + + +@contextmanager +def create_temporary_copy(path): + temp_dir = tempfile.mkdtemp() + temp_path = os.path.join(temp_dir, 'temp_itunes_lib') + shutil.copyfile(path, temp_path) + try: + yield temp_path + finally: + shutil.rmtree(temp_dir) + + +def _norm_itunes_path(path): + # Itunes prepends the location with 'file://' on posix systems, + # and with 'file://localhost/' on Windows systems. + # The actual path to the file is always saved as posix form + # E.g., 'file://Users/Music/bar' or 'file://localhost/G:/Music/bar' + + # The entire path will also be capitalized (e.g., '/Music/Alt-J') + # Note that this means the path will always have a leading separator, + # which is unwanted in the case of Windows systems. + # E.g., '\\G:\\Music\\bar' needs to be stripped to 'G:\\Music\\bar' + + return util.bytestring_path(os.path.normpath( + urllib.unquote(urlparse(path).path)).lstrip('\\')).lower() + + +class Itunes(MetaSource): + + item_types = { + 'itunes_rating': types.INTEGER, # 0..100 scale + 'itunes_playcount': types.INTEGER, + 'itunes_skipcount': types.INTEGER, + 'itunes_lastplayed': DateType(), + 'itunes_lastskipped': DateType(), + } + + def __init__(self, config, log): + super(Itunes, self).__init__(config, log) + + config.add({'itunes': { + 'library': '~/Music/iTunes/iTunes Library.xml' + }}) + + # Load the iTunes library, which has to be the .xml one (not the .itl) + library_path = config['itunes']['library'].as_filename() + + try: + self._log.debug( + u'loading iTunes library from {0}'.format(library_path)) + with create_temporary_copy(library_path) as library_copy: + raw_library = plistlib.readPlist(library_copy) + except IOError as e: + raise ConfigValueError(u'invalid iTunes library: ' + e.strerror) + except Exception: + # It's likely the user configured their '.itl' library (<> xml) + if os.path.splitext(library_path)[1].lower() != '.xml': + hint = u': please ensure that the configured path' \ + u' points to the .XML library' + else: + hint = '' + raise ConfigValueError(u'invalid iTunes library' + hint) + + # Make the iTunes library queryable using the path + self.collection = {_norm_itunes_path(track['Location']): track + for track in raw_library['Tracks'].values() + if 'Location' in track} + + def sync_from_source(self, item): + result = self.collection.get(util.bytestring_path(item.path).lower()) + + if not result: + self._log.warning(u'no iTunes match found for {0}'.format(item)) + return + + item.itunes_rating = result.get('Rating') + item.itunes_playcount = result.get('Play Count') + item.itunes_skipcount = result.get('Skip Count') + + if result.get('Play Date UTC'): + item.itunes_lastplayed = mktime( + result.get('Play Date UTC').timetuple()) + + if result.get('Skip Date'): + item.itunes_lastskipped = mktime( + result.get('Skip Date').timetuple()) diff --git a/libs/beetsplug/missing.py b/libs/beetsplug/missing.py new file mode 100644 index 00000000..8fff659f --- /dev/null +++ b/libs/beetsplug/missing.py @@ -0,0 +1,146 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Pedro Silva. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""List missing tracks. +""" +from __future__ import division, absolute_import, print_function + +from beets.autotag import hooks +from beets.library import Item +from beets.plugins import BeetsPlugin +from beets.ui import decargs, print_, Subcommand +from beets import config + + +def _missing_count(album): + """Return number of missing items in `album`. + """ + return (album.albumtotal or 0) - len(album.items()) + + +def _item(track_info, album_info, album_id): + """Build and return `item` from `track_info` and `album info` + objects. `item` is missing what fields cannot be obtained from + MusicBrainz alone (encoder, rg_track_gain, rg_track_peak, + rg_album_gain, rg_album_peak, original_year, original_month, + original_day, length, bitrate, format, samplerate, bitdepth, + channels, mtime.) + """ + t = track_info + a = album_info + + return Item(**{ + 'album_id': album_id, + 'album': a.album, + 'albumartist': a.artist, + 'albumartist_credit': a.artist_credit, + 'albumartist_sort': a.artist_sort, + 'albumdisambig': a.albumdisambig, + 'albumstatus': a.albumstatus, + 'albumtype': a.albumtype, + 'artist': t.artist, + 'artist_credit': t.artist_credit, + 'artist_sort': t.artist_sort, + 'asin': a.asin, + 'catalognum': a.catalognum, + 'comp': a.va, + 'country': a.country, + 'day': a.day, + 'disc': t.medium, + 'disctitle': t.disctitle, + 'disctotal': a.mediums, + 'label': a.label, + 'language': a.language, + 'length': t.length, + 'mb_albumid': a.album_id, + 'mb_artistid': t.artist_id, + 'mb_releasegroupid': a.releasegroup_id, + 'mb_trackid': t.track_id, + 'media': t.media, + 'month': a.month, + 'script': a.script, + 'title': t.title, + 'track': t.index, + 'tracktotal': len(a.tracks), + 'year': a.year, + }) + + +class MissingPlugin(BeetsPlugin): + """List missing tracks + """ + def __init__(self): + super(MissingPlugin, self).__init__() + + self.config.add({ + 'count': False, + 'total': False, + }) + + self.album_template_fields['missing'] = _missing_count + + self._command = Subcommand('missing', + help=__doc__, + aliases=['miss']) + self._command.parser.add_option( + u'-c', u'--count', dest='count', action='store_true', + help=u'count missing tracks per album') + self._command.parser.add_option( + u'-t', u'--total', dest='total', action='store_true', + help=u'count total of missing tracks') + self._command.parser.add_format_option() + + def commands(self): + def _miss(lib, opts, args): + self.config.set_args(opts) + count = self.config['count'].get() + total = self.config['total'].get() + fmt = config['format_album' if count else 'format_item'].get() + + albums = lib.albums(decargs(args)) + if total: + print(sum([_missing_count(a) for a in albums])) + return + + # Default format string for count mode. + if count: + fmt += ': $missing' + + for album in albums: + if count: + if _missing_count(album): + print_(format(album, fmt)) + + else: + for item in self._missing(album): + print_(format(item, fmt)) + + self._command.func = _miss + return [self._command] + + def _missing(self, album): + """Query MusicBrainz to determine items missing from `album`. + """ + item_mbids = map(lambda x: x.mb_trackid, album.items()) + if len([i for i in album.items()]) < album.albumtotal: + # fetch missing items + # TODO: Implement caching that without breaking other stuff + album_info = hooks.album_for_mbid(album.mb_albumid) + for track_info in getattr(album_info, 'tracks', []): + if track_info.track_id not in item_mbids: + item = _item(track_info, album_info, album.id) + self._log.debug(u'track {0} in album {1}', + track_info.track_id, album_info.album_id) + yield item diff --git a/libs/beetsplug/mpdstats.py b/libs/beetsplug/mpdstats.py new file mode 100644 index 00000000..2b642294 --- /dev/null +++ b/libs/beetsplug/mpdstats.py @@ -0,0 +1,368 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Peter Schnebel and Johann Klähn. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +from __future__ import division, absolute_import, print_function + +import mpd +import socket +import select +import time +import os + +from beets import ui +from beets import config +from beets import plugins +from beets import library +from beets.util import displayable_path +from beets.dbcore import types + +# If we lose the connection, how many times do we want to retry and how +# much time should we wait between retries? +RETRIES = 10 +RETRY_INTERVAL = 5 + + +mpd_config = config['mpd'] + + +def is_url(path): + """Try to determine if the path is an URL. + """ + return path.split('://', 1)[0] in ['http', 'https'] + + +# Use the MPDClient internals to get unicode. +# see http://www.tarmack.eu/code/mpdunicode.py for the general idea +class MPDClient(mpd.MPDClient): + def _write_command(self, command, args=[]): + args = [unicode(arg).encode('utf-8') for arg in args] + super(MPDClient, self)._write_command(command, args) + + def _read_line(self): + line = super(MPDClient, self)._read_line() + if line is not None: + return line.decode('utf-8') + return None + + +class MPDClientWrapper(object): + def __init__(self, log): + self._log = log + + self.music_directory = ( + mpd_config['music_directory'].get(unicode)) + + self.client = MPDClient() + + def connect(self): + """Connect to the MPD. + """ + host = mpd_config['host'].get(unicode) + port = mpd_config['port'].get(int) + + if host[0] in ['/', '~']: + host = os.path.expanduser(host) + + self._log.info(u'connecting to {0}:{1}', host, port) + try: + self.client.connect(host, port) + except socket.error as e: + raise ui.UserError(u'could not connect to MPD: {0}'.format(e)) + + password = mpd_config['password'].get(unicode) + if password: + try: + self.client.password(password) + except mpd.CommandError as e: + raise ui.UserError( + u'could not authenticate to MPD: {0}'.format(e) + ) + + def disconnect(self): + """Disconnect from the MPD. + """ + self.client.close() + self.client.disconnect() + + def get(self, command, retries=RETRIES): + """Wrapper for requests to the MPD server. Tries to re-connect if the + connection was lost (f.ex. during MPD's library refresh). + """ + try: + return getattr(self.client, command)() + except (select.error, mpd.ConnectionError) as err: + self._log.error(u'{0}', err) + + if retries <= 0: + # if we exited without breaking, we couldn't reconnect in time :( + raise ui.UserError(u'communication with MPD server failed') + + time.sleep(RETRY_INTERVAL) + + try: + self.disconnect() + except mpd.ConnectionError: + pass + + self.connect() + return self.get(command, retries=retries - 1) + + def playlist(self): + """Return the currently active playlist. Prefixes paths with the + music_directory, to get the absolute path. + """ + result = {} + for entry in self.get('playlistinfo'): + if not is_url(entry['file']): + result[entry['id']] = os.path.join( + self.music_directory, entry['file']) + else: + result[entry['id']] = entry['file'] + return result + + def status(self): + """Return the current status of the MPD. + """ + return self.get('status') + + def events(self): + """Return list of events. This may block a long time while waiting for + an answer from MPD. + """ + return self.get('idle') + + +class MPDStats(object): + def __init__(self, lib, log): + self.lib = lib + self._log = log + + self.do_rating = mpd_config['rating'].get(bool) + self.rating_mix = mpd_config['rating_mix'].get(float) + self.time_threshold = 10.0 # TODO: maybe add config option? + + self.now_playing = None + self.mpd = MPDClientWrapper(log) + + def rating(self, play_count, skip_count, rating, skipped): + """Calculate a new rating for a song based on play count, skip count, + old rating and the fact if it was skipped or not. + """ + if skipped: + rolling = (rating - rating / 2.0) + else: + rolling = (rating + (1.0 - rating) / 2.0) + stable = (play_count + 1.0) / (play_count + skip_count + 2.0) + return (self.rating_mix * stable + + (1.0 - self.rating_mix) * rolling) + + def get_item(self, path): + """Return the beets item related to path. + """ + query = library.PathQuery('path', path) + item = self.lib.items(query).get() + if item: + return item + else: + self._log.info(u'item not found: {0}', displayable_path(path)) + + def update_item(self, item, attribute, value=None, increment=None): + """Update the beets item. Set attribute to value or increment the value + of attribute. If the increment argument is used the value is cast to + the corresponding type. + """ + if item is None: + return + + if increment is not None: + item.load() + value = type(increment)(item.get(attribute, 0)) + increment + + if value is not None: + item[attribute] = value + item.store() + + self._log.debug(u'updated: {0} = {1} [{2}]', + attribute, + item[attribute], + displayable_path(item.path)) + + def update_rating(self, item, skipped): + """Update the rating for a beets item. The `item` can either be a + beets `Item` or None. If the item is None, nothing changes. + """ + if item is None: + return + + item.load() + rating = self.rating( + int(item.get('play_count', 0)), + int(item.get('skip_count', 0)), + float(item.get('rating', 0.5)), + skipped) + + self.update_item(item, 'rating', rating) + + def handle_song_change(self, song): + """Determine if a song was skipped or not and update its attributes. + To this end the difference between the song's supposed end time + and the current time is calculated. If it's greater than a threshold, + the song is considered skipped. + + Returns whether the change was manual (skipped previous song or not) + """ + diff = abs(song['remaining'] - (time.time() - song['started'])) + + skipped = diff >= self.time_threshold + + if skipped: + self.handle_skipped(song) + else: + self.handle_played(song) + + if self.do_rating: + self.update_rating(song['beets_item'], skipped) + + return skipped + + def handle_played(self, song): + """Updates the play count of a song. + """ + self.update_item(song['beets_item'], 'play_count', increment=1) + self._log.info(u'played {0}', displayable_path(song['path'])) + + def handle_skipped(self, song): + """Updates the skip count of a song. + """ + self.update_item(song['beets_item'], 'skip_count', increment=1) + self._log.info(u'skipped {0}', displayable_path(song['path'])) + + def on_stop(self, status): + self._log.info(u'stop') + + if self.now_playing: + self.handle_song_change(self.now_playing) + + self.now_playing = None + + def on_pause(self, status): + self._log.info(u'pause') + self.now_playing = None + + def on_play(self, status): + playlist = self.mpd.playlist() + path = playlist.get(status['songid']) + + if not path: + return + + if is_url(path): + self._log.info(u'playing stream {0}', displayable_path(path)) + return + + played, duration = map(int, status['time'].split(':', 1)) + remaining = duration - played + + if self.now_playing and self.now_playing['path'] != path: + skipped = self.handle_song_change(self.now_playing) + # mpd responds twice on a natural new song start + going_to_happen_twice = not skipped + else: + going_to_happen_twice = False + + if not going_to_happen_twice: + self._log.info(u'playing {0}', displayable_path(path)) + + self.now_playing = { + 'started': time.time(), + 'remaining': remaining, + 'path': path, + 'beets_item': self.get_item(path), + } + + self.update_item(self.now_playing['beets_item'], + 'last_played', value=int(time.time())) + + def run(self): + self.mpd.connect() + events = ['player'] + + while True: + if 'player' in events: + status = self.mpd.status() + + handler = getattr(self, 'on_' + status['state'], None) + + if handler: + handler(status) + else: + self._log.debug(u'unhandled status "{0}"', status) + + events = self.mpd.events() + + +class MPDStatsPlugin(plugins.BeetsPlugin): + + item_types = { + 'play_count': types.INTEGER, + 'skip_count': types.INTEGER, + 'last_played': library.DateType(), + 'rating': types.FLOAT, + } + + def __init__(self): + super(MPDStatsPlugin, self).__init__() + mpd_config.add({ + 'music_directory': config['directory'].as_filename(), + 'rating': True, + 'rating_mix': 0.75, + 'host': u'localhost', + 'port': 6600, + 'password': u'', + }) + mpd_config['password'].redact = True + + def commands(self): + cmd = ui.Subcommand( + 'mpdstats', + help=u'run a MPD client to gather play statistics') + cmd.parser.add_option( + u'--host', dest='host', type='string', + help=u'set the hostname of the server to connect to') + cmd.parser.add_option( + u'--port', dest='port', type='int', + help=u'set the port of the MPD server to connect to') + cmd.parser.add_option( + u'--password', dest='password', type='string', + help=u'set the password of the MPD server to connect to') + + def func(lib, opts, args): + mpd_config.set_args(opts) + + # Overrides for MPD settings. + if opts.host: + mpd_config['host'] = opts.host.decode('utf8') + if opts.port: + mpd_config['host'] = int(opts.port) + if opts.password: + mpd_config['password'] = opts.password.decode('utf8') + + try: + MPDStats(lib, self._log).run() + except KeyboardInterrupt: + pass + + cmd.func = func + return [cmd] diff --git a/libs/beetsplug/mpdupdate.py b/libs/beetsplug/mpdupdate.py new file mode 100644 index 00000000..f828ba5d --- /dev/null +++ b/libs/beetsplug/mpdupdate.py @@ -0,0 +1,128 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Adrian Sampson. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Updates an MPD index whenever the library is changed. + +Put something like the following in your config.yaml to configure: + mpd: + host: localhost + port: 6600 + password: seekrit +""" +from __future__ import division, absolute_import, print_function + +from beets.plugins import BeetsPlugin +import os +import socket +from beets import config + + +# No need to introduce a dependency on an MPD library for such a +# simple use case. Here's a simple socket abstraction to make things +# easier. +class BufferedSocket(object): + """Socket abstraction that allows reading by line.""" + def __init__(self, host, port, sep='\n'): + if host[0] in ['/', '~']: + self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.sock.connect(os.path.expanduser(host)) + else: + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.connect((host, port)) + self.buf = '' + self.sep = sep + + def readline(self): + while self.sep not in self.buf: + data = self.sock.recv(1024) + if not data: + break + self.buf += data + if '\n' in self.buf: + res, self.buf = self.buf.split(self.sep, 1) + return res + self.sep + else: + return '' + + def send(self, data): + self.sock.send(data) + + def close(self): + self.sock.close() + + +class MPDUpdatePlugin(BeetsPlugin): + def __init__(self): + super(MPDUpdatePlugin, self).__init__() + config['mpd'].add({ + 'host': u'localhost', + 'port': 6600, + 'password': u'', + }) + config['mpd']['password'].redact = True + + # For backwards compatibility, use any values from the + # plugin-specific "mpdupdate" section. + for key in config['mpd'].keys(): + if self.config[key].exists(): + config['mpd'][key] = self.config[key].get() + + self.register_listener('database_change', self.db_change) + + def db_change(self, lib, model): + self.register_listener('cli_exit', self.update) + + def update(self, lib): + self.update_mpd( + config['mpd']['host'].get(unicode), + config['mpd']['port'].get(int), + config['mpd']['password'].get(unicode), + ) + + def update_mpd(self, host='localhost', port=6600, password=None): + """Sends the "update" command to the MPD server indicated, + possibly authenticating with a password first. + """ + self._log.info('Updating MPD database...') + + try: + s = BufferedSocket(host, port) + except socket.error as e: + self._log.warning(u'MPD connection failed: {0}', + unicode(e.strerror)) + return + + resp = s.readline() + if 'OK MPD' not in resp: + self._log.warning(u'MPD connection failed: {0!r}', resp) + return + + if password: + s.send('password "%s"\n' % password) + resp = s.readline() + if 'OK' not in resp: + self._log.warning(u'Authentication failed: {0!r}', resp) + s.send('close\n') + s.close() + return + + s.send('update\n') + resp = s.readline() + if 'updating_db' not in resp: + self._log.warning(u'Update failed: {0!r}', resp) + + s.send('close\n') + s.close() + self._log.info(u'Database updated.') diff --git a/libs/beetsplug/permissions.py b/libs/beetsplug/permissions.py new file mode 100644 index 00000000..0de8978c --- /dev/null +++ b/libs/beetsplug/permissions.py @@ -0,0 +1,102 @@ +# -*- coding: utf-8 -*- + +from __future__ import division, absolute_import, print_function + +"""Fixes file permissions after the file gets written on import. Put something +like the following in your config.yaml to configure: + + permissions: + file: 644 + dir: 755 +""" +import os +from beets import config, util +from beets.plugins import BeetsPlugin +from beets.util import ancestry + + +def convert_perm(perm): + """If the perm is a int it will first convert it to a string and back + to an oct int. Else it just converts it to oct. + """ + if isinstance(perm, int): + return int(bytes(perm), 8) + else: + return int(perm, 8) + + +def check_permissions(path, permission): + """Checks the permissions of a path. + """ + return oct(os.stat(path).st_mode & 0o777) == oct(permission) + + +def dirs_in_library(library, item): + """Creates a list of ancestor directories in the beets library path. + """ + return [ancestor + for ancestor in ancestry(item) + if ancestor.startswith(library)][1:] + + +class Permissions(BeetsPlugin): + def __init__(self): + super(Permissions, self).__init__() + + # Adding defaults. + self.config.add({ + u'file': 644, + u'dir': 755 + }) + + self.register_listener('item_imported', permissions) + self.register_listener('album_imported', permissions) + + +def permissions(lib, item=None, album=None): + """Running the permission fixer. + """ + # Getting the config. + file_perm = config['permissions']['file'].get() + dir_perm = config['permissions']['dir'].get() + + # Converts permissions to oct. + file_perm = convert_perm(file_perm) + dir_perm = convert_perm(dir_perm) + + # Create chmod_queue. + file_chmod_queue = [] + if item: + file_chmod_queue.append(item.path) + elif album: + for album_item in album.items(): + file_chmod_queue.append(album_item.path) + + # A set of directories to change permissions for. + dir_chmod_queue = set() + + for path in file_chmod_queue: + # Changing permissions on the destination file. + os.chmod(util.bytestring_path(path), file_perm) + + # Checks if the destination path has the permissions configured. + if not check_permissions(util.bytestring_path(path), file_perm): + message = u'There was a problem setting permission on {}'.format( + path) + print(message) + + # Adding directories to the directory chmod queue. + dir_chmod_queue.update( + dirs_in_library(lib.directory, + path)) + + # Change permissions for the directories. + for path in dir_chmod_queue: + # Chaning permissions on the destination directory. + os.chmod(util.bytestring_path(path), dir_perm) + + # Checks if the destination path has the permissions configured. + if not check_permissions(util.bytestring_path(path), dir_perm): + message = u'There was a problem setting permission on {}'.format( + path) + print(message) diff --git a/libs/beetsplug/play.py b/libs/beetsplug/play.py new file mode 100644 index 00000000..fa70f2bc --- /dev/null +++ b/libs/beetsplug/play.py @@ -0,0 +1,155 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, David Hamp-Gonsalves +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Send the results of a query to the configured music player as a playlist. +""" +from __future__ import division, absolute_import, print_function + +from beets.plugins import BeetsPlugin +from beets.ui import Subcommand +from beets import config +from beets import ui +from beets import util +from os.path import relpath +from tempfile import NamedTemporaryFile + +# Indicate where arguments should be inserted into the command string. +# If this is missing, they're placed at the end. +ARGS_MARKER = '$args' + + +class PlayPlugin(BeetsPlugin): + + def __init__(self): + super(PlayPlugin, self).__init__() + + config['play'].add({ + 'command': None, + 'use_folders': False, + 'relative_to': None, + 'raw': False, + # Backwards compatibility. See #1803 and line 74 + 'warning_threshold': -2, + 'warning_treshold': 100, + }) + + def commands(self): + play_command = Subcommand( + 'play', + help=u'send music to a player as a playlist' + ) + play_command.parser.add_album_option() + play_command.parser.add_option( + u'-A', u'--args', + action='store', + help=u'add additional arguments to the command', + ) + play_command.func = self.play_music + return [play_command] + + def play_music(self, lib, opts, args): + """Execute query, create temporary playlist and execute player + command passing that playlist, at request insert optional arguments. + """ + command_str = config['play']['command'].get() + if not command_str: + command_str = util.open_anything() + use_folders = config['play']['use_folders'].get(bool) + relative_to = config['play']['relative_to'].get() + raw = config['play']['raw'].get(bool) + warning_threshold = config['play']['warning_threshold'].get(int) + # We use -2 as a default value for warning_threshold to detect if it is + # set or not. We can't use a falsey value because it would have an + # actual meaning in the configuration of this plugin, and we do not use + # -1 because some people might use it as a value to obtain no warning, + # which wouldn't be that bad of a practice. + if warning_threshold == -2: + # if warning_threshold has not been set by user, look for + # warning_treshold, to preserve backwards compatibility. See #1803. + # warning_treshold has the correct default value of 100. + warning_threshold = config['play']['warning_treshold'].get(int) + + if relative_to: + relative_to = util.normpath(relative_to) + + # Add optional arguments to the player command. + if opts.args: + if ARGS_MARKER in command_str: + command_str = command_str.replace(ARGS_MARKER, opts.args) + else: + command_str = u"{} {}".format(command_str, opts.args) + + # Perform search by album and add folders rather than tracks to + # playlist. + if opts.album: + selection = lib.albums(ui.decargs(args)) + paths = [] + + sort = lib.get_default_album_sort() + for album in selection: + if use_folders: + paths.append(album.item_dir()) + else: + paths.extend(item.path + for item in sort.sort(album.items())) + item_type = 'album' + + # Perform item query and add tracks to playlist. + else: + selection = lib.items(ui.decargs(args)) + paths = [item.path for item in selection] + if relative_to: + paths = [relpath(path, relative_to) for path in paths] + item_type = 'track' + + item_type += 's' if len(selection) > 1 else '' + + if not selection: + ui.print_(ui.colorize('text_warning', + u'No {0} to play.'.format(item_type))) + return + + # Warn user before playing any huge playlists. + if warning_threshold and len(selection) > warning_threshold: + ui.print_(ui.colorize( + 'text_warning', + u'You are about to queue {0} {1}.'.format( + len(selection), item_type))) + + if ui.input_options(('Continue', 'Abort')) == 'a': + return + + ui.print_(u'Playing {0} {1}.'.format(len(selection), item_type)) + if raw: + open_args = paths + else: + open_args = [self._create_tmp_playlist(paths)] + + self._log.debug(u'executing command: {} {}', command_str, + b' '.join(open_args)) + try: + util.interactive_open(open_args, command_str) + except OSError as exc: + raise ui.UserError( + "Could not play the query: {0}".format(exc)) + + def _create_tmp_playlist(self, paths_list): + """Create a temporary .m3u file. Return the filename. + """ + m3u = NamedTemporaryFile('w', suffix='.m3u', delete=False) + for item in paths_list: + m3u.write(item + b'\n') + m3u.close() + return m3u.name diff --git a/libs/beetsplug/plexupdate.py b/libs/beetsplug/plexupdate.py new file mode 100644 index 00000000..ef50fde7 --- /dev/null +++ b/libs/beetsplug/plexupdate.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- + +"""Updates an Plex library whenever the beets library is changed. + +Plex Home users enter the Plex Token to enable updating. +Put something like the following in your config.yaml to configure: + plex: + host: localhost + port: 32400 + token: token +""" +from __future__ import division, absolute_import, print_function + +import requests +from urlparse import urljoin +from urllib import urlencode +import xml.etree.ElementTree as ET +from beets import config +from beets.plugins import BeetsPlugin + + +def get_music_section(host, port, token, library_name): + """Getting the section key for the music library in Plex. + """ + api_endpoint = append_token('library/sections', token) + url = urljoin('http://{0}:{1}'.format(host, port), api_endpoint) + + # Sends request. + r = requests.get(url) + + # Parse xml tree and extract music section key. + tree = ET.fromstring(r.content) + for child in tree.findall('Directory'): + if child.get('title') == library_name: + return child.get('key') + + +def update_plex(host, port, token, library_name): + """Sends request to the Plex api to start a library refresh. + """ + # Getting section key and build url. + section_key = get_music_section(host, port, token, library_name) + api_endpoint = 'library/sections/{0}/refresh'.format(section_key) + api_endpoint = append_token(api_endpoint, token) + url = urljoin('http://{0}:{1}'.format(host, port), api_endpoint) + + # Sends request and returns requests object. + r = requests.get(url) + return r + + +def append_token(url, token): + """Appends the Plex Home token to the api call if required. + """ + if token: + url += '?' + urlencode({'X-Plex-Token': token}) + return url + + +class PlexUpdate(BeetsPlugin): + def __init__(self): + super(PlexUpdate, self).__init__() + + # Adding defaults. + config['plex'].add({ + u'host': u'localhost', + u'port': 32400, + u'token': u'', + u'library_name': u'Music'}) + + self.register_listener('database_change', self.listen_for_db_change) + + def listen_for_db_change(self, lib, model): + """Listens for beets db change and register the update for the end""" + self.register_listener('cli_exit', self.update) + + def update(self, lib): + """When the client exists try to send refresh request to Plex server. + """ + self._log.info(u'Updating Plex library...') + + # Try to send update request. + try: + update_plex( + config['plex']['host'].get(), + config['plex']['port'].get(), + config['plex']['token'].get(), + config['plex']['library_name'].get()) + self._log.info(u'... started.') + + except requests.exceptions.RequestException: + self._log.warning(u'Update failed.') diff --git a/libs/beetsplug/random.py b/libs/beetsplug/random.py new file mode 100644 index 00000000..e1c6fea4 --- /dev/null +++ b/libs/beetsplug/random.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Philippe Mongeau. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Get a random song or album from the library. +""" +from __future__ import division, absolute_import, print_function + +from beets.plugins import BeetsPlugin +from beets.ui import Subcommand, decargs, print_ +import random +from operator import attrgetter +from itertools import groupby + + +def random_item(lib, opts, args): + query = decargs(args) + + if opts.album: + objs = list(lib.albums(query)) + else: + objs = list(lib.items(query)) + + if opts.equal_chance: + # Group the objects by artist so we can sample from them. + key = attrgetter('albumartist') + objs.sort(key=key) + objs_by_artists = {} + for artist, v in groupby(objs, key): + objs_by_artists[artist] = list(v) + + objs = [] + for _ in range(opts.number): + # Terminate early if we're out of objects to select. + if not objs_by_artists: + break + + # Choose an artist and an object for that artist, removing + # this choice from the pool. + artist = random.choice(objs_by_artists.keys()) + objs_from_artist = objs_by_artists[artist] + i = random.randint(0, len(objs_from_artist) - 1) + objs.append(objs_from_artist.pop(i)) + + # Remove the artist if we've used up all of its objects. + if not objs_from_artist: + del objs_by_artists[artist] + + else: + number = min(len(objs), opts.number) + objs = random.sample(objs, number) + + for item in objs: + print_(format(item)) + +random_cmd = Subcommand('random', + help=u'chose a random track or album') +random_cmd.parser.add_option( + u'-n', u'--number', action='store', type="int", + help=u'number of objects to choose', default=1) +random_cmd.parser.add_option( + u'-e', u'--equal-chance', action='store_true', + help=u'each artist has the same chance') +random_cmd.parser.add_all_common_options() +random_cmd.func = random_item + + +class Random(BeetsPlugin): + def commands(self): + return [random_cmd] diff --git a/libs/beetsplug/replaygain.py b/libs/beetsplug/replaygain.py new file mode 100644 index 00000000..7bb2aa39 --- /dev/null +++ b/libs/beetsplug/replaygain.py @@ -0,0 +1,953 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Fabrice Laporte, Yevgeny Bezman, and Adrian Sampson. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +from __future__ import division, absolute_import, print_function + +import subprocess +import os +import collections +import itertools +import sys +import warnings +import re + +from beets import logging +from beets import ui +from beets.plugins import BeetsPlugin +from beets.util import syspath, command_output, displayable_path + + +# Utilities. + +class ReplayGainError(Exception): + """Raised when a local (to a track or an album) error occurs in one + of the backends. + """ + + +class FatalReplayGainError(Exception): + """Raised when a fatal error occurs in one of the backends. + """ + + +class FatalGstreamerPluginReplayGainError(FatalReplayGainError): + """Raised when a fatal error occurs in the GStreamerBackend when + loading the required plugins.""" + + +def call(args): + """Execute the command and return its output or raise a + ReplayGainError on failure. + """ + try: + return command_output(args) + except subprocess.CalledProcessError as e: + raise ReplayGainError( + u"{0} exited with status {1}".format(args[0], e.returncode) + ) + except UnicodeEncodeError: + # Due to a bug in Python 2's subprocess on Windows, Unicode + # filenames can fail to encode on that platform. See: + # http://code.google.com/p/beets/issues/detail?id=499 + raise ReplayGainError(u"argument encoding failed") + + +# Backend base and plumbing classes. + +Gain = collections.namedtuple("Gain", "gain peak") +AlbumGain = collections.namedtuple("AlbumGain", "album_gain track_gains") + + +class Backend(object): + """An abstract class representing engine for calculating RG values. + """ + + def __init__(self, config, log): + """Initialize the backend with the configuration view for the + plugin. + """ + self._log = log + + def compute_track_gain(self, items): + raise NotImplementedError() + + def compute_album_gain(self, album): + # TODO: implement album gain in terms of track gain of the + # individual tracks which can be used for any backend. + raise NotImplementedError() + + +# bsg1770gain backend +class Bs1770gainBackend(Backend): + """bs1770gain is a loudness scanner compliant with ITU-R BS.1770 and + its flavors EBU R128, ATSC A/85 and Replaygain 2.0. + """ + + def __init__(self, config, log): + super(Bs1770gainBackend, self).__init__(config, log) + config.add({ + 'chunk_at': 5000, + 'method': 'replaygain', + }) + self.chunk_at = config['chunk_at'].as_number() + self.method = b'--' + bytes(config['method'].get(unicode)) + + cmd = b'bs1770gain' + try: + call([cmd, self.method]) + self.command = cmd + except OSError: + raise FatalReplayGainError( + u'Is bs1770gain installed? Is your method in config correct?' + ) + if not self.command: + raise FatalReplayGainError( + u'no replaygain command found: install bs1770gain' + ) + + def compute_track_gain(self, items): + """Computes the track gain of the given tracks, returns a list + of TrackGain objects. + """ + + output = self.compute_gain(items, False) + return output + + def compute_album_gain(self, album): + """Computes the album gain of the given album, returns an + AlbumGain object. + """ + # TODO: What should be done when not all tracks in the album are + # supported? + + supported_items = album.items() + output = self.compute_gain(supported_items, True) + + if not output: + raise ReplayGainError(u'no output from bs1770gain') + return AlbumGain(output[-1], output[:-1]) + + def isplitter(self, items, chunk_at): + """Break an iterable into chunks of at most size `chunk_at`, + generating lists for each chunk. + """ + iterable = iter(items) + while True: + result = [] + for i in range(chunk_at): + try: + a = next(iterable) + except StopIteration: + break + else: + result.append(a) + if result: + yield result + else: + break + + def compute_gain(self, items, is_album): + """Computes the track or album gain of a list of items, returns + a list of TrackGain objects. + When computing album gain, the last TrackGain object returned is + the album gain + """ + + if len(items) == 0: + return [] + + albumgaintot = 0.0 + albumpeaktot = 0.0 + returnchunks = [] + + # In the case of very large sets of music, we break the tracks + # into smaller chunks and process them one at a time. This + # avoids running out of memory. + if len(items) > self.chunk_at: + i = 0 + for chunk in self.isplitter(items, self.chunk_at): + i += 1 + returnchunk = self.compute_chunk_gain(chunk, is_album) + albumgaintot += returnchunk[-1].gain + albumpeaktot += returnchunk[-1].peak + returnchunks = returnchunks + returnchunk[0:-1] + returnchunks.append(Gain(albumgaintot / i, albumpeaktot / i)) + return returnchunks + else: + return self.compute_chunk_gain(items, is_album) + + def compute_chunk_gain(self, items, is_album): + """Compute ReplayGain values and return a list of results + dictionaries as given by `parse_tool_output`. + """ + # Construct shell command. + cmd = [self.command] + cmd = cmd + [self.method] + cmd = cmd + [b'-it'] + + # Workaround for Windows: the underlying tool fails on paths + # with the \\?\ prefix, so we don't use it here. This + # prevents the backend from working with long paths. + args = cmd + [syspath(i.path, prefix=False) for i in items] + + # Invoke the command. + self._log.debug( + u'executing {0}', u' '.join(map(displayable_path, args)) + ) + output = call(args) + + self._log.debug(u'analysis finished: {0}', output) + results = self.parse_tool_output(output, + len(items) + is_album) + self._log.debug(u'{0} items, {1} results', len(items), len(results)) + return results + + def parse_tool_output(self, text, num_lines): + """Given the output from bs1770gain, parse the text and + return a list of dictionaries + containing information about each analyzed file. + """ + out = [] + data = text.decode('utf8', errors='ignore') + regex = re.compile( + ur'(\s{2,2}\[\d+\/\d+\].*?|\[ALBUM\].*?)' + '(?=\s{2,2}\[\d+\/\d+\]|\s{2,2}\[ALBUM\]' + ':|done\.\s)', re.DOTALL | re.UNICODE) + results = re.findall(regex, data) + for parts in results[0:num_lines]: + part = parts.split(b'\n') + if len(part) == 0: + self._log.debug(u'bad tool output: {0!r}', text) + raise ReplayGainError(u'bs1770gain failed') + + try: + song = { + 'file': part[0], + 'gain': float((part[1].split('/'))[1].split('LU')[0]), + 'peak': float(part[2].split('/')[1]), + } + except IndexError: + self._log.info(u'bs1770gain reports (faulty file?): {}', parts) + continue + + out.append(Gain(song['gain'], song['peak'])) + return out + + +# mpgain/aacgain CLI tool backend. +class CommandBackend(Backend): + + def __init__(self, config, log): + super(CommandBackend, self).__init__(config, log) + config.add({ + 'command': u"", + 'noclip': True, + }) + + self.command = config["command"].get(unicode) + + if self.command: + # Explicit executable path. + if not os.path.isfile(self.command): + raise FatalReplayGainError( + u'replaygain command does not exist: {0}'.format( + self.command) + ) + else: + # Check whether the program is in $PATH. + for cmd in (b'mp3gain', b'aacgain'): + try: + call([cmd, b'-v']) + self.command = cmd + except OSError: + pass + if not self.command: + raise FatalReplayGainError( + u'no replaygain command found: install mp3gain or aacgain' + ) + + self.noclip = config['noclip'].get(bool) + target_level = config['targetlevel'].as_number() + self.gain_offset = int(target_level - 89) + + def compute_track_gain(self, items): + """Computes the track gain of the given tracks, returns a list + of TrackGain objects. + """ + supported_items = filter(self.format_supported, items) + output = self.compute_gain(supported_items, False) + return output + + def compute_album_gain(self, album): + """Computes the album gain of the given album, returns an + AlbumGain object. + """ + # TODO: What should be done when not all tracks in the album are + # supported? + + supported_items = filter(self.format_supported, album.items()) + if len(supported_items) != len(album.items()): + self._log.debug(u'tracks are of unsupported format') + return AlbumGain(None, []) + + output = self.compute_gain(supported_items, True) + return AlbumGain(output[-1], output[:-1]) + + def format_supported(self, item): + """Checks whether the given item is supported by the selected tool. + """ + if 'mp3gain' in self.command and item.format != 'MP3': + return False + elif 'aacgain' in self.command and item.format not in ('MP3', 'AAC'): + return False + return True + + def compute_gain(self, items, is_album): + """Computes the track or album gain of a list of items, returns + a list of TrackGain objects. + + When computing album gain, the last TrackGain object returned is + the album gain + """ + if len(items) == 0: + self._log.debug(u'no supported tracks to analyze') + return [] + + """Compute ReplayGain values and return a list of results + dictionaries as given by `parse_tool_output`. + """ + # Construct shell command. The "-o" option makes the output + # easily parseable (tab-delimited). "-s s" forces gain + # recalculation even if tags are already present and disables + # tag-writing; this turns the mp3gain/aacgain tool into a gain + # calculator rather than a tag manipulator because we take care + # of changing tags ourselves. + cmd = [self.command, b'-o', b'-s', b's'] + if self.noclip: + # Adjust to avoid clipping. + cmd = cmd + [b'-k'] + else: + # Disable clipping warning. + cmd = cmd + [b'-c'] + cmd = cmd + [b'-d', bytes(self.gain_offset)] + cmd = cmd + [syspath(i.path) for i in items] + + self._log.debug(u'analyzing {0} files', len(items)) + self._log.debug(u"executing {0}", " ".join(map(displayable_path, cmd))) + output = call(cmd) + self._log.debug(u'analysis finished') + return self.parse_tool_output(output, + len(items) + (1 if is_album else 0)) + + def parse_tool_output(self, text, num_lines): + """Given the tab-delimited output from an invocation of mp3gain + or aacgain, parse the text and return a list of dictionaries + containing information about each analyzed file. + """ + out = [] + for line in text.split(b'\n')[1:num_lines + 1]: + parts = line.split(b'\t') + if len(parts) != 6 or parts[0] == b'File': + self._log.debug(u'bad tool output: {0}', text) + raise ReplayGainError(u'mp3gain failed') + d = { + 'file': parts[0], + 'mp3gain': int(parts[1]), + 'gain': float(parts[2]), + 'peak': float(parts[3]) / (1 << 15), + 'maxgain': int(parts[4]), + 'mingain': int(parts[5]), + + } + out.append(Gain(d['gain'], d['peak'])) + return out + + +# GStreamer-based backend. + +class GStreamerBackend(Backend): + + def __init__(self, config, log): + super(GStreamerBackend, self).__init__(config, log) + self._import_gst() + + # Initialized a GStreamer pipeline of the form filesrc -> + # decodebin -> audioconvert -> audioresample -> rganalysis -> + # fakesink The connection between decodebin and audioconvert is + # handled dynamically after decodebin figures out the type of + # the input file. + self._src = self.Gst.ElementFactory.make("filesrc", "src") + self._decbin = self.Gst.ElementFactory.make("decodebin", "decbin") + self._conv = self.Gst.ElementFactory.make("audioconvert", "conv") + self._res = self.Gst.ElementFactory.make("audioresample", "res") + self._rg = self.Gst.ElementFactory.make("rganalysis", "rg") + + if self._src is None or self._decbin is None or self._conv is None \ + or self._res is None or self._rg is None: + raise FatalGstreamerPluginReplayGainError( + u"Failed to load required GStreamer plugins" + ) + + # We check which files need gain ourselves, so all files given + # to rganalsys should have their gain computed, even if it + # already exists. + self._rg.set_property("forced", True) + self._rg.set_property("reference-level", + config["targetlevel"].as_number()) + self._sink = self.Gst.ElementFactory.make("fakesink", "sink") + + self._pipe = self.Gst.Pipeline() + self._pipe.add(self._src) + self._pipe.add(self._decbin) + self._pipe.add(self._conv) + self._pipe.add(self._res) + self._pipe.add(self._rg) + self._pipe.add(self._sink) + + self._src.link(self._decbin) + self._conv.link(self._res) + self._res.link(self._rg) + self._rg.link(self._sink) + + self._bus = self._pipe.get_bus() + self._bus.add_signal_watch() + self._bus.connect("message::eos", self._on_eos) + self._bus.connect("message::error", self._on_error) + self._bus.connect("message::tag", self._on_tag) + # Needed for handling the dynamic connection between decodebin + # and audioconvert + self._decbin.connect("pad-added", self._on_pad_added) + self._decbin.connect("pad-removed", self._on_pad_removed) + + self._main_loop = self.GLib.MainLoop() + + self._files = [] + + def _import_gst(self): + """Import the necessary GObject-related modules and assign `Gst` + and `GObject` fields on this object. + """ + + try: + import gi + except ImportError: + raise FatalReplayGainError( + u"Failed to load GStreamer: python-gi not found" + ) + + try: + gi.require_version('Gst', '1.0') + except ValueError as e: + raise FatalReplayGainError( + u"Failed to load GStreamer 1.0: {0}".format(e) + ) + + from gi.repository import GObject, Gst, GLib + # Calling GObject.threads_init() is not needed for + # PyGObject 3.10.2+ + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + GObject.threads_init() + Gst.init([sys.argv[0]]) + + self.GObject = GObject + self.GLib = GLib + self.Gst = Gst + + def compute(self, files, album): + self._error = None + self._files = list(files) + + if len(self._files) == 0: + return + + self._file_tags = collections.defaultdict(dict) + + if album: + self._rg.set_property("num-tracks", len(self._files)) + + if self._set_first_file(): + self._main_loop.run() + if self._error is not None: + raise self._error + + def compute_track_gain(self, items): + self.compute(items, False) + if len(self._file_tags) != len(items): + raise ReplayGainError(u"Some tracks did not receive tags") + + ret = [] + for item in items: + ret.append(Gain(self._file_tags[item]["TRACK_GAIN"], + self._file_tags[item]["TRACK_PEAK"])) + + return ret + + def compute_album_gain(self, album): + items = list(album.items()) + self.compute(items, True) + if len(self._file_tags) != len(items): + raise ReplayGainError(u"Some items in album did not receive tags") + + # Collect track gains. + track_gains = [] + for item in items: + try: + gain = self._file_tags[item]["TRACK_GAIN"] + peak = self._file_tags[item]["TRACK_PEAK"] + except KeyError: + raise ReplayGainError(u"results missing for track") + track_gains.append(Gain(gain, peak)) + + # Get album gain information from the last track. + last_tags = self._file_tags[items[-1]] + try: + gain = last_tags["ALBUM_GAIN"] + peak = last_tags["ALBUM_PEAK"] + except KeyError: + raise ReplayGainError(u"results missing for album") + + return AlbumGain(Gain(gain, peak), track_gains) + + def close(self): + self._bus.remove_signal_watch() + + def _on_eos(self, bus, message): + # A file finished playing in all elements of the pipeline. The + # RG tags have already been propagated. If we don't have a next + # file, we stop processing. + if not self._set_next_file(): + self._pipe.set_state(self.Gst.State.NULL) + self._main_loop.quit() + + def _on_error(self, bus, message): + self._pipe.set_state(self.Gst.State.NULL) + self._main_loop.quit() + err, debug = message.parse_error() + f = self._src.get_property("location") + # A GStreamer error, either an unsupported format or a bug. + self._error = ReplayGainError( + u"Error {0!r} - {1!r} on file {2!r}".format(err, debug, f) + ) + + def _on_tag(self, bus, message): + tags = message.parse_tag() + + def handle_tag(taglist, tag, userdata): + # The rganalysis element provides both the existing tags for + # files and the new computes tags. In order to ensure we + # store the computed tags, we overwrite the RG values of + # received a second time. + if tag == self.Gst.TAG_TRACK_GAIN: + self._file_tags[self._file]["TRACK_GAIN"] = \ + taglist.get_double(tag)[1] + elif tag == self.Gst.TAG_TRACK_PEAK: + self._file_tags[self._file]["TRACK_PEAK"] = \ + taglist.get_double(tag)[1] + elif tag == self.Gst.TAG_ALBUM_GAIN: + self._file_tags[self._file]["ALBUM_GAIN"] = \ + taglist.get_double(tag)[1] + elif tag == self.Gst.TAG_ALBUM_PEAK: + self._file_tags[self._file]["ALBUM_PEAK"] = \ + taglist.get_double(tag)[1] + elif tag == self.Gst.TAG_REFERENCE_LEVEL: + self._file_tags[self._file]["REFERENCE_LEVEL"] = \ + taglist.get_double(tag)[1] + + tags.foreach(handle_tag, None) + + def _set_first_file(self): + if len(self._files) == 0: + return False + + self._file = self._files.pop(0) + self._pipe.set_state(self.Gst.State.NULL) + self._src.set_property("location", syspath(self._file.path)) + self._pipe.set_state(self.Gst.State.PLAYING) + return True + + def _set_file(self): + """Initialize the filesrc element with the next file to be analyzed. + """ + # No more files, we're done + if len(self._files) == 0: + return False + + self._file = self._files.pop(0) + + # Disconnect the decodebin element from the pipeline, set its + # state to READY to to clear it. + self._decbin.unlink(self._conv) + self._decbin.set_state(self.Gst.State.READY) + + # Set a new file on the filesrc element, can only be done in the + # READY state + self._src.set_state(self.Gst.State.READY) + self._src.set_property("location", syspath(self._file.path)) + + # Ensure the filesrc element received the paused state of the + # pipeline in a blocking manner + self._src.sync_state_with_parent() + self._src.get_state(self.Gst.CLOCK_TIME_NONE) + + # Ensure the decodebin element receives the paused state of the + # pipeline in a blocking manner + self._decbin.sync_state_with_parent() + self._decbin.get_state(self.Gst.CLOCK_TIME_NONE) + + return True + + def _set_next_file(self): + """Set the next file to be analyzed while keeping the pipeline + in the PAUSED state so that the rganalysis element can correctly + handle album gain. + """ + # A blocking pause + self._pipe.set_state(self.Gst.State.PAUSED) + self._pipe.get_state(self.Gst.CLOCK_TIME_NONE) + + # Try setting the next file + ret = self._set_file() + if ret: + # Seek to the beginning in order to clear the EOS state of the + # various elements of the pipeline + self._pipe.seek_simple(self.Gst.Format.TIME, + self.Gst.SeekFlags.FLUSH, + 0) + self._pipe.set_state(self.Gst.State.PLAYING) + + return ret + + def _on_pad_added(self, decbin, pad): + sink_pad = self._conv.get_compatible_pad(pad, None) + assert(sink_pad is not None) + pad.link(sink_pad) + + def _on_pad_removed(self, decbin, pad): + # Called when the decodebin element is disconnected from the + # rest of the pipeline while switching input files + peer = pad.get_peer() + assert(peer is None) + + +class AudioToolsBackend(Backend): + """ReplayGain backend that uses `Python Audio Tools + <http://audiotools.sourceforge.net/>`_ and its capabilities to read more + file formats and compute ReplayGain values using it replaygain module. + """ + + def __init__(self, config, log): + super(AudioToolsBackend, self).__init__(config, log) + self._import_audiotools() + + def _import_audiotools(self): + """Check whether it's possible to import the necessary modules. + There is no check on the file formats at runtime. + + :raises :exc:`ReplayGainError`: if the modules cannot be imported + """ + try: + import audiotools + import audiotools.replaygain + except ImportError: + raise FatalReplayGainError( + u"Failed to load audiotools: audiotools not found" + ) + self._mod_audiotools = audiotools + self._mod_replaygain = audiotools.replaygain + + def open_audio_file(self, item): + """Open the file to read the PCM stream from the using + ``item.path``. + + :return: the audiofile instance + :rtype: :class:`audiotools.AudioFile` + :raises :exc:`ReplayGainError`: if the file is not found or the + file format is not supported + """ + try: + audiofile = self._mod_audiotools.open(item.path) + except IOError: + raise ReplayGainError( + u"File {} was not found".format(item.path) + ) + except self._mod_audiotools.UnsupportedFile: + raise ReplayGainError( + u"Unsupported file type {}".format(item.format) + ) + + return audiofile + + def init_replaygain(self, audiofile, item): + """Return an initialized :class:`audiotools.replaygain.ReplayGain` + instance, which requires the sample rate of the song(s) on which + the ReplayGain values will be computed. The item is passed in case + the sample rate is invalid to log the stored item sample rate. + + :return: initialized replagain object + :rtype: :class:`audiotools.replaygain.ReplayGain` + :raises: :exc:`ReplayGainError` if the sample rate is invalid + """ + try: + rg = self._mod_replaygain.ReplayGain(audiofile.sample_rate()) + except ValueError: + raise ReplayGainError( + u"Unsupported sample rate {}".format(item.samplerate)) + return + return rg + + def compute_track_gain(self, items): + """Compute ReplayGain values for the requested items. + + :return list: list of :class:`Gain` objects + """ + return [self._compute_track_gain(item) for item in items] + + def _title_gain(self, rg, audiofile): + """Get the gain result pair from PyAudioTools using the `ReplayGain` + instance `rg` for the given `audiofile`. + + Wraps `rg.title_gain(audiofile.to_pcm())` and throws a + `ReplayGainError` when the library fails. + """ + try: + # The method needs an audiotools.PCMReader instance that can + # be obtained from an audiofile instance. + return rg.title_gain(audiofile.to_pcm()) + except ValueError as exc: + # `audiotools.replaygain` can raise a `ValueError` if the sample + # rate is incorrect. + self._log.debug(u'error in rg.title_gain() call: {}', exc) + raise ReplayGainError(u'audiotools audio data error') + + def _compute_track_gain(self, item): + """Compute ReplayGain value for the requested item. + + :rtype: :class:`Gain` + """ + audiofile = self.open_audio_file(item) + rg = self.init_replaygain(audiofile, item) + + # Each call to title_gain on a ReplayGain object returns peak and gain + # of the track. + rg_track_gain, rg_track_peak = self._title_gain(rg, audiofile) + + self._log.debug(u'ReplayGain for track {0} - {1}: {2:.2f}, {3:.2f}', + item.artist, item.title, rg_track_gain, rg_track_peak) + return Gain(gain=rg_track_gain, peak=rg_track_peak) + + def compute_album_gain(self, album): + """Compute ReplayGain values for the requested album and its items. + + :rtype: :class:`AlbumGain` + """ + self._log.debug(u'Analysing album {0}', album) + + # The first item is taken and opened to get the sample rate to + # initialize the replaygain object. The object is used for all the + # tracks in the album to get the album values. + item = list(album.items())[0] + audiofile = self.open_audio_file(item) + rg = self.init_replaygain(audiofile, item) + + track_gains = [] + for item in album.items(): + audiofile = self.open_audio_file(item) + rg_track_gain, rg_track_peak = self._title_gain(rg, audiofile) + track_gains.append( + Gain(gain=rg_track_gain, peak=rg_track_peak) + ) + self._log.debug(u'ReplayGain for track {0}: {1:.2f}, {2:.2f}', + item, rg_track_gain, rg_track_peak) + + # After getting the values for all tracks, it's possible to get the + # album values. + rg_album_gain, rg_album_peak = rg.album_gain() + self._log.debug(u'ReplayGain for album {0}: {1:.2f}, {2:.2f}', + album, rg_album_gain, rg_album_peak) + + return AlbumGain( + Gain(gain=rg_album_gain, peak=rg_album_peak), + track_gains=track_gains + ) + + +# Main plugin logic. + +class ReplayGainPlugin(BeetsPlugin): + """Provides ReplayGain analysis. + """ + + backends = { + "command": CommandBackend, + "gstreamer": GStreamerBackend, + "audiotools": AudioToolsBackend, + "bs1770gain": Bs1770gainBackend + } + + def __init__(self): + super(ReplayGainPlugin, self).__init__() + + # default backend is 'command' for backward-compatibility. + self.config.add({ + 'overwrite': False, + 'auto': True, + 'backend': u'command', + 'targetlevel': 89, + }) + + self.overwrite = self.config['overwrite'].get(bool) + backend_name = self.config['backend'].get(unicode) + if backend_name not in self.backends: + raise ui.UserError( + u"Selected ReplayGain backend {0} is not supported. " + u"Please select one of: {1}".format( + backend_name, + u', '.join(self.backends.keys()) + ) + ) + + # On-import analysis. + if self.config['auto']: + self.import_stages = [self.imported] + + try: + self.backend_instance = self.backends[backend_name]( + self.config, self._log + ) + except (ReplayGainError, FatalReplayGainError) as e: + raise ui.UserError( + u'replaygain initialization failed: {0}'.format(e)) + + def track_requires_gain(self, item): + return self.overwrite or \ + (not item.rg_track_gain or not item.rg_track_peak) + + def album_requires_gain(self, album): + # Skip calculating gain only when *all* files don't need + # recalculation. This way, if any file among an album's tracks + # needs recalculation, we still get an accurate album gain + # value. + return self.overwrite or \ + any([not item.rg_album_gain or not item.rg_album_peak + for item in album.items()]) + + def store_track_gain(self, item, track_gain): + item.rg_track_gain = track_gain.gain + item.rg_track_peak = track_gain.peak + item.store() + + self._log.debug(u'applied track gain {0}, peak {1}', + item.rg_track_gain, item.rg_track_peak) + + def store_album_gain(self, album, album_gain): + album.rg_album_gain = album_gain.gain + album.rg_album_peak = album_gain.peak + album.store() + + self._log.debug(u'applied album gain {0}, peak {1}', + album.rg_album_gain, album.rg_album_peak) + + def handle_album(self, album, write): + """Compute album and track replay gain store it in all of the + album's items. + + If ``write`` is truthy then ``item.write()`` is called for each + item. If replay gain information is already present in all + items, nothing is done. + """ + if not self.album_requires_gain(album): + self._log.info(u'Skipping album {0}', album) + return + + self._log.info(u'analyzing {0}', album) + + try: + album_gain = self.backend_instance.compute_album_gain(album) + if len(album_gain.track_gains) != len(album.items()): + raise ReplayGainError( + u"ReplayGain backend failed " + u"for some tracks in album {0}".format(album) + ) + + self.store_album_gain(album, album_gain.album_gain) + for item, track_gain in itertools.izip(album.items(), + album_gain.track_gains): + self.store_track_gain(item, track_gain) + if write: + item.try_write() + except ReplayGainError as e: + self._log.info(u"ReplayGain error: {0}", e) + except FatalReplayGainError as e: + raise ui.UserError( + u"Fatal replay gain error: {0}".format(e)) + + def handle_track(self, item, write): + """Compute track replay gain and store it in the item. + + If ``write`` is truthy then ``item.write()`` is called to write + the data to disk. If replay gain information is already present + in the item, nothing is done. + """ + if not self.track_requires_gain(item): + self._log.info(u'Skipping track {0}', item) + return + + self._log.info(u'analyzing {0}', item) + + try: + track_gains = self.backend_instance.compute_track_gain([item]) + if len(track_gains) != 1: + raise ReplayGainError( + u"ReplayGain backend failed for track {0}".format(item) + ) + + self.store_track_gain(item, track_gains[0]) + if write: + item.try_write() + except ReplayGainError as e: + self._log.info(u"ReplayGain error: {0}", e) + except FatalReplayGainError as e: + raise ui.UserError( + u"Fatal replay gain error: {0}".format(e)) + + def imported(self, session, task): + """Add replay gain info to items or albums of ``task``. + """ + if task.is_album: + self.handle_album(task.album, False) + else: + self.handle_track(task.item, False) + + def commands(self): + """Return the "replaygain" ui subcommand. + """ + def func(lib, opts, args): + self._log.setLevel(logging.INFO) + + write = ui.should_write() + + if opts.album: + for album in lib.albums(ui.decargs(args)): + self.handle_album(album, write) + + else: + for item in lib.items(ui.decargs(args)): + self.handle_track(item, write) + + cmd = ui.Subcommand('replaygain', help=u'analyze for ReplayGain') + cmd.parser.add_album_option() + cmd.func = func + return [cmd] diff --git a/libs/beetsplug/rewrite.py b/libs/beetsplug/rewrite.py new file mode 100644 index 00000000..b0104a11 --- /dev/null +++ b/libs/beetsplug/rewrite.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Adrian Sampson. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Uses user-specified rewriting rules to canonicalize names for path +formats. +""" +from __future__ import division, absolute_import, print_function + +import re +from collections import defaultdict + +from beets.plugins import BeetsPlugin +from beets import ui +from beets import library + + +def rewriter(field, rules): + """Create a template field function that rewrites the given field + with the given rewriting rules. ``rules`` must be a list of + (pattern, replacement) pairs. + """ + def fieldfunc(item): + value = item._values_fixed[field] + for pattern, replacement in rules: + if pattern.match(value.lower()): + # Rewrite activated. + return replacement + # Not activated; return original value. + return value + return fieldfunc + + +class RewritePlugin(BeetsPlugin): + def __init__(self): + super(RewritePlugin, self).__init__() + + self.config.add({}) + + # Gather all the rewrite rules for each field. + rules = defaultdict(list) + for key, view in self.config.items(): + value = view.get(unicode) + try: + fieldname, pattern = key.split(None, 1) + except ValueError: + raise ui.UserError(u"invalid rewrite specification") + if fieldname not in library.Item._fields: + raise ui.UserError(u"invalid field name (%s) in rewriter" % + fieldname) + self._log.debug(u'adding template field {0}', key) + pattern = re.compile(pattern.lower()) + rules[fieldname].append((pattern, value)) + if fieldname == 'artist': + # Special case for the artist field: apply the same + # rewrite for "albumartist" as well. + rules['albumartist'].append((pattern, value)) + + # Replace each template field with the new rewriter function. + for fieldname, fieldrules in rules.iteritems(): + getter = rewriter(fieldname, fieldrules) + self.template_fields[fieldname] = getter + if fieldname in library.Album._fields: + self.album_template_fields[fieldname] = getter diff --git a/libs/beetsplug/scrub.py b/libs/beetsplug/scrub.py new file mode 100644 index 00000000..ed4040d5 --- /dev/null +++ b/libs/beetsplug/scrub.py @@ -0,0 +1,146 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Adrian Sampson. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Cleans extraneous metadata from files' tags via a command or +automatically whenever tags are written. +""" + +from __future__ import division, absolute_import, print_function + +from beets.plugins import BeetsPlugin +from beets import ui +from beets import util +from beets import config +from beets import mediafile + +_MUTAGEN_FORMATS = { + 'asf': 'ASF', + 'apev2': 'APEv2File', + 'flac': 'FLAC', + 'id3': 'ID3FileType', + 'mp3': 'MP3', + 'mp4': 'MP4', + 'oggflac': 'OggFLAC', + 'oggspeex': 'OggSpeex', + 'oggtheora': 'OggTheora', + 'oggvorbis': 'OggVorbis', + 'oggopus': 'OggOpus', + 'trueaudio': 'TrueAudio', + 'wavpack': 'WavPack', + 'monkeysaudio': 'MonkeysAudio', + 'optimfrog': 'OptimFROG', +} + + +class ScrubPlugin(BeetsPlugin): + """Removes extraneous metadata from files' tags.""" + def __init__(self): + super(ScrubPlugin, self).__init__() + self.config.add({ + 'auto': True, + }) + + if self.config['auto']: + self.register_listener("import_task_files", self.import_task_files) + + def commands(self): + def scrub_func(lib, opts, args): + # Walk through matching files and remove tags. + for item in lib.items(ui.decargs(args)): + self._log.info(u'scrubbing: {0}', + util.displayable_path(item.path)) + self._scrub_item(item, opts.write) + + scrub_cmd = ui.Subcommand('scrub', help=u'clean audio tags') + scrub_cmd.parser.add_option( + u'-W', u'--nowrite', dest='write', + action='store_false', default=True, + help=u'leave tags empty') + scrub_cmd.func = scrub_func + + return [scrub_cmd] + + @staticmethod + def _mutagen_classes(): + """Get a list of file type classes from the Mutagen module. + """ + classes = [] + for modname, clsname in _MUTAGEN_FORMATS.items(): + mod = __import__('mutagen.{0}'.format(modname), + fromlist=[clsname]) + classes.append(getattr(mod, clsname)) + return classes + + def _scrub(self, path): + """Remove all tags from a file. + """ + for cls in self._mutagen_classes(): + # Try opening the file with this type, but just skip in the + # event of any error. + try: + f = cls(util.syspath(path)) + except Exception: + continue + if f.tags is None: + continue + + # Remove the tag for this type. + try: + f.delete() + except NotImplementedError: + # Some Mutagen metadata subclasses (namely, ASFTag) do not + # support .delete(), presumably because it is impossible to + # remove them. In this case, we just remove all the tags. + for tag in f.keys(): + del f[tag] + f.save() + except IOError as exc: + self._log.error(u'could not scrub {0}: {1}', + util.displayable_path(path), exc) + + def _scrub_item(self, item, restore=True): + """Remove tags from an Item's associated file and, if `restore` + is enabled, write the database's tags back to the file. + """ + # Get album art if we need to restore it. + if restore: + try: + mf = mediafile.MediaFile(util.syspath(item.path), + config['id3v23'].get(bool)) + except IOError as exc: + self._log.error(u'could not open file to scrub: {0}', + exc) + art = mf.art + + # Remove all tags. + self._scrub(item.path) + + # Restore tags, if enabled. + if restore: + self._log.debug(u'writing new tags after scrub') + item.try_write() + if art: + self._log.debug(u'restoring art') + mf = mediafile.MediaFile(util.syspath(item.path), + config['id3v23'].get(bool)) + mf.art = art + mf.save() + + def import_task_files(self, session, task): + """Automatically scrub imported files.""" + for item in task.imported_items(): + self._log.debug(u'auto-scrubbing {0}', + util.displayable_path(item.path)) + self._scrub_item(item) diff --git a/libs/beetsplug/smartplaylist.py b/libs/beetsplug/smartplaylist.py new file mode 100644 index 00000000..f6d7f715 --- /dev/null +++ b/libs/beetsplug/smartplaylist.py @@ -0,0 +1,202 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Dang Mai <contact@dangmai.net>. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Generates smart playlists based on beets queries. +""" + +from __future__ import division, absolute_import, print_function + +from beets.plugins import BeetsPlugin +from beets import ui +from beets.util import mkdirall, normpath, syspath +from beets.library import Item, Album, parse_query_string +from beets.dbcore import OrQuery +from beets.dbcore.query import MultipleSort, ParsingError +import os + + +class SmartPlaylistPlugin(BeetsPlugin): + + def __init__(self): + super(SmartPlaylistPlugin, self).__init__() + self.config.add({ + 'relative_to': None, + 'playlist_dir': u'.', + 'auto': True, + 'playlists': [] + }) + + self._matched_playlists = None + self._unmatched_playlists = None + + if self.config['auto']: + self.register_listener('database_change', self.db_change) + + def commands(self): + spl_update = ui.Subcommand( + 'splupdate', + help=u'update the smart playlists. Playlist names may be ' + u'passed as arguments.' + ) + spl_update.func = self.update_cmd + return [spl_update] + + def update_cmd(self, lib, opts, args): + self.build_queries() + if args: + args = set(ui.decargs(args)) + for a in list(args): + if not a.endswith(".m3u"): + args.add("{0}.m3u".format(a)) + + playlists = set((name, q, a_q) + for name, q, a_q in self._unmatched_playlists + if name in args) + if not playlists: + raise ui.UserError( + u'No playlist matching any of {0} found'.format( + [name for name, _, _ in self._unmatched_playlists]) + ) + + self._matched_playlists = playlists + self._unmatched_playlists -= playlists + else: + self._matched_playlists = self._unmatched_playlists + + self.update_playlists(lib) + + def build_queries(self): + """ + Instanciate queries for the playlists. + + Each playlist has 2 queries: one or items one for albums, each with a + sort. We must also remember its name. _unmatched_playlists is a set of + tuples (name, (q, q_sort), (album_q, album_q_sort)). + + sort may be any sort, or NullSort, or None. None and NullSort are + equivalent and both eval to False. + More precisely + - it will be NullSort when a playlist query ('query' or 'album_query') + is a single item or a list with 1 element + - it will be None when there are multiple items i a query + """ + self._unmatched_playlists = set() + self._matched_playlists = set() + + for playlist in self.config['playlists'].get(list): + if 'name' not in playlist: + self._log.warn(u"playlist configuration is missing name") + continue + + playlist_data = (playlist['name'],) + try: + for key, Model in (('query', Item), ('album_query', Album)): + qs = playlist.get(key) + if qs is None: + query_and_sort = None, None + elif isinstance(qs, basestring): + query_and_sort = parse_query_string(qs, Model) + elif len(qs) == 1: + query_and_sort = parse_query_string(qs[0], Model) + else: + # multiple queries and sorts + queries, sorts = zip(*(parse_query_string(q, Model) + for q in qs)) + query = OrQuery(queries) + final_sorts = [] + for s in sorts: + if s: + if isinstance(s, MultipleSort): + final_sorts += s.sorts + else: + final_sorts.append(s) + if not final_sorts: + sort = None + elif len(final_sorts) == 1: + sort, = final_sorts + else: + sort = MultipleSort(final_sorts) + query_and_sort = query, sort + + playlist_data += (query_and_sort,) + + except ParsingError as exc: + self._log.warn(u"invalid query in playlist {}: {}", + playlist['name'], exc) + continue + + self._unmatched_playlists.add(playlist_data) + + def matches(self, model, query, album_query): + if album_query and isinstance(model, Album): + return album_query.match(model) + if query and isinstance(model, Item): + return query.match(model) + return False + + def db_change(self, lib, model): + if self._unmatched_playlists is None: + self.build_queries() + + for playlist in self._unmatched_playlists: + n, (q, _), (a_q, _) = playlist + if self.matches(model, q, a_q): + self._log.debug( + u"{0} will be updated because of {1}", n, model) + self._matched_playlists.add(playlist) + self.register_listener('cli_exit', self.update_playlists) + + self._unmatched_playlists -= self._matched_playlists + + def update_playlists(self, lib): + self._log.info(u"Updating {0} smart playlists...", + len(self._matched_playlists)) + + playlist_dir = self.config['playlist_dir'].as_filename() + relative_to = self.config['relative_to'].get() + if relative_to: + relative_to = normpath(relative_to) + + for playlist in self._matched_playlists: + name, (query, q_sort), (album_query, a_q_sort) = playlist + self._log.debug(u"Creating playlist {0}", name) + items = [] + + if query: + items.extend(lib.items(query, q_sort)) + if album_query: + for album in lib.albums(album_query, a_q_sort): + items.extend(album.items()) + + m3us = {} + # As we allow tags in the m3u names, we'll need to iterate through + # the items and generate the correct m3u file names. + for item in items: + m3u_name = item.evaluate_template(name, True) + if m3u_name not in m3us: + m3us[m3u_name] = [] + item_path = item.path + if relative_to: + item_path = os.path.relpath(item.path, relative_to) + if item_path not in m3us[m3u_name]: + m3us[m3u_name].append(item_path) + # Now iterate through the m3us that we need to generate + for m3u in m3us: + m3u_path = normpath(os.path.join(playlist_dir, m3u)) + mkdirall(m3u_path) + with open(syspath(m3u_path), 'w') as f: + for path in m3us[m3u]: + f.write(path + b'\n') + self._log.info(u"{0} playlists updated", len(self._matched_playlists)) diff --git a/libs/beetsplug/spotify.py b/libs/beetsplug/spotify.py new file mode 100644 index 00000000..081a027f --- /dev/null +++ b/libs/beetsplug/spotify.py @@ -0,0 +1,176 @@ +# -*- coding: utf-8 -*- + +from __future__ import division, absolute_import, print_function + +import re +import webbrowser +import requests +from beets.plugins import BeetsPlugin +from beets.ui import decargs +from beets import ui +from requests.exceptions import HTTPError + + +class SpotifyPlugin(BeetsPlugin): + + # URL for the Web API of Spotify + # Documentation here: https://developer.spotify.com/web-api/search-item/ + base_url = "https://api.spotify.com/v1/search" + open_url = "http://open.spotify.com/track/" + playlist_partial = "spotify:trackset:Playlist:" + + def __init__(self): + super(SpotifyPlugin, self).__init__() + self.config.add({ + 'mode': 'list', + 'tiebreak': 'popularity', + 'show_failures': False, + 'artist_field': 'albumartist', + 'album_field': 'album', + 'track_field': 'title', + 'region_filter': None, + 'regex': [] + }) + + def commands(self): + def queries(lib, opts, args): + success = self.parse_opts(opts) + if success: + results = self.query_spotify(lib, decargs(args)) + self.output_results(results) + spotify_cmd = ui.Subcommand( + 'spotify', + help=u'build a Spotify playlist' + ) + spotify_cmd.parser.add_option( + u'-m', u'--mode', action='store', + help=u'"open" to open Spotify with playlist, ' + u'"list" to print (default)' + ) + spotify_cmd.parser.add_option( + u'-f', u'--show-failures', + action='store_true', dest='show_failures', + help=u'list tracks that did not match a Spotify ID' + ) + spotify_cmd.func = queries + return [spotify_cmd] + + def parse_opts(self, opts): + if opts.mode: + self.config['mode'].set(opts.mode) + + if opts.show_failures: + self.config['show_failures'].set(True) + + if self.config['mode'].get() not in ['list', 'open']: + self._log.warn(u'{0} is not a valid mode', + self.config['mode'].get()) + return False + + self.opts = opts + return True + + def query_spotify(self, lib, query): + + results = [] + failures = [] + + items = lib.items(query) + + if not items: + self._log.debug(u'Your beets query returned no items, ' + u'skipping spotify') + return + + self._log.info(u'Processing {0} tracks...', len(items)) + + for item in items: + + # Apply regex transformations if provided + for regex in self.config['regex'].get(): + if ( + not regex['field'] or + not regex['search'] or + not regex['replace'] + ): + continue + + value = item[regex['field']] + item[regex['field']] = re.sub( + regex['search'], regex['replace'], value + ) + + # Custom values can be passed in the config (just in case) + artist = item[self.config['artist_field'].get()] + album = item[self.config['album_field'].get()] + query = item[self.config['track_field'].get()] + search_url = query + " album:" + album + " artist:" + artist + + # Query the Web API for each track, look for the items' JSON data + r = requests.get(self.base_url, params={ + "q": search_url, "type": "track" + }) + self._log.debug('{}', r.url) + try: + r.raise_for_status() + except HTTPError as e: + self._log.debug(u'URL returned a {0} error', + e.response.status_code) + failures.append(search_url) + continue + + r_data = r.json()['tracks']['items'] + + # Apply market filter if requested + region_filter = self.config['region_filter'].get() + if region_filter: + r_data = filter( + lambda x: region_filter in x['available_markets'], r_data + ) + + # Simplest, take the first result + chosen_result = None + if len(r_data) == 1 or self.config['tiebreak'].get() == "first": + self._log.debug(u'Spotify track(s) found, count: {0}', + len(r_data)) + chosen_result = r_data[0] + elif len(r_data) > 1: + # Use the popularity filter + self._log.debug(u'Most popular track chosen, count: {0}', + len(r_data)) + chosen_result = max(r_data, key=lambda x: x['popularity']) + + if chosen_result: + results.append(chosen_result) + else: + self._log.debug(u'No spotify track found: {0}', search_url) + failures.append(search_url) + + failure_count = len(failures) + if failure_count > 0: + if self.config['show_failures'].get(): + self._log.info(u'{0} track(s) did not match a Spotify ID:', + failure_count) + for track in failures: + self._log.info(u'track: {0}', track) + self._log.info(u'') + else: + self._log.warn(u'{0} track(s) did not match a Spotify ID;\n' + u'use --show-failures to display', + failure_count) + + return results + + def output_results(self, results): + if results: + ids = map(lambda x: x['id'], results) + if self.config['mode'].get() == "open": + self._log.info(u'Attempting to open Spotify with playlist') + spotify_url = self.playlist_partial + ",".join(ids) + webbrowser.open(spotify_url) + + else: + for item in ids: + print(unicode.encode(self.open_url + item)) + else: + self._log.warn(u'No Spotify tracks found from beets query') diff --git a/libs/beetsplug/the.py b/libs/beetsplug/the.py new file mode 100644 index 00000000..6bed4c6e --- /dev/null +++ b/libs/beetsplug/the.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Blemjhoo Tezoulbr <baobab@heresiarch.info>. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Moves patterns in path formats (suitable for moving articles).""" + +from __future__ import division, absolute_import, print_function + +import re +from beets.plugins import BeetsPlugin + +__author__ = 'baobab@heresiarch.info' +__version__ = '1.1' + +PATTERN_THE = u'^[the]{3}\s' +PATTERN_A = u'^[a][n]?\s' +FORMAT = u'{0}, {1}' + + +class ThePlugin(BeetsPlugin): + + patterns = [] + + def __init__(self): + super(ThePlugin, self).__init__() + + self.template_funcs['the'] = self.the_template_func + + self.config.add({ + 'the': True, + 'a': True, + 'format': u'{0}, {1}', + 'strip': False, + 'patterns': [], + }) + + self.patterns = self.config['patterns'].as_str_seq() + for p in self.patterns: + if p: + try: + re.compile(p) + except re.error: + self._log.error(u'invalid pattern: {0}', p) + else: + if not (p.startswith('^') or p.endswith('$')): + self._log.warn(u'warning: \"{0}\" will not ' + u'match string start/end', p) + if self.config['a']: + self.patterns = [PATTERN_A] + self.patterns + if self.config['the']: + self.patterns = [PATTERN_THE] + self.patterns + if not self.patterns: + self._log.warn(u'no patterns defined!') + + def unthe(self, text, pattern): + """Moves pattern in the path format string or strips it + + text -- text to handle + pattern -- regexp pattern (case ignore is already on) + strip -- if True, pattern will be removed + """ + if text: + r = re.compile(pattern, flags=re.IGNORECASE) + try: + t = r.findall(text)[0] + except IndexError: + return text + else: + r = re.sub(r, '', text).strip() + if self.config['strip']: + return r + else: + fmt = self.config['format'].get(unicode) + return fmt.format(r, t.strip()).strip() + else: + return u'' + + def the_template_func(self, text): + if not self.patterns: + return text + if text: + for p in self.patterns: + r = self.unthe(text, p) + if r != text: + break + self._log.debug(u'\"{0}\" -> \"{1}\"', text, r) + return r + else: + return u'' diff --git a/libs/beetsplug/thumbnails.py b/libs/beetsplug/thumbnails.py new file mode 100644 index 00000000..0e7fbc6e --- /dev/null +++ b/libs/beetsplug/thumbnails.py @@ -0,0 +1,289 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Bruno Cauet +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Create freedesktop.org-compliant thumbnails for album folders + +This plugin is POSIX-only. +Spec: standards.freedesktop.org/thumbnail-spec/latest/index.html +""" + +from __future__ import division, absolute_import, print_function + +from hashlib import md5 +import os +import shutil +from itertools import chain +from pathlib import PurePosixPath +import ctypes +import ctypes.util + +from xdg import BaseDirectory + +from beets.plugins import BeetsPlugin +from beets.ui import Subcommand, decargs +from beets import util +from beets.util.artresizer import ArtResizer, get_im_version, get_pil_version + + +BASE_DIR = os.path.join(BaseDirectory.xdg_cache_home, "thumbnails") +NORMAL_DIR = util.bytestring_path(os.path.join(BASE_DIR, "normal")) +LARGE_DIR = util.bytestring_path(os.path.join(BASE_DIR, "large")) + + +class ThumbnailsPlugin(BeetsPlugin): + def __init__(self): + super(ThumbnailsPlugin, self).__init__() + self.config.add({ + 'auto': True, + 'force': False, + 'dolphin': False, + }) + + self.write_metadata = None + if self.config['auto'] and self._check_local_ok(): + self.register_listener('art_set', self.process_album) + + def commands(self): + thumbnails_command = Subcommand("thumbnails", + help=u"Create album thumbnails") + thumbnails_command.parser.add_option( + u'-f', u'--force', + dest='force', action='store_true', default=False, + help=u'force regeneration of thumbnails deemed fine (existing & ' + u'recent enough)') + thumbnails_command.parser.add_option( + u'--dolphin', dest='dolphin', action='store_true', default=False, + help=u"create Dolphin-compatible thumbnail information (for KDE)") + thumbnails_command.func = self.process_query + + return [thumbnails_command] + + def process_query(self, lib, opts, args): + self.config.set_args(opts) + if self._check_local_ok(): + for album in lib.albums(decargs(args)): + self.process_album(album) + + def _check_local_ok(self): + """Check that's everythings ready: + - local capability to resize images + - thumbnail dirs exist (create them if needed) + - detect whether we'll use PIL or IM + - detect whether we'll use GIO or Python to get URIs + """ + if not ArtResizer.shared.local: + self._log.warning(u"No local image resizing capabilities, " + u"cannot generate thumbnails") + return False + + for dir in (NORMAL_DIR, LARGE_DIR): + if not os.path.exists(dir): + os.makedirs(dir) + + if get_im_version(): + self.write_metadata = write_metadata_im + tool = "IM" + else: + assert get_pil_version() # since we're local + self.write_metadata = write_metadata_pil + tool = "PIL" + self._log.debug(u"using {0} to write metadata", tool) + + uri_getter = GioURI() + if not uri_getter.available: + uri_getter = PathlibURI() + self._log.debug(u"using {0.name} to compute URIs", uri_getter) + self.get_uri = uri_getter.uri + + return True + + def process_album(self, album): + """Produce thumbnails for the album folder. + """ + self._log.debug(u'generating thumbnail for {0}', album) + if not album.artpath: + self._log.info(u'album {0} has no art', album) + return + + if self.config['dolphin']: + self.make_dolphin_cover_thumbnail(album) + + size = ArtResizer.shared.get_size(album.artpath) + if not size: + self._log.warning(u'problem getting the picture size for {0}', + album.artpath) + return + + wrote = True + if max(size) >= 256: + wrote &= self.make_cover_thumbnail(album, 256, LARGE_DIR) + wrote &= self.make_cover_thumbnail(album, 128, NORMAL_DIR) + + if wrote: + self._log.info(u'wrote thumbnail for {0}', album) + else: + self._log.info(u'nothing to do for {0}', album) + + def make_cover_thumbnail(self, album, size, target_dir): + """Make a thumbnail of given size for `album` and put it in + `target_dir`. + """ + target = os.path.join(target_dir, self.thumbnail_file_name(album.path)) + + if os.path.exists(target) and \ + os.stat(target).st_mtime > os.stat(album.artpath).st_mtime: + if self.config['force']: + self._log.debug(u"found a suitable {1}x{1} thumbnail for {0}, " + u"forcing regeneration", album, size) + else: + self._log.debug(u"{1}x{1} thumbnail for {0} exists and is " + u"recent enough", album, size) + return False + resized = ArtResizer.shared.resize(size, album.artpath, + util.syspath(target)) + self.add_tags(album, util.syspath(resized)) + shutil.move(resized, target) + return True + + def thumbnail_file_name(self, path): + """Compute the thumbnail file name + See http://standards.freedesktop.org/thumbnail-spec/latest/x227.html + """ + uri = self.get_uri(path) + hash = md5(uri).hexdigest() + return b"{0}.png".format(hash) + + def add_tags(self, album, image_path): + """Write required metadata to the thumbnail + See http://standards.freedesktop.org/thumbnail-spec/latest/x142.html + """ + metadata = {"Thumb::URI": self.get_uri(album.artpath), + "Thumb::MTime": unicode(os.stat(album.artpath).st_mtime)} + try: + self.write_metadata(image_path, metadata) + except Exception: + self._log.exception(u"could not write metadata to {0}", + util.displayable_path(image_path)) + + def make_dolphin_cover_thumbnail(self, album): + outfilename = os.path.join(album.path, b".directory") + if os.path.exists(outfilename): + return + artfile = os.path.split(album.artpath)[1] + with open(outfilename, 'w') as f: + f.write(b"[Desktop Entry]\nIcon=./{0}".format(artfile)) + f.close() + self._log.debug(u"Wrote file {0}", util.displayable_path(outfilename)) + + +def write_metadata_im(file, metadata): + """Enrich the file metadata with `metadata` dict thanks to IM.""" + command = ['convert', file] + \ + list(chain.from_iterable(('-set', k, v) + for k, v in metadata.items())) + [file] + util.command_output(command) + return True + + +def write_metadata_pil(file, metadata): + """Enrich the file metadata with `metadata` dict thanks to PIL.""" + from PIL import Image, PngImagePlugin + im = Image.open(file) + meta = PngImagePlugin.PngInfo() + for k, v in metadata.items(): + meta.add_text(k, v, 0) + im.save(file, "PNG", pnginfo=meta) + return True + + +class URIGetter(object): + available = False + name = "Abstract base" + + def uri(self, path): + raise NotImplementedError() + + +class PathlibURI(URIGetter): + available = True + name = "Python Pathlib" + + def uri(self, path): + return PurePosixPath(path).as_uri() + + +def copy_c_string(c_string): + """Copy a `ctypes.POINTER(ctypes.c_char)` value into a new Python + string and return it. The old memory is then safe to free. + """ + # This is a pretty dumb way to get a string copy, but it seems to + # work. A more surefire way would be to allocate a ctypes buffer and copy + # the data with `memcpy` or somesuch. + s = ctypes.cast(c_string, ctypes.c_char_p).value + return '' + s + + +class GioURI(URIGetter): + """Use gio URI function g_file_get_uri. Paths must be utf-8 encoded. + """ + name = "GIO" + + def __init__(self): + self.libgio = self.get_library() + self.available = bool(self.libgio) + if self.available: + self.libgio.g_type_init() # for glib < 2.36 + + self.libgio.g_file_get_uri.argtypes = [ctypes.c_char_p] + self.libgio.g_file_new_for_path.restype = ctypes.c_void_p + + self.libgio.g_file_get_uri.argtypes = [ctypes.c_void_p] + self.libgio.g_file_get_uri.restype = ctypes.POINTER(ctypes.c_char) + + self.libgio.g_object_unref.argtypes = [ctypes.c_void_p] + + def get_library(self): + lib_name = ctypes.util.find_library("gio-2") + try: + if not lib_name: + return False + return ctypes.cdll.LoadLibrary(lib_name) + except OSError: + return False + + def uri(self, path): + g_file_ptr = self.libgio.g_file_new_for_path(path) + if not g_file_ptr: + raise RuntimeError(u"No gfile pointer received for {0}".format( + util.displayable_path(path))) + + try: + uri_ptr = self.libgio.g_file_get_uri(g_file_ptr) + except: + raise + finally: + self.libgio.g_object_unref(g_file_ptr) + if not uri_ptr: + self.libgio.g_free(uri_ptr) + raise RuntimeError(u"No URI received from the gfile pointer for " + u"{0}".format(util.displayable_path(path))) + + try: + uri = copy_c_string(uri_ptr) + except: + raise + finally: + self.libgio.g_free(uri_ptr) + return uri diff --git a/libs/beetsplug/types.py b/libs/beetsplug/types.py new file mode 100644 index 00000000..0c078881 --- /dev/null +++ b/libs/beetsplug/types.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Thomas Scholtes. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +from __future__ import division, absolute_import, print_function + +from beets.plugins import BeetsPlugin +from beets.dbcore import types +from beets.util.confit import ConfigValueError +from beets import library + + +class TypesPlugin(BeetsPlugin): + + @property + def item_types(self): + return self._types() + + @property + def album_types(self): + return self._types() + + def _types(self): + if not self.config.exists(): + return {} + + mytypes = {} + for key, value in self.config.items(): + if value.get() == 'int': + mytypes[key] = types.INTEGER + elif value.get() == 'float': + mytypes[key] = types.FLOAT + elif value.get() == 'bool': + mytypes[key] = types.BOOLEAN + elif value.get() == 'date': + mytypes[key] = library.DateType() + else: + raise ConfigValueError( + u"unknown type '{0}' for the '{1}' field" + .format(value, key)) + return mytypes diff --git a/libs/beetsplug/web/__init__.py b/libs/beetsplug/web/__init__.py new file mode 100644 index 00000000..67d99db6 --- /dev/null +++ b/libs/beetsplug/web/__init__.py @@ -0,0 +1,328 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Adrian Sampson. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""A Web interface to beets.""" +from __future__ import division, absolute_import, print_function + +from beets.plugins import BeetsPlugin +from beets import ui +from beets import util +import beets.library +import flask +from flask import g +from werkzeug.routing import BaseConverter, PathConverter +import os +import json + + +# Utilities. + +def _rep(obj, expand=False): + """Get a flat -- i.e., JSON-ish -- representation of a beets Item or + Album object. For Albums, `expand` dictates whether tracks are + included. + """ + out = dict(obj) + + if isinstance(obj, beets.library.Item): + del out['path'] + + # Get the size (in bytes) of the backing file. This is useful + # for the Tomahawk resolver API. + try: + out['size'] = os.path.getsize(util.syspath(obj.path)) + except OSError: + out['size'] = 0 + + return out + + elif isinstance(obj, beets.library.Album): + del out['artpath'] + if expand: + out['items'] = [_rep(item) for item in obj.items()] + return out + + +def json_generator(items, root): + """Generator that dumps list of beets Items or Albums as JSON + + :param root: root key for JSON + :param items: list of :class:`Item` or :class:`Album` to dump + :returns: generator that yields strings + """ + yield '{"%s":[' % root + first = True + for item in items: + if first: + first = False + else: + yield ',' + yield json.dumps(_rep(item)) + yield ']}' + + +def resource(name): + """Decorates a function to handle RESTful HTTP requests for a resource. + """ + def make_responder(retriever): + def responder(ids): + entities = [retriever(id) for id in ids] + entities = [entity for entity in entities if entity] + + if len(entities) == 1: + return flask.jsonify(_rep(entities[0])) + elif entities: + return app.response_class( + json_generator(entities, root=name), + mimetype='application/json' + ) + else: + return flask.abort(404) + responder.__name__ = 'get_{0}'.format(name) + return responder + return make_responder + + +def resource_query(name): + """Decorates a function to handle RESTful HTTP queries for resources. + """ + def make_responder(query_func): + def responder(queries): + return app.response_class( + json_generator(query_func(queries), root='results'), + mimetype='application/json' + ) + responder.__name__ = 'query_{0}'.format(name) + return responder + return make_responder + + +def resource_list(name): + """Decorates a function to handle RESTful HTTP request for a list of + resources. + """ + def make_responder(list_all): + def responder(): + return app.response_class( + json_generator(list_all(), root=name), + mimetype='application/json' + ) + responder.__name__ = 'all_{0}'.format(name) + return responder + return make_responder + + +def _get_unique_table_field_values(model, field, sort_field): + """ retrieve all unique values belonging to a key from a model """ + if field not in model.all_keys() or sort_field not in model.all_keys(): + raise KeyError + with g.lib.transaction() as tx: + rows = tx.query('SELECT DISTINCT "{0}" FROM "{1}" ORDER BY "{2}"' + .format(field, model._table, sort_field)) + return [row[0] for row in rows] + + +class IdListConverter(BaseConverter): + """Converts comma separated lists of ids in urls to integer lists. + """ + + def to_python(self, value): + ids = [] + for id in value.split(','): + try: + ids.append(int(id)) + except ValueError: + pass + return ids + + def to_url(self, value): + return ','.join(value) + + +class QueryConverter(PathConverter): + """Converts slash separated lists of queries in the url to string list. + """ + + def to_python(self, value): + return value.split('/') + + def to_url(self, value): + return ','.join(value) + + +# Flask setup. + +app = flask.Flask(__name__) +app.url_map.converters['idlist'] = IdListConverter +app.url_map.converters['query'] = QueryConverter + + +@app.before_request +def before_request(): + g.lib = app.config['lib'] + + +# Items. + +@app.route('/item/<idlist:ids>') +@resource('items') +def get_item(id): + return g.lib.get_item(id) + + +@app.route('/item/') +@app.route('/item/query/') +@resource_list('items') +def all_items(): + return g.lib.items() + + +@app.route('/item/<int:item_id>/file') +def item_file(item_id): + item = g.lib.get_item(item_id) + response = flask.send_file(item.path, as_attachment=True, + attachment_filename=os.path.basename(item.path)) + response.headers['Content-Length'] = os.path.getsize(item.path) + return response + + +@app.route('/item/query/<query:queries>') +@resource_query('items') +def item_query(queries): + return g.lib.items(queries) + + +@app.route('/item/values/<string:key>') +def item_unique_field_values(key): + sort_key = flask.request.args.get('sort_key', key) + try: + values = _get_unique_table_field_values(beets.library.Item, key, + sort_key) + except KeyError: + return flask.abort(404) + return flask.jsonify(values=values) + + +# Albums. + +@app.route('/album/<idlist:ids>') +@resource('albums') +def get_album(id): + return g.lib.get_album(id) + + +@app.route('/album/') +@app.route('/album/query/') +@resource_list('albums') +def all_albums(): + return g.lib.albums() + + +@app.route('/album/query/<query:queries>') +@resource_query('albums') +def album_query(queries): + return g.lib.albums(queries) + + +@app.route('/album/<int:album_id>/art') +def album_art(album_id): + album = g.lib.get_album(album_id) + if album.artpath: + return flask.send_file(album.artpath) + else: + return flask.abort(404) + + +@app.route('/album/values/<string:key>') +def album_unique_field_values(key): + sort_key = flask.request.args.get('sort_key', key) + try: + values = _get_unique_table_field_values(beets.library.Album, key, + sort_key) + except KeyError: + return flask.abort(404) + return flask.jsonify(values=values) + + +# Artists. + +@app.route('/artist/') +def all_artists(): + with g.lib.transaction() as tx: + rows = tx.query("SELECT DISTINCT albumartist FROM albums") + all_artists = [row[0] for row in rows] + return flask.jsonify(artist_names=all_artists) + + +# Library information. + +@app.route('/stats') +def stats(): + with g.lib.transaction() as tx: + item_rows = tx.query("SELECT COUNT(*) FROM items") + album_rows = tx.query("SELECT COUNT(*) FROM albums") + return flask.jsonify({ + 'items': item_rows[0][0], + 'albums': album_rows[0][0], + }) + + +# UI. + +@app.route('/') +def home(): + return flask.render_template('index.html') + + +# Plugin hook. + +class WebPlugin(BeetsPlugin): + def __init__(self): + super(WebPlugin, self).__init__() + self.config.add({ + 'host': u'127.0.0.1', + 'port': 8337, + 'cors': '', + }) + + def commands(self): + cmd = ui.Subcommand('web', help=u'start a Web interface') + cmd.parser.add_option(u'-d', u'--debug', action='store_true', + default=False, help=u'debug mode') + + def func(lib, opts, args): + args = ui.decargs(args) + if args: + self.config['host'] = args.pop(0) + if args: + self.config['port'] = int(args.pop(0)) + + app.config['lib'] = lib + # Enable CORS if required. + if self.config['cors']: + self._log.info(u'Enabling CORS with origin: {0}', + self.config['cors']) + from flask.ext.cors import CORS + app.config['CORS_ALLOW_HEADERS'] = "Content-Type" + app.config['CORS_RESOURCES'] = { + r"/*": {"origins": self.config['cors'].get(str)} + } + CORS(app) + # Start the web application. + app.run(host=self.config['host'].get(unicode), + port=self.config['port'].get(int), + debug=opts.debug, threaded=True) + cmd.func = func + return [cmd] diff --git a/libs/beetsplug/web/static/backbone.js b/libs/beetsplug/web/static/backbone.js new file mode 100644 index 00000000..b2e49322 --- /dev/null +++ b/libs/beetsplug/web/static/backbone.js @@ -0,0 +1,1158 @@ +// Backbone.js 0.5.3 +// (c) 2010 Jeremy Ashkenas, DocumentCloud Inc. +// Backbone may be freely distributed under the MIT license. +// For all details and documentation: +// http://documentcloud.github.com/backbone + +(function(){ + + // Initial Setup + // ------------- + + // Save a reference to the global object. + var root = this; + + // Save the previous value of the `Backbone` variable. + var previousBackbone = root.Backbone; + + // The top-level namespace. All public Backbone classes and modules will + // be attached to this. Exported for both CommonJS and the browser. + var Backbone; + if (typeof exports !== 'undefined') { + Backbone = exports; + } else { + Backbone = root.Backbone = {}; + } + + // Current version of the library. Keep in sync with `package.json`. + Backbone.VERSION = '0.5.3'; + + // Require Underscore, if we're on the server, and it's not already present. + var _ = root._; + if (!_ && (typeof require !== 'undefined')) _ = require('underscore')._; + + // For Backbone's purposes, jQuery or Zepto owns the `$` variable. + var $ = root.jQuery || root.Zepto; + + // Runs Backbone.js in *noConflict* mode, returning the `Backbone` variable + // to its previous owner. Returns a reference to this Backbone object. + Backbone.noConflict = function() { + root.Backbone = previousBackbone; + return this; + }; + + // Turn on `emulateHTTP` to support legacy HTTP servers. Setting this option will + // fake `"PUT"` and `"DELETE"` requests via the `_method` parameter and set a + // `X-Http-Method-Override` header. + Backbone.emulateHTTP = false; + + // Turn on `emulateJSON` to support legacy servers that can't deal with direct + // `application/json` requests ... will encode the body as + // `application/x-www-form-urlencoded` instead and will send the model in a + // form param named `model`. + Backbone.emulateJSON = false; + + // Backbone.Events + // ----------------- + + // A module that can be mixed in to *any object* in order to provide it with + // custom events. You may `bind` or `unbind` a callback function to an event; + // `trigger`-ing an event fires all callbacks in succession. + // + // var object = {}; + // _.extend(object, Backbone.Events); + // object.bind('expand', function(){ alert('expanded'); }); + // object.trigger('expand'); + // + Backbone.Events = { + + // Bind an event, specified by a string name, `ev`, to a `callback` function. + // Passing `"all"` will bind the callback to all events fired. + bind : function(ev, callback, context) { + var calls = this._callbacks || (this._callbacks = {}); + var list = calls[ev] || (calls[ev] = []); + list.push([callback, context]); + return this; + }, + + // Remove one or many callbacks. If `callback` is null, removes all + // callbacks for the event. If `ev` is null, removes all bound callbacks + // for all events. + unbind : function(ev, callback) { + var calls; + if (!ev) { + this._callbacks = {}; + } else if (calls = this._callbacks) { + if (!callback) { + calls[ev] = []; + } else { + var list = calls[ev]; + if (!list) return this; + for (var i = 0, l = list.length; i < l; i++) { + if (list[i] && callback === list[i][0]) { + list[i] = null; + break; + } + } + } + } + return this; + }, + + // Trigger an event, firing all bound callbacks. Callbacks are passed the + // same arguments as `trigger` is, apart from the event name. + // Listening for `"all"` passes the true event name as the first argument. + trigger : function(eventName) { + var list, calls, ev, callback, args; + var both = 2; + if (!(calls = this._callbacks)) return this; + while (both--) { + ev = both ? eventName : 'all'; + if (list = calls[ev]) { + for (var i = 0, l = list.length; i < l; i++) { + if (!(callback = list[i])) { + list.splice(i, 1); i--; l--; + } else { + args = both ? Array.prototype.slice.call(arguments, 1) : arguments; + callback[0].apply(callback[1] || this, args); + } + } + } + } + return this; + } + + }; + + // Backbone.Model + // -------------- + + // Create a new model, with defined attributes. A client id (`cid`) + // is automatically generated and assigned for you. + Backbone.Model = function(attributes, options) { + var defaults; + attributes || (attributes = {}); + if (defaults = this.defaults) { + if (_.isFunction(defaults)) defaults = defaults.call(this); + attributes = _.extend({}, defaults, attributes); + } + this.attributes = {}; + this._escapedAttributes = {}; + this.cid = _.uniqueId('c'); + this.set(attributes, {silent : true}); + this._changed = false; + this._previousAttributes = _.clone(this.attributes); + if (options && options.collection) this.collection = options.collection; + this.initialize(attributes, options); + }; + + // Attach all inheritable methods to the Model prototype. + _.extend(Backbone.Model.prototype, Backbone.Events, { + + // A snapshot of the model's previous attributes, taken immediately + // after the last `"change"` event was fired. + _previousAttributes : null, + + // Has the item been changed since the last `"change"` event? + _changed : false, + + // The default name for the JSON `id` attribute is `"id"`. MongoDB and + // CouchDB users may want to set this to `"_id"`. + idAttribute : 'id', + + // Initialize is an empty function by default. Override it with your own + // initialization logic. + initialize : function(){}, + + // Return a copy of the model's `attributes` object. + toJSON : function() { + return _.clone(this.attributes); + }, + + // Get the value of an attribute. + get : function(attr) { + return this.attributes[attr]; + }, + + // Get the HTML-escaped value of an attribute. + escape : function(attr) { + var html; + if (html = this._escapedAttributes[attr]) return html; + var val = this.attributes[attr]; + return this._escapedAttributes[attr] = escapeHTML(val == null ? '' : '' + val); + }, + + // Returns `true` if the attribute contains a value that is not null + // or undefined. + has : function(attr) { + return this.attributes[attr] != null; + }, + + // Set a hash of model attributes on the object, firing `"change"` unless you + // choose to silence it. + set : function(attrs, options) { + + // Extract attributes and options. + options || (options = {}); + if (!attrs) return this; + if (attrs.attributes) attrs = attrs.attributes; + var now = this.attributes, escaped = this._escapedAttributes; + + // Run validation. + if (!options.silent && this.validate && !this._performValidation(attrs, options)) return false; + + // Check for changes of `id`. + if (this.idAttribute in attrs) this.id = attrs[this.idAttribute]; + + // We're about to start triggering change events. + var alreadyChanging = this._changing; + this._changing = true; + + // Update attributes. + for (var attr in attrs) { + var val = attrs[attr]; + if (!_.isEqual(now[attr], val)) { + now[attr] = val; + delete escaped[attr]; + this._changed = true; + if (!options.silent) this.trigger('change:' + attr, this, val, options); + } + } + + // Fire the `"change"` event, if the model has been changed. + if (!alreadyChanging && !options.silent && this._changed) this.change(options); + this._changing = false; + return this; + }, + + // Remove an attribute from the model, firing `"change"` unless you choose + // to silence it. `unset` is a noop if the attribute doesn't exist. + unset : function(attr, options) { + if (!(attr in this.attributes)) return this; + options || (options = {}); + var value = this.attributes[attr]; + + // Run validation. + var validObj = {}; + validObj[attr] = void 0; + if (!options.silent && this.validate && !this._performValidation(validObj, options)) return false; + + // Remove the attribute. + delete this.attributes[attr]; + delete this._escapedAttributes[attr]; + if (attr == this.idAttribute) delete this.id; + this._changed = true; + if (!options.silent) { + this.trigger('change:' + attr, this, void 0, options); + this.change(options); + } + return this; + }, + + // Clear all attributes on the model, firing `"change"` unless you choose + // to silence it. + clear : function(options) { + options || (options = {}); + var attr; + var old = this.attributes; + + // Run validation. + var validObj = {}; + for (attr in old) validObj[attr] = void 0; + if (!options.silent && this.validate && !this._performValidation(validObj, options)) return false; + + this.attributes = {}; + this._escapedAttributes = {}; + this._changed = true; + if (!options.silent) { + for (attr in old) { + this.trigger('change:' + attr, this, void 0, options); + } + this.change(options); + } + return this; + }, + + // Fetch the model from the server. If the server's representation of the + // model differs from its current attributes, they will be overriden, + // triggering a `"change"` event. + fetch : function(options) { + options || (options = {}); + var model = this; + var success = options.success; + options.success = function(resp, status, xhr) { + if (!model.set(model.parse(resp, xhr), options)) return false; + if (success) success(model, resp); + }; + options.error = wrapError(options.error, model, options); + return (this.sync || Backbone.sync).call(this, 'read', this, options); + }, + + // Set a hash of model attributes, and sync the model to the server. + // If the server returns an attributes hash that differs, the model's + // state will be `set` again. + save : function(attrs, options) { + options || (options = {}); + if (attrs && !this.set(attrs, options)) return false; + var model = this; + var success = options.success; + options.success = function(resp, status, xhr) { + if (!model.set(model.parse(resp, xhr), options)) return false; + if (success) success(model, resp, xhr); + }; + options.error = wrapError(options.error, model, options); + var method = this.isNew() ? 'create' : 'update'; + return (this.sync || Backbone.sync).call(this, method, this, options); + }, + + // Destroy this model on the server if it was already persisted. Upon success, the model is removed + // from its collection, if it has one. + destroy : function(options) { + options || (options = {}); + if (this.isNew()) return this.trigger('destroy', this, this.collection, options); + var model = this; + var success = options.success; + options.success = function(resp) { + model.trigger('destroy', model, model.collection, options); + if (success) success(model, resp); + }; + options.error = wrapError(options.error, model, options); + return (this.sync || Backbone.sync).call(this, 'delete', this, options); + }, + + // Default URL for the model's representation on the server -- if you're + // using Backbone's restful methods, override this to change the endpoint + // that will be called. + url : function() { + var base = getUrl(this.collection) || this.urlRoot || urlError(); + if (this.isNew()) return base; + return base + (base.charAt(base.length - 1) == '/' ? '' : '/') + encodeURIComponent(this.id); + }, + + // **parse** converts a response into the hash of attributes to be `set` on + // the model. The default implementation is just to pass the response along. + parse : function(resp, xhr) { + return resp; + }, + + // Create a new model with identical attributes to this one. + clone : function() { + return new this.constructor(this); + }, + + // A model is new if it has never been saved to the server, and lacks an id. + isNew : function() { + return this.id == null; + }, + + // Call this method to manually fire a `change` event for this model. + // Calling this will cause all objects observing the model to update. + change : function(options) { + this.trigger('change', this, options); + this._previousAttributes = _.clone(this.attributes); + this._changed = false; + }, + + // Determine if the model has changed since the last `"change"` event. + // If you specify an attribute name, determine if that attribute has changed. + hasChanged : function(attr) { + if (attr) return this._previousAttributes[attr] != this.attributes[attr]; + return this._changed; + }, + + // Return an object containing all the attributes that have changed, or false + // if there are no changed attributes. Useful for determining what parts of a + // view need to be updated and/or what attributes need to be persisted to + // the server. + changedAttributes : function(now) { + now || (now = this.attributes); + var old = this._previousAttributes; + var changed = false; + for (var attr in now) { + if (!_.isEqual(old[attr], now[attr])) { + changed = changed || {}; + changed[attr] = now[attr]; + } + } + return changed; + }, + + // Get the previous value of an attribute, recorded at the time the last + // `"change"` event was fired. + previous : function(attr) { + if (!attr || !this._previousAttributes) return null; + return this._previousAttributes[attr]; + }, + + // Get all of the attributes of the model at the time of the previous + // `"change"` event. + previousAttributes : function() { + return _.clone(this._previousAttributes); + }, + + // Run validation against a set of incoming attributes, returning `true` + // if all is well. If a specific `error` callback has been passed, + // call that instead of firing the general `"error"` event. + _performValidation : function(attrs, options) { + var error = this.validate(attrs); + if (error) { + if (options.error) { + options.error(this, error, options); + } else { + this.trigger('error', this, error, options); + } + return false; + } + return true; + } + + }); + + // Backbone.Collection + // ------------------- + + // Provides a standard collection class for our sets of models, ordered + // or unordered. If a `comparator` is specified, the Collection will maintain + // its models in sort order, as they're added and removed. + Backbone.Collection = function(models, options) { + options || (options = {}); + if (options.comparator) this.comparator = options.comparator; + _.bindAll(this, '_onModelEvent', '_removeReference'); + this._reset(); + if (models) this.reset(models, {silent: true}); + this.initialize.apply(this, arguments); + }; + + // Define the Collection's inheritable methods. + _.extend(Backbone.Collection.prototype, Backbone.Events, { + + // The default model for a collection is just a **Backbone.Model**. + // This should be overridden in most cases. + model : Backbone.Model, + + // Initialize is an empty function by default. Override it with your own + // initialization logic. + initialize : function(){}, + + // The JSON representation of a Collection is an array of the + // models' attributes. + toJSON : function() { + return this.map(function(model){ return model.toJSON(); }); + }, + + // Add a model, or list of models to the set. Pass **silent** to avoid + // firing the `added` event for every new model. + add : function(models, options) { + if (_.isArray(models)) { + for (var i = 0, l = models.length; i < l; i++) { + this._add(models[i], options); + } + } else { + this._add(models, options); + } + return this; + }, + + // Remove a model, or a list of models from the set. Pass silent to avoid + // firing the `removed` event for every model removed. + remove : function(models, options) { + if (_.isArray(models)) { + for (var i = 0, l = models.length; i < l; i++) { + this._remove(models[i], options); + } + } else { + this._remove(models, options); + } + return this; + }, + + // Get a model from the set by id. + get : function(id) { + if (id == null) return null; + return this._byId[id.id != null ? id.id : id]; + }, + + // Get a model from the set by client id. + getByCid : function(cid) { + return cid && this._byCid[cid.cid || cid]; + }, + + // Get the model at the given index. + at: function(index) { + return this.models[index]; + }, + + // Force the collection to re-sort itself. You don't need to call this under normal + // circumstances, as the set will maintain sort order as each item is added. + sort : function(options) { + options || (options = {}); + if (!this.comparator) throw new Error('Cannot sort a set without a comparator'); + this.models = this.sortBy(this.comparator); + if (!options.silent) this.trigger('reset', this, options); + return this; + }, + + // Pluck an attribute from each model in the collection. + pluck : function(attr) { + return _.map(this.models, function(model){ return model.get(attr); }); + }, + + // When you have more items than you want to add or remove individually, + // you can reset the entire set with a new list of models, without firing + // any `added` or `removed` events. Fires `reset` when finished. + reset : function(models, options) { + models || (models = []); + options || (options = {}); + this.each(this._removeReference); + this._reset(); + this.add(models, {silent: true}); + if (!options.silent) this.trigger('reset', this, options); + return this; + }, + + // Fetch the default set of models for this collection, resetting the + // collection when they arrive. If `add: true` is passed, appends the + // models to the collection instead of resetting. + fetch : function(options) { + options || (options = {}); + var collection = this; + var success = options.success; + options.success = function(resp, status, xhr) { + collection[options.add ? 'add' : 'reset'](collection.parse(resp, xhr), options); + if (success) success(collection, resp); + }; + options.error = wrapError(options.error, collection, options); + return (this.sync || Backbone.sync).call(this, 'read', this, options); + }, + + // Create a new instance of a model in this collection. After the model + // has been created on the server, it will be added to the collection. + // Returns the model, or 'false' if validation on a new model fails. + create : function(model, options) { + var coll = this; + options || (options = {}); + model = this._prepareModel(model, options); + if (!model) return false; + var success = options.success; + options.success = function(nextModel, resp, xhr) { + coll.add(nextModel, options); + if (success) success(nextModel, resp, xhr); + }; + model.save(null, options); + return model; + }, + + // **parse** converts a response into a list of models to be added to the + // collection. The default implementation is just to pass it through. + parse : function(resp, xhr) { + return resp; + }, + + // Proxy to _'s chain. Can't be proxied the same way the rest of the + // underscore methods are proxied because it relies on the underscore + // constructor. + chain: function () { + return _(this.models).chain(); + }, + + // Reset all internal state. Called when the collection is reset. + _reset : function(options) { + this.length = 0; + this.models = []; + this._byId = {}; + this._byCid = {}; + }, + + // Prepare a model to be added to this collection + _prepareModel: function(model, options) { + if (!(model instanceof Backbone.Model)) { + var attrs = model; + model = new this.model(attrs, {collection: this}); + if (model.validate && !model._performValidation(attrs, options)) model = false; + } else if (!model.collection) { + model.collection = this; + } + return model; + }, + + // Internal implementation of adding a single model to the set, updating + // hash indexes for `id` and `cid` lookups. + // Returns the model, or 'false' if validation on a new model fails. + _add : function(model, options) { + options || (options = {}); + model = this._prepareModel(model, options); + if (!model) return false; + var already = this.getByCid(model); + if (already) throw new Error(["Can't add the same model to a set twice", already.id]); + this._byId[model.id] = model; + this._byCid[model.cid] = model; + var index = options.at != null ? options.at : + this.comparator ? this.sortedIndex(model, this.comparator) : + this.length; + this.models.splice(index, 0, model); + model.bind('all', this._onModelEvent); + this.length++; + if (!options.silent) model.trigger('add', model, this, options); + return model; + }, + + // Internal implementation of removing a single model from the set, updating + // hash indexes for `id` and `cid` lookups. + _remove : function(model, options) { + options || (options = {}); + model = this.getByCid(model) || this.get(model); + if (!model) return null; + delete this._byId[model.id]; + delete this._byCid[model.cid]; + this.models.splice(this.indexOf(model), 1); + this.length--; + if (!options.silent) model.trigger('remove', model, this, options); + this._removeReference(model); + return model; + }, + + // Internal method to remove a model's ties to a collection. + _removeReference : function(model) { + if (this == model.collection) { + delete model.collection; + } + model.unbind('all', this._onModelEvent); + }, + + // Internal method called every time a model in the set fires an event. + // Sets need to update their indexes when models change ids. All other + // events simply proxy through. "add" and "remove" events that originate + // in other collections are ignored. + _onModelEvent : function(ev, model, collection, options) { + if ((ev == 'add' || ev == 'remove') && collection != this) return; + if (ev == 'destroy') { + this._remove(model, options); + } + if (model && ev === 'change:' + model.idAttribute) { + delete this._byId[model.previous(model.idAttribute)]; + this._byId[model.id] = model; + } + this.trigger.apply(this, arguments); + } + + }); + + // Underscore methods that we want to implement on the Collection. + var methods = ['forEach', 'each', 'map', 'reduce', 'reduceRight', 'find', 'detect', + 'filter', 'select', 'reject', 'every', 'all', 'some', 'any', 'include', + 'contains', 'invoke', 'max', 'min', 'sortBy', 'sortedIndex', 'toArray', 'size', + 'first', 'rest', 'last', 'without', 'indexOf', 'lastIndexOf', 'isEmpty', 'groupBy']; + + // Mix in each Underscore method as a proxy to `Collection#models`. + _.each(methods, function(method) { + Backbone.Collection.prototype[method] = function() { + return _[method].apply(_, [this.models].concat(_.toArray(arguments))); + }; + }); + + // Backbone.Router + // ------------------- + + // Routers map faux-URLs to actions, and fire events when routes are + // matched. Creating a new one sets its `routes` hash, if not set statically. + Backbone.Router = function(options) { + options || (options = {}); + if (options.routes) this.routes = options.routes; + this._bindRoutes(); + this.initialize.apply(this, arguments); + }; + + // Cached regular expressions for matching named param parts and splatted + // parts of route strings. + var namedParam = /:([\w\d]+)/g; + var splatParam = /\*([\w\d]+)/g; + var escapeRegExp = /[-[\]{}()+?.,\\^$|#\s]/g; + + // Set up all inheritable **Backbone.Router** properties and methods. + _.extend(Backbone.Router.prototype, Backbone.Events, { + + // Initialize is an empty function by default. Override it with your own + // initialization logic. + initialize : function(){}, + + // Manually bind a single named route to a callback. For example: + // + // this.route('search/:query/p:num', 'search', function(query, num) { + // ... + // }); + // + route : function(route, name, callback) { + Backbone.history || (Backbone.history = new Backbone.History); + if (!_.isRegExp(route)) route = this._routeToRegExp(route); + Backbone.history.route(route, _.bind(function(fragment) { + var args = this._extractParameters(route, fragment); + callback.apply(this, args); + this.trigger.apply(this, ['route:' + name].concat(args)); + }, this)); + }, + + // Simple proxy to `Backbone.history` to save a fragment into the history. + navigate : function(fragment, triggerRoute) { + Backbone.history.navigate(fragment, triggerRoute); + }, + + // Bind all defined routes to `Backbone.history`. We have to reverse the + // order of the routes here to support behavior where the most general + // routes can be defined at the bottom of the route map. + _bindRoutes : function() { + if (!this.routes) return; + var routes = []; + for (var route in this.routes) { + routes.unshift([route, this.routes[route]]); + } + for (var i = 0, l = routes.length; i < l; i++) { + this.route(routes[i][0], routes[i][1], this[routes[i][1]]); + } + }, + + // Convert a route string into a regular expression, suitable for matching + // against the current location hash. + _routeToRegExp : function(route) { + route = route.replace(escapeRegExp, "\\$&") + .replace(namedParam, "([^\/]*)") + .replace(splatParam, "(.*?)"); + return new RegExp('^' + route + '$'); + }, + + // Given a route, and a URL fragment that it matches, return the array of + // extracted parameters. + _extractParameters : function(route, fragment) { + return route.exec(fragment).slice(1); + } + + }); + + // Backbone.History + // ---------------- + + // Handles cross-browser history management, based on URL fragments. If the + // browser does not support `onhashchange`, falls back to polling. + Backbone.History = function() { + this.handlers = []; + _.bindAll(this, 'checkUrl'); + }; + + // Cached regex for cleaning hashes. + var hashStrip = /^#*/; + + // Cached regex for detecting MSIE. + var isExplorer = /msie [\w.]+/; + + // Has the history handling already been started? + var historyStarted = false; + + // Set up all inheritable **Backbone.History** properties and methods. + _.extend(Backbone.History.prototype, { + + // The default interval to poll for hash changes, if necessary, is + // twenty times a second. + interval: 50, + + // Get the cross-browser normalized URL fragment, either from the URL, + // the hash, or the override. + getFragment : function(fragment, forcePushState) { + if (fragment == null) { + if (this._hasPushState || forcePushState) { + fragment = window.location.pathname; + var search = window.location.search; + if (search) fragment += search; + if (fragment.indexOf(this.options.root) == 0) fragment = fragment.substr(this.options.root.length); + } else { + fragment = window.location.hash; + } + } + return decodeURIComponent(fragment.replace(hashStrip, '')); + }, + + // Start the hash change handling, returning `true` if the current URL matches + // an existing route, and `false` otherwise. + start : function(options) { + + // Figure out the initial configuration. Do we need an iframe? + // Is pushState desired ... is it available? + if (historyStarted) throw new Error("Backbone.history has already been started"); + this.options = _.extend({}, {root: '/'}, this.options, options); + this._wantsPushState = !!this.options.pushState; + this._hasPushState = !!(this.options.pushState && window.history && window.history.pushState); + var fragment = this.getFragment(); + var docMode = document.documentMode; + var oldIE = (isExplorer.exec(navigator.userAgent.toLowerCase()) && (!docMode || docMode <= 7)); + if (oldIE) { + this.iframe = $('<iframe src="javascript:0" tabindex="-1" />').hide().appendTo('body')[0].contentWindow; + this.navigate(fragment); + } + + // Depending on whether we're using pushState or hashes, and whether + // 'onhashchange' is supported, determine how we check the URL state. + if (this._hasPushState) { + $(window).bind('popstate', this.checkUrl); + } else if ('onhashchange' in window && !oldIE) { + $(window).bind('hashchange', this.checkUrl); + } else { + setInterval(this.checkUrl, this.interval); + } + + // Determine if we need to change the base url, for a pushState link + // opened by a non-pushState browser. + this.fragment = fragment; + historyStarted = true; + var loc = window.location; + var atRoot = loc.pathname == this.options.root; + if (this._wantsPushState && !this._hasPushState && !atRoot) { + this.fragment = this.getFragment(null, true); + window.location.replace(this.options.root + '#' + this.fragment); + // Return immediately as browser will do redirect to new url + return true; + } else if (this._wantsPushState && this._hasPushState && atRoot && loc.hash) { + this.fragment = loc.hash.replace(hashStrip, ''); + window.history.replaceState({}, document.title, loc.protocol + '//' + loc.host + this.options.root + this.fragment); + } + + if (!this.options.silent) { + return this.loadUrl(); + } + }, + + // Add a route to be tested when the fragment changes. Routes added later may + // override previous routes. + route : function(route, callback) { + this.handlers.unshift({route : route, callback : callback}); + }, + + // Checks the current URL to see if it has changed, and if it has, + // calls `loadUrl`, normalizing across the hidden iframe. + checkUrl : function(e) { + var current = this.getFragment(); + if (current == this.fragment && this.iframe) current = this.getFragment(this.iframe.location.hash); + if (current == this.fragment || current == decodeURIComponent(this.fragment)) return false; + if (this.iframe) this.navigate(current); + this.loadUrl() || this.loadUrl(window.location.hash); + }, + + // Attempt to load the current URL fragment. If a route succeeds with a + // match, returns `true`. If no defined routes matches the fragment, + // returns `false`. + loadUrl : function(fragmentOverride) { + var fragment = this.fragment = this.getFragment(fragmentOverride); + var matched = _.any(this.handlers, function(handler) { + if (handler.route.test(fragment)) { + handler.callback(fragment); + return true; + } + }); + return matched; + }, + + // Save a fragment into the hash history. You are responsible for properly + // URL-encoding the fragment in advance. This does not trigger + // a `hashchange` event. + navigate : function(fragment, triggerRoute) { + var frag = (fragment || '').replace(hashStrip, ''); + if (this.fragment == frag || this.fragment == decodeURIComponent(frag)) return; + if (this._hasPushState) { + var loc = window.location; + if (frag.indexOf(this.options.root) != 0) frag = this.options.root + frag; + this.fragment = frag; + window.history.pushState({}, document.title, loc.protocol + '//' + loc.host + frag); + } else { + window.location.hash = this.fragment = frag; + if (this.iframe && (frag != this.getFragment(this.iframe.location.hash))) { + this.iframe.document.open().close(); + this.iframe.location.hash = frag; + } + } + if (triggerRoute) this.loadUrl(fragment); + } + + }); + + // Backbone.View + // ------------- + + // Creating a Backbone.View creates its initial element outside of the DOM, + // if an existing element is not provided... + Backbone.View = function(options) { + this.cid = _.uniqueId('view'); + this._configure(options || {}); + this._ensureElement(); + this.delegateEvents(); + this.initialize.apply(this, arguments); + }; + + // Element lookup, scoped to DOM elements within the current view. + // This should be prefered to global lookups, if you're dealing with + // a specific view. + var selectorDelegate = function(selector) { + return $(selector, this.el); + }; + + // Cached regex to split keys for `delegate`. + var eventSplitter = /^(\S+)\s*(.*)$/; + + // List of view options to be merged as properties. + var viewOptions = ['model', 'collection', 'el', 'id', 'attributes', 'className', 'tagName']; + + // Set up all inheritable **Backbone.View** properties and methods. + _.extend(Backbone.View.prototype, Backbone.Events, { + + // The default `tagName` of a View's element is `"div"`. + tagName : 'div', + + // Attach the `selectorDelegate` function as the `$` property. + $ : selectorDelegate, + + // Initialize is an empty function by default. Override it with your own + // initialization logic. + initialize : function(){}, + + // **render** is the core function that your view should override, in order + // to populate its element (`this.el`), with the appropriate HTML. The + // convention is for **render** to always return `this`. + render : function() { + return this; + }, + + // Remove this view from the DOM. Note that the view isn't present in the + // DOM by default, so calling this method may be a no-op. + remove : function() { + $(this.el).remove(); + return this; + }, + + // For small amounts of DOM Elements, where a full-blown template isn't + // needed, use **make** to manufacture elements, one at a time. + // + // var el = this.make('li', {'class': 'row'}, this.model.escape('title')); + // + make : function(tagName, attributes, content) { + var el = document.createElement(tagName); + if (attributes) $(el).attr(attributes); + if (content) $(el).html(content); + return el; + }, + + // Set callbacks, where `this.callbacks` is a hash of + // + // *{"event selector": "callback"}* + // + // { + // 'mousedown .title': 'edit', + // 'click .button': 'save' + // } + // + // pairs. Callbacks will be bound to the view, with `this` set properly. + // Uses event delegation for efficiency. + // Omitting the selector binds the event to `this.el`. + // This only works for delegate-able events: not `focus`, `blur`, and + // not `change`, `submit`, and `reset` in Internet Explorer. + delegateEvents : function(events) { + if (!(events || (events = this.events))) return; + if (_.isFunction(events)) events = events.call(this); + $(this.el).unbind('.delegateEvents' + this.cid); + for (var key in events) { + var method = this[events[key]]; + if (!method) throw new Error('Event "' + events[key] + '" does not exist'); + var match = key.match(eventSplitter); + var eventName = match[1], selector = match[2]; + method = _.bind(method, this); + eventName += '.delegateEvents' + this.cid; + if (selector === '') { + $(this.el).bind(eventName, method); + } else { + $(this.el).delegate(selector, eventName, method); + } + } + }, + + // Performs the initial configuration of a View with a set of options. + // Keys with special meaning *(model, collection, id, className)*, are + // attached directly to the view. + _configure : function(options) { + if (this.options) options = _.extend({}, this.options, options); + for (var i = 0, l = viewOptions.length; i < l; i++) { + var attr = viewOptions[i]; + if (options[attr]) this[attr] = options[attr]; + } + this.options = options; + }, + + // Ensure that the View has a DOM element to render into. + // If `this.el` is a string, pass it through `$()`, take the first + // matching element, and re-assign it to `el`. Otherwise, create + // an element from the `id`, `className` and `tagName` proeprties. + _ensureElement : function() { + if (!this.el) { + var attrs = this.attributes || {}; + if (this.id) attrs.id = this.id; + if (this.className) attrs['class'] = this.className; + this.el = this.make(this.tagName, attrs); + } else if (_.isString(this.el)) { + this.el = $(this.el).get(0); + } + } + + }); + + // The self-propagating extend function that Backbone classes use. + var extend = function (protoProps, classProps) { + var child = inherits(this, protoProps, classProps); + child.extend = this.extend; + return child; + }; + + // Set up inheritance for the model, collection, and view. + Backbone.Model.extend = Backbone.Collection.extend = + Backbone.Router.extend = Backbone.View.extend = extend; + + // Map from CRUD to HTTP for our default `Backbone.sync` implementation. + var methodMap = { + 'create': 'POST', + 'update': 'PUT', + 'delete': 'DELETE', + 'read' : 'GET' + }; + + // Backbone.sync + // ------------- + + // Override this function to change the manner in which Backbone persists + // models to the server. You will be passed the type of request, and the + // model in question. By default, uses makes a RESTful Ajax request + // to the model's `url()`. Some possible customizations could be: + // + // * Use `setTimeout` to batch rapid-fire updates into a single request. + // * Send up the models as XML instead of JSON. + // * Persist models via WebSockets instead of Ajax. + // + // Turn on `Backbone.emulateHTTP` in order to send `PUT` and `DELETE` requests + // as `POST`, with a `_method` parameter containing the true HTTP method, + // as well as all requests with the body as `application/x-www-form-urlencoded` instead of + // `application/json` with the model in a param named `model`. + // Useful when interfacing with server-side languages like **PHP** that make + // it difficult to read the body of `PUT` requests. + Backbone.sync = function(method, model, options) { + var type = methodMap[method]; + + // Default JSON-request options. + var params = _.extend({ + type: type, + dataType: 'json' + }, options); + + // Ensure that we have a URL. + if (!params.url) { + params.url = getUrl(model) || urlError(); + } + + // Ensure that we have the appropriate request data. + if (!params.data && model && (method == 'create' || method == 'update')) { + params.contentType = 'application/json'; + params.data = JSON.stringify(model.toJSON()); + } + + // For older servers, emulate JSON by encoding the request into an HTML-form. + if (Backbone.emulateJSON) { + params.contentType = 'application/x-www-form-urlencoded'; + params.data = params.data ? {model : params.data} : {}; + } + + // For older servers, emulate HTTP by mimicking the HTTP method with `_method` + // And an `X-HTTP-Method-Override` header. + if (Backbone.emulateHTTP) { + if (type === 'PUT' || type === 'DELETE') { + if (Backbone.emulateJSON) params.data._method = type; + params.type = 'POST'; + params.beforeSend = function(xhr) { + xhr.setRequestHeader('X-HTTP-Method-Override', type); + }; + } + } + + // Don't process data on a non-GET request. + if (params.type !== 'GET' && !Backbone.emulateJSON) { + params.processData = false; + } + + // Make the request. + return $.ajax(params); + }; + + // Helpers + // ------- + + // Shared empty constructor function to aid in prototype-chain creation. + var ctor = function(){}; + + // Helper function to correctly set up the prototype chain, for subclasses. + // Similar to `goog.inherits`, but uses a hash of prototype properties and + // class properties to be extended. + var inherits = function(parent, protoProps, staticProps) { + var child; + + // The constructor function for the new subclass is either defined by you + // (the "constructor" property in your `extend` definition), or defaulted + // by us to simply call `super()`. + if (protoProps && protoProps.hasOwnProperty('constructor')) { + child = protoProps.constructor; + } else { + child = function(){ return parent.apply(this, arguments); }; + } + + // Inherit class (static) properties from parent. + _.extend(child, parent); + + // Set the prototype chain to inherit from `parent`, without calling + // `parent`'s constructor function. + ctor.prototype = parent.prototype; + child.prototype = new ctor(); + + // Add prototype properties (instance properties) to the subclass, + // if supplied. + if (protoProps) _.extend(child.prototype, protoProps); + + // Add static properties to the constructor function, if supplied. + if (staticProps) _.extend(child, staticProps); + + // Correctly set child's `prototype.constructor`. + child.prototype.constructor = child; + + // Set a convenience property in case the parent's prototype is needed later. + child.__super__ = parent.prototype; + + return child; + }; + + // Helper function to get a URL from a Model or Collection as a property + // or as a function. + var getUrl = function(object) { + if (!(object && object.url)) return null; + return _.isFunction(object.url) ? object.url() : object.url; + }; + + // Throw an error when a URL is needed, and none is supplied. + var urlError = function() { + throw new Error('A "url" property or function must be specified'); + }; + + // Wrap an optional error callback with a fallback error event. + var wrapError = function(onError, model, options) { + return function(resp) { + if (onError) { + onError(model, resp, options); + } else { + model.trigger('error', model, resp, options); + } + }; + }; + + // Helper function to escape a string for HTML rendering. + var escapeHTML = function(string) { + return string.replace(/&(?!\w+;|#\d+;|#x[\da-f]+;)/gi, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"').replace(/'/g, ''').replace(/\//g,'/'); + }; + +}).call(this); diff --git a/libs/beetsplug/web/static/beets.css b/libs/beetsplug/web/static/beets.css new file mode 100644 index 00000000..2ca7fc83 --- /dev/null +++ b/libs/beetsplug/web/static/beets.css @@ -0,0 +1,160 @@ +body { + font-family: Helvetica, Arial, sans-serif; +} + +#header { + position: fixed; + left: 0; + right: 0; + top: 0; + height: 36px; + + color: white; + + cursor: default; + + /* shadowy border */ + box-shadow: 0 0 20px #999; + -webkit-box-shadow: 0 0 20px #999; + -moz-box-shadow: 0 0 20px #999; + + /* background gradient */ + background: #0e0e0e; + background: -moz-linear-gradient(top, #6b6b6b 0%, #0e0e0e 100%); + background: -webkit-linear-gradient(top, #6b6b6b 0%,#0e0e0e 100%); +} +#header h1 { + font-size: 1.1em; + font-weight: bold; + color: white; + margin: 0.35em; + float: left; +} + +#entities { + width: 17em; + + position: fixed; + top: 36px; + left: 0; + bottom: 0; + margin: 0; + + z-index: 1; + background: #dde4eb; + + /* shadowy border */ + box-shadow: 0 0 20px #666; + -webkit-box-shadow: 0 0 20px #666; + -moz-box-shadow: 0 0 20px #666; +} +#queryForm { + display: block; + text-align: center; + margin: 0.25em 0; +} +#query { + width: 95%; + font-size: 1em; +} +#entities ul { + width: 17em; + + position: fixed; + top: 36px; + left: 0; + bottom: 0; + margin: 2.2em 0 0 0; + padding: 0; + + overflow-y: auto; + overflow-x: hidden; +} +#entities ul li { + list-style: none; + padding: 4px 8px; + margin: 0; + cursor: default; +} +#entities ul li.selected { + background: #7abcff; + background: -moz-linear-gradient(top, #7abcff 0%, #60abf8 44%, #4096ee 100%); + background: -webkit-gradient(linear, left top, left bottom, color-stop(0%,#7abcff), color-stop(44%,#60abf8), color-stop(100%,#4096ee)); + color: white; +} +#entities ul li .playing { + margin-left: 5px; + font-size: 0.9em; +} + + +#main-detail, #extra-detail { + position: fixed; + left: 17em; + margin: 1.0em 0 0 1.5em; +} +#main-detail { + top: 36px; + height: 98px; +} +#main-detail .artist, #main-detail .album, #main-detail .title { + display: block; +} +#main-detail .title { + font-size: 1.3em; + font-weight: bold; +} +#main-detail .albumtitle { + font-style: italic; +} + +#extra-detail { + overflow-x: hidden; + overflow-y: auto; + top: 134px; + bottom: 0; + right: 0; +} +/*Fix for correctly displaying line breaks in lyrics*/ +#extra-detail .lyrics { + white-space: pre-wrap; +} +#extra-detail dl dt, #extra-detail dl dd { + list-style: none; + margin: 0; + padding: 0; +} +#extra-detail dl dt { + width: 10em; + float: left; + text-align: right; + font-weight: bold; + clear: both; +} +#extra-detail dl dd { + margin-left: 10.5em; +} + + +#player { + float: left; + width: 150px; + height: 36px; +} +#player .play, #player .pause, #player .disabled { + -webkit-appearance: none; + font-size: 1em; + font-family: Helvetica, Arial, sans-serif; + background: none; + border: none; + color: white; + padding: 5px; + margin: 0; + text-align: center; + + width: 36px; + height: 36px; +} +#player .disabled { + color: #666; +} diff --git a/libs/beetsplug/web/static/beets.js b/libs/beetsplug/web/static/beets.js new file mode 100644 index 00000000..757f2cda --- /dev/null +++ b/libs/beetsplug/web/static/beets.js @@ -0,0 +1,314 @@ +// Format times as minutes and seconds. +var timeFormat = function(secs) { + if (secs == undefined || isNaN(secs)) { + return '0:00'; + } + secs = Math.round(secs); + var mins = '' + Math.round(secs / 60); + secs = '' + (secs % 60); + if (secs.length < 2) { + secs = '0' + secs; + } + return mins + ':' + secs; +} + +// jQuery extension encapsulating event hookups for audio element controls. +$.fn.player = function(debug) { + // Selected element should contain an HTML5 Audio element. + var audio = $('audio', this).get(0); + + // Control elements that may be present, identified by class. + var playBtn = $('.play', this); + var pauseBtn = $('.pause', this); + var disabledInd = $('.disabled', this); + var timesEl = $('.times', this); + var curTimeEl = $('.currentTime', this); + var totalTimeEl = $('.totalTime', this); + var sliderPlayedEl = $('.slider .played', this); + var sliderLoadedEl = $('.slider .loaded', this); + + // Button events. + playBtn.click(function() { + audio.play(); + }); + pauseBtn.click(function(ev) { + audio.pause(); + }); + + // Utilities. + var timePercent = function(cur, total) { + if (cur == undefined || isNaN(cur) || + total == undefined || isNaN(total) || total == 0) { + return 0; + } + var ratio = cur / total; + if (ratio > 1.0) { + ratio = 1.0; + } + return (Math.round(ratio * 10000) / 100) + '%'; + } + + // Event helpers. + var dbg = function(msg) { + if (debug) + console.log(msg); + } + var showState = function() { + if (audio.duration == undefined || isNaN(audio.duration)) { + playBtn.hide(); + pauseBtn.hide(); + disabledInd.show(); + timesEl.hide(); + } else if (audio.paused) { + playBtn.show(); + pauseBtn.hide(); + disabledInd.hide(); + timesEl.show(); + } else { + playBtn.hide(); + pauseBtn.show(); + disabledInd.hide(); + timesEl.show(); + } + } + var showTimes = function() { + curTimeEl.text(timeFormat(audio.currentTime)); + totalTimeEl.text(timeFormat(audio.duration)); + + sliderPlayedEl.css('width', + timePercent(audio.currentTime, audio.duration)); + + // last time buffered + var bufferEnd = 0; + for (var i = 0; i < audio.buffered.length; ++i) { + if (audio.buffered.end(i) > bufferEnd) + bufferEnd = audio.buffered.end(i); + } + sliderLoadedEl.css('width', + timePercent(bufferEnd, audio.duration)); + } + + // Initialize controls. + showState(); + showTimes(); + + // Bind events. + $('audio', this).bind({ + playing: function() { + dbg('playing'); + showState(); + }, + pause: function() { + dbg('pause'); + showState(); + }, + ended: function() { + dbg('ended'); + showState(); + }, + progress: function() { + dbg('progress ' + audio.buffered); + }, + timeupdate: function() { + dbg('timeupdate ' + audio.currentTime); + showTimes(); + }, + durationchange: function() { + dbg('durationchange ' + audio.duration); + showState(); + showTimes(); + }, + loadeddata: function() { + dbg('loadeddata'); + }, + loadedmetadata: function() { + dbg('loadedmetadata'); + } + }); +} + +// Simple selection disable for jQuery. +// Cut-and-paste from: +// http://stackoverflow.com/questions/2700000 +$.fn.disableSelection = function() { + $(this).attr('unselectable', 'on') + .css('-moz-user-select', 'none') + .each(function() { + this.onselectstart = function() { return false; }; + }); +}; + +$(function() { + +// Routes. +var BeetsRouter = Backbone.Router.extend({ + routes: { + "item/query/:query": "itemQuery", + }, + itemQuery: function(query) { + var queryURL = query.split(/\s+/).map(encodeURIComponent).join('/'); + $.getJSON('/item/query/' + queryURL, function(data) { + var models = _.map( + data['results'], + function(d) { return new Item(d); } + ); + var results = new Items(models); + app.showItems(results); + }); + } +}); +var router = new BeetsRouter(); + +// Model. +var Item = Backbone.Model.extend({ + urlRoot: '/item' +}); +var Items = Backbone.Collection.extend({ + model: Item +}); + +// Item views. +var ItemEntryView = Backbone.View.extend({ + tagName: "li", + template: _.template($('#item-entry-template').html()), + events: { + 'click': 'select', + 'dblclick': 'play' + }, + initialize: function() { + this.playing = false; + }, + render: function() { + $(this.el).html(this.template(this.model.toJSON())); + this.setPlaying(this.playing); + return this; + }, + select: function() { + app.selectItem(this); + }, + play: function() { + app.playItem(this.model); + }, + setPlaying: function(val) { + this.playing = val; + if (val) + this.$('.playing').show(); + else + this.$('.playing').hide(); + } +}); +//Holds Title, Artist, Album etc. +var ItemMainDetailView = Backbone.View.extend({ + tagName: "div", + template: _.template($('#item-main-detail-template').html()), + events: { + 'click .play': 'play', + }, + render: function() { + $(this.el).html(this.template(this.model.toJSON())); + return this; + }, + play: function() { + app.playItem(this.model); + } +}); +// Holds Track no., Format, MusicBrainz link, Lyrics, Comments etc. +var ItemExtraDetailView = Backbone.View.extend({ + tagName: "div", + template: _.template($('#item-extra-detail-template').html()), + render: function() { + $(this.el).html(this.template(this.model.toJSON())); + return this; + } +}); +// Main app view. +var AppView = Backbone.View.extend({ + el: $('body'), + events: { + 'submit #queryForm': 'querySubmit', + }, + querySubmit: function(ev) { + ev.preventDefault(); + router.navigate('item/query/' + encodeURIComponent($('#query').val()), true); + }, + initialize: function() { + this.playingItem = null; + this.shownItems = null; + + // Not sure why these events won't bind automatically. + this.$('audio').bind({ + 'play': _.bind(this.audioPlay, this), + 'pause': _.bind(this.audioPause, this), + 'ended': _.bind(this.audioEnded, this) + }); + }, + showItems: function(items) { + this.shownItems = items; + $('#results').empty(); + items.each(function(item) { + var view = new ItemEntryView({model: item}); + item.entryView = view; + $('#results').append(view.render().el); + }); + }, + selectItem: function(view) { + // Mark row as selected. + $('#results li').removeClass("selected"); + $(view.el).addClass("selected"); + + // Show main and extra detail. + var mainDetailView = new ItemMainDetailView({model: view.model}); + $('#main-detail').empty().append(mainDetailView.render().el); + + var extraDetailView = new ItemExtraDetailView({model: view.model}); + $('#extra-detail').empty().append(extraDetailView.render().el); + }, + playItem: function(item) { + var url = '/item/' + item.get('id') + '/file'; + $('#player audio').attr('src', url); + $('#player audio').get(0).play(); + + if (this.playingItem != null) { + this.playingItem.entryView.setPlaying(false); + } + item.entryView.setPlaying(true); + this.playingItem = item; + }, + + audioPause: function() { + this.playingItem.entryView.setPlaying(false); + }, + audioPlay: function() { + if (this.playingItem != null) + this.playingItem.entryView.setPlaying(true); + }, + audioEnded: function() { + this.playingItem.entryView.setPlaying(false); + + // Try to play the next track. + var idx = this.shownItems.indexOf(this.playingItem); + if (idx == -1) { + // Not in current list. + return; + } + var nextIdx = idx + 1; + if (nextIdx >= this.shownItems.size()) { + // End of list. + return; + } + this.playItem(this.shownItems.at(nextIdx)); + } +}); +var app = new AppView(); + +// App setup. +Backbone.history.start({pushState: false}); + +// Disable selection on UI elements. +$('#entities ul').disableSelection(); +$('#header').disableSelection(); + +// Audio player setup. +$('#player').player(); + +}); diff --git a/libs/beetsplug/web/static/jquery.js b/libs/beetsplug/web/static/jquery.js new file mode 100644 index 00000000..e1414212 --- /dev/null +++ b/libs/beetsplug/web/static/jquery.js @@ -0,0 +1,9266 @@ +/*! + * jQuery JavaScript Library v1.7.1 + * http://jquery.com/ + * + * Copyright 2016, John Resig + * Dual licensed under the MIT or GPL Version 2 licenses. + * http://jquery.org/license + * + * Includes Sizzle.js + * http://sizzlejs.com/ + * Copyright 2016, The Dojo Foundation + * Released under the MIT, BSD, and GPL Licenses. + * + * Date: Mon Nov 21 21:11:03 2011 -0500 + */ +(function( window, undefined ) { + +// Use the correct document accordingly with window argument (sandbox) +var document = window.document, + navigator = window.navigator, + location = window.location; +var jQuery = (function() { + +// Define a local copy of jQuery +var jQuery = function( selector, context ) { + // The jQuery object is actually just the init constructor 'enhanced' + return new jQuery.fn.init( selector, context, rootjQuery ); + }, + + // Map over jQuery in case of overwrite + _jQuery = window.jQuery, + + // Map over the $ in case of overwrite + _$ = window.$, + + // A central reference to the root jQuery(document) + rootjQuery, + + // A simple way to check for HTML strings or ID strings + // Prioritize #id over <tag> to avoid XSS via location.hash (#9521) + quickExpr = /^(?:[^#<]*(<[\w\W]+>)[^>]*$|#([\w\-]*)$)/, + + // Check if a string has a non-whitespace character in it + rnotwhite = /\S/, + + // Used for trimming whitespace + trimLeft = /^\s+/, + trimRight = /\s+$/, + + // Match a standalone tag + rsingleTag = /^<(\w+)\s*\/?>(?:<\/\1>)?$/, + + // JSON RegExp + rvalidchars = /^[\],:{}\s]*$/, + rvalidescape = /\\(?:["\\\/bfnrt]|u[0-9a-fA-F]{4})/g, + rvalidtokens = /"[^"\\\n\r]*"|true|false|null|-?\d+(?:\.\d*)?(?:[eE][+\-]?\d+)?/g, + rvalidbraces = /(?:^|:|,)(?:\s*\[)+/g, + + // Useragent RegExp + rwebkit = /(webkit)[ \/]([\w.]+)/, + ropera = /(opera)(?:.*version)?[ \/]([\w.]+)/, + rmsie = /(msie) ([\w.]+)/, + rmozilla = /(mozilla)(?:.*? rv:([\w.]+))?/, + + // Matches dashed string for camelizing + rdashAlpha = /-([a-z]|[0-9])/ig, + rmsPrefix = /^-ms-/, + + // Used by jQuery.camelCase as callback to replace() + fcamelCase = function( all, letter ) { + return ( letter + "" ).toUpperCase(); + }, + + // Keep a UserAgent string for use with jQuery.browser + userAgent = navigator.userAgent, + + // For matching the engine and version of the browser + browserMatch, + + // The deferred used on DOM ready + readyList, + + // The ready event handler + DOMContentLoaded, + + // Save a reference to some core methods + toString = Object.prototype.toString, + hasOwn = Object.prototype.hasOwnProperty, + push = Array.prototype.push, + slice = Array.prototype.slice, + trim = String.prototype.trim, + indexOf = Array.prototype.indexOf, + + // [[Class]] -> type pairs + class2type = {}; + +jQuery.fn = jQuery.prototype = { + constructor: jQuery, + init: function( selector, context, rootjQuery ) { + var match, elem, ret, doc; + + // Handle $(""), $(null), or $(undefined) + if ( !selector ) { + return this; + } + + // Handle $(DOMElement) + if ( selector.nodeType ) { + this.context = this[0] = selector; + this.length = 1; + return this; + } + + // The body element only exists once, optimize finding it + if ( selector === "body" && !context && document.body ) { + this.context = document; + this[0] = document.body; + this.selector = selector; + this.length = 1; + return this; + } + + // Handle HTML strings + if ( typeof selector === "string" ) { + // Are we dealing with HTML string or an ID? + if ( selector.charAt(0) === "<" && selector.charAt( selector.length - 1 ) === ">" && selector.length >= 3 ) { + // Assume that strings that start and end with <> are HTML and skip the regex check + match = [ null, selector, null ]; + + } else { + match = quickExpr.exec( selector ); + } + + // Verify a match, and that no context was specified for #id + if ( match && (match[1] || !context) ) { + + // HANDLE: $(html) -> $(array) + if ( match[1] ) { + context = context instanceof jQuery ? context[0] : context; + doc = ( context ? context.ownerDocument || context : document ); + + // If a single string is passed in and it's a single tag + // just do a createElement and skip the rest + ret = rsingleTag.exec( selector ); + + if ( ret ) { + if ( jQuery.isPlainObject( context ) ) { + selector = [ document.createElement( ret[1] ) ]; + jQuery.fn.attr.call( selector, context, true ); + + } else { + selector = [ doc.createElement( ret[1] ) ]; + } + + } else { + ret = jQuery.buildFragment( [ match[1] ], [ doc ] ); + selector = ( ret.cacheable ? jQuery.clone(ret.fragment) : ret.fragment ).childNodes; + } + + return jQuery.merge( this, selector ); + + // HANDLE: $("#id") + } else { + elem = document.getElementById( match[2] ); + + // Check parentNode to catch when Blackberry 4.6 returns + // nodes that are no longer in the document #6963 + if ( elem && elem.parentNode ) { + // Handle the case where IE and Opera return items + // by name instead of ID + if ( elem.id !== match[2] ) { + return rootjQuery.find( selector ); + } + + // Otherwise, we inject the element directly into the jQuery object + this.length = 1; + this[0] = elem; + } + + this.context = document; + this.selector = selector; + return this; + } + + // HANDLE: $(expr, $(...)) + } else if ( !context || context.jquery ) { + return ( context || rootjQuery ).find( selector ); + + // HANDLE: $(expr, context) + // (which is just equivalent to: $(context).find(expr) + } else { + return this.constructor( context ).find( selector ); + } + + // HANDLE: $(function) + // Shortcut for document ready + } else if ( jQuery.isFunction( selector ) ) { + return rootjQuery.ready( selector ); + } + + if ( selector.selector !== undefined ) { + this.selector = selector.selector; + this.context = selector.context; + } + + return jQuery.makeArray( selector, this ); + }, + + // Start with an empty selector + selector: "", + + // The current version of jQuery being used + jquery: "1.7.1", + + // The default length of a jQuery object is 0 + length: 0, + + // The number of elements contained in the matched element set + size: function() { + return this.length; + }, + + toArray: function() { + return slice.call( this, 0 ); + }, + + // Get the Nth element in the matched element set OR + // Get the whole matched element set as a clean array + get: function( num ) { + return num == null ? + + // Return a 'clean' array + this.toArray() : + + // Return just the object + ( num < 0 ? this[ this.length + num ] : this[ num ] ); + }, + + // Take an array of elements and push it onto the stack + // (returning the new matched element set) + pushStack: function( elems, name, selector ) { + // Build a new jQuery matched element set + var ret = this.constructor(); + + if ( jQuery.isArray( elems ) ) { + push.apply( ret, elems ); + + } else { + jQuery.merge( ret, elems ); + } + + // Add the old object onto the stack (as a reference) + ret.prevObject = this; + + ret.context = this.context; + + if ( name === "find" ) { + ret.selector = this.selector + ( this.selector ? " " : "" ) + selector; + } else if ( name ) { + ret.selector = this.selector + "." + name + "(" + selector + ")"; + } + + // Return the newly-formed element set + return ret; + }, + + // Execute a callback for every element in the matched set. + // (You can seed the arguments with an array of args, but this is + // only used internally.) + each: function( callback, args ) { + return jQuery.each( this, callback, args ); + }, + + ready: function( fn ) { + // Attach the listeners + jQuery.bindReady(); + + // Add the callback + readyList.add( fn ); + + return this; + }, + + eq: function( i ) { + i = +i; + return i === -1 ? + this.slice( i ) : + this.slice( i, i + 1 ); + }, + + first: function() { + return this.eq( 0 ); + }, + + last: function() { + return this.eq( -1 ); + }, + + slice: function() { + return this.pushStack( slice.apply( this, arguments ), + "slice", slice.call(arguments).join(",") ); + }, + + map: function( callback ) { + return this.pushStack( jQuery.map(this, function( elem, i ) { + return callback.call( elem, i, elem ); + })); + }, + + end: function() { + return this.prevObject || this.constructor(null); + }, + + // For internal use only. + // Behaves like an Array's method, not like a jQuery method. + push: push, + sort: [].sort, + splice: [].splice +}; + +// Give the init function the jQuery prototype for later instantiation +jQuery.fn.init.prototype = jQuery.fn; + +jQuery.extend = jQuery.fn.extend = function() { + var options, name, src, copy, copyIsArray, clone, + target = arguments[0] || {}, + i = 1, + length = arguments.length, + deep = false; + + // Handle a deep copy situation + if ( typeof target === "boolean" ) { + deep = target; + target = arguments[1] || {}; + // skip the boolean and the target + i = 2; + } + + // Handle case when target is a string or something (possible in deep copy) + if ( typeof target !== "object" && !jQuery.isFunction(target) ) { + target = {}; + } + + // extend jQuery itself if only one argument is passed + if ( length === i ) { + target = this; + --i; + } + + for ( ; i < length; i++ ) { + // Only deal with non-null/undefined values + if ( (options = arguments[ i ]) != null ) { + // Extend the base object + for ( name in options ) { + src = target[ name ]; + copy = options[ name ]; + + // Prevent never-ending loop + if ( target === copy ) { + continue; + } + + // Recurse if we're merging plain objects or arrays + if ( deep && copy && ( jQuery.isPlainObject(copy) || (copyIsArray = jQuery.isArray(copy)) ) ) { + if ( copyIsArray ) { + copyIsArray = false; + clone = src && jQuery.isArray(src) ? src : []; + + } else { + clone = src && jQuery.isPlainObject(src) ? src : {}; + } + + // Never move original objects, clone them + target[ name ] = jQuery.extend( deep, clone, copy ); + + // Don't bring in undefined values + } else if ( copy !== undefined ) { + target[ name ] = copy; + } + } + } + } + + // Return the modified object + return target; +}; + +jQuery.extend({ + noConflict: function( deep ) { + if ( window.$ === jQuery ) { + window.$ = _$; + } + + if ( deep && window.jQuery === jQuery ) { + window.jQuery = _jQuery; + } + + return jQuery; + }, + + // Is the DOM ready to be used? Set to true once it occurs. + isReady: false, + + // A counter to track how many items to wait for before + // the ready event fires. See #6781 + readyWait: 1, + + // Hold (or release) the ready event + holdReady: function( hold ) { + if ( hold ) { + jQuery.readyWait++; + } else { + jQuery.ready( true ); + } + }, + + // Handle when the DOM is ready + ready: function( wait ) { + // Either a released hold or an DOMready/load event and not yet ready + if ( (wait === true && !--jQuery.readyWait) || (wait !== true && !jQuery.isReady) ) { + // Make sure body exists, at least, in case IE gets a little overzealous (ticket #5443). + if ( !document.body ) { + return setTimeout( jQuery.ready, 1 ); + } + + // Remember that the DOM is ready + jQuery.isReady = true; + + // If a normal DOM Ready event fired, decrement, and wait if need be + if ( wait !== true && --jQuery.readyWait > 0 ) { + return; + } + + // If there are functions bound, to execute + readyList.fireWith( document, [ jQuery ] ); + + // Trigger any bound ready events + if ( jQuery.fn.trigger ) { + jQuery( document ).trigger( "ready" ).off( "ready" ); + } + } + }, + + bindReady: function() { + if ( readyList ) { + return; + } + + readyList = jQuery.Callbacks( "once memory" ); + + // Catch cases where $(document).ready() is called after the + // browser event has already occurred. + if ( document.readyState === "complete" ) { + // Handle it asynchronously to allow scripts the opportunity to delay ready + return setTimeout( jQuery.ready, 1 ); + } + + // Mozilla, Opera and webkit nightlies currently support this event + if ( document.addEventListener ) { + // Use the handy event callback + document.addEventListener( "DOMContentLoaded", DOMContentLoaded, false ); + + // A fallback to window.onload, that will always work + window.addEventListener( "load", jQuery.ready, false ); + + // If IE event model is used + } else if ( document.attachEvent ) { + // ensure firing before onload, + // maybe late but safe also for iframes + document.attachEvent( "onreadystatechange", DOMContentLoaded ); + + // A fallback to window.onload, that will always work + window.attachEvent( "onload", jQuery.ready ); + + // If IE and not a frame + // continually check to see if the document is ready + var toplevel = false; + + try { + toplevel = window.frameElement == null; + } catch(e) {} + + if ( document.documentElement.doScroll && toplevel ) { + doScrollCheck(); + } + } + }, + + // See test/unit/core.js for details concerning isFunction. + // Since version 1.3, DOM methods and functions like alert + // aren't supported. They return false on IE (#2968). + isFunction: function( obj ) { + return jQuery.type(obj) === "function"; + }, + + isArray: Array.isArray || function( obj ) { + return jQuery.type(obj) === "array"; + }, + + // A crude way of determining if an object is a window + isWindow: function( obj ) { + return obj && typeof obj === "object" && "setInterval" in obj; + }, + + isNumeric: function( obj ) { + return !isNaN( parseFloat(obj) ) && isFinite( obj ); + }, + + type: function( obj ) { + return obj == null ? + String( obj ) : + class2type[ toString.call(obj) ] || "object"; + }, + + isPlainObject: function( obj ) { + // Must be an Object. + // Because of IE, we also have to check the presence of the constructor property. + // Make sure that DOM nodes and window objects don't pass through, as well + if ( !obj || jQuery.type(obj) !== "object" || obj.nodeType || jQuery.isWindow( obj ) ) { + return false; + } + + try { + // Not own constructor property must be Object + if ( obj.constructor && + !hasOwn.call(obj, "constructor") && + !hasOwn.call(obj.constructor.prototype, "isPrototypeOf") ) { + return false; + } + } catch ( e ) { + // IE8,9 Will throw exceptions on certain host objects #9897 + return false; + } + + // Own properties are enumerated firstly, so to speed up, + // if last one is own, then all properties are own. + + var key; + for ( key in obj ) {} + + return key === undefined || hasOwn.call( obj, key ); + }, + + isEmptyObject: function( obj ) { + for ( var name in obj ) { + return false; + } + return true; + }, + + error: function( msg ) { + throw new Error( msg ); + }, + + parseJSON: function( data ) { + if ( typeof data !== "string" || !data ) { + return null; + } + + // Make sure leading/trailing whitespace is removed (IE can't handle it) + data = jQuery.trim( data ); + + // Attempt to parse using the native JSON parser first + if ( window.JSON && window.JSON.parse ) { + return window.JSON.parse( data ); + } + + // Make sure the incoming data is actual JSON + // Logic borrowed from http://json.org/json2.js + if ( rvalidchars.test( data.replace( rvalidescape, "@" ) + .replace( rvalidtokens, "]" ) + .replace( rvalidbraces, "")) ) { + + return ( new Function( "return " + data ) )(); + + } + jQuery.error( "Invalid JSON: " + data ); + }, + + // Cross-browser xml parsing + parseXML: function( data ) { + var xml, tmp; + try { + if ( window.DOMParser ) { // Standard + tmp = new DOMParser(); + xml = tmp.parseFromString( data , "text/xml" ); + } else { // IE + xml = new ActiveXObject( "Microsoft.XMLDOM" ); + xml.async = "false"; + xml.loadXML( data ); + } + } catch( e ) { + xml = undefined; + } + if ( !xml || !xml.documentElement || xml.getElementsByTagName( "parsererror" ).length ) { + jQuery.error( "Invalid XML: " + data ); + } + return xml; + }, + + noop: function() {}, + + // Evaluates a script in a global context + // Workarounds based on findings by Jim Driscoll + // http://weblogs.java.net/blog/driscoll/archive/2009/09/08/eval-javascript-global-context + globalEval: function( data ) { + if ( data && rnotwhite.test( data ) ) { + // We use execScript on Internet Explorer + // We use an anonymous function so that context is window + // rather than jQuery in Firefox + ( window.execScript || function( data ) { + window[ "eval" ].call( window, data ); + } )( data ); + } + }, + + // Convert dashed to camelCase; used by the css and data modules + // Microsoft forgot to hump their vendor prefix (#9572) + camelCase: function( string ) { + return string.replace( rmsPrefix, "ms-" ).replace( rdashAlpha, fcamelCase ); + }, + + nodeName: function( elem, name ) { + return elem.nodeName && elem.nodeName.toUpperCase() === name.toUpperCase(); + }, + + // args is for internal usage only + each: function( object, callback, args ) { + var name, i = 0, + length = object.length, + isObj = length === undefined || jQuery.isFunction( object ); + + if ( args ) { + if ( isObj ) { + for ( name in object ) { + if ( callback.apply( object[ name ], args ) === false ) { + break; + } + } + } else { + for ( ; i < length; ) { + if ( callback.apply( object[ i++ ], args ) === false ) { + break; + } + } + } + + // A special, fast, case for the most common use of each + } else { + if ( isObj ) { + for ( name in object ) { + if ( callback.call( object[ name ], name, object[ name ] ) === false ) { + break; + } + } + } else { + for ( ; i < length; ) { + if ( callback.call( object[ i ], i, object[ i++ ] ) === false ) { + break; + } + } + } + } + + return object; + }, + + // Use native String.trim function wherever possible + trim: trim ? + function( text ) { + return text == null ? + "" : + trim.call( text ); + } : + + // Otherwise use our own trimming functionality + function( text ) { + return text == null ? + "" : + text.toString().replace( trimLeft, "" ).replace( trimRight, "" ); + }, + + // results is for internal usage only + makeArray: function( array, results ) { + var ret = results || []; + + if ( array != null ) { + // The window, strings (and functions) also have 'length' + // Tweaked logic slightly to handle Blackberry 4.7 RegExp issues #6930 + var type = jQuery.type( array ); + + if ( array.length == null || type === "string" || type === "function" || type === "regexp" || jQuery.isWindow( array ) ) { + push.call( ret, array ); + } else { + jQuery.merge( ret, array ); + } + } + + return ret; + }, + + inArray: function( elem, array, i ) { + var len; + + if ( array ) { + if ( indexOf ) { + return indexOf.call( array, elem, i ); + } + + len = array.length; + i = i ? i < 0 ? Math.max( 0, len + i ) : i : 0; + + for ( ; i < len; i++ ) { + // Skip accessing in sparse arrays + if ( i in array && array[ i ] === elem ) { + return i; + } + } + } + + return -1; + }, + + merge: function( first, second ) { + var i = first.length, + j = 0; + + if ( typeof second.length === "number" ) { + for ( var l = second.length; j < l; j++ ) { + first[ i++ ] = second[ j ]; + } + + } else { + while ( second[j] !== undefined ) { + first[ i++ ] = second[ j++ ]; + } + } + + first.length = i; + + return first; + }, + + grep: function( elems, callback, inv ) { + var ret = [], retVal; + inv = !!inv; + + // Go through the array, only saving the items + // that pass the validator function + for ( var i = 0, length = elems.length; i < length; i++ ) { + retVal = !!callback( elems[ i ], i ); + if ( inv !== retVal ) { + ret.push( elems[ i ] ); + } + } + + return ret; + }, + + // arg is for internal usage only + map: function( elems, callback, arg ) { + var value, key, ret = [], + i = 0, + length = elems.length, + // jquery objects are treated as arrays + isArray = elems instanceof jQuery || length !== undefined && typeof length === "number" && ( ( length > 0 && elems[ 0 ] && elems[ length -1 ] ) || length === 0 || jQuery.isArray( elems ) ) ; + + // Go through the array, translating each of the items to their + if ( isArray ) { + for ( ; i < length; i++ ) { + value = callback( elems[ i ], i, arg ); + + if ( value != null ) { + ret[ ret.length ] = value; + } + } + + // Go through every key on the object, + } else { + for ( key in elems ) { + value = callback( elems[ key ], key, arg ); + + if ( value != null ) { + ret[ ret.length ] = value; + } + } + } + + // Flatten any nested arrays + return ret.concat.apply( [], ret ); + }, + + // A global GUID counter for objects + guid: 1, + + // Bind a function to a context, optionally partially applying any + // arguments. + proxy: function( fn, context ) { + if ( typeof context === "string" ) { + var tmp = fn[ context ]; + context = fn; + fn = tmp; + } + + // Quick check to determine if target is callable, in the spec + // this throws a TypeError, but we will just return undefined. + if ( !jQuery.isFunction( fn ) ) { + return undefined; + } + + // Simulated bind + var args = slice.call( arguments, 2 ), + proxy = function() { + return fn.apply( context, args.concat( slice.call( arguments ) ) ); + }; + + // Set the guid of unique handler to the same of original handler, so it can be removed + proxy.guid = fn.guid = fn.guid || proxy.guid || jQuery.guid++; + + return proxy; + }, + + // Mutifunctional method to get and set values to a collection + // The value/s can optionally be executed if it's a function + access: function( elems, key, value, exec, fn, pass ) { + var length = elems.length; + + // Setting many attributes + if ( typeof key === "object" ) { + for ( var k in key ) { + jQuery.access( elems, k, key[k], exec, fn, value ); + } + return elems; + } + + // Setting one attribute + if ( value !== undefined ) { + // Optionally, function values get executed if exec is true + exec = !pass && exec && jQuery.isFunction(value); + + for ( var i = 0; i < length; i++ ) { + fn( elems[i], key, exec ? value.call( elems[i], i, fn( elems[i], key ) ) : value, pass ); + } + + return elems; + } + + // Getting an attribute + return length ? fn( elems[0], key ) : undefined; + }, + + now: function() { + return ( new Date() ).getTime(); + }, + + // Use of jQuery.browser is frowned upon. + // More details: http://docs.jquery.com/Utilities/jQuery.browser + uaMatch: function( ua ) { + ua = ua.toLowerCase(); + + var match = rwebkit.exec( ua ) || + ropera.exec( ua ) || + rmsie.exec( ua ) || + ua.indexOf("compatible") < 0 && rmozilla.exec( ua ) || + []; + + return { browser: match[1] || "", version: match[2] || "0" }; + }, + + sub: function() { + function jQuerySub( selector, context ) { + return new jQuerySub.fn.init( selector, context ); + } + jQuery.extend( true, jQuerySub, this ); + jQuerySub.superclass = this; + jQuerySub.fn = jQuerySub.prototype = this(); + jQuerySub.fn.constructor = jQuerySub; + jQuerySub.sub = this.sub; + jQuerySub.fn.init = function init( selector, context ) { + if ( context && context instanceof jQuery && !(context instanceof jQuerySub) ) { + context = jQuerySub( context ); + } + + return jQuery.fn.init.call( this, selector, context, rootjQuerySub ); + }; + jQuerySub.fn.init.prototype = jQuerySub.fn; + var rootjQuerySub = jQuerySub(document); + return jQuerySub; + }, + + browser: {} +}); + +// Populate the class2type map +jQuery.each("Boolean Number String Function Array Date RegExp Object".split(" "), function(i, name) { + class2type[ "[object " + name + "]" ] = name.toLowerCase(); +}); + +browserMatch = jQuery.uaMatch( userAgent ); +if ( browserMatch.browser ) { + jQuery.browser[ browserMatch.browser ] = true; + jQuery.browser.version = browserMatch.version; +} + +// Deprecated, use jQuery.browser.webkit instead +if ( jQuery.browser.webkit ) { + jQuery.browser.safari = true; +} + +// IE doesn't match non-breaking spaces with \s +if ( rnotwhite.test( "\xA0" ) ) { + trimLeft = /^[\s\xA0]+/; + trimRight = /[\s\xA0]+$/; +} + +// All jQuery objects should point back to these +rootjQuery = jQuery(document); + +// Cleanup functions for the document ready method +if ( document.addEventListener ) { + DOMContentLoaded = function() { + document.removeEventListener( "DOMContentLoaded", DOMContentLoaded, false ); + jQuery.ready(); + }; + +} else if ( document.attachEvent ) { + DOMContentLoaded = function() { + // Make sure body exists, at least, in case IE gets a little overzealous (ticket #5443). + if ( document.readyState === "complete" ) { + document.detachEvent( "onreadystatechange", DOMContentLoaded ); + jQuery.ready(); + } + }; +} + +// The DOM ready check for Internet Explorer +function doScrollCheck() { + if ( jQuery.isReady ) { + return; + } + + try { + // If IE is used, use the trick by Diego Perini + // http://javascript.nwbox.com/IEContentLoaded/ + document.documentElement.doScroll("left"); + } catch(e) { + setTimeout( doScrollCheck, 1 ); + return; + } + + // and execute any waiting functions + jQuery.ready(); +} + +return jQuery; + +})(); + + +// String to Object flags format cache +var flagsCache = {}; + +// Convert String-formatted flags into Object-formatted ones and store in cache +function createFlags( flags ) { + var object = flagsCache[ flags ] = {}, + i, length; + flags = flags.split( /\s+/ ); + for ( i = 0, length = flags.length; i < length; i++ ) { + object[ flags[i] ] = true; + } + return object; +} + +/* + * Create a callback list using the following parameters: + * + * flags: an optional list of space-separated flags that will change how + * the callback list behaves + * + * By default a callback list will act like an event callback list and can be + * "fired" multiple times. + * + * Possible flags: + * + * once: will ensure the callback list can only be fired once (like a Deferred) + * + * memory: will keep track of previous values and will call any callback added + * after the list has been fired right away with the latest "memorized" + * values (like a Deferred) + * + * unique: will ensure a callback can only be added once (no duplicate in the list) + * + * stopOnFalse: interrupt callings when a callback returns false + * + */ +jQuery.Callbacks = function( flags ) { + + // Convert flags from String-formatted to Object-formatted + // (we check in cache first) + flags = flags ? ( flagsCache[ flags ] || createFlags( flags ) ) : {}; + + var // Actual callback list + list = [], + // Stack of fire calls for repeatable lists + stack = [], + // Last fire value (for non-forgettable lists) + memory, + // Flag to know if list is currently firing + firing, + // First callback to fire (used internally by add and fireWith) + firingStart, + // End of the loop when firing + firingLength, + // Index of currently firing callback (modified by remove if needed) + firingIndex, + // Add one or several callbacks to the list + add = function( args ) { + var i, + length, + elem, + type, + actual; + for ( i = 0, length = args.length; i < length; i++ ) { + elem = args[ i ]; + type = jQuery.type( elem ); + if ( type === "array" ) { + // Inspect recursively + add( elem ); + } else if ( type === "function" ) { + // Add if not in unique mode and callback is not in + if ( !flags.unique || !self.has( elem ) ) { + list.push( elem ); + } + } + } + }, + // Fire callbacks + fire = function( context, args ) { + args = args || []; + memory = !flags.memory || [ context, args ]; + firing = true; + firingIndex = firingStart || 0; + firingStart = 0; + firingLength = list.length; + for ( ; list && firingIndex < firingLength; firingIndex++ ) { + if ( list[ firingIndex ].apply( context, args ) === false && flags.stopOnFalse ) { + memory = true; // Mark as halted + break; + } + } + firing = false; + if ( list ) { + if ( !flags.once ) { + if ( stack && stack.length ) { + memory = stack.shift(); + self.fireWith( memory[ 0 ], memory[ 1 ] ); + } + } else if ( memory === true ) { + self.disable(); + } else { + list = []; + } + } + }, + // Actual Callbacks object + self = { + // Add a callback or a collection of callbacks to the list + add: function() { + if ( list ) { + var length = list.length; + add( arguments ); + // Do we need to add the callbacks to the + // current firing batch? + if ( firing ) { + firingLength = list.length; + // With memory, if we're not firing then + // we should call right away, unless previous + // firing was halted (stopOnFalse) + } else if ( memory && memory !== true ) { + firingStart = length; + fire( memory[ 0 ], memory[ 1 ] ); + } + } + return this; + }, + // Remove a callback from the list + remove: function() { + if ( list ) { + var args = arguments, + argIndex = 0, + argLength = args.length; + for ( ; argIndex < argLength ; argIndex++ ) { + for ( var i = 0; i < list.length; i++ ) { + if ( args[ argIndex ] === list[ i ] ) { + // Handle firingIndex and firingLength + if ( firing ) { + if ( i <= firingLength ) { + firingLength--; + if ( i <= firingIndex ) { + firingIndex--; + } + } + } + // Remove the element + list.splice( i--, 1 ); + // If we have some unicity property then + // we only need to do this once + if ( flags.unique ) { + break; + } + } + } + } + } + return this; + }, + // Control if a given callback is in the list + has: function( fn ) { + if ( list ) { + var i = 0, + length = list.length; + for ( ; i < length; i++ ) { + if ( fn === list[ i ] ) { + return true; + } + } + } + return false; + }, + // Remove all callbacks from the list + empty: function() { + list = []; + return this; + }, + // Have the list do nothing anymore + disable: function() { + list = stack = memory = undefined; + return this; + }, + // Is it disabled? + disabled: function() { + return !list; + }, + // Lock the list in its current state + lock: function() { + stack = undefined; + if ( !memory || memory === true ) { + self.disable(); + } + return this; + }, + // Is it locked? + locked: function() { + return !stack; + }, + // Call all callbacks with the given context and arguments + fireWith: function( context, args ) { + if ( stack ) { + if ( firing ) { + if ( !flags.once ) { + stack.push( [ context, args ] ); + } + } else if ( !( flags.once && memory ) ) { + fire( context, args ); + } + } + return this; + }, + // Call all the callbacks with the given arguments + fire: function() { + self.fireWith( this, arguments ); + return this; + }, + // To know if the callbacks have already been called at least once + fired: function() { + return !!memory; + } + }; + + return self; +}; + + + + +var // Static reference to slice + sliceDeferred = [].slice; + +jQuery.extend({ + + Deferred: function( func ) { + var doneList = jQuery.Callbacks( "once memory" ), + failList = jQuery.Callbacks( "once memory" ), + progressList = jQuery.Callbacks( "memory" ), + state = "pending", + lists = { + resolve: doneList, + reject: failList, + notify: progressList + }, + promise = { + done: doneList.add, + fail: failList.add, + progress: progressList.add, + + state: function() { + return state; + }, + + // Deprecated + isResolved: doneList.fired, + isRejected: failList.fired, + + then: function( doneCallbacks, failCallbacks, progressCallbacks ) { + deferred.done( doneCallbacks ).fail( failCallbacks ).progress( progressCallbacks ); + return this; + }, + always: function() { + deferred.done.apply( deferred, arguments ).fail.apply( deferred, arguments ); + return this; + }, + pipe: function( fnDone, fnFail, fnProgress ) { + return jQuery.Deferred(function( newDefer ) { + jQuery.each( { + done: [ fnDone, "resolve" ], + fail: [ fnFail, "reject" ], + progress: [ fnProgress, "notify" ] + }, function( handler, data ) { + var fn = data[ 0 ], + action = data[ 1 ], + returned; + if ( jQuery.isFunction( fn ) ) { + deferred[ handler ](function() { + returned = fn.apply( this, arguments ); + if ( returned && jQuery.isFunction( returned.promise ) ) { + returned.promise().then( newDefer.resolve, newDefer.reject, newDefer.notify ); + } else { + newDefer[ action + "With" ]( this === deferred ? newDefer : this, [ returned ] ); + } + }); + } else { + deferred[ handler ]( newDefer[ action ] ); + } + }); + }).promise(); + }, + // Get a promise for this deferred + // If obj is provided, the promise aspect is added to the object + promise: function( obj ) { + if ( obj == null ) { + obj = promise; + } else { + for ( var key in promise ) { + obj[ key ] = promise[ key ]; + } + } + return obj; + } + }, + deferred = promise.promise({}), + key; + + for ( key in lists ) { + deferred[ key ] = lists[ key ].fire; + deferred[ key + "With" ] = lists[ key ].fireWith; + } + + // Handle state + deferred.done( function() { + state = "resolved"; + }, failList.disable, progressList.lock ).fail( function() { + state = "rejected"; + }, doneList.disable, progressList.lock ); + + // Call given func if any + if ( func ) { + func.call( deferred, deferred ); + } + + // All done! + return deferred; + }, + + // Deferred helper + when: function( firstParam ) { + var args = sliceDeferred.call( arguments, 0 ), + i = 0, + length = args.length, + pValues = new Array( length ), + count = length, + pCount = length, + deferred = length <= 1 && firstParam && jQuery.isFunction( firstParam.promise ) ? + firstParam : + jQuery.Deferred(), + promise = deferred.promise(); + function resolveFunc( i ) { + return function( value ) { + args[ i ] = arguments.length > 1 ? sliceDeferred.call( arguments, 0 ) : value; + if ( !( --count ) ) { + deferred.resolveWith( deferred, args ); + } + }; + } + function progressFunc( i ) { + return function( value ) { + pValues[ i ] = arguments.length > 1 ? sliceDeferred.call( arguments, 0 ) : value; + deferred.notifyWith( promise, pValues ); + }; + } + if ( length > 1 ) { + for ( ; i < length; i++ ) { + if ( args[ i ] && args[ i ].promise && jQuery.isFunction( args[ i ].promise ) ) { + args[ i ].promise().then( resolveFunc(i), deferred.reject, progressFunc(i) ); + } else { + --count; + } + } + if ( !count ) { + deferred.resolveWith( deferred, args ); + } + } else if ( deferred !== firstParam ) { + deferred.resolveWith( deferred, length ? [ firstParam ] : [] ); + } + return promise; + } +}); + + + + +jQuery.support = (function() { + + var support, + all, + a, + select, + opt, + input, + marginDiv, + fragment, + tds, + events, + eventName, + i, + isSupported, + div = document.createElement( "div" ), + documentElement = document.documentElement; + + // Preliminary tests + div.setAttribute("className", "t"); + div.innerHTML = " <link/><table></table><a href='/a' style='top:1px;float:left;opacity:.55;'>a</a><input type='checkbox'/>"; + + all = div.getElementsByTagName( "*" ); + a = div.getElementsByTagName( "a" )[ 0 ]; + + // Can't get basic test support + if ( !all || !all.length || !a ) { + return {}; + } + + // First batch of supports tests + select = document.createElement( "select" ); + opt = select.appendChild( document.createElement("option") ); + input = div.getElementsByTagName( "input" )[ 0 ]; + + support = { + // IE strips leading whitespace when .innerHTML is used + leadingWhitespace: ( div.firstChild.nodeType === 3 ), + + // Make sure that tbody elements aren't automatically inserted + // IE will insert them into empty tables + tbody: !div.getElementsByTagName("tbody").length, + + // Make sure that link elements get serialized correctly by innerHTML + // This requires a wrapper element in IE + htmlSerialize: !!div.getElementsByTagName("link").length, + + // Get the style information from getAttribute + // (IE uses .cssText instead) + style: /top/.test( a.getAttribute("style") ), + + // Make sure that URLs aren't manipulated + // (IE normalizes it by default) + hrefNormalized: ( a.getAttribute("href") === "/a" ), + + // Make sure that element opacity exists + // (IE uses filter instead) + // Use a regex to work around a WebKit issue. See #5145 + opacity: /^0.55/.test( a.style.opacity ), + + // Verify style float existence + // (IE uses styleFloat instead of cssFloat) + cssFloat: !!a.style.cssFloat, + + // Make sure that if no value is specified for a checkbox + // that it defaults to "on". + // (WebKit defaults to "" instead) + checkOn: ( input.value === "on" ), + + // Make sure that a selected-by-default option has a working selected property. + // (WebKit defaults to false instead of true, IE too, if it's in an optgroup) + optSelected: opt.selected, + + // Test setAttribute on camelCase class. If it works, we need attrFixes when doing get/setAttribute (ie6/7) + getSetAttribute: div.className !== "t", + + // Tests for enctype support on a form(#6743) + enctype: !!document.createElement("form").enctype, + + // Makes sure cloning an html5 element does not cause problems + // Where outerHTML is undefined, this still works + html5Clone: document.createElement("nav").cloneNode( true ).outerHTML !== "<:nav></:nav>", + + // Will be defined later + submitBubbles: true, + changeBubbles: true, + focusinBubbles: false, + deleteExpando: true, + noCloneEvent: true, + inlineBlockNeedsLayout: false, + shrinkWrapBlocks: false, + reliableMarginRight: true + }; + + // Make sure checked status is properly cloned + input.checked = true; + support.noCloneChecked = input.cloneNode( true ).checked; + + // Make sure that the options inside disabled selects aren't marked as disabled + // (WebKit marks them as disabled) + select.disabled = true; + support.optDisabled = !opt.disabled; + + // Test to see if it's possible to delete an expando from an element + // Fails in Internet Explorer + try { + delete div.test; + } catch( e ) { + support.deleteExpando = false; + } + + if ( !div.addEventListener && div.attachEvent && div.fireEvent ) { + div.attachEvent( "onclick", function() { + // Cloning a node shouldn't copy over any + // bound event handlers (IE does this) + support.noCloneEvent = false; + }); + div.cloneNode( true ).fireEvent( "onclick" ); + } + + // Check if a radio maintains its value + // after being appended to the DOM + input = document.createElement("input"); + input.value = "t"; + input.setAttribute("type", "radio"); + support.radioValue = input.value === "t"; + + input.setAttribute("checked", "checked"); + div.appendChild( input ); + fragment = document.createDocumentFragment(); + fragment.appendChild( div.lastChild ); + + // WebKit doesn't clone checked state correctly in fragments + support.checkClone = fragment.cloneNode( true ).cloneNode( true ).lastChild.checked; + + // Check if a disconnected checkbox will retain its checked + // value of true after appended to the DOM (IE6/7) + support.appendChecked = input.checked; + + fragment.removeChild( input ); + fragment.appendChild( div ); + + div.innerHTML = ""; + + // Check if div with explicit width and no margin-right incorrectly + // gets computed margin-right based on width of container. For more + // info see bug #3333 + // Fails in WebKit before Feb 2011 nightlies + // WebKit Bug 13343 - getComputedStyle returns wrong value for margin-right + if ( window.getComputedStyle ) { + marginDiv = document.createElement( "div" ); + marginDiv.style.width = "0"; + marginDiv.style.marginRight = "0"; + div.style.width = "2px"; + div.appendChild( marginDiv ); + support.reliableMarginRight = + ( parseInt( ( window.getComputedStyle( marginDiv, null ) || { marginRight: 0 } ).marginRight, 10 ) || 0 ) === 0; + } + + // Technique from Juriy Zaytsev + // http://perfectionkills.com/detecting-event-support-without-browser-sniffing/ + // We only care about the case where non-standard event systems + // are used, namely in IE. Short-circuiting here helps us to + // avoid an eval call (in setAttribute) which can cause CSP + // to go haywire. See: https://developer.mozilla.org/en/Security/CSP + if ( div.attachEvent ) { + for( i in { + submit: 1, + change: 1, + focusin: 1 + }) { + eventName = "on" + i; + isSupported = ( eventName in div ); + if ( !isSupported ) { + div.setAttribute( eventName, "return;" ); + isSupported = ( typeof div[ eventName ] === "function" ); + } + support[ i + "Bubbles" ] = isSupported; + } + } + + fragment.removeChild( div ); + + // Null elements to avoid leaks in IE + fragment = select = opt = marginDiv = div = input = null; + + // Run tests that need a body at doc ready + jQuery(function() { + var container, outer, inner, table, td, offsetSupport, + conMarginTop, ptlm, vb, style, html, + body = document.getElementsByTagName("body")[0]; + + if ( !body ) { + // Return for frameset docs that don't have a body + return; + } + + conMarginTop = 1; + ptlm = "position:absolute;top:0;left:0;width:1px;height:1px;margin:0;"; + vb = "visibility:hidden;border:0;"; + style = "style='" + ptlm + "border:5px solid #000;padding:0;'"; + html = "<div " + style + "><div></div></div>" + + "<table " + style + " cellpadding='0' cellspacing='0'>" + + "<tr><td></td></tr></table>"; + + container = document.createElement("div"); + container.style.cssText = vb + "width:0;height:0;position:static;top:0;margin-top:" + conMarginTop + "px"; + body.insertBefore( container, body.firstChild ); + + // Construct the test element + div = document.createElement("div"); + container.appendChild( div ); + + // Check if table cells still have offsetWidth/Height when they are set + // to display:none and there are still other visible table cells in a + // table row; if so, offsetWidth/Height are not reliable for use when + // determining if an element has been hidden directly using + // display:none (it is still safe to use offsets if a parent element is + // hidden; don safety goggles and see bug #4512 for more information). + // (only IE 8 fails this test) + div.innerHTML = "<table><tr><td style='padding:0;border:0;display:none'></td><td>t</td></tr></table>"; + tds = div.getElementsByTagName( "td" ); + isSupported = ( tds[ 0 ].offsetHeight === 0 ); + + tds[ 0 ].style.display = ""; + tds[ 1 ].style.display = "none"; + + // Check if empty table cells still have offsetWidth/Height + // (IE <= 8 fail this test) + support.reliableHiddenOffsets = isSupported && ( tds[ 0 ].offsetHeight === 0 ); + + // Figure out if the W3C box model works as expected + div.innerHTML = ""; + div.style.width = div.style.paddingLeft = "1px"; + jQuery.boxModel = support.boxModel = div.offsetWidth === 2; + + if ( typeof div.style.zoom !== "undefined" ) { + // Check if natively block-level elements act like inline-block + // elements when setting their display to 'inline' and giving + // them layout + // (IE < 8 does this) + div.style.display = "inline"; + div.style.zoom = 1; + support.inlineBlockNeedsLayout = ( div.offsetWidth === 2 ); + + // Check if elements with layout shrink-wrap their children + // (IE 6 does this) + div.style.display = ""; + div.innerHTML = "<div style='width:4px;'></div>"; + support.shrinkWrapBlocks = ( div.offsetWidth !== 2 ); + } + + div.style.cssText = ptlm + vb; + div.innerHTML = html; + + outer = div.firstChild; + inner = outer.firstChild; + td = outer.nextSibling.firstChild.firstChild; + + offsetSupport = { + doesNotAddBorder: ( inner.offsetTop !== 5 ), + doesAddBorderForTableAndCells: ( td.offsetTop === 5 ) + }; + + inner.style.position = "fixed"; + inner.style.top = "20px"; + + // safari subtracts parent border width here which is 5px + offsetSupport.fixedPosition = ( inner.offsetTop === 20 || inner.offsetTop === 15 ); + inner.style.position = inner.style.top = ""; + + outer.style.overflow = "hidden"; + outer.style.position = "relative"; + + offsetSupport.subtractsBorderForOverflowNotVisible = ( inner.offsetTop === -5 ); + offsetSupport.doesNotIncludeMarginInBodyOffset = ( body.offsetTop !== conMarginTop ); + + body.removeChild( container ); + div = container = null; + + jQuery.extend( support, offsetSupport ); + }); + + return support; +})(); + + + + +var rbrace = /^(?:\{.*\}|\[.*\])$/, + rmultiDash = /([A-Z])/g; + +jQuery.extend({ + cache: {}, + + // Please use with caution + uuid: 0, + + // Unique for each copy of jQuery on the page + // Non-digits removed to match rinlinejQuery + expando: "jQuery" + ( jQuery.fn.jquery + Math.random() ).replace( /\D/g, "" ), + + // The following elements throw uncatchable exceptions if you + // attempt to add expando properties to them. + noData: { + "embed": true, + // Ban all objects except for Flash (which handle expandos) + "object": "clsid:D27CDB6E-AE6D-11cf-96B8-444553540000", + "applet": true + }, + + hasData: function( elem ) { + elem = elem.nodeType ? jQuery.cache[ elem[jQuery.expando] ] : elem[ jQuery.expando ]; + return !!elem && !isEmptyDataObject( elem ); + }, + + data: function( elem, name, data, pvt /* Internal Use Only */ ) { + if ( !jQuery.acceptData( elem ) ) { + return; + } + + var privateCache, thisCache, ret, + internalKey = jQuery.expando, + getByName = typeof name === "string", + + // We have to handle DOM nodes and JS objects differently because IE6-7 + // can't GC object references properly across the DOM-JS boundary + isNode = elem.nodeType, + + // Only DOM nodes need the global jQuery cache; JS object data is + // attached directly to the object so GC can occur automatically + cache = isNode ? jQuery.cache : elem, + + // Only defining an ID for JS objects if its cache already exists allows + // the code to shortcut on the same path as a DOM node with no cache + id = isNode ? elem[ internalKey ] : elem[ internalKey ] && internalKey, + isEvents = name === "events"; + + // Avoid doing any more work than we need to when trying to get data on an + // object that has no data at all + if ( (!id || !cache[id] || (!isEvents && !pvt && !cache[id].data)) && getByName && data === undefined ) { + return; + } + + if ( !id ) { + // Only DOM nodes need a new unique ID for each element since their data + // ends up in the global cache + if ( isNode ) { + elem[ internalKey ] = id = ++jQuery.uuid; + } else { + id = internalKey; + } + } + + if ( !cache[ id ] ) { + cache[ id ] = {}; + + // Avoids exposing jQuery metadata on plain JS objects when the object + // is serialized using JSON.stringify + if ( !isNode ) { + cache[ id ].toJSON = jQuery.noop; + } + } + + // An object can be passed to jQuery.data instead of a key/value pair; this gets + // shallow copied over onto the existing cache + if ( typeof name === "object" || typeof name === "function" ) { + if ( pvt ) { + cache[ id ] = jQuery.extend( cache[ id ], name ); + } else { + cache[ id ].data = jQuery.extend( cache[ id ].data, name ); + } + } + + privateCache = thisCache = cache[ id ]; + + // jQuery data() is stored in a separate object inside the object's internal data + // cache in order to avoid key collisions between internal data and user-defined + // data. + if ( !pvt ) { + if ( !thisCache.data ) { + thisCache.data = {}; + } + + thisCache = thisCache.data; + } + + if ( data !== undefined ) { + thisCache[ jQuery.camelCase( name ) ] = data; + } + + // Users should not attempt to inspect the internal events object using jQuery.data, + // it is undocumented and subject to change. But does anyone listen? No. + if ( isEvents && !thisCache[ name ] ) { + return privateCache.events; + } + + // Check for both converted-to-camel and non-converted data property names + // If a data property was specified + if ( getByName ) { + + // First Try to find as-is property data + ret = thisCache[ name ]; + + // Test for null|undefined property data + if ( ret == null ) { + + // Try to find the camelCased property + ret = thisCache[ jQuery.camelCase( name ) ]; + } + } else { + ret = thisCache; + } + + return ret; + }, + + removeData: function( elem, name, pvt /* Internal Use Only */ ) { + if ( !jQuery.acceptData( elem ) ) { + return; + } + + var thisCache, i, l, + + // Reference to internal data cache key + internalKey = jQuery.expando, + + isNode = elem.nodeType, + + // See jQuery.data for more information + cache = isNode ? jQuery.cache : elem, + + // See jQuery.data for more information + id = isNode ? elem[ internalKey ] : internalKey; + + // If there is already no cache entry for this object, there is no + // purpose in continuing + if ( !cache[ id ] ) { + return; + } + + if ( name ) { + + thisCache = pvt ? cache[ id ] : cache[ id ].data; + + if ( thisCache ) { + + // Support array or space separated string names for data keys + if ( !jQuery.isArray( name ) ) { + + // try the string as a key before any manipulation + if ( name in thisCache ) { + name = [ name ]; + } else { + + // split the camel cased version by spaces unless a key with the spaces exists + name = jQuery.camelCase( name ); + if ( name in thisCache ) { + name = [ name ]; + } else { + name = name.split( " " ); + } + } + } + + for ( i = 0, l = name.length; i < l; i++ ) { + delete thisCache[ name[i] ]; + } + + // If there is no data left in the cache, we want to continue + // and let the cache object itself get destroyed + if ( !( pvt ? isEmptyDataObject : jQuery.isEmptyObject )( thisCache ) ) { + return; + } + } + } + + // See jQuery.data for more information + if ( !pvt ) { + delete cache[ id ].data; + + // Don't destroy the parent cache unless the internal data object + // had been the only thing left in it + if ( !isEmptyDataObject(cache[ id ]) ) { + return; + } + } + + // Browsers that fail expando deletion also refuse to delete expandos on + // the window, but it will allow it on all other JS objects; other browsers + // don't care + // Ensure that `cache` is not a window object #10080 + if ( jQuery.support.deleteExpando || !cache.setInterval ) { + delete cache[ id ]; + } else { + cache[ id ] = null; + } + + // We destroyed the cache and need to eliminate the expando on the node to avoid + // false lookups in the cache for entries that no longer exist + if ( isNode ) { + // IE does not allow us to delete expando properties from nodes, + // nor does it have a removeAttribute function on Document nodes; + // we must handle all of these cases + if ( jQuery.support.deleteExpando ) { + delete elem[ internalKey ]; + } else if ( elem.removeAttribute ) { + elem.removeAttribute( internalKey ); + } else { + elem[ internalKey ] = null; + } + } + }, + + // For internal use only. + _data: function( elem, name, data ) { + return jQuery.data( elem, name, data, true ); + }, + + // A method for determining if a DOM node can handle the data expando + acceptData: function( elem ) { + if ( elem.nodeName ) { + var match = jQuery.noData[ elem.nodeName.toLowerCase() ]; + + if ( match ) { + return !(match === true || elem.getAttribute("classid") !== match); + } + } + + return true; + } +}); + +jQuery.fn.extend({ + data: function( key, value ) { + var parts, attr, name, + data = null; + + if ( typeof key === "undefined" ) { + if ( this.length ) { + data = jQuery.data( this[0] ); + + if ( this[0].nodeType === 1 && !jQuery._data( this[0], "parsedAttrs" ) ) { + attr = this[0].attributes; + for ( var i = 0, l = attr.length; i < l; i++ ) { + name = attr[i].name; + + if ( name.indexOf( "data-" ) === 0 ) { + name = jQuery.camelCase( name.substring(5) ); + + dataAttr( this[0], name, data[ name ] ); + } + } + jQuery._data( this[0], "parsedAttrs", true ); + } + } + + return data; + + } else if ( typeof key === "object" ) { + return this.each(function() { + jQuery.data( this, key ); + }); + } + + parts = key.split("."); + parts[1] = parts[1] ? "." + parts[1] : ""; + + if ( value === undefined ) { + data = this.triggerHandler("getData" + parts[1] + "!", [parts[0]]); + + // Try to fetch any internally stored data first + if ( data === undefined && this.length ) { + data = jQuery.data( this[0], key ); + data = dataAttr( this[0], key, data ); + } + + return data === undefined && parts[1] ? + this.data( parts[0] ) : + data; + + } else { + return this.each(function() { + var self = jQuery( this ), + args = [ parts[0], value ]; + + self.triggerHandler( "setData" + parts[1] + "!", args ); + jQuery.data( this, key, value ); + self.triggerHandler( "changeData" + parts[1] + "!", args ); + }); + } + }, + + removeData: function( key ) { + return this.each(function() { + jQuery.removeData( this, key ); + }); + } +}); + +function dataAttr( elem, key, data ) { + // If nothing was found internally, try to fetch any + // data from the HTML5 data-* attribute + if ( data === undefined && elem.nodeType === 1 ) { + + var name = "data-" + key.replace( rmultiDash, "-$1" ).toLowerCase(); + + data = elem.getAttribute( name ); + + if ( typeof data === "string" ) { + try { + data = data === "true" ? true : + data === "false" ? false : + data === "null" ? null : + jQuery.isNumeric( data ) ? parseFloat( data ) : + rbrace.test( data ) ? jQuery.parseJSON( data ) : + data; + } catch( e ) {} + + // Make sure we set the data so it isn't changed later + jQuery.data( elem, key, data ); + + } else { + data = undefined; + } + } + + return data; +} + +// checks a cache object for emptiness +function isEmptyDataObject( obj ) { + for ( var name in obj ) { + + // if the public data object is empty, the private is still empty + if ( name === "data" && jQuery.isEmptyObject( obj[name] ) ) { + continue; + } + if ( name !== "toJSON" ) { + return false; + } + } + + return true; +} + + + + +function handleQueueMarkDefer( elem, type, src ) { + var deferDataKey = type + "defer", + queueDataKey = type + "queue", + markDataKey = type + "mark", + defer = jQuery._data( elem, deferDataKey ); + if ( defer && + ( src === "queue" || !jQuery._data(elem, queueDataKey) ) && + ( src === "mark" || !jQuery._data(elem, markDataKey) ) ) { + // Give room for hard-coded callbacks to fire first + // and eventually mark/queue something else on the element + setTimeout( function() { + if ( !jQuery._data( elem, queueDataKey ) && + !jQuery._data( elem, markDataKey ) ) { + jQuery.removeData( elem, deferDataKey, true ); + defer.fire(); + } + }, 0 ); + } +} + +jQuery.extend({ + + _mark: function( elem, type ) { + if ( elem ) { + type = ( type || "fx" ) + "mark"; + jQuery._data( elem, type, (jQuery._data( elem, type ) || 0) + 1 ); + } + }, + + _unmark: function( force, elem, type ) { + if ( force !== true ) { + type = elem; + elem = force; + force = false; + } + if ( elem ) { + type = type || "fx"; + var key = type + "mark", + count = force ? 0 : ( (jQuery._data( elem, key ) || 1) - 1 ); + if ( count ) { + jQuery._data( elem, key, count ); + } else { + jQuery.removeData( elem, key, true ); + handleQueueMarkDefer( elem, type, "mark" ); + } + } + }, + + queue: function( elem, type, data ) { + var q; + if ( elem ) { + type = ( type || "fx" ) + "queue"; + q = jQuery._data( elem, type ); + + // Speed up dequeue by getting out quickly if this is just a lookup + if ( data ) { + if ( !q || jQuery.isArray(data) ) { + q = jQuery._data( elem, type, jQuery.makeArray(data) ); + } else { + q.push( data ); + } + } + return q || []; + } + }, + + dequeue: function( elem, type ) { + type = type || "fx"; + + var queue = jQuery.queue( elem, type ), + fn = queue.shift(), + hooks = {}; + + // If the fx queue is dequeued, always remove the progress sentinel + if ( fn === "inprogress" ) { + fn = queue.shift(); + } + + if ( fn ) { + // Add a progress sentinel to prevent the fx queue from being + // automatically dequeued + if ( type === "fx" ) { + queue.unshift( "inprogress" ); + } + + jQuery._data( elem, type + ".run", hooks ); + fn.call( elem, function() { + jQuery.dequeue( elem, type ); + }, hooks ); + } + + if ( !queue.length ) { + jQuery.removeData( elem, type + "queue " + type + ".run", true ); + handleQueueMarkDefer( elem, type, "queue" ); + } + } +}); + +jQuery.fn.extend({ + queue: function( type, data ) { + if ( typeof type !== "string" ) { + data = type; + type = "fx"; + } + + if ( data === undefined ) { + return jQuery.queue( this[0], type ); + } + return this.each(function() { + var queue = jQuery.queue( this, type, data ); + + if ( type === "fx" && queue[0] !== "inprogress" ) { + jQuery.dequeue( this, type ); + } + }); + }, + dequeue: function( type ) { + return this.each(function() { + jQuery.dequeue( this, type ); + }); + }, + // Based off of the plugin by Clint Helfers, with permission. + // http://blindsignals.com/index.php/2009/07/jquery-delay/ + delay: function( time, type ) { + time = jQuery.fx ? jQuery.fx.speeds[ time ] || time : time; + type = type || "fx"; + + return this.queue( type, function( next, hooks ) { + var timeout = setTimeout( next, time ); + hooks.stop = function() { + clearTimeout( timeout ); + }; + }); + }, + clearQueue: function( type ) { + return this.queue( type || "fx", [] ); + }, + // Get a promise resolved when queues of a certain type + // are emptied (fx is the type by default) + promise: function( type, object ) { + if ( typeof type !== "string" ) { + object = type; + type = undefined; + } + type = type || "fx"; + var defer = jQuery.Deferred(), + elements = this, + i = elements.length, + count = 1, + deferDataKey = type + "defer", + queueDataKey = type + "queue", + markDataKey = type + "mark", + tmp; + function resolve() { + if ( !( --count ) ) { + defer.resolveWith( elements, [ elements ] ); + } + } + while( i-- ) { + if (( tmp = jQuery.data( elements[ i ], deferDataKey, undefined, true ) || + ( jQuery.data( elements[ i ], queueDataKey, undefined, true ) || + jQuery.data( elements[ i ], markDataKey, undefined, true ) ) && + jQuery.data( elements[ i ], deferDataKey, jQuery.Callbacks( "once memory" ), true ) )) { + count++; + tmp.add( resolve ); + } + } + resolve(); + return defer.promise(); + } +}); + + + + +var rclass = /[\n\t\r]/g, + rspace = /\s+/, + rreturn = /\r/g, + rtype = /^(?:button|input)$/i, + rfocusable = /^(?:button|input|object|select|textarea)$/i, + rclickable = /^a(?:rea)?$/i, + rboolean = /^(?:autofocus|autoplay|async|checked|controls|defer|disabled|hidden|loop|multiple|open|readonly|required|scoped|selected)$/i, + getSetAttribute = jQuery.support.getSetAttribute, + nodeHook, boolHook, fixSpecified; + +jQuery.fn.extend({ + attr: function( name, value ) { + return jQuery.access( this, name, value, true, jQuery.attr ); + }, + + removeAttr: function( name ) { + return this.each(function() { + jQuery.removeAttr( this, name ); + }); + }, + + prop: function( name, value ) { + return jQuery.access( this, name, value, true, jQuery.prop ); + }, + + removeProp: function( name ) { + name = jQuery.propFix[ name ] || name; + return this.each(function() { + // try/catch handles cases where IE balks (such as removing a property on window) + try { + this[ name ] = undefined; + delete this[ name ]; + } catch( e ) {} + }); + }, + + addClass: function( value ) { + var classNames, i, l, elem, + setClass, c, cl; + + if ( jQuery.isFunction( value ) ) { + return this.each(function( j ) { + jQuery( this ).addClass( value.call(this, j, this.className) ); + }); + } + + if ( value && typeof value === "string" ) { + classNames = value.split( rspace ); + + for ( i = 0, l = this.length; i < l; i++ ) { + elem = this[ i ]; + + if ( elem.nodeType === 1 ) { + if ( !elem.className && classNames.length === 1 ) { + elem.className = value; + + } else { + setClass = " " + elem.className + " "; + + for ( c = 0, cl = classNames.length; c < cl; c++ ) { + if ( !~setClass.indexOf( " " + classNames[ c ] + " " ) ) { + setClass += classNames[ c ] + " "; + } + } + elem.className = jQuery.trim( setClass ); + } + } + } + } + + return this; + }, + + removeClass: function( value ) { + var classNames, i, l, elem, className, c, cl; + + if ( jQuery.isFunction( value ) ) { + return this.each(function( j ) { + jQuery( this ).removeClass( value.call(this, j, this.className) ); + }); + } + + if ( (value && typeof value === "string") || value === undefined ) { + classNames = ( value || "" ).split( rspace ); + + for ( i = 0, l = this.length; i < l; i++ ) { + elem = this[ i ]; + + if ( elem.nodeType === 1 && elem.className ) { + if ( value ) { + className = (" " + elem.className + " ").replace( rclass, " " ); + for ( c = 0, cl = classNames.length; c < cl; c++ ) { + className = className.replace(" " + classNames[ c ] + " ", " "); + } + elem.className = jQuery.trim( className ); + + } else { + elem.className = ""; + } + } + } + } + + return this; + }, + + toggleClass: function( value, stateVal ) { + var type = typeof value, + isBool = typeof stateVal === "boolean"; + + if ( jQuery.isFunction( value ) ) { + return this.each(function( i ) { + jQuery( this ).toggleClass( value.call(this, i, this.className, stateVal), stateVal ); + }); + } + + return this.each(function() { + if ( type === "string" ) { + // toggle individual class names + var className, + i = 0, + self = jQuery( this ), + state = stateVal, + classNames = value.split( rspace ); + + while ( (className = classNames[ i++ ]) ) { + // check each className given, space seperated list + state = isBool ? state : !self.hasClass( className ); + self[ state ? "addClass" : "removeClass" ]( className ); + } + + } else if ( type === "undefined" || type === "boolean" ) { + if ( this.className ) { + // store className if set + jQuery._data( this, "__className__", this.className ); + } + + // toggle whole className + this.className = this.className || value === false ? "" : jQuery._data( this, "__className__" ) || ""; + } + }); + }, + + hasClass: function( selector ) { + var className = " " + selector + " ", + i = 0, + l = this.length; + for ( ; i < l; i++ ) { + if ( this[i].nodeType === 1 && (" " + this[i].className + " ").replace(rclass, " ").indexOf( className ) > -1 ) { + return true; + } + } + + return false; + }, + + val: function( value ) { + var hooks, ret, isFunction, + elem = this[0]; + + if ( !arguments.length ) { + if ( elem ) { + hooks = jQuery.valHooks[ elem.nodeName.toLowerCase() ] || jQuery.valHooks[ elem.type ]; + + if ( hooks && "get" in hooks && (ret = hooks.get( elem, "value" )) !== undefined ) { + return ret; + } + + ret = elem.value; + + return typeof ret === "string" ? + // handle most common string cases + ret.replace(rreturn, "") : + // handle cases where value is null/undef or number + ret == null ? "" : ret; + } + + return; + } + + isFunction = jQuery.isFunction( value ); + + return this.each(function( i ) { + var self = jQuery(this), val; + + if ( this.nodeType !== 1 ) { + return; + } + + if ( isFunction ) { + val = value.call( this, i, self.val() ); + } else { + val = value; + } + + // Treat null/undefined as ""; convert numbers to string + if ( val == null ) { + val = ""; + } else if ( typeof val === "number" ) { + val += ""; + } else if ( jQuery.isArray( val ) ) { + val = jQuery.map(val, function ( value ) { + return value == null ? "" : value + ""; + }); + } + + hooks = jQuery.valHooks[ this.nodeName.toLowerCase() ] || jQuery.valHooks[ this.type ]; + + // If set returns undefined, fall back to normal setting + if ( !hooks || !("set" in hooks) || hooks.set( this, val, "value" ) === undefined ) { + this.value = val; + } + }); + } +}); + +jQuery.extend({ + valHooks: { + option: { + get: function( elem ) { + // attributes.value is undefined in Blackberry 4.7 but + // uses .value. See #6932 + var val = elem.attributes.value; + return !val || val.specified ? elem.value : elem.text; + } + }, + select: { + get: function( elem ) { + var value, i, max, option, + index = elem.selectedIndex, + values = [], + options = elem.options, + one = elem.type === "select-one"; + + // Nothing was selected + if ( index < 0 ) { + return null; + } + + // Loop through all the selected options + i = one ? index : 0; + max = one ? index + 1 : options.length; + for ( ; i < max; i++ ) { + option = options[ i ]; + + // Don't return options that are disabled or in a disabled optgroup + if ( option.selected && (jQuery.support.optDisabled ? !option.disabled : option.getAttribute("disabled") === null) && + (!option.parentNode.disabled || !jQuery.nodeName( option.parentNode, "optgroup" )) ) { + + // Get the specific value for the option + value = jQuery( option ).val(); + + // We don't need an array for one selects + if ( one ) { + return value; + } + + // Multi-Selects return an array + values.push( value ); + } + } + + // Fixes Bug #2551 -- select.val() broken in IE after form.reset() + if ( one && !values.length && options.length ) { + return jQuery( options[ index ] ).val(); + } + + return values; + }, + + set: function( elem, value ) { + var values = jQuery.makeArray( value ); + + jQuery(elem).find("option").each(function() { + this.selected = jQuery.inArray( jQuery(this).val(), values ) >= 0; + }); + + if ( !values.length ) { + elem.selectedIndex = -1; + } + return values; + } + } + }, + + attrFn: { + val: true, + css: true, + html: true, + text: true, + data: true, + width: true, + height: true, + offset: true + }, + + attr: function( elem, name, value, pass ) { + var ret, hooks, notxml, + nType = elem.nodeType; + + // don't get/set attributes on text, comment and attribute nodes + if ( !elem || nType === 3 || nType === 8 || nType === 2 ) { + return; + } + + if ( pass && name in jQuery.attrFn ) { + return jQuery( elem )[ name ]( value ); + } + + // Fallback to prop when attributes are not supported + if ( typeof elem.getAttribute === "undefined" ) { + return jQuery.prop( elem, name, value ); + } + + notxml = nType !== 1 || !jQuery.isXMLDoc( elem ); + + // All attributes are lowercase + // Grab necessary hook if one is defined + if ( notxml ) { + name = name.toLowerCase(); + hooks = jQuery.attrHooks[ name ] || ( rboolean.test( name ) ? boolHook : nodeHook ); + } + + if ( value !== undefined ) { + + if ( value === null ) { + jQuery.removeAttr( elem, name ); + return; + + } else if ( hooks && "set" in hooks && notxml && (ret = hooks.set( elem, value, name )) !== undefined ) { + return ret; + + } else { + elem.setAttribute( name, "" + value ); + return value; + } + + } else if ( hooks && "get" in hooks && notxml && (ret = hooks.get( elem, name )) !== null ) { + return ret; + + } else { + + ret = elem.getAttribute( name ); + + // Non-existent attributes return null, we normalize to undefined + return ret === null ? + undefined : + ret; + } + }, + + removeAttr: function( elem, value ) { + var propName, attrNames, name, l, + i = 0; + + if ( value && elem.nodeType === 1 ) { + attrNames = value.toLowerCase().split( rspace ); + l = attrNames.length; + + for ( ; i < l; i++ ) { + name = attrNames[ i ]; + + if ( name ) { + propName = jQuery.propFix[ name ] || name; + + // See #9699 for explanation of this approach (setting first, then removal) + jQuery.attr( elem, name, "" ); + elem.removeAttribute( getSetAttribute ? name : propName ); + + // Set corresponding property to false for boolean attributes + if ( rboolean.test( name ) && propName in elem ) { + elem[ propName ] = false; + } + } + } + } + }, + + attrHooks: { + type: { + set: function( elem, value ) { + // We can't allow the type property to be changed (since it causes problems in IE) + if ( rtype.test( elem.nodeName ) && elem.parentNode ) { + jQuery.error( "type property can't be changed" ); + } else if ( !jQuery.support.radioValue && value === "radio" && jQuery.nodeName(elem, "input") ) { + // Setting the type on a radio button after the value resets the value in IE6-9 + // Reset value to it's default in case type is set after value + // This is for element creation + var val = elem.value; + elem.setAttribute( "type", value ); + if ( val ) { + elem.value = val; + } + return value; + } + } + }, + // Use the value property for back compat + // Use the nodeHook for button elements in IE6/7 (#1954) + value: { + get: function( elem, name ) { + if ( nodeHook && jQuery.nodeName( elem, "button" ) ) { + return nodeHook.get( elem, name ); + } + return name in elem ? + elem.value : + null; + }, + set: function( elem, value, name ) { + if ( nodeHook && jQuery.nodeName( elem, "button" ) ) { + return nodeHook.set( elem, value, name ); + } + // Does not return so that setAttribute is also used + elem.value = value; + } + } + }, + + propFix: { + tabindex: "tabIndex", + readonly: "readOnly", + "for": "htmlFor", + "class": "className", + maxlength: "maxLength", + cellspacing: "cellSpacing", + cellpadding: "cellPadding", + rowspan: "rowSpan", + colspan: "colSpan", + usemap: "useMap", + frameborder: "frameBorder", + contenteditable: "contentEditable" + }, + + prop: function( elem, name, value ) { + var ret, hooks, notxml, + nType = elem.nodeType; + + // don't get/set properties on text, comment and attribute nodes + if ( !elem || nType === 3 || nType === 8 || nType === 2 ) { + return; + } + + notxml = nType !== 1 || !jQuery.isXMLDoc( elem ); + + if ( notxml ) { + // Fix name and attach hooks + name = jQuery.propFix[ name ] || name; + hooks = jQuery.propHooks[ name ]; + } + + if ( value !== undefined ) { + if ( hooks && "set" in hooks && (ret = hooks.set( elem, value, name )) !== undefined ) { + return ret; + + } else { + return ( elem[ name ] = value ); + } + + } else { + if ( hooks && "get" in hooks && (ret = hooks.get( elem, name )) !== null ) { + return ret; + + } else { + return elem[ name ]; + } + } + }, + + propHooks: { + tabIndex: { + get: function( elem ) { + // elem.tabIndex doesn't always return the correct value when it hasn't been explicitly set + // http://fluidproject.org/blog/2008/01/09/getting-setting-and-removing-tabindex-values-with-javascript/ + var attributeNode = elem.getAttributeNode("tabindex"); + + return attributeNode && attributeNode.specified ? + parseInt( attributeNode.value, 10 ) : + rfocusable.test( elem.nodeName ) || rclickable.test( elem.nodeName ) && elem.href ? + 0 : + undefined; + } + } + } +}); + +// Add the tabIndex propHook to attrHooks for back-compat (different case is intentional) +jQuery.attrHooks.tabindex = jQuery.propHooks.tabIndex; + +// Hook for boolean attributes +boolHook = { + get: function( elem, name ) { + // Align boolean attributes with corresponding properties + // Fall back to attribute presence where some booleans are not supported + var attrNode, + property = jQuery.prop( elem, name ); + return property === true || typeof property !== "boolean" && ( attrNode = elem.getAttributeNode(name) ) && attrNode.nodeValue !== false ? + name.toLowerCase() : + undefined; + }, + set: function( elem, value, name ) { + var propName; + if ( value === false ) { + // Remove boolean attributes when set to false + jQuery.removeAttr( elem, name ); + } else { + // value is true since we know at this point it's type boolean and not false + // Set boolean attributes to the same name and set the DOM property + propName = jQuery.propFix[ name ] || name; + if ( propName in elem ) { + // Only set the IDL specifically if it already exists on the element + elem[ propName ] = true; + } + + elem.setAttribute( name, name.toLowerCase() ); + } + return name; + } +}; + +// IE6/7 do not support getting/setting some attributes with get/setAttribute +if ( !getSetAttribute ) { + + fixSpecified = { + name: true, + id: true + }; + + // Use this for any attribute in IE6/7 + // This fixes almost every IE6/7 issue + nodeHook = jQuery.valHooks.button = { + get: function( elem, name ) { + var ret; + ret = elem.getAttributeNode( name ); + return ret && ( fixSpecified[ name ] ? ret.nodeValue !== "" : ret.specified ) ? + ret.nodeValue : + undefined; + }, + set: function( elem, value, name ) { + // Set the existing or create a new attribute node + var ret = elem.getAttributeNode( name ); + if ( !ret ) { + ret = document.createAttribute( name ); + elem.setAttributeNode( ret ); + } + return ( ret.nodeValue = value + "" ); + } + }; + + // Apply the nodeHook to tabindex + jQuery.attrHooks.tabindex.set = nodeHook.set; + + // Set width and height to auto instead of 0 on empty string( Bug #8150 ) + // This is for removals + jQuery.each([ "width", "height" ], function( i, name ) { + jQuery.attrHooks[ name ] = jQuery.extend( jQuery.attrHooks[ name ], { + set: function( elem, value ) { + if ( value === "" ) { + elem.setAttribute( name, "auto" ); + return value; + } + } + }); + }); + + // Set contenteditable to false on removals(#10429) + // Setting to empty string throws an error as an invalid value + jQuery.attrHooks.contenteditable = { + get: nodeHook.get, + set: function( elem, value, name ) { + if ( value === "" ) { + value = "false"; + } + nodeHook.set( elem, value, name ); + } + }; +} + + +// Some attributes require a special call on IE +if ( !jQuery.support.hrefNormalized ) { + jQuery.each([ "href", "src", "width", "height" ], function( i, name ) { + jQuery.attrHooks[ name ] = jQuery.extend( jQuery.attrHooks[ name ], { + get: function( elem ) { + var ret = elem.getAttribute( name, 2 ); + return ret === null ? undefined : ret; + } + }); + }); +} + +if ( !jQuery.support.style ) { + jQuery.attrHooks.style = { + get: function( elem ) { + // Return undefined in the case of empty string + // Normalize to lowercase since IE uppercases css property names + return elem.style.cssText.toLowerCase() || undefined; + }, + set: function( elem, value ) { + return ( elem.style.cssText = "" + value ); + } + }; +} + +// Safari mis-reports the default selected property of an option +// Accessing the parent's selectedIndex property fixes it +if ( !jQuery.support.optSelected ) { + jQuery.propHooks.selected = jQuery.extend( jQuery.propHooks.selected, { + get: function( elem ) { + var parent = elem.parentNode; + + if ( parent ) { + parent.selectedIndex; + + // Make sure that it also works with optgroups, see #5701 + if ( parent.parentNode ) { + parent.parentNode.selectedIndex; + } + } + return null; + } + }); +} + +// IE6/7 call enctype encoding +if ( !jQuery.support.enctype ) { + jQuery.propFix.enctype = "encoding"; +} + +// Radios and checkboxes getter/setter +if ( !jQuery.support.checkOn ) { + jQuery.each([ "radio", "checkbox" ], function() { + jQuery.valHooks[ this ] = { + get: function( elem ) { + // Handle the case where in Webkit "" is returned instead of "on" if a value isn't specified + return elem.getAttribute("value") === null ? "on" : elem.value; + } + }; + }); +} +jQuery.each([ "radio", "checkbox" ], function() { + jQuery.valHooks[ this ] = jQuery.extend( jQuery.valHooks[ this ], { + set: function( elem, value ) { + if ( jQuery.isArray( value ) ) { + return ( elem.checked = jQuery.inArray( jQuery(elem).val(), value ) >= 0 ); + } + } + }); +}); + + + + +var rformElems = /^(?:textarea|input|select)$/i, + rtypenamespace = /^([^\.]*)?(?:\.(.+))?$/, + rhoverHack = /\bhover(\.\S+)?\b/, + rkeyEvent = /^key/, + rmouseEvent = /^(?:mouse|contextmenu)|click/, + rfocusMorph = /^(?:focusinfocus|focusoutblur)$/, + rquickIs = /^(\w*)(?:#([\w\-]+))?(?:\.([\w\-]+))?$/, + quickParse = function( selector ) { + var quick = rquickIs.exec( selector ); + if ( quick ) { + // 0 1 2 3 + // [ _, tag, id, class ] + quick[1] = ( quick[1] || "" ).toLowerCase(); + quick[3] = quick[3] && new RegExp( "(?:^|\\s)" + quick[3] + "(?:\\s|$)" ); + } + return quick; + }, + quickIs = function( elem, m ) { + var attrs = elem.attributes || {}; + return ( + (!m[1] || elem.nodeName.toLowerCase() === m[1]) && + (!m[2] || (attrs.id || {}).value === m[2]) && + (!m[3] || m[3].test( (attrs[ "class" ] || {}).value )) + ); + }, + hoverHack = function( events ) { + return jQuery.event.special.hover ? events : events.replace( rhoverHack, "mouseenter$1 mouseleave$1" ); + }; + +/* + * Helper functions for managing events -- not part of the public interface. + * Props to Dean Edwards' addEvent library for many of the ideas. + */ +jQuery.event = { + + add: function( elem, types, handler, data, selector ) { + + var elemData, eventHandle, events, + t, tns, type, namespaces, handleObj, + handleObjIn, quick, handlers, special; + + // Don't attach events to noData or text/comment nodes (allow plain objects tho) + if ( elem.nodeType === 3 || elem.nodeType === 8 || !types || !handler || !(elemData = jQuery._data( elem )) ) { + return; + } + + // Caller can pass in an object of custom data in lieu of the handler + if ( handler.handler ) { + handleObjIn = handler; + handler = handleObjIn.handler; + } + + // Make sure that the handler has a unique ID, used to find/remove it later + if ( !handler.guid ) { + handler.guid = jQuery.guid++; + } + + // Init the element's event structure and main handler, if this is the first + events = elemData.events; + if ( !events ) { + elemData.events = events = {}; + } + eventHandle = elemData.handle; + if ( !eventHandle ) { + elemData.handle = eventHandle = function( e ) { + // Discard the second event of a jQuery.event.trigger() and + // when an event is called after a page has unloaded + return typeof jQuery !== "undefined" && (!e || jQuery.event.triggered !== e.type) ? + jQuery.event.dispatch.apply( eventHandle.elem, arguments ) : + undefined; + }; + // Add elem as a property of the handle fn to prevent a memory leak with IE non-native events + eventHandle.elem = elem; + } + + // Handle multiple events separated by a space + // jQuery(...).bind("mouseover mouseout", fn); + types = jQuery.trim( hoverHack(types) ).split( " " ); + for ( t = 0; t < types.length; t++ ) { + + tns = rtypenamespace.exec( types[t] ) || []; + type = tns[1]; + namespaces = ( tns[2] || "" ).split( "." ).sort(); + + // If event changes its type, use the special event handlers for the changed type + special = jQuery.event.special[ type ] || {}; + + // If selector defined, determine special event api type, otherwise given type + type = ( selector ? special.delegateType : special.bindType ) || type; + + // Update special based on newly reset type + special = jQuery.event.special[ type ] || {}; + + // handleObj is passed to all event handlers + handleObj = jQuery.extend({ + type: type, + origType: tns[1], + data: data, + handler: handler, + guid: handler.guid, + selector: selector, + quick: quickParse( selector ), + namespace: namespaces.join(".") + }, handleObjIn ); + + // Init the event handler queue if we're the first + handlers = events[ type ]; + if ( !handlers ) { + handlers = events[ type ] = []; + handlers.delegateCount = 0; + + // Only use addEventListener/attachEvent if the special events handler returns false + if ( !special.setup || special.setup.call( elem, data, namespaces, eventHandle ) === false ) { + // Bind the global event handler to the element + if ( elem.addEventListener ) { + elem.addEventListener( type, eventHandle, false ); + + } else if ( elem.attachEvent ) { + elem.attachEvent( "on" + type, eventHandle ); + } + } + } + + if ( special.add ) { + special.add.call( elem, handleObj ); + + if ( !handleObj.handler.guid ) { + handleObj.handler.guid = handler.guid; + } + } + + // Add to the element's handler list, delegates in front + if ( selector ) { + handlers.splice( handlers.delegateCount++, 0, handleObj ); + } else { + handlers.push( handleObj ); + } + + // Keep track of which events have ever been used, for event optimization + jQuery.event.global[ type ] = true; + } + + // Nullify elem to prevent memory leaks in IE + elem = null; + }, + + global: {}, + + // Detach an event or set of events from an element + remove: function( elem, types, handler, selector, mappedTypes ) { + + var elemData = jQuery.hasData( elem ) && jQuery._data( elem ), + t, tns, type, origType, namespaces, origCount, + j, events, special, handle, eventType, handleObj; + + if ( !elemData || !(events = elemData.events) ) { + return; + } + + // Once for each type.namespace in types; type may be omitted + types = jQuery.trim( hoverHack( types || "" ) ).split(" "); + for ( t = 0; t < types.length; t++ ) { + tns = rtypenamespace.exec( types[t] ) || []; + type = origType = tns[1]; + namespaces = tns[2]; + + // Unbind all events (on this namespace, if provided) for the element + if ( !type ) { + for ( type in events ) { + jQuery.event.remove( elem, type + types[ t ], handler, selector, true ); + } + continue; + } + + special = jQuery.event.special[ type ] || {}; + type = ( selector? special.delegateType : special.bindType ) || type; + eventType = events[ type ] || []; + origCount = eventType.length; + namespaces = namespaces ? new RegExp("(^|\\.)" + namespaces.split(".").sort().join("\\.(?:.*\\.)?") + "(\\.|$)") : null; + + // Remove matching events + for ( j = 0; j < eventType.length; j++ ) { + handleObj = eventType[ j ]; + + if ( ( mappedTypes || origType === handleObj.origType ) && + ( !handler || handler.guid === handleObj.guid ) && + ( !namespaces || namespaces.test( handleObj.namespace ) ) && + ( !selector || selector === handleObj.selector || selector === "**" && handleObj.selector ) ) { + eventType.splice( j--, 1 ); + + if ( handleObj.selector ) { + eventType.delegateCount--; + } + if ( special.remove ) { + special.remove.call( elem, handleObj ); + } + } + } + + // Remove generic event handler if we removed something and no more handlers exist + // (avoids potential for endless recursion during removal of special event handlers) + if ( eventType.length === 0 && origCount !== eventType.length ) { + if ( !special.teardown || special.teardown.call( elem, namespaces ) === false ) { + jQuery.removeEvent( elem, type, elemData.handle ); + } + + delete events[ type ]; + } + } + + // Remove the expando if it's no longer used + if ( jQuery.isEmptyObject( events ) ) { + handle = elemData.handle; + if ( handle ) { + handle.elem = null; + } + + // removeData also checks for emptiness and clears the expando if empty + // so use it instead of delete + jQuery.removeData( elem, [ "events", "handle" ], true ); + } + }, + + // Events that are safe to short-circuit if no handlers are attached. + // Native DOM events should not be added, they may have inline handlers. + customEvent: { + "getData": true, + "setData": true, + "changeData": true + }, + + trigger: function( event, data, elem, onlyHandlers ) { + // Don't do events on text and comment nodes + if ( elem && (elem.nodeType === 3 || elem.nodeType === 8) ) { + return; + } + + // Event object or event type + var type = event.type || event, + namespaces = [], + cache, exclusive, i, cur, old, ontype, special, handle, eventPath, bubbleType; + + // focus/blur morphs to focusin/out; ensure we're not firing them right now + if ( rfocusMorph.test( type + jQuery.event.triggered ) ) { + return; + } + + if ( type.indexOf( "!" ) >= 0 ) { + // Exclusive events trigger only for the exact event (no namespaces) + type = type.slice(0, -1); + exclusive = true; + } + + if ( type.indexOf( "." ) >= 0 ) { + // Namespaced trigger; create a regexp to match event type in handle() + namespaces = type.split("."); + type = namespaces.shift(); + namespaces.sort(); + } + + if ( (!elem || jQuery.event.customEvent[ type ]) && !jQuery.event.global[ type ] ) { + // No jQuery handlers for this event type, and it can't have inline handlers + return; + } + + // Caller can pass in an Event, Object, or just an event type string + event = typeof event === "object" ? + // jQuery.Event object + event[ jQuery.expando ] ? event : + // Object literal + new jQuery.Event( type, event ) : + // Just the event type (string) + new jQuery.Event( type ); + + event.type = type; + event.isTrigger = true; + event.exclusive = exclusive; + event.namespace = namespaces.join( "." ); + event.namespace_re = event.namespace? new RegExp("(^|\\.)" + namespaces.join("\\.(?:.*\\.)?") + "(\\.|$)") : null; + ontype = type.indexOf( ":" ) < 0 ? "on" + type : ""; + + // Handle a global trigger + if ( !elem ) { + + // TODO: Stop taunting the data cache; remove global events and always attach to document + cache = jQuery.cache; + for ( i in cache ) { + if ( cache[ i ].events && cache[ i ].events[ type ] ) { + jQuery.event.trigger( event, data, cache[ i ].handle.elem, true ); + } + } + return; + } + + // Clean up the event in case it is being reused + event.result = undefined; + if ( !event.target ) { + event.target = elem; + } + + // Clone any incoming data and prepend the event, creating the handler arg list + data = data != null ? jQuery.makeArray( data ) : []; + data.unshift( event ); + + // Allow special events to draw outside the lines + special = jQuery.event.special[ type ] || {}; + if ( special.trigger && special.trigger.apply( elem, data ) === false ) { + return; + } + + // Determine event propagation path in advance, per W3C events spec (#9951) + // Bubble up to document, then to window; watch for a global ownerDocument var (#9724) + eventPath = [[ elem, special.bindType || type ]]; + if ( !onlyHandlers && !special.noBubble && !jQuery.isWindow( elem ) ) { + + bubbleType = special.delegateType || type; + cur = rfocusMorph.test( bubbleType + type ) ? elem : elem.parentNode; + old = null; + for ( ; cur; cur = cur.parentNode ) { + eventPath.push([ cur, bubbleType ]); + old = cur; + } + + // Only add window if we got to document (e.g., not plain obj or detached DOM) + if ( old && old === elem.ownerDocument ) { + eventPath.push([ old.defaultView || old.parentWindow || window, bubbleType ]); + } + } + + // Fire handlers on the event path + for ( i = 0; i < eventPath.length && !event.isPropagationStopped(); i++ ) { + + cur = eventPath[i][0]; + event.type = eventPath[i][1]; + + handle = ( jQuery._data( cur, "events" ) || {} )[ event.type ] && jQuery._data( cur, "handle" ); + if ( handle ) { + handle.apply( cur, data ); + } + // Note that this is a bare JS function and not a jQuery handler + handle = ontype && cur[ ontype ]; + if ( handle && jQuery.acceptData( cur ) && handle.apply( cur, data ) === false ) { + event.preventDefault(); + } + } + event.type = type; + + // If nobody prevented the default action, do it now + if ( !onlyHandlers && !event.isDefaultPrevented() ) { + + if ( (!special._default || special._default.apply( elem.ownerDocument, data ) === false) && + !(type === "click" && jQuery.nodeName( elem, "a" )) && jQuery.acceptData( elem ) ) { + + // Call a native DOM method on the target with the same name name as the event. + // Can't use an .isFunction() check here because IE6/7 fails that test. + // Don't do default actions on window, that's where global variables be (#6170) + // IE<9 dies on focus/blur to hidden element (#1486) + if ( ontype && elem[ type ] && ((type !== "focus" && type !== "blur") || event.target.offsetWidth !== 0) && !jQuery.isWindow( elem ) ) { + + // Don't re-trigger an onFOO event when we call its FOO() method + old = elem[ ontype ]; + + if ( old ) { + elem[ ontype ] = null; + } + + // Prevent re-triggering of the same event, since we already bubbled it above + jQuery.event.triggered = type; + elem[ type ](); + jQuery.event.triggered = undefined; + + if ( old ) { + elem[ ontype ] = old; + } + } + } + } + + return event.result; + }, + + dispatch: function( event ) { + + // Make a writable jQuery.Event from the native event object + event = jQuery.event.fix( event || window.event ); + + var handlers = ( (jQuery._data( this, "events" ) || {} )[ event.type ] || []), + delegateCount = handlers.delegateCount, + args = [].slice.call( arguments, 0 ), + run_all = !event.exclusive && !event.namespace, + handlerQueue = [], + i, j, cur, jqcur, ret, selMatch, matched, matches, handleObj, sel, related; + + // Use the fix-ed jQuery.Event rather than the (read-only) native event + args[0] = event; + event.delegateTarget = this; + + // Determine handlers that should run if there are delegated events + // Avoid disabled elements in IE (#6911) and non-left-click bubbling in Firefox (#3861) + if ( delegateCount && !event.target.disabled && !(event.button && event.type === "click") ) { + + // Pregenerate a single jQuery object for reuse with .is() + jqcur = jQuery(this); + jqcur.context = this.ownerDocument || this; + + for ( cur = event.target; cur != this; cur = cur.parentNode || this ) { + selMatch = {}; + matches = []; + jqcur[0] = cur; + for ( i = 0; i < delegateCount; i++ ) { + handleObj = handlers[ i ]; + sel = handleObj.selector; + + if ( selMatch[ sel ] === undefined ) { + selMatch[ sel ] = ( + handleObj.quick ? quickIs( cur, handleObj.quick ) : jqcur.is( sel ) + ); + } + if ( selMatch[ sel ] ) { + matches.push( handleObj ); + } + } + if ( matches.length ) { + handlerQueue.push({ elem: cur, matches: matches }); + } + } + } + + // Add the remaining (directly-bound) handlers + if ( handlers.length > delegateCount ) { + handlerQueue.push({ elem: this, matches: handlers.slice( delegateCount ) }); + } + + // Run delegates first; they may want to stop propagation beneath us + for ( i = 0; i < handlerQueue.length && !event.isPropagationStopped(); i++ ) { + matched = handlerQueue[ i ]; + event.currentTarget = matched.elem; + + for ( j = 0; j < matched.matches.length && !event.isImmediatePropagationStopped(); j++ ) { + handleObj = matched.matches[ j ]; + + // Triggered event must either 1) be non-exclusive and have no namespace, or + // 2) have namespace(s) a subset or equal to those in the bound event (both can have no namespace). + if ( run_all || (!event.namespace && !handleObj.namespace) || event.namespace_re && event.namespace_re.test( handleObj.namespace ) ) { + + event.data = handleObj.data; + event.handleObj = handleObj; + + ret = ( (jQuery.event.special[ handleObj.origType ] || {}).handle || handleObj.handler ) + .apply( matched.elem, args ); + + if ( ret !== undefined ) { + event.result = ret; + if ( ret === false ) { + event.preventDefault(); + event.stopPropagation(); + } + } + } + } + } + + return event.result; + }, + + // Includes some event props shared by KeyEvent and MouseEvent + // *** attrChange attrName relatedNode srcElement are not normalized, non-W3C, deprecated, will be removed in 1.8 *** + props: "attrChange attrName relatedNode srcElement altKey bubbles cancelable ctrlKey currentTarget eventPhase metaKey relatedTarget shiftKey target timeStamp view which".split(" "), + + fixHooks: {}, + + keyHooks: { + props: "char charCode key keyCode".split(" "), + filter: function( event, original ) { + + // Add which for key events + if ( event.which == null ) { + event.which = original.charCode != null ? original.charCode : original.keyCode; + } + + return event; + } + }, + + mouseHooks: { + props: "button buttons clientX clientY fromElement offsetX offsetY pageX pageY screenX screenY toElement".split(" "), + filter: function( event, original ) { + var eventDoc, doc, body, + button = original.button, + fromElement = original.fromElement; + + // Calculate pageX/Y if missing and clientX/Y available + if ( event.pageX == null && original.clientX != null ) { + eventDoc = event.target.ownerDocument || document; + doc = eventDoc.documentElement; + body = eventDoc.body; + + event.pageX = original.clientX + ( doc && doc.scrollLeft || body && body.scrollLeft || 0 ) - ( doc && doc.clientLeft || body && body.clientLeft || 0 ); + event.pageY = original.clientY + ( doc && doc.scrollTop || body && body.scrollTop || 0 ) - ( doc && doc.clientTop || body && body.clientTop || 0 ); + } + + // Add relatedTarget, if necessary + if ( !event.relatedTarget && fromElement ) { + event.relatedTarget = fromElement === event.target ? original.toElement : fromElement; + } + + // Add which for click: 1 === left; 2 === middle; 3 === right + // Note: button is not normalized, so don't use it + if ( !event.which && button !== undefined ) { + event.which = ( button & 1 ? 1 : ( button & 2 ? 3 : ( button & 4 ? 2 : 0 ) ) ); + } + + return event; + } + }, + + fix: function( event ) { + if ( event[ jQuery.expando ] ) { + return event; + } + + // Create a writable copy of the event object and normalize some properties + var i, prop, + originalEvent = event, + fixHook = jQuery.event.fixHooks[ event.type ] || {}, + copy = fixHook.props ? this.props.concat( fixHook.props ) : this.props; + + event = jQuery.Event( originalEvent ); + + for ( i = copy.length; i; ) { + prop = copy[ --i ]; + event[ prop ] = originalEvent[ prop ]; + } + + // Fix target property, if necessary (#1925, IE 6/7/8 & Safari2) + if ( !event.target ) { + event.target = originalEvent.srcElement || document; + } + + // Target should not be a text node (#504, Safari) + if ( event.target.nodeType === 3 ) { + event.target = event.target.parentNode; + } + + // For mouse/key events; add metaKey if it's not there (#3368, IE6/7/8) + if ( event.metaKey === undefined ) { + event.metaKey = event.ctrlKey; + } + + return fixHook.filter? fixHook.filter( event, originalEvent ) : event; + }, + + special: { + ready: { + // Make sure the ready event is setup + setup: jQuery.bindReady + }, + + load: { + // Prevent triggered image.load events from bubbling to window.load + noBubble: true + }, + + focus: { + delegateType: "focusin" + }, + blur: { + delegateType: "focusout" + }, + + beforeunload: { + setup: function( data, namespaces, eventHandle ) { + // We only want to do this special case on windows + if ( jQuery.isWindow( this ) ) { + this.onbeforeunload = eventHandle; + } + }, + + teardown: function( namespaces, eventHandle ) { + if ( this.onbeforeunload === eventHandle ) { + this.onbeforeunload = null; + } + } + } + }, + + simulate: function( type, elem, event, bubble ) { + // Piggyback on a donor event to simulate a different one. + // Fake originalEvent to avoid donor's stopPropagation, but if the + // simulated event prevents default then we do the same on the donor. + var e = jQuery.extend( + new jQuery.Event(), + event, + { type: type, + isSimulated: true, + originalEvent: {} + } + ); + if ( bubble ) { + jQuery.event.trigger( e, null, elem ); + } else { + jQuery.event.dispatch.call( elem, e ); + } + if ( e.isDefaultPrevented() ) { + event.preventDefault(); + } + } +}; + +// Some plugins are using, but it's undocumented/deprecated and will be removed. +// The 1.7 special event interface should provide all the hooks needed now. +jQuery.event.handle = jQuery.event.dispatch; + +jQuery.removeEvent = document.removeEventListener ? + function( elem, type, handle ) { + if ( elem.removeEventListener ) { + elem.removeEventListener( type, handle, false ); + } + } : + function( elem, type, handle ) { + if ( elem.detachEvent ) { + elem.detachEvent( "on" + type, handle ); + } + }; + +jQuery.Event = function( src, props ) { + // Allow instantiation without the 'new' keyword + if ( !(this instanceof jQuery.Event) ) { + return new jQuery.Event( src, props ); + } + + // Event object + if ( src && src.type ) { + this.originalEvent = src; + this.type = src.type; + + // Events bubbling up the document may have been marked as prevented + // by a handler lower down the tree; reflect the correct value. + this.isDefaultPrevented = ( src.defaultPrevented || src.returnValue === false || + src.getPreventDefault && src.getPreventDefault() ) ? returnTrue : returnFalse; + + // Event type + } else { + this.type = src; + } + + // Put explicitly provided properties onto the event object + if ( props ) { + jQuery.extend( this, props ); + } + + // Create a timestamp if incoming event doesn't have one + this.timeStamp = src && src.timeStamp || jQuery.now(); + + // Mark it as fixed + this[ jQuery.expando ] = true; +}; + +function returnFalse() { + return false; +} +function returnTrue() { + return true; +} + +// jQuery.Event is based on DOM3 Events as specified by the ECMAScript Language Binding +// http://www.w3.org/TR/2003/WD-DOM-Level-3-Events-20030331/ecma-script-binding.html +jQuery.Event.prototype = { + preventDefault: function() { + this.isDefaultPrevented = returnTrue; + + var e = this.originalEvent; + if ( !e ) { + return; + } + + // if preventDefault exists run it on the original event + if ( e.preventDefault ) { + e.preventDefault(); + + // otherwise set the returnValue property of the original event to false (IE) + } else { + e.returnValue = false; + } + }, + stopPropagation: function() { + this.isPropagationStopped = returnTrue; + + var e = this.originalEvent; + if ( !e ) { + return; + } + // if stopPropagation exists run it on the original event + if ( e.stopPropagation ) { + e.stopPropagation(); + } + // otherwise set the cancelBubble property of the original event to true (IE) + e.cancelBubble = true; + }, + stopImmediatePropagation: function() { + this.isImmediatePropagationStopped = returnTrue; + this.stopPropagation(); + }, + isDefaultPrevented: returnFalse, + isPropagationStopped: returnFalse, + isImmediatePropagationStopped: returnFalse +}; + +// Create mouseenter/leave events using mouseover/out and event-time checks +jQuery.each({ + mouseenter: "mouseover", + mouseleave: "mouseout" +}, function( orig, fix ) { + jQuery.event.special[ orig ] = { + delegateType: fix, + bindType: fix, + + handle: function( event ) { + var target = this, + related = event.relatedTarget, + handleObj = event.handleObj, + selector = handleObj.selector, + ret; + + // For mousenter/leave call the handler if related is outside the target. + // NB: No relatedTarget if the mouse left/entered the browser window + if ( !related || (related !== target && !jQuery.contains( target, related )) ) { + event.type = handleObj.origType; + ret = handleObj.handler.apply( this, arguments ); + event.type = fix; + } + return ret; + } + }; +}); + +// IE submit delegation +if ( !jQuery.support.submitBubbles ) { + + jQuery.event.special.submit = { + setup: function() { + // Only need this for delegated form submit events + if ( jQuery.nodeName( this, "form" ) ) { + return false; + } + + // Lazy-add a submit handler when a descendant form may potentially be submitted + jQuery.event.add( this, "click._submit keypress._submit", function( e ) { + // Node name check avoids a VML-related crash in IE (#9807) + var elem = e.target, + form = jQuery.nodeName( elem, "input" ) || jQuery.nodeName( elem, "button" ) ? elem.form : undefined; + if ( form && !form._submit_attached ) { + jQuery.event.add( form, "submit._submit", function( event ) { + // If form was submitted by the user, bubble the event up the tree + if ( this.parentNode && !event.isTrigger ) { + jQuery.event.simulate( "submit", this.parentNode, event, true ); + } + }); + form._submit_attached = true; + } + }); + // return undefined since we don't need an event listener + }, + + teardown: function() { + // Only need this for delegated form submit events + if ( jQuery.nodeName( this, "form" ) ) { + return false; + } + + // Remove delegated handlers; cleanData eventually reaps submit handlers attached above + jQuery.event.remove( this, "._submit" ); + } + }; +} + +// IE change delegation and checkbox/radio fix +if ( !jQuery.support.changeBubbles ) { + + jQuery.event.special.change = { + + setup: function() { + + if ( rformElems.test( this.nodeName ) ) { + // IE doesn't fire change on a check/radio until blur; trigger it on click + // after a propertychange. Eat the blur-change in special.change.handle. + // This still fires onchange a second time for check/radio after blur. + if ( this.type === "checkbox" || this.type === "radio" ) { + jQuery.event.add( this, "propertychange._change", function( event ) { + if ( event.originalEvent.propertyName === "checked" ) { + this._just_changed = true; + } + }); + jQuery.event.add( this, "click._change", function( event ) { + if ( this._just_changed && !event.isTrigger ) { + this._just_changed = false; + jQuery.event.simulate( "change", this, event, true ); + } + }); + } + return false; + } + // Delegated event; lazy-add a change handler on descendant inputs + jQuery.event.add( this, "beforeactivate._change", function( e ) { + var elem = e.target; + + if ( rformElems.test( elem.nodeName ) && !elem._change_attached ) { + jQuery.event.add( elem, "change._change", function( event ) { + if ( this.parentNode && !event.isSimulated && !event.isTrigger ) { + jQuery.event.simulate( "change", this.parentNode, event, true ); + } + }); + elem._change_attached = true; + } + }); + }, + + handle: function( event ) { + var elem = event.target; + + // Swallow native change events from checkbox/radio, we already triggered them above + if ( this !== elem || event.isSimulated || event.isTrigger || (elem.type !== "radio" && elem.type !== "checkbox") ) { + return event.handleObj.handler.apply( this, arguments ); + } + }, + + teardown: function() { + jQuery.event.remove( this, "._change" ); + + return rformElems.test( this.nodeName ); + } + }; +} + +// Create "bubbling" focus and blur events +if ( !jQuery.support.focusinBubbles ) { + jQuery.each({ focus: "focusin", blur: "focusout" }, function( orig, fix ) { + + // Attach a single capturing handler while someone wants focusin/focusout + var attaches = 0, + handler = function( event ) { + jQuery.event.simulate( fix, event.target, jQuery.event.fix( event ), true ); + }; + + jQuery.event.special[ fix ] = { + setup: function() { + if ( attaches++ === 0 ) { + document.addEventListener( orig, handler, true ); + } + }, + teardown: function() { + if ( --attaches === 0 ) { + document.removeEventListener( orig, handler, true ); + } + } + }; + }); +} + +jQuery.fn.extend({ + + on: function( types, selector, data, fn, /*INTERNAL*/ one ) { + var origFn, type; + + // Types can be a map of types/handlers + if ( typeof types === "object" ) { + // ( types-Object, selector, data ) + if ( typeof selector !== "string" ) { + // ( types-Object, data ) + data = selector; + selector = undefined; + } + for ( type in types ) { + this.on( type, selector, data, types[ type ], one ); + } + return this; + } + + if ( data == null && fn == null ) { + // ( types, fn ) + fn = selector; + data = selector = undefined; + } else if ( fn == null ) { + if ( typeof selector === "string" ) { + // ( types, selector, fn ) + fn = data; + data = undefined; + } else { + // ( types, data, fn ) + fn = data; + data = selector; + selector = undefined; + } + } + if ( fn === false ) { + fn = returnFalse; + } else if ( !fn ) { + return this; + } + + if ( one === 1 ) { + origFn = fn; + fn = function( event ) { + // Can use an empty set, since event contains the info + jQuery().off( event ); + return origFn.apply( this, arguments ); + }; + // Use same guid so caller can remove using origFn + fn.guid = origFn.guid || ( origFn.guid = jQuery.guid++ ); + } + return this.each( function() { + jQuery.event.add( this, types, fn, data, selector ); + }); + }, + one: function( types, selector, data, fn ) { + return this.on.call( this, types, selector, data, fn, 1 ); + }, + off: function( types, selector, fn ) { + if ( types && types.preventDefault && types.handleObj ) { + // ( event ) dispatched jQuery.Event + var handleObj = types.handleObj; + jQuery( types.delegateTarget ).off( + handleObj.namespace? handleObj.type + "." + handleObj.namespace : handleObj.type, + handleObj.selector, + handleObj.handler + ); + return this; + } + if ( typeof types === "object" ) { + // ( types-object [, selector] ) + for ( var type in types ) { + this.off( type, selector, types[ type ] ); + } + return this; + } + if ( selector === false || typeof selector === "function" ) { + // ( types [, fn] ) + fn = selector; + selector = undefined; + } + if ( fn === false ) { + fn = returnFalse; + } + return this.each(function() { + jQuery.event.remove( this, types, fn, selector ); + }); + }, + + bind: function( types, data, fn ) { + return this.on( types, null, data, fn ); + }, + unbind: function( types, fn ) { + return this.off( types, null, fn ); + }, + + live: function( types, data, fn ) { + jQuery( this.context ).on( types, this.selector, data, fn ); + return this; + }, + die: function( types, fn ) { + jQuery( this.context ).off( types, this.selector || "**", fn ); + return this; + }, + + delegate: function( selector, types, data, fn ) { + return this.on( types, selector, data, fn ); + }, + undelegate: function( selector, types, fn ) { + // ( namespace ) or ( selector, types [, fn] ) + return arguments.length == 1? this.off( selector, "**" ) : this.off( types, selector, fn ); + }, + + trigger: function( type, data ) { + return this.each(function() { + jQuery.event.trigger( type, data, this ); + }); + }, + triggerHandler: function( type, data ) { + if ( this[0] ) { + return jQuery.event.trigger( type, data, this[0], true ); + } + }, + + toggle: function( fn ) { + // Save reference to arguments for access in closure + var args = arguments, + guid = fn.guid || jQuery.guid++, + i = 0, + toggler = function( event ) { + // Figure out which function to execute + var lastToggle = ( jQuery._data( this, "lastToggle" + fn.guid ) || 0 ) % i; + jQuery._data( this, "lastToggle" + fn.guid, lastToggle + 1 ); + + // Make sure that clicks stop + event.preventDefault(); + + // and execute the function + return args[ lastToggle ].apply( this, arguments ) || false; + }; + + // link all the functions, so any of them can unbind this click handler + toggler.guid = guid; + while ( i < args.length ) { + args[ i++ ].guid = guid; + } + + return this.click( toggler ); + }, + + hover: function( fnOver, fnOut ) { + return this.mouseenter( fnOver ).mouseleave( fnOut || fnOver ); + } +}); + +jQuery.each( ("blur focus focusin focusout load resize scroll unload click dblclick " + + "mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave " + + "change select submit keydown keypress keyup error contextmenu").split(" "), function( i, name ) { + + // Handle event binding + jQuery.fn[ name ] = function( data, fn ) { + if ( fn == null ) { + fn = data; + data = null; + } + + return arguments.length > 0 ? + this.on( name, null, data, fn ) : + this.trigger( name ); + }; + + if ( jQuery.attrFn ) { + jQuery.attrFn[ name ] = true; + } + + if ( rkeyEvent.test( name ) ) { + jQuery.event.fixHooks[ name ] = jQuery.event.keyHooks; + } + + if ( rmouseEvent.test( name ) ) { + jQuery.event.fixHooks[ name ] = jQuery.event.mouseHooks; + } +}); + + + +/*! + * Sizzle CSS Selector Engine + * Copyright 2016, The Dojo Foundation + * Released under the MIT, BSD, and GPL Licenses. + * More information: http://sizzlejs.com/ + */ +(function(){ + +var chunker = /((?:\((?:\([^()]+\)|[^()]+)+\)|\[(?:\[[^\[\]]*\]|['"][^'"]*['"]|[^\[\]'"]+)+\]|\\.|[^ >+~,(\[\\]+)+|[>+~])(\s*,\s*)?((?:.|\r|\n)*)/g, + expando = "sizcache" + (Math.random() + '').replace('.', ''), + done = 0, + toString = Object.prototype.toString, + hasDuplicate = false, + baseHasDuplicate = true, + rBackslash = /\\/g, + rReturn = /\r\n/g, + rNonWord = /\W/; + +// Here we check if the JavaScript engine is using some sort of +// optimization where it does not always call our comparision +// function. If that is the case, discard the hasDuplicate value. +// Thus far that includes Google Chrome. +[0, 0].sort(function() { + baseHasDuplicate = false; + return 0; +}); + +var Sizzle = function( selector, context, results, seed ) { + results = results || []; + context = context || document; + + var origContext = context; + + if ( context.nodeType !== 1 && context.nodeType !== 9 ) { + return []; + } + + if ( !selector || typeof selector !== "string" ) { + return results; + } + + var m, set, checkSet, extra, ret, cur, pop, i, + prune = true, + contextXML = Sizzle.isXML( context ), + parts = [], + soFar = selector; + + // Reset the position of the chunker regexp (start from head) + do { + chunker.exec( "" ); + m = chunker.exec( soFar ); + + if ( m ) { + soFar = m[3]; + + parts.push( m[1] ); + + if ( m[2] ) { + extra = m[3]; + break; + } + } + } while ( m ); + + if ( parts.length > 1 && origPOS.exec( selector ) ) { + + if ( parts.length === 2 && Expr.relative[ parts[0] ] ) { + set = posProcess( parts[0] + parts[1], context, seed ); + + } else { + set = Expr.relative[ parts[0] ] ? + [ context ] : + Sizzle( parts.shift(), context ); + + while ( parts.length ) { + selector = parts.shift(); + + if ( Expr.relative[ selector ] ) { + selector += parts.shift(); + } + + set = posProcess( selector, set, seed ); + } + } + + } else { + // Take a shortcut and set the context if the root selector is an ID + // (but not if it'll be faster if the inner selector is an ID) + if ( !seed && parts.length > 1 && context.nodeType === 9 && !contextXML && + Expr.match.ID.test(parts[0]) && !Expr.match.ID.test(parts[parts.length - 1]) ) { + + ret = Sizzle.find( parts.shift(), context, contextXML ); + context = ret.expr ? + Sizzle.filter( ret.expr, ret.set )[0] : + ret.set[0]; + } + + if ( context ) { + ret = seed ? + { expr: parts.pop(), set: makeArray(seed) } : + Sizzle.find( parts.pop(), parts.length === 1 && (parts[0] === "~" || parts[0] === "+") && context.parentNode ? context.parentNode : context, contextXML ); + + set = ret.expr ? + Sizzle.filter( ret.expr, ret.set ) : + ret.set; + + if ( parts.length > 0 ) { + checkSet = makeArray( set ); + + } else { + prune = false; + } + + while ( parts.length ) { + cur = parts.pop(); + pop = cur; + + if ( !Expr.relative[ cur ] ) { + cur = ""; + } else { + pop = parts.pop(); + } + + if ( pop == null ) { + pop = context; + } + + Expr.relative[ cur ]( checkSet, pop, contextXML ); + } + + } else { + checkSet = parts = []; + } + } + + if ( !checkSet ) { + checkSet = set; + } + + if ( !checkSet ) { + Sizzle.error( cur || selector ); + } + + if ( toString.call(checkSet) === "[object Array]" ) { + if ( !prune ) { + results.push.apply( results, checkSet ); + + } else if ( context && context.nodeType === 1 ) { + for ( i = 0; checkSet[i] != null; i++ ) { + if ( checkSet[i] && (checkSet[i] === true || checkSet[i].nodeType === 1 && Sizzle.contains(context, checkSet[i])) ) { + results.push( set[i] ); + } + } + + } else { + for ( i = 0; checkSet[i] != null; i++ ) { + if ( checkSet[i] && checkSet[i].nodeType === 1 ) { + results.push( set[i] ); + } + } + } + + } else { + makeArray( checkSet, results ); + } + + if ( extra ) { + Sizzle( extra, origContext, results, seed ); + Sizzle.uniqueSort( results ); + } + + return results; +}; + +Sizzle.uniqueSort = function( results ) { + if ( sortOrder ) { + hasDuplicate = baseHasDuplicate; + results.sort( sortOrder ); + + if ( hasDuplicate ) { + for ( var i = 1; i < results.length; i++ ) { + if ( results[i] === results[ i - 1 ] ) { + results.splice( i--, 1 ); + } + } + } + } + + return results; +}; + +Sizzle.matches = function( expr, set ) { + return Sizzle( expr, null, null, set ); +}; + +Sizzle.matchesSelector = function( node, expr ) { + return Sizzle( expr, null, null, [node] ).length > 0; +}; + +Sizzle.find = function( expr, context, isXML ) { + var set, i, len, match, type, left; + + if ( !expr ) { + return []; + } + + for ( i = 0, len = Expr.order.length; i < len; i++ ) { + type = Expr.order[i]; + + if ( (match = Expr.leftMatch[ type ].exec( expr )) ) { + left = match[1]; + match.splice( 1, 1 ); + + if ( left.substr( left.length - 1 ) !== "\\" ) { + match[1] = (match[1] || "").replace( rBackslash, "" ); + set = Expr.find[ type ]( match, context, isXML ); + + if ( set != null ) { + expr = expr.replace( Expr.match[ type ], "" ); + break; + } + } + } + } + + if ( !set ) { + set = typeof context.getElementsByTagName !== "undefined" ? + context.getElementsByTagName( "*" ) : + []; + } + + return { set: set, expr: expr }; +}; + +Sizzle.filter = function( expr, set, inplace, not ) { + var match, anyFound, + type, found, item, filter, left, + i, pass, + old = expr, + result = [], + curLoop = set, + isXMLFilter = set && set[0] && Sizzle.isXML( set[0] ); + + while ( expr && set.length ) { + for ( type in Expr.filter ) { + if ( (match = Expr.leftMatch[ type ].exec( expr )) != null && match[2] ) { + filter = Expr.filter[ type ]; + left = match[1]; + + anyFound = false; + + match.splice(1,1); + + if ( left.substr( left.length - 1 ) === "\\" ) { + continue; + } + + if ( curLoop === result ) { + result = []; + } + + if ( Expr.preFilter[ type ] ) { + match = Expr.preFilter[ type ]( match, curLoop, inplace, result, not, isXMLFilter ); + + if ( !match ) { + anyFound = found = true; + + } else if ( match === true ) { + continue; + } + } + + if ( match ) { + for ( i = 0; (item = curLoop[i]) != null; i++ ) { + if ( item ) { + found = filter( item, match, i, curLoop ); + pass = not ^ found; + + if ( inplace && found != null ) { + if ( pass ) { + anyFound = true; + + } else { + curLoop[i] = false; + } + + } else if ( pass ) { + result.push( item ); + anyFound = true; + } + } + } + } + + if ( found !== undefined ) { + if ( !inplace ) { + curLoop = result; + } + + expr = expr.replace( Expr.match[ type ], "" ); + + if ( !anyFound ) { + return []; + } + + break; + } + } + } + + // Improper expression + if ( expr === old ) { + if ( anyFound == null ) { + Sizzle.error( expr ); + + } else { + break; + } + } + + old = expr; + } + + return curLoop; +}; + +Sizzle.error = function( msg ) { + throw new Error( "Syntax error, unrecognized expression: " + msg ); +}; + +/** + * Utility function for retreiving the text value of an array of DOM nodes + * @param {Array|Element} elem + */ +var getText = Sizzle.getText = function( elem ) { + var i, node, + nodeType = elem.nodeType, + ret = ""; + + if ( nodeType ) { + if ( nodeType === 1 || nodeType === 9 ) { + // Use textContent || innerText for elements + if ( typeof elem.textContent === 'string' ) { + return elem.textContent; + } else if ( typeof elem.innerText === 'string' ) { + // Replace IE's carriage returns + return elem.innerText.replace( rReturn, '' ); + } else { + // Traverse it's children + for ( elem = elem.firstChild; elem; elem = elem.nextSibling) { + ret += getText( elem ); + } + } + } else if ( nodeType === 3 || nodeType === 4 ) { + return elem.nodeValue; + } + } else { + + // If no nodeType, this is expected to be an array + for ( i = 0; (node = elem[i]); i++ ) { + // Do not traverse comment nodes + if ( node.nodeType !== 8 ) { + ret += getText( node ); + } + } + } + return ret; +}; + +var Expr = Sizzle.selectors = { + order: [ "ID", "NAME", "TAG" ], + + match: { + ID: /#((?:[\w\u00c0-\uFFFF\-]|\\.)+)/, + CLASS: /\.((?:[\w\u00c0-\uFFFF\-]|\\.)+)/, + NAME: /\[name=['"]*((?:[\w\u00c0-\uFFFF\-]|\\.)+)['"]*\]/, + ATTR: /\[\s*((?:[\w\u00c0-\uFFFF\-]|\\.)+)\s*(?:(\S?=)\s*(?:(['"])(.*?)\3|(#?(?:[\w\u00c0-\uFFFF\-]|\\.)*)|)|)\s*\]/, + TAG: /^((?:[\w\u00c0-\uFFFF\*\-]|\\.)+)/, + CHILD: /:(only|nth|last|first)-child(?:\(\s*(even|odd|(?:[+\-]?\d+|(?:[+\-]?\d*)?n\s*(?:[+\-]\s*\d+)?))\s*\))?/, + POS: /:(nth|eq|gt|lt|first|last|even|odd)(?:\((\d*)\))?(?=[^\-]|$)/, + PSEUDO: /:((?:[\w\u00c0-\uFFFF\-]|\\.)+)(?:\((['"]?)((?:\([^\)]+\)|[^\(\)]*)+)\2\))?/ + }, + + leftMatch: {}, + + attrMap: { + "class": "className", + "for": "htmlFor" + }, + + attrHandle: { + href: function( elem ) { + return elem.getAttribute( "href" ); + }, + type: function( elem ) { + return elem.getAttribute( "type" ); + } + }, + + relative: { + "+": function(checkSet, part){ + var isPartStr = typeof part === "string", + isTag = isPartStr && !rNonWord.test( part ), + isPartStrNotTag = isPartStr && !isTag; + + if ( isTag ) { + part = part.toLowerCase(); + } + + for ( var i = 0, l = checkSet.length, elem; i < l; i++ ) { + if ( (elem = checkSet[i]) ) { + while ( (elem = elem.previousSibling) && elem.nodeType !== 1 ) {} + + checkSet[i] = isPartStrNotTag || elem && elem.nodeName.toLowerCase() === part ? + elem || false : + elem === part; + } + } + + if ( isPartStrNotTag ) { + Sizzle.filter( part, checkSet, true ); + } + }, + + ">": function( checkSet, part ) { + var elem, + isPartStr = typeof part === "string", + i = 0, + l = checkSet.length; + + if ( isPartStr && !rNonWord.test( part ) ) { + part = part.toLowerCase(); + + for ( ; i < l; i++ ) { + elem = checkSet[i]; + + if ( elem ) { + var parent = elem.parentNode; + checkSet[i] = parent.nodeName.toLowerCase() === part ? parent : false; + } + } + + } else { + for ( ; i < l; i++ ) { + elem = checkSet[i]; + + if ( elem ) { + checkSet[i] = isPartStr ? + elem.parentNode : + elem.parentNode === part; + } + } + + if ( isPartStr ) { + Sizzle.filter( part, checkSet, true ); + } + } + }, + + "": function(checkSet, part, isXML){ + var nodeCheck, + doneName = done++, + checkFn = dirCheck; + + if ( typeof part === "string" && !rNonWord.test( part ) ) { + part = part.toLowerCase(); + nodeCheck = part; + checkFn = dirNodeCheck; + } + + checkFn( "parentNode", part, doneName, checkSet, nodeCheck, isXML ); + }, + + "~": function( checkSet, part, isXML ) { + var nodeCheck, + doneName = done++, + checkFn = dirCheck; + + if ( typeof part === "string" && !rNonWord.test( part ) ) { + part = part.toLowerCase(); + nodeCheck = part; + checkFn = dirNodeCheck; + } + + checkFn( "previousSibling", part, doneName, checkSet, nodeCheck, isXML ); + } + }, + + find: { + ID: function( match, context, isXML ) { + if ( typeof context.getElementById !== "undefined" && !isXML ) { + var m = context.getElementById(match[1]); + // Check parentNode to catch when Blackberry 4.6 returns + // nodes that are no longer in the document #6963 + return m && m.parentNode ? [m] : []; + } + }, + + NAME: function( match, context ) { + if ( typeof context.getElementsByName !== "undefined" ) { + var ret = [], + results = context.getElementsByName( match[1] ); + + for ( var i = 0, l = results.length; i < l; i++ ) { + if ( results[i].getAttribute("name") === match[1] ) { + ret.push( results[i] ); + } + } + + return ret.length === 0 ? null : ret; + } + }, + + TAG: function( match, context ) { + if ( typeof context.getElementsByTagName !== "undefined" ) { + return context.getElementsByTagName( match[1] ); + } + } + }, + preFilter: { + CLASS: function( match, curLoop, inplace, result, not, isXML ) { + match = " " + match[1].replace( rBackslash, "" ) + " "; + + if ( isXML ) { + return match; + } + + for ( var i = 0, elem; (elem = curLoop[i]) != null; i++ ) { + if ( elem ) { + if ( not ^ (elem.className && (" " + elem.className + " ").replace(/[\t\n\r]/g, " ").indexOf(match) >= 0) ) { + if ( !inplace ) { + result.push( elem ); + } + + } else if ( inplace ) { + curLoop[i] = false; + } + } + } + + return false; + }, + + ID: function( match ) { + return match[1].replace( rBackslash, "" ); + }, + + TAG: function( match, curLoop ) { + return match[1].replace( rBackslash, "" ).toLowerCase(); + }, + + CHILD: function( match ) { + if ( match[1] === "nth" ) { + if ( !match[2] ) { + Sizzle.error( match[0] ); + } + + match[2] = match[2].replace(/^\+|\s*/g, ''); + + // parse equations like 'even', 'odd', '5', '2n', '3n+2', '4n-1', '-n+6' + var test = /(-?)(\d*)(?:n([+\-]?\d*))?/.exec( + match[2] === "even" && "2n" || match[2] === "odd" && "2n+1" || + !/\D/.test( match[2] ) && "0n+" + match[2] || match[2]); + + // calculate the numbers (first)n+(last) including if they are negative + match[2] = (test[1] + (test[2] || 1)) - 0; + match[3] = test[3] - 0; + } + else if ( match[2] ) { + Sizzle.error( match[0] ); + } + + // TODO: Move to normal caching system + match[0] = done++; + + return match; + }, + + ATTR: function( match, curLoop, inplace, result, not, isXML ) { + var name = match[1] = match[1].replace( rBackslash, "" ); + + if ( !isXML && Expr.attrMap[name] ) { + match[1] = Expr.attrMap[name]; + } + + // Handle if an un-quoted value was used + match[4] = ( match[4] || match[5] || "" ).replace( rBackslash, "" ); + + if ( match[2] === "~=" ) { + match[4] = " " + match[4] + " "; + } + + return match; + }, + + PSEUDO: function( match, curLoop, inplace, result, not ) { + if ( match[1] === "not" ) { + // If we're dealing with a complex expression, or a simple one + if ( ( chunker.exec(match[3]) || "" ).length > 1 || /^\w/.test(match[3]) ) { + match[3] = Sizzle(match[3], null, null, curLoop); + + } else { + var ret = Sizzle.filter(match[3], curLoop, inplace, true ^ not); + + if ( !inplace ) { + result.push.apply( result, ret ); + } + + return false; + } + + } else if ( Expr.match.POS.test( match[0] ) || Expr.match.CHILD.test( match[0] ) ) { + return true; + } + + return match; + }, + + POS: function( match ) { + match.unshift( true ); + + return match; + } + }, + + filters: { + enabled: function( elem ) { + return elem.disabled === false && elem.type !== "hidden"; + }, + + disabled: function( elem ) { + return elem.disabled === true; + }, + + checked: function( elem ) { + return elem.checked === true; + }, + + selected: function( elem ) { + // Accessing this property makes selected-by-default + // options in Safari work properly + if ( elem.parentNode ) { + elem.parentNode.selectedIndex; + } + + return elem.selected === true; + }, + + parent: function( elem ) { + return !!elem.firstChild; + }, + + empty: function( elem ) { + return !elem.firstChild; + }, + + has: function( elem, i, match ) { + return !!Sizzle( match[3], elem ).length; + }, + + header: function( elem ) { + return (/h\d/i).test( elem.nodeName ); + }, + + text: function( elem ) { + var attr = elem.getAttribute( "type" ), type = elem.type; + // IE6 and 7 will map elem.type to 'text' for new HTML5 types (search, etc) + // use getAttribute instead to test this case + return elem.nodeName.toLowerCase() === "input" && "text" === type && ( attr === type || attr === null ); + }, + + radio: function( elem ) { + return elem.nodeName.toLowerCase() === "input" && "radio" === elem.type; + }, + + checkbox: function( elem ) { + return elem.nodeName.toLowerCase() === "input" && "checkbox" === elem.type; + }, + + file: function( elem ) { + return elem.nodeName.toLowerCase() === "input" && "file" === elem.type; + }, + + password: function( elem ) { + return elem.nodeName.toLowerCase() === "input" && "password" === elem.type; + }, + + submit: function( elem ) { + var name = elem.nodeName.toLowerCase(); + return (name === "input" || name === "button") && "submit" === elem.type; + }, + + image: function( elem ) { + return elem.nodeName.toLowerCase() === "input" && "image" === elem.type; + }, + + reset: function( elem ) { + var name = elem.nodeName.toLowerCase(); + return (name === "input" || name === "button") && "reset" === elem.type; + }, + + button: function( elem ) { + var name = elem.nodeName.toLowerCase(); + return name === "input" && "button" === elem.type || name === "button"; + }, + + input: function( elem ) { + return (/input|select|textarea|button/i).test( elem.nodeName ); + }, + + focus: function( elem ) { + return elem === elem.ownerDocument.activeElement; + } + }, + setFilters: { + first: function( elem, i ) { + return i === 0; + }, + + last: function( elem, i, match, array ) { + return i === array.length - 1; + }, + + even: function( elem, i ) { + return i % 2 === 0; + }, + + odd: function( elem, i ) { + return i % 2 === 1; + }, + + lt: function( elem, i, match ) { + return i < match[3] - 0; + }, + + gt: function( elem, i, match ) { + return i > match[3] - 0; + }, + + nth: function( elem, i, match ) { + return match[3] - 0 === i; + }, + + eq: function( elem, i, match ) { + return match[3] - 0 === i; + } + }, + filter: { + PSEUDO: function( elem, match, i, array ) { + var name = match[1], + filter = Expr.filters[ name ]; + + if ( filter ) { + return filter( elem, i, match, array ); + + } else if ( name === "contains" ) { + return (elem.textContent || elem.innerText || getText([ elem ]) || "").indexOf(match[3]) >= 0; + + } else if ( name === "not" ) { + var not = match[3]; + + for ( var j = 0, l = not.length; j < l; j++ ) { + if ( not[j] === elem ) { + return false; + } + } + + return true; + + } else { + Sizzle.error( name ); + } + }, + + CHILD: function( elem, match ) { + var first, last, + doneName, parent, cache, + count, diff, + type = match[1], + node = elem; + + switch ( type ) { + case "only": + case "first": + while ( (node = node.previousSibling) ) { + if ( node.nodeType === 1 ) { + return false; + } + } + + if ( type === "first" ) { + return true; + } + + node = elem; + + case "last": + while ( (node = node.nextSibling) ) { + if ( node.nodeType === 1 ) { + return false; + } + } + + return true; + + case "nth": + first = match[2]; + last = match[3]; + + if ( first === 1 && last === 0 ) { + return true; + } + + doneName = match[0]; + parent = elem.parentNode; + + if ( parent && (parent[ expando ] !== doneName || !elem.nodeIndex) ) { + count = 0; + + for ( node = parent.firstChild; node; node = node.nextSibling ) { + if ( node.nodeType === 1 ) { + node.nodeIndex = ++count; + } + } + + parent[ expando ] = doneName; + } + + diff = elem.nodeIndex - last; + + if ( first === 0 ) { + return diff === 0; + + } else { + return ( diff % first === 0 && diff / first >= 0 ); + } + } + }, + + ID: function( elem, match ) { + return elem.nodeType === 1 && elem.getAttribute("id") === match; + }, + + TAG: function( elem, match ) { + return (match === "*" && elem.nodeType === 1) || !!elem.nodeName && elem.nodeName.toLowerCase() === match; + }, + + CLASS: function( elem, match ) { + return (" " + (elem.className || elem.getAttribute("class")) + " ") + .indexOf( match ) > -1; + }, + + ATTR: function( elem, match ) { + var name = match[1], + result = Sizzle.attr ? + Sizzle.attr( elem, name ) : + Expr.attrHandle[ name ] ? + Expr.attrHandle[ name ]( elem ) : + elem[ name ] != null ? + elem[ name ] : + elem.getAttribute( name ), + value = result + "", + type = match[2], + check = match[4]; + + return result == null ? + type === "!=" : + !type && Sizzle.attr ? + result != null : + type === "=" ? + value === check : + type === "*=" ? + value.indexOf(check) >= 0 : + type === "~=" ? + (" " + value + " ").indexOf(check) >= 0 : + !check ? + value && result !== false : + type === "!=" ? + value !== check : + type === "^=" ? + value.indexOf(check) === 0 : + type === "$=" ? + value.substr(value.length - check.length) === check : + type === "|=" ? + value === check || value.substr(0, check.length + 1) === check + "-" : + false; + }, + + POS: function( elem, match, i, array ) { + var name = match[2], + filter = Expr.setFilters[ name ]; + + if ( filter ) { + return filter( elem, i, match, array ); + } + } + } +}; + +var origPOS = Expr.match.POS, + fescape = function(all, num){ + return "\\" + (num - 0 + 1); + }; + +for ( var type in Expr.match ) { + Expr.match[ type ] = new RegExp( Expr.match[ type ].source + (/(?![^\[]*\])(?![^\(]*\))/.source) ); + Expr.leftMatch[ type ] = new RegExp( /(^(?:.|\r|\n)*?)/.source + Expr.match[ type ].source.replace(/\\(\d+)/g, fescape) ); +} + +var makeArray = function( array, results ) { + array = Array.prototype.slice.call( array, 0 ); + + if ( results ) { + results.push.apply( results, array ); + return results; + } + + return array; +}; + +// Perform a simple check to determine if the browser is capable of +// converting a NodeList to an array using builtin methods. +// Also verifies that the returned array holds DOM nodes +// (which is not the case in the Blackberry browser) +try { + Array.prototype.slice.call( document.documentElement.childNodes, 0 )[0].nodeType; + +// Provide a fallback method if it does not work +} catch( e ) { + makeArray = function( array, results ) { + var i = 0, + ret = results || []; + + if ( toString.call(array) === "[object Array]" ) { + Array.prototype.push.apply( ret, array ); + + } else { + if ( typeof array.length === "number" ) { + for ( var l = array.length; i < l; i++ ) { + ret.push( array[i] ); + } + + } else { + for ( ; array[i]; i++ ) { + ret.push( array[i] ); + } + } + } + + return ret; + }; +} + +var sortOrder, siblingCheck; + +if ( document.documentElement.compareDocumentPosition ) { + sortOrder = function( a, b ) { + if ( a === b ) { + hasDuplicate = true; + return 0; + } + + if ( !a.compareDocumentPosition || !b.compareDocumentPosition ) { + return a.compareDocumentPosition ? -1 : 1; + } + + return a.compareDocumentPosition(b) & 4 ? -1 : 1; + }; + +} else { + sortOrder = function( a, b ) { + // The nodes are identical, we can exit early + if ( a === b ) { + hasDuplicate = true; + return 0; + + // Fallback to using sourceIndex (in IE) if it's available on both nodes + } else if ( a.sourceIndex && b.sourceIndex ) { + return a.sourceIndex - b.sourceIndex; + } + + var al, bl, + ap = [], + bp = [], + aup = a.parentNode, + bup = b.parentNode, + cur = aup; + + // If the nodes are siblings (or identical) we can do a quick check + if ( aup === bup ) { + return siblingCheck( a, b ); + + // If no parents were found then the nodes are disconnected + } else if ( !aup ) { + return -1; + + } else if ( !bup ) { + return 1; + } + + // Otherwise they're somewhere else in the tree so we need + // to build up a full list of the parentNodes for comparison + while ( cur ) { + ap.unshift( cur ); + cur = cur.parentNode; + } + + cur = bup; + + while ( cur ) { + bp.unshift( cur ); + cur = cur.parentNode; + } + + al = ap.length; + bl = bp.length; + + // Start walking down the tree looking for a discrepancy + for ( var i = 0; i < al && i < bl; i++ ) { + if ( ap[i] !== bp[i] ) { + return siblingCheck( ap[i], bp[i] ); + } + } + + // We ended someplace up the tree so do a sibling check + return i === al ? + siblingCheck( a, bp[i], -1 ) : + siblingCheck( ap[i], b, 1 ); + }; + + siblingCheck = function( a, b, ret ) { + if ( a === b ) { + return ret; + } + + var cur = a.nextSibling; + + while ( cur ) { + if ( cur === b ) { + return -1; + } + + cur = cur.nextSibling; + } + + return 1; + }; +} + +// Check to see if the browser returns elements by name when +// querying by getElementById (and provide a workaround) +(function(){ + // We're going to inject a fake input element with a specified name + var form = document.createElement("div"), + id = "script" + (new Date()).getTime(), + root = document.documentElement; + + form.innerHTML = "<a name='" + id + "'/>"; + + // Inject it into the root element, check its status, and remove it quickly + root.insertBefore( form, root.firstChild ); + + // The workaround has to do additional checks after a getElementById + // Which slows things down for other browsers (hence the branching) + if ( document.getElementById( id ) ) { + Expr.find.ID = function( match, context, isXML ) { + if ( typeof context.getElementById !== "undefined" && !isXML ) { + var m = context.getElementById(match[1]); + + return m ? + m.id === match[1] || typeof m.getAttributeNode !== "undefined" && m.getAttributeNode("id").nodeValue === match[1] ? + [m] : + undefined : + []; + } + }; + + Expr.filter.ID = function( elem, match ) { + var node = typeof elem.getAttributeNode !== "undefined" && elem.getAttributeNode("id"); + + return elem.nodeType === 1 && node && node.nodeValue === match; + }; + } + + root.removeChild( form ); + + // release memory in IE + root = form = null; +})(); + +(function(){ + // Check to see if the browser returns only elements + // when doing getElementsByTagName("*") + + // Create a fake element + var div = document.createElement("div"); + div.appendChild( document.createComment("") ); + + // Make sure no comments are found + if ( div.getElementsByTagName("*").length > 0 ) { + Expr.find.TAG = function( match, context ) { + var results = context.getElementsByTagName( match[1] ); + + // Filter out possible comments + if ( match[1] === "*" ) { + var tmp = []; + + for ( var i = 0; results[i]; i++ ) { + if ( results[i].nodeType === 1 ) { + tmp.push( results[i] ); + } + } + + results = tmp; + } + + return results; + }; + } + + // Check to see if an attribute returns normalized href attributes + div.innerHTML = "<a href='#'></a>"; + + if ( div.firstChild && typeof div.firstChild.getAttribute !== "undefined" && + div.firstChild.getAttribute("href") !== "#" ) { + + Expr.attrHandle.href = function( elem ) { + return elem.getAttribute( "href", 2 ); + }; + } + + // release memory in IE + div = null; +})(); + +if ( document.querySelectorAll ) { + (function(){ + var oldSizzle = Sizzle, + div = document.createElement("div"), + id = "__sizzle__"; + + div.innerHTML = "<p class='TEST'></p>"; + + // Safari can't handle uppercase or unicode characters when + // in quirks mode. + if ( div.querySelectorAll && div.querySelectorAll(".TEST").length === 0 ) { + return; + } + + Sizzle = function( query, context, extra, seed ) { + context = context || document; + + // Only use querySelectorAll on non-XML documents + // (ID selectors don't work in non-HTML documents) + if ( !seed && !Sizzle.isXML(context) ) { + // See if we find a selector to speed up + var match = /^(\w+$)|^\.([\w\-]+$)|^#([\w\-]+$)/.exec( query ); + + if ( match && (context.nodeType === 1 || context.nodeType === 9) ) { + // Speed-up: Sizzle("TAG") + if ( match[1] ) { + return makeArray( context.getElementsByTagName( query ), extra ); + + // Speed-up: Sizzle(".CLASS") + } else if ( match[2] && Expr.find.CLASS && context.getElementsByClassName ) { + return makeArray( context.getElementsByClassName( match[2] ), extra ); + } + } + + if ( context.nodeType === 9 ) { + // Speed-up: Sizzle("body") + // The body element only exists once, optimize finding it + if ( query === "body" && context.body ) { + return makeArray( [ context.body ], extra ); + + // Speed-up: Sizzle("#ID") + } else if ( match && match[3] ) { + var elem = context.getElementById( match[3] ); + + // Check parentNode to catch when Blackberry 4.6 returns + // nodes that are no longer in the document #6963 + if ( elem && elem.parentNode ) { + // Handle the case where IE and Opera return items + // by name instead of ID + if ( elem.id === match[3] ) { + return makeArray( [ elem ], extra ); + } + + } else { + return makeArray( [], extra ); + } + } + + try { + return makeArray( context.querySelectorAll(query), extra ); + } catch(qsaError) {} + + // qSA works strangely on Element-rooted queries + // We can work around this by specifying an extra ID on the root + // and working up from there (Thanks to Andrew Dupont for the technique) + // IE 8 doesn't work on object elements + } else if ( context.nodeType === 1 && context.nodeName.toLowerCase() !== "object" ) { + var oldContext = context, + old = context.getAttribute( "id" ), + nid = old || id, + hasParent = context.parentNode, + relativeHierarchySelector = /^\s*[+~]/.test( query ); + + if ( !old ) { + context.setAttribute( "id", nid ); + } else { + nid = nid.replace( /'/g, "\\$&" ); + } + if ( relativeHierarchySelector && hasParent ) { + context = context.parentNode; + } + + try { + if ( !relativeHierarchySelector || hasParent ) { + return makeArray( context.querySelectorAll( "[id='" + nid + "'] " + query ), extra ); + } + + } catch(pseudoError) { + } finally { + if ( !old ) { + oldContext.removeAttribute( "id" ); + } + } + } + } + + return oldSizzle(query, context, extra, seed); + }; + + for ( var prop in oldSizzle ) { + Sizzle[ prop ] = oldSizzle[ prop ]; + } + + // release memory in IE + div = null; + })(); +} + +(function(){ + var html = document.documentElement, + matches = html.matchesSelector || html.mozMatchesSelector || html.webkitMatchesSelector || html.msMatchesSelector; + + if ( matches ) { + // Check to see if it's possible to do matchesSelector + // on a disconnected node (IE 9 fails this) + var disconnectedMatch = !matches.call( document.createElement( "div" ), "div" ), + pseudoWorks = false; + + try { + // This should fail with an exception + // Gecko does not error, returns false instead + matches.call( document.documentElement, "[test!='']:sizzle" ); + + } catch( pseudoError ) { + pseudoWorks = true; + } + + Sizzle.matchesSelector = function( node, expr ) { + // Make sure that attribute selectors are quoted + expr = expr.replace(/\=\s*([^'"\]]*)\s*\]/g, "='$1']"); + + if ( !Sizzle.isXML( node ) ) { + try { + if ( pseudoWorks || !Expr.match.PSEUDO.test( expr ) && !/!=/.test( expr ) ) { + var ret = matches.call( node, expr ); + + // IE 9's matchesSelector returns false on disconnected nodes + if ( ret || !disconnectedMatch || + // As well, disconnected nodes are said to be in a document + // fragment in IE 9, so check for that + node.document && node.document.nodeType !== 11 ) { + return ret; + } + } + } catch(e) {} + } + + return Sizzle(expr, null, null, [node]).length > 0; + }; + } +})(); + +(function(){ + var div = document.createElement("div"); + + div.innerHTML = "<div class='test e'></div><div class='test'></div>"; + + // Opera can't find a second classname (in 9.6) + // Also, make sure that getElementsByClassName actually exists + if ( !div.getElementsByClassName || div.getElementsByClassName("e").length === 0 ) { + return; + } + + // Safari caches class attributes, doesn't catch changes (in 3.2) + div.lastChild.className = "e"; + + if ( div.getElementsByClassName("e").length === 1 ) { + return; + } + + Expr.order.splice(1, 0, "CLASS"); + Expr.find.CLASS = function( match, context, isXML ) { + if ( typeof context.getElementsByClassName !== "undefined" && !isXML ) { + return context.getElementsByClassName(match[1]); + } + }; + + // release memory in IE + div = null; +})(); + +function dirNodeCheck( dir, cur, doneName, checkSet, nodeCheck, isXML ) { + for ( var i = 0, l = checkSet.length; i < l; i++ ) { + var elem = checkSet[i]; + + if ( elem ) { + var match = false; + + elem = elem[dir]; + + while ( elem ) { + if ( elem[ expando ] === doneName ) { + match = checkSet[elem.sizset]; + break; + } + + if ( elem.nodeType === 1 && !isXML ){ + elem[ expando ] = doneName; + elem.sizset = i; + } + + if ( elem.nodeName.toLowerCase() === cur ) { + match = elem; + break; + } + + elem = elem[dir]; + } + + checkSet[i] = match; + } + } +} + +function dirCheck( dir, cur, doneName, checkSet, nodeCheck, isXML ) { + for ( var i = 0, l = checkSet.length; i < l; i++ ) { + var elem = checkSet[i]; + + if ( elem ) { + var match = false; + + elem = elem[dir]; + + while ( elem ) { + if ( elem[ expando ] === doneName ) { + match = checkSet[elem.sizset]; + break; + } + + if ( elem.nodeType === 1 ) { + if ( !isXML ) { + elem[ expando ] = doneName; + elem.sizset = i; + } + + if ( typeof cur !== "string" ) { + if ( elem === cur ) { + match = true; + break; + } + + } else if ( Sizzle.filter( cur, [elem] ).length > 0 ) { + match = elem; + break; + } + } + + elem = elem[dir]; + } + + checkSet[i] = match; + } + } +} + +if ( document.documentElement.contains ) { + Sizzle.contains = function( a, b ) { + return a !== b && (a.contains ? a.contains(b) : true); + }; + +} else if ( document.documentElement.compareDocumentPosition ) { + Sizzle.contains = function( a, b ) { + return !!(a.compareDocumentPosition(b) & 16); + }; + +} else { + Sizzle.contains = function() { + return false; + }; +} + +Sizzle.isXML = function( elem ) { + // documentElement is verified for cases where it doesn't yet exist + // (such as loading iframes in IE - #4833) + var documentElement = (elem ? elem.ownerDocument || elem : 0).documentElement; + + return documentElement ? documentElement.nodeName !== "HTML" : false; +}; + +var posProcess = function( selector, context, seed ) { + var match, + tmpSet = [], + later = "", + root = context.nodeType ? [context] : context; + + // Position selectors must be done after the filter + // And so must :not(positional) so we move all PSEUDOs to the end + while ( (match = Expr.match.PSEUDO.exec( selector )) ) { + later += match[0]; + selector = selector.replace( Expr.match.PSEUDO, "" ); + } + + selector = Expr.relative[selector] ? selector + "*" : selector; + + for ( var i = 0, l = root.length; i < l; i++ ) { + Sizzle( selector, root[i], tmpSet, seed ); + } + + return Sizzle.filter( later, tmpSet ); +}; + +// EXPOSE +// Override sizzle attribute retrieval +Sizzle.attr = jQuery.attr; +Sizzle.selectors.attrMap = {}; +jQuery.find = Sizzle; +jQuery.expr = Sizzle.selectors; +jQuery.expr[":"] = jQuery.expr.filters; +jQuery.unique = Sizzle.uniqueSort; +jQuery.text = Sizzle.getText; +jQuery.isXMLDoc = Sizzle.isXML; +jQuery.contains = Sizzle.contains; + + +})(); + + +var runtil = /Until$/, + rparentsprev = /^(?:parents|prevUntil|prevAll)/, + // Note: This RegExp should be improved, or likely pulled from Sizzle + rmultiselector = /,/, + isSimple = /^.[^:#\[\.,]*$/, + slice = Array.prototype.slice, + POS = jQuery.expr.match.POS, + // methods guaranteed to produce a unique set when starting from a unique set + guaranteedUnique = { + children: true, + contents: true, + next: true, + prev: true + }; + +jQuery.fn.extend({ + find: function( selector ) { + var self = this, + i, l; + + if ( typeof selector !== "string" ) { + return jQuery( selector ).filter(function() { + for ( i = 0, l = self.length; i < l; i++ ) { + if ( jQuery.contains( self[ i ], this ) ) { + return true; + } + } + }); + } + + var ret = this.pushStack( "", "find", selector ), + length, n, r; + + for ( i = 0, l = this.length; i < l; i++ ) { + length = ret.length; + jQuery.find( selector, this[i], ret ); + + if ( i > 0 ) { + // Make sure that the results are unique + for ( n = length; n < ret.length; n++ ) { + for ( r = 0; r < length; r++ ) { + if ( ret[r] === ret[n] ) { + ret.splice(n--, 1); + break; + } + } + } + } + } + + return ret; + }, + + has: function( target ) { + var targets = jQuery( target ); + return this.filter(function() { + for ( var i = 0, l = targets.length; i < l; i++ ) { + if ( jQuery.contains( this, targets[i] ) ) { + return true; + } + } + }); + }, + + not: function( selector ) { + return this.pushStack( winnow(this, selector, false), "not", selector); + }, + + filter: function( selector ) { + return this.pushStack( winnow(this, selector, true), "filter", selector ); + }, + + is: function( selector ) { + return !!selector && ( + typeof selector === "string" ? + // If this is a positional selector, check membership in the returned set + // so $("p:first").is("p:last") won't return true for a doc with two "p". + POS.test( selector ) ? + jQuery( selector, this.context ).index( this[0] ) >= 0 : + jQuery.filter( selector, this ).length > 0 : + this.filter( selector ).length > 0 ); + }, + + closest: function( selectors, context ) { + var ret = [], i, l, cur = this[0]; + + // Array (deprecated as of jQuery 1.7) + if ( jQuery.isArray( selectors ) ) { + var level = 1; + + while ( cur && cur.ownerDocument && cur !== context ) { + for ( i = 0; i < selectors.length; i++ ) { + + if ( jQuery( cur ).is( selectors[ i ] ) ) { + ret.push({ selector: selectors[ i ], elem: cur, level: level }); + } + } + + cur = cur.parentNode; + level++; + } + + return ret; + } + + // String + var pos = POS.test( selectors ) || typeof selectors !== "string" ? + jQuery( selectors, context || this.context ) : + 0; + + for ( i = 0, l = this.length; i < l; i++ ) { + cur = this[i]; + + while ( cur ) { + if ( pos ? pos.index(cur) > -1 : jQuery.find.matchesSelector(cur, selectors) ) { + ret.push( cur ); + break; + + } else { + cur = cur.parentNode; + if ( !cur || !cur.ownerDocument || cur === context || cur.nodeType === 11 ) { + break; + } + } + } + } + + ret = ret.length > 1 ? jQuery.unique( ret ) : ret; + + return this.pushStack( ret, "closest", selectors ); + }, + + // Determine the position of an element within + // the matched set of elements + index: function( elem ) { + + // No argument, return index in parent + if ( !elem ) { + return ( this[0] && this[0].parentNode ) ? this.prevAll().length : -1; + } + + // index in selector + if ( typeof elem === "string" ) { + return jQuery.inArray( this[0], jQuery( elem ) ); + } + + // Locate the position of the desired element + return jQuery.inArray( + // If it receives a jQuery object, the first element is used + elem.jquery ? elem[0] : elem, this ); + }, + + add: function( selector, context ) { + var set = typeof selector === "string" ? + jQuery( selector, context ) : + jQuery.makeArray( selector && selector.nodeType ? [ selector ] : selector ), + all = jQuery.merge( this.get(), set ); + + return this.pushStack( isDisconnected( set[0] ) || isDisconnected( all[0] ) ? + all : + jQuery.unique( all ) ); + }, + + andSelf: function() { + return this.add( this.prevObject ); + } +}); + +// A painfully simple check to see if an element is disconnected +// from a document (should be improved, where feasible). +function isDisconnected( node ) { + return !node || !node.parentNode || node.parentNode.nodeType === 11; +} + +jQuery.each({ + parent: function( elem ) { + var parent = elem.parentNode; + return parent && parent.nodeType !== 11 ? parent : null; + }, + parents: function( elem ) { + return jQuery.dir( elem, "parentNode" ); + }, + parentsUntil: function( elem, i, until ) { + return jQuery.dir( elem, "parentNode", until ); + }, + next: function( elem ) { + return jQuery.nth( elem, 2, "nextSibling" ); + }, + prev: function( elem ) { + return jQuery.nth( elem, 2, "previousSibling" ); + }, + nextAll: function( elem ) { + return jQuery.dir( elem, "nextSibling" ); + }, + prevAll: function( elem ) { + return jQuery.dir( elem, "previousSibling" ); + }, + nextUntil: function( elem, i, until ) { + return jQuery.dir( elem, "nextSibling", until ); + }, + prevUntil: function( elem, i, until ) { + return jQuery.dir( elem, "previousSibling", until ); + }, + siblings: function( elem ) { + return jQuery.sibling( elem.parentNode.firstChild, elem ); + }, + children: function( elem ) { + return jQuery.sibling( elem.firstChild ); + }, + contents: function( elem ) { + return jQuery.nodeName( elem, "iframe" ) ? + elem.contentDocument || elem.contentWindow.document : + jQuery.makeArray( elem.childNodes ); + } +}, function( name, fn ) { + jQuery.fn[ name ] = function( until, selector ) { + var ret = jQuery.map( this, fn, until ); + + if ( !runtil.test( name ) ) { + selector = until; + } + + if ( selector && typeof selector === "string" ) { + ret = jQuery.filter( selector, ret ); + } + + ret = this.length > 1 && !guaranteedUnique[ name ] ? jQuery.unique( ret ) : ret; + + if ( (this.length > 1 || rmultiselector.test( selector )) && rparentsprev.test( name ) ) { + ret = ret.reverse(); + } + + return this.pushStack( ret, name, slice.call( arguments ).join(",") ); + }; +}); + +jQuery.extend({ + filter: function( expr, elems, not ) { + if ( not ) { + expr = ":not(" + expr + ")"; + } + + return elems.length === 1 ? + jQuery.find.matchesSelector(elems[0], expr) ? [ elems[0] ] : [] : + jQuery.find.matches(expr, elems); + }, + + dir: function( elem, dir, until ) { + var matched = [], + cur = elem[ dir ]; + + while ( cur && cur.nodeType !== 9 && (until === undefined || cur.nodeType !== 1 || !jQuery( cur ).is( until )) ) { + if ( cur.nodeType === 1 ) { + matched.push( cur ); + } + cur = cur[dir]; + } + return matched; + }, + + nth: function( cur, result, dir, elem ) { + result = result || 1; + var num = 0; + + for ( ; cur; cur = cur[dir] ) { + if ( cur.nodeType === 1 && ++num === result ) { + break; + } + } + + return cur; + }, + + sibling: function( n, elem ) { + var r = []; + + for ( ; n; n = n.nextSibling ) { + if ( n.nodeType === 1 && n !== elem ) { + r.push( n ); + } + } + + return r; + } +}); + +// Implement the identical functionality for filter and not +function winnow( elements, qualifier, keep ) { + + // Can't pass null or undefined to indexOf in Firefox 4 + // Set to 0 to skip string check + qualifier = qualifier || 0; + + if ( jQuery.isFunction( qualifier ) ) { + return jQuery.grep(elements, function( elem, i ) { + var retVal = !!qualifier.call( elem, i, elem ); + return retVal === keep; + }); + + } else if ( qualifier.nodeType ) { + return jQuery.grep(elements, function( elem, i ) { + return ( elem === qualifier ) === keep; + }); + + } else if ( typeof qualifier === "string" ) { + var filtered = jQuery.grep(elements, function( elem ) { + return elem.nodeType === 1; + }); + + if ( isSimple.test( qualifier ) ) { + return jQuery.filter(qualifier, filtered, !keep); + } else { + qualifier = jQuery.filter( qualifier, filtered ); + } + } + + return jQuery.grep(elements, function( elem, i ) { + return ( jQuery.inArray( elem, qualifier ) >= 0 ) === keep; + }); +} + + + + +function createSafeFragment( document ) { + var list = nodeNames.split( "|" ), + safeFrag = document.createDocumentFragment(); + + if ( safeFrag.createElement ) { + while ( list.length ) { + safeFrag.createElement( + list.pop() + ); + } + } + return safeFrag; +} + +var nodeNames = "abbr|article|aside|audio|canvas|datalist|details|figcaption|figure|footer|" + + "header|hgroup|mark|meter|nav|output|progress|section|summary|time|video", + rinlinejQuery = / jQuery\d+="(?:\d+|null)"/g, + rleadingWhitespace = /^\s+/, + rxhtmlTag = /<(?!area|br|col|embed|hr|img|input|link|meta|param)(([\w:]+)[^>]*)\/>/ig, + rtagName = /<([\w:]+)/, + rtbody = /<tbody/i, + rhtml = /<|&#?\w+;/, + rnoInnerhtml = /<(?:script|style)/i, + rnocache = /<(?:script|object|embed|option|style)/i, + rnoshimcache = new RegExp("<(?:" + nodeNames + ")", "i"), + // checked="checked" or checked + rchecked = /checked\s*(?:[^=]|=\s*.checked.)/i, + rscriptType = /\/(java|ecma)script/i, + rcleanScript = /^\s*<!(?:\[CDATA\[|\-\-)/, + wrapMap = { + option: [ 1, "<select multiple='multiple'>", "</select>" ], + legend: [ 1, "<fieldset>", "</fieldset>" ], + thead: [ 1, "<table>", "</table>" ], + tr: [ 2, "<table><tbody>", "</tbody></table>" ], + td: [ 3, "<table><tbody><tr>", "</tr></tbody></table>" ], + col: [ 2, "<table><tbody></tbody><colgroup>", "</colgroup></table>" ], + area: [ 1, "<map>", "</map>" ], + _default: [ 0, "", "" ] + }, + safeFragment = createSafeFragment( document ); + +wrapMap.optgroup = wrapMap.option; +wrapMap.tbody = wrapMap.tfoot = wrapMap.colgroup = wrapMap.caption = wrapMap.thead; +wrapMap.th = wrapMap.td; + +// IE can't serialize <link> and <script> tags normally +if ( !jQuery.support.htmlSerialize ) { + wrapMap._default = [ 1, "div<div>", "</div>" ]; +} + +jQuery.fn.extend({ + text: function( text ) { + if ( jQuery.isFunction(text) ) { + return this.each(function(i) { + var self = jQuery( this ); + + self.text( text.call(this, i, self.text()) ); + }); + } + + if ( typeof text !== "object" && text !== undefined ) { + return this.empty().append( (this[0] && this[0].ownerDocument || document).createTextNode( text ) ); + } + + return jQuery.text( this ); + }, + + wrapAll: function( html ) { + if ( jQuery.isFunction( html ) ) { + return this.each(function(i) { + jQuery(this).wrapAll( html.call(this, i) ); + }); + } + + if ( this[0] ) { + // The elements to wrap the target around + var wrap = jQuery( html, this[0].ownerDocument ).eq(0).clone(true); + + if ( this[0].parentNode ) { + wrap.insertBefore( this[0] ); + } + + wrap.map(function() { + var elem = this; + + while ( elem.firstChild && elem.firstChild.nodeType === 1 ) { + elem = elem.firstChild; + } + + return elem; + }).append( this ); + } + + return this; + }, + + wrapInner: function( html ) { + if ( jQuery.isFunction( html ) ) { + return this.each(function(i) { + jQuery(this).wrapInner( html.call(this, i) ); + }); + } + + return this.each(function() { + var self = jQuery( this ), + contents = self.contents(); + + if ( contents.length ) { + contents.wrapAll( html ); + + } else { + self.append( html ); + } + }); + }, + + wrap: function( html ) { + var isFunction = jQuery.isFunction( html ); + + return this.each(function(i) { + jQuery( this ).wrapAll( isFunction ? html.call(this, i) : html ); + }); + }, + + unwrap: function() { + return this.parent().each(function() { + if ( !jQuery.nodeName( this, "body" ) ) { + jQuery( this ).replaceWith( this.childNodes ); + } + }).end(); + }, + + append: function() { + return this.domManip(arguments, true, function( elem ) { + if ( this.nodeType === 1 ) { + this.appendChild( elem ); + } + }); + }, + + prepend: function() { + return this.domManip(arguments, true, function( elem ) { + if ( this.nodeType === 1 ) { + this.insertBefore( elem, this.firstChild ); + } + }); + }, + + before: function() { + if ( this[0] && this[0].parentNode ) { + return this.domManip(arguments, false, function( elem ) { + this.parentNode.insertBefore( elem, this ); + }); + } else if ( arguments.length ) { + var set = jQuery.clean( arguments ); + set.push.apply( set, this.toArray() ); + return this.pushStack( set, "before", arguments ); + } + }, + + after: function() { + if ( this[0] && this[0].parentNode ) { + return this.domManip(arguments, false, function( elem ) { + this.parentNode.insertBefore( elem, this.nextSibling ); + }); + } else if ( arguments.length ) { + var set = this.pushStack( this, "after", arguments ); + set.push.apply( set, jQuery.clean(arguments) ); + return set; + } + }, + + // keepData is for internal use only--do not document + remove: function( selector, keepData ) { + for ( var i = 0, elem; (elem = this[i]) != null; i++ ) { + if ( !selector || jQuery.filter( selector, [ elem ] ).length ) { + if ( !keepData && elem.nodeType === 1 ) { + jQuery.cleanData( elem.getElementsByTagName("*") ); + jQuery.cleanData( [ elem ] ); + } + + if ( elem.parentNode ) { + elem.parentNode.removeChild( elem ); + } + } + } + + return this; + }, + + empty: function() { + for ( var i = 0, elem; (elem = this[i]) != null; i++ ) { + // Remove element nodes and prevent memory leaks + if ( elem.nodeType === 1 ) { + jQuery.cleanData( elem.getElementsByTagName("*") ); + } + + // Remove any remaining nodes + while ( elem.firstChild ) { + elem.removeChild( elem.firstChild ); + } + } + + return this; + }, + + clone: function( dataAndEvents, deepDataAndEvents ) { + dataAndEvents = dataAndEvents == null ? false : dataAndEvents; + deepDataAndEvents = deepDataAndEvents == null ? dataAndEvents : deepDataAndEvents; + + return this.map( function () { + return jQuery.clone( this, dataAndEvents, deepDataAndEvents ); + }); + }, + + html: function( value ) { + if ( value === undefined ) { + return this[0] && this[0].nodeType === 1 ? + this[0].innerHTML.replace(rinlinejQuery, "") : + null; + + // See if we can take a shortcut and just use innerHTML + } else if ( typeof value === "string" && !rnoInnerhtml.test( value ) && + (jQuery.support.leadingWhitespace || !rleadingWhitespace.test( value )) && + !wrapMap[ (rtagName.exec( value ) || ["", ""])[1].toLowerCase() ] ) { + + value = value.replace(rxhtmlTag, "<$1></$2>"); + + try { + for ( var i = 0, l = this.length; i < l; i++ ) { + // Remove element nodes and prevent memory leaks + if ( this[i].nodeType === 1 ) { + jQuery.cleanData( this[i].getElementsByTagName("*") ); + this[i].innerHTML = value; + } + } + + // If using innerHTML throws an exception, use the fallback method + } catch(e) { + this.empty().append( value ); + } + + } else if ( jQuery.isFunction( value ) ) { + this.each(function(i){ + var self = jQuery( this ); + + self.html( value.call(this, i, self.html()) ); + }); + + } else { + this.empty().append( value ); + } + + return this; + }, + + replaceWith: function( value ) { + if ( this[0] && this[0].parentNode ) { + // Make sure that the elements are removed from the DOM before they are inserted + // this can help fix replacing a parent with child elements + if ( jQuery.isFunction( value ) ) { + return this.each(function(i) { + var self = jQuery(this), old = self.html(); + self.replaceWith( value.call( this, i, old ) ); + }); + } + + if ( typeof value !== "string" ) { + value = jQuery( value ).detach(); + } + + return this.each(function() { + var next = this.nextSibling, + parent = this.parentNode; + + jQuery( this ).remove(); + + if ( next ) { + jQuery(next).before( value ); + } else { + jQuery(parent).append( value ); + } + }); + } else { + return this.length ? + this.pushStack( jQuery(jQuery.isFunction(value) ? value() : value), "replaceWith", value ) : + this; + } + }, + + detach: function( selector ) { + return this.remove( selector, true ); + }, + + domManip: function( args, table, callback ) { + var results, first, fragment, parent, + value = args[0], + scripts = []; + + // We can't cloneNode fragments that contain checked, in WebKit + if ( !jQuery.support.checkClone && arguments.length === 3 && typeof value === "string" && rchecked.test( value ) ) { + return this.each(function() { + jQuery(this).domManip( args, table, callback, true ); + }); + } + + if ( jQuery.isFunction(value) ) { + return this.each(function(i) { + var self = jQuery(this); + args[0] = value.call(this, i, table ? self.html() : undefined); + self.domManip( args, table, callback ); + }); + } + + if ( this[0] ) { + parent = value && value.parentNode; + + // If we're in a fragment, just use that instead of building a new one + if ( jQuery.support.parentNode && parent && parent.nodeType === 11 && parent.childNodes.length === this.length ) { + results = { fragment: parent }; + + } else { + results = jQuery.buildFragment( args, this, scripts ); + } + + fragment = results.fragment; + + if ( fragment.childNodes.length === 1 ) { + first = fragment = fragment.firstChild; + } else { + first = fragment.firstChild; + } + + if ( first ) { + table = table && jQuery.nodeName( first, "tr" ); + + for ( var i = 0, l = this.length, lastIndex = l - 1; i < l; i++ ) { + callback.call( + table ? + root(this[i], first) : + this[i], + // Make sure that we do not leak memory by inadvertently discarding + // the original fragment (which might have attached data) instead of + // using it; in addition, use the original fragment object for the last + // item instead of first because it can end up being emptied incorrectly + // in certain situations (Bug #8070). + // Fragments from the fragment cache must always be cloned and never used + // in place. + results.cacheable || ( l > 1 && i < lastIndex ) ? + jQuery.clone( fragment, true, true ) : + fragment + ); + } + } + + if ( scripts.length ) { + jQuery.each( scripts, evalScript ); + } + } + + return this; + } +}); + +function root( elem, cur ) { + return jQuery.nodeName(elem, "table") ? + (elem.getElementsByTagName("tbody")[0] || + elem.appendChild(elem.ownerDocument.createElement("tbody"))) : + elem; +} + +function cloneCopyEvent( src, dest ) { + + if ( dest.nodeType !== 1 || !jQuery.hasData( src ) ) { + return; + } + + var type, i, l, + oldData = jQuery._data( src ), + curData = jQuery._data( dest, oldData ), + events = oldData.events; + + if ( events ) { + delete curData.handle; + curData.events = {}; + + for ( type in events ) { + for ( i = 0, l = events[ type ].length; i < l; i++ ) { + jQuery.event.add( dest, type + ( events[ type ][ i ].namespace ? "." : "" ) + events[ type ][ i ].namespace, events[ type ][ i ], events[ type ][ i ].data ); + } + } + } + + // make the cloned public data object a copy from the original + if ( curData.data ) { + curData.data = jQuery.extend( {}, curData.data ); + } +} + +function cloneFixAttributes( src, dest ) { + var nodeName; + + // We do not need to do anything for non-Elements + if ( dest.nodeType !== 1 ) { + return; + } + + // clearAttributes removes the attributes, which we don't want, + // but also removes the attachEvent events, which we *do* want + if ( dest.clearAttributes ) { + dest.clearAttributes(); + } + + // mergeAttributes, in contrast, only merges back on the + // original attributes, not the events + if ( dest.mergeAttributes ) { + dest.mergeAttributes( src ); + } + + nodeName = dest.nodeName.toLowerCase(); + + // IE6-8 fail to clone children inside object elements that use + // the proprietary classid attribute value (rather than the type + // attribute) to identify the type of content to display + if ( nodeName === "object" ) { + dest.outerHTML = src.outerHTML; + + } else if ( nodeName === "input" && (src.type === "checkbox" || src.type === "radio") ) { + // IE6-8 fails to persist the checked state of a cloned checkbox + // or radio button. Worse, IE6-7 fail to give the cloned element + // a checked appearance if the defaultChecked value isn't also set + if ( src.checked ) { + dest.defaultChecked = dest.checked = src.checked; + } + + // IE6-7 get confused and end up setting the value of a cloned + // checkbox/radio button to an empty string instead of "on" + if ( dest.value !== src.value ) { + dest.value = src.value; + } + + // IE6-8 fails to return the selected option to the default selected + // state when cloning options + } else if ( nodeName === "option" ) { + dest.selected = src.defaultSelected; + + // IE6-8 fails to set the defaultValue to the correct value when + // cloning other types of input fields + } else if ( nodeName === "input" || nodeName === "textarea" ) { + dest.defaultValue = src.defaultValue; + } + + // Event data gets referenced instead of copied if the expando + // gets copied too + dest.removeAttribute( jQuery.expando ); +} + +jQuery.buildFragment = function( args, nodes, scripts ) { + var fragment, cacheable, cacheresults, doc, + first = args[ 0 ]; + + // nodes may contain either an explicit document object, + // a jQuery collection or context object. + // If nodes[0] contains a valid object to assign to doc + if ( nodes && nodes[0] ) { + doc = nodes[0].ownerDocument || nodes[0]; + } + + // Ensure that an attr object doesn't incorrectly stand in as a document object + // Chrome and Firefox seem to allow this to occur and will throw exception + // Fixes #8950 + if ( !doc.createDocumentFragment ) { + doc = document; + } + + // Only cache "small" (1/2 KB) HTML strings that are associated with the main document + // Cloning options loses the selected state, so don't cache them + // IE 6 doesn't like it when you put <object> or <embed> elements in a fragment + // Also, WebKit does not clone 'checked' attributes on cloneNode, so don't cache + // Lastly, IE6,7,8 will not correctly reuse cached fragments that were created from unknown elems #10501 + if ( args.length === 1 && typeof first === "string" && first.length < 512 && doc === document && + first.charAt(0) === "<" && !rnocache.test( first ) && + (jQuery.support.checkClone || !rchecked.test( first )) && + (jQuery.support.html5Clone || !rnoshimcache.test( first )) ) { + + cacheable = true; + + cacheresults = jQuery.fragments[ first ]; + if ( cacheresults && cacheresults !== 1 ) { + fragment = cacheresults; + } + } + + if ( !fragment ) { + fragment = doc.createDocumentFragment(); + jQuery.clean( args, doc, fragment, scripts ); + } + + if ( cacheable ) { + jQuery.fragments[ first ] = cacheresults ? fragment : 1; + } + + return { fragment: fragment, cacheable: cacheable }; +}; + +jQuery.fragments = {}; + +jQuery.each({ + appendTo: "append", + prependTo: "prepend", + insertBefore: "before", + insertAfter: "after", + replaceAll: "replaceWith" +}, function( name, original ) { + jQuery.fn[ name ] = function( selector ) { + var ret = [], + insert = jQuery( selector ), + parent = this.length === 1 && this[0].parentNode; + + if ( parent && parent.nodeType === 11 && parent.childNodes.length === 1 && insert.length === 1 ) { + insert[ original ]( this[0] ); + return this; + + } else { + for ( var i = 0, l = insert.length; i < l; i++ ) { + var elems = ( i > 0 ? this.clone(true) : this ).get(); + jQuery( insert[i] )[ original ]( elems ); + ret = ret.concat( elems ); + } + + return this.pushStack( ret, name, insert.selector ); + } + }; +}); + +function getAll( elem ) { + if ( typeof elem.getElementsByTagName !== "undefined" ) { + return elem.getElementsByTagName( "*" ); + + } else if ( typeof elem.querySelectorAll !== "undefined" ) { + return elem.querySelectorAll( "*" ); + + } else { + return []; + } +} + +// Used in clean, fixes the defaultChecked property +function fixDefaultChecked( elem ) { + if ( elem.type === "checkbox" || elem.type === "radio" ) { + elem.defaultChecked = elem.checked; + } +} +// Finds all inputs and passes them to fixDefaultChecked +function findInputs( elem ) { + var nodeName = ( elem.nodeName || "" ).toLowerCase(); + if ( nodeName === "input" ) { + fixDefaultChecked( elem ); + // Skip scripts, get other children + } else if ( nodeName !== "script" && typeof elem.getElementsByTagName !== "undefined" ) { + jQuery.grep( elem.getElementsByTagName("input"), fixDefaultChecked ); + } +} + +// Derived From: http://www.iecss.com/shimprove/javascript/shimprove.1-0-1.js +function shimCloneNode( elem ) { + var div = document.createElement( "div" ); + safeFragment.appendChild( div ); + + div.innerHTML = elem.outerHTML; + return div.firstChild; +} + +jQuery.extend({ + clone: function( elem, dataAndEvents, deepDataAndEvents ) { + var srcElements, + destElements, + i, + // IE<=8 does not properly clone detached, unknown element nodes + clone = jQuery.support.html5Clone || !rnoshimcache.test( "<" + elem.nodeName ) ? + elem.cloneNode( true ) : + shimCloneNode( elem ); + + if ( (!jQuery.support.noCloneEvent || !jQuery.support.noCloneChecked) && + (elem.nodeType === 1 || elem.nodeType === 11) && !jQuery.isXMLDoc(elem) ) { + // IE copies events bound via attachEvent when using cloneNode. + // Calling detachEvent on the clone will also remove the events + // from the original. In order to get around this, we use some + // proprietary methods to clear the events. Thanks to MooTools + // guys for this hotness. + + cloneFixAttributes( elem, clone ); + + // Using Sizzle here is crazy slow, so we use getElementsByTagName instead + srcElements = getAll( elem ); + destElements = getAll( clone ); + + // Weird iteration because IE will replace the length property + // with an element if you are cloning the body and one of the + // elements on the page has a name or id of "length" + for ( i = 0; srcElements[i]; ++i ) { + // Ensure that the destination node is not null; Fixes #9587 + if ( destElements[i] ) { + cloneFixAttributes( srcElements[i], destElements[i] ); + } + } + } + + // Copy the events from the original to the clone + if ( dataAndEvents ) { + cloneCopyEvent( elem, clone ); + + if ( deepDataAndEvents ) { + srcElements = getAll( elem ); + destElements = getAll( clone ); + + for ( i = 0; srcElements[i]; ++i ) { + cloneCopyEvent( srcElements[i], destElements[i] ); + } + } + } + + srcElements = destElements = null; + + // Return the cloned set + return clone; + }, + + clean: function( elems, context, fragment, scripts ) { + var checkScriptType; + + context = context || document; + + // !context.createElement fails in IE with an error but returns typeof 'object' + if ( typeof context.createElement === "undefined" ) { + context = context.ownerDocument || context[0] && context[0].ownerDocument || document; + } + + var ret = [], j; + + for ( var i = 0, elem; (elem = elems[i]) != null; i++ ) { + if ( typeof elem === "number" ) { + elem += ""; + } + + if ( !elem ) { + continue; + } + + // Convert html string into DOM nodes + if ( typeof elem === "string" ) { + if ( !rhtml.test( elem ) ) { + elem = context.createTextNode( elem ); + } else { + // Fix "XHTML"-style tags in all browsers + elem = elem.replace(rxhtmlTag, "<$1></$2>"); + + // Trim whitespace, otherwise indexOf won't work as expected + var tag = ( rtagName.exec( elem ) || ["", ""] )[1].toLowerCase(), + wrap = wrapMap[ tag ] || wrapMap._default, + depth = wrap[0], + div = context.createElement("div"); + + // Append wrapper element to unknown element safe doc fragment + if ( context === document ) { + // Use the fragment we've already created for this document + safeFragment.appendChild( div ); + } else { + // Use a fragment created with the owner document + createSafeFragment( context ).appendChild( div ); + } + + // Go to html and back, then peel off extra wrappers + div.innerHTML = wrap[1] + elem + wrap[2]; + + // Move to the right depth + while ( depth-- ) { + div = div.lastChild; + } + + // Remove IE's autoinserted <tbody> from table fragments + if ( !jQuery.support.tbody ) { + + // String was a <table>, *may* have spurious <tbody> + var hasBody = rtbody.test(elem), + tbody = tag === "table" && !hasBody ? + div.firstChild && div.firstChild.childNodes : + + // String was a bare <thead> or <tfoot> + wrap[1] === "<table>" && !hasBody ? + div.childNodes : + []; + + for ( j = tbody.length - 1; j >= 0 ; --j ) { + if ( jQuery.nodeName( tbody[ j ], "tbody" ) && !tbody[ j ].childNodes.length ) { + tbody[ j ].parentNode.removeChild( tbody[ j ] ); + } + } + } + + // IE completely kills leading whitespace when innerHTML is used + if ( !jQuery.support.leadingWhitespace && rleadingWhitespace.test( elem ) ) { + div.insertBefore( context.createTextNode( rleadingWhitespace.exec(elem)[0] ), div.firstChild ); + } + + elem = div.childNodes; + } + } + + // Resets defaultChecked for any radios and checkboxes + // about to be appended to the DOM in IE 6/7 (#8060) + var len; + if ( !jQuery.support.appendChecked ) { + if ( elem[0] && typeof (len = elem.length) === "number" ) { + for ( j = 0; j < len; j++ ) { + findInputs( elem[j] ); + } + } else { + findInputs( elem ); + } + } + + if ( elem.nodeType ) { + ret.push( elem ); + } else { + ret = jQuery.merge( ret, elem ); + } + } + + if ( fragment ) { + checkScriptType = function( elem ) { + return !elem.type || rscriptType.test( elem.type ); + }; + for ( i = 0; ret[i]; i++ ) { + if ( scripts && jQuery.nodeName( ret[i], "script" ) && (!ret[i].type || ret[i].type.toLowerCase() === "text/javascript") ) { + scripts.push( ret[i].parentNode ? ret[i].parentNode.removeChild( ret[i] ) : ret[i] ); + + } else { + if ( ret[i].nodeType === 1 ) { + var jsTags = jQuery.grep( ret[i].getElementsByTagName( "script" ), checkScriptType ); + + ret.splice.apply( ret, [i + 1, 0].concat( jsTags ) ); + } + fragment.appendChild( ret[i] ); + } + } + } + + return ret; + }, + + cleanData: function( elems ) { + var data, id, + cache = jQuery.cache, + special = jQuery.event.special, + deleteExpando = jQuery.support.deleteExpando; + + for ( var i = 0, elem; (elem = elems[i]) != null; i++ ) { + if ( elem.nodeName && jQuery.noData[elem.nodeName.toLowerCase()] ) { + continue; + } + + id = elem[ jQuery.expando ]; + + if ( id ) { + data = cache[ id ]; + + if ( data && data.events ) { + for ( var type in data.events ) { + if ( special[ type ] ) { + jQuery.event.remove( elem, type ); + + // This is a shortcut to avoid jQuery.event.remove's overhead + } else { + jQuery.removeEvent( elem, type, data.handle ); + } + } + + // Null the DOM reference to avoid IE6/7/8 leak (#7054) + if ( data.handle ) { + data.handle.elem = null; + } + } + + if ( deleteExpando ) { + delete elem[ jQuery.expando ]; + + } else if ( elem.removeAttribute ) { + elem.removeAttribute( jQuery.expando ); + } + + delete cache[ id ]; + } + } + } +}); + +function evalScript( i, elem ) { + if ( elem.src ) { + jQuery.ajax({ + url: elem.src, + async: false, + dataType: "script" + }); + } else { + jQuery.globalEval( ( elem.text || elem.textContent || elem.innerHTML || "" ).replace( rcleanScript, "/*$0*/" ) ); + } + + if ( elem.parentNode ) { + elem.parentNode.removeChild( elem ); + } +} + + + + +var ralpha = /alpha\([^)]*\)/i, + ropacity = /opacity=([^)]*)/, + // fixed for IE9, see #8346 + rupper = /([A-Z]|^ms)/g, + rnumpx = /^-?\d+(?:px)?$/i, + rnum = /^-?\d/, + rrelNum = /^([\-+])=([\-+.\de]+)/, + + cssShow = { position: "absolute", visibility: "hidden", display: "block" }, + cssWidth = [ "Left", "Right" ], + cssHeight = [ "Top", "Bottom" ], + curCSS, + + getComputedStyle, + currentStyle; + +jQuery.fn.css = function( name, value ) { + // Setting 'undefined' is a no-op + if ( arguments.length === 2 && value === undefined ) { + return this; + } + + return jQuery.access( this, name, value, true, function( elem, name, value ) { + return value !== undefined ? + jQuery.style( elem, name, value ) : + jQuery.css( elem, name ); + }); +}; + +jQuery.extend({ + // Add in style property hooks for overriding the default + // behavior of getting and setting a style property + cssHooks: { + opacity: { + get: function( elem, computed ) { + if ( computed ) { + // We should always get a number back from opacity + var ret = curCSS( elem, "opacity", "opacity" ); + return ret === "" ? "1" : ret; + + } else { + return elem.style.opacity; + } + } + } + }, + + // Exclude the following css properties to add px + cssNumber: { + "fillOpacity": true, + "fontWeight": true, + "lineHeight": true, + "opacity": true, + "orphans": true, + "widows": true, + "zIndex": true, + "zoom": true + }, + + // Add in properties whose names you wish to fix before + // setting or getting the value + cssProps: { + // normalize float css property + "float": jQuery.support.cssFloat ? "cssFloat" : "styleFloat" + }, + + // Get and set the style property on a DOM Node + style: function( elem, name, value, extra ) { + // Don't set styles on text and comment nodes + if ( !elem || elem.nodeType === 3 || elem.nodeType === 8 || !elem.style ) { + return; + } + + // Make sure that we're working with the right name + var ret, type, origName = jQuery.camelCase( name ), + style = elem.style, hooks = jQuery.cssHooks[ origName ]; + + name = jQuery.cssProps[ origName ] || origName; + + // Check if we're setting a value + if ( value !== undefined ) { + type = typeof value; + + // convert relative number strings (+= or -=) to relative numbers. #7345 + if ( type === "string" && (ret = rrelNum.exec( value )) ) { + value = ( +( ret[1] + 1) * +ret[2] ) + parseFloat( jQuery.css( elem, name ) ); + // Fixes bug #9237 + type = "number"; + } + + // Make sure that NaN and null values aren't set. See: #7116 + if ( value == null || type === "number" && isNaN( value ) ) { + return; + } + + // If a number was passed in, add 'px' to the (except for certain CSS properties) + if ( type === "number" && !jQuery.cssNumber[ origName ] ) { + value += "px"; + } + + // If a hook was provided, use that value, otherwise just set the specified value + if ( !hooks || !("set" in hooks) || (value = hooks.set( elem, value )) !== undefined ) { + // Wrapped to prevent IE from throwing errors when 'invalid' values are provided + // Fixes bug #5509 + try { + style[ name ] = value; + } catch(e) {} + } + + } else { + // If a hook was provided get the non-computed value from there + if ( hooks && "get" in hooks && (ret = hooks.get( elem, false, extra )) !== undefined ) { + return ret; + } + + // Otherwise just get the value from the style object + return style[ name ]; + } + }, + + css: function( elem, name, extra ) { + var ret, hooks; + + // Make sure that we're working with the right name + name = jQuery.camelCase( name ); + hooks = jQuery.cssHooks[ name ]; + name = jQuery.cssProps[ name ] || name; + + // cssFloat needs a special treatment + if ( name === "cssFloat" ) { + name = "float"; + } + + // If a hook was provided get the computed value from there + if ( hooks && "get" in hooks && (ret = hooks.get( elem, true, extra )) !== undefined ) { + return ret; + + // Otherwise, if a way to get the computed value exists, use that + } else if ( curCSS ) { + return curCSS( elem, name ); + } + }, + + // A method for quickly swapping in/out CSS properties to get correct calculations + swap: function( elem, options, callback ) { + var old = {}; + + // Remember the old values, and insert the new ones + for ( var name in options ) { + old[ name ] = elem.style[ name ]; + elem.style[ name ] = options[ name ]; + } + + callback.call( elem ); + + // Revert the old values + for ( name in options ) { + elem.style[ name ] = old[ name ]; + } + } +}); + +// DEPRECATED, Use jQuery.css() instead +jQuery.curCSS = jQuery.css; + +jQuery.each(["height", "width"], function( i, name ) { + jQuery.cssHooks[ name ] = { + get: function( elem, computed, extra ) { + var val; + + if ( computed ) { + if ( elem.offsetWidth !== 0 ) { + return getWH( elem, name, extra ); + } else { + jQuery.swap( elem, cssShow, function() { + val = getWH( elem, name, extra ); + }); + } + + return val; + } + }, + + set: function( elem, value ) { + if ( rnumpx.test( value ) ) { + // ignore negative width and height values #1599 + value = parseFloat( value ); + + if ( value >= 0 ) { + return value + "px"; + } + + } else { + return value; + } + } + }; +}); + +if ( !jQuery.support.opacity ) { + jQuery.cssHooks.opacity = { + get: function( elem, computed ) { + // IE uses filters for opacity + return ropacity.test( (computed && elem.currentStyle ? elem.currentStyle.filter : elem.style.filter) || "" ) ? + ( parseFloat( RegExp.$1 ) / 100 ) + "" : + computed ? "1" : ""; + }, + + set: function( elem, value ) { + var style = elem.style, + currentStyle = elem.currentStyle, + opacity = jQuery.isNumeric( value ) ? "alpha(opacity=" + value * 100 + ")" : "", + filter = currentStyle && currentStyle.filter || style.filter || ""; + + // IE has trouble with opacity if it does not have layout + // Force it by setting the zoom level + style.zoom = 1; + + // if setting opacity to 1, and no other filters exist - attempt to remove filter attribute #6652 + if ( value >= 1 && jQuery.trim( filter.replace( ralpha, "" ) ) === "" ) { + + // Setting style.filter to null, "" & " " still leave "filter:" in the cssText + // if "filter:" is present at all, clearType is disabled, we want to avoid this + // style.removeAttribute is IE Only, but so apparently is this code path... + style.removeAttribute( "filter" ); + + // if there there is no filter style applied in a css rule, we are done + if ( currentStyle && !currentStyle.filter ) { + return; + } + } + + // otherwise, set new filter values + style.filter = ralpha.test( filter ) ? + filter.replace( ralpha, opacity ) : + filter + " " + opacity; + } + }; +} + +jQuery(function() { + // This hook cannot be added until DOM ready because the support test + // for it is not run until after DOM ready + if ( !jQuery.support.reliableMarginRight ) { + jQuery.cssHooks.marginRight = { + get: function( elem, computed ) { + // WebKit Bug 13343 - getComputedStyle returns wrong value for margin-right + // Work around by temporarily setting element display to inline-block + var ret; + jQuery.swap( elem, { "display": "inline-block" }, function() { + if ( computed ) { + ret = curCSS( elem, "margin-right", "marginRight" ); + } else { + ret = elem.style.marginRight; + } + }); + return ret; + } + }; + } +}); + +if ( document.defaultView && document.defaultView.getComputedStyle ) { + getComputedStyle = function( elem, name ) { + var ret, defaultView, computedStyle; + + name = name.replace( rupper, "-$1" ).toLowerCase(); + + if ( (defaultView = elem.ownerDocument.defaultView) && + (computedStyle = defaultView.getComputedStyle( elem, null )) ) { + ret = computedStyle.getPropertyValue( name ); + if ( ret === "" && !jQuery.contains( elem.ownerDocument.documentElement, elem ) ) { + ret = jQuery.style( elem, name ); + } + } + + return ret; + }; +} + +if ( document.documentElement.currentStyle ) { + currentStyle = function( elem, name ) { + var left, rsLeft, uncomputed, + ret = elem.currentStyle && elem.currentStyle[ name ], + style = elem.style; + + // Avoid setting ret to empty string here + // so we don't default to auto + if ( ret === null && style && (uncomputed = style[ name ]) ) { + ret = uncomputed; + } + + // From the awesome hack by Dean Edwards + // http://erik.eae.net/archives/2007/07/27/18.54.15/#comment-102291 + + // If we're not dealing with a regular pixel number + // but a number that has a weird ending, we need to convert it to pixels + if ( !rnumpx.test( ret ) && rnum.test( ret ) ) { + + // Remember the original values + left = style.left; + rsLeft = elem.runtimeStyle && elem.runtimeStyle.left; + + // Put in the new values to get a computed value out + if ( rsLeft ) { + elem.runtimeStyle.left = elem.currentStyle.left; + } + style.left = name === "fontSize" ? "1em" : ( ret || 0 ); + ret = style.pixelLeft + "px"; + + // Revert the changed values + style.left = left; + if ( rsLeft ) { + elem.runtimeStyle.left = rsLeft; + } + } + + return ret === "" ? "auto" : ret; + }; +} + +curCSS = getComputedStyle || currentStyle; + +function getWH( elem, name, extra ) { + + // Start with offset property + var val = name === "width" ? elem.offsetWidth : elem.offsetHeight, + which = name === "width" ? cssWidth : cssHeight, + i = 0, + len = which.length; + + if ( val > 0 ) { + if ( extra !== "border" ) { + for ( ; i < len; i++ ) { + if ( !extra ) { + val -= parseFloat( jQuery.css( elem, "padding" + which[ i ] ) ) || 0; + } + if ( extra === "margin" ) { + val += parseFloat( jQuery.css( elem, extra + which[ i ] ) ) || 0; + } else { + val -= parseFloat( jQuery.css( elem, "border" + which[ i ] + "Width" ) ) || 0; + } + } + } + + return val + "px"; + } + + // Fall back to computed then uncomputed css if necessary + val = curCSS( elem, name, name ); + if ( val < 0 || val == null ) { + val = elem.style[ name ] || 0; + } + // Normalize "", auto, and prepare for extra + val = parseFloat( val ) || 0; + + // Add padding, border, margin + if ( extra ) { + for ( ; i < len; i++ ) { + val += parseFloat( jQuery.css( elem, "padding" + which[ i ] ) ) || 0; + if ( extra !== "padding" ) { + val += parseFloat( jQuery.css( elem, "border" + which[ i ] + "Width" ) ) || 0; + } + if ( extra === "margin" ) { + val += parseFloat( jQuery.css( elem, extra + which[ i ] ) ) || 0; + } + } + } + + return val + "px"; +} + +if ( jQuery.expr && jQuery.expr.filters ) { + jQuery.expr.filters.hidden = function( elem ) { + var width = elem.offsetWidth, + height = elem.offsetHeight; + + return ( width === 0 && height === 0 ) || (!jQuery.support.reliableHiddenOffsets && ((elem.style && elem.style.display) || jQuery.css( elem, "display" )) === "none"); + }; + + jQuery.expr.filters.visible = function( elem ) { + return !jQuery.expr.filters.hidden( elem ); + }; +} + + + + +var r20 = /%20/g, + rbracket = /\[\]$/, + rCRLF = /\r?\n/g, + rhash = /#.*$/, + rheaders = /^(.*?):[ \t]*([^\r\n]*)\r?$/mg, // IE leaves an \r character at EOL + rinput = /^(?:color|date|datetime|datetime-local|email|hidden|month|number|password|range|search|tel|text|time|url|week)$/i, + // #7653, #8125, #8152: local protocol detection + rlocalProtocol = /^(?:about|app|app\-storage|.+\-extension|file|res|widget):$/, + rnoContent = /^(?:GET|HEAD)$/, + rprotocol = /^\/\//, + rquery = /\?/, + rscript = /<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, + rselectTextarea = /^(?:select|textarea)/i, + rspacesAjax = /\s+/, + rts = /([?&])_=[^&]*/, + rurl = /^([\w\+\.\-]+:)(?:\/\/([^\/?#:]*)(?::(\d+))?)?/, + + // Keep a copy of the old load method + _load = jQuery.fn.load, + + /* Prefilters + * 1) They are useful to introduce custom dataTypes (see ajax/jsonp.js for an example) + * 2) These are called: + * - BEFORE asking for a transport + * - AFTER param serialization (s.data is a string if s.processData is true) + * 3) key is the dataType + * 4) the catchall symbol "*" can be used + * 5) execution will start with transport dataType and THEN continue down to "*" if needed + */ + prefilters = {}, + + /* Transports bindings + * 1) key is the dataType + * 2) the catchall symbol "*" can be used + * 3) selection will start with transport dataType and THEN go to "*" if needed + */ + transports = {}, + + // Document location + ajaxLocation, + + // Document location segments + ajaxLocParts, + + // Avoid comment-prolog char sequence (#10098); must appease lint and evade compression + allTypes = ["*/"] + ["*"]; + +// #8138, IE may throw an exception when accessing +// a field from window.location if document.domain has been set +try { + ajaxLocation = location.href; +} catch( e ) { + // Use the href attribute of an A element + // since IE will modify it given document.location + ajaxLocation = document.createElement( "a" ); + ajaxLocation.href = ""; + ajaxLocation = ajaxLocation.href; +} + +// Segment location into parts +ajaxLocParts = rurl.exec( ajaxLocation.toLowerCase() ) || []; + +// Base "constructor" for jQuery.ajaxPrefilter and jQuery.ajaxTransport +function addToPrefiltersOrTransports( structure ) { + + // dataTypeExpression is optional and defaults to "*" + return function( dataTypeExpression, func ) { + + if ( typeof dataTypeExpression !== "string" ) { + func = dataTypeExpression; + dataTypeExpression = "*"; + } + + if ( jQuery.isFunction( func ) ) { + var dataTypes = dataTypeExpression.toLowerCase().split( rspacesAjax ), + i = 0, + length = dataTypes.length, + dataType, + list, + placeBefore; + + // For each dataType in the dataTypeExpression + for ( ; i < length; i++ ) { + dataType = dataTypes[ i ]; + // We control if we're asked to add before + // any existing element + placeBefore = /^\+/.test( dataType ); + if ( placeBefore ) { + dataType = dataType.substr( 1 ) || "*"; + } + list = structure[ dataType ] = structure[ dataType ] || []; + // then we add to the structure accordingly + list[ placeBefore ? "unshift" : "push" ]( func ); + } + } + }; +} + +// Base inspection function for prefilters and transports +function inspectPrefiltersOrTransports( structure, options, originalOptions, jqXHR, + dataType /* internal */, inspected /* internal */ ) { + + dataType = dataType || options.dataTypes[ 0 ]; + inspected = inspected || {}; + + inspected[ dataType ] = true; + + var list = structure[ dataType ], + i = 0, + length = list ? list.length : 0, + executeOnly = ( structure === prefilters ), + selection; + + for ( ; i < length && ( executeOnly || !selection ); i++ ) { + selection = list[ i ]( options, originalOptions, jqXHR ); + // If we got redirected to another dataType + // we try there if executing only and not done already + if ( typeof selection === "string" ) { + if ( !executeOnly || inspected[ selection ] ) { + selection = undefined; + } else { + options.dataTypes.unshift( selection ); + selection = inspectPrefiltersOrTransports( + structure, options, originalOptions, jqXHR, selection, inspected ); + } + } + } + // If we're only executing or nothing was selected + // we try the catchall dataType if not done already + if ( ( executeOnly || !selection ) && !inspected[ "*" ] ) { + selection = inspectPrefiltersOrTransports( + structure, options, originalOptions, jqXHR, "*", inspected ); + } + // unnecessary when only executing (prefilters) + // but it'll be ignored by the caller in that case + return selection; +} + +// A special extend for ajax options +// that takes "flat" options (not to be deep extended) +// Fixes #9887 +function ajaxExtend( target, src ) { + var key, deep, + flatOptions = jQuery.ajaxSettings.flatOptions || {}; + for ( key in src ) { + if ( src[ key ] !== undefined ) { + ( flatOptions[ key ] ? target : ( deep || ( deep = {} ) ) )[ key ] = src[ key ]; + } + } + if ( deep ) { + jQuery.extend( true, target, deep ); + } +} + +jQuery.fn.extend({ + load: function( url, params, callback ) { + if ( typeof url !== "string" && _load ) { + return _load.apply( this, arguments ); + + // Don't do a request if no elements are being requested + } else if ( !this.length ) { + return this; + } + + var off = url.indexOf( " " ); + if ( off >= 0 ) { + var selector = url.slice( off, url.length ); + url = url.slice( 0, off ); + } + + // Default to a GET request + var type = "GET"; + + // If the second parameter was provided + if ( params ) { + // If it's a function + if ( jQuery.isFunction( params ) ) { + // We assume that it's the callback + callback = params; + params = undefined; + + // Otherwise, build a param string + } else if ( typeof params === "object" ) { + params = jQuery.param( params, jQuery.ajaxSettings.traditional ); + type = "POST"; + } + } + + var self = this; + + // Request the remote document + jQuery.ajax({ + url: url, + type: type, + dataType: "html", + data: params, + // Complete callback (responseText is used internally) + complete: function( jqXHR, status, responseText ) { + // Store the response as specified by the jqXHR object + responseText = jqXHR.responseText; + // If successful, inject the HTML into all the matched elements + if ( jqXHR.isResolved() ) { + // #4825: Get the actual response in case + // a dataFilter is present in ajaxSettings + jqXHR.done(function( r ) { + responseText = r; + }); + // See if a selector was specified + self.html( selector ? + // Create a dummy div to hold the results + jQuery("<div>") + // inject the contents of the document in, removing the scripts + // to avoid any 'Permission Denied' errors in IE + .append(responseText.replace(rscript, "")) + + // Locate the specified elements + .find(selector) : + + // If not, just inject the full result + responseText ); + } + + if ( callback ) { + self.each( callback, [ responseText, status, jqXHR ] ); + } + } + }); + + return this; + }, + + serialize: function() { + return jQuery.param( this.serializeArray() ); + }, + + serializeArray: function() { + return this.map(function(){ + return this.elements ? jQuery.makeArray( this.elements ) : this; + }) + .filter(function(){ + return this.name && !this.disabled && + ( this.checked || rselectTextarea.test( this.nodeName ) || + rinput.test( this.type ) ); + }) + .map(function( i, elem ){ + var val = jQuery( this ).val(); + + return val == null ? + null : + jQuery.isArray( val ) ? + jQuery.map( val, function( val, i ){ + return { name: elem.name, value: val.replace( rCRLF, "\r\n" ) }; + }) : + { name: elem.name, value: val.replace( rCRLF, "\r\n" ) }; + }).get(); + } +}); + +// Attach a bunch of functions for handling common AJAX events +jQuery.each( "ajaxStart ajaxStop ajaxComplete ajaxError ajaxSuccess ajaxSend".split( " " ), function( i, o ){ + jQuery.fn[ o ] = function( f ){ + return this.on( o, f ); + }; +}); + +jQuery.each( [ "get", "post" ], function( i, method ) { + jQuery[ method ] = function( url, data, callback, type ) { + // shift arguments if data argument was omitted + if ( jQuery.isFunction( data ) ) { + type = type || callback; + callback = data; + data = undefined; + } + + return jQuery.ajax({ + type: method, + url: url, + data: data, + success: callback, + dataType: type + }); + }; +}); + +jQuery.extend({ + + getScript: function( url, callback ) { + return jQuery.get( url, undefined, callback, "script" ); + }, + + getJSON: function( url, data, callback ) { + return jQuery.get( url, data, callback, "json" ); + }, + + // Creates a full fledged settings object into target + // with both ajaxSettings and settings fields. + // If target is omitted, writes into ajaxSettings. + ajaxSetup: function( target, settings ) { + if ( settings ) { + // Building a settings object + ajaxExtend( target, jQuery.ajaxSettings ); + } else { + // Extending ajaxSettings + settings = target; + target = jQuery.ajaxSettings; + } + ajaxExtend( target, settings ); + return target; + }, + + ajaxSettings: { + url: ajaxLocation, + isLocal: rlocalProtocol.test( ajaxLocParts[ 1 ] ), + global: true, + type: "GET", + contentType: "application/x-www-form-urlencoded", + processData: true, + async: true, + /* + timeout: 0, + data: null, + dataType: null, + username: null, + password: null, + cache: null, + traditional: false, + headers: {}, + */ + + accepts: { + xml: "application/xml, text/xml", + html: "text/html", + text: "text/plain", + json: "application/json, text/javascript", + "*": allTypes + }, + + contents: { + xml: /xml/, + html: /html/, + json: /json/ + }, + + responseFields: { + xml: "responseXML", + text: "responseText" + }, + + // List of data converters + // 1) key format is "source_type destination_type" (a single space in-between) + // 2) the catchall symbol "*" can be used for source_type + converters: { + + // Convert anything to text + "* text": window.String, + + // Text to html (true = no transformation) + "text html": true, + + // Evaluate text as a json expression + "text json": jQuery.parseJSON, + + // Parse text as xml + "text xml": jQuery.parseXML + }, + + // For options that shouldn't be deep extended: + // you can add your own custom options here if + // and when you create one that shouldn't be + // deep extended (see ajaxExtend) + flatOptions: { + context: true, + url: true + } + }, + + ajaxPrefilter: addToPrefiltersOrTransports( prefilters ), + ajaxTransport: addToPrefiltersOrTransports( transports ), + + // Main method + ajax: function( url, options ) { + + // If url is an object, simulate pre-1.5 signature + if ( typeof url === "object" ) { + options = url; + url = undefined; + } + + // Force options to be an object + options = options || {}; + + var // Create the final options object + s = jQuery.ajaxSetup( {}, options ), + // Callbacks context + callbackContext = s.context || s, + // Context for global events + // It's the callbackContext if one was provided in the options + // and if it's a DOM node or a jQuery collection + globalEventContext = callbackContext !== s && + ( callbackContext.nodeType || callbackContext instanceof jQuery ) ? + jQuery( callbackContext ) : jQuery.event, + // Deferreds + deferred = jQuery.Deferred(), + completeDeferred = jQuery.Callbacks( "once memory" ), + // Status-dependent callbacks + statusCode = s.statusCode || {}, + // ifModified key + ifModifiedKey, + // Headers (they are sent all at once) + requestHeaders = {}, + requestHeadersNames = {}, + // Response headers + responseHeadersString, + responseHeaders, + // transport + transport, + // timeout handle + timeoutTimer, + // Cross-domain detection vars + parts, + // The jqXHR state + state = 0, + // To know if global events are to be dispatched + fireGlobals, + // Loop variable + i, + // Fake xhr + jqXHR = { + + readyState: 0, + + // Caches the header + setRequestHeader: function( name, value ) { + if ( !state ) { + var lname = name.toLowerCase(); + name = requestHeadersNames[ lname ] = requestHeadersNames[ lname ] || name; + requestHeaders[ name ] = value; + } + return this; + }, + + // Raw string + getAllResponseHeaders: function() { + return state === 2 ? responseHeadersString : null; + }, + + // Builds headers hashtable if needed + getResponseHeader: function( key ) { + var match; + if ( state === 2 ) { + if ( !responseHeaders ) { + responseHeaders = {}; + while( ( match = rheaders.exec( responseHeadersString ) ) ) { + responseHeaders[ match[1].toLowerCase() ] = match[ 2 ]; + } + } + match = responseHeaders[ key.toLowerCase() ]; + } + return match === undefined ? null : match; + }, + + // Overrides response content-type header + overrideMimeType: function( type ) { + if ( !state ) { + s.mimeType = type; + } + return this; + }, + + // Cancel the request + abort: function( statusText ) { + statusText = statusText || "abort"; + if ( transport ) { + transport.abort( statusText ); + } + done( 0, statusText ); + return this; + } + }; + + // Callback for when everything is done + // It is defined here because jslint complains if it is declared + // at the end of the function (which would be more logical and readable) + function done( status, nativeStatusText, responses, headers ) { + + // Called once + if ( state === 2 ) { + return; + } + + // State is "done" now + state = 2; + + // Clear timeout if it exists + if ( timeoutTimer ) { + clearTimeout( timeoutTimer ); + } + + // Dereference transport for early garbage collection + // (no matter how long the jqXHR object will be used) + transport = undefined; + + // Cache response headers + responseHeadersString = headers || ""; + + // Set readyState + jqXHR.readyState = status > 0 ? 4 : 0; + + var isSuccess, + success, + error, + statusText = nativeStatusText, + response = responses ? ajaxHandleResponses( s, jqXHR, responses ) : undefined, + lastModified, + etag; + + // If successful, handle type chaining + if ( status >= 200 && status < 300 || status === 304 ) { + + // Set the If-Modified-Since and/or If-None-Match header, if in ifModified mode. + if ( s.ifModified ) { + + if ( ( lastModified = jqXHR.getResponseHeader( "Last-Modified" ) ) ) { + jQuery.lastModified[ ifModifiedKey ] = lastModified; + } + if ( ( etag = jqXHR.getResponseHeader( "Etag" ) ) ) { + jQuery.etag[ ifModifiedKey ] = etag; + } + } + + // If not modified + if ( status === 304 ) { + + statusText = "notmodified"; + isSuccess = true; + + // If we have data + } else { + + try { + success = ajaxConvert( s, response ); + statusText = "success"; + isSuccess = true; + } catch(e) { + // We have a parsererror + statusText = "parsererror"; + error = e; + } + } + } else { + // We extract error from statusText + // then normalize statusText and status for non-aborts + error = statusText; + if ( !statusText || status ) { + statusText = "error"; + if ( status < 0 ) { + status = 0; + } + } + } + + // Set data for the fake xhr object + jqXHR.status = status; + jqXHR.statusText = "" + ( nativeStatusText || statusText ); + + // Success/Error + if ( isSuccess ) { + deferred.resolveWith( callbackContext, [ success, statusText, jqXHR ] ); + } else { + deferred.rejectWith( callbackContext, [ jqXHR, statusText, error ] ); + } + + // Status-dependent callbacks + jqXHR.statusCode( statusCode ); + statusCode = undefined; + + if ( fireGlobals ) { + globalEventContext.trigger( "ajax" + ( isSuccess ? "Success" : "Error" ), + [ jqXHR, s, isSuccess ? success : error ] ); + } + + // Complete + completeDeferred.fireWith( callbackContext, [ jqXHR, statusText ] ); + + if ( fireGlobals ) { + globalEventContext.trigger( "ajaxComplete", [ jqXHR, s ] ); + // Handle the global AJAX counter + if ( !( --jQuery.active ) ) { + jQuery.event.trigger( "ajaxStop" ); + } + } + } + + // Attach deferreds + deferred.promise( jqXHR ); + jqXHR.success = jqXHR.done; + jqXHR.error = jqXHR.fail; + jqXHR.complete = completeDeferred.add; + + // Status-dependent callbacks + jqXHR.statusCode = function( map ) { + if ( map ) { + var tmp; + if ( state < 2 ) { + for ( tmp in map ) { + statusCode[ tmp ] = [ statusCode[tmp], map[tmp] ]; + } + } else { + tmp = map[ jqXHR.status ]; + jqXHR.then( tmp, tmp ); + } + } + return this; + }; + + // Remove hash character (#7531: and string promotion) + // Add protocol if not provided (#5866: IE7 issue with protocol-less urls) + // We also use the url parameter if available + s.url = ( ( url || s.url ) + "" ).replace( rhash, "" ).replace( rprotocol, ajaxLocParts[ 1 ] + "//" ); + + // Extract dataTypes list + s.dataTypes = jQuery.trim( s.dataType || "*" ).toLowerCase().split( rspacesAjax ); + + // Determine if a cross-domain request is in order + if ( s.crossDomain == null ) { + parts = rurl.exec( s.url.toLowerCase() ); + s.crossDomain = !!( parts && + ( parts[ 1 ] != ajaxLocParts[ 1 ] || parts[ 2 ] != ajaxLocParts[ 2 ] || + ( parts[ 3 ] || ( parts[ 1 ] === "http:" ? 80 : 443 ) ) != + ( ajaxLocParts[ 3 ] || ( ajaxLocParts[ 1 ] === "http:" ? 80 : 443 ) ) ) + ); + } + + // Convert data if not already a string + if ( s.data && s.processData && typeof s.data !== "string" ) { + s.data = jQuery.param( s.data, s.traditional ); + } + + // Apply prefilters + inspectPrefiltersOrTransports( prefilters, s, options, jqXHR ); + + // If request was aborted inside a prefiler, stop there + if ( state === 2 ) { + return false; + } + + // We can fire global events as of now if asked to + fireGlobals = s.global; + + // Uppercase the type + s.type = s.type.toUpperCase(); + + // Determine if request has content + s.hasContent = !rnoContent.test( s.type ); + + // Watch for a new set of requests + if ( fireGlobals && jQuery.active++ === 0 ) { + jQuery.event.trigger( "ajaxStart" ); + } + + // More options handling for requests with no content + if ( !s.hasContent ) { + + // If data is available, append data to url + if ( s.data ) { + s.url += ( rquery.test( s.url ) ? "&" : "?" ) + s.data; + // #9682: remove data so that it's not used in an eventual retry + delete s.data; + } + + // Get ifModifiedKey before adding the anti-cache parameter + ifModifiedKey = s.url; + + // Add anti-cache in url if needed + if ( s.cache === false ) { + + var ts = jQuery.now(), + // try replacing _= if it is there + ret = s.url.replace( rts, "$1_=" + ts ); + + // if nothing was replaced, add timestamp to the end + s.url = ret + ( ( ret === s.url ) ? ( rquery.test( s.url ) ? "&" : "?" ) + "_=" + ts : "" ); + } + } + + // Set the correct header, if data is being sent + if ( s.data && s.hasContent && s.contentType !== false || options.contentType ) { + jqXHR.setRequestHeader( "Content-Type", s.contentType ); + } + + // Set the If-Modified-Since and/or If-None-Match header, if in ifModified mode. + if ( s.ifModified ) { + ifModifiedKey = ifModifiedKey || s.url; + if ( jQuery.lastModified[ ifModifiedKey ] ) { + jqXHR.setRequestHeader( "If-Modified-Since", jQuery.lastModified[ ifModifiedKey ] ); + } + if ( jQuery.etag[ ifModifiedKey ] ) { + jqXHR.setRequestHeader( "If-None-Match", jQuery.etag[ ifModifiedKey ] ); + } + } + + // Set the Accepts header for the server, depending on the dataType + jqXHR.setRequestHeader( + "Accept", + s.dataTypes[ 0 ] && s.accepts[ s.dataTypes[0] ] ? + s.accepts[ s.dataTypes[0] ] + ( s.dataTypes[ 0 ] !== "*" ? ", " + allTypes + "; q=0.01" : "" ) : + s.accepts[ "*" ] + ); + + // Check for headers option + for ( i in s.headers ) { + jqXHR.setRequestHeader( i, s.headers[ i ] ); + } + + // Allow custom headers/mimetypes and early abort + if ( s.beforeSend && ( s.beforeSend.call( callbackContext, jqXHR, s ) === false || state === 2 ) ) { + // Abort if not done already + jqXHR.abort(); + return false; + + } + + // Install callbacks on deferreds + for ( i in { success: 1, error: 1, complete: 1 } ) { + jqXHR[ i ]( s[ i ] ); + } + + // Get transport + transport = inspectPrefiltersOrTransports( transports, s, options, jqXHR ); + + // If no transport, we auto-abort + if ( !transport ) { + done( -1, "No Transport" ); + } else { + jqXHR.readyState = 1; + // Send global event + if ( fireGlobals ) { + globalEventContext.trigger( "ajaxSend", [ jqXHR, s ] ); + } + // Timeout + if ( s.async && s.timeout > 0 ) { + timeoutTimer = setTimeout( function(){ + jqXHR.abort( "timeout" ); + }, s.timeout ); + } + + try { + state = 1; + transport.send( requestHeaders, done ); + } catch (e) { + // Propagate exception as error if not done + if ( state < 2 ) { + done( -1, e ); + // Simply rethrow otherwise + } else { + throw e; + } + } + } + + return jqXHR; + }, + + // Serialize an array of form elements or a set of + // key/values into a query string + param: function( a, traditional ) { + var s = [], + add = function( key, value ) { + // If value is a function, invoke it and return its value + value = jQuery.isFunction( value ) ? value() : value; + s[ s.length ] = encodeURIComponent( key ) + "=" + encodeURIComponent( value ); + }; + + // Set traditional to true for jQuery <= 1.3.2 behavior. + if ( traditional === undefined ) { + traditional = jQuery.ajaxSettings.traditional; + } + + // If an array was passed in, assume that it is an array of form elements. + if ( jQuery.isArray( a ) || ( a.jquery && !jQuery.isPlainObject( a ) ) ) { + // Serialize the form elements + jQuery.each( a, function() { + add( this.name, this.value ); + }); + + } else { + // If traditional, encode the "old" way (the way 1.3.2 or older + // did it), otherwise encode params recursively. + for ( var prefix in a ) { + buildParams( prefix, a[ prefix ], traditional, add ); + } + } + + // Return the resulting serialization + return s.join( "&" ).replace( r20, "+" ); + } +}); + +function buildParams( prefix, obj, traditional, add ) { + if ( jQuery.isArray( obj ) ) { + // Serialize array item. + jQuery.each( obj, function( i, v ) { + if ( traditional || rbracket.test( prefix ) ) { + // Treat each array item as a scalar. + add( prefix, v ); + + } else { + // If array item is non-scalar (array or object), encode its + // numeric index to resolve deserialization ambiguity issues. + // Note that rack (as of 1.0.0) can't currently deserialize + // nested arrays properly, and attempting to do so may cause + // a server error. Possible fixes are to modify rack's + // deserialization algorithm or to provide an option or flag + // to force array serialization to be shallow. + buildParams( prefix + "[" + ( typeof v === "object" || jQuery.isArray(v) ? i : "" ) + "]", v, traditional, add ); + } + }); + + } else if ( !traditional && obj != null && typeof obj === "object" ) { + // Serialize object item. + for ( var name in obj ) { + buildParams( prefix + "[" + name + "]", obj[ name ], traditional, add ); + } + + } else { + // Serialize scalar item. + add( prefix, obj ); + } +} + +// This is still on the jQuery object... for now +// Want to move this to jQuery.ajax some day +jQuery.extend({ + + // Counter for holding the number of active queries + active: 0, + + // Last-Modified header cache for next request + lastModified: {}, + etag: {} + +}); + +/* Handles responses to an ajax request: + * - sets all responseXXX fields accordingly + * - finds the right dataType (mediates between content-type and expected dataType) + * - returns the corresponding response + */ +function ajaxHandleResponses( s, jqXHR, responses ) { + + var contents = s.contents, + dataTypes = s.dataTypes, + responseFields = s.responseFields, + ct, + type, + finalDataType, + firstDataType; + + // Fill responseXXX fields + for ( type in responseFields ) { + if ( type in responses ) { + jqXHR[ responseFields[type] ] = responses[ type ]; + } + } + + // Remove auto dataType and get content-type in the process + while( dataTypes[ 0 ] === "*" ) { + dataTypes.shift(); + if ( ct === undefined ) { + ct = s.mimeType || jqXHR.getResponseHeader( "content-type" ); + } + } + + // Check if we're dealing with a known content-type + if ( ct ) { + for ( type in contents ) { + if ( contents[ type ] && contents[ type ].test( ct ) ) { + dataTypes.unshift( type ); + break; + } + } + } + + // Check to see if we have a response for the expected dataType + if ( dataTypes[ 0 ] in responses ) { + finalDataType = dataTypes[ 0 ]; + } else { + // Try convertible dataTypes + for ( type in responses ) { + if ( !dataTypes[ 0 ] || s.converters[ type + " " + dataTypes[0] ] ) { + finalDataType = type; + break; + } + if ( !firstDataType ) { + firstDataType = type; + } + } + // Or just use first one + finalDataType = finalDataType || firstDataType; + } + + // If we found a dataType + // We add the dataType to the list if needed + // and return the corresponding response + if ( finalDataType ) { + if ( finalDataType !== dataTypes[ 0 ] ) { + dataTypes.unshift( finalDataType ); + } + return responses[ finalDataType ]; + } +} + +// Chain conversions given the request and the original response +function ajaxConvert( s, response ) { + + // Apply the dataFilter if provided + if ( s.dataFilter ) { + response = s.dataFilter( response, s.dataType ); + } + + var dataTypes = s.dataTypes, + converters = {}, + i, + key, + length = dataTypes.length, + tmp, + // Current and previous dataTypes + current = dataTypes[ 0 ], + prev, + // Conversion expression + conversion, + // Conversion function + conv, + // Conversion functions (transitive conversion) + conv1, + conv2; + + // For each dataType in the chain + for ( i = 1; i < length; i++ ) { + + // Create converters map + // with lowercased keys + if ( i === 1 ) { + for ( key in s.converters ) { + if ( typeof key === "string" ) { + converters[ key.toLowerCase() ] = s.converters[ key ]; + } + } + } + + // Get the dataTypes + prev = current; + current = dataTypes[ i ]; + + // If current is auto dataType, update it to prev + if ( current === "*" ) { + current = prev; + // If no auto and dataTypes are actually different + } else if ( prev !== "*" && prev !== current ) { + + // Get the converter + conversion = prev + " " + current; + conv = converters[ conversion ] || converters[ "* " + current ]; + + // If there is no direct converter, search transitively + if ( !conv ) { + conv2 = undefined; + for ( conv1 in converters ) { + tmp = conv1.split( " " ); + if ( tmp[ 0 ] === prev || tmp[ 0 ] === "*" ) { + conv2 = converters[ tmp[1] + " " + current ]; + if ( conv2 ) { + conv1 = converters[ conv1 ]; + if ( conv1 === true ) { + conv = conv2; + } else if ( conv2 === true ) { + conv = conv1; + } + break; + } + } + } + } + // If we found no converter, dispatch an error + if ( !( conv || conv2 ) ) { + jQuery.error( "No conversion from " + conversion.replace(" "," to ") ); + } + // If found converter is not an equivalence + if ( conv !== true ) { + // Convert with 1 or 2 converters accordingly + response = conv ? conv( response ) : conv2( conv1(response) ); + } + } + } + return response; +} + + + + +var jsc = jQuery.now(), + jsre = /(\=)\?(&|$)|\?\?/i; + +// Default jsonp settings +jQuery.ajaxSetup({ + jsonp: "callback", + jsonpCallback: function() { + return jQuery.expando + "_" + ( jsc++ ); + } +}); + +// Detect, normalize options and install callbacks for jsonp requests +jQuery.ajaxPrefilter( "json jsonp", function( s, originalSettings, jqXHR ) { + + var inspectData = s.contentType === "application/x-www-form-urlencoded" && + ( typeof s.data === "string" ); + + if ( s.dataTypes[ 0 ] === "jsonp" || + s.jsonp !== false && ( jsre.test( s.url ) || + inspectData && jsre.test( s.data ) ) ) { + + var responseContainer, + jsonpCallback = s.jsonpCallback = + jQuery.isFunction( s.jsonpCallback ) ? s.jsonpCallback() : s.jsonpCallback, + previous = window[ jsonpCallback ], + url = s.url, + data = s.data, + replace = "$1" + jsonpCallback + "$2"; + + if ( s.jsonp !== false ) { + url = url.replace( jsre, replace ); + if ( s.url === url ) { + if ( inspectData ) { + data = data.replace( jsre, replace ); + } + if ( s.data === data ) { + // Add callback manually + url += (/\?/.test( url ) ? "&" : "?") + s.jsonp + "=" + jsonpCallback; + } + } + } + + s.url = url; + s.data = data; + + // Install callback + window[ jsonpCallback ] = function( response ) { + responseContainer = [ response ]; + }; + + // Clean-up function + jqXHR.always(function() { + // Set callback back to previous value + window[ jsonpCallback ] = previous; + // Call if it was a function and we have a response + if ( responseContainer && jQuery.isFunction( previous ) ) { + window[ jsonpCallback ]( responseContainer[ 0 ] ); + } + }); + + // Use data converter to retrieve json after script execution + s.converters["script json"] = function() { + if ( !responseContainer ) { + jQuery.error( jsonpCallback + " was not called" ); + } + return responseContainer[ 0 ]; + }; + + // force json dataType + s.dataTypes[ 0 ] = "json"; + + // Delegate to script + return "script"; + } +}); + + + + +// Install script dataType +jQuery.ajaxSetup({ + accepts: { + script: "text/javascript, application/javascript, application/ecmascript, application/x-ecmascript" + }, + contents: { + script: /javascript|ecmascript/ + }, + converters: { + "text script": function( text ) { + jQuery.globalEval( text ); + return text; + } + } +}); + +// Handle cache's special case and global +jQuery.ajaxPrefilter( "script", function( s ) { + if ( s.cache === undefined ) { + s.cache = false; + } + if ( s.crossDomain ) { + s.type = "GET"; + s.global = false; + } +}); + +// Bind script tag hack transport +jQuery.ajaxTransport( "script", function(s) { + + // This transport only deals with cross domain requests + if ( s.crossDomain ) { + + var script, + head = document.head || document.getElementsByTagName( "head" )[0] || document.documentElement; + + return { + + send: function( _, callback ) { + + script = document.createElement( "script" ); + + script.async = "async"; + + if ( s.scriptCharset ) { + script.charset = s.scriptCharset; + } + + script.src = s.url; + + // Attach handlers for all browsers + script.onload = script.onreadystatechange = function( _, isAbort ) { + + if ( isAbort || !script.readyState || /loaded|complete/.test( script.readyState ) ) { + + // Handle memory leak in IE + script.onload = script.onreadystatechange = null; + + // Remove the script + if ( head && script.parentNode ) { + head.removeChild( script ); + } + + // Dereference the script + script = undefined; + + // Callback if not abort + if ( !isAbort ) { + callback( 200, "success" ); + } + } + }; + // Use insertBefore instead of appendChild to circumvent an IE6 bug. + // This arises when a base node is used (#2709 and #4378). + head.insertBefore( script, head.firstChild ); + }, + + abort: function() { + if ( script ) { + script.onload( 0, 1 ); + } + } + }; + } +}); + + + + +var // #5280: Internet Explorer will keep connections alive if we don't abort on unload + xhrOnUnloadAbort = window.ActiveXObject ? function() { + // Abort all pending requests + for ( var key in xhrCallbacks ) { + xhrCallbacks[ key ]( 0, 1 ); + } + } : false, + xhrId = 0, + xhrCallbacks; + +// Functions to create xhrs +function createStandardXHR() { + try { + return new window.XMLHttpRequest(); + } catch( e ) {} +} + +function createActiveXHR() { + try { + return new window.ActiveXObject( "Microsoft.XMLHTTP" ); + } catch( e ) {} +} + +// Create the request object +// (This is still attached to ajaxSettings for backward compatibility) +jQuery.ajaxSettings.xhr = window.ActiveXObject ? + /* Microsoft failed to properly + * implement the XMLHttpRequest in IE7 (can't request local files), + * so we use the ActiveXObject when it is available + * Additionally XMLHttpRequest can be disabled in IE7/IE8 so + * we need a fallback. + */ + function() { + return !this.isLocal && createStandardXHR() || createActiveXHR(); + } : + // For all other browsers, use the standard XMLHttpRequest object + createStandardXHR; + +// Determine support properties +(function( xhr ) { + jQuery.extend( jQuery.support, { + ajax: !!xhr, + cors: !!xhr && ( "withCredentials" in xhr ) + }); +})( jQuery.ajaxSettings.xhr() ); + +// Create transport if the browser can provide an xhr +if ( jQuery.support.ajax ) { + + jQuery.ajaxTransport(function( s ) { + // Cross domain only allowed if supported through XMLHttpRequest + if ( !s.crossDomain || jQuery.support.cors ) { + + var callback; + + return { + send: function( headers, complete ) { + + // Get a new xhr + var xhr = s.xhr(), + handle, + i; + + // Open the socket + // Passing null username, generates a login popup on Opera (#2865) + if ( s.username ) { + xhr.open( s.type, s.url, s.async, s.username, s.password ); + } else { + xhr.open( s.type, s.url, s.async ); + } + + // Apply custom fields if provided + if ( s.xhrFields ) { + for ( i in s.xhrFields ) { + xhr[ i ] = s.xhrFields[ i ]; + } + } + + // Override mime type if needed + if ( s.mimeType && xhr.overrideMimeType ) { + xhr.overrideMimeType( s.mimeType ); + } + + // X-Requested-With header + // For cross-domain requests, seeing as conditions for a preflight are + // akin to a jigsaw puzzle, we simply never set it to be sure. + // (it can always be set on a per-request basis or even using ajaxSetup) + // For same-domain requests, won't change header if already provided. + if ( !s.crossDomain && !headers["X-Requested-With"] ) { + headers[ "X-Requested-With" ] = "XMLHttpRequest"; + } + + // Need an extra try/catch for cross domain requests in Firefox 3 + try { + for ( i in headers ) { + xhr.setRequestHeader( i, headers[ i ] ); + } + } catch( _ ) {} + + // Do send the request + // This may raise an exception which is actually + // handled in jQuery.ajax (so no try/catch here) + xhr.send( ( s.hasContent && s.data ) || null ); + + // Listener + callback = function( _, isAbort ) { + + var status, + statusText, + responseHeaders, + responses, + xml; + + // Firefox throws exceptions when accessing properties + // of an xhr when a network error occured + // http://helpful.knobs-dials.com/index.php/Component_returned_failure_code:_0x80040111_(NS_ERROR_NOT_AVAILABLE) + try { + + // Was never called and is aborted or complete + if ( callback && ( isAbort || xhr.readyState === 4 ) ) { + + // Only called once + callback = undefined; + + // Do not keep as active anymore + if ( handle ) { + xhr.onreadystatechange = jQuery.noop; + if ( xhrOnUnloadAbort ) { + delete xhrCallbacks[ handle ]; + } + } + + // If it's an abort + if ( isAbort ) { + // Abort it manually if needed + if ( xhr.readyState !== 4 ) { + xhr.abort(); + } + } else { + status = xhr.status; + responseHeaders = xhr.getAllResponseHeaders(); + responses = {}; + xml = xhr.responseXML; + + // Construct response list + if ( xml && xml.documentElement /* #4958 */ ) { + responses.xml = xml; + } + responses.text = xhr.responseText; + + // Firefox throws an exception when accessing + // statusText for faulty cross-domain requests + try { + statusText = xhr.statusText; + } catch( e ) { + // We normalize with Webkit giving an empty statusText + statusText = ""; + } + + // Filter status for non standard behaviors + + // If the request is local and we have data: assume a success + // (success with no data won't get notified, that's the best we + // can do given current implementations) + if ( !status && s.isLocal && !s.crossDomain ) { + status = responses.text ? 200 : 404; + // IE - #1450: sometimes returns 1223 when it should be 204 + } else if ( status === 1223 ) { + status = 204; + } + } + } + } catch( firefoxAccessException ) { + if ( !isAbort ) { + complete( -1, firefoxAccessException ); + } + } + + // Call complete if needed + if ( responses ) { + complete( status, statusText, responses, responseHeaders ); + } + }; + + // if we're in sync mode or it's in cache + // and has been retrieved directly (IE6 & IE7) + // we need to manually fire the callback + if ( !s.async || xhr.readyState === 4 ) { + callback(); + } else { + handle = ++xhrId; + if ( xhrOnUnloadAbort ) { + // Create the active xhrs callbacks list if needed + // and attach the unload handler + if ( !xhrCallbacks ) { + xhrCallbacks = {}; + jQuery( window ).unload( xhrOnUnloadAbort ); + } + // Add to list of active xhrs callbacks + xhrCallbacks[ handle ] = callback; + } + xhr.onreadystatechange = callback; + } + }, + + abort: function() { + if ( callback ) { + callback(0,1); + } + } + }; + } + }); +} + + + + +var elemdisplay = {}, + iframe, iframeDoc, + rfxtypes = /^(?:toggle|show|hide)$/, + rfxnum = /^([+\-]=)?([\d+.\-]+)([a-z%]*)$/i, + timerId, + fxAttrs = [ + // height animations + [ "height", "marginTop", "marginBottom", "paddingTop", "paddingBottom" ], + // width animations + [ "width", "marginLeft", "marginRight", "paddingLeft", "paddingRight" ], + // opacity animations + [ "opacity" ] + ], + fxNow; + +jQuery.fn.extend({ + show: function( speed, easing, callback ) { + var elem, display; + + if ( speed || speed === 0 ) { + return this.animate( genFx("show", 3), speed, easing, callback ); + + } else { + for ( var i = 0, j = this.length; i < j; i++ ) { + elem = this[ i ]; + + if ( elem.style ) { + display = elem.style.display; + + // Reset the inline display of this element to learn if it is + // being hidden by cascaded rules or not + if ( !jQuery._data(elem, "olddisplay") && display === "none" ) { + display = elem.style.display = ""; + } + + // Set elements which have been overridden with display: none + // in a stylesheet to whatever the default browser style is + // for such an element + if ( display === "" && jQuery.css(elem, "display") === "none" ) { + jQuery._data( elem, "olddisplay", defaultDisplay(elem.nodeName) ); + } + } + } + + // Set the display of most of the elements in a second loop + // to avoid the constant reflow + for ( i = 0; i < j; i++ ) { + elem = this[ i ]; + + if ( elem.style ) { + display = elem.style.display; + + if ( display === "" || display === "none" ) { + elem.style.display = jQuery._data( elem, "olddisplay" ) || ""; + } + } + } + + return this; + } + }, + + hide: function( speed, easing, callback ) { + if ( speed || speed === 0 ) { + return this.animate( genFx("hide", 3), speed, easing, callback); + + } else { + var elem, display, + i = 0, + j = this.length; + + for ( ; i < j; i++ ) { + elem = this[i]; + if ( elem.style ) { + display = jQuery.css( elem, "display" ); + + if ( display !== "none" && !jQuery._data( elem, "olddisplay" ) ) { + jQuery._data( elem, "olddisplay", display ); + } + } + } + + // Set the display of the elements in a second loop + // to avoid the constant reflow + for ( i = 0; i < j; i++ ) { + if ( this[i].style ) { + this[i].style.display = "none"; + } + } + + return this; + } + }, + + // Save the old toggle function + _toggle: jQuery.fn.toggle, + + toggle: function( fn, fn2, callback ) { + var bool = typeof fn === "boolean"; + + if ( jQuery.isFunction(fn) && jQuery.isFunction(fn2) ) { + this._toggle.apply( this, arguments ); + + } else if ( fn == null || bool ) { + this.each(function() { + var state = bool ? fn : jQuery(this).is(":hidden"); + jQuery(this)[ state ? "show" : "hide" ](); + }); + + } else { + this.animate(genFx("toggle", 3), fn, fn2, callback); + } + + return this; + }, + + fadeTo: function( speed, to, easing, callback ) { + return this.filter(":hidden").css("opacity", 0).show().end() + .animate({opacity: to}, speed, easing, callback); + }, + + animate: function( prop, speed, easing, callback ) { + var optall = jQuery.speed( speed, easing, callback ); + + if ( jQuery.isEmptyObject( prop ) ) { + return this.each( optall.complete, [ false ] ); + } + + // Do not change referenced properties as per-property easing will be lost + prop = jQuery.extend( {}, prop ); + + function doAnimation() { + // XXX 'this' does not always have a nodeName when running the + // test suite + + if ( optall.queue === false ) { + jQuery._mark( this ); + } + + var opt = jQuery.extend( {}, optall ), + isElement = this.nodeType === 1, + hidden = isElement && jQuery(this).is(":hidden"), + name, val, p, e, + parts, start, end, unit, + method; + + // will store per property easing and be used to determine when an animation is complete + opt.animatedProperties = {}; + + for ( p in prop ) { + + // property name normalization + name = jQuery.camelCase( p ); + if ( p !== name ) { + prop[ name ] = prop[ p ]; + delete prop[ p ]; + } + + val = prop[ name ]; + + // easing resolution: per property > opt.specialEasing > opt.easing > 'swing' (default) + if ( jQuery.isArray( val ) ) { + opt.animatedProperties[ name ] = val[ 1 ]; + val = prop[ name ] = val[ 0 ]; + } else { + opt.animatedProperties[ name ] = opt.specialEasing && opt.specialEasing[ name ] || opt.easing || 'swing'; + } + + if ( val === "hide" && hidden || val === "show" && !hidden ) { + return opt.complete.call( this ); + } + + if ( isElement && ( name === "height" || name === "width" ) ) { + // Make sure that nothing sneaks out + // Record all 3 overflow attributes because IE does not + // change the overflow attribute when overflowX and + // overflowY are set to the same value + opt.overflow = [ this.style.overflow, this.style.overflowX, this.style.overflowY ]; + + // Set display property to inline-block for height/width + // animations on inline elements that are having width/height animated + if ( jQuery.css( this, "display" ) === "inline" && + jQuery.css( this, "float" ) === "none" ) { + + // inline-level elements accept inline-block; + // block-level elements need to be inline with layout + if ( !jQuery.support.inlineBlockNeedsLayout || defaultDisplay( this.nodeName ) === "inline" ) { + this.style.display = "inline-block"; + + } else { + this.style.zoom = 1; + } + } + } + } + + if ( opt.overflow != null ) { + this.style.overflow = "hidden"; + } + + for ( p in prop ) { + e = new jQuery.fx( this, opt, p ); + val = prop[ p ]; + + if ( rfxtypes.test( val ) ) { + + // Tracks whether to show or hide based on private + // data attached to the element + method = jQuery._data( this, "toggle" + p ) || ( val === "toggle" ? hidden ? "show" : "hide" : 0 ); + if ( method ) { + jQuery._data( this, "toggle" + p, method === "show" ? "hide" : "show" ); + e[ method ](); + } else { + e[ val ](); + } + + } else { + parts = rfxnum.exec( val ); + start = e.cur(); + + if ( parts ) { + end = parseFloat( parts[2] ); + unit = parts[3] || ( jQuery.cssNumber[ p ] ? "" : "px" ); + + // We need to compute starting value + if ( unit !== "px" ) { + jQuery.style( this, p, (end || 1) + unit); + start = ( (end || 1) / e.cur() ) * start; + jQuery.style( this, p, start + unit); + } + + // If a +=/-= token was provided, we're doing a relative animation + if ( parts[1] ) { + end = ( (parts[ 1 ] === "-=" ? -1 : 1) * end ) + start; + } + + e.custom( start, end, unit ); + + } else { + e.custom( start, val, "" ); + } + } + } + + // For JS strict compliance + return true; + } + + return optall.queue === false ? + this.each( doAnimation ) : + this.queue( optall.queue, doAnimation ); + }, + + stop: function( type, clearQueue, gotoEnd ) { + if ( typeof type !== "string" ) { + gotoEnd = clearQueue; + clearQueue = type; + type = undefined; + } + if ( clearQueue && type !== false ) { + this.queue( type || "fx", [] ); + } + + return this.each(function() { + var index, + hadTimers = false, + timers = jQuery.timers, + data = jQuery._data( this ); + + // clear marker counters if we know they won't be + if ( !gotoEnd ) { + jQuery._unmark( true, this ); + } + + function stopQueue( elem, data, index ) { + var hooks = data[ index ]; + jQuery.removeData( elem, index, true ); + hooks.stop( gotoEnd ); + } + + if ( type == null ) { + for ( index in data ) { + if ( data[ index ] && data[ index ].stop && index.indexOf(".run") === index.length - 4 ) { + stopQueue( this, data, index ); + } + } + } else if ( data[ index = type + ".run" ] && data[ index ].stop ){ + stopQueue( this, data, index ); + } + + for ( index = timers.length; index--; ) { + if ( timers[ index ].elem === this && (type == null || timers[ index ].queue === type) ) { + if ( gotoEnd ) { + + // force the next step to be the last + timers[ index ]( true ); + } else { + timers[ index ].saveState(); + } + hadTimers = true; + timers.splice( index, 1 ); + } + } + + // start the next in the queue if the last step wasn't forced + // timers currently will call their complete callbacks, which will dequeue + // but only if they were gotoEnd + if ( !( gotoEnd && hadTimers ) ) { + jQuery.dequeue( this, type ); + } + }); + } + +}); + +// Animations created synchronously will run synchronously +function createFxNow() { + setTimeout( clearFxNow, 0 ); + return ( fxNow = jQuery.now() ); +} + +function clearFxNow() { + fxNow = undefined; +} + +// Generate parameters to create a standard animation +function genFx( type, num ) { + var obj = {}; + + jQuery.each( fxAttrs.concat.apply([], fxAttrs.slice( 0, num )), function() { + obj[ this ] = type; + }); + + return obj; +} + +// Generate shortcuts for custom animations +jQuery.each({ + slideDown: genFx( "show", 1 ), + slideUp: genFx( "hide", 1 ), + slideToggle: genFx( "toggle", 1 ), + fadeIn: { opacity: "show" }, + fadeOut: { opacity: "hide" }, + fadeToggle: { opacity: "toggle" } +}, function( name, props ) { + jQuery.fn[ name ] = function( speed, easing, callback ) { + return this.animate( props, speed, easing, callback ); + }; +}); + +jQuery.extend({ + speed: function( speed, easing, fn ) { + var opt = speed && typeof speed === "object" ? jQuery.extend( {}, speed ) : { + complete: fn || !fn && easing || + jQuery.isFunction( speed ) && speed, + duration: speed, + easing: fn && easing || easing && !jQuery.isFunction( easing ) && easing + }; + + opt.duration = jQuery.fx.off ? 0 : typeof opt.duration === "number" ? opt.duration : + opt.duration in jQuery.fx.speeds ? jQuery.fx.speeds[ opt.duration ] : jQuery.fx.speeds._default; + + // normalize opt.queue - true/undefined/null -> "fx" + if ( opt.queue == null || opt.queue === true ) { + opt.queue = "fx"; + } + + // Queueing + opt.old = opt.complete; + + opt.complete = function( noUnmark ) { + if ( jQuery.isFunction( opt.old ) ) { + opt.old.call( this ); + } + + if ( opt.queue ) { + jQuery.dequeue( this, opt.queue ); + } else if ( noUnmark !== false ) { + jQuery._unmark( this ); + } + }; + + return opt; + }, + + easing: { + linear: function( p, n, firstNum, diff ) { + return firstNum + diff * p; + }, + swing: function( p, n, firstNum, diff ) { + return ( ( -Math.cos( p*Math.PI ) / 2 ) + 0.5 ) * diff + firstNum; + } + }, + + timers: [], + + fx: function( elem, options, prop ) { + this.options = options; + this.elem = elem; + this.prop = prop; + + options.orig = options.orig || {}; + } + +}); + +jQuery.fx.prototype = { + // Simple function for setting a style value + update: function() { + if ( this.options.step ) { + this.options.step.call( this.elem, this.now, this ); + } + + ( jQuery.fx.step[ this.prop ] || jQuery.fx.step._default )( this ); + }, + + // Get the current size + cur: function() { + if ( this.elem[ this.prop ] != null && (!this.elem.style || this.elem.style[ this.prop ] == null) ) { + return this.elem[ this.prop ]; + } + + var parsed, + r = jQuery.css( this.elem, this.prop ); + // Empty strings, null, undefined and "auto" are converted to 0, + // complex values such as "rotate(1rad)" are returned as is, + // simple values such as "10px" are parsed to Float. + return isNaN( parsed = parseFloat( r ) ) ? !r || r === "auto" ? 0 : r : parsed; + }, + + // Start an animation from one number to another + custom: function( from, to, unit ) { + var self = this, + fx = jQuery.fx; + + this.startTime = fxNow || createFxNow(); + this.end = to; + this.now = this.start = from; + this.pos = this.state = 0; + this.unit = unit || this.unit || ( jQuery.cssNumber[ this.prop ] ? "" : "px" ); + + function t( gotoEnd ) { + return self.step( gotoEnd ); + } + + t.queue = this.options.queue; + t.elem = this.elem; + t.saveState = function() { + if ( self.options.hide && jQuery._data( self.elem, "fxshow" + self.prop ) === undefined ) { + jQuery._data( self.elem, "fxshow" + self.prop, self.start ); + } + }; + + if ( t() && jQuery.timers.push(t) && !timerId ) { + timerId = setInterval( fx.tick, fx.interval ); + } + }, + + // Simple 'show' function + show: function() { + var dataShow = jQuery._data( this.elem, "fxshow" + this.prop ); + + // Remember where we started, so that we can go back to it later + this.options.orig[ this.prop ] = dataShow || jQuery.style( this.elem, this.prop ); + this.options.show = true; + + // Begin the animation + // Make sure that we start at a small width/height to avoid any flash of content + if ( dataShow !== undefined ) { + // This show is picking up where a previous hide or show left off + this.custom( this.cur(), dataShow ); + } else { + this.custom( this.prop === "width" || this.prop === "height" ? 1 : 0, this.cur() ); + } + + // Start by showing the element + jQuery( this.elem ).show(); + }, + + // Simple 'hide' function + hide: function() { + // Remember where we started, so that we can go back to it later + this.options.orig[ this.prop ] = jQuery._data( this.elem, "fxshow" + this.prop ) || jQuery.style( this.elem, this.prop ); + this.options.hide = true; + + // Begin the animation + this.custom( this.cur(), 0 ); + }, + + // Each step of an animation + step: function( gotoEnd ) { + var p, n, complete, + t = fxNow || createFxNow(), + done = true, + elem = this.elem, + options = this.options; + + if ( gotoEnd || t >= options.duration + this.startTime ) { + this.now = this.end; + this.pos = this.state = 1; + this.update(); + + options.animatedProperties[ this.prop ] = true; + + for ( p in options.animatedProperties ) { + if ( options.animatedProperties[ p ] !== true ) { + done = false; + } + } + + if ( done ) { + // Reset the overflow + if ( options.overflow != null && !jQuery.support.shrinkWrapBlocks ) { + + jQuery.each( [ "", "X", "Y" ], function( index, value ) { + elem.style[ "overflow" + value ] = options.overflow[ index ]; + }); + } + + // Hide the element if the "hide" operation was done + if ( options.hide ) { + jQuery( elem ).hide(); + } + + // Reset the properties, if the item has been hidden or shown + if ( options.hide || options.show ) { + for ( p in options.animatedProperties ) { + jQuery.style( elem, p, options.orig[ p ] ); + jQuery.removeData( elem, "fxshow" + p, true ); + // Toggle data is no longer needed + jQuery.removeData( elem, "toggle" + p, true ); + } + } + + // Execute the complete function + // in the event that the complete function throws an exception + // we must ensure it won't be called twice. #5684 + + complete = options.complete; + if ( complete ) { + + options.complete = false; + complete.call( elem ); + } + } + + return false; + + } else { + // classical easing cannot be used with an Infinity duration + if ( options.duration == Infinity ) { + this.now = t; + } else { + n = t - this.startTime; + this.state = n / options.duration; + + // Perform the easing function, defaults to swing + this.pos = jQuery.easing[ options.animatedProperties[this.prop] ]( this.state, n, 0, 1, options.duration ); + this.now = this.start + ( (this.end - this.start) * this.pos ); + } + // Perform the next step of the animation + this.update(); + } + + return true; + } +}; + +jQuery.extend( jQuery.fx, { + tick: function() { + var timer, + timers = jQuery.timers, + i = 0; + + for ( ; i < timers.length; i++ ) { + timer = timers[ i ]; + // Checks the timer has not already been removed + if ( !timer() && timers[ i ] === timer ) { + timers.splice( i--, 1 ); + } + } + + if ( !timers.length ) { + jQuery.fx.stop(); + } + }, + + interval: 13, + + stop: function() { + clearInterval( timerId ); + timerId = null; + }, + + speeds: { + slow: 600, + fast: 200, + // Default speed + _default: 400 + }, + + step: { + opacity: function( fx ) { + jQuery.style( fx.elem, "opacity", fx.now ); + }, + + _default: function( fx ) { + if ( fx.elem.style && fx.elem.style[ fx.prop ] != null ) { + fx.elem.style[ fx.prop ] = fx.now + fx.unit; + } else { + fx.elem[ fx.prop ] = fx.now; + } + } + } +}); + +// Adds width/height step functions +// Do not set anything below 0 +jQuery.each([ "width", "height" ], function( i, prop ) { + jQuery.fx.step[ prop ] = function( fx ) { + jQuery.style( fx.elem, prop, Math.max(0, fx.now) + fx.unit ); + }; +}); + +if ( jQuery.expr && jQuery.expr.filters ) { + jQuery.expr.filters.animated = function( elem ) { + return jQuery.grep(jQuery.timers, function( fn ) { + return elem === fn.elem; + }).length; + }; +} + +// Try to restore the default display value of an element +function defaultDisplay( nodeName ) { + + if ( !elemdisplay[ nodeName ] ) { + + var body = document.body, + elem = jQuery( "<" + nodeName + ">" ).appendTo( body ), + display = elem.css( "display" ); + elem.remove(); + + // If the simple way fails, + // get element's real default display by attaching it to a temp iframe + if ( display === "none" || display === "" ) { + // No iframe to use yet, so create it + if ( !iframe ) { + iframe = document.createElement( "iframe" ); + iframe.frameBorder = iframe.width = iframe.height = 0; + } + + body.appendChild( iframe ); + + // Create a cacheable copy of the iframe document on first call. + // IE and Opera will allow us to reuse the iframeDoc without re-writing the fake HTML + // document to it; WebKit & Firefox won't allow reusing the iframe document. + if ( !iframeDoc || !iframe.createElement ) { + iframeDoc = ( iframe.contentWindow || iframe.contentDocument ).document; + iframeDoc.write( ( document.compatMode === "CSS1Compat" ? "<!doctype html>" : "" ) + "<html><body>" ); + iframeDoc.close(); + } + + elem = iframeDoc.createElement( nodeName ); + + iframeDoc.body.appendChild( elem ); + + display = jQuery.css( elem, "display" ); + body.removeChild( iframe ); + } + + // Store the correct default display + elemdisplay[ nodeName ] = display; + } + + return elemdisplay[ nodeName ]; +} + + + + +var rtable = /^t(?:able|d|h)$/i, + rroot = /^(?:body|html)$/i; + +if ( "getBoundingClientRect" in document.documentElement ) { + jQuery.fn.offset = function( options ) { + var elem = this[0], box; + + if ( options ) { + return this.each(function( i ) { + jQuery.offset.setOffset( this, options, i ); + }); + } + + if ( !elem || !elem.ownerDocument ) { + return null; + } + + if ( elem === elem.ownerDocument.body ) { + return jQuery.offset.bodyOffset( elem ); + } + + try { + box = elem.getBoundingClientRect(); + } catch(e) {} + + var doc = elem.ownerDocument, + docElem = doc.documentElement; + + // Make sure we're not dealing with a disconnected DOM node + if ( !box || !jQuery.contains( docElem, elem ) ) { + return box ? { top: box.top, left: box.left } : { top: 0, left: 0 }; + } + + var body = doc.body, + win = getWindow(doc), + clientTop = docElem.clientTop || body.clientTop || 0, + clientLeft = docElem.clientLeft || body.clientLeft || 0, + scrollTop = win.pageYOffset || jQuery.support.boxModel && docElem.scrollTop || body.scrollTop, + scrollLeft = win.pageXOffset || jQuery.support.boxModel && docElem.scrollLeft || body.scrollLeft, + top = box.top + scrollTop - clientTop, + left = box.left + scrollLeft - clientLeft; + + return { top: top, left: left }; + }; + +} else { + jQuery.fn.offset = function( options ) { + var elem = this[0]; + + if ( options ) { + return this.each(function( i ) { + jQuery.offset.setOffset( this, options, i ); + }); + } + + if ( !elem || !elem.ownerDocument ) { + return null; + } + + if ( elem === elem.ownerDocument.body ) { + return jQuery.offset.bodyOffset( elem ); + } + + var computedStyle, + offsetParent = elem.offsetParent, + prevOffsetParent = elem, + doc = elem.ownerDocument, + docElem = doc.documentElement, + body = doc.body, + defaultView = doc.defaultView, + prevComputedStyle = defaultView ? defaultView.getComputedStyle( elem, null ) : elem.currentStyle, + top = elem.offsetTop, + left = elem.offsetLeft; + + while ( (elem = elem.parentNode) && elem !== body && elem !== docElem ) { + if ( jQuery.support.fixedPosition && prevComputedStyle.position === "fixed" ) { + break; + } + + computedStyle = defaultView ? defaultView.getComputedStyle(elem, null) : elem.currentStyle; + top -= elem.scrollTop; + left -= elem.scrollLeft; + + if ( elem === offsetParent ) { + top += elem.offsetTop; + left += elem.offsetLeft; + + if ( jQuery.support.doesNotAddBorder && !(jQuery.support.doesAddBorderForTableAndCells && rtable.test(elem.nodeName)) ) { + top += parseFloat( computedStyle.borderTopWidth ) || 0; + left += parseFloat( computedStyle.borderLeftWidth ) || 0; + } + + prevOffsetParent = offsetParent; + offsetParent = elem.offsetParent; + } + + if ( jQuery.support.subtractsBorderForOverflowNotVisible && computedStyle.overflow !== "visible" ) { + top += parseFloat( computedStyle.borderTopWidth ) || 0; + left += parseFloat( computedStyle.borderLeftWidth ) || 0; + } + + prevComputedStyle = computedStyle; + } + + if ( prevComputedStyle.position === "relative" || prevComputedStyle.position === "static" ) { + top += body.offsetTop; + left += body.offsetLeft; + } + + if ( jQuery.support.fixedPosition && prevComputedStyle.position === "fixed" ) { + top += Math.max( docElem.scrollTop, body.scrollTop ); + left += Math.max( docElem.scrollLeft, body.scrollLeft ); + } + + return { top: top, left: left }; + }; +} + +jQuery.offset = { + + bodyOffset: function( body ) { + var top = body.offsetTop, + left = body.offsetLeft; + + if ( jQuery.support.doesNotIncludeMarginInBodyOffset ) { + top += parseFloat( jQuery.css(body, "marginTop") ) || 0; + left += parseFloat( jQuery.css(body, "marginLeft") ) || 0; + } + + return { top: top, left: left }; + }, + + setOffset: function( elem, options, i ) { + var position = jQuery.css( elem, "position" ); + + // set position first, in-case top/left are set even on static elem + if ( position === "static" ) { + elem.style.position = "relative"; + } + + var curElem = jQuery( elem ), + curOffset = curElem.offset(), + curCSSTop = jQuery.css( elem, "top" ), + curCSSLeft = jQuery.css( elem, "left" ), + calculatePosition = ( position === "absolute" || position === "fixed" ) && jQuery.inArray("auto", [curCSSTop, curCSSLeft]) > -1, + props = {}, curPosition = {}, curTop, curLeft; + + // need to be able to calculate position if either top or left is auto and position is either absolute or fixed + if ( calculatePosition ) { + curPosition = curElem.position(); + curTop = curPosition.top; + curLeft = curPosition.left; + } else { + curTop = parseFloat( curCSSTop ) || 0; + curLeft = parseFloat( curCSSLeft ) || 0; + } + + if ( jQuery.isFunction( options ) ) { + options = options.call( elem, i, curOffset ); + } + + if ( options.top != null ) { + props.top = ( options.top - curOffset.top ) + curTop; + } + if ( options.left != null ) { + props.left = ( options.left - curOffset.left ) + curLeft; + } + + if ( "using" in options ) { + options.using.call( elem, props ); + } else { + curElem.css( props ); + } + } +}; + + +jQuery.fn.extend({ + + position: function() { + if ( !this[0] ) { + return null; + } + + var elem = this[0], + + // Get *real* offsetParent + offsetParent = this.offsetParent(), + + // Get correct offsets + offset = this.offset(), + parentOffset = rroot.test(offsetParent[0].nodeName) ? { top: 0, left: 0 } : offsetParent.offset(); + + // Subtract element margins + // note: when an element has margin: auto the offsetLeft and marginLeft + // are the same in Safari causing offset.left to incorrectly be 0 + offset.top -= parseFloat( jQuery.css(elem, "marginTop") ) || 0; + offset.left -= parseFloat( jQuery.css(elem, "marginLeft") ) || 0; + + // Add offsetParent borders + parentOffset.top += parseFloat( jQuery.css(offsetParent[0], "borderTopWidth") ) || 0; + parentOffset.left += parseFloat( jQuery.css(offsetParent[0], "borderLeftWidth") ) || 0; + + // Subtract the two offsets + return { + top: offset.top - parentOffset.top, + left: offset.left - parentOffset.left + }; + }, + + offsetParent: function() { + return this.map(function() { + var offsetParent = this.offsetParent || document.body; + while ( offsetParent && (!rroot.test(offsetParent.nodeName) && jQuery.css(offsetParent, "position") === "static") ) { + offsetParent = offsetParent.offsetParent; + } + return offsetParent; + }); + } +}); + + +// Create scrollLeft and scrollTop methods +jQuery.each( ["Left", "Top"], function( i, name ) { + var method = "scroll" + name; + + jQuery.fn[ method ] = function( val ) { + var elem, win; + + if ( val === undefined ) { + elem = this[ 0 ]; + + if ( !elem ) { + return null; + } + + win = getWindow( elem ); + + // Return the scroll offset + return win ? ("pageXOffset" in win) ? win[ i ? "pageYOffset" : "pageXOffset" ] : + jQuery.support.boxModel && win.document.documentElement[ method ] || + win.document.body[ method ] : + elem[ method ]; + } + + // Set the scroll offset + return this.each(function() { + win = getWindow( this ); + + if ( win ) { + win.scrollTo( + !i ? val : jQuery( win ).scrollLeft(), + i ? val : jQuery( win ).scrollTop() + ); + + } else { + this[ method ] = val; + } + }); + }; +}); + +function getWindow( elem ) { + return jQuery.isWindow( elem ) ? + elem : + elem.nodeType === 9 ? + elem.defaultView || elem.parentWindow : + false; +} + + + + +// Create width, height, innerHeight, innerWidth, outerHeight and outerWidth methods +jQuery.each([ "Height", "Width" ], function( i, name ) { + + var type = name.toLowerCase(); + + // innerHeight and innerWidth + jQuery.fn[ "inner" + name ] = function() { + var elem = this[0]; + return elem ? + elem.style ? + parseFloat( jQuery.css( elem, type, "padding" ) ) : + this[ type ]() : + null; + }; + + // outerHeight and outerWidth + jQuery.fn[ "outer" + name ] = function( margin ) { + var elem = this[0]; + return elem ? + elem.style ? + parseFloat( jQuery.css( elem, type, margin ? "margin" : "border" ) ) : + this[ type ]() : + null; + }; + + jQuery.fn[ type ] = function( size ) { + // Get window width or height + var elem = this[0]; + if ( !elem ) { + return size == null ? null : this; + } + + if ( jQuery.isFunction( size ) ) { + return this.each(function( i ) { + var self = jQuery( this ); + self[ type ]( size.call( this, i, self[ type ]() ) ); + }); + } + + if ( jQuery.isWindow( elem ) ) { + // Everyone else use document.documentElement or document.body depending on Quirks vs Standards mode + // 3rd condition allows Nokia support, as it supports the docElem prop but not CSS1Compat + var docElemProp = elem.document.documentElement[ "client" + name ], + body = elem.document.body; + return elem.document.compatMode === "CSS1Compat" && docElemProp || + body && body[ "client" + name ] || docElemProp; + + // Get document width or height + } else if ( elem.nodeType === 9 ) { + // Either scroll[Width/Height] or offset[Width/Height], whichever is greater + return Math.max( + elem.documentElement["client" + name], + elem.body["scroll" + name], elem.documentElement["scroll" + name], + elem.body["offset" + name], elem.documentElement["offset" + name] + ); + + // Get or set width or height on the element + } else if ( size === undefined ) { + var orig = jQuery.css( elem, type ), + ret = parseFloat( orig ); + + return jQuery.isNumeric( ret ) ? ret : orig; + + // Set the width or height on the element (default to pixels if value is unitless) + } else { + return this.css( type, typeof size === "string" ? size : size + "px" ); + } + }; + +}); + + + + +// Expose jQuery to the global object +window.jQuery = window.$ = jQuery; + +// Expose jQuery as an AMD module, but only for AMD loaders that +// understand the issues with loading multiple versions of jQuery +// in a page that all might call define(). The loader will indicate +// they have special allowances for multiple jQuery versions by +// specifying define.amd.jQuery = true. Register as a named module, +// since jQuery can be concatenated with other files that may use define, +// but not use a proper concatenation script that understands anonymous +// AMD modules. A named AMD is safest and most robust way to register. +// Lowercase jquery is used because AMD module names are derived from +// file names, and jQuery is normally delivered in a lowercase file name. +// Do this after creating the global so that if an AMD module wants to call +// noConflict to hide this version of jQuery, it will work. +if ( typeof define === "function" && define.amd && define.amd.jQuery ) { + define( "jquery", [], function () { return jQuery; } ); +} + + + +})( window ); diff --git a/libs/beetsplug/web/static/underscore.js b/libs/beetsplug/web/static/underscore.js new file mode 100644 index 00000000..5579c07d --- /dev/null +++ b/libs/beetsplug/web/static/underscore.js @@ -0,0 +1,977 @@ +// Underscore.js 1.2.2 +// (c) 2011 Jeremy Ashkenas, DocumentCloud Inc. +// Underscore is freely distributable under the MIT license. +// Portions of Underscore are inspired or borrowed from Prototype, +// Oliver Steele's Functional, and John Resig's Micro-Templating. +// For all details and documentation: +// http://documentcloud.github.com/underscore + +(function() { + + // Baseline setup + // -------------- + + // Establish the root object, `window` in the browser, or `global` on the server. + var root = this; + + // Save the previous value of the `_` variable. + var previousUnderscore = root._; + + // Establish the object that gets returned to break out of a loop iteration. + var breaker = {}; + + // Save bytes in the minified (but not gzipped) version: + var ArrayProto = Array.prototype, ObjProto = Object.prototype, FuncProto = Function.prototype; + + // Create quick reference variables for speed access to core prototypes. + var slice = ArrayProto.slice, + unshift = ArrayProto.unshift, + toString = ObjProto.toString, + hasOwnProperty = ObjProto.hasOwnProperty; + + // All **ECMAScript 5** native function implementations that we hope to use + // are declared here. + var + nativeForEach = ArrayProto.forEach, + nativeMap = ArrayProto.map, + nativeReduce = ArrayProto.reduce, + nativeReduceRight = ArrayProto.reduceRight, + nativeFilter = ArrayProto.filter, + nativeEvery = ArrayProto.every, + nativeSome = ArrayProto.some, + nativeIndexOf = ArrayProto.indexOf, + nativeLastIndexOf = ArrayProto.lastIndexOf, + nativeIsArray = Array.isArray, + nativeKeys = Object.keys, + nativeBind = FuncProto.bind; + + // Create a safe reference to the Underscore object for use below. + var _ = function(obj) { return new wrapper(obj); }; + + // Export the Underscore object for **Node.js** and **"CommonJS"**, with + // backwards-compatibility for the old `require()` API. If we're not in + // CommonJS, add `_` to the global object. + if (typeof exports !== 'undefined') { + if (typeof module !== 'undefined' && module.exports) { + exports = module.exports = _; + } + exports._ = _; + } else if (typeof define === 'function' && define.amd) { + // Register as a named module with AMD. + define('underscore', function() { + return _; + }); + } else { + // Exported as a string, for Closure Compiler "advanced" mode. + root['_'] = _; + } + + // Current version. + _.VERSION = '1.2.2'; + + // Collection Functions + // -------------------- + + // The cornerstone, an `each` implementation, aka `forEach`. + // Handles objects with the built-in `forEach`, arrays, and raw objects. + // Delegates to **ECMAScript 5**'s native `forEach` if available. + var each = _.each = _.forEach = function(obj, iterator, context) { + if (obj == null) return; + if (nativeForEach && obj.forEach === nativeForEach) { + obj.forEach(iterator, context); + } else if (obj.length === +obj.length) { + for (var i = 0, l = obj.length; i < l; i++) { + if (i in obj && iterator.call(context, obj[i], i, obj) === breaker) return; + } + } else { + for (var key in obj) { + if (hasOwnProperty.call(obj, key)) { + if (iterator.call(context, obj[key], key, obj) === breaker) return; + } + } + } + }; + + // Return the results of applying the iterator to each element. + // Delegates to **ECMAScript 5**'s native `map` if available. + _.map = function(obj, iterator, context) { + var results = []; + if (obj == null) return results; + if (nativeMap && obj.map === nativeMap) return obj.map(iterator, context); + each(obj, function(value, index, list) { + results[results.length] = iterator.call(context, value, index, list); + }); + return results; + }; + + // **Reduce** builds up a single result from a list of values, aka `inject`, + // or `foldl`. Delegates to **ECMAScript 5**'s native `reduce` if available. + _.reduce = _.foldl = _.inject = function(obj, iterator, memo, context) { + var initial = memo !== void 0; + if (obj == null) obj = []; + if (nativeReduce && obj.reduce === nativeReduce) { + if (context) iterator = _.bind(iterator, context); + return initial ? obj.reduce(iterator, memo) : obj.reduce(iterator); + } + each(obj, function(value, index, list) { + if (!initial) { + memo = value; + initial = true; + } else { + memo = iterator.call(context, memo, value, index, list); + } + }); + if (!initial) throw new TypeError("Reduce of empty array with no initial value"); + return memo; + }; + + // The right-associative version of reduce, also known as `foldr`. + // Delegates to **ECMAScript 5**'s native `reduceRight` if available. + _.reduceRight = _.foldr = function(obj, iterator, memo, context) { + if (obj == null) obj = []; + if (nativeReduceRight && obj.reduceRight === nativeReduceRight) { + if (context) iterator = _.bind(iterator, context); + return memo !== void 0 ? obj.reduceRight(iterator, memo) : obj.reduceRight(iterator); + } + var reversed = (_.isArray(obj) ? obj.slice() : _.toArray(obj)).reverse(); + return _.reduce(reversed, iterator, memo, context); + }; + + // Return the first value which passes a truth test. Aliased as `detect`. + _.find = _.detect = function(obj, iterator, context) { + var result; + any(obj, function(value, index, list) { + if (iterator.call(context, value, index, list)) { + result = value; + return true; + } + }); + return result; + }; + + // Return all the elements that pass a truth test. + // Delegates to **ECMAScript 5**'s native `filter` if available. + // Aliased as `select`. + _.filter = _.select = function(obj, iterator, context) { + var results = []; + if (obj == null) return results; + if (nativeFilter && obj.filter === nativeFilter) return obj.filter(iterator, context); + each(obj, function(value, index, list) { + if (iterator.call(context, value, index, list)) results[results.length] = value; + }); + return results; + }; + + // Return all the elements for which a truth test fails. + _.reject = function(obj, iterator, context) { + var results = []; + if (obj == null) return results; + each(obj, function(value, index, list) { + if (!iterator.call(context, value, index, list)) results[results.length] = value; + }); + return results; + }; + + // Determine whether all of the elements match a truth test. + // Delegates to **ECMAScript 5**'s native `every` if available. + // Aliased as `all`. + _.every = _.all = function(obj, iterator, context) { + var result = true; + if (obj == null) return result; + if (nativeEvery && obj.every === nativeEvery) return obj.every(iterator, context); + each(obj, function(value, index, list) { + if (!(result = result && iterator.call(context, value, index, list))) return breaker; + }); + return result; + }; + + // Determine if at least one element in the object matches a truth test. + // Delegates to **ECMAScript 5**'s native `some` if available. + // Aliased as `any`. + var any = _.some = _.any = function(obj, iterator, context) { + iterator = iterator || _.identity; + var result = false; + if (obj == null) return result; + if (nativeSome && obj.some === nativeSome) return obj.some(iterator, context); + each(obj, function(value, index, list) { + if (result || (result = iterator.call(context, value, index, list))) return breaker; + }); + return !!result; + }; + + // Determine if a given value is included in the array or object using `===`. + // Aliased as `contains`. + _.include = _.contains = function(obj, target) { + var found = false; + if (obj == null) return found; + if (nativeIndexOf && obj.indexOf === nativeIndexOf) return obj.indexOf(target) != -1; + found = any(obj, function(value) { + return value === target; + }); + return found; + }; + + // Invoke a method (with arguments) on every item in a collection. + _.invoke = function(obj, method) { + var args = slice.call(arguments, 2); + return _.map(obj, function(value) { + return (method.call ? method || value : value[method]).apply(value, args); + }); + }; + + // Convenience version of a common use case of `map`: fetching a property. + _.pluck = function(obj, key) { + return _.map(obj, function(value){ return value[key]; }); + }; + + // Return the maximum element or (element-based computation). + _.max = function(obj, iterator, context) { + if (!iterator && _.isArray(obj)) return Math.max.apply(Math, obj); + if (!iterator && _.isEmpty(obj)) return -Infinity; + var result = {computed : -Infinity}; + each(obj, function(value, index, list) { + var computed = iterator ? iterator.call(context, value, index, list) : value; + computed >= result.computed && (result = {value : value, computed : computed}); + }); + return result.value; + }; + + // Return the minimum element (or element-based computation). + _.min = function(obj, iterator, context) { + if (!iterator && _.isArray(obj)) return Math.min.apply(Math, obj); + if (!iterator && _.isEmpty(obj)) return Infinity; + var result = {computed : Infinity}; + each(obj, function(value, index, list) { + var computed = iterator ? iterator.call(context, value, index, list) : value; + computed < result.computed && (result = {value : value, computed : computed}); + }); + return result.value; + }; + + // Shuffle an array. + _.shuffle = function(obj) { + var shuffled = [], rand; + each(obj, function(value, index, list) { + if (index == 0) { + shuffled[0] = value; + } else { + rand = Math.floor(Math.random() * (index + 1)); + shuffled[index] = shuffled[rand]; + shuffled[rand] = value; + } + }); + return shuffled; + }; + + // Sort the object's values by a criterion produced by an iterator. + _.sortBy = function(obj, iterator, context) { + return _.pluck(_.map(obj, function(value, index, list) { + return { + value : value, + criteria : iterator.call(context, value, index, list) + }; + }).sort(function(left, right) { + var a = left.criteria, b = right.criteria; + return a < b ? -1 : a > b ? 1 : 0; + }), 'value'); + }; + + // Groups the object's values by a criterion. Pass either a string attribute + // to group by, or a function that returns the criterion. + _.groupBy = function(obj, val) { + var result = {}; + var iterator = _.isFunction(val) ? val : function(obj) { return obj[val]; }; + each(obj, function(value, index) { + var key = iterator(value, index); + (result[key] || (result[key] = [])).push(value); + }); + return result; + }; + + // Use a comparator function to figure out at what index an object should + // be inserted so as to maintain order. Uses binary search. + _.sortedIndex = function(array, obj, iterator) { + iterator || (iterator = _.identity); + var low = 0, high = array.length; + while (low < high) { + var mid = (low + high) >> 1; + iterator(array[mid]) < iterator(obj) ? low = mid + 1 : high = mid; + } + return low; + }; + + // Safely convert anything iterable into a real, live array. + _.toArray = function(iterable) { + if (!iterable) return []; + if (iterable.toArray) return iterable.toArray(); + if (_.isArray(iterable)) return slice.call(iterable); + if (_.isArguments(iterable)) return slice.call(iterable); + return _.values(iterable); + }; + + // Return the number of elements in an object. + _.size = function(obj) { + return _.toArray(obj).length; + }; + + // Array Functions + // --------------- + + // Get the first element of an array. Passing **n** will return the first N + // values in the array. Aliased as `head`. The **guard** check allows it to work + // with `_.map`. + _.first = _.head = function(array, n, guard) { + return (n != null) && !guard ? slice.call(array, 0, n) : array[0]; + }; + + // Returns everything but the last entry of the array. Especcialy useful on + // the arguments object. Passing **n** will return all the values in + // the array, excluding the last N. The **guard** check allows it to work with + // `_.map`. + _.initial = function(array, n, guard) { + return slice.call(array, 0, array.length - ((n == null) || guard ? 1 : n)); + }; + + // Get the last element of an array. Passing **n** will return the last N + // values in the array. The **guard** check allows it to work with `_.map`. + _.last = function(array, n, guard) { + if ((n != null) && !guard) { + return slice.call(array, Math.max(array.length - n, 0)); + } else { + return array[array.length - 1]; + } + }; + + // Returns everything but the first entry of the array. Aliased as `tail`. + // Especially useful on the arguments object. Passing an **index** will return + // the rest of the values in the array from that index onward. The **guard** + // check allows it to work with `_.map`. + _.rest = _.tail = function(array, index, guard) { + return slice.call(array, (index == null) || guard ? 1 : index); + }; + + // Trim out all falsy values from an array. + _.compact = function(array) { + return _.filter(array, function(value){ return !!value; }); + }; + + // Return a completely flattened version of an array. + _.flatten = function(array, shallow) { + return _.reduce(array, function(memo, value) { + if (_.isArray(value)) return memo.concat(shallow ? value : _.flatten(value)); + memo[memo.length] = value; + return memo; + }, []); + }; + + // Return a version of the array that does not contain the specified value(s). + _.without = function(array) { + return _.difference(array, slice.call(arguments, 1)); + }; + + // Produce a duplicate-free version of the array. If the array has already + // been sorted, you have the option of using a faster algorithm. + // Aliased as `unique`. + _.uniq = _.unique = function(array, isSorted, iterator) { + var initial = iterator ? _.map(array, iterator) : array; + var result = []; + _.reduce(initial, function(memo, el, i) { + if (0 == i || (isSorted === true ? _.last(memo) != el : !_.include(memo, el))) { + memo[memo.length] = el; + result[result.length] = array[i]; + } + return memo; + }, []); + return result; + }; + + // Produce an array that contains the union: each distinct element from all of + // the passed-in arrays. + _.union = function() { + return _.uniq(_.flatten(arguments, true)); + }; + + // Produce an array that contains every item shared between all the + // passed-in arrays. (Aliased as "intersect" for back-compat.) + _.intersection = _.intersect = function(array) { + var rest = slice.call(arguments, 1); + return _.filter(_.uniq(array), function(item) { + return _.every(rest, function(other) { + return _.indexOf(other, item) >= 0; + }); + }); + }; + + // Take the difference between one array and another. + // Only the elements present in just the first array will remain. + _.difference = function(array, other) { + return _.filter(array, function(value){ return !_.include(other, value); }); + }; + + // Zip together multiple lists into a single array -- elements that share + // an index go together. + _.zip = function() { + var args = slice.call(arguments); + var length = _.max(_.pluck(args, 'length')); + var results = new Array(length); + for (var i = 0; i < length; i++) results[i] = _.pluck(args, "" + i); + return results; + }; + + // If the browser doesn't supply us with indexOf (I'm looking at you, **MSIE**), + // we need this function. Return the position of the first occurrence of an + // item in an array, or -1 if the item is not included in the array. + // Delegates to **ECMAScript 5**'s native `indexOf` if available. + // If the array is large and already in sort order, pass `true` + // for **isSorted** to use binary search. + _.indexOf = function(array, item, isSorted) { + if (array == null) return -1; + var i, l; + if (isSorted) { + i = _.sortedIndex(array, item); + return array[i] === item ? i : -1; + } + if (nativeIndexOf && array.indexOf === nativeIndexOf) return array.indexOf(item); + for (i = 0, l = array.length; i < l; i++) if (array[i] === item) return i; + return -1; + }; + + // Delegates to **ECMAScript 5**'s native `lastIndexOf` if available. + _.lastIndexOf = function(array, item) { + if (array == null) return -1; + if (nativeLastIndexOf && array.lastIndexOf === nativeLastIndexOf) return array.lastIndexOf(item); + var i = array.length; + while (i--) if (array[i] === item) return i; + return -1; + }; + + // Generate an integer Array containing an arithmetic progression. A port of + // the native Python `range()` function. See + // [the Python documentation](http://docs.python.org/library/functions.html#range). + _.range = function(start, stop, step) { + if (arguments.length <= 1) { + stop = start || 0; + start = 0; + } + step = arguments[2] || 1; + + var len = Math.max(Math.ceil((stop - start) / step), 0); + var idx = 0; + var range = new Array(len); + + while(idx < len) { + range[idx++] = start; + start += step; + } + + return range; + }; + + // Function (ahem) Functions + // ------------------ + + // Reusable constructor function for prototype setting. + var ctor = function(){}; + + // Create a function bound to a given object (assigning `this`, and arguments, + // optionally). Binding with arguments is also known as `curry`. + // Delegates to **ECMAScript 5**'s native `Function.bind` if available. + // We check for `func.bind` first, to fail fast when `func` is undefined. + _.bind = function bind(func, context) { + var bound, args; + if (func.bind === nativeBind && nativeBind) return nativeBind.apply(func, slice.call(arguments, 1)); + if (!_.isFunction(func)) throw new TypeError; + args = slice.call(arguments, 2); + return bound = function() { + if (!(this instanceof bound)) return func.apply(context, args.concat(slice.call(arguments))); + ctor.prototype = func.prototype; + var self = new ctor; + var result = func.apply(self, args.concat(slice.call(arguments))); + if (Object(result) === result) return result; + return self; + }; + }; + + // Bind all of an object's methods to that object. Useful for ensuring that + // all callbacks defined on an object belong to it. + _.bindAll = function(obj) { + var funcs = slice.call(arguments, 1); + if (funcs.length == 0) funcs = _.functions(obj); + each(funcs, function(f) { obj[f] = _.bind(obj[f], obj); }); + return obj; + }; + + // Memoize an expensive function by storing its results. + _.memoize = function(func, hasher) { + var memo = {}; + hasher || (hasher = _.identity); + return function() { + var key = hasher.apply(this, arguments); + return hasOwnProperty.call(memo, key) ? memo[key] : (memo[key] = func.apply(this, arguments)); + }; + }; + + // Delays a function for the given number of milliseconds, and then calls + // it with the arguments supplied. + _.delay = function(func, wait) { + var args = slice.call(arguments, 2); + return setTimeout(function(){ return func.apply(func, args); }, wait); + }; + + // Defers a function, scheduling it to run after the current call stack has + // cleared. + _.defer = function(func) { + return _.delay.apply(_, [func, 1].concat(slice.call(arguments, 1))); + }; + + // Returns a function, that, when invoked, will only be triggered at most once + // during a given window of time. + _.throttle = function(func, wait) { + var context, args, timeout, throttling, more; + var whenDone = _.debounce(function(){ more = throttling = false; }, wait); + return function() { + context = this; args = arguments; + var later = function() { + timeout = null; + if (more) func.apply(context, args); + whenDone(); + }; + if (!timeout) timeout = setTimeout(later, wait); + if (throttling) { + more = true; + } else { + func.apply(context, args); + } + whenDone(); + throttling = true; + }; + }; + + // Returns a function, that, as long as it continues to be invoked, will not + // be triggered. The function will be called after it stops being called for + // N milliseconds. + _.debounce = function(func, wait) { + var timeout; + return function() { + var context = this, args = arguments; + var later = function() { + timeout = null; + func.apply(context, args); + }; + clearTimeout(timeout); + timeout = setTimeout(later, wait); + }; + }; + + // Returns a function that will be executed at most one time, no matter how + // often you call it. Useful for lazy initialization. + _.once = function(func) { + var ran = false, memo; + return function() { + if (ran) return memo; + ran = true; + return memo = func.apply(this, arguments); + }; + }; + + // Returns the first function passed as an argument to the second, + // allowing you to adjust arguments, run code before and after, and + // conditionally execute the original function. + _.wrap = function(func, wrapper) { + return function() { + var args = [func].concat(slice.call(arguments)); + return wrapper.apply(this, args); + }; + }; + + // Returns a function that is the composition of a list of functions, each + // consuming the return value of the function that follows. + _.compose = function() { + var funcs = slice.call(arguments); + return function() { + var args = slice.call(arguments); + for (var i = funcs.length - 1; i >= 0; i--) { + args = [funcs[i].apply(this, args)]; + } + return args[0]; + }; + }; + + // Returns a function that will only be executed after being called N times. + _.after = function(times, func) { + if (times <= 0) return func(); + return function() { + if (--times < 1) { return func.apply(this, arguments); } + }; + }; + + // Object Functions + // ---------------- + + // Retrieve the names of an object's properties. + // Delegates to **ECMAScript 5**'s native `Object.keys` + _.keys = nativeKeys || function(obj) { + if (obj !== Object(obj)) throw new TypeError('Invalid object'); + var keys = []; + for (var key in obj) if (hasOwnProperty.call(obj, key)) keys[keys.length] = key; + return keys; + }; + + // Retrieve the values of an object's properties. + _.values = function(obj) { + return _.map(obj, _.identity); + }; + + // Return a sorted list of the function names available on the object. + // Aliased as `methods` + _.functions = _.methods = function(obj) { + var names = []; + for (var key in obj) { + if (_.isFunction(obj[key])) names.push(key); + } + return names.sort(); + }; + + // Extend a given object with all the properties in passed-in object(s). + _.extend = function(obj) { + each(slice.call(arguments, 1), function(source) { + for (var prop in source) { + if (source[prop] !== void 0) obj[prop] = source[prop]; + } + }); + return obj; + }; + + // Fill in a given object with default properties. + _.defaults = function(obj) { + each(slice.call(arguments, 1), function(source) { + for (var prop in source) { + if (obj[prop] == null) obj[prop] = source[prop]; + } + }); + return obj; + }; + + // Create a (shallow-cloned) duplicate of an object. + _.clone = function(obj) { + if (!_.isObject(obj)) return obj; + return _.isArray(obj) ? obj.slice() : _.extend({}, obj); + }; + + // Invokes interceptor with the obj, and then returns obj. + // The primary purpose of this method is to "tap into" a method chain, in + // order to perform operations on intermediate results within the chain. + _.tap = function(obj, interceptor) { + interceptor(obj); + return obj; + }; + + // Internal recursive comparison function. + function eq(a, b, stack) { + // Identical objects are equal. `0 === -0`, but they aren't identical. + // See the Harmony `egal` proposal: http://wiki.ecmascript.org/doku.php?id=harmony:egal. + if (a === b) return a !== 0 || 1 / a == 1 / b; + // A strict comparison is necessary because `null == undefined`. + if (a == null || b == null) return a === b; + // Unwrap any wrapped objects. + if (a._chain) a = a._wrapped; + if (b._chain) b = b._wrapped; + // Invoke a custom `isEqual` method if one is provided. + if (_.isFunction(a.isEqual)) return a.isEqual(b); + if (_.isFunction(b.isEqual)) return b.isEqual(a); + // Compare `[[Class]]` names. + var className = toString.call(a); + if (className != toString.call(b)) return false; + switch (className) { + // Strings, numbers, dates, and booleans are compared by value. + case '[object String]': + // Primitives and their corresponding object wrappers are equivalent; thus, `"5"` is + // equivalent to `new String("5")`. + return String(a) == String(b); + case '[object Number]': + a = +a; + b = +b; + // `NaN`s are equivalent, but non-reflexive. An `egal` comparison is performed for + // other numeric values. + return a != a ? b != b : (a == 0 ? 1 / a == 1 / b : a == b); + case '[object Date]': + case '[object Boolean]': + // Coerce dates and booleans to numeric primitive values. Dates are compared by their + // millisecond representations. Note that invalid dates with millisecond representations + // of `NaN` are not equivalent. + return +a == +b; + // RegExps are compared by their source patterns and flags. + case '[object RegExp]': + return a.source == b.source && + a.global == b.global && + a.multiline == b.multiline && + a.ignoreCase == b.ignoreCase; + } + if (typeof a != 'object' || typeof b != 'object') return false; + // Assume equality for cyclic structures. The algorithm for detecting cyclic + // structures is adapted from ES 5.1 section 15.12.3, abstract operation `JO`. + var length = stack.length; + while (length--) { + // Linear search. Performance is inversely proportional to the number of + // unique nested structures. + if (stack[length] == a) return true; + } + // Add the first object to the stack of traversed objects. + stack.push(a); + var size = 0, result = true; + // Recursively compare objects and arrays. + if (className == '[object Array]') { + // Compare array lengths to determine if a deep comparison is necessary. + size = a.length; + result = size == b.length; + if (result) { + // Deep compare the contents, ignoring non-numeric properties. + while (size--) { + // Ensure commutative equality for sparse arrays. + if (!(result = size in a == size in b && eq(a[size], b[size], stack))) break; + } + } + } else { + // Objects with different constructors are not equivalent. + if ("constructor" in a != "constructor" in b || a.constructor != b.constructor) return false; + // Deep compare objects. + for (var key in a) { + if (hasOwnProperty.call(a, key)) { + // Count the expected number of properties. + size++; + // Deep compare each member. + if (!(result = hasOwnProperty.call(b, key) && eq(a[key], b[key], stack))) break; + } + } + // Ensure that both objects contain the same number of properties. + if (result) { + for (key in b) { + if (hasOwnProperty.call(b, key) && !(size--)) break; + } + result = !size; + } + } + // Remove the first object from the stack of traversed objects. + stack.pop(); + return result; + } + + // Perform a deep comparison to check if two objects are equal. + _.isEqual = function(a, b) { + return eq(a, b, []); + }; + + // Is a given array, string, or object empty? + // An "empty" object has no enumerable own-properties. + _.isEmpty = function(obj) { + if (_.isArray(obj) || _.isString(obj)) return obj.length === 0; + for (var key in obj) if (hasOwnProperty.call(obj, key)) return false; + return true; + }; + + // Is a given value a DOM element? + _.isElement = function(obj) { + return !!(obj && obj.nodeType == 1); + }; + + // Is a given value an array? + // Delegates to ECMA5's native Array.isArray + _.isArray = nativeIsArray || function(obj) { + return toString.call(obj) == '[object Array]'; + }; + + // Is a given variable an object? + _.isObject = function(obj) { + return obj === Object(obj); + }; + + // Is a given variable an arguments object? + if (toString.call(arguments) == '[object Arguments]') { + _.isArguments = function(obj) { + return toString.call(obj) == '[object Arguments]'; + }; + } else { + _.isArguments = function(obj) { + return !!(obj && hasOwnProperty.call(obj, 'callee')); + }; + } + + // Is a given value a function? + _.isFunction = function(obj) { + return toString.call(obj) == '[object Function]'; + }; + + // Is a given value a string? + _.isString = function(obj) { + return toString.call(obj) == '[object String]'; + }; + + // Is a given value a number? + _.isNumber = function(obj) { + return toString.call(obj) == '[object Number]'; + }; + + // Is the given value `NaN`? + _.isNaN = function(obj) { + // `NaN` is the only value for which `===` is not reflexive. + return obj !== obj; + }; + + // Is a given value a boolean? + _.isBoolean = function(obj) { + return obj === true || obj === false || toString.call(obj) == '[object Boolean]'; + }; + + // Is a given value a date? + _.isDate = function(obj) { + return toString.call(obj) == '[object Date]'; + }; + + // Is the given value a regular expression? + _.isRegExp = function(obj) { + return toString.call(obj) == '[object RegExp]'; + }; + + // Is a given value equal to null? + _.isNull = function(obj) { + return obj === null; + }; + + // Is a given variable undefined? + _.isUndefined = function(obj) { + return obj === void 0; + }; + + // Utility Functions + // ----------------- + + // Run Underscore.js in *noConflict* mode, returning the `_` variable to its + // previous owner. Returns a reference to the Underscore object. + _.noConflict = function() { + root._ = previousUnderscore; + return this; + }; + + // Keep the identity function around for default iterators. + _.identity = function(value) { + return value; + }; + + // Run a function **n** times. + _.times = function (n, iterator, context) { + for (var i = 0; i < n; i++) iterator.call(context, i); + }; + + // Escape a string for HTML interpolation. + _.escape = function(string) { + return (''+string).replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"').replace(/'/g, ''').replace(/\//g,'/'); + }; + + // Add your own custom functions to the Underscore object, ensuring that + // they're correctly added to the OOP wrapper as well. + _.mixin = function(obj) { + each(_.functions(obj), function(name){ + addToWrapper(name, _[name] = obj[name]); + }); + }; + + // Generate a unique integer id (unique within the entire client session). + // Useful for temporary DOM ids. + var idCounter = 0; + _.uniqueId = function(prefix) { + var id = idCounter++; + return prefix ? prefix + id : id; + }; + + // By default, Underscore uses ERB-style template delimiters, change the + // following template settings to use alternative delimiters. + _.templateSettings = { + evaluate : /<%([\s\S]+?)%>/g, + interpolate : /<%=([\s\S]+?)%>/g, + escape : /<%-([\s\S]+?)%>/g + }; + + // JavaScript micro-templating, similar to John Resig's implementation. + // Underscore templating handles arbitrary delimiters, preserves whitespace, + // and correctly escapes quotes within interpolated code. + _.template = function(str, data) { + var c = _.templateSettings; + var tmpl = 'var __p=[],print=function(){__p.push.apply(__p,arguments);};' + + 'with(obj||{}){__p.push(\'' + + str.replace(/\\/g, '\\\\') + .replace(/'/g, "\\'") + .replace(c.escape, function(match, code) { + return "',_.escape(" + code.replace(/\\'/g, "'") + "),'"; + }) + .replace(c.interpolate, function(match, code) { + return "'," + code.replace(/\\'/g, "'") + ",'"; + }) + .replace(c.evaluate || null, function(match, code) { + return "');" + code.replace(/\\'/g, "'") + .replace(/[\r\n\t]/g, ' ') + ";__p.push('"; + }) + .replace(/\r/g, '\\r') + .replace(/\n/g, '\\n') + .replace(/\t/g, '\\t') + + "');}return __p.join('');"; + var func = new Function('obj', '_', tmpl); + return data ? func(data, _) : function(data) { return func(data, _) }; + }; + + // The OOP Wrapper + // --------------- + + // If Underscore is called as a function, it returns a wrapped object that + // can be used OO-style. This wrapper holds altered versions of all the + // underscore functions. Wrapped objects may be chained. + var wrapper = function(obj) { this._wrapped = obj; }; + + // Expose `wrapper.prototype` as `_.prototype` + _.prototype = wrapper.prototype; + + // Helper function to continue chaining intermediate results. + var result = function(obj, chain) { + return chain ? _(obj).chain() : obj; + }; + + // A method to easily add functions to the OOP wrapper. + var addToWrapper = function(name, func) { + wrapper.prototype[name] = function() { + var args = slice.call(arguments); + unshift.call(args, this._wrapped); + return result(func.apply(_, args), this._chain); + }; + }; + + // Add all of the Underscore functions to the wrapper object. + _.mixin(_); + + // Add all mutator Array functions to the wrapper. + each(['pop', 'push', 'reverse', 'shift', 'sort', 'splice', 'unshift'], function(name) { + var method = ArrayProto[name]; + wrapper.prototype[name] = function() { + method.apply(this._wrapped, arguments); + return result(this._wrapped, this._chain); + }; + }); + + // Add all accessor Array functions to the wrapper. + each(['concat', 'join', 'slice'], function(name) { + var method = ArrayProto[name]; + wrapper.prototype[name] = function() { + return result(method.apply(this._wrapped, arguments), this._chain); + }; + }); + + // Start chaining a wrapped Underscore object. + wrapper.prototype.chain = function() { + this._chain = true; + return this; + }; + + // Extracts the result from a wrapped and chained object. + wrapper.prototype.value = function() { + return this._wrapped; + }; + +}).call(this); diff --git a/libs/beetsplug/web/templates/index.html b/libs/beetsplug/web/templates/index.html new file mode 100644 index 00000000..7c37c82d --- /dev/null +++ b/libs/beetsplug/web/templates/index.html @@ -0,0 +1,98 @@ +<!DOCTYPE html> +<html> + <head> + <title>beets + + + + + + + + + + + +
+
+ +
+
    +
+
+ +
+
+ +
+
+ + + + + + + diff --git a/libs/beetsplug/zero.py b/libs/beetsplug/zero.py new file mode 100644 index 00000000..d20f7616 --- /dev/null +++ b/libs/beetsplug/zero.py @@ -0,0 +1,140 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Blemjhoo Tezoulbr . +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +""" Clears tag fields in media files.""" + +from __future__ import division, absolute_import, print_function + +import re +from beets.plugins import BeetsPlugin +from beets.mediafile import MediaFile +from beets.importer import action +from beets.util import confit + +__author__ = 'baobab@heresiarch.info' +__version__ = '0.10' + + +class ZeroPlugin(BeetsPlugin): + + _instance = None + + def __init__(self): + super(ZeroPlugin, self).__init__() + + # Listeners. + self.register_listener('write', self.write_event) + self.register_listener('import_task_choice', + self.import_task_choice_event) + + self.config.add({ + 'fields': [], + 'keep_fields': [], + 'update_database': False, + }) + + self.patterns = {} + self.warned = False + + # We'll only handle `fields` or `keep_fields`, but not both. + if self.config['fields'] and self.config['keep_fields']: + self._log.warn(u'cannot blacklist and whitelist at the same time') + + # Blacklist mode. + if self.config['fields']: + self.validate_config('fields') + for field in self.config['fields'].as_str_seq(): + self.set_pattern(field) + + # Whitelist mode. + elif self.config['keep_fields']: + self.validate_config('keep_fields') + + for field in MediaFile.fields(): + if field in self.config['keep_fields'].as_str_seq(): + continue + self.set_pattern(field) + + # These fields should always be preserved. + for key in ('id', 'path', 'album_id'): + if key in self.patterns: + del self.patterns[key] + + def validate_config(self, mode): + """Check whether fields in the configuration are valid. + + `mode` should either be "fields" or "keep_fields", indicating + the section of the configuration to validate. + """ + for field in self.config[mode].as_str_seq(): + if field not in MediaFile.fields(): + self._log.error(u'invalid field: {0}', field) + continue + if mode == 'fields' and field in ('id', 'path', 'album_id'): + self._log.warn(u'field \'{0}\' ignored, zeroing ' + u'it would be dangerous', field) + continue + + def set_pattern(self, field): + """Set a field in `self.patterns` to a string list corresponding to + the configuration, or `True` if the field has no specific + configuration. + """ + try: + self.patterns[field] = self.config[field].as_str_seq() + except confit.NotFoundError: + # Matches everything + self.patterns[field] = True + + def import_task_choice_event(self, session, task): + """Listen for import_task_choice event.""" + if task.choice_flag == action.ASIS and not self.warned: + self._log.warn(u'cannot zero in \"as-is\" mode') + self.warned = True + # TODO request write in as-is mode + + @classmethod + def match_patterns(cls, field, patterns): + """Check if field (as string) is matching any of the patterns in + the list. + """ + if patterns is True: + return True + for p in patterns: + if re.search(p, unicode(field), flags=re.IGNORECASE): + return True + return False + + def write_event(self, item, path, tags): + """Set values in tags to `None` if the key and value are matched + by `self.patterns`. + """ + if not self.patterns: + self._log.warn(u'no fields, nothing to do') + return + + for field, patterns in self.patterns.items(): + if field in tags: + value = tags[field] + match = self.match_patterns(tags[field], patterns) + else: + value = '' + match = patterns is True + + if match: + self._log.debug(u'{0}: {1} -> None', field, value) + tags[field] = None + if self.config['update_database']: + item[field] = None diff --git a/libs/colorama/__init__.py b/libs/colorama/__init__.py new file mode 100644 index 00000000..670e6b39 --- /dev/null +++ b/libs/colorama/__init__.py @@ -0,0 +1,7 @@ +# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file. +from .initialise import init, deinit, reinit, colorama_text +from .ansi import Fore, Back, Style, Cursor +from .ansitowin32 import AnsiToWin32 + +__version__ = '0.3.7' + diff --git a/libs/colorama/ansi.py b/libs/colorama/ansi.py new file mode 100644 index 00000000..78776588 --- /dev/null +++ b/libs/colorama/ansi.py @@ -0,0 +1,102 @@ +# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file. +''' +This module generates ANSI character codes to printing colors to terminals. +See: http://en.wikipedia.org/wiki/ANSI_escape_code +''' + +CSI = '\033[' +OSC = '\033]' +BEL = '\007' + + +def code_to_chars(code): + return CSI + str(code) + 'm' + +def set_title(title): + return OSC + '2;' + title + BEL + +def clear_screen(mode=2): + return CSI + str(mode) + 'J' + +def clear_line(mode=2): + return CSI + str(mode) + 'K' + + +class AnsiCodes(object): + def __init__(self): + # the subclasses declare class attributes which are numbers. + # Upon instantiation we define instance attributes, which are the same + # as the class attributes but wrapped with the ANSI escape sequence + for name in dir(self): + if not name.startswith('_'): + value = getattr(self, name) + setattr(self, name, code_to_chars(value)) + + +class AnsiCursor(object): + def UP(self, n=1): + return CSI + str(n) + 'A' + def DOWN(self, n=1): + return CSI + str(n) + 'B' + def FORWARD(self, n=1): + return CSI + str(n) + 'C' + def BACK(self, n=1): + return CSI + str(n) + 'D' + def POS(self, x=1, y=1): + return CSI + str(y) + ';' + str(x) + 'H' + + +class AnsiFore(AnsiCodes): + BLACK = 30 + RED = 31 + GREEN = 32 + YELLOW = 33 + BLUE = 34 + MAGENTA = 35 + CYAN = 36 + WHITE = 37 + RESET = 39 + + # These are fairly well supported, but not part of the standard. + LIGHTBLACK_EX = 90 + LIGHTRED_EX = 91 + LIGHTGREEN_EX = 92 + LIGHTYELLOW_EX = 93 + LIGHTBLUE_EX = 94 + LIGHTMAGENTA_EX = 95 + LIGHTCYAN_EX = 96 + LIGHTWHITE_EX = 97 + + +class AnsiBack(AnsiCodes): + BLACK = 40 + RED = 41 + GREEN = 42 + YELLOW = 43 + BLUE = 44 + MAGENTA = 45 + CYAN = 46 + WHITE = 47 + RESET = 49 + + # These are fairly well supported, but not part of the standard. + LIGHTBLACK_EX = 100 + LIGHTRED_EX = 101 + LIGHTGREEN_EX = 102 + LIGHTYELLOW_EX = 103 + LIGHTBLUE_EX = 104 + LIGHTMAGENTA_EX = 105 + LIGHTCYAN_EX = 106 + LIGHTWHITE_EX = 107 + + +class AnsiStyle(AnsiCodes): + BRIGHT = 1 + DIM = 2 + NORMAL = 22 + RESET_ALL = 0 + +Fore = AnsiFore() +Back = AnsiBack() +Style = AnsiStyle() +Cursor = AnsiCursor() diff --git a/libs/colorama/ansitowin32.py b/libs/colorama/ansitowin32.py new file mode 100644 index 00000000..b7ff6f21 --- /dev/null +++ b/libs/colorama/ansitowin32.py @@ -0,0 +1,236 @@ +# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file. +import re +import sys +import os + +from .ansi import AnsiFore, AnsiBack, AnsiStyle, Style +from .winterm import WinTerm, WinColor, WinStyle +from .win32 import windll, winapi_test + + +winterm = None +if windll is not None: + winterm = WinTerm() + + +def is_stream_closed(stream): + return not hasattr(stream, 'closed') or stream.closed + + +def is_a_tty(stream): + return hasattr(stream, 'isatty') and stream.isatty() + + +class StreamWrapper(object): + ''' + Wraps a stream (such as stdout), acting as a transparent proxy for all + attribute access apart from method 'write()', which is delegated to our + Converter instance. + ''' + def __init__(self, wrapped, converter): + # double-underscore everything to prevent clashes with names of + # attributes on the wrapped stream object. + self.__wrapped = wrapped + self.__convertor = converter + + def __getattr__(self, name): + return getattr(self.__wrapped, name) + + def write(self, text): + self.__convertor.write(text) + + +class AnsiToWin32(object): + ''' + Implements a 'write()' method which, on Windows, will strip ANSI character + sequences from the text, and if outputting to a tty, will convert them into + win32 function calls. + ''' + ANSI_CSI_RE = re.compile('\001?\033\[((?:\d|;)*)([a-zA-Z])\002?') # Control Sequence Introducer + ANSI_OSC_RE = re.compile('\001?\033\]((?:.|;)*?)(\x07)\002?') # Operating System Command + + def __init__(self, wrapped, convert=None, strip=None, autoreset=False): + # The wrapped stream (normally sys.stdout or sys.stderr) + self.wrapped = wrapped + + # should we reset colors to defaults after every .write() + self.autoreset = autoreset + + # create the proxy wrapping our output stream + self.stream = StreamWrapper(wrapped, self) + + on_windows = os.name == 'nt' + # We test if the WinAPI works, because even if we are on Windows + # we may be using a terminal that doesn't support the WinAPI + # (e.g. Cygwin Terminal). In this case it's up to the terminal + # to support the ANSI codes. + conversion_supported = on_windows and winapi_test() + + # should we strip ANSI sequences from our output? + if strip is None: + strip = conversion_supported or (not is_stream_closed(wrapped) and not is_a_tty(wrapped)) + self.strip = strip + + # should we should convert ANSI sequences into win32 calls? + if convert is None: + convert = conversion_supported and not is_stream_closed(wrapped) and is_a_tty(wrapped) + self.convert = convert + + # dict of ansi codes to win32 functions and parameters + self.win32_calls = self.get_win32_calls() + + # are we wrapping stderr? + self.on_stderr = self.wrapped is sys.stderr + + def should_wrap(self): + ''' + True if this class is actually needed. If false, then the output + stream will not be affected, nor will win32 calls be issued, so + wrapping stdout is not actually required. This will generally be + False on non-Windows platforms, unless optional functionality like + autoreset has been requested using kwargs to init() + ''' + return self.convert or self.strip or self.autoreset + + def get_win32_calls(self): + if self.convert and winterm: + return { + AnsiStyle.RESET_ALL: (winterm.reset_all, ), + AnsiStyle.BRIGHT: (winterm.style, WinStyle.BRIGHT), + AnsiStyle.DIM: (winterm.style, WinStyle.NORMAL), + AnsiStyle.NORMAL: (winterm.style, WinStyle.NORMAL), + AnsiFore.BLACK: (winterm.fore, WinColor.BLACK), + AnsiFore.RED: (winterm.fore, WinColor.RED), + AnsiFore.GREEN: (winterm.fore, WinColor.GREEN), + AnsiFore.YELLOW: (winterm.fore, WinColor.YELLOW), + AnsiFore.BLUE: (winterm.fore, WinColor.BLUE), + AnsiFore.MAGENTA: (winterm.fore, WinColor.MAGENTA), + AnsiFore.CYAN: (winterm.fore, WinColor.CYAN), + AnsiFore.WHITE: (winterm.fore, WinColor.GREY), + AnsiFore.RESET: (winterm.fore, ), + AnsiFore.LIGHTBLACK_EX: (winterm.fore, WinColor.BLACK, True), + AnsiFore.LIGHTRED_EX: (winterm.fore, WinColor.RED, True), + AnsiFore.LIGHTGREEN_EX: (winterm.fore, WinColor.GREEN, True), + AnsiFore.LIGHTYELLOW_EX: (winterm.fore, WinColor.YELLOW, True), + AnsiFore.LIGHTBLUE_EX: (winterm.fore, WinColor.BLUE, True), + AnsiFore.LIGHTMAGENTA_EX: (winterm.fore, WinColor.MAGENTA, True), + AnsiFore.LIGHTCYAN_EX: (winterm.fore, WinColor.CYAN, True), + AnsiFore.LIGHTWHITE_EX: (winterm.fore, WinColor.GREY, True), + AnsiBack.BLACK: (winterm.back, WinColor.BLACK), + AnsiBack.RED: (winterm.back, WinColor.RED), + AnsiBack.GREEN: (winterm.back, WinColor.GREEN), + AnsiBack.YELLOW: (winterm.back, WinColor.YELLOW), + AnsiBack.BLUE: (winterm.back, WinColor.BLUE), + AnsiBack.MAGENTA: (winterm.back, WinColor.MAGENTA), + AnsiBack.CYAN: (winterm.back, WinColor.CYAN), + AnsiBack.WHITE: (winterm.back, WinColor.GREY), + AnsiBack.RESET: (winterm.back, ), + AnsiBack.LIGHTBLACK_EX: (winterm.back, WinColor.BLACK, True), + AnsiBack.LIGHTRED_EX: (winterm.back, WinColor.RED, True), + AnsiBack.LIGHTGREEN_EX: (winterm.back, WinColor.GREEN, True), + AnsiBack.LIGHTYELLOW_EX: (winterm.back, WinColor.YELLOW, True), + AnsiBack.LIGHTBLUE_EX: (winterm.back, WinColor.BLUE, True), + AnsiBack.LIGHTMAGENTA_EX: (winterm.back, WinColor.MAGENTA, True), + AnsiBack.LIGHTCYAN_EX: (winterm.back, WinColor.CYAN, True), + AnsiBack.LIGHTWHITE_EX: (winterm.back, WinColor.GREY, True), + } + return dict() + + def write(self, text): + if self.strip or self.convert: + self.write_and_convert(text) + else: + self.wrapped.write(text) + self.wrapped.flush() + if self.autoreset: + self.reset_all() + + + def reset_all(self): + if self.convert: + self.call_win32('m', (0,)) + elif not self.strip and not is_stream_closed(self.wrapped): + self.wrapped.write(Style.RESET_ALL) + + + def write_and_convert(self, text): + ''' + Write the given text to our wrapped stream, stripping any ANSI + sequences from the text, and optionally converting them into win32 + calls. + ''' + cursor = 0 + text = self.convert_osc(text) + for match in self.ANSI_CSI_RE.finditer(text): + start, end = match.span() + self.write_plain_text(text, cursor, start) + self.convert_ansi(*match.groups()) + cursor = end + self.write_plain_text(text, cursor, len(text)) + + + def write_plain_text(self, text, start, end): + if start < end: + self.wrapped.write(text[start:end]) + self.wrapped.flush() + + + def convert_ansi(self, paramstring, command): + if self.convert: + params = self.extract_params(command, paramstring) + self.call_win32(command, params) + + + def extract_params(self, command, paramstring): + if command in 'Hf': + params = tuple(int(p) if len(p) != 0 else 1 for p in paramstring.split(';')) + while len(params) < 2: + # defaults: + params = params + (1,) + else: + params = tuple(int(p) for p in paramstring.split(';') if len(p) != 0) + if len(params) == 0: + # defaults: + if command in 'JKm': + params = (0,) + elif command in 'ABCD': + params = (1,) + + return params + + + def call_win32(self, command, params): + if command == 'm': + for param in params: + if param in self.win32_calls: + func_args = self.win32_calls[param] + func = func_args[0] + args = func_args[1:] + kwargs = dict(on_stderr=self.on_stderr) + func(*args, **kwargs) + elif command in 'J': + winterm.erase_screen(params[0], on_stderr=self.on_stderr) + elif command in 'K': + winterm.erase_line(params[0], on_stderr=self.on_stderr) + elif command in 'Hf': # cursor position - absolute + winterm.set_cursor_position(params, on_stderr=self.on_stderr) + elif command in 'ABCD': # cursor position - relative + n = params[0] + # A - up, B - down, C - forward, D - back + x, y = {'A': (0, -n), 'B': (0, n), 'C': (n, 0), 'D': (-n, 0)}[command] + winterm.cursor_adjust(x, y, on_stderr=self.on_stderr) + + + def convert_osc(self, text): + for match in self.ANSI_OSC_RE.finditer(text): + start, end = match.span() + text = text[:start] + text[end:] + paramstring, command = match.groups() + if command in '\x07': # \x07 = BEL + params = paramstring.split(";") + # 0 - change title and icon (we will only change title) + # 1 - change icon (we don't support this) + # 2 - change title + if params[0] in '02': + winterm.set_title(params[1]) + return text diff --git a/libs/colorama/initialise.py b/libs/colorama/initialise.py new file mode 100644 index 00000000..834962a3 --- /dev/null +++ b/libs/colorama/initialise.py @@ -0,0 +1,82 @@ +# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file. +import atexit +import contextlib +import sys + +from .ansitowin32 import AnsiToWin32 + + +orig_stdout = None +orig_stderr = None + +wrapped_stdout = None +wrapped_stderr = None + +atexit_done = False + + +def reset_all(): + if AnsiToWin32 is not None: # Issue #74: objects might become None at exit + AnsiToWin32(orig_stdout).reset_all() + + +def init(autoreset=False, convert=None, strip=None, wrap=True): + + if not wrap and any([autoreset, convert, strip]): + raise ValueError('wrap=False conflicts with any other arg=True') + + global wrapped_stdout, wrapped_stderr + global orig_stdout, orig_stderr + + orig_stdout = sys.stdout + orig_stderr = sys.stderr + + if sys.stdout is None: + wrapped_stdout = None + else: + sys.stdout = wrapped_stdout = \ + wrap_stream(orig_stdout, convert, strip, autoreset, wrap) + if sys.stderr is None: + wrapped_stderr = None + else: + sys.stderr = wrapped_stderr = \ + wrap_stream(orig_stderr, convert, strip, autoreset, wrap) + + global atexit_done + if not atexit_done: + atexit.register(reset_all) + atexit_done = True + + +def deinit(): + if orig_stdout is not None: + sys.stdout = orig_stdout + if orig_stderr is not None: + sys.stderr = orig_stderr + + +@contextlib.contextmanager +def colorama_text(*args, **kwargs): + init(*args, **kwargs) + try: + yield + finally: + deinit() + + +def reinit(): + if wrapped_stdout is not None: + sys.stdout = wrapped_stdout + if wrapped_stderr is not None: + sys.stderr = wrapped_stderr + + +def wrap_stream(stream, convert, strip, autoreset, wrap): + if wrap: + wrapper = AnsiToWin32(stream, + convert=convert, strip=strip, autoreset=autoreset) + if wrapper.should_wrap(): + stream = wrapper.stream + return stream + + diff --git a/libs/colorama/win32.py b/libs/colorama/win32.py new file mode 100644 index 00000000..3d1d2f2d --- /dev/null +++ b/libs/colorama/win32.py @@ -0,0 +1,154 @@ +# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file. + +# from winbase.h +STDOUT = -11 +STDERR = -12 + +try: + import ctypes + from ctypes import LibraryLoader + windll = LibraryLoader(ctypes.WinDLL) + from ctypes import wintypes +except (AttributeError, ImportError): + windll = None + SetConsoleTextAttribute = lambda *_: None + winapi_test = lambda *_: None +else: + from ctypes import byref, Structure, c_char, POINTER + + COORD = wintypes._COORD + + class CONSOLE_SCREEN_BUFFER_INFO(Structure): + """struct in wincon.h.""" + _fields_ = [ + ("dwSize", COORD), + ("dwCursorPosition", COORD), + ("wAttributes", wintypes.WORD), + ("srWindow", wintypes.SMALL_RECT), + ("dwMaximumWindowSize", COORD), + ] + def __str__(self): + return '(%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d)' % ( + self.dwSize.Y, self.dwSize.X + , self.dwCursorPosition.Y, self.dwCursorPosition.X + , self.wAttributes + , self.srWindow.Top, self.srWindow.Left, self.srWindow.Bottom, self.srWindow.Right + , self.dwMaximumWindowSize.Y, self.dwMaximumWindowSize.X + ) + + _GetStdHandle = windll.kernel32.GetStdHandle + _GetStdHandle.argtypes = [ + wintypes.DWORD, + ] + _GetStdHandle.restype = wintypes.HANDLE + + _GetConsoleScreenBufferInfo = windll.kernel32.GetConsoleScreenBufferInfo + _GetConsoleScreenBufferInfo.argtypes = [ + wintypes.HANDLE, + POINTER(CONSOLE_SCREEN_BUFFER_INFO), + ] + _GetConsoleScreenBufferInfo.restype = wintypes.BOOL + + _SetConsoleTextAttribute = windll.kernel32.SetConsoleTextAttribute + _SetConsoleTextAttribute.argtypes = [ + wintypes.HANDLE, + wintypes.WORD, + ] + _SetConsoleTextAttribute.restype = wintypes.BOOL + + _SetConsoleCursorPosition = windll.kernel32.SetConsoleCursorPosition + _SetConsoleCursorPosition.argtypes = [ + wintypes.HANDLE, + COORD, + ] + _SetConsoleCursorPosition.restype = wintypes.BOOL + + _FillConsoleOutputCharacterA = windll.kernel32.FillConsoleOutputCharacterA + _FillConsoleOutputCharacterA.argtypes = [ + wintypes.HANDLE, + c_char, + wintypes.DWORD, + COORD, + POINTER(wintypes.DWORD), + ] + _FillConsoleOutputCharacterA.restype = wintypes.BOOL + + _FillConsoleOutputAttribute = windll.kernel32.FillConsoleOutputAttribute + _FillConsoleOutputAttribute.argtypes = [ + wintypes.HANDLE, + wintypes.WORD, + wintypes.DWORD, + COORD, + POINTER(wintypes.DWORD), + ] + _FillConsoleOutputAttribute.restype = wintypes.BOOL + + _SetConsoleTitleW = windll.kernel32.SetConsoleTitleA + _SetConsoleTitleW.argtypes = [ + wintypes.LPCSTR + ] + _SetConsoleTitleW.restype = wintypes.BOOL + + handles = { + STDOUT: _GetStdHandle(STDOUT), + STDERR: _GetStdHandle(STDERR), + } + + def winapi_test(): + handle = handles[STDOUT] + csbi = CONSOLE_SCREEN_BUFFER_INFO() + success = _GetConsoleScreenBufferInfo( + handle, byref(csbi)) + return bool(success) + + def GetConsoleScreenBufferInfo(stream_id=STDOUT): + handle = handles[stream_id] + csbi = CONSOLE_SCREEN_BUFFER_INFO() + success = _GetConsoleScreenBufferInfo( + handle, byref(csbi)) + return csbi + + def SetConsoleTextAttribute(stream_id, attrs): + handle = handles[stream_id] + return _SetConsoleTextAttribute(handle, attrs) + + def SetConsoleCursorPosition(stream_id, position, adjust=True): + position = COORD(*position) + # If the position is out of range, do nothing. + if position.Y <= 0 or position.X <= 0: + return + # Adjust for Windows' SetConsoleCursorPosition: + # 1. being 0-based, while ANSI is 1-based. + # 2. expecting (x,y), while ANSI uses (y,x). + adjusted_position = COORD(position.Y - 1, position.X - 1) + if adjust: + # Adjust for viewport's scroll position + sr = GetConsoleScreenBufferInfo(STDOUT).srWindow + adjusted_position.Y += sr.Top + adjusted_position.X += sr.Left + # Resume normal processing + handle = handles[stream_id] + return _SetConsoleCursorPosition(handle, adjusted_position) + + def FillConsoleOutputCharacter(stream_id, char, length, start): + handle = handles[stream_id] + char = c_char(char.encode()) + length = wintypes.DWORD(length) + num_written = wintypes.DWORD(0) + # Note that this is hard-coded for ANSI (vs wide) bytes. + success = _FillConsoleOutputCharacterA( + handle, char, length, start, byref(num_written)) + return num_written.value + + def FillConsoleOutputAttribute(stream_id, attr, length, start): + ''' FillConsoleOutputAttribute( hConsole, csbi.wAttributes, dwConSize, coordScreen, &cCharsWritten )''' + handle = handles[stream_id] + attribute = wintypes.WORD(attr) + length = wintypes.DWORD(length) + num_written = wintypes.DWORD(0) + # Note that this is hard-coded for ANSI (vs wide) bytes. + return _FillConsoleOutputAttribute( + handle, attribute, length, start, byref(num_written)) + + def SetConsoleTitle(title): + return _SetConsoleTitleW(title) diff --git a/libs/colorama/winterm.py b/libs/colorama/winterm.py new file mode 100644 index 00000000..60309d3c --- /dev/null +++ b/libs/colorama/winterm.py @@ -0,0 +1,162 @@ +# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file. +from . import win32 + + +# from wincon.h +class WinColor(object): + BLACK = 0 + BLUE = 1 + GREEN = 2 + CYAN = 3 + RED = 4 + MAGENTA = 5 + YELLOW = 6 + GREY = 7 + +# from wincon.h +class WinStyle(object): + NORMAL = 0x00 # dim text, dim background + BRIGHT = 0x08 # bright text, dim background + BRIGHT_BACKGROUND = 0x80 # dim text, bright background + +class WinTerm(object): + + def __init__(self): + self._default = win32.GetConsoleScreenBufferInfo(win32.STDOUT).wAttributes + self.set_attrs(self._default) + self._default_fore = self._fore + self._default_back = self._back + self._default_style = self._style + # In order to emulate LIGHT_EX in windows, we borrow the BRIGHT style. + # So that LIGHT_EX colors and BRIGHT style do not clobber each other, + # we track them separately, since LIGHT_EX is overwritten by Fore/Back + # and BRIGHT is overwritten by Style codes. + self._light = 0 + + def get_attrs(self): + return self._fore + self._back * 16 + (self._style | self._light) + + def set_attrs(self, value): + self._fore = value & 7 + self._back = (value >> 4) & 7 + self._style = value & (WinStyle.BRIGHT | WinStyle.BRIGHT_BACKGROUND) + + def reset_all(self, on_stderr=None): + self.set_attrs(self._default) + self.set_console(attrs=self._default) + + def fore(self, fore=None, light=False, on_stderr=False): + if fore is None: + fore = self._default_fore + self._fore = fore + # Emulate LIGHT_EX with BRIGHT Style + if light: + self._light |= WinStyle.BRIGHT + else: + self._light &= ~WinStyle.BRIGHT + self.set_console(on_stderr=on_stderr) + + def back(self, back=None, light=False, on_stderr=False): + if back is None: + back = self._default_back + self._back = back + # Emulate LIGHT_EX with BRIGHT_BACKGROUND Style + if light: + self._light |= WinStyle.BRIGHT_BACKGROUND + else: + self._light &= ~WinStyle.BRIGHT_BACKGROUND + self.set_console(on_stderr=on_stderr) + + def style(self, style=None, on_stderr=False): + if style is None: + style = self._default_style + self._style = style + self.set_console(on_stderr=on_stderr) + + def set_console(self, attrs=None, on_stderr=False): + if attrs is None: + attrs = self.get_attrs() + handle = win32.STDOUT + if on_stderr: + handle = win32.STDERR + win32.SetConsoleTextAttribute(handle, attrs) + + def get_position(self, handle): + position = win32.GetConsoleScreenBufferInfo(handle).dwCursorPosition + # Because Windows coordinates are 0-based, + # and win32.SetConsoleCursorPosition expects 1-based. + position.X += 1 + position.Y += 1 + return position + + def set_cursor_position(self, position=None, on_stderr=False): + if position is None: + # I'm not currently tracking the position, so there is no default. + # position = self.get_position() + return + handle = win32.STDOUT + if on_stderr: + handle = win32.STDERR + win32.SetConsoleCursorPosition(handle, position) + + def cursor_adjust(self, x, y, on_stderr=False): + handle = win32.STDOUT + if on_stderr: + handle = win32.STDERR + position = self.get_position(handle) + adjusted_position = (position.Y + y, position.X + x) + win32.SetConsoleCursorPosition(handle, adjusted_position, adjust=False) + + def erase_screen(self, mode=0, on_stderr=False): + # 0 should clear from the cursor to the end of the screen. + # 1 should clear from the cursor to the beginning of the screen. + # 2 should clear the entire screen, and move cursor to (1,1) + handle = win32.STDOUT + if on_stderr: + handle = win32.STDERR + csbi = win32.GetConsoleScreenBufferInfo(handle) + # get the number of character cells in the current buffer + cells_in_screen = csbi.dwSize.X * csbi.dwSize.Y + # get number of character cells before current cursor position + cells_before_cursor = csbi.dwSize.X * csbi.dwCursorPosition.Y + csbi.dwCursorPosition.X + if mode == 0: + from_coord = csbi.dwCursorPosition + cells_to_erase = cells_in_screen - cells_before_cursor + if mode == 1: + from_coord = win32.COORD(0, 0) + cells_to_erase = cells_before_cursor + elif mode == 2: + from_coord = win32.COORD(0, 0) + cells_to_erase = cells_in_screen + # fill the entire screen with blanks + win32.FillConsoleOutputCharacter(handle, ' ', cells_to_erase, from_coord) + # now set the buffer's attributes accordingly + win32.FillConsoleOutputAttribute(handle, self.get_attrs(), cells_to_erase, from_coord) + if mode == 2: + # put the cursor where needed + win32.SetConsoleCursorPosition(handle, (1, 1)) + + def erase_line(self, mode=0, on_stderr=False): + # 0 should clear from the cursor to the end of the line. + # 1 should clear from the cursor to the beginning of the line. + # 2 should clear the entire line. + handle = win32.STDOUT + if on_stderr: + handle = win32.STDERR + csbi = win32.GetConsoleScreenBufferInfo(handle) + if mode == 0: + from_coord = csbi.dwCursorPosition + cells_to_erase = csbi.dwSize.X - csbi.dwCursorPosition.X + if mode == 1: + from_coord = win32.COORD(0, csbi.dwCursorPosition.Y) + cells_to_erase = csbi.dwCursorPosition.X + elif mode == 2: + from_coord = win32.COORD(0, csbi.dwCursorPosition.Y) + cells_to_erase = csbi.dwSize.X + # fill the entire screen with blanks + win32.FillConsoleOutputCharacter(handle, ' ', cells_to_erase, from_coord) + # now set the buffer's attributes accordingly + win32.FillConsoleOutputAttribute(handle, self.get_attrs(), cells_to_erase, from_coord) + + def set_title(self, title): + win32.SetConsoleTitle(title) diff --git a/libs/enum/LICENSE b/libs/enum/LICENSE new file mode 100644 index 00000000..9003b885 --- /dev/null +++ b/libs/enum/LICENSE @@ -0,0 +1,32 @@ +Copyright (c) 2013, Ethan Furman. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + Redistributions of source code must retain the above + copyright notice, this list of conditions and the + following disclaimer. + + Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials + provided with the distribution. + + Neither the name Ethan Furman nor the names of any + contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/libs/enum/README b/libs/enum/README new file mode 100644 index 00000000..aa2333d8 --- /dev/null +++ b/libs/enum/README @@ -0,0 +1,3 @@ +enum34 is the new Python stdlib enum module available in Python 3.4 +backported for previous versions of Python from 2.4 to 3.3. +tested on 2.6, 2.7, and 3.3+ diff --git a/libs/enum/__init__.py b/libs/enum/__init__.py new file mode 100644 index 00000000..d6ffb3a4 --- /dev/null +++ b/libs/enum/__init__.py @@ -0,0 +1,837 @@ +"""Python Enumerations""" + +import sys as _sys + +__all__ = ['Enum', 'IntEnum', 'unique'] + +version = 1, 1, 6 + +pyver = float('%s.%s' % _sys.version_info[:2]) + +try: + any +except NameError: + def any(iterable): + for element in iterable: + if element: + return True + return False + +try: + from collections import OrderedDict +except ImportError: + OrderedDict = None + +try: + basestring +except NameError: + # In Python 2 basestring is the ancestor of both str and unicode + # in Python 3 it's just str, but was missing in 3.1 + basestring = str + +try: + unicode +except NameError: + # In Python 3 unicode no longer exists (it's just str) + unicode = str + +class _RouteClassAttributeToGetattr(object): + """Route attribute access on a class to __getattr__. + + This is a descriptor, used to define attributes that act differently when + accessed through an instance and through a class. Instance access remains + normal, but access to an attribute through a class will be routed to the + class's __getattr__ method; this is done by raising AttributeError. + + """ + def __init__(self, fget=None): + self.fget = fget + + def __get__(self, instance, ownerclass=None): + if instance is None: + raise AttributeError() + return self.fget(instance) + + def __set__(self, instance, value): + raise AttributeError("can't set attribute") + + def __delete__(self, instance): + raise AttributeError("can't delete attribute") + + +def _is_descriptor(obj): + """Returns True if obj is a descriptor, False otherwise.""" + return ( + hasattr(obj, '__get__') or + hasattr(obj, '__set__') or + hasattr(obj, '__delete__')) + + +def _is_dunder(name): + """Returns True if a __dunder__ name, False otherwise.""" + return (name[:2] == name[-2:] == '__' and + name[2:3] != '_' and + name[-3:-2] != '_' and + len(name) > 4) + + +def _is_sunder(name): + """Returns True if a _sunder_ name, False otherwise.""" + return (name[0] == name[-1] == '_' and + name[1:2] != '_' and + name[-2:-1] != '_' and + len(name) > 2) + + +def _make_class_unpicklable(cls): + """Make the given class un-picklable.""" + def _break_on_call_reduce(self, protocol=None): + raise TypeError('%r cannot be pickled' % self) + cls.__reduce_ex__ = _break_on_call_reduce + cls.__module__ = '' + + +class _EnumDict(dict): + """Track enum member order and ensure member names are not reused. + + EnumMeta will use the names found in self._member_names as the + enumeration member names. + + """ + def __init__(self): + super(_EnumDict, self).__init__() + self._member_names = [] + + def __setitem__(self, key, value): + """Changes anything not dundered or not a descriptor. + + If a descriptor is added with the same name as an enum member, the name + is removed from _member_names (this may leave a hole in the numerical + sequence of values). + + If an enum member name is used twice, an error is raised; duplicate + values are not checked for. + + Single underscore (sunder) names are reserved. + + Note: in 3.x __order__ is simply discarded as a not necessary piece + leftover from 2.x + + """ + if pyver >= 3.0 and key in ('_order_', '__order__'): + return + elif key == '__order__': + key = '_order_' + if _is_sunder(key): + if key != '_order_': + raise ValueError('_names_ are reserved for future Enum use') + elif _is_dunder(key): + pass + elif key in self._member_names: + # descriptor overwriting an enum? + raise TypeError('Attempted to reuse key: %r' % key) + elif not _is_descriptor(value): + if key in self: + # enum overwriting a descriptor? + raise TypeError('Key already defined as: %r' % self[key]) + self._member_names.append(key) + super(_EnumDict, self).__setitem__(key, value) + + +# Dummy value for Enum as EnumMeta explicity checks for it, but of course until +# EnumMeta finishes running the first time the Enum class doesn't exist. This +# is also why there are checks in EnumMeta like `if Enum is not None` +Enum = None + + +class EnumMeta(type): + """Metaclass for Enum""" + @classmethod + def __prepare__(metacls, cls, bases): + return _EnumDict() + + def __new__(metacls, cls, bases, classdict): + # an Enum class is final once enumeration items have been defined; it + # cannot be mixed with other types (int, float, etc.) if it has an + # inherited __new__ unless a new __new__ is defined (or the resulting + # class will fail). + if type(classdict) is dict: + original_dict = classdict + classdict = _EnumDict() + for k, v in original_dict.items(): + classdict[k] = v + + member_type, first_enum = metacls._get_mixins_(bases) + __new__, save_new, use_args = metacls._find_new_(classdict, member_type, + first_enum) + # save enum items into separate mapping so they don't get baked into + # the new class + members = dict((k, classdict[k]) for k in classdict._member_names) + for name in classdict._member_names: + del classdict[name] + + # py2 support for definition order + _order_ = classdict.get('_order_') + if _order_ is None: + if pyver < 3.0: + try: + _order_ = [name for (name, value) in sorted(members.items(), key=lambda item: item[1])] + except TypeError: + _order_ = [name for name in sorted(members.keys())] + else: + _order_ = classdict._member_names + else: + del classdict['_order_'] + if pyver < 3.0: + _order_ = _order_.replace(',', ' ').split() + aliases = [name for name in members if name not in _order_] + _order_ += aliases + + # check for illegal enum names (any others?) + invalid_names = set(members) & set(['mro']) + if invalid_names: + raise ValueError('Invalid enum member name(s): %s' % ( + ', '.join(invalid_names), )) + + # save attributes from super classes so we know if we can take + # the shortcut of storing members in the class dict + base_attributes = set([a for b in bases for a in b.__dict__]) + # create our new Enum type + enum_class = super(EnumMeta, metacls).__new__(metacls, cls, bases, classdict) + enum_class._member_names_ = [] # names in random order + if OrderedDict is not None: + enum_class._member_map_ = OrderedDict() + else: + enum_class._member_map_ = {} # name->value map + enum_class._member_type_ = member_type + + # Reverse value->name map for hashable values. + enum_class._value2member_map_ = {} + + # instantiate them, checking for duplicates as we go + # we instantiate first instead of checking for duplicates first in case + # a custom __new__ is doing something funky with the values -- such as + # auto-numbering ;) + if __new__ is None: + __new__ = enum_class.__new__ + for member_name in _order_: + value = members[member_name] + if not isinstance(value, tuple): + args = (value, ) + else: + args = value + if member_type is tuple: # special case for tuple enums + args = (args, ) # wrap it one more time + if not use_args or not args: + enum_member = __new__(enum_class) + if not hasattr(enum_member, '_value_'): + enum_member._value_ = value + else: + enum_member = __new__(enum_class, *args) + if not hasattr(enum_member, '_value_'): + enum_member._value_ = member_type(*args) + value = enum_member._value_ + enum_member._name_ = member_name + enum_member.__objclass__ = enum_class + enum_member.__init__(*args) + # If another member with the same value was already defined, the + # new member becomes an alias to the existing one. + for name, canonical_member in enum_class._member_map_.items(): + if canonical_member.value == enum_member._value_: + enum_member = canonical_member + break + else: + # Aliases don't appear in member names (only in __members__). + enum_class._member_names_.append(member_name) + # performance boost for any member that would not shadow + # a DynamicClassAttribute (aka _RouteClassAttributeToGetattr) + if member_name not in base_attributes: + setattr(enum_class, member_name, enum_member) + # now add to _member_map_ + enum_class._member_map_[member_name] = enum_member + try: + # This may fail if value is not hashable. We can't add the value + # to the map, and by-value lookups for this value will be + # linear. + enum_class._value2member_map_[value] = enum_member + except TypeError: + pass + + + # If a custom type is mixed into the Enum, and it does not know how + # to pickle itself, pickle.dumps will succeed but pickle.loads will + # fail. Rather than have the error show up later and possibly far + # from the source, sabotage the pickle protocol for this class so + # that pickle.dumps also fails. + # + # However, if the new class implements its own __reduce_ex__, do not + # sabotage -- it's on them to make sure it works correctly. We use + # __reduce_ex__ instead of any of the others as it is preferred by + # pickle over __reduce__, and it handles all pickle protocols. + unpicklable = False + if '__reduce_ex__' not in classdict: + if member_type is not object: + methods = ('__getnewargs_ex__', '__getnewargs__', + '__reduce_ex__', '__reduce__') + if not any(m in member_type.__dict__ for m in methods): + _make_class_unpicklable(enum_class) + unpicklable = True + + + # double check that repr and friends are not the mixin's or various + # things break (such as pickle) + for name in ('__repr__', '__str__', '__format__', '__reduce_ex__'): + class_method = getattr(enum_class, name) + obj_method = getattr(member_type, name, None) + enum_method = getattr(first_enum, name, None) + if name not in classdict and class_method is not enum_method: + if name == '__reduce_ex__' and unpicklable: + continue + setattr(enum_class, name, enum_method) + + # method resolution and int's are not playing nice + # Python's less than 2.6 use __cmp__ + + if pyver < 2.6: + + if issubclass(enum_class, int): + setattr(enum_class, '__cmp__', getattr(int, '__cmp__')) + + elif pyver < 3.0: + + if issubclass(enum_class, int): + for method in ( + '__le__', + '__lt__', + '__gt__', + '__ge__', + '__eq__', + '__ne__', + '__hash__', + ): + setattr(enum_class, method, getattr(int, method)) + + # replace any other __new__ with our own (as long as Enum is not None, + # anyway) -- again, this is to support pickle + if Enum is not None: + # if the user defined their own __new__, save it before it gets + # clobbered in case they subclass later + if save_new: + setattr(enum_class, '__member_new__', enum_class.__dict__['__new__']) + setattr(enum_class, '__new__', Enum.__dict__['__new__']) + return enum_class + + def __bool__(cls): + """ + classes/types should always be True. + """ + return True + + def __call__(cls, value, names=None, module=None, type=None, start=1): + """Either returns an existing member, or creates a new enum class. + + This method is used both when an enum class is given a value to match + to an enumeration member (i.e. Color(3)) and for the functional API + (i.e. Color = Enum('Color', names='red green blue')). + + When used for the functional API: `module`, if set, will be stored in + the new class' __module__ attribute; `type`, if set, will be mixed in + as the first base class. + + Note: if `module` is not set this routine will attempt to discover the + calling module by walking the frame stack; if this is unsuccessful + the resulting class will not be pickleable. + + """ + if names is None: # simple value lookup + return cls.__new__(cls, value) + # otherwise, functional API: we're creating a new Enum type + return cls._create_(value, names, module=module, type=type, start=start) + + def __contains__(cls, member): + return isinstance(member, cls) and member.name in cls._member_map_ + + def __delattr__(cls, attr): + # nicer error message when someone tries to delete an attribute + # (see issue19025). + if attr in cls._member_map_: + raise AttributeError( + "%s: cannot delete Enum member." % cls.__name__) + super(EnumMeta, cls).__delattr__(attr) + + def __dir__(self): + return (['__class__', '__doc__', '__members__', '__module__'] + + self._member_names_) + + @property + def __members__(cls): + """Returns a mapping of member name->value. + + This mapping lists all enum members, including aliases. Note that this + is a copy of the internal mapping. + + """ + return cls._member_map_.copy() + + def __getattr__(cls, name): + """Return the enum member matching `name` + + We use __getattr__ instead of descriptors or inserting into the enum + class' __dict__ in order to support `name` and `value` being both + properties for enum members (which live in the class' __dict__) and + enum members themselves. + + """ + if _is_dunder(name): + raise AttributeError(name) + try: + return cls._member_map_[name] + except KeyError: + raise AttributeError(name) + + def __getitem__(cls, name): + return cls._member_map_[name] + + def __iter__(cls): + return (cls._member_map_[name] for name in cls._member_names_) + + def __reversed__(cls): + return (cls._member_map_[name] for name in reversed(cls._member_names_)) + + def __len__(cls): + return len(cls._member_names_) + + __nonzero__ = __bool__ + + def __repr__(cls): + return "" % cls.__name__ + + def __setattr__(cls, name, value): + """Block attempts to reassign Enum members. + + A simple assignment to the class namespace only changes one of the + several possible ways to get an Enum member from the Enum class, + resulting in an inconsistent Enumeration. + + """ + member_map = cls.__dict__.get('_member_map_', {}) + if name in member_map: + raise AttributeError('Cannot reassign members.') + super(EnumMeta, cls).__setattr__(name, value) + + def _create_(cls, class_name, names=None, module=None, type=None, start=1): + """Convenience method to create a new Enum class. + + `names` can be: + + * A string containing member names, separated either with spaces or + commas. Values are auto-numbered from 1. + * An iterable of member names. Values are auto-numbered from 1. + * An iterable of (member name, value) pairs. + * A mapping of member name -> value. + + """ + if pyver < 3.0: + # if class_name is unicode, attempt a conversion to ASCII + if isinstance(class_name, unicode): + try: + class_name = class_name.encode('ascii') + except UnicodeEncodeError: + raise TypeError('%r is not representable in ASCII' % class_name) + metacls = cls.__class__ + if type is None: + bases = (cls, ) + else: + bases = (type, cls) + classdict = metacls.__prepare__(class_name, bases) + _order_ = [] + + # special processing needed for names? + if isinstance(names, basestring): + names = names.replace(',', ' ').split() + if isinstance(names, (tuple, list)) and isinstance(names[0], basestring): + names = [(e, i+start) for (i, e) in enumerate(names)] + + # Here, names is either an iterable of (name, value) or a mapping. + item = None # in case names is empty + for item in names: + if isinstance(item, basestring): + member_name, member_value = item, names[item] + else: + member_name, member_value = item + classdict[member_name] = member_value + _order_.append(member_name) + # only set _order_ in classdict if name/value was not from a mapping + if not isinstance(item, basestring): + classdict['_order_'] = ' '.join(_order_) + enum_class = metacls.__new__(metacls, class_name, bases, classdict) + + # TODO: replace the frame hack if a blessed way to know the calling + # module is ever developed + if module is None: + try: + module = _sys._getframe(2).f_globals['__name__'] + except (AttributeError, ValueError): + pass + if module is None: + _make_class_unpicklable(enum_class) + else: + enum_class.__module__ = module + + return enum_class + + @staticmethod + def _get_mixins_(bases): + """Returns the type for creating enum members, and the first inherited + enum class. + + bases: the tuple of bases that was given to __new__ + + """ + if not bases or Enum is None: + return object, Enum + + + # double check that we are not subclassing a class with existing + # enumeration members; while we're at it, see if any other data + # type has been mixed in so we can use the correct __new__ + member_type = first_enum = None + for base in bases: + if (base is not Enum and + issubclass(base, Enum) and + base._member_names_): + raise TypeError("Cannot extend enumerations") + # base is now the last base in bases + if not issubclass(base, Enum): + raise TypeError("new enumerations must be created as " + "`ClassName([mixin_type,] enum_type)`") + + # get correct mix-in type (either mix-in type of Enum subclass, or + # first base if last base is Enum) + if not issubclass(bases[0], Enum): + member_type = bases[0] # first data type + first_enum = bases[-1] # enum type + else: + for base in bases[0].__mro__: + # most common: (IntEnum, int, Enum, object) + # possible: (, , + # , , + # ) + if issubclass(base, Enum): + if first_enum is None: + first_enum = base + else: + if member_type is None: + member_type = base + + return member_type, first_enum + + if pyver < 3.0: + @staticmethod + def _find_new_(classdict, member_type, first_enum): + """Returns the __new__ to be used for creating the enum members. + + classdict: the class dictionary given to __new__ + member_type: the data type whose __new__ will be used by default + first_enum: enumeration to check for an overriding __new__ + + """ + # now find the correct __new__, checking to see of one was defined + # by the user; also check earlier enum classes in case a __new__ was + # saved as __member_new__ + __new__ = classdict.get('__new__', None) + if __new__: + return None, True, True # __new__, save_new, use_args + + N__new__ = getattr(None, '__new__') + O__new__ = getattr(object, '__new__') + if Enum is None: + E__new__ = N__new__ + else: + E__new__ = Enum.__dict__['__new__'] + # check all possibles for __member_new__ before falling back to + # __new__ + for method in ('__member_new__', '__new__'): + for possible in (member_type, first_enum): + try: + target = possible.__dict__[method] + except (AttributeError, KeyError): + target = getattr(possible, method, None) + if target not in [ + None, + N__new__, + O__new__, + E__new__, + ]: + if method == '__member_new__': + classdict['__new__'] = target + return None, False, True + if isinstance(target, staticmethod): + target = target.__get__(member_type) + __new__ = target + break + if __new__ is not None: + break + else: + __new__ = object.__new__ + + # if a non-object.__new__ is used then whatever value/tuple was + # assigned to the enum member name will be passed to __new__ and to the + # new enum member's __init__ + if __new__ is object.__new__: + use_args = False + else: + use_args = True + + return __new__, False, use_args + else: + @staticmethod + def _find_new_(classdict, member_type, first_enum): + """Returns the __new__ to be used for creating the enum members. + + classdict: the class dictionary given to __new__ + member_type: the data type whose __new__ will be used by default + first_enum: enumeration to check for an overriding __new__ + + """ + # now find the correct __new__, checking to see of one was defined + # by the user; also check earlier enum classes in case a __new__ was + # saved as __member_new__ + __new__ = classdict.get('__new__', None) + + # should __new__ be saved as __member_new__ later? + save_new = __new__ is not None + + if __new__ is None: + # check all possibles for __member_new__ before falling back to + # __new__ + for method in ('__member_new__', '__new__'): + for possible in (member_type, first_enum): + target = getattr(possible, method, None) + if target not in ( + None, + None.__new__, + object.__new__, + Enum.__new__, + ): + __new__ = target + break + if __new__ is not None: + break + else: + __new__ = object.__new__ + + # if a non-object.__new__ is used then whatever value/tuple was + # assigned to the enum member name will be passed to __new__ and to the + # new enum member's __init__ + if __new__ is object.__new__: + use_args = False + else: + use_args = True + + return __new__, save_new, use_args + + +######################################################## +# In order to support Python 2 and 3 with a single +# codebase we have to create the Enum methods separately +# and then use the `type(name, bases, dict)` method to +# create the class. +######################################################## +temp_enum_dict = {} +temp_enum_dict['__doc__'] = "Generic enumeration.\n\n Derive from this class to define new enumerations.\n\n" + +def __new__(cls, value): + # all enum instances are actually created during class construction + # without calling this method; this method is called by the metaclass' + # __call__ (i.e. Color(3) ), and by pickle + if type(value) is cls: + # For lookups like Color(Color.red) + value = value.value + #return value + # by-value search for a matching enum member + # see if it's in the reverse mapping (for hashable values) + try: + if value in cls._value2member_map_: + return cls._value2member_map_[value] + except TypeError: + # not there, now do long search -- O(n) behavior + for member in cls._member_map_.values(): + if member.value == value: + return member + raise ValueError("%s is not a valid %s" % (value, cls.__name__)) +temp_enum_dict['__new__'] = __new__ +del __new__ + +def __repr__(self): + return "<%s.%s: %r>" % ( + self.__class__.__name__, self._name_, self._value_) +temp_enum_dict['__repr__'] = __repr__ +del __repr__ + +def __str__(self): + return "%s.%s" % (self.__class__.__name__, self._name_) +temp_enum_dict['__str__'] = __str__ +del __str__ + +if pyver >= 3.0: + def __dir__(self): + added_behavior = [ + m + for cls in self.__class__.mro() + for m in cls.__dict__ + if m[0] != '_' and m not in self._member_map_ + ] + return (['__class__', '__doc__', '__module__', ] + added_behavior) + temp_enum_dict['__dir__'] = __dir__ + del __dir__ + +def __format__(self, format_spec): + # mixed-in Enums should use the mixed-in type's __format__, otherwise + # we can get strange results with the Enum name showing up instead of + # the value + + # pure Enum branch + if self._member_type_ is object: + cls = str + val = str(self) + # mix-in branch + else: + cls = self._member_type_ + val = self.value + return cls.__format__(val, format_spec) +temp_enum_dict['__format__'] = __format__ +del __format__ + + +#################################### +# Python's less than 2.6 use __cmp__ + +if pyver < 2.6: + + def __cmp__(self, other): + if type(other) is self.__class__: + if self is other: + return 0 + return -1 + return NotImplemented + raise TypeError("unorderable types: %s() and %s()" % (self.__class__.__name__, other.__class__.__name__)) + temp_enum_dict['__cmp__'] = __cmp__ + del __cmp__ + +else: + + def __le__(self, other): + raise TypeError("unorderable types: %s() <= %s()" % (self.__class__.__name__, other.__class__.__name__)) + temp_enum_dict['__le__'] = __le__ + del __le__ + + def __lt__(self, other): + raise TypeError("unorderable types: %s() < %s()" % (self.__class__.__name__, other.__class__.__name__)) + temp_enum_dict['__lt__'] = __lt__ + del __lt__ + + def __ge__(self, other): + raise TypeError("unorderable types: %s() >= %s()" % (self.__class__.__name__, other.__class__.__name__)) + temp_enum_dict['__ge__'] = __ge__ + del __ge__ + + def __gt__(self, other): + raise TypeError("unorderable types: %s() > %s()" % (self.__class__.__name__, other.__class__.__name__)) + temp_enum_dict['__gt__'] = __gt__ + del __gt__ + + +def __eq__(self, other): + if type(other) is self.__class__: + return self is other + return NotImplemented +temp_enum_dict['__eq__'] = __eq__ +del __eq__ + +def __ne__(self, other): + if type(other) is self.__class__: + return self is not other + return NotImplemented +temp_enum_dict['__ne__'] = __ne__ +del __ne__ + +def __hash__(self): + return hash(self._name_) +temp_enum_dict['__hash__'] = __hash__ +del __hash__ + +def __reduce_ex__(self, proto): + return self.__class__, (self._value_, ) +temp_enum_dict['__reduce_ex__'] = __reduce_ex__ +del __reduce_ex__ + +# _RouteClassAttributeToGetattr is used to provide access to the `name` +# and `value` properties of enum members while keeping some measure of +# protection from modification, while still allowing for an enumeration +# to have members named `name` and `value`. This works because enumeration +# members are not set directly on the enum class -- __getattr__ is +# used to look them up. + +@_RouteClassAttributeToGetattr +def name(self): + return self._name_ +temp_enum_dict['name'] = name +del name + +@_RouteClassAttributeToGetattr +def value(self): + return self._value_ +temp_enum_dict['value'] = value +del value + +@classmethod +def _convert(cls, name, module, filter, source=None): + """ + Create a new Enum subclass that replaces a collection of global constants + """ + # convert all constants from source (or module) that pass filter() to + # a new Enum called name, and export the enum and its members back to + # module; + # also, replace the __reduce_ex__ method so unpickling works in + # previous Python versions + module_globals = vars(_sys.modules[module]) + if source: + source = vars(source) + else: + source = module_globals + members = dict((name, value) for name, value in source.items() if filter(name)) + cls = cls(name, members, module=module) + cls.__reduce_ex__ = _reduce_ex_by_name + module_globals.update(cls.__members__) + module_globals[name] = cls + return cls +temp_enum_dict['_convert'] = _convert +del _convert + +Enum = EnumMeta('Enum', (object, ), temp_enum_dict) +del temp_enum_dict + +# Enum has now been created +########################### + +class IntEnum(int, Enum): + """Enum where members are also (and must be) ints""" + +def _reduce_ex_by_name(self, proto): + return self.name + +def unique(enumeration): + """Class decorator that ensures only unique members exist in an enumeration.""" + duplicates = [] + for name, member in enumeration.__members__.items(): + if name != member.name: + duplicates.append((name, member.name)) + if duplicates: + duplicate_names = ', '.join( + ["%s -> %s" % (alias, name) for (alias, name) in duplicates] + ) + raise ValueError('duplicate names found in %r: %s' % + (enumeration, duplicate_names) + ) + return enumeration diff --git a/libs/jellyfish/__init__.py b/libs/jellyfish/__init__.py new file mode 100644 index 00000000..78345699 --- /dev/null +++ b/libs/jellyfish/__init__.py @@ -0,0 +1,4 @@ +try: + from .cjellyfish import * # noqa +except ImportError: + from ._jellyfish import * # noqa diff --git a/libs/jellyfish/_jellyfish.py b/libs/jellyfish/_jellyfish.py new file mode 100644 index 00000000..a596bb73 --- /dev/null +++ b/libs/jellyfish/_jellyfish.py @@ -0,0 +1,489 @@ +import unicodedata +from collections import defaultdict +from .compat import _range, _zip_longest, _no_bytes_err +from .porter import Stemmer + + +def _normalize(s): + return unicodedata.normalize('NFKD', s) + + +def levenshtein_distance(s1, s2): + if isinstance(s1, bytes) or isinstance(s2, bytes): + raise TypeError(_no_bytes_err) + + if s1 == s2: + return 0 + rows = len(s1)+1 + cols = len(s2)+1 + + if not s1: + return cols-1 + if not s2: + return rows-1 + + prev = None + cur = range(cols) + for r in _range(1, rows): + prev, cur = cur, [r] + [0]*(cols-1) + for c in _range(1, cols): + deletion = prev[c] + 1 + insertion = cur[c-1] + 1 + edit = prev[c-1] + (0 if s1[r-1] == s2[c-1] else 1) + cur[c] = min(edit, deletion, insertion) + + return cur[-1] + + +def _jaro_winkler(ying, yang, long_tolerance, winklerize): + if isinstance(ying, bytes) or isinstance(yang, bytes): + raise TypeError(_no_bytes_err) + + ying_len = len(ying) + yang_len = len(yang) + + if not ying_len or not yang_len: + return 0 + + min_len = max(ying_len, yang_len) + search_range = (min_len // 2) - 1 + if search_range < 0: + search_range = 0 + + ying_flags = [False]*ying_len + yang_flags = [False]*yang_len + + # looking only within search range, count & flag matched pairs + common_chars = 0 + for i, ying_ch in enumerate(ying): + low = i - search_range if i > search_range else 0 + hi = i + search_range if i + search_range < yang_len else yang_len - 1 + for j in _range(low, hi+1): + if not yang_flags[j] and yang[j] == ying_ch: + ying_flags[i] = yang_flags[j] = True + common_chars += 1 + break + + # short circuit if no characters match + if not common_chars: + return 0 + + # count transpositions + k = trans_count = 0 + for i, ying_f in enumerate(ying_flags): + if ying_f: + for j in _range(k, yang_len): + if yang_flags[j]: + k = j + 1 + break + if ying[i] != yang[j]: + trans_count += 1 + trans_count /= 2 + + # adjust for similarities in nonmatched characters + common_chars = float(common_chars) + weight = ((common_chars/ying_len + common_chars/yang_len + + (common_chars-trans_count) / common_chars)) / 3 + + # winkler modification: continue to boost if strings are similar + if winklerize and weight > 0.7 and ying_len > 3 and yang_len > 3: + # adjust for up to first 4 chars in common + j = min(min_len, 4) + i = 0 + while i < j and ying[i] == yang[i] and ying[i]: + i += 1 + if i: + weight += i * 0.1 * (1.0 - weight) + + # optionally adjust for long strings + # after agreeing beginning chars, at least two or more must agree and + # agreed characters must be > half of remaining characters + if (long_tolerance and min_len > 4 and common_chars > i+1 and + 2 * common_chars >= min_len + i): + weight += ((1.0 - weight) * (float(common_chars-i-1) / float(ying_len+yang_len-i*2+2))) + + return weight + + +def damerau_levenshtein_distance(s1, s2): + if isinstance(s1, bytes) or isinstance(s2, bytes): + raise TypeError(_no_bytes_err) + + len1 = len(s1) + len2 = len(s2) + infinite = len1 + len2 + + # character array + da = defaultdict(int) + + # distance matrix + score = [[0]*(len2+2) for x in _range(len1+2)] + + score[0][0] = infinite + for i in _range(0, len1+1): + score[i+1][0] = infinite + score[i+1][1] = i + for i in _range(0, len2+1): + score[0][i+1] = infinite + score[1][i+1] = i + + for i in _range(1, len1+1): + db = 0 + for j in _range(1, len2+1): + i1 = da[s2[j-1]] + j1 = db + cost = 1 + if s1[i-1] == s2[j-1]: + cost = 0 + db = j + + score[i+1][j+1] = min(score[i][j] + cost, + score[i+1][j] + 1, + score[i][j+1] + 1, + score[i1][j1] + (i-i1-1) + 1 + (j-j1-1)) + da[s1[i-1]] = i + + return score[len1+1][len2+1] + + +def jaro_distance(s1, s2): + return _jaro_winkler(s1, s2, False, False) + + +def jaro_winkler(s1, s2, long_tolerance=False): + return _jaro_winkler(s1, s2, long_tolerance, True) + + +def soundex(s): + if not s: + return s + if isinstance(s, bytes): + raise TypeError(_no_bytes_err) + + s = _normalize(s) + + replacements = (('bfpv', '1'), + ('cgjkqsxz', '2'), + ('dt', '3'), + ('l', '4'), + ('mn', '5'), + ('r', '6')) + result = [s[0]] + count = 1 + + # find would-be replacment for first character + for lset, sub in replacements: + if s[0].lower() in lset: + last = sub + break + else: + last = None + + for letter in s[1:]: + for lset, sub in replacements: + if letter.lower() in lset: + if sub != last: + result.append(sub) + count += 1 + last = sub + break + else: + last = None + if count == 4: + break + + result += '0'*(4-count) + return ''.join(result) + + +def hamming_distance(s1, s2): + if isinstance(s1, bytes) or isinstance(s2, bytes): + raise TypeError(_no_bytes_err) + + # ensure length of s1 >= s2 + if len(s2) > len(s1): + s1, s2 = s2, s1 + + # distance is difference in length + differing chars + distance = len(s1) - len(s2) + for i, c in enumerate(s2): + if c != s1[i]: + distance += 1 + + return distance + + +def nysiis(s): + if isinstance(s, bytes): + raise TypeError(_no_bytes_err) + if not s: + return '' + + s = s.upper() + key = [] + + # step 1 - prefixes + if s.startswith('MAC'): + s = 'MCC' + s[3:] + elif s.startswith('KN'): + s = s[1:] + elif s.startswith('K'): + s = 'C' + s[1:] + elif s.startswith(('PH', 'PF')): + s = 'FF' + s[2:] + elif s.startswith('SCH'): + s = 'SSS' + s[3:] + + # step 2 - suffixes + if s.endswith(('IE', 'EE')): + s = s[:-2] + 'Y' + elif s.endswith(('DT', 'RT', 'RD', 'NT', 'ND')): + s = s[:-2] + 'D' + + # step 3 - first character of key comes from name + key.append(s[0]) + + # step 4 - translate remaining chars + i = 1 + len_s = len(s) + while i < len_s: + ch = s[i] + if ch == 'E' and i+1 < len_s and s[i+1] == 'V': + ch = 'AF' + i += 1 + elif ch in 'AEIOU': + ch = 'A' + elif ch == 'Q': + ch = 'G' + elif ch == 'Z': + ch = 'S' + elif ch == 'M': + ch = 'N' + elif ch == 'K': + if i+1 < len(s) and s[i+1] == 'N': + ch = 'N' + else: + ch = 'C' + elif ch == 'S' and s[i+1:i+3] == 'CH': + ch = 'SS' + i += 2 + elif ch == 'P' and i+1 < len(s) and s[i+1] == 'H': + ch = 'F' + i += 1 + elif ch == 'H' and (s[i-1] not in 'AEIOU' or (i+1 < len(s) and s[i+1] not in 'AEIOU')): + if s[i-1] in 'AEIOU': + ch = 'A' + else: + ch = s[i-1] + elif ch == 'W' and s[i-1] in 'AEIOU': + ch = s[i-1] + + if ch[-1] != key[-1][-1]: + key.append(ch) + + i += 1 + + key = ''.join(key) + + # step 5 - remove trailing S + if key.endswith('S') and key != 'S': + key = key[:-1] + + # step 6 - replace AY w/ Y + if key.endswith('AY'): + key = key[:-2] + 'Y' + + # step 7 - remove trailing A + if key.endswith('A') and key != 'A': + key = key[:-1] + + # step 8 was already done + + return key + + +def match_rating_codex(s): + if isinstance(s, bytes): + raise TypeError(_no_bytes_err) + s = s.upper() + codex = [] + + prev = None + for i, c in enumerate(s): + # not a space OR + # starting character & vowel + # or consonant not preceded by same consonant + if (c != ' ' and (i == 0 and c in 'AEIOU') or (c not in 'AEIOU' and c != prev)): + codex.append(c) + + prev = c + + # just use first/last 3 + if len(codex) > 6: + return ''.join(codex[:3]+codex[-3:]) + else: + return ''.join(codex) + + +def match_rating_comparison(s1, s2): + codex1 = match_rating_codex(s1) + codex2 = match_rating_codex(s2) + len1 = len(codex1) + len2 = len(codex2) + res1 = [] + res2 = [] + + # length differs by 3 or more, no result + if abs(len1-len2) >= 3: + return None + + # get minimum rating based on sums of codexes + lensum = len1 + len2 + if lensum <= 4: + min_rating = 5 + elif lensum <= 7: + min_rating = 4 + elif lensum <= 11: + min_rating = 3 + else: + min_rating = 2 + + # strip off common prefixes + for c1, c2 in _zip_longest(codex1, codex2): + if c1 != c2: + if c1: + res1.append(c1) + if c2: + res2.append(c2) + + unmatched_count1 = unmatched_count2 = 0 + for c1, c2 in _zip_longest(reversed(res1), reversed(res2)): + if c1 != c2: + if c1: + unmatched_count1 += 1 + if c2: + unmatched_count2 += 1 + + return (6 - max(unmatched_count1, unmatched_count2)) >= min_rating + + +def metaphone(s): + if isinstance(s, bytes): + raise TypeError(_no_bytes_err) + + result = [] + + s = _normalize(s.lower()) + + # skip first character if s starts with these + if s.startswith(('kn', 'gn', 'pn', 'ac', 'wr', 'ae')): + s = s[1:] + + i = 0 + + while i < len(s): + c = s[i] + next = s[i+1] if i < len(s)-1 else '*****' + nextnext = s[i+2] if i < len(s)-2 else '*****' + + # skip doubles except for cc + if c == next and c != 'c': + i += 1 + continue + + if c in 'aeiou': + if i == 0 or s[i-1] == ' ': + result.append(c) + elif c == 'b': + if (not (i != 0 and s[i-1] == 'm')) or next: + result.append('b') + elif c == 'c': + if next == 'i' and nextnext == 'a' or next == 'h': + result.append('x') + i += 1 + elif next in 'iey': + result.append('s') + i += 1 + else: + result.append('k') + elif c == 'd': + if next == 'g' and nextnext in 'iey': + result.append('j') + i += 2 + else: + result.append('t') + elif c in 'fjlmnr': + result.append(c) + elif c == 'g': + if next in 'iey': + result.append('j') + elif next not in 'hn': + result.append('k') + elif next == 'h' and nextnext and nextnext not in 'aeiou': + i += 1 + elif c == 'h': + if i == 0 or next in 'aeiou' or s[i-1] not in 'aeiou': + result.append('h') + elif c == 'k': + if i == 0 or s[i-1] != 'c': + result.append('k') + elif c == 'p': + if next == 'h': + result.append('f') + i += 1 + else: + result.append('p') + elif c == 'q': + result.append('k') + elif c == 's': + if next == 'h': + result.append('x') + i += 1 + elif next == 'i' and nextnext in 'oa': + result.append('x') + i += 2 + else: + result.append('s') + elif c == 't': + if next == 'i' and nextnext in 'oa': + result.append('x') + elif next == 'h': + result.append('0') + i += 1 + elif next != 'c' or nextnext != 'h': + result.append('t') + elif c == 'v': + result.append('f') + elif c == 'w': + if i == 0 and next == 'h': + i += 1 + next = s[i+1] + if next in 'aeiou': + result.append('w') + elif c == 'x': + if i == 0: + if next == 'h' or (next == 'i' and nextnext in 'oa'): + result.append('x') + else: + result.append('s') + else: + result.append('k') + result.append('s') + elif c == 'y': + if next in 'aeiou': + result.append('y') + elif c == 'z': + result.append('s') + elif c == ' ': + if len(result) > 0 and result[-1] != ' ': + result.append(' ') + + i += 1 + + return ''.join(result).upper() + + +def porter_stem(s): + if isinstance(s, bytes): + raise TypeError(_no_bytes_err) + return Stemmer(s).stem() diff --git a/libs/jellyfish/cjellyfish.pyd b/libs/jellyfish/cjellyfish.pyd new file mode 100644 index 00000000..fb20e5d7 Binary files /dev/null and b/libs/jellyfish/cjellyfish.pyd differ diff --git a/libs/jellyfish/compat.py b/libs/jellyfish/compat.py new file mode 100644 index 00000000..b5e09792 --- /dev/null +++ b/libs/jellyfish/compat.py @@ -0,0 +1,13 @@ +import sys +import itertools + +IS_PY3 = sys.version_info[0] == 3 + +if IS_PY3: + _range = range + _zip_longest = itertools.zip_longest + _no_bytes_err = 'expected str, got bytes' +else: + _range = xrange + _zip_longest = itertools.izip_longest + _no_bytes_err = 'expected unicode, got str' diff --git a/libs/jellyfish/porter.py b/libs/jellyfish/porter.py new file mode 100644 index 00000000..2945b22d --- /dev/null +++ b/libs/jellyfish/porter.py @@ -0,0 +1,218 @@ +from .compat import _range + +_s2_options = { + 'a': ((['a', 't', 'i', 'o', 'n', 'a', 'l'], ['a', 't', 'e']), + (['t', 'i', 'o', 'n', 'a', 'l'], ['t', 'i', 'o', 'n'])), + 'c': ((['e', 'n', 'c', 'i'], ['e', 'n', 'c', 'e']), + (['a', 'n', 'c', 'i'], ['a', 'n', 'c', 'e']),), + 'e': ((['i', 'z', 'e', 'r'], ['i', 'z', 'e']),), + 'l': ((['b', 'l', 'i'], ['b', 'l', 'e']), + (['a', 'l', 'l', 'i'], ['a', 'l']), + (['e', 'n', 't', 'l', 'i'], ['e', 'n', 't']), + (['e', 'l', 'i'], ['e']), + (['o', 'u', 's', 'l', 'i'], ['o', 'u', 's']),), + 'o': ((['i', 'z', 'a', 't', 'i', 'o', 'n'], ['i', 'z', 'e']), + (['a', 't', 'i', 'o', 'n'], ['a', 't', 'e']), + (['a', 't', 'o', 'r'], ['a', 't', 'e']),), + 's': ((['a', 'l', 'i', 's', 'm'], ['a', 'l']), + (['i', 'v', 'e', 'n', 'e', 's', 's'], ['i', 'v', 'e']), + (['f', 'u', 'l', 'n', 'e', 's', 's'], ['f', 'u', 'l']), + (['o', 'u', 's', 'n', 'e', 's', 's'], ['o', 'u', 's']),), + 't': ((['a', 'l', 'i', 't', 'i'], ['a', 'l']), + (['i', 'v', 'i', 't', 'i'], ['i', 'v', 'e']), + (['b', 'i', 'l', 'i', 't', 'i'], ['b', 'l', 'e']),), + 'g': ((['l', 'o', 'g', 'i'], ['l', 'o', 'g']),), +} + + +_s3_options = { + 'e': ((['i', 'c', 'a', 't', 'e'], ['i', 'c']), + (['a', 't', 'i', 'v', 'e'], []), + (['a', 'l', 'i', 'z', 'e'], ['a', 'l']),), + 'i': ((['i', 'c', 'i', 't', 'i'], ['i', 'c']),), + 'l': ((['i', 'c', 'a', 'l'], ['i', 'c']), + (['f', 'u', 'l'], []),), + 's': ((['n', 'e', 's', 's'], []),), +} + +_s4_endings = { + 'a': (['a', 'l'],), + 'c': (['a', 'n', 'c', 'e'], ['e', 'n', 'c', 'e']), + 'e': (['e', 'r'],), + 'i': (['i', 'c'],), + 'l': (['a', 'b', 'l', 'e'], ['i', 'b', 'l', 'e']), + 'n': (['a', 'n', 't'], ['e', 'm', 'e', 'n', 't'], ['m', 'e', 'n', 't'], + ['e', 'n', 't']), + # handle 'o' separately + 's': (['i', 's', 'm'],), + 't': (['a', 't', 'e'], ['i', 't', 'i']), + 'u': (['o', 'u', 's'],), + 'v': (['i', 'v', 'e'],), + 'z': (['i', 'z', 'e'],), +} + + +class Stemmer(object): + def __init__(self, b): + self.b = list(b) + self.k = len(b)-1 + self.j = 0 + + def cons(self, i): + """ True iff b[i] is a consonant """ + if self.b[i] in 'aeiou': + return False + elif self.b[i] == 'y': + return True if i == 0 else not self.cons(i-1) + return True + + def m(self): + n = i = 0 + while True: + if i > self.j: + return n + if not self.cons(i): + break + i += 1 + i += 1 + while True: + while True: + if i > self.j: + return n + if self.cons(i): + break + i += 1 + + i += 1 + n += 1 + + while True: + if i > self.j: + return n + if not self.cons(i): + break + i += 1 + i += 1 + + def vowel_in_stem(self): + """ True iff 0...j contains vowel """ + for i in _range(0, self.j+1): + if not self.cons(i): + return True + return False + + def doublec(self, j): + """ True iff j, j-1 contains double consonant """ + if j < 1 or self.b[j] != self.b[j-1]: + return False + return self.cons(j) + + def cvc(self, i): + """ True iff i-2,i-1,i is consonent-vowel consonant + and if second c isn't w,x, or y. + used to restore e at end of short words like cave, love, hope, crime + """ + if (i < 2 or not self.cons(i) or self.cons(i-1) or not self.cons(i-2) or + self.b[i] in 'wxy'): + return False + return True + + def ends(self, s): + length = len(s) + """ True iff 0...k ends with string s """ + res = (self.b[self.k-length+1:self.k+1] == s) + if res: + self.j = self.k - length + return res + + def setto(self, s): + """ set j+1...k to string s, readjusting k """ + length = len(s) + self.b[self.j+1:self.j+1+length] = s + self.k = self.j + length + + def r(self, s): + if self.m() > 0: + self.setto(s) + + def step1ab(self): + if self.b[self.k] == 's': + if self.ends(['s', 's', 'e', 's']): + self.k -= 2 + elif self.ends(['i', 'e', 's']): + self.setto(['i']) + elif self.b[self.k-1] != 's': + self.k -= 1 + if self.ends(['e', 'e', 'd']): + if self.m() > 0: + self.k -= 1 + elif ((self.ends(['e', 'd']) or self.ends(['i', 'n', 'g'])) and + self.vowel_in_stem()): + self.k = self.j + if self.ends(['a', 't']): + self.setto(['a', 't', 'e']) + elif self.ends(['b', 'l']): + self.setto(['b', 'l', 'e']) + elif self.ends(['i', 'z']): + self.setto(['i', 'z', 'e']) + elif self.doublec(self.k): + self.k -= 1 + if self.b[self.k] in 'lsz': + self.k += 1 + elif self.m() == 1 and self.cvc(self.k): + self.setto(['e']) + + def step1c(self): + """ turn terminal y into i if there's a vowel in stem """ + if self.ends(['y']) and self.vowel_in_stem(): + self.b[self.k] = 'i' + + def step2and3(self): + for end, repl in _s2_options.get(self.b[self.k-1], []): + if self.ends(end): + self.r(repl) + break + + for end, repl in _s3_options.get(self.b[self.k], []): + if self.ends(end): + self.r(repl) + break + + def step4(self): + ch = self.b[self.k-1] + + if ch == 'o': + if not ((self.ends(['i', 'o', 'n']) and self.b[self.j] in 'st') or + self.ends(['o', 'u'])): + return + else: + endings = _s4_endings.get(ch, []) + for end in endings: + if self.ends(end): + break + else: + return + + if self.m() > 1: + self.k = self.j + + def step5(self): + self.j = self.k + if self.b[self.k] == 'e': + a = self.m() + if a > 1 or a == 1 and not self.cvc(self.k-1): + self.k -= 1 + if self.b[self.k] == 'l' and self.doublec(self.k) and self.m() > 1: + self.k -= 1 + + def result(self): + return ''.join(self.b[:self.k+1]) + + def stem(self): + if self.k > 1: + self.step1ab() + self.step1c() + self.step2and3() + self.step4() + self.step5() + return self.result() diff --git a/libs/jellyfish/test.py b/libs/jellyfish/test.py new file mode 100644 index 00000000..72ef9344 --- /dev/null +++ b/libs/jellyfish/test.py @@ -0,0 +1,213 @@ +# -*- coding: utf-8 -*- +import sys +if sys.version_info[0] < 3: + import unicodecsv as csv + open_kwargs = {} +else: + import csv + open_kwargs = {'encoding': 'utf8'} +import platform +import pytest + + +def assertAlmostEqual(a, b, places=3): + assert abs(a - b) < (0.1**places) + + +if platform.python_implementation() == 'CPython': + implementations = ['python', 'c'] +else: + implementations = ['python'] + + +@pytest.fixture(params=implementations) +def jf(request): + if request.param == 'python': + from jellyfish import _jellyfish as jf + else: + from jellyfish import cjellyfish as jf + return jf + + +def _load_data(name): + with open('testdata/{}.csv'.format(name), **open_kwargs) as f: + for data in csv.reader(f): + yield data + + +@pytest.mark.parametrize("s1,s2,value", _load_data('jaro_winkler'), ids=str) +def test_jaro_winkler(jf, s1, s2, value): + value = float(value) + assertAlmostEqual(jf.jaro_winkler(s1, s2), value, places=3) + + +@pytest.mark.parametrize("s1,s2,value", _load_data('jaro_distance'), ids=str) +def test_jaro_distance(jf, s1, s2, value): + value = float(value) + assertAlmostEqual(jf.jaro_distance(s1, s2), value, places=3) + + +@pytest.mark.parametrize("s1,s2,value", _load_data('hamming'), ids=str) +def test_hamming_distance(jf, s1, s2, value): + value = int(value) + assert jf.hamming_distance(s1, s2) == value + + +@pytest.mark.parametrize("s1,s2,value", _load_data('levenshtein'), ids=str) +def test_levenshtein_distance(jf, s1, s2, value): + value = int(value) + assert jf.levenshtein_distance(s1, s2) == value + + +@pytest.mark.parametrize("s1,s2,value", _load_data('damerau_levenshtein'), ids=str) +def test_damerau_levenshtein_distance(jf, s1, s2, value): + value = int(value) + assert jf.damerau_levenshtein_distance(s1, s2) == value + + +@pytest.mark.parametrize("s1,code", _load_data('soundex'), ids=str) +def test_soundex(jf, s1, code): + assert jf.soundex(s1) == code + + +@pytest.mark.parametrize("s1,code", _load_data('metaphone'), ids=str) +def test_metaphone(jf, s1, code): + assert jf.metaphone(s1) == code + + +@pytest.mark.parametrize("s1,s2", _load_data('nysiis'), ids=str) +def test_nysiis(jf, s1, s2): + assert jf.nysiis(s1) == s2 + + +@pytest.mark.parametrize("s1,s2", _load_data('match_rating_codex'), ids=str) +def test_match_rating_codex(jf, s1, s2): + assert jf.match_rating_codex(s1) == s2 + + +@pytest.mark.parametrize("s1,s2,value", _load_data('match_rating_comparison'), ids=str) +def test_match_rating_comparison(jf, s1, s2, value): + value = {'True': True, 'False': False, 'None': None}[value] + assert jf.match_rating_comparison(s1, s2) is value + + +# use non-parameterized version for speed +# @pytest.mark.parametrize("a,b", _load_data('porter'), ids=str) +# def test_porter_stem(jf, a, b): +# assert jf.porter_stem(a) == b + +def test_porter_stem(jf): + with open('testdata/porter.csv', **open_kwargs) as f: + reader = csv.reader(f) + for (a, b) in reader: + assert jf.porter_stem(a) == b + + +if platform.python_implementation() == 'CPython': + def test_match_rating_comparison_segfault(): + import hashlib + from jellyfish import cjellyfish as jf + sha1s = [u'{}'.format(hashlib.sha1(str(v).encode('ascii')).hexdigest()) + for v in range(100)] + # this segfaulted on 0.1.2 + assert [[jf.match_rating_comparison(h1, h2) for h1 in sha1s] for h2 in sha1s] + + + def test_damerau_levenshtein_unicode_segfault(): + # unfortunate difference in behavior between Py & C versions + from jellyfish.cjellyfish import damerau_levenshtein_distance as c_dl + from jellyfish._jellyfish import damerau_levenshtein_distance as py_dl + s1 = u'mylifeoutdoors' + s2 = u'нахлыст' + with pytest.raises(ValueError): + c_dl(s1, s2) + with pytest.raises(ValueError): + c_dl(s2, s1) + + assert py_dl(s1, s2) == 14 + assert py_dl(s2, s1) == 14 + + +def test_jaro_winkler_long_tolerance(jf): + no_lt = jf.jaro_winkler(u'two long strings', u'two long stringz', long_tolerance=False) + with_lt = jf.jaro_winkler(u'two long strings', u'two long stringz', long_tolerance=True) + # make sure long_tolerance does something + assertAlmostEqual(no_lt, 0.975) + assertAlmostEqual(with_lt, 0.984) + + +def test_damerau_levenshtein_distance_type(jf): + jf.damerau_levenshtein_distance(u'abc', u'abc') + with pytest.raises(TypeError) as exc: + jf.damerau_levenshtein_distance(b'abc', b'abc') + assert 'expected' in str(exc.value) + + +def test_levenshtein_distance_type(jf): + assert jf.levenshtein_distance(u'abc', u'abc') == 0 + with pytest.raises(TypeError) as exc: + jf.levenshtein_distance(b'abc', b'abc') + assert 'expected' in str(exc.value) + + +def test_jaro_distance_type(jf): + assert jf.jaro_distance(u'abc', u'abc') == 1 + with pytest.raises(TypeError) as exc: + jf.jaro_distance(b'abc', b'abc') + assert 'expected' in str(exc.value) + + +def test_jaro_winkler_type(jf): + assert jf.jaro_winkler(u'abc', u'abc') == 1 + with pytest.raises(TypeError) as exc: + jf.jaro_winkler(b'abc', b'abc') + assert 'expected' in str(exc.value) + + +def test_mra_comparison_type(jf): + assert jf.match_rating_comparison(u'abc', u'abc') is True + with pytest.raises(TypeError) as exc: + jf.match_rating_comparison(b'abc', b'abc') + assert 'expected' in str(exc.value) + + +def test_hamming_type(jf): + assert jf.hamming_distance(u'abc', u'abc') == 0 + with pytest.raises(TypeError) as exc: + jf.hamming_distance(b'abc', b'abc') + assert 'expected' in str(exc.value) + + +def test_soundex_type(jf): + assert jf.soundex(u'ABC') == 'A120' + with pytest.raises(TypeError) as exc: + jf.soundex(b'ABC') + assert 'expected' in str(exc.value) + + +def test_metaphone_type(jf): + assert jf.metaphone(u'abc') == 'ABK' + with pytest.raises(TypeError) as exc: + jf.metaphone(b'abc') + assert 'expected' in str(exc.value) + + +def test_nysiis_type(jf): + assert jf.nysiis(u'abc') == 'ABC' + with pytest.raises(TypeError) as exc: + jf.nysiis(b'abc') + assert 'expected' in str(exc.value) + + +def test_mr_codex_type(jf): + assert jf.match_rating_codex(u'abc') == 'ABC' + with pytest.raises(TypeError) as exc: + jf.match_rating_codex(b'abc') + assert 'expected' in str(exc.value) + + +def test_porter_type(jf): + assert jf.porter_stem(u'abc') == 'abc' + with pytest.raises(TypeError) as exc: + jf.porter_stem(b'abc') + assert 'expected' in str(exc.value) diff --git a/libs/munkres.py b/libs/munkres.py new file mode 100644 index 00000000..187333b3 --- /dev/null +++ b/libs/munkres.py @@ -0,0 +1,786 @@ +#!/usr/bin/env python +# -*- coding: iso-8859-1 -*- + +# Documentation is intended to be processed by Epydoc. + +""" +Introduction +============ + +The Munkres module provides an implementation of the Munkres algorithm +(also called the Hungarian algorithm or the Kuhn-Munkres algorithm), +useful for solving the Assignment Problem. + +Assignment Problem +================== + +Let *C* be an *n*\ x\ *n* matrix representing the costs of each of *n* workers +to perform any of *n* jobs. The assignment problem is to assign jobs to +workers in a way that minimizes the total cost. Since each worker can perform +only one job and each job can be assigned to only one worker the assignments +represent an independent set of the matrix *C*. + +One way to generate the optimal set is to create all permutations of +the indexes necessary to traverse the matrix so that no row and column +are used more than once. For instance, given this matrix (expressed in +Python):: + + matrix = [[5, 9, 1], + [10, 3, 2], + [8, 7, 4]] + +You could use this code to generate the traversal indexes:: + + def permute(a, results): + if len(a) == 1: + results.insert(len(results), a) + + else: + for i in range(0, len(a)): + element = a[i] + a_copy = [a[j] for j in range(0, len(a)) if j != i] + subresults = [] + permute(a_copy, subresults) + for subresult in subresults: + result = [element] + subresult + results.insert(len(results), result) + + results = [] + permute(range(len(matrix)), results) # [0, 1, 2] for a 3x3 matrix + +After the call to permute(), the results matrix would look like this:: + + [[0, 1, 2], + [0, 2, 1], + [1, 0, 2], + [1, 2, 0], + [2, 0, 1], + [2, 1, 0]] + +You could then use that index matrix to loop over the original cost matrix +and calculate the smallest cost of the combinations:: + + n = len(matrix) + minval = sys.maxsize + for row in range(n): + cost = 0 + for col in range(n): + cost += matrix[row][col] + minval = min(cost, minval) + + print minval + +While this approach works fine for small matrices, it does not scale. It +executes in O(*n*!) time: Calculating the permutations for an *n*\ x\ *n* +matrix requires *n*! operations. For a 12x12 matrix, that's 479,001,600 +traversals. Even if you could manage to perform each traversal in just one +millisecond, it would still take more than 133 hours to perform the entire +traversal. A 20x20 matrix would take 2,432,902,008,176,640,000 operations. At +an optimistic millisecond per operation, that's more than 77 million years. + +The Munkres algorithm runs in O(*n*\ ^3) time, rather than O(*n*!). This +package provides an implementation of that algorithm. + +This version is based on +http://www.public.iastate.edu/~ddoty/HungarianAlgorithm.html. + +This version was written for Python by Brian Clapper from the (Ada) algorithm +at the above web site. (The ``Algorithm::Munkres`` Perl version, in CPAN, was +clearly adapted from the same web site.) + +Usage +===== + +Construct a Munkres object:: + + from munkres import Munkres + + m = Munkres() + +Then use it to compute the lowest cost assignment from a cost matrix. Here's +a sample program:: + + from munkres import Munkres, print_matrix + + matrix = [[5, 9, 1], + [10, 3, 2], + [8, 7, 4]] + m = Munkres() + indexes = m.compute(matrix) + print_matrix(matrix, msg='Lowest cost through this matrix:') + total = 0 + for row, column in indexes: + value = matrix[row][column] + total += value + print '(%d, %d) -> %d' % (row, column, value) + print 'total cost: %d' % total + +Running that program produces:: + + Lowest cost through this matrix: + [5, 9, 1] + [10, 3, 2] + [8, 7, 4] + (0, 0) -> 5 + (1, 1) -> 3 + (2, 2) -> 4 + total cost=12 + +The instantiated Munkres object can be used multiple times on different +matrices. + +Non-square Cost Matrices +======================== + +The Munkres algorithm assumes that the cost matrix is square. However, it's +possible to use a rectangular matrix if you first pad it with 0 values to make +it square. This module automatically pads rectangular cost matrices to make +them square. + +Notes: + +- The module operates on a *copy* of the caller's matrix, so any padding will + not be seen by the caller. +- The cost matrix must be rectangular or square. An irregular matrix will + *not* work. + +Calculating Profit, Rather than Cost +==================================== + +The cost matrix is just that: A cost matrix. The Munkres algorithm finds +the combination of elements (one from each row and column) that results in +the smallest cost. It's also possible to use the algorithm to maximize +profit. To do that, however, you have to convert your profit matrix to a +cost matrix. The simplest way to do that is to subtract all elements from a +large value. For example:: + + from munkres import Munkres, print_matrix + + matrix = [[5, 9, 1], + [10, 3, 2], + [8, 7, 4]] + cost_matrix = [] + for row in matrix: + cost_row = [] + for col in row: + cost_row += [sys.maxsize - col] + cost_matrix += [cost_row] + + m = Munkres() + indexes = m.compute(cost_matrix) + print_matrix(matrix, msg='Highest profit through this matrix:') + total = 0 + for row, column in indexes: + value = matrix[row][column] + total += value + print '(%d, %d) -> %d' % (row, column, value) + + print 'total profit=%d' % total + +Running that program produces:: + + Highest profit through this matrix: + [5, 9, 1] + [10, 3, 2] + [8, 7, 4] + (0, 1) -> 9 + (1, 0) -> 10 + (2, 2) -> 4 + total profit=23 + +The ``munkres`` module provides a convenience method for creating a cost +matrix from a profit matrix. Since it doesn't know whether the matrix contains +floating point numbers, decimals, or integers, you have to provide the +conversion function; but the convenience method takes care of the actual +creation of the cost matrix:: + + import munkres + + cost_matrix = munkres.make_cost_matrix(matrix, + lambda cost: sys.maxsize - cost) + +So, the above profit-calculation program can be recast as:: + + from munkres import Munkres, print_matrix, make_cost_matrix + + matrix = [[5, 9, 1], + [10, 3, 2], + [8, 7, 4]] + cost_matrix = make_cost_matrix(matrix, lambda cost: sys.maxsize - cost) + m = Munkres() + indexes = m.compute(cost_matrix) + print_matrix(matrix, msg='Lowest cost through this matrix:') + total = 0 + for row, column in indexes: + value = matrix[row][column] + total += value + print '(%d, %d) -> %d' % (row, column, value) + print 'total profit=%d' % total + +References +========== + +1. http://www.public.iastate.edu/~ddoty/HungarianAlgorithm.html + +2. Harold W. Kuhn. The Hungarian Method for the assignment problem. + *Naval Research Logistics Quarterly*, 2:83-97, 1955. + +3. Harold W. Kuhn. Variants of the Hungarian method for assignment + problems. *Naval Research Logistics Quarterly*, 3: 253-258, 1956. + +4. Munkres, J. Algorithms for the Assignment and Transportation Problems. + *Journal of the Society of Industrial and Applied Mathematics*, + 5(1):32-38, March, 1957. + +5. http://en.wikipedia.org/wiki/Hungarian_algorithm + +Copyright and License +===================== + +This software is released under a BSD license, adapted from + + +Copyright (c) 2008 Brian M. Clapper +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name "clapper.org" nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +""" + +__docformat__ = 'restructuredtext' + +# --------------------------------------------------------------------------- +# Imports +# --------------------------------------------------------------------------- + +import sys +import copy + +# --------------------------------------------------------------------------- +# Exports +# --------------------------------------------------------------------------- + +__all__ = ['Munkres', 'make_cost_matrix'] + +# --------------------------------------------------------------------------- +# Globals +# --------------------------------------------------------------------------- + +# Info about the module +__version__ = "1.0.7" +__author__ = "Brian Clapper, bmc@clapper.org" +__url__ = "http://software.clapper.org/munkres/" +__copyright__ = "(c) 2008 Brian M. Clapper" +__license__ = "BSD-style license" + +# --------------------------------------------------------------------------- +# Classes +# --------------------------------------------------------------------------- + +class Munkres: + """ + Calculate the Munkres solution to the classical assignment problem. + See the module documentation for usage. + """ + + def __init__(self): + """Create a new instance""" + self.C = None + self.row_covered = [] + self.col_covered = [] + self.n = 0 + self.Z0_r = 0 + self.Z0_c = 0 + self.marked = None + self.path = None + + def make_cost_matrix(profit_matrix, inversion_function): + """ + **DEPRECATED** + + Please use the module function ``make_cost_matrix()``. + """ + import munkres + return munkres.make_cost_matrix(profit_matrix, inversion_function) + + make_cost_matrix = staticmethod(make_cost_matrix) + + def pad_matrix(self, matrix, pad_value=0): + """ + Pad a possibly non-square matrix to make it square. + + :Parameters: + matrix : list of lists + matrix to pad + + pad_value : int + value to use to pad the matrix + + :rtype: list of lists + :return: a new, possibly padded, matrix + """ + max_columns = 0 + total_rows = len(matrix) + + for row in matrix: + max_columns = max(max_columns, len(row)) + + total_rows = max(max_columns, total_rows) + + new_matrix = [] + for row in matrix: + row_len = len(row) + new_row = row[:] + if total_rows > row_len: + # Row too short. Pad it. + new_row += [pad_value] * (total_rows - row_len) + new_matrix += [new_row] + + while len(new_matrix) < total_rows: + new_matrix += [[pad_value] * total_rows] + + return new_matrix + + def compute(self, cost_matrix): + """ + Compute the indexes for the lowest-cost pairings between rows and + columns in the database. Returns a list of (row, column) tuples + that can be used to traverse the matrix. + + :Parameters: + cost_matrix : list of lists + The cost matrix. If this cost matrix is not square, it + will be padded with zeros, via a call to ``pad_matrix()``. + (This method does *not* modify the caller's matrix. It + operates on a copy of the matrix.) + + **WARNING**: This code handles square and rectangular + matrices. It does *not* handle irregular matrices. + + :rtype: list + :return: A list of ``(row, column)`` tuples that describe the lowest + cost path through the matrix + + """ + self.C = self.pad_matrix(cost_matrix) + self.n = len(self.C) + self.original_length = len(cost_matrix) + self.original_width = len(cost_matrix[0]) + self.row_covered = [False for i in range(self.n)] + self.col_covered = [False for i in range(self.n)] + self.Z0_r = 0 + self.Z0_c = 0 + self.path = self.__make_matrix(self.n * 2, 0) + self.marked = self.__make_matrix(self.n, 0) + + done = False + step = 1 + + steps = { 1 : self.__step1, + 2 : self.__step2, + 3 : self.__step3, + 4 : self.__step4, + 5 : self.__step5, + 6 : self.__step6 } + + while not done: + try: + func = steps[step] + step = func() + except KeyError: + done = True + + # Look for the starred columns + results = [] + for i in range(self.original_length): + for j in range(self.original_width): + if self.marked[i][j] == 1: + results += [(i, j)] + + return results + + def __copy_matrix(self, matrix): + """Return an exact copy of the supplied matrix""" + return copy.deepcopy(matrix) + + def __make_matrix(self, n, val): + """Create an *n*x*n* matrix, populating it with the specific value.""" + matrix = [] + for i in range(n): + matrix += [[val for j in range(n)]] + return matrix + + def __step1(self): + """ + For each row of the matrix, find the smallest element and + subtract it from every element in its row. Go to Step 2. + """ + C = self.C + n = self.n + for i in range(n): + minval = min(self.C[i]) + # Find the minimum value for this row and subtract that minimum + # from every element in the row. + for j in range(n): + self.C[i][j] -= minval + + return 2 + + def __step2(self): + """ + Find a zero (Z) in the resulting matrix. If there is no starred + zero in its row or column, star Z. Repeat for each element in the + matrix. Go to Step 3. + """ + n = self.n + for i in range(n): + for j in range(n): + if (self.C[i][j] == 0) and \ + (not self.col_covered[j]) and \ + (not self.row_covered[i]): + self.marked[i][j] = 1 + self.col_covered[j] = True + self.row_covered[i] = True + + self.__clear_covers() + return 3 + + def __step3(self): + """ + Cover each column containing a starred zero. If K columns are + covered, the starred zeros describe a complete set of unique + assignments. In this case, Go to DONE, otherwise, Go to Step 4. + """ + n = self.n + count = 0 + for i in range(n): + for j in range(n): + if self.marked[i][j] == 1: + self.col_covered[j] = True + count += 1 + + if count >= n: + step = 7 # done + else: + step = 4 + + return step + + def __step4(self): + """ + Find a noncovered zero and prime it. If there is no starred zero + in the row containing this primed zero, Go to Step 5. Otherwise, + cover this row and uncover the column containing the starred + zero. Continue in this manner until there are no uncovered zeros + left. Save the smallest uncovered value and Go to Step 6. + """ + step = 0 + done = False + row = -1 + col = -1 + star_col = -1 + while not done: + (row, col) = self.__find_a_zero() + if row < 0: + done = True + step = 6 + else: + self.marked[row][col] = 2 + star_col = self.__find_star_in_row(row) + if star_col >= 0: + col = star_col + self.row_covered[row] = True + self.col_covered[col] = False + else: + done = True + self.Z0_r = row + self.Z0_c = col + step = 5 + + return step + + def __step5(self): + """ + Construct a series of alternating primed and starred zeros as + follows. Let Z0 represent the uncovered primed zero found in Step 4. + Let Z1 denote the starred zero in the column of Z0 (if any). + Let Z2 denote the primed zero in the row of Z1 (there will always + be one). Continue until the series terminates at a primed zero + that has no starred zero in its column. Unstar each starred zero + of the series, star each primed zero of the series, erase all + primes and uncover every line in the matrix. Return to Step 3 + """ + count = 0 + path = self.path + path[count][0] = self.Z0_r + path[count][1] = self.Z0_c + done = False + while not done: + row = self.__find_star_in_col(path[count][1]) + if row >= 0: + count += 1 + path[count][0] = row + path[count][1] = path[count-1][1] + else: + done = True + + if not done: + col = self.__find_prime_in_row(path[count][0]) + count += 1 + path[count][0] = path[count-1][0] + path[count][1] = col + + self.__convert_path(path, count) + self.__clear_covers() + self.__erase_primes() + return 3 + + def __step6(self): + """ + Add the value found in Step 4 to every element of each covered + row, and subtract it from every element of each uncovered column. + Return to Step 4 without altering any stars, primes, or covered + lines. + """ + minval = self.__find_smallest() + for i in range(self.n): + for j in range(self.n): + if self.row_covered[i]: + self.C[i][j] += minval + if not self.col_covered[j]: + self.C[i][j] -= minval + return 4 + + def __find_smallest(self): + """Find the smallest uncovered value in the matrix.""" + minval = sys.maxsize + for i in range(self.n): + for j in range(self.n): + if (not self.row_covered[i]) and (not self.col_covered[j]): + if minval > self.C[i][j]: + minval = self.C[i][j] + return minval + + def __find_a_zero(self): + """Find the first uncovered element with value 0""" + row = -1 + col = -1 + i = 0 + n = self.n + done = False + + while not done: + j = 0 + while True: + if (self.C[i][j] == 0) and \ + (not self.row_covered[i]) and \ + (not self.col_covered[j]): + row = i + col = j + done = True + j += 1 + if j >= n: + break + i += 1 + if i >= n: + done = True + + return (row, col) + + def __find_star_in_row(self, row): + """ + Find the first starred element in the specified row. Returns + the column index, or -1 if no starred element was found. + """ + col = -1 + for j in range(self.n): + if self.marked[row][j] == 1: + col = j + break + + return col + + def __find_star_in_col(self, col): + """ + Find the first starred element in the specified row. Returns + the row index, or -1 if no starred element was found. + """ + row = -1 + for i in range(self.n): + if self.marked[i][col] == 1: + row = i + break + + return row + + def __find_prime_in_row(self, row): + """ + Find the first prime element in the specified row. Returns + the column index, or -1 if no starred element was found. + """ + col = -1 + for j in range(self.n): + if self.marked[row][j] == 2: + col = j + break + + return col + + def __convert_path(self, path, count): + for i in range(count+1): + if self.marked[path[i][0]][path[i][1]] == 1: + self.marked[path[i][0]][path[i][1]] = 0 + else: + self.marked[path[i][0]][path[i][1]] = 1 + + def __clear_covers(self): + """Clear all covered matrix cells""" + for i in range(self.n): + self.row_covered[i] = False + self.col_covered[i] = False + + def __erase_primes(self): + """Erase all prime markings""" + for i in range(self.n): + for j in range(self.n): + if self.marked[i][j] == 2: + self.marked[i][j] = 0 + +# --------------------------------------------------------------------------- +# Functions +# --------------------------------------------------------------------------- + +def make_cost_matrix(profit_matrix, inversion_function): + """ + Create a cost matrix from a profit matrix by calling + 'inversion_function' to invert each value. The inversion + function must take one numeric argument (of any type) and return + another numeric argument which is presumed to be the cost inverse + of the original profit. + + This is a static method. Call it like this: + + .. python:: + + cost_matrix = Munkres.make_cost_matrix(matrix, inversion_func) + + For example: + + .. python:: + + cost_matrix = Munkres.make_cost_matrix(matrix, lambda x : sys.maxsize - x) + + :Parameters: + profit_matrix : list of lists + The matrix to convert from a profit to a cost matrix + + inversion_function : function + The function to use to invert each entry in the profit matrix + + :rtype: list of lists + :return: The converted matrix + """ + cost_matrix = [] + for row in profit_matrix: + cost_matrix.append([inversion_function(value) for value in row]) + return cost_matrix + +def print_matrix(matrix, msg=None): + """ + Convenience function: Displays the contents of a matrix of integers. + + :Parameters: + matrix : list of lists + Matrix to print + + msg : str + Optional message to print before displaying the matrix + """ + import math + + if msg is not None: + print(msg) + + # Calculate the appropriate format width. + width = 0 + for row in matrix: + for val in row: + width = max(width, int(math.log10(val)) + 1) + + # Make the format string + format = '%%%dd' % width + + # Print the matrix + for row in matrix: + sep = '[' + for val in row: + sys.stdout.write(sep + format % val) + sep = ', ' + sys.stdout.write(']\n') + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +if __name__ == '__main__': + + matrices = [ + # Square + ([[400, 150, 400], + [400, 450, 600], + [300, 225, 300]], + 850), # expected cost + + # Rectangular variant + ([[400, 150, 400, 1], + [400, 450, 600, 2], + [300, 225, 300, 3]], + 452), # expected cost + + + # Square + ([[10, 10, 8], + [9, 8, 1], + [9, 7, 4]], + 18), + + # Rectangular variant + ([[10, 10, 8, 11], + [9, 8, 1, 1], + [9, 7, 4, 10]], + 15)] + + m = Munkres() + for cost_matrix, expected_total in matrices: + print_matrix(cost_matrix, msg='cost matrix') + indexes = m.compute(cost_matrix) + total_cost = 0 + for r, c in indexes: + x = cost_matrix[r][c] + total_cost += x + print('(%d, %d) -> %d' % (r, c, x)) + print('lowest cost=%d' % total_cost) + assert expected_total == total_cost diff --git a/libs/musicbrainzngs/__init__.py b/libs/musicbrainzngs/__init__.py new file mode 100644 index 00000000..22fed80d --- /dev/null +++ b/libs/musicbrainzngs/__init__.py @@ -0,0 +1,2 @@ +from musicbrainzngs.musicbrainz import * +from musicbrainzngs.caa import * diff --git a/libs/musicbrainzngs/caa.py b/libs/musicbrainzngs/caa.py new file mode 100644 index 00000000..12fa8d35 --- /dev/null +++ b/libs/musicbrainzngs/caa.py @@ -0,0 +1,177 @@ +# This file is part of the musicbrainzngs library +# Copyright (C) Alastair Porter, Wieland Hoffmann, and others +# This file is distributed under a BSD-2-Clause type license. +# See the COPYING file for more information. + +__all__ = [ + 'set_caa_hostname', 'get_image_list', 'get_release_group_image_list', + 'get_release_group_image_front', 'get_image_front', 'get_image_back', + 'get_image' + ] + +import json + +from musicbrainzngs import compat +from musicbrainzngs import musicbrainz + +hostname = "coverartarchive.org" + + +def set_caa_hostname(new_hostname): + """Set the base hostname for Cover Art Archive requests. + Defaults to 'coverartarchive.org'.""" + global hostname + hostname = new_hostname + + +def _caa_request(mbid, imageid=None, size=None, entitytype="release"): + """ Make a CAA request. + + :param imageid: ``front``, ``back`` or a number from the listing obtained + with :meth:`get_image_list`. + :type imageid: str + + :param size: 250, 500 + :type size: str or None + + :param entitytype: ``release`` or ``release-group`` + :type entitytype: str + """ + # Construct the full URL for the request, including hostname and + # query string. + path = [entitytype, mbid] + if imageid and size: + path.append("%s-%s" % (imageid, size)) + elif imageid: + path.append(imageid) + url = compat.urlunparse(( + 'http', + hostname, + '/%s' % '/'.join(path), + '', + '', + '' + )) + musicbrainz._log.debug("GET request for %s" % (url, )) + + # Set up HTTP request handler and URL opener. + httpHandler = compat.HTTPHandler(debuglevel=0) + handlers = [httpHandler] + + opener = compat.build_opener(*handlers) + + # Make request. + req = musicbrainz._MusicbrainzHttpRequest("GET", url, None) + # Useragent isn't needed for CAA, but we'll add it if it exists + if musicbrainz._useragent != "": + req.add_header('User-Agent', musicbrainz._useragent) + musicbrainz._log.debug("requesting with UA %s" % musicbrainz._useragent) + + resp = musicbrainz._safe_read(opener, req, None) + + # TODO: The content type declared by the CAA for JSON files is + # 'applicaiton/octet-stream'. This is not useful to detect whether the + # content is JSON, so default to decoding JSON if no imageid was supplied. + # http://tickets.musicbrainz.org/browse/CAA-75 + if imageid: + # If we asked for an image, return the image + return resp + else: + # Otherwise it's json + return json.loads(resp) + + +def get_image_list(releaseid): + """Get the list of cover art associated with a release. + + The return value is the deserialized response of the `JSON listing + `_ + returned by the Cover Art Archive API. + + If an error occurs then a :class:`~musicbrainzngs.ResponseError` will + be raised with one of the following HTTP codes: + + * 400: `Releaseid` is not a valid UUID + * 404: No release exists with an MBID of `releaseid` + * 503: Ratelimit exceeded + """ + return _caa_request(releaseid) + + +def get_release_group_image_list(releasegroupid): + """Get the list of cover art associated with a release group. + + The return value is the deserialized response of the `JSON listing + `_ + returned by the Cover Art Archive API. + + If an error occurs then a :class:`~musicbrainzngs.ResponseError` will + be raised with one of the following HTTP codes: + + * 400: `Releaseid` is not a valid UUID + * 404: No release exists with an MBID of `releaseid` + * 503: Ratelimit exceeded + """ + return _caa_request(releasegroupid, entitytype="release-group") + + +def get_release_group_image_front(releasegroupid, size=None): + """Download the front cover art for a release group. + The `size` argument and the possible error conditions are the same as for + :meth:`get_image`. + """ + return get_image(releasegroupid, "front", size=size, + entitytype="release-group") + + +def get_image_front(releaseid, size=None): + """Download the front cover art for a release. + The `size` argument and the possible error conditions are the same as for + :meth:`get_image`. + """ + return get_image(releaseid, "front", size=size) + + +def get_image_back(releaseid, size=None): + """Download the back cover art for a release. + The `size` argument and the possible error conditions are the same as for + :meth:`get_image`. + """ + return get_image(releaseid, "back", size=size) + + +def get_image(mbid, coverid, size=None, entitytype="release"): + """Download cover art for a release. The coverart file to download + is specified by the `coverid` argument. + + If `size` is not specified, download the largest copy present, which can be + very large. + + If an error occurs then a :class:`~musicbrainzngs.ResponseError` + will be raised with one of the following HTTP codes: + + * 400: `Releaseid` is not a valid UUID or `coverid` is invalid + * 404: No release exists with an MBID of `releaseid` + * 503: Ratelimit exceeded + + :param coverid: ``front``, ``back`` or a number from the listing obtained with + :meth:`get_image_list` + :type coverid: int or str + + :param size: 250, 500 or None. If it is None, the largest available picture + will be downloaded. If the image originally uploaded to the + Cover Art Archive was smaller than the requested size, only + the original image will be returned. + :type size: str or None + + :param entitytype: The type of entity for which to download the cover art. + This is either ``release`` or ``release-group``. + :type entitytype: str + :return: The binary image data + :type: str + """ + if isinstance(coverid, int): + coverid = "%d" % (coverid, ) + if isinstance(size, int): + size = "%d" % (size, ) + return _caa_request(mbid, coverid, size=size, entitytype=entitytype) diff --git a/libs/musicbrainzngs/compat.py b/libs/musicbrainzngs/compat.py new file mode 100644 index 00000000..36574b5c --- /dev/null +++ b/libs/musicbrainzngs/compat.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2012 Kenneth Reitz. + +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. + +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +""" +pythoncompat +""" + + +import sys + +# ------- +# Pythons +# ------- + +# Syntax sugar. +_ver = sys.version_info + +#: Python 2.x? +is_py2 = (_ver[0] == 2) + +#: Python 3.x? +is_py3 = (_ver[0] == 3) + +# --------- +# Specifics +# --------- + +if is_py2: + from StringIO import StringIO + from urllib2 import HTTPPasswordMgr, HTTPDigestAuthHandler, Request,\ + HTTPHandler, build_opener, HTTPError, URLError,\ + build_opener + from httplib import BadStatusLine, HTTPException + from urlparse import urlunparse + from urllib import urlencode + + bytes = str + unicode = unicode + basestring = basestring +elif is_py3: + from io import StringIO + from urllib.request import HTTPPasswordMgr, HTTPDigestAuthHandler, Request,\ + HTTPHandler, build_opener + from urllib.error import HTTPError, URLError + from http.client import HTTPException, BadStatusLine + from urllib.parse import urlunparse, urlencode + + unicode = str + bytes = bytes + basestring = (str,bytes) diff --git a/libs/musicbrainzngs/mbxml.py b/libs/musicbrainzngs/mbxml.py new file mode 100644 index 00000000..60236dc7 --- /dev/null +++ b/libs/musicbrainzngs/mbxml.py @@ -0,0 +1,821 @@ +# This file is part of the musicbrainzngs library +# Copyright (C) Alastair Porter, Adrian Sampson, and others +# This file is distributed under a BSD-2-Clause type license. +# See the COPYING file for more information. + +import re +import xml.etree.ElementTree as ET +import logging + +from musicbrainzngs import util + +try: + from ET import fixtag +except: + # Python < 2.7 + def fixtag(tag, namespaces): + # given a decorated tag (of the form {uri}tag), return prefixed + # tag and namespace declaration, if any + if isinstance(tag, ET.QName): + tag = tag.text + namespace_uri, tag = tag[1:].split("}", 1) + prefix = namespaces.get(namespace_uri) + if prefix is None: + prefix = "ns%d" % len(namespaces) + namespaces[namespace_uri] = prefix + if prefix == "xml": + xmlns = None + else: + xmlns = ("xmlns:%s" % prefix, namespace_uri) + else: + xmlns = None + return "%s:%s" % (prefix, tag), xmlns + + +NS_MAP = {"http://musicbrainz.org/ns/mmd-2.0#": "ws2", + "http://musicbrainz.org/ns/ext#-2.0": "ext"} +_log = logging.getLogger("musicbrainzngs") + +def get_error_message(error): + """ Given an error XML message from the webservice containing + xy, return a list + of [x, y]""" + try: + tree = util.bytes_to_elementtree(error) + root = tree.getroot() + errors = [] + if root.tag == "error": + for ch in root: + if ch.tag == "text": + errors.append(ch.text) + return errors + except ET.ParseError: + return None + +def make_artist_credit(artists): + names = [] + for artist in artists: + if isinstance(artist, dict): + if "name" in artist: + names.append(artist.get("name", "")) + else: + names.append(artist.get("artist", {}).get("name", "")) + else: + names.append(artist) + return "".join(names) + +def parse_elements(valid_els, inner_els, element): + """ Extract single level subelements from an element. + For example, given the element: + + Text + + and a list valid_els that contains "subelement", + return a dict {'subelement': 'Text'} + + Delegate the parsing of multi-level subelements to another function. + For example, given the element: + + + FooBar + + + and a dictionary {'subelement': parse_subelement}, + call parse_subelement() and + return a dict {'subelement': } + if parse_subelement returns a tuple of the form + (True, {'subelement-key': }) + then merge the second element of the tuple into the + result (which may have a key other than 'subelement' or + more than 1 key) + """ + result = {} + for sub in element: + t = fixtag(sub.tag, NS_MAP)[0] + if ":" in t: + t = t.split(":")[1] + if t in valid_els: + result[t] = sub.text or "" + elif t in inner_els.keys(): + inner_result = inner_els[t](sub) + if isinstance(inner_result, tuple) and inner_result[0]: + result.update(inner_result[1]) + else: + result[t] = inner_result + # add counts for lists when available + m = re.match(r'([a-z0-9-]+)-list', t) + if m and "count" in sub.attrib: + result["%s-count" % m.group(1)] = int(sub.attrib["count"]) + else: + _log.info("in <%s>, uncaught <%s>", + fixtag(element.tag, NS_MAP)[0], t) + return result + +def parse_attributes(attributes, element): + """ Extract attributes from an element. + For example, given the element: + + and a list attributes that contains "type", + return a dict {'type': 'Group'} + """ + result = {} + for attr in element.attrib: + if "{" in attr: + a = fixtag(attr, NS_MAP)[0] + else: + a = attr + if a in attributes: + result[a] = element.attrib[attr] + else: + _log.info("in <%s>, uncaught attribute %s", fixtag(element.tag, NS_MAP)[0], attr) + + return result + +def parse_message(message): + tree = util.bytes_to_elementtree(message) + root = tree.getroot() + result = {} + valid_elements = {"area": parse_area, + "artist": parse_artist, + "instrument": parse_instrument, + "label": parse_label, + "place": parse_place, + "event": parse_event, + "release": parse_release, + "release-group": parse_release_group, + "series": parse_series, + "recording": parse_recording, + "work": parse_work, + "url": parse_url, + + "disc": parse_disc, + "cdstub": parse_cdstub, + "isrc": parse_isrc, + + "annotation-list": parse_annotation_list, + "area-list": parse_area_list, + "artist-list": parse_artist_list, + "label-list": parse_label_list, + "place-list": parse_place_list, + "event-list": parse_event_list, + "instrument-list": parse_instrument_list, + "release-list": parse_release_list, + "release-group-list": parse_release_group_list, + "series-list": parse_series_list, + "recording-list": parse_recording_list, + "work-list": parse_work_list, + "url-list": parse_url_list, + + "collection-list": parse_collection_list, + "collection": parse_collection, + + "message": parse_response_message + } + result.update(parse_elements([], valid_elements, root)) + return result + +def parse_response_message(message): + return parse_elements(["text"], {}, message) + +def parse_collection_list(cl): + return [parse_collection(c) for c in cl] + +def parse_collection(collection): + result = {} + attribs = ["id", "type", "entity-type"] + elements = ["name", "editor"] + inner_els = {"release-list": parse_release_list, + "artist-list": parse_artist_list, + "event-list": parse_event_list, + "place-list": parse_place_list, + "recording-list": parse_recording_list, + "work-list": parse_work_list} + result.update(parse_attributes(attribs, collection)) + result.update(parse_elements(elements, inner_els, collection)) + + return result + +def parse_annotation_list(al): + return [parse_annotation(a) for a in al] + +def parse_annotation(annotation): + result = {} + attribs = ["type", "ext:score"] + elements = ["entity", "name", "text"] + result.update(parse_attributes(attribs, annotation)) + result.update(parse_elements(elements, {}, annotation)) + return result + +def parse_lifespan(lifespan): + parts = parse_elements(["begin", "end", "ended"], {}, lifespan) + + return parts + +def parse_area_list(al): + return [parse_area(a) for a in al] + +def parse_area(area): + result = {} + attribs = ["id", "type", "ext:score"] + elements = ["name", "sort-name", "disambiguation"] + inner_els = {"life-span": parse_lifespan, + "alias-list": parse_alias_list, + "relation-list": parse_relation_list, + "annotation": parse_annotation, + "iso-3166-1-code-list": parse_element_list, + "iso-3166-2-code-list": parse_element_list, + "iso-3166-3-code-list": parse_element_list} + + result.update(parse_attributes(attribs, area)) + result.update(parse_elements(elements, inner_els, area)) + + return result + +def parse_artist_list(al): + return [parse_artist(a) for a in al] + +def parse_artist(artist): + result = {} + attribs = ["id", "type", "ext:score"] + elements = ["name", "sort-name", "country", "user-rating", + "disambiguation", "gender", "ipi"] + inner_els = {"area": parse_area, + "begin-area": parse_area, + "end-area": parse_area, + "life-span": parse_lifespan, + "recording-list": parse_recording_list, + "relation-list": parse_relation_list, + "release-list": parse_release_list, + "release-group-list": parse_release_group_list, + "work-list": parse_work_list, + "tag-list": parse_tag_list, + "user-tag-list": parse_tag_list, + "rating": parse_rating, + "ipi-list": parse_element_list, + "isni-list": parse_element_list, + "alias-list": parse_alias_list, + "annotation": parse_annotation} + + result.update(parse_attributes(attribs, artist)) + result.update(parse_elements(elements, inner_els, artist)) + + return result + +def parse_coordinates(c): + return parse_elements(['latitude', 'longitude'], {}, c) + +def parse_place_list(pl): + return [parse_place(p) for p in pl] + +def parse_place(place): + result = {} + attribs = ["id", "type", "ext:score"] + elements = ["name", "address", + "ipi", "disambiguation"] + inner_els = {"area": parse_area, + "coordinates": parse_coordinates, + "life-span": parse_lifespan, + "tag-list": parse_tag_list, + "user-tag-list": parse_tag_list, + "alias-list": parse_alias_list, + "relation-list": parse_relation_list, + "annotation": parse_annotation} + + result.update(parse_attributes(attribs, place)) + result.update(parse_elements(elements, inner_els, place)) + + return result + +def parse_event_list(el): + return [parse_event(e) for e in el] + +def parse_event(event): + result = {} + attribs = ["id", "type", "ext:score"] + elements = ["name", "time", "setlist", "cancelled", "disambiguation", "user-rating"] + inner_els = {"life-span": parse_lifespan, + "relation-list": parse_relation_list, + "alias-list": parse_alias_list, + "tag-list": parse_tag_list, + "user-tag-list": parse_tag_list, + "rating": parse_rating} + + result.update(parse_attributes(attribs, event)) + result.update(parse_elements(elements, inner_els, event)) + + return result + +def parse_instrument(instrument): + result = {} + attribs = ["id", "type", "ext:score"] + elements = ["name", "description", "disambiguation"] + inner_els = {"relation-list": parse_relation_list, + "tag-list": parse_tag_list, + "alias-list": parse_alias_list, + "annotation": parse_annotation} + result.update(parse_attributes(attribs, instrument)) + result.update(parse_elements(elements, inner_els, instrument)) + + return result + +def parse_label_list(ll): + return [parse_label(l) for l in ll] + +def parse_label(label): + result = {} + attribs = ["id", "type", "ext:score"] + elements = ["name", "sort-name", "country", "label-code", "user-rating", + "ipi", "disambiguation"] + inner_els = {"area": parse_area, + "life-span": parse_lifespan, + "release-list": parse_release_list, + "tag-list": parse_tag_list, + "user-tag-list": parse_tag_list, + "rating": parse_rating, + "ipi-list": parse_element_list, + "alias-list": parse_alias_list, + "relation-list": parse_relation_list, + "annotation": parse_annotation} + + result.update(parse_attributes(attribs, label)) + result.update(parse_elements(elements, inner_els, label)) + + return result + +def parse_relation_target(tgt): + attributes = parse_attributes(['id'], tgt) + if 'id' in attributes: + return (True, {'target-id': attributes['id']}) + else: + return (True, {'target-id': tgt.text}) + +def parse_relation_list(rl): + attribs = ["target-type"] + ttype = parse_attributes(attribs, rl) + key = "%s-relation-list" % ttype["target-type"] + return (True, {key: [parse_relation(r) for r in rl]}) + +def parse_relation(relation): + result = {} + attribs = ["type", "type-id"] + elements = ["target", "direction", "begin", "end", "ended", "ordering-key"] + inner_els = {"area": parse_area, + "artist": parse_artist, + "instrument": parse_instrument, + "label": parse_label, + "place": parse_place, + "event": parse_event, + "recording": parse_recording, + "release": parse_release, + "release-group": parse_release_group, + "series": parse_series, + "attribute-list": parse_element_list, + "work": parse_work, + "target": parse_relation_target + } + result.update(parse_attributes(attribs, relation)) + result.update(parse_elements(elements, inner_els, relation)) + # We parse attribute-list again to get attributes that have both + # text and attribute values + result.update(parse_elements([], {"attribute-list": parse_relation_attribute_list}, relation)) + + return result + +def parse_relation_attribute_list(attributelist): + ret = [] + for attribute in attributelist: + ret.append(parse_relation_attribute_element(attribute)) + return (True, {"attributes": ret}) + +def parse_relation_attribute_element(element): + # Parses an attribute into a dictionary containing an element + # {"attribute": } and also an additional element + # containing any xml attributes. + # e.g number + # -> {"attribute": "number", "value": "BuxWV 1"} + result = {} + for attr in element.attrib: + if "{" in attr: + a = fixtag(attr, NS_MAP)[0] + else: + a = attr + result[a] = element.attrib[attr] + result["attribute"] = element.text + return result + +def parse_release(release): + result = {} + attribs = ["id", "ext:score"] + elements = ["title", "status", "disambiguation", "quality", "country", + "barcode", "date", "packaging", "asin"] + inner_els = {"text-representation": parse_text_representation, + "artist-credit": parse_artist_credit, + "label-info-list": parse_label_info_list, + "medium-list": parse_medium_list, + "release-group": parse_release_group, + "tag-list": parse_tag_list, + "user-tag-list": parse_tag_list, + "relation-list": parse_relation_list, + "annotation": parse_annotation, + "cover-art-archive": parse_caa, + "release-event-list": parse_release_event_list} + + result.update(parse_attributes(attribs, release)) + result.update(parse_elements(elements, inner_els, release)) + if "artist-credit" in result: + result["artist-credit-phrase"] = make_artist_credit( + result["artist-credit"]) + + return result + +def parse_medium_list(ml): + """medium-list results from search have an additional + element containing the number of tracks + over all mediums. Optionally add this""" + medium_list = [] + track_count = None + for m in ml: + tag = fixtag(m.tag, NS_MAP)[0] + if tag == "ws2:medium": + medium_list.append(parse_medium(m)) + elif tag == "ws2:track-count": + track_count = int(m.text) + ret = {"medium-list": medium_list} + if track_count is not None: + ret["medium-track-count"] = track_count + + return (True, ret) + +def parse_release_event_list(rel): + return [parse_release_event(re) for re in rel] + +def parse_release_event(event): + result = {} + elements = ["date"] + inner_els = {"area": parse_area} + + result.update(parse_elements(elements, inner_els, event)) + return result + +def parse_medium(medium): + result = {} + elements = ["position", "format", "title"] + inner_els = {"disc-list": parse_disc_list, + "pregap": parse_track, + "track-list": parse_track_list, + "data-track-list": parse_track_list} + + result.update(parse_elements(elements, inner_els, medium)) + return result + +def parse_disc_list(dl): + return [parse_disc(d) for d in dl] + +def parse_text_representation(textr): + return parse_elements(["language", "script"], {}, textr) + +def parse_release_group(rg): + result = {} + attribs = ["id", "type", "ext:score"] + elements = ["title", "user-rating", "first-release-date", "primary-type", + "disambiguation"] + inner_els = {"artist-credit": parse_artist_credit, + "release-list": parse_release_list, + "tag-list": parse_tag_list, + "user-tag-list": parse_tag_list, + "secondary-type-list": parse_element_list, + "relation-list": parse_relation_list, + "rating": parse_rating, + "annotation": parse_annotation} + + result.update(parse_attributes(attribs, rg)) + result.update(parse_elements(elements, inner_els, rg)) + if "artist-credit" in result: + result["artist-credit-phrase"] = make_artist_credit(result["artist-credit"]) + + return result + +def parse_recording(recording): + result = {} + attribs = ["id", "ext:score"] + elements = ["title", "length", "user-rating", "disambiguation", "video"] + inner_els = {"artist-credit": parse_artist_credit, + "release-list": parse_release_list, + "tag-list": parse_tag_list, + "user-tag-list": parse_tag_list, + "rating": parse_rating, + "isrc-list": parse_external_id_list, + "echoprint-list": parse_external_id_list, + "relation-list": parse_relation_list, + "annotation": parse_annotation} + + result.update(parse_attributes(attribs, recording)) + result.update(parse_elements(elements, inner_els, recording)) + if "artist-credit" in result: + result["artist-credit-phrase"] = make_artist_credit(result["artist-credit"]) + + return result + +def parse_series_list(sl): + return [parse_series(s) for s in sl] + +def parse_series(series): + result = {} + attribs = ["id", "type", "ext:score"] + elements = ["name", "disambiguation"] + inner_els = {"alias-list": parse_alias_list, + "relation-list": parse_relation_list, + "annotation": parse_annotation} + + result.update(parse_attributes(attribs, series)) + result.update(parse_elements(elements, inner_els, series)) + + return result + +def parse_external_id_list(pl): + return [parse_attributes(["id"], p)["id"] for p in pl] + +def parse_element_list(el): + return [e.text for e in el] + +def parse_work_list(wl): + return [parse_work(w) for w in wl] + +def parse_work(work): + result = {} + attribs = ["id", "ext:score", "type"] + elements = ["title", "user-rating", "language", "iswc", "disambiguation"] + inner_els = {"tag-list": parse_tag_list, + "user-tag-list": parse_tag_list, + "rating": parse_rating, + "alias-list": parse_alias_list, + "iswc-list": parse_element_list, + "relation-list": parse_relation_list, + "annotation": parse_response_message, + "attribute-list": parse_work_attribute_list + } + + result.update(parse_attributes(attribs, work)) + result.update(parse_elements(elements, inner_els, work)) + + return result + +def parse_work_attribute_list(wal): + return [parse_work_attribute(wa) for wa in wal] + +def parse_work_attribute(wa): + attribs = ["type"] + typeinfo = parse_attributes(attribs, wa) + result = {} + if typeinfo: + result = {"attribute": typeinfo["type"], + "value": wa.text} + + return result + + +def parse_url_list(ul): + return [parse_url(u) for u in ul] + +def parse_url(url): + result = {} + attribs = ["id"] + elements = ["resource"] + inner_els = {"relation-list": parse_relation_list} + + result.update(parse_attributes(attribs, url)) + result.update(parse_elements(elements, inner_els, url)) + + return result + +def parse_disc(disc): + result = {} + attribs = ["id"] + elements = ["sectors"] + inner_els = {"release-list": parse_release_list, + "offset-list": parse_offset_list + } + + result.update(parse_attributes(attribs, disc)) + result.update(parse_elements(elements, inner_els, disc)) + + return result + +def parse_cdstub(cdstub): + result = {} + attribs = ["id"] + elements = ["title", "artist", "barcode"] + inner_els = {"track-list": parse_track_list} + + result.update(parse_attributes(attribs, cdstub)) + result.update(parse_elements(elements, inner_els, cdstub)) + + return result + +def parse_offset_list(ol): + return [int(o.text) for o in ol] + +def parse_instrument_list(rl): + result = [] + for r in rl: + result.append(parse_instrument(r)) + return result + +def parse_release_list(rl): + result = [] + for r in rl: + result.append(parse_release(r)) + return result + +def parse_release_group_list(rgl): + result = [] + for rg in rgl: + result.append(parse_release_group(rg)) + return result + +def parse_isrc(isrc): + result = {} + attribs = ["id"] + inner_els = {"recording-list": parse_recording_list} + + result.update(parse_attributes(attribs, isrc)) + result.update(parse_elements([], inner_els, isrc)) + + return result + +def parse_recording_list(recs): + result = [] + for r in recs: + result.append(parse_recording(r)) + return result + +def parse_artist_credit(ac): + result = [] + for namecredit in ac: + result.append(parse_name_credit(namecredit)) + join = parse_attributes(["joinphrase"], namecredit) + if "joinphrase" in join: + result.append(join["joinphrase"]) + return result + +def parse_name_credit(nc): + result = {} + elements = ["name"] + inner_els = {"artist": parse_artist} + + result.update(parse_elements(elements, inner_els, nc)) + + return result + +def parse_label_info_list(lil): + result = [] + + for li in lil: + result.append(parse_label_info(li)) + return result + +def parse_label_info(li): + result = {} + elements = ["catalog-number"] + inner_els = {"label": parse_label} + + result.update(parse_elements(elements, inner_els, li)) + return result + +def parse_track_list(tl): + result = [] + for t in tl: + result.append(parse_track(t)) + return result + +def parse_track(track): + result = {} + attribs = ["id"] + elements = ["number", "position", "title", "length"] + inner_els = {"recording": parse_recording, + "artist-credit": parse_artist_credit} + + result.update(parse_attributes(attribs, track)) + result.update(parse_elements(elements, inner_els, track)) + if "artist-credit" in result.get("recording", {}) and "artist-credit" not in result: + result["artist-credit"] = result["recording"]["artist-credit"] + if "artist-credit" in result: + result["artist-credit-phrase"] = make_artist_credit(result["artist-credit"]) + # Make a length field that contains track length or recording length + track_or_recording = None + if "length" in result: + track_or_recording = result["length"] + elif result.get("recording", {}).get("length"): + track_or_recording = result.get("recording", {}).get("length") + if track_or_recording: + result["track_or_recording_length"] = track_or_recording + return result + +def parse_tag_list(tl): + return [parse_tag(t) for t in tl] + +def parse_tag(tag): + result = {} + attribs = ["count"] + elements = ["name"] + + result.update(parse_attributes(attribs, tag)) + result.update(parse_elements(elements, {}, tag)) + + return result + +def parse_rating(rating): + result = {} + attribs = ["votes-count"] + + result.update(parse_attributes(attribs, rating)) + result["rating"] = rating.text + + return result + +def parse_alias_list(al): + return [parse_alias(a) for a in al] + +def parse_alias(alias): + result = {} + attribs = ["locale", "sort-name", "type", "primary", + "begin-date", "end-date"] + + result.update(parse_attributes(attribs, alias)) + result["alias"] = alias.text + + return result + +def parse_caa(caa_element): + result = {} + elements = ["artwork", "count", "front", "back", "darkened"] + + result.update(parse_elements(elements, {}, caa_element)) + return result + + +### + +def make_barcode_request(release2barcode): + NS = "http://musicbrainz.org/ns/mmd-2.0#" + root = ET.Element("{%s}metadata" % NS) + rel_list = ET.SubElement(root, "{%s}release-list" % NS) + for release, barcode in release2barcode.items(): + rel_xml = ET.SubElement(rel_list, "{%s}release" % NS) + bar_xml = ET.SubElement(rel_xml, "{%s}barcode" % NS) + rel_xml.set("{%s}id" % NS, release) + bar_xml.text = barcode + + return ET.tostring(root, "utf-8") + +def make_tag_request(**kwargs): + NS = "http://musicbrainz.org/ns/mmd-2.0#" + root = ET.Element("{%s}metadata" % NS) + for entity_type in ['artist', 'label', 'place', 'recording', 'release', 'release_group', 'work']: + entity_tags = kwargs.pop(entity_type + '_tags', None) + if entity_tags is not None: + e_list = ET.SubElement(root, "{%s}%s-list" % (NS, entity_type.replace('_', '-'))) + for e, tags in entity_tags.items(): + e_xml = ET.SubElement(e_list, "{%s}%s" % (NS, entity_type.replace('_', '-'))) + e_xml.set("{%s}id" % NS, e) + taglist = ET.SubElement(e_xml, "{%s}user-tag-list" % NS) + for tag in tags: + usertag_xml = ET.SubElement(taglist, "{%s}user-tag" % NS) + name_xml = ET.SubElement(usertag_xml, "{%s}name" % NS) + name_xml.text = tag + if kwargs.keys(): + raise TypeError("make_tag_request() got an unexpected keyword argument '%s'" % kwargs.popitem()[0]) + + return ET.tostring(root, "utf-8") + +def make_rating_request(**kwargs): + NS = "http://musicbrainz.org/ns/mmd-2.0#" + root = ET.Element("{%s}metadata" % NS) + for entity_type in ['artist', 'label', 'recording', 'release_group', 'work']: + entity_ratings = kwargs.pop(entity_type + '_ratings', None) + if entity_ratings is not None: + e_list = ET.SubElement(root, "{%s}%s-list" % (NS, entity_type.replace('_', '-'))) + for e, rating in entity_ratings.items(): + e_xml = ET.SubElement(e_list, "{%s}%s" % (NS, entity_type.replace('_', '-'))) + e_xml.set("{%s}id" % NS, e) + rating_xml = ET.SubElement(e_xml, "{%s}user-rating" % NS) + rating_xml.text = str(rating) + if kwargs.keys(): + raise TypeError("make_rating_request() got an unexpected keyword argument '%s'" % kwargs.popitem()[0]) + + return ET.tostring(root, "utf-8") + +def make_isrc_request(recording2isrcs): + NS = "http://musicbrainz.org/ns/mmd-2.0#" + root = ET.Element("{%s}metadata" % NS) + rec_list = ET.SubElement(root, "{%s}recording-list" % NS) + for rec, isrcs in recording2isrcs.items(): + if len(isrcs) > 0: + rec_xml = ET.SubElement(rec_list, "{%s}recording" % NS) + rec_xml.set("{%s}id" % NS, rec) + isrc_list_xml = ET.SubElement(rec_xml, "{%s}isrc-list" % NS) + isrc_list_xml.set("{%s}count" % NS, str(len(isrcs))) + for isrc in isrcs: + isrc_xml = ET.SubElement(isrc_list_xml, "{%s}isrc" % NS) + isrc_xml.set("{%s}id" % NS, isrc) + return ET.tostring(root, "utf-8") diff --git a/libs/musicbrainzngs/musicbrainz.py b/libs/musicbrainzngs/musicbrainz.py new file mode 100644 index 00000000..953c79b8 --- /dev/null +++ b/libs/musicbrainzngs/musicbrainz.py @@ -0,0 +1,1337 @@ +# This file is part of the musicbrainzngs library +# Copyright (C) Alastair Porter, Adrian Sampson, and others +# This file is distributed under a BSD-2-Clause type license. +# See the COPYING file for more information. + +import re +import threading +import time +import logging +import socket +import hashlib +import locale +import sys +import json +import xml.etree.ElementTree as etree +from xml.parsers import expat +from warnings import warn + +from musicbrainzngs import mbxml +from musicbrainzngs import util +from musicbrainzngs import compat + +_version = "0.6" +_log = logging.getLogger("musicbrainzngs") + +LUCENE_SPECIAL = r'([+\-&|!(){}\[\]\^"~*?:\\\/])' + +# Constants for validation. + +RELATABLE_TYPES = ['area', 'artist', 'label', 'place', 'event', 'recording', 'release', 'release-group', 'series', 'url', 'work', 'instrument'] +RELATION_INCLUDES = [entity + '-rels' for entity in RELATABLE_TYPES] +TAG_INCLUDES = ["tags", "user-tags"] +RATING_INCLUDES = ["ratings", "user-ratings"] + +VALID_INCLUDES = { + 'area' : ["aliases", "annotation"] + RELATION_INCLUDES, + 'artist': [ + "recordings", "releases", "release-groups", "works", # Subqueries + "various-artists", "discids", "media", "isrcs", + "aliases", "annotation" + ] + RELATION_INCLUDES + TAG_INCLUDES + RATING_INCLUDES, + 'annotation': [ + + ], + 'instrument': ["aliases", "annotation" + ] + RELATION_INCLUDES + TAG_INCLUDES, + 'label': [ + "releases", # Subqueries + "discids", "media", + "aliases", "annotation" + ] + RELATION_INCLUDES + TAG_INCLUDES + RATING_INCLUDES, + 'place' : ["aliases", "annotation"] + RELATION_INCLUDES + TAG_INCLUDES, + 'event' : ["aliases"] + RELATION_INCLUDES + TAG_INCLUDES + RATING_INCLUDES, + 'recording': [ + "artists", "releases", # Subqueries + "discids", "media", "artist-credits", "isrcs", + "annotation", "aliases" + ] + TAG_INCLUDES + RATING_INCLUDES + RELATION_INCLUDES, + 'release': [ + "artists", "labels", "recordings", "release-groups", "media", + "artist-credits", "discids", "puids", "isrcs", + "recording-level-rels", "work-level-rels", "annotation", "aliases" + ] + TAG_INCLUDES + RELATION_INCLUDES, + 'release-group': [ + "artists", "releases", "discids", "media", + "artist-credits", "annotation", "aliases" + ] + TAG_INCLUDES + RATING_INCLUDES + RELATION_INCLUDES, + 'series': [ + "annotation", "aliases" + ] + RELATION_INCLUDES, + 'work': [ + "artists", # Subqueries + "aliases", "annotation" + ] + TAG_INCLUDES + RATING_INCLUDES + RELATION_INCLUDES, + 'url': RELATION_INCLUDES, + 'discid': [ # Discid should be the same as release + "artists", "labels", "recordings", "release-groups", "media", + "artist-credits", "discids", "puids", "isrcs", + "recording-level-rels", "work-level-rels", "annotation", "aliases" + ] + RELATION_INCLUDES, + 'isrc': ["artists", "releases", "puids", "isrcs"], + 'iswc': ["artists"], + 'collection': ['releases'], +} +VALID_BROWSE_INCLUDES = { + 'artist': ["aliases"] + TAG_INCLUDES + RATING_INCLUDES + RELATION_INCLUDES, + 'event': ["aliases"] + TAG_INCLUDES + RATING_INCLUDES + RELATION_INCLUDES, + 'label': ["aliases"] + TAG_INCLUDES + RATING_INCLUDES + RELATION_INCLUDES, + 'recording': ["artist-credits", "isrcs"] + TAG_INCLUDES + RATING_INCLUDES + RELATION_INCLUDES, + 'release': ["artist-credits", "labels", "recordings", "isrcs", + "release-groups", "media", "discids"] + RELATION_INCLUDES, + 'place': ["aliases"] + TAG_INCLUDES + RELATION_INCLUDES, + 'release-group': ["artist-credits"] + TAG_INCLUDES + RATING_INCLUDES + RELATION_INCLUDES, + 'url': RELATION_INCLUDES, + 'work': ["aliases", "annotation"] + TAG_INCLUDES + RATING_INCLUDES + RELATION_INCLUDES, +} + +#: These can be used to filter whenever releases are includes or browsed +VALID_RELEASE_TYPES = [ + "nat", + "album", "single", "ep", "broadcast", "other", # primary types + "compilation", "soundtrack", "spokenword", "interview", "audiobook", + "live", "remix", "dj-mix", "mixtape/street", # secondary types +] +#: These can be used to filter whenever releases or release-groups are involved +VALID_RELEASE_STATUSES = ["official", "promotion", "bootleg", "pseudo-release"] +VALID_SEARCH_FIELDS = { + 'annotation': [ + 'entity', 'name', 'text', 'type' + ], + 'area': [ + 'aid', 'area', 'alias', 'begin', 'comment', 'end', 'ended', + 'iso', 'iso1', 'iso2', 'iso3', 'type' + ], + 'artist': [ + 'arid', 'artist', 'artistaccent', 'alias', 'begin', 'comment', + 'country', 'end', 'ended', 'gender', 'ipi', 'sortname', 'tag', 'type', + 'area', 'beginarea', 'endarea' + ], + 'label': [ + 'alias', 'begin', 'code', 'comment', 'country', 'end', 'ended', + 'ipi', 'label', 'labelaccent', 'laid', 'sortname', 'type', 'tag', + 'area' + ], + 'recording': [ + 'arid', 'artist', 'artistname', 'creditname', 'comment', + 'country', 'date', 'dur', 'format', 'isrc', 'number', + 'position', 'primarytype', 'puid', 'qdur', 'recording', + 'recordingaccent', 'reid', 'release', 'rgid', 'rid', + 'secondarytype', 'status', 'tnum', 'tracks', 'tracksrelease', + 'tag', 'type', 'video' + ], + 'release-group': [ + 'arid', 'artist', 'artistname', 'comment', 'creditname', + 'primarytype', 'rgid', 'releasegroup', 'releasegroupaccent', + 'releases', 'release', 'reid', 'secondarytype', 'status', + 'tag', 'type' + ], + 'release': [ + 'arid', 'artist', 'artistname', 'asin', 'barcode', 'creditname', + 'catno', 'comment', 'country', 'creditname', 'date', 'discids', + 'discidsmedium', 'format', 'laid', 'label', 'lang', 'mediums', + 'primarytype', 'puid', 'quality', 'reid', 'release', 'releaseaccent', + 'rgid', 'script', 'secondarytype', 'status', 'tag', 'tracks', + 'tracksmedium', 'type' + ], + 'series': [ + 'alias', 'comment', 'sid', 'series', 'type' + ], + 'work': [ + 'alias', 'arid', 'artist', 'comment', 'iswc', 'lang', 'tag', + 'type', 'wid', 'work', 'workaccent' + ], +} + +# Constants +class AUTH_YES: pass +class AUTH_NO: pass +class AUTH_IFSET: pass + + +# Exceptions. + +class MusicBrainzError(Exception): + """Base class for all exceptions related to MusicBrainz.""" + pass + +class UsageError(MusicBrainzError): + """Error related to misuse of the module API.""" + pass + +class InvalidSearchFieldError(UsageError): + pass + +class InvalidIncludeError(UsageError): + def __init__(self, msg='Invalid Includes', reason=None): + super(InvalidIncludeError, self).__init__(self) + self.msg = msg + self.reason = reason + + def __str__(self): + return self.msg + +class InvalidFilterError(UsageError): + def __init__(self, msg='Invalid Includes', reason=None): + super(InvalidFilterError, self).__init__(self) + self.msg = msg + self.reason = reason + + def __str__(self): + return self.msg + +class WebServiceError(MusicBrainzError): + """Error related to MusicBrainz API requests.""" + def __init__(self, message=None, cause=None): + """Pass ``cause`` if this exception was caused by another + exception. + """ + self.message = message + self.cause = cause + + def __str__(self): + if self.message: + msg = "%s, " % self.message + else: + msg = "" + msg += "caused by: %s" % str(self.cause) + return msg + +class NetworkError(WebServiceError): + """Problem communicating with the MB server.""" + pass + +class ResponseError(WebServiceError): + """Bad response sent by the MB server.""" + pass + +class AuthenticationError(WebServiceError): + """Received a HTTP 401 response while accessing a protected resource.""" + pass + + +# Helpers for validating and formatting allowed sets. + +def _check_includes_impl(includes, valid_includes): + for i in includes: + if i not in valid_includes: + raise InvalidIncludeError("Bad includes: " + "%s is not a valid include" % i) +def _check_includes(entity, inc): + _check_includes_impl(inc, VALID_INCLUDES[entity]) + +def _check_filter(values, valid): + for v in values: + if v not in valid: + raise InvalidFilterError(v) + +def _check_filter_and_make_params(entity, includes, release_status=[], release_type=[]): + """Check that the status or type values are valid. Then, check that + the filters can be used with the given includes. Return a params + dict that can be passed to _do_mb_query. + """ + if isinstance(release_status, compat.basestring): + release_status = [release_status] + if isinstance(release_type, compat.basestring): + release_type = [release_type] + _check_filter(release_status, VALID_RELEASE_STATUSES) + _check_filter(release_type, VALID_RELEASE_TYPES) + + if (release_status + and "releases" not in includes and entity != "release"): + raise InvalidFilterError("Can't have a status with no release include") + if (release_type + and "release-groups" not in includes and "releases" not in includes + and entity not in ["release-group", "release"]): + raise InvalidFilterError("Can't have a release type " + "with no releases or release-groups involved") + + # Build parameters. + params = {} + if len(release_status): + params["status"] = "|".join(release_status) + if len(release_type): + params["type"] = "|".join(release_type) + return params + +def _docstring_get(entity): + includes = list(VALID_INCLUDES.get(entity, [])) + return _docstring_impl("includes", includes) + +def _docstring_browse(entity): + includes = list(VALID_BROWSE_INCLUDES.get(entity, [])) + return _docstring_impl("includes", includes) + +def _docstring_search(entity): + search_fields = list(VALID_SEARCH_FIELDS.get(entity, [])) + return _docstring_impl("fields", search_fields) + +def _docstring_impl(name, values): + def _decorator(func): + # puids are allowed so nothing breaks, but not documented + if "puids" in values: values.remove("puids") + vstr = ", ".join(values) + args = {name: vstr} + if func.__doc__: + func.__doc__ = func.__doc__.format(**args) + return func + + return _decorator + + +# Global authentication and endpoint details. + +user = password = "" +hostname = "musicbrainz.org" +_client = "" +_useragent = "" + +def auth(u, p): + """Set the username and password to be used in subsequent queries to + the MusicBrainz XML API that require authentication. + """ + global user, password + user = u + password = p + +def set_useragent(app, version, contact=None): + """Set the User-Agent to be used for requests to the MusicBrainz webservice. + This must be set before requests are made.""" + global _useragent, _client + if not app or not version: + raise ValueError("App and version can not be empty") + if contact is not None: + _useragent = "%s/%s python-musicbrainzngs/%s ( %s )" % (app, version, _version, contact) + else: + _useragent = "%s/%s python-musicbrainzngs/%s" % (app, version, _version) + _client = "%s-%s" % (app, version) + _log.debug("set user-agent to %s" % _useragent) + +def set_hostname(new_hostname): + """Set the hostname for MusicBrainz webservice requests. + Defaults to 'musicbrainz.org'. + You can also include a port: 'localhost:8000'.""" + global hostname + hostname = new_hostname + +# Rate limiting. + +limit_interval = 1.0 +limit_requests = 1 +do_rate_limit = True + +def set_rate_limit(limit_or_interval=1.0, new_requests=1): + """Sets the rate limiting behavior of the module. Must be invoked + before the first Web service call. + If the `limit_or_interval` parameter is set to False then + rate limiting will be disabled. If it is a number then only + a set number of requests (`new_requests`) will be made per + given interval (`limit_or_interval`). + """ + global limit_interval + global limit_requests + global do_rate_limit + if isinstance(limit_or_interval, bool): + do_rate_limit = limit_or_interval + else: + if limit_or_interval <= 0.0: + raise ValueError("limit_or_interval can't be less than 0") + if new_requests <= 0: + raise ValueError("new_requests can't be less than 0") + do_rate_limit = True + limit_interval = limit_or_interval + limit_requests = new_requests + +class _rate_limit(object): + """A decorator that limits the rate at which the function may be + called. The rate is controlled by the `limit_interval` and + `limit_requests` global variables. The limiting is thread-safe; + only one thread may be in the function at a time (acts like a + monitor in this sense). The globals must be set before the first + call to the limited function. + """ + def __init__(self, fun): + self.fun = fun + self.last_call = 0.0 + self.lock = threading.Lock() + self.remaining_requests = None # Set on first invocation. + + def _update_remaining(self): + """Update remaining requests based on the elapsed time since + they were last calculated. + """ + # On first invocation, we have the maximum number of requests + # available. + if self.remaining_requests is None: + self.remaining_requests = float(limit_requests) + + else: + since_last_call = time.time() - self.last_call + self.remaining_requests += since_last_call * \ + (limit_requests / limit_interval) + self.remaining_requests = min(self.remaining_requests, + float(limit_requests)) + + self.last_call = time.time() + + def __call__(self, *args, **kwargs): + with self.lock: + if do_rate_limit: + self._update_remaining() + + # Delay if necessary. + while self.remaining_requests < 0.999: + time.sleep((1.0 - self.remaining_requests) * + (limit_requests / limit_interval)) + self._update_remaining() + + # Call the original function, "paying" for this call. + self.remaining_requests -= 1.0 + return self.fun(*args, **kwargs) + +# From pymb2 +class _RedirectPasswordMgr(compat.HTTPPasswordMgr): + def __init__(self): + self._realms = { } + + def find_user_password(self, realm, uri): + # ignoring the uri parameter intentionally + try: + return self._realms[realm] + except KeyError: + return (None, None) + + def add_password(self, realm, uri, username, password): + # ignoring the uri parameter intentionally + self._realms[realm] = (username, password) + +class _DigestAuthHandler(compat.HTTPDigestAuthHandler): + def get_authorization (self, req, chal): + qop = chal.get ('qop', None) + if qop and ',' in qop and 'auth' in qop.split (','): + chal['qop'] = 'auth' + + return compat.HTTPDigestAuthHandler.get_authorization (self, req, chal) + + def _encode_utf8(self, msg): + """The MusicBrainz server also accepts UTF-8 encoded passwords.""" + encoding = sys.stdin.encoding or locale.getpreferredencoding() + try: + # This works on Python 2 (msg in bytes) + msg = msg.decode(encoding) + except AttributeError: + # on Python 3 (msg is already in unicode) + pass + return msg.encode("utf-8") + + def get_algorithm_impls(self, algorithm): + # algorithm should be case-insensitive according to RFC2617 + algorithm = algorithm.upper() + # lambdas assume digest modules are imported at the top level + if algorithm == 'MD5': + H = lambda x: hashlib.md5(self._encode_utf8(x)).hexdigest() + elif algorithm == 'SHA': + H = lambda x: hashlib.sha1(self._encode_utf8(x)).hexdigest() + # XXX MD5-sess + KD = lambda s, d: H("%s:%s" % (s, d)) + return H, KD + +class _MusicbrainzHttpRequest(compat.Request): + """ A custom request handler that allows DELETE and PUT""" + def __init__(self, method, url, data=None): + compat.Request.__init__(self, url, data) + allowed_m = ["GET", "POST", "DELETE", "PUT"] + if method not in allowed_m: + raise ValueError("invalid method: %s" % method) + self.method = method + + def get_method(self): + return self.method + + +# Core (internal) functions for calling the MB API. + +def _safe_read(opener, req, body=None, max_retries=8, retry_delay_delta=2.0): + """Open an HTTP request with a given URL opener and (optionally) a + request body. Transient errors lead to retries. Permanent errors + and repeated errors are translated into a small set of handleable + exceptions. Return a bytestring. + """ + last_exc = None + for retry_num in range(max_retries): + if retry_num: # Not the first try: delay an increasing amount. + _log.info("retrying after delay (#%i)" % retry_num) + time.sleep(retry_num * retry_delay_delta) + + try: + if body: + f = opener.open(req, body) + else: + f = opener.open(req) + return f.read() + + except compat.HTTPError as exc: + if exc.code in (400, 404, 411): + # Bad request, not found, etc. + raise ResponseError(cause=exc) + elif exc.code in (503, 502, 500): + # Rate limiting, internal overloading... + _log.info("HTTP error %i" % exc.code) + elif exc.code in (401, ): + raise AuthenticationError(cause=exc) + else: + # Other, unknown error. Should handle more cases, but + # retrying for now. + _log.info("unknown HTTP error %i" % exc.code) + last_exc = exc + except compat.BadStatusLine as exc: + _log.info("bad status line") + last_exc = exc + except compat.HTTPException as exc: + _log.info("miscellaneous HTTP exception: %s" % str(exc)) + last_exc = exc + except compat.URLError as exc: + if isinstance(exc.reason, socket.error): + code = exc.reason.errno + if code == 104: # "Connection reset by peer." + continue + raise NetworkError(cause=exc) + except socket.timeout as exc: + _log.info("socket timeout") + last_exc = exc + except socket.error as exc: + if exc.errno == 104: + continue + raise NetworkError(cause=exc) + except IOError as exc: + raise NetworkError(cause=exc) + + # Out of retries! + raise NetworkError("retried %i times" % max_retries, last_exc) + +# Get the XML parsing exceptions to catch. The behavior chnaged with Python 2.7 +# and ElementTree 1.3. +if hasattr(etree, 'ParseError'): + ETREE_EXCEPTIONS = (etree.ParseError, expat.ExpatError) +else: + ETREE_EXCEPTIONS = (expat.ExpatError) + + +# Parsing setup + +def mb_parser_null(resp): + """Return the raw response (XML)""" + return resp + +def mb_parser_xml(resp): + """Return a Python dict representing the XML response""" + # Parse the response. + try: + return mbxml.parse_message(resp) + except UnicodeError as exc: + raise ResponseError(cause=exc) + except Exception as exc: + if isinstance(exc, ETREE_EXCEPTIONS): + raise ResponseError(cause=exc) + else: + raise + +# Defaults +parser_fun = mb_parser_xml +ws_format = "xml" + +def set_parser(new_parser_fun=None): + """Sets the function used to parse the response from the + MusicBrainz web service. + + If no parser is given, the parser is reset to the default parser + :func:`mb_parser_xml`. + """ + global parser_fun + if new_parser_fun is None: + new_parser_fun = mb_parser_xml + if not callable(new_parser_fun): + raise ValueError("new_parser_fun must be callable") + parser_fun = new_parser_fun + +def set_format(fmt="xml"): + """Sets the format that should be returned by the Web Service. + The server currently supports `xml` and `json`. + + This method will set a default parser for the specified format, + but you can modify it with :func:`set_parser`. + + .. warning:: The json format used by the server is different from + the json format returned by the `musicbrainzngs` internal parser + when using the `xml` format! This format may change at any time. + """ + global ws_format + if fmt == "xml": + ws_format = fmt + set_parser() # set to default + elif fmt == "json": + ws_format = fmt + warn("The json format is non-official and may change at any time") + set_parser(json.loads) + else: + raise ValueError("invalid format: %s" % fmt) + + +@_rate_limit +def _mb_request(path, method='GET', auth_required=AUTH_NO, + client_required=False, args=None, data=None, body=None): + """Makes a request for the specified `path` (endpoint) on /ws/2 on + the globally-specified hostname. Parses the responses and returns + the resulting object. `auth_required` and `client_required` control + whether exceptions should be raised if the username/password and + client are left unspecified, respectively. + """ + global parser_fun + + if args is None: + args = {} + else: + args = dict(args) or {} + + if _useragent == "": + raise UsageError("set a proper user-agent with " + "set_useragent(\"application name\", \"application version\", \"contact info (preferably URL or email for your application)\")") + + if client_required: + args["client"] = _client + + if ws_format != "xml": + args["fmt"] = ws_format + + # Convert args from a dictionary to a list of tuples + # so that the ordering of elements is stable for easy + # testing (in this case we order alphabetically) + # Encode Unicode arguments using UTF-8. + newargs = [] + for key, value in sorted(args.items()): + if isinstance(value, compat.unicode): + value = value.encode('utf8') + newargs.append((key, value)) + + # Construct the full URL for the request, including hostname and + # query string. + url = compat.urlunparse(( + 'http', + hostname, + '/ws/2/%s' % path, + '', + compat.urlencode(newargs), + '' + )) + _log.debug("%s request for %s" % (method, url)) + + # Set up HTTP request handler and URL opener. + httpHandler = compat.HTTPHandler(debuglevel=0) + handlers = [httpHandler] + + # Add credentials if required. + add_auth = False + if auth_required == AUTH_YES: + _log.debug("Auth required for %s" % url) + if not user: + raise UsageError("authorization required; " + "use auth(user, pass) first") + add_auth = True + + if auth_required == AUTH_IFSET and user: + _log.debug("Using auth for %s because user and pass is set" % url) + add_auth = True + + if add_auth: + passwordMgr = _RedirectPasswordMgr() + authHandler = _DigestAuthHandler(passwordMgr) + authHandler.add_password("musicbrainz.org", (), user, password) + handlers.append(authHandler) + + opener = compat.build_opener(*handlers) + + # Make request. + req = _MusicbrainzHttpRequest(method, url, data) + req.add_header('User-Agent', _useragent) + _log.debug("requesting with UA %s" % _useragent) + if body: + req.add_header('Content-Type', 'application/xml; charset=UTF-8') + elif not data and not req.has_header('Content-Length'): + # Explicitly indicate zero content length if no request data + # will be sent (avoids HTTP 411 error). + req.add_header('Content-Length', '0') + resp = _safe_read(opener, req, body) + + return parser_fun(resp) + +def _get_auth_type(entity, id, includes): + """ Some calls require authentication. This returns + True if a call does, False otherwise + """ + if "user-tags" in includes or "user-ratings" in includes: + return AUTH_YES + elif entity.startswith("collection"): + if not id: + return AUTH_YES + else: + return AUTH_IFSET + else: + return AUTH_NO + +def _do_mb_query(entity, id, includes=[], params={}): + """Make a single GET call to the MusicBrainz XML API. `entity` is a + string indicated the type of object to be retrieved. The id may be + empty, in which case the query is a search. `includes` is a list + of strings that must be valid includes for the entity type. `params` + is a dictionary of additional parameters for the API call. The + response is parsed and returned. + """ + # Build arguments. + if not isinstance(includes, list): + includes = [includes] + _check_includes(entity, includes) + auth_required = _get_auth_type(entity, id, includes) + args = dict(params) + if len(includes) > 0: + inc = " ".join(includes) + args["inc"] = inc + + # Build the endpoint components. + path = '%s/%s' % (entity, id) + return _mb_request(path, 'GET', auth_required, args=args) + +def _do_mb_search(entity, query='', fields={}, + limit=None, offset=None, strict=False): + """Perform a full-text search on the MusicBrainz search server. + `query` is a lucene query string when no fields are set, + but is escaped when any fields are given. `fields` is a dictionary + of key/value query parameters. They keys in `fields` must be valid + for the given entity type. + """ + # Encode the query terms as a Lucene query string. + query_parts = [] + if query: + clean_query = util._unicode(query) + if fields: + clean_query = re.sub(LUCENE_SPECIAL, r'\\\1', + clean_query) + if strict: + query_parts.append('"%s"' % clean_query) + else: + query_parts.append(clean_query.lower()) + else: + query_parts.append(clean_query) + for key, value in fields.items(): + # Ensure this is a valid search field. + if key not in VALID_SEARCH_FIELDS[entity]: + raise InvalidSearchFieldError( + '%s is not a valid search field for %s' % (key, entity) + ) + elif key == "puid": + warn("PUID support was removed from server\n" + "the 'puid' field is ignored", + Warning, stacklevel=2) + + # Escape Lucene's special characters. + value = util._unicode(value) + value = re.sub(LUCENE_SPECIAL, r'\\\1', value) + if value: + if strict: + query_parts.append('%s:"%s"' % (key, value)) + else: + value = value.lower() # avoid AND / OR + query_parts.append('%s:(%s)' % (key, value)) + if strict: + full_query = ' AND '.join(query_parts).strip() + else: + full_query = ' '.join(query_parts).strip() + + if not full_query: + raise ValueError('at least one query term is required') + + # Additional parameters to the search. + params = {'query': full_query} + if limit: + params['limit'] = str(limit) + if offset: + params['offset'] = str(offset) + + return _do_mb_query(entity, '', [], params) + +def _do_mb_delete(path): + """Send a DELETE request for the specified object. + """ + return _mb_request(path, 'DELETE', AUTH_YES, True) + +def _do_mb_put(path): + """Send a PUT request for the specified object. + """ + return _mb_request(path, 'PUT', AUTH_YES, True) + +def _do_mb_post(path, body): + """Perform a single POST call for an endpoint with a specified + request body. + """ + return _mb_request(path, 'POST', AUTH_YES, True, body=body) + + +# The main interface! + +# Single entity by ID + +@_docstring_get("area") +def get_area_by_id(id, includes=[], release_status=[], release_type=[]): + """Get the area with the MusicBrainz `id` as a dict with an 'area' key. + + *Available includes*: {includes}""" + params = _check_filter_and_make_params("area", includes, + release_status, release_type) + return _do_mb_query("area", id, includes, params) + +@_docstring_get("artist") +def get_artist_by_id(id, includes=[], release_status=[], release_type=[]): + """Get the artist with the MusicBrainz `id` as a dict with an 'artist' key. + + *Available includes*: {includes}""" + params = _check_filter_and_make_params("artist", includes, + release_status, release_type) + return _do_mb_query("artist", id, includes, params) + +@_docstring_get("instrument") +def get_instrument_by_id(id, includes=[], release_status=[], release_type=[]): + """Get the instrument with the MusicBrainz `id` as a dict with an 'artist' key. + + *Available includes*: {includes}""" + params = _check_filter_and_make_params("instrument", includes, + release_status, release_type) + return _do_mb_query("instrument", id, includes, params) + +@_docstring_get("label") +def get_label_by_id(id, includes=[], release_status=[], release_type=[]): + """Get the label with the MusicBrainz `id` as a dict with a 'label' key. + + *Available includes*: {includes}""" + params = _check_filter_and_make_params("label", includes, + release_status, release_type) + return _do_mb_query("label", id, includes, params) + +@_docstring_get("place") +def get_place_by_id(id, includes=[], release_status=[], release_type=[]): + """Get the place with the MusicBrainz `id` as a dict with an 'place' key. + + *Available includes*: {includes}""" + params = _check_filter_and_make_params("place", includes, + release_status, release_type) + return _do_mb_query("place", id, includes, params) + +@_docstring_get("event") +def get_event_by_id(id, includes=[], release_status=[], release_type=[]): + """Get the event with the MusicBrainz `id` as a dict with an 'event' key. + + The event dict has the following keys: + `id`, `type`, `name`, `time`, `disambiguation` and `life-span`. + + *Available includes*: {includes}""" + params = _check_filter_and_make_params("event", includes, + release_status, release_type) + return _do_mb_query("event", id, includes, params) + +@_docstring_get("recording") +def get_recording_by_id(id, includes=[], release_status=[], release_type=[]): + """Get the recording with the MusicBrainz `id` as a dict + with a 'recording' key. + + *Available includes*: {includes}""" + params = _check_filter_and_make_params("recording", includes, + release_status, release_type) + return _do_mb_query("recording", id, includes, params) + +@_docstring_get("release") +def get_release_by_id(id, includes=[], release_status=[], release_type=[]): + """Get the release with the MusicBrainz `id` as a dict with a 'release' key. + + *Available includes*: {includes}""" + params = _check_filter_and_make_params("release", includes, + release_status, release_type) + return _do_mb_query("release", id, includes, params) + +@_docstring_get("release-group") +def get_release_group_by_id(id, includes=[], + release_status=[], release_type=[]): + """Get the release group with the MusicBrainz `id` as a dict + with a 'release-group' key. + + *Available includes*: {includes}""" + params = _check_filter_and_make_params("release-group", includes, + release_status, release_type) + return _do_mb_query("release-group", id, includes, params) + +@_docstring_get("series") +def get_series_by_id(id, includes=[]): + """Get the series with the MusicBrainz `id` as a dict with a 'series' key. + + *Available includes*: {includes}""" + return _do_mb_query("series", id, includes) + +@_docstring_get("work") +def get_work_by_id(id, includes=[]): + """Get the work with the MusicBrainz `id` as a dict with a 'work' key. + + *Available includes*: {includes}""" + return _do_mb_query("work", id, includes) + +@_docstring_get("url") +def get_url_by_id(id, includes=[]): + """Get the url with the MusicBrainz `id` as a dict with a 'url' key. + + *Available includes*: {includes}""" + return _do_mb_query("url", id, includes) + + +# Searching + +@_docstring_search("annotation") +def search_annotations(query='', limit=None, offset=None, strict=False, **fields): + """Search for annotations and return a dict with an 'annotation-list' key. + + *Available search fields*: {fields}""" + return _do_mb_search('annotation', query, fields, limit, offset, strict) + +@_docstring_search("area") +def search_areas(query='', limit=None, offset=None, strict=False, **fields): + """Search for areas and return a dict with an 'area-list' key. + + *Available search fields*: {fields}""" + return _do_mb_search('area', query, fields, limit, offset, strict) + +@_docstring_search("artist") +def search_artists(query='', limit=None, offset=None, strict=False, **fields): + """Search for artists and return a dict with an 'artist-list' key. + + *Available search fields*: {fields}""" + return _do_mb_search('artist', query, fields, limit, offset, strict) + +@_docstring_search("event") +def search_events(query='', limit=None, offset=None, strict=False, **fields): + """Search for events and return a dict with an 'event-list' key. + + *Available search fields*: {fields}""" + return _do_mb_search('event', query, fields, limit, offset, strict) + +@_docstring_search("instrument") +def search_instruments(query='', limit=None, offset=None, strict=False, **fields): + """Search for instruments and return a dict with a 'instrument-list' key. + + *Available search fields*: {fields}""" + return _do_mb_search('instrument', query, fields, limit, offset, strict) + +@_docstring_search("label") +def search_labels(query='', limit=None, offset=None, strict=False, **fields): + """Search for labels and return a dict with a 'label-list' key. + + *Available search fields*: {fields}""" + return _do_mb_search('label', query, fields, limit, offset, strict) + +@_docstring_search("place") +def search_places(query='', limit=None, offset=None, strict=False, **fields): + """Search for places and return a dict with a 'place-list' key. + + *Available search fields*: {fields}""" + return _do_mb_search('place', query, fields, limit, offset, strict) + +@_docstring_search("recording") +def search_recordings(query='', limit=None, offset=None, + strict=False, **fields): + """Search for recordings and return a dict with a 'recording-list' key. + + *Available search fields*: {fields}""" + return _do_mb_search('recording', query, fields, limit, offset, strict) + +@_docstring_search("release") +def search_releases(query='', limit=None, offset=None, strict=False, **fields): + """Search for recordings and return a dict with a 'recording-list' key. + + *Available search fields*: {fields}""" + return _do_mb_search('release', query, fields, limit, offset, strict) + +@_docstring_search("release-group") +def search_release_groups(query='', limit=None, offset=None, + strict=False, **fields): + """Search for release groups and return a dict + with a 'release-group-list' key. + + *Available search fields*: {fields}""" + return _do_mb_search('release-group', query, fields, limit, offset, strict) + +@_docstring_search("series") +def search_series(query='', limit=None, offset=None, strict=False, **fields): + """Search for series and return a dict with a 'series-list' key. + + *Available search fields*: {fields}""" + return _do_mb_search('series', query, fields, limit, offset, strict) + +@_docstring_search("work") +def search_works(query='', limit=None, offset=None, strict=False, **fields): + """Search for works and return a dict with a 'work-list' key. + + *Available search fields*: {fields}""" + return _do_mb_search('work', query, fields, limit, offset, strict) + + +# Lists of entities +@_docstring_get("discid") +def get_releases_by_discid(id, includes=[], toc=None, cdstubs=True, media_format=None): + """Search for releases with a :musicbrainz:`Disc ID` or table of contents. + + When a `toc` is provided and no release with the disc ID is found, + a fuzzy search by the toc is done. + The `toc` should have to same format as :attr:`discid.Disc.toc_string`. + When a `toc` is provided, the format of the discid itself is not + checked server-side, so any value may be passed if searching by only + `toc` is desired. + + If no toc matches in musicbrainz but a :musicbrainz:`CD Stub` does, + the CD Stub will be returned. Prevent this from happening by + passing `cdstubs=False`. + + By default only results that match a format that allows discids + (e.g. CD) are included. To include all media formats, pass + `media_format='all'`. + + The result is a dict with either a 'disc' , a 'cdstub' key + or a 'release-list' (fuzzy match with TOC). + A 'disc' has an 'offset-count', an 'offset-list' and a 'release-list'. + A 'cdstub' key has direct 'artist' and 'title' keys. + + *Available includes*: {includes}""" + params = _check_filter_and_make_params("discid", includes, release_status=[], + release_type=[]) + if toc: + params["toc"] = toc + if not cdstubs: + params["cdstubs"] = "no" + if media_format: + params["media-format"] = media_format + return _do_mb_query("discid", id, includes, params) + +@_docstring_get("recording") +def get_recordings_by_echoprint(echoprint, includes=[], release_status=[], + release_type=[]): + """Search for recordings with an `echoprint `_. + (not available on server)""" + warn("Echoprints were never introduced\n" + "and will not be found (404)", + Warning, stacklevel=2) + raise ResponseError(cause=compat.HTTPError( + None, 404, "Not Found", None, None)) + +@_docstring_get("recording") +def get_recordings_by_puid(puid, includes=[], release_status=[], + release_type=[]): + """Search for recordings with a :musicbrainz:`PUID`. + (not available on server)""" + warn("PUID support was removed from the server\n" + "and no PUIDs will be found (404)", + Warning, stacklevel=2) + raise ResponseError(cause=compat.HTTPError( + None, 404, "Not Found", None, None)) + +@_docstring_get("recording") +def get_recordings_by_isrc(isrc, includes=[], release_status=[], + release_type=[]): + """Search for recordings with an :musicbrainz:`ISRC`. + The result is a dict with an 'isrc' key, + which again includes a 'recording-list'. + + *Available includes*: {includes}""" + params = _check_filter_and_make_params("isrc", includes, + release_status, release_type) + return _do_mb_query("isrc", isrc, includes, params) + +@_docstring_get("work") +def get_works_by_iswc(iswc, includes=[]): + """Search for works with an :musicbrainz:`ISWC`. + The result is a dict with a`work-list`. + + *Available includes*: {includes}""" + return _do_mb_query("iswc", iswc, includes) + + +def _browse_impl(entity, includes, limit, offset, params, release_status=[], release_type=[]): + includes = includes if isinstance(includes, list) else [includes] + valid_includes = VALID_BROWSE_INCLUDES[entity] + _check_includes_impl(includes, valid_includes) + p = {} + for k,v in params.items(): + if v: + p[k] = v + if len(p) > 1: + raise Exception("Can't have more than one of " + ", ".join(params.keys())) + if limit: p["limit"] = limit + if offset: p["offset"] = offset + filterp = _check_filter_and_make_params(entity, includes, release_status, release_type) + p.update(filterp) + return _do_mb_query(entity, "", includes, p) + +# Browse methods +# Browse include are a subset of regular get includes, so we check them here +# and the test in _do_mb_query will pass anyway. +@_docstring_browse("artist") +def browse_artists(recording=None, release=None, release_group=None, + work=None, includes=[], limit=None, offset=None): + """Get all artists linked to a recording, a release or a release group. + You need to give one MusicBrainz ID. + + *Available includes*: {includes}""" + params = {"recording": recording, + "release": release, + "release-group": release_group, + "work": work} + return _browse_impl("artist", includes, limit, offset, params) + +@_docstring_browse("event") +def browse_events(area=None, artist=None, place=None, + includes=[], limit=None, offset=None): + """Get all events linked to a area, a artist or a place. + You need to give one MusicBrainz ID. + + *Available includes*: {includes}""" + params = {"area": area, + "artist": artist, + "place": place} + return _browse_impl("event", includes, limit, offset, params) + +@_docstring_browse("label") +def browse_labels(release=None, includes=[], limit=None, offset=None): + """Get all labels linked to a relase. You need to give a MusicBrainz ID. + + *Available includes*: {includes}""" + params = {"release": release} + return _browse_impl("label", includes, limit, offset, params) + +@_docstring_browse("place") +def browse_places(area=None, includes=[], limit=None, offset=None): + """Get all places linked to an area. You need to give a MusicBrainz ID. + + *Available includes*: {includes}""" + params = {"area": area} + return _browse_impl("place", includes, limit, offset, params) + +@_docstring_browse("recording") +def browse_recordings(artist=None, release=None, includes=[], + limit=None, offset=None): + """Get all recordings linked to an artist or a release. + You need to give one MusicBrainz ID. + + *Available includes*: {includes}""" + params = {"artist": artist, + "release": release} + return _browse_impl("recording", includes, limit, offset, params) + +@_docstring_browse("release") +def browse_releases(artist=None, track_artist=None, label=None, recording=None, + release_group=None, release_status=[], release_type=[], + includes=[], limit=None, offset=None): + """Get all releases linked to an artist, a label, a recording + or a release group. You need to give one MusicBrainz ID. + + You can also browse by `track_artist`, which gives all releases where some + tracks are attributed to that artist, but not the whole release. + + You can filter by :data:`musicbrainz.VALID_RELEASE_TYPES` or + :data:`musicbrainz.VALID_RELEASE_STATUSES`. + + *Available includes*: {includes}""" + # track_artist param doesn't work yet + params = {"artist": artist, + "track_artist": track_artist, + "label": label, + "recording": recording, + "release-group": release_group} + return _browse_impl("release", includes, limit, offset, + params, release_status, release_type) + +@_docstring_browse("release-group") +def browse_release_groups(artist=None, release=None, release_type=[], + includes=[], limit=None, offset=None): + """Get all release groups linked to an artist or a release. + You need to give one MusicBrainz ID. + + You can filter by :data:`musicbrainz.VALID_RELEASE_TYPES`. + + *Available includes*: {includes}""" + params = {"artist": artist, + "release": release} + return _browse_impl("release-group", includes, limit, + offset, params, [], release_type) + +@_docstring_browse("url") +def browse_urls(resource=None, includes=[], limit=None, offset=None): + """Get urls by actual URL string. + You need to give a URL string as 'resource' + + *Available includes*: {includes}""" + params = {"resource": resource} + return _browse_impl("url", includes, limit, offset, params) + +@_docstring_browse("work") +def browse_works(artist=None, includes=[], limit=None, offset=None): + """Get all works linked to an artist + + *Available includes*: {includes}""" + params = {"artist": artist} + return _browse_impl("work", includes, limit, offset, params) + +# Collections +def get_collections(): + """List the collections for the currently :func:`authenticated ` user + as a dict with a 'collection-list' key.""" + # Missing the count in the reply + return _do_mb_query("collection", '') + +def _do_collection_query(collection, collection_type, limit, offset): + params = {} + if limit: params["limit"] = limit + if offset: params["offset"] = offset + return _do_mb_query("collection", "%s/%s" % (collection, collection_type), [], params) + +def get_artists_in_collection(collection, limit=None, offset=None): + """List the artists in a collection. + Returns a dict with a 'collection' key, which again has a 'artist-list'. + + See `Browsing`_ for how to use `limit` and `offset`. + """ + return _do_collection_query(collection, "artists", limit, offset) + +def get_releases_in_collection(collection, limit=None, offset=None): + """List the releases in a collection. + Returns a dict with a 'collection' key, which again has a 'release-list'. + + See `Browsing`_ for how to use `limit` and `offset`. + """ + return _do_collection_query(collection, "releases", limit, offset) + +def get_events_in_collection(collection, limit=None, offset=None): + """List the events in a collection. + Returns a dict with a 'collection' key, which again has a 'event-list'. + + See `Browsing`_ for how to use `limit` and `offset`. + """ + return _do_collection_query(collection, "events", limit, offset) + +def get_places_in_collection(collection, limit=None, offset=None): + """List the places in a collection. + Returns a dict with a 'collection' key, which again has a 'place-list'. + + See `Browsing`_ for how to use `limit` and `offset`. + """ + return _do_collection_query(collection, "places", limit, offset) + +def get_recordings_in_collection(collection, limit=None, offset=None): + """List the recordings in a collection. + Returns a dict with a 'collection' key, which again has a 'recording-list'. + + See `Browsing`_ for how to use `limit` and `offset`. + """ + return _do_collection_query(collection, "recordings", limit, offset) + +def get_works_in_collection(collection, limit=None, offset=None): + """List the works in a collection. + Returns a dict with a 'collection' key, which again has a 'work-list'. + + See `Browsing`_ for how to use `limit` and `offset`. + """ + return _do_collection_query(collection, "works", limit, offset) + + +# Submission methods + +def submit_barcodes(release_barcode): + """Submits a set of {release_id1: barcode, ...}""" + query = mbxml.make_barcode_request(release_barcode) + return _do_mb_post("release", query) + +def submit_puids(recording_puids): + """Submit PUIDs. + (Functionality removed from server) + """ + warn("PUID support was dropped at the server\n" + "nothing will be submitted", + Warning, stacklevel=2) + return {'message': {'text': 'OK'}} + +def submit_echoprints(recording_echoprints): + """Submit echoprints. + (Functionality removed from server) + """ + warn("Echoprints were never introduced\n" + "nothing will be submitted", + Warning, stacklevel=2) + return {'message': {'text': 'OK'}} + +def submit_isrcs(recording_isrcs): + """Submit ISRCs. + Submits a set of {recording-id1: [isrc1, ...], ...} + or {recording_id1: isrc, ...}. + """ + rec2isrcs = dict() + for (rec, isrcs) in recording_isrcs.items(): + rec2isrcs[rec] = isrcs if isinstance(isrcs, list) else [isrcs] + query = mbxml.make_isrc_request(rec2isrcs) + return _do_mb_post("recording", query) + +def submit_tags(**kwargs): + """Submit user tags. + Takes parameters named e.g. 'artist_tags', 'recording_tags', etc., + and of the form: + {entity_id1: [tag1, ...], ...} + If you only have one tag for an entity you can use a string instead + of a list. + + The user's tags for each entity will be set to that list, adding or + removing tags as necessary. Submitting an empty list for an entity + will remove all tags for that entity by the user. + """ + for k, v in kwargs.items(): + for id, tags in v.items(): + kwargs[k][id] = tags if isinstance(tags, list) else [tags] + + query = mbxml.make_tag_request(**kwargs) + return _do_mb_post("tag", query) + +def submit_ratings(**kwargs): + """Submit user ratings. + Takes parameters named e.g. 'artist_ratings', 'recording_ratings', etc., + and of the form: + {entity_id1: rating, ...} + + Ratings are numbers from 0-100, at intervals of 20 (20 per 'star'). + Submitting a rating of 0 will remove the user's rating. + """ + query = mbxml.make_rating_request(**kwargs) + return _do_mb_post("rating", query) + +def add_releases_to_collection(collection, releases=[]): + """Add releases to a collection. + Collection and releases should be identified by their MBIDs + """ + # XXX: Maximum URI length of 16kb means we should only allow ~400 releases + releaselist = ";".join(releases) + return _do_mb_put("collection/%s/releases/%s" % (collection, releaselist)) + +def remove_releases_from_collection(collection, releases=[]): + """Remove releases from a collection. + Collection and releases should be identified by their MBIDs + """ + releaselist = ";".join(releases) + return _do_mb_delete("collection/%s/releases/%s" % (collection, releaselist)) diff --git a/libs/musicbrainzngs/util.py b/libs/musicbrainzngs/util.py new file mode 100644 index 00000000..37316f53 --- /dev/null +++ b/libs/musicbrainzngs/util.py @@ -0,0 +1,44 @@ +# This file is part of the musicbrainzngs library +# Copyright (C) Alastair Porter, Adrian Sampson, and others +# This file is distributed under a BSD-2-Clause type license. +# See the COPYING file for more information. + +import sys +import locale +import xml.etree.ElementTree as ET + +from . import compat + +def _unicode(string, encoding=None): + """Try to decode byte strings to unicode. + This can only be a guess, but this might be better than failing. + It is safe to use this on numbers or strings that are already unicode. + """ + if isinstance(string, compat.unicode): + unicode_string = string + elif isinstance(string, compat.bytes): + # use given encoding, stdin, preferred until something != None is found + if encoding is None: + encoding = sys.stdin.encoding + if encoding is None: + encoding = locale.getpreferredencoding() + unicode_string = string.decode(encoding, "ignore") + else: + unicode_string = compat.unicode(string) + return unicode_string.replace('\x00', '').strip() + +def bytes_to_elementtree(bytes_or_file): + """Given a bytestring or a file-like object that will produce them, + parse and return an ElementTree. + """ + if isinstance(bytes_or_file, compat.basestring): + s = bytes_or_file + else: + s = bytes_or_file.read() + + if compat.is_py3: + s = _unicode(s, "utf-8") + + f = compat.StringIO(s) + tree = ET.ElementTree(file=f) + return tree diff --git a/libs/mutagen/__init__.py b/libs/mutagen/__init__.py index 28febab3..c1abc0b1 100644 --- a/libs/mutagen/__init__.py +++ b/libs/mutagen/__init__.py @@ -1,4 +1,5 @@ -# mutagen aims to be an all purpose media tagging library +# -*- coding: utf-8 -*- + # Copyright (C) 2005 Michael Urman # # This program is free software; you can redistribute it and/or modify @@ -6,7 +7,7 @@ # published by the Free Software Foundation. -"""Mutagen aims to be an all purpose tagging library. +"""Mutagen aims to be an all purpose multimedia tagging library. :: @@ -19,245 +20,26 @@ depending on tag or format. They may also be entirely different objects for certain keys, again depending on format. """ -version = (1, 22) +from mutagen._util import MutagenError +from mutagen._file import FileType, StreamInfo, File +from mutagen._tags import Tags, Metadata, PaddingInfo + +version = (1, 32) """Version tuple.""" version_string = ".".join(map(str, version)) """Version string.""" +MutagenError -import warnings +FileType -import mutagen._util +StreamInfo +File -class Metadata(object): - """An abstract dict-like object. +Tags - Metadata is the base class for many of the tag objects in Mutagen. - """ +Metadata - def __init__(self, *args, **kwargs): - if args or kwargs: - self.load(*args, **kwargs) - - def load(self, *args, **kwargs): - raise NotImplementedError - - def save(self, filename=None): - """Save changes to a file.""" - - raise NotImplementedError - - def delete(self, filename=None): - """Remove tags from a file.""" - - raise NotImplementedError - - -class FileType(mutagen._util.DictMixin): - """An abstract object wrapping tags and audio stream information. - - Attributes: - - * info -- stream information (length, bitrate, sample rate) - * tags -- metadata tags, if any - - Each file format has different potential tags and stream - information. - - FileTypes implement an interface very similar to Metadata; the - dict interface, save, load, and delete calls on a FileType call - the appropriate methods on its tag data. - """ - - info = None - tags = None - filename = None - _mimes = ["application/octet-stream"] - - def __init__(self, filename=None, *args, **kwargs): - if filename is None: - warnings.warn("FileType constructor requires a filename", - DeprecationWarning) - else: - self.load(filename, *args, **kwargs) - - def load(self, filename, *args, **kwargs): - raise NotImplementedError - - def __getitem__(self, key): - """Look up a metadata tag key. - - If the file has no tags at all, a KeyError is raised. - """ - - if self.tags is None: - raise KeyError(key) - else: - return self.tags[key] - - def __setitem__(self, key, value): - """Set a metadata tag. - - If the file has no tags, an appropriate format is added (but - not written until save is called). - """ - - if self.tags is None: - self.add_tags() - self.tags[key] = value - - def __delitem__(self, key): - """Delete a metadata tag key. - - If the file has no tags at all, a KeyError is raised. - """ - - if self.tags is None: - raise KeyError(key) - else: - del(self.tags[key]) - - def keys(self): - """Return a list of keys in the metadata tag. - - If the file has no tags at all, an empty list is returned. - """ - - if self.tags is None: - return [] - else: - return self.tags.keys() - - def delete(self, filename=None): - """Remove tags from a file.""" - - if self.tags is not None: - if filename is None: - filename = self.filename - else: - warnings.warn( - "delete(filename=...) is deprecated, reload the file", - DeprecationWarning) - return self.tags.delete(filename) - - def save(self, filename=None, **kwargs): - """Save metadata tags.""" - - if filename is None: - filename = self.filename - else: - warnings.warn( - "save(filename=...) is deprecated, reload the file", - DeprecationWarning) - if self.tags is not None: - return self.tags.save(filename, **kwargs) - else: - raise ValueError("no tags in file") - - def pprint(self): - """Print stream information and comment key=value pairs.""" - - stream = "%s (%s)" % (self.info.pprint(), self.mime[0]) - try: - tags = self.tags.pprint() - except AttributeError: - return stream - else: - return stream + ((tags and "\n" + tags) or "") - - def add_tags(self): - """Adds new tags to the file. - - Raises if tags already exist. - """ - - raise NotImplementedError - - @property - def mime(self): - """A list of mime types""" - - mimes = [] - for Kind in type(self).__mro__: - for mime in getattr(Kind, '_mimes', []): - if mime not in mimes: - mimes.append(mime) - return mimes - - @staticmethod - def score(filename, fileobj, header): - raise NotImplementedError - - -def File(filename, options=None, easy=False): - """Guess the type of the file and try to open it. - - The file type is decided by several things, such as the first 128 - bytes (which usually contains a file type identifier), the - filename extension, and the presence of existing tags. - - If no appropriate type could be found, None is returned. - - :param options: Sequence of :class:`FileType` implementations, defaults to - all included ones. - - :param easy: If the easy wrappers should be returnd if available. - For example :class:`EasyMP3 ` instead - of :class:`MP3 `. - """ - - if options is None: - from mutagen.asf import ASF - from mutagen.apev2 import APEv2File - from mutagen.flac import FLAC - if easy: - from mutagen.easyid3 import EasyID3FileType as ID3FileType - else: - from mutagen.id3 import ID3FileType - if easy: - from mutagen.mp3 import EasyMP3 as MP3 - else: - from mutagen.mp3 import MP3 - from mutagen.oggflac import OggFLAC - from mutagen.oggspeex import OggSpeex - from mutagen.oggtheora import OggTheora - from mutagen.oggvorbis import OggVorbis - from mutagen.oggopus import OggOpus - if easy: - from mutagen.trueaudio import EasyTrueAudio as TrueAudio - else: - from mutagen.trueaudio import TrueAudio - from mutagen.wavpack import WavPack - if easy: - from mutagen.easymp4 import EasyMP4 as MP4 - else: - from mutagen.mp4 import MP4 - from mutagen.musepack import Musepack - from mutagen.monkeysaudio import MonkeysAudio - from mutagen.optimfrog import OptimFROG - options = [MP3, TrueAudio, OggTheora, OggSpeex, OggVorbis, OggFLAC, - FLAC, APEv2File, MP4, ID3FileType, WavPack, Musepack, - MonkeysAudio, OptimFROG, ASF, OggOpus] - - if not options: - return None - - fileobj = open(filename, "rb") - try: - header = fileobj.read(128) - # Sort by name after score. Otherwise import order affects - # Kind sort order, which affects treatment of things with - # equals scores. - results = [(Kind.score(filename, fileobj, header), Kind.__name__) - for Kind in options] - finally: - fileobj.close() - results = zip(results, options) - results.sort() - (score, name), Kind = results[-1] - if score > 0: - return Kind(filename) - else: - return None +PaddingInfo diff --git a/libs/mutagen/_compat.py b/libs/mutagen/_compat.py new file mode 100644 index 00000000..77c465f1 --- /dev/null +++ b/libs/mutagen/_compat.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2013 Christoph Reiter +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of version 2 of the GNU General Public License as +# published by the Free Software Foundation. + +import sys + + +PY2 = sys.version_info[0] == 2 +PY3 = not PY2 + +if PY2: + from StringIO import StringIO + BytesIO = StringIO + from cStringIO import StringIO as cBytesIO + from itertools import izip + + long_ = long + integer_types = (int, long) + string_types = (str, unicode) + text_type = unicode + + xrange = xrange + cmp = cmp + chr_ = chr + + def endswith(text, end): + return text.endswith(end) + + iteritems = lambda d: d.iteritems() + itervalues = lambda d: d.itervalues() + iterkeys = lambda d: d.iterkeys() + + iterbytes = lambda b: iter(b) + + exec("def reraise(tp, value, tb):\n raise tp, value, tb") + + def swap_to_string(cls): + if "__str__" in cls.__dict__: + cls.__unicode__ = cls.__str__ + + if "__bytes__" in cls.__dict__: + cls.__str__ = cls.__bytes__ + + return cls + +elif PY3: + from io import StringIO + StringIO = StringIO + from io import BytesIO + cBytesIO = BytesIO + + long_ = int + integer_types = (int,) + string_types = (str,) + text_type = str + + izip = zip + xrange = range + cmp = lambda a, b: (a > b) - (a < b) + chr_ = lambda x: bytes([x]) + + def endswith(text, end): + # usefull for paths which can be both, str and bytes + if isinstance(text, str): + if not isinstance(end, str): + end = end.decode("ascii") + else: + if not isinstance(end, bytes): + end = end.encode("ascii") + return text.endswith(end) + + iteritems = lambda d: iter(d.items()) + itervalues = lambda d: iter(d.values()) + iterkeys = lambda d: iter(d.keys()) + + iterbytes = lambda b: (bytes([v]) for v in b) + + def reraise(tp, value, tb): + raise tp(value).with_traceback(tb) + + def swap_to_string(cls): + return cls diff --git a/libs/mutagen/_constants.py b/libs/mutagen/_constants.py index f5ecd90c..62c1ce02 100644 --- a/libs/mutagen/_constants.py +++ b/libs/mutagen/_constants.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + """Constants used by Mutagen.""" GENRES = [ diff --git a/libs/mutagen/_file.py b/libs/mutagen/_file.py new file mode 100644 index 00000000..95f400cf --- /dev/null +++ b/libs/mutagen/_file.py @@ -0,0 +1,255 @@ +# Copyright (C) 2005 Michael Urman +# -*- coding: utf-8 -*- +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of version 2 of the GNU General Public License as +# published by the Free Software Foundation. + +import warnings + +from mutagen._util import DictMixin +from mutagen._compat import izip + + +class FileType(DictMixin): + """An abstract object wrapping tags and audio stream information. + + Attributes: + + * info -- :class:`StreamInfo` -- (length, bitrate, sample rate) + * tags -- :class:`Tags` -- metadata tags, if any + + Each file format has different potential tags and stream + information. + + FileTypes implement an interface very similar to Metadata; the + dict interface, save, load, and delete calls on a FileType call + the appropriate methods on its tag data. + """ + + __module__ = "mutagen" + + info = None + tags = None + filename = None + _mimes = ["application/octet-stream"] + + def __init__(self, filename=None, *args, **kwargs): + if filename is None: + warnings.warn("FileType constructor requires a filename", + DeprecationWarning) + else: + self.load(filename, *args, **kwargs) + + def load(self, filename, *args, **kwargs): + raise NotImplementedError + + def __getitem__(self, key): + """Look up a metadata tag key. + + If the file has no tags at all, a KeyError is raised. + """ + + if self.tags is None: + raise KeyError(key) + else: + return self.tags[key] + + def __setitem__(self, key, value): + """Set a metadata tag. + + If the file has no tags, an appropriate format is added (but + not written until save is called). + """ + + if self.tags is None: + self.add_tags() + self.tags[key] = value + + def __delitem__(self, key): + """Delete a metadata tag key. + + If the file has no tags at all, a KeyError is raised. + """ + + if self.tags is None: + raise KeyError(key) + else: + del(self.tags[key]) + + def keys(self): + """Return a list of keys in the metadata tag. + + If the file has no tags at all, an empty list is returned. + """ + + if self.tags is None: + return [] + else: + return self.tags.keys() + + def delete(self, filename=None): + """Remove tags from a file. + + In cases where the tagging format is independent of the file type + (for example `mutagen.ID3`) all traces of the tagging format will + be removed. + In cases where the tag is part of the file type, all tags and + padding will be removed. + + The tags attribute will be cleared as well if there is one. + + Does nothing if the file has no tags. + + :raises mutagen.MutagenError: if deleting wasn't possible + """ + + if self.tags is not None: + if filename is None: + filename = self.filename + else: + warnings.warn( + "delete(filename=...) is deprecated, reload the file", + DeprecationWarning) + return self.tags.delete(filename) + + def save(self, filename=None, **kwargs): + """Save metadata tags. + + :raises mutagen.MutagenError: if saving wasn't possible + """ + + if filename is None: + filename = self.filename + else: + warnings.warn( + "save(filename=...) is deprecated, reload the file", + DeprecationWarning) + + if self.tags is not None: + return self.tags.save(filename, **kwargs) + + def pprint(self): + """Print stream information and comment key=value pairs.""" + + stream = "%s (%s)" % (self.info.pprint(), self.mime[0]) + try: + tags = self.tags.pprint() + except AttributeError: + return stream + else: + return stream + ((tags and "\n" + tags) or "") + + def add_tags(self): + """Adds new tags to the file. + + :raises mutagen.MutagenError: if tags already exist or adding is not + possible. + """ + + raise NotImplementedError + + @property + def mime(self): + """A list of mime types""" + + mimes = [] + for Kind in type(self).__mro__: + for mime in getattr(Kind, '_mimes', []): + if mime not in mimes: + mimes.append(mime) + return mimes + + @staticmethod + def score(filename, fileobj, header): + raise NotImplementedError + + +class StreamInfo(object): + """Abstract stream information object. + + Provides attributes for length, bitrate, sample rate etc. + + See the implementations for details. + """ + + __module__ = "mutagen" + + def pprint(self): + """Print stream information""" + + raise NotImplementedError + + +def File(filename, options=None, easy=False): + """Guess the type of the file and try to open it. + + The file type is decided by several things, such as the first 128 + bytes (which usually contains a file type identifier), the + filename extension, and the presence of existing tags. + + If no appropriate type could be found, None is returned. + + :param options: Sequence of :class:`FileType` implementations, defaults to + all included ones. + + :param easy: If the easy wrappers should be returnd if available. + For example :class:`EasyMP3 ` instead + of :class:`MP3 `. + """ + + if options is None: + from mutagen.asf import ASF + from mutagen.apev2 import APEv2File + from mutagen.flac import FLAC + if easy: + from mutagen.easyid3 import EasyID3FileType as ID3FileType + else: + from mutagen.id3 import ID3FileType + if easy: + from mutagen.mp3 import EasyMP3 as MP3 + else: + from mutagen.mp3 import MP3 + from mutagen.oggflac import OggFLAC + from mutagen.oggspeex import OggSpeex + from mutagen.oggtheora import OggTheora + from mutagen.oggvorbis import OggVorbis + from mutagen.oggopus import OggOpus + if easy: + from mutagen.trueaudio import EasyTrueAudio as TrueAudio + else: + from mutagen.trueaudio import TrueAudio + from mutagen.wavpack import WavPack + if easy: + from mutagen.easymp4 import EasyMP4 as MP4 + else: + from mutagen.mp4 import MP4 + from mutagen.musepack import Musepack + from mutagen.monkeysaudio import MonkeysAudio + from mutagen.optimfrog import OptimFROG + from mutagen.aiff import AIFF + from mutagen.aac import AAC + from mutagen.smf import SMF + options = [MP3, TrueAudio, OggTheora, OggSpeex, OggVorbis, OggFLAC, + FLAC, AIFF, APEv2File, MP4, ID3FileType, WavPack, + Musepack, MonkeysAudio, OptimFROG, ASF, OggOpus, AAC, + SMF] + + if not options: + return None + + with open(filename, "rb") as fileobj: + header = fileobj.read(128) + # Sort by name after score. Otherwise import order affects + # Kind sort order, which affects treatment of things with + # equals scores. + results = [(Kind.score(filename, fileobj, header), Kind.__name__) + for Kind in options] + + results = list(izip(results, options)) + results.sort() + (score, name), Kind = results[-1] + if score > 0: + return Kind(filename) + else: + return None diff --git a/libs/mutagen/_mp3util.py b/libs/mutagen/_mp3util.py new file mode 100644 index 00000000..409cadcb --- /dev/null +++ b/libs/mutagen/_mp3util.py @@ -0,0 +1,420 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 Christoph Reiter +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of version 2 of the GNU General Public License as +# published by the Free Software Foundation. + +""" +http://www.codeproject.com/Articles/8295/MPEG-Audio-Frame-Header +http://wiki.hydrogenaud.io/index.php?title=MP3 +""" + +from functools import partial + +from ._util import cdata, BitReader +from ._compat import xrange, iterbytes, cBytesIO + + +class LAMEError(Exception): + pass + + +class LAMEHeader(object): + """http://gabriel.mp3-tech.org/mp3infotag.html""" + + vbr_method = 0 + """0: unknown, 1: CBR, 2: ABR, 3/4/5: VBR, others: see the docs""" + + lowpass_filter = 0 + """lowpass filter value in Hz. 0 means unknown""" + + quality = -1 + """Encoding quality: 0..9""" + + vbr_quality = -1 + """VBR quality: 0..9""" + + track_peak = None + """Peak signal amplitude as float. None if unknown.""" + + track_gain_origin = 0 + """see the docs""" + + track_gain_adjustment = None + """Track gain adjustment as float (for 89db replay gain) or None""" + + album_gain_origin = 0 + """see the docs""" + + album_gain_adjustment = None + """Album gain adjustment as float (for 89db replay gain) or None""" + + encoding_flags = 0 + """see docs""" + + ath_type = -1 + """see docs""" + + bitrate = -1 + """Bitrate in kbps. For VBR the minimum bitrate, for anything else + (CBR, ABR, ..) the target bitrate. + """ + + encoder_delay_start = 0 + """Encoder delay in samples""" + + encoder_padding_end = 0 + """Padding in samples added at the end""" + + source_sample_frequency_enum = -1 + """see docs""" + + unwise_setting_used = False + """see docs""" + + stereo_mode = 0 + """see docs""" + + noise_shaping = 0 + """see docs""" + + mp3_gain = 0 + """Applied MP3 gain -127..127. Factor is 2 ** (mp3_gain / 4)""" + + surround_info = 0 + """see docs""" + + preset_used = 0 + """lame preset""" + + music_length = 0 + """Length in bytes excluding any ID3 tags""" + + music_crc = -1 + """CRC16 of the data specified by music_length""" + + header_crc = -1 + """CRC16 of this header and everything before (not checked)""" + + def __init__(self, xing, fileobj): + """Raises LAMEError if parsing fails""" + + payload = fileobj.read(27) + if len(payload) != 27: + raise LAMEError("Not enough data") + + # extended lame header + r = BitReader(cBytesIO(payload)) + revision = r.bits(4) + if revision != 0: + raise LAMEError("unsupported header revision %d" % revision) + + self.vbr_method = r.bits(4) + self.lowpass_filter = r.bits(8) * 100 + + # these have a different meaning for lame; expose them again here + self.quality = (100 - xing.vbr_scale) % 10 + self.vbr_quality = (100 - xing.vbr_scale) // 10 + + track_peak_data = r.bytes(4) + if track_peak_data == b"\x00\x00\x00\x00": + self.track_peak = None + else: + # see PutLameVBR() in LAME's VbrTag.c + self.track_peak = ( + cdata.uint32_be(track_peak_data) - 0.5) / 2 ** 23 + track_gain_type = r.bits(3) + self.track_gain_origin = r.bits(3) + sign = r.bits(1) + gain_adj = r.bits(9) / 10.0 + if sign: + gain_adj *= -1 + if track_gain_type == 1: + self.track_gain_adjustment = gain_adj + else: + self.track_gain_adjustment = None + assert r.is_aligned() + + album_gain_type = r.bits(3) + self.album_gain_origin = r.bits(3) + sign = r.bits(1) + album_gain_adj = r.bits(9) / 10.0 + if album_gain_type == 2: + self.album_gain_adjustment = album_gain_adj + else: + self.album_gain_adjustment = None + + self.encoding_flags = r.bits(4) + self.ath_type = r.bits(4) + + self.bitrate = r.bits(8) + + self.encoder_delay_start = r.bits(12) + self.encoder_padding_end = r.bits(12) + + self.source_sample_frequency_enum = r.bits(2) + self.unwise_setting_used = r.bits(1) + self.stereo_mode = r.bits(3) + self.noise_shaping = r.bits(2) + + sign = r.bits(1) + mp3_gain = r.bits(7) + if sign: + mp3_gain *= -1 + self.mp3_gain = mp3_gain + + r.skip(2) + self.surround_info = r.bits(3) + self.preset_used = r.bits(11) + self.music_length = r.bits(32) + self.music_crc = r.bits(16) + + self.header_crc = r.bits(16) + assert r.is_aligned() + + @classmethod + def parse_version(cls, fileobj): + """Returns a version string and True if a LAMEHeader follows. + The passed file object will be positioned right before the + lame header if True. + + Raises LAMEError if there is no lame version info. + """ + + # http://wiki.hydrogenaud.io/index.php?title=LAME_version_string + + data = fileobj.read(20) + if len(data) != 20: + raise LAMEError("Not a lame header") + if not data.startswith((b"LAME", b"L3.99")): + raise LAMEError("Not a lame header") + + data = data.lstrip(b"EMAL") + major, data = data[0:1], data[1:].lstrip(b".") + minor = b"" + for c in iterbytes(data): + if not c.isdigit(): + break + minor += c + data = data[len(minor):] + + try: + major = int(major.decode("ascii")) + minor = int(minor.decode("ascii")) + except ValueError: + raise LAMEError + + # the extended header was added sometimes in the 3.90 cycle + # e.g. "LAME3.90 (alpha)" should still stop here. + # (I have seen such a file) + if (major, minor) < (3, 90) or ( + (major, minor) == (3, 90) and data[-11:-10] == b"("): + flag = data.strip(b"\x00").rstrip().decode("ascii") + return u"%d.%d%s" % (major, minor, flag), False + + if len(data) <= 11: + raise LAMEError("Invalid version: too long") + + flag = data[:-11].rstrip(b"\x00") + + flag_string = u"" + patch = u"" + if flag == b"a": + flag_string = u" (alpha)" + elif flag == b"b": + flag_string = u" (beta)" + elif flag == b"r": + patch = u".1+" + elif flag == b" ": + if (major, minor) > (3, 96): + patch = u".0" + else: + patch = u".0+" + elif flag == b"" or flag == b".": + patch = u".0+" + else: + flag_string = u" (?)" + + # extended header, seek back to 9 bytes for the caller + fileobj.seek(-11, 1) + + return u"%d.%d%s%s" % (major, minor, patch, flag_string), True + + +class XingHeaderError(Exception): + pass + + +class XingHeaderFlags(object): + FRAMES = 0x1 + BYTES = 0x2 + TOC = 0x4 + VBR_SCALE = 0x8 + + +class XingHeader(object): + + frames = -1 + """Number of frames, -1 if unknown""" + + bytes = -1 + """Number of bytes, -1 if unknown""" + + toc = [] + """List of 100 file offsets in percent encoded as 0-255. E.g. entry + 50 contains the file offset in percent at 50% play time. + Empty if unknown. + """ + + vbr_scale = -1 + """VBR quality indicator 0-100. -1 if unknown""" + + lame_header = None + """A LAMEHeader instance or None""" + + lame_version = u"" + """The version of the LAME encoder e.g. '3.99.0'. Empty if unknown""" + + is_info = False + """If the header started with 'Info' and not 'Xing'""" + + def __init__(self, fileobj): + """Parses the Xing header or raises XingHeaderError. + + The file position after this returns is undefined. + """ + + data = fileobj.read(8) + if len(data) != 8 or data[:4] not in (b"Xing", b"Info"): + raise XingHeaderError("Not a Xing header") + + self.is_info = (data[:4] == b"Info") + + flags = cdata.uint32_be_from(data, 4)[0] + + if flags & XingHeaderFlags.FRAMES: + data = fileobj.read(4) + if len(data) != 4: + raise XingHeaderError("Xing header truncated") + self.frames = cdata.uint32_be(data) + + if flags & XingHeaderFlags.BYTES: + data = fileobj.read(4) + if len(data) != 4: + raise XingHeaderError("Xing header truncated") + self.bytes = cdata.uint32_be(data) + + if flags & XingHeaderFlags.TOC: + data = fileobj.read(100) + if len(data) != 100: + raise XingHeaderError("Xing header truncated") + self.toc = list(bytearray(data)) + + if flags & XingHeaderFlags.VBR_SCALE: + data = fileobj.read(4) + if len(data) != 4: + raise XingHeaderError("Xing header truncated") + self.vbr_scale = cdata.uint32_be(data) + + try: + self.lame_version, has_header = LAMEHeader.parse_version(fileobj) + if has_header: + self.lame_header = LAMEHeader(self, fileobj) + except LAMEError: + pass + + @classmethod + def get_offset(cls, info): + """Calculate the offset to the Xing header from the start of the + MPEG header including sync based on the MPEG header's content. + """ + + assert info.layer == 3 + + if info.version == 1: + if info.mode != 3: + return 36 + else: + return 21 + else: + if info.mode != 3: + return 21 + else: + return 13 + + +class VBRIHeaderError(Exception): + pass + + +class VBRIHeader(object): + + version = 0 + """VBRI header version""" + + quality = 0 + """Quality indicator""" + + bytes = 0 + """Number of bytes""" + + frames = 0 + """Number of frames""" + + toc_scale_factor = 0 + """Scale factor of TOC entries""" + + toc_frames = 0 + """Number of frames per table entry""" + + toc = [] + """TOC""" + + def __init__(self, fileobj): + """Reads the VBRI header or raises VBRIHeaderError. + + The file position is undefined after this returns + """ + + data = fileobj.read(26) + if len(data) != 26 or not data.startswith(b"VBRI"): + raise VBRIHeaderError("Not a VBRI header") + + offset = 4 + self.version, offset = cdata.uint16_be_from(data, offset) + if self.version != 1: + raise VBRIHeaderError( + "Unsupported header version: %r" % self.version) + + offset += 2 # float16.. can't do + self.quality, offset = cdata.uint16_be_from(data, offset) + self.bytes, offset = cdata.uint32_be_from(data, offset) + self.frames, offset = cdata.uint32_be_from(data, offset) + + toc_num_entries, offset = cdata.uint16_be_from(data, offset) + self.toc_scale_factor, offset = cdata.uint16_be_from(data, offset) + toc_entry_size, offset = cdata.uint16_be_from(data, offset) + self.toc_frames, offset = cdata.uint16_be_from(data, offset) + toc_size = toc_entry_size * toc_num_entries + toc_data = fileobj.read(toc_size) + if len(toc_data) != toc_size: + raise VBRIHeaderError("VBRI header truncated") + + self.toc = [] + if toc_entry_size == 2: + unpack = partial(cdata.uint16_be_from, toc_data) + elif toc_entry_size == 4: + unpack = partial(cdata.uint32_be_from, toc_data) + else: + raise VBRIHeaderError("Invalid TOC entry size") + + self.toc = [unpack(i)[0] for i in xrange(0, toc_size, toc_entry_size)] + + @classmethod + def get_offset(cls, info): + """Offset in bytes from the start of the MPEG header including sync""" + + assert info.layer == 3 + + return 36 diff --git a/libs/mutagen/_tags.py b/libs/mutagen/_tags.py new file mode 100644 index 00000000..e6365f0a --- /dev/null +++ b/libs/mutagen/_tags.py @@ -0,0 +1,124 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2005 Michael Urman +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of version 2 of the GNU General Public License as +# published by the Free Software Foundation. + + +class PaddingInfo(object): + """Abstract padding information object. + + This will be passed to the callback function that can be used + for saving tags. + + :: + + def my_callback(info: PaddingInfo): + return info.get_default_padding() + + The callback should return the amount of padding to use (>= 0) based on + the content size and the padding of the file after saving. The actual used + amount of padding might vary depending on the file format (due to + alignment etc.) + + The default implementation can be accessed using the + :meth:`get_default_padding` method in the callback. + """ + + padding = 0 + """The amount of padding left after saving in bytes (can be negative if + more data needs to be added as padding is available) + """ + + size = 0 + """The amount of data following the padding""" + + def __init__(self, padding, size): + self.padding = padding + self.size = size + + def get_default_padding(self): + """The default implementation which tries to select a reasonable + amount of padding and which might change in future versions. + + :return: Amount of padding after saving + :rtype: int + """ + + high = 1024 * 10 + self.size // 100 # 10 KiB + 1% of trailing data + low = 1024 + self.size // 1000 # 1 KiB + 0.1% of trailing data + + if self.padding >= 0: + # enough padding left + if self.padding > high: + # padding too large, reduce + return low + # just use existing padding as is + return self.padding + else: + # not enough padding, add some + return low + + def _get_padding(self, user_func): + if user_func is None: + return self.get_default_padding() + else: + return user_func(self) + + def __repr__(self): + return "<%s size=%d padding=%d>" % ( + type(self).__name__, self.size, self.padding) + + +class Tags(object): + """`Tags` is the base class for many of the tag objects in Mutagen. + + In many cases it has a dict like interface. + """ + + __module__ = "mutagen" + + def pprint(self): + """ + :returns: tag information + :rtype: mutagen.text + """ + + raise NotImplementedError + + +class Metadata(Tags): + """Like :class:`Tags` but for standalone tagging formats that are not + solely managed by a container format. + + Provides methods to load, save and delete tags. + """ + + __module__ = "mutagen" + + def __init__(self, *args, **kwargs): + if args or kwargs: + self.load(*args, **kwargs) + + def load(self, filename, **kwargs): + raise NotImplementedError + + def save(self, filename=None): + """Save changes to a file. + + :raises mutagen.MutagenError: if saving wasn't possible + """ + + raise NotImplementedError + + def delete(self, filename=None): + """Remove tags from a file. + + In most cases this means any traces of the tag will be removed + from the file. + + :raises mutagen.MutagenError: if deleting wasn't possible + """ + + raise NotImplementedError diff --git a/libs/mutagen/_toolsutil.py b/libs/mutagen/_toolsutil.py new file mode 100644 index 00000000..e9074b71 --- /dev/null +++ b/libs/mutagen/_toolsutil.py @@ -0,0 +1,231 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Christoph Reiter +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +import os +import sys +import signal +import locale +import contextlib +import optparse +import ctypes + +from ._compat import text_type, PY2, PY3, iterbytes + + +def split_escape(string, sep, maxsplit=None, escape_char="\\"): + """Like unicode/str/bytes.split but allows for the separator to be escaped + + If passed unicode/str/bytes will only return list of unicode/str/bytes. + """ + + assert len(sep) == 1 + assert len(escape_char) == 1 + + if isinstance(string, bytes): + if isinstance(escape_char, text_type): + escape_char = escape_char.encode("ascii") + iter_ = iterbytes + else: + iter_ = iter + + if maxsplit is None: + maxsplit = len(string) + + empty = string[:0] + result = [] + current = empty + escaped = False + for char in iter_(string): + if escaped: + if char != escape_char and char != sep: + current += escape_char + current += char + escaped = False + else: + if char == escape_char: + escaped = True + elif char == sep and len(result) < maxsplit: + result.append(current) + current = empty + else: + current += char + result.append(current) + return result + + +class SignalHandler(object): + + def __init__(self): + self._interrupted = False + self._nosig = False + self._init = False + + def init(self): + signal.signal(signal.SIGINT, self._handler) + signal.signal(signal.SIGTERM, self._handler) + if os.name != "nt": + signal.signal(signal.SIGHUP, self._handler) + + def _handler(self, signum, frame): + self._interrupted = True + if not self._nosig: + raise SystemExit("Aborted...") + + @contextlib.contextmanager + def block(self): + """While this context manager is active any signals for aborting + the process will be queued and exit the program once the context + is left. + """ + + self._nosig = True + yield + self._nosig = False + if self._interrupted: + raise SystemExit("Aborted...") + + +def get_win32_unicode_argv(): + """Returns a unicode argv under Windows and standard sys.argv otherwise""" + + if os.name != "nt" or not PY2: + return sys.argv + + import ctypes + from ctypes import cdll, windll, wintypes + + GetCommandLineW = cdll.kernel32.GetCommandLineW + GetCommandLineW.argtypes = [] + GetCommandLineW.restype = wintypes.LPCWSTR + + CommandLineToArgvW = windll.shell32.CommandLineToArgvW + CommandLineToArgvW.argtypes = [ + wintypes.LPCWSTR, ctypes.POINTER(ctypes.c_int)] + CommandLineToArgvW.restype = ctypes.POINTER(wintypes.LPWSTR) + + LocalFree = windll.kernel32.LocalFree + LocalFree.argtypes = [wintypes.HLOCAL] + LocalFree.restype = wintypes.HLOCAL + + argc = ctypes.c_int() + argv = CommandLineToArgvW(GetCommandLineW(), ctypes.byref(argc)) + if not argv: + return + + res = argv[max(0, argc.value - len(sys.argv)):argc.value] + + LocalFree(argv) + + return res + + +def fsencoding(): + """The encoding used for paths, argv, environ, stdout and stdin""" + + if os.name == "nt": + return "" + + return locale.getpreferredencoding() or "utf-8" + + +def fsnative(text=u""): + """Returns the passed text converted to the preferred path type + for each platform. + """ + + assert isinstance(text, text_type) + + if os.name == "nt" or PY3: + return text + else: + return text.encode(fsencoding(), "replace") + return text + + +def is_fsnative(arg): + """If the passed value is of the preferred path type for each platform. + Note that on Python3+linux, paths can be bytes or str but this returns + False for bytes there. + """ + + if PY3 or os.name == "nt": + return isinstance(arg, text_type) + else: + return isinstance(arg, bytes) + + +def print_(*objects, **kwargs): + """A print which supports bytes and str+surrogates under python3. + + Needed so we can print anything passed to us through argv and environ. + Under Windows only text_type is allowed. + + Arguments: + objects: one or more bytes/text + linesep (bool): whether a line separator should be appended + sep (bool): whether objects should be printed separated by spaces + """ + + linesep = kwargs.pop("linesep", True) + sep = kwargs.pop("sep", True) + file_ = kwargs.pop("file", None) + if file_ is None: + file_ = sys.stdout + + old_cp = None + if os.name == "nt": + # Try to force the output to cp65001 aka utf-8. + # If that fails use the current one (most likely cp850, so + # most of unicode will be replaced with '?') + encoding = "utf-8" + old_cp = ctypes.windll.kernel32.GetConsoleOutputCP() + if ctypes.windll.kernel32.SetConsoleOutputCP(65001) == 0: + encoding = getattr(sys.stdout, "encoding", None) or "utf-8" + old_cp = None + else: + encoding = fsencoding() + + try: + if linesep: + objects = list(objects) + [os.linesep] + + parts = [] + for text in objects: + if isinstance(text, text_type): + if PY3: + try: + text = text.encode(encoding, 'surrogateescape') + except UnicodeEncodeError: + text = text.encode(encoding, 'replace') + else: + text = text.encode(encoding, 'replace') + parts.append(text) + + data = (b" " if sep else b"").join(parts) + try: + fileno = file_.fileno() + except (AttributeError, OSError, ValueError): + # for tests when stdout is replaced + try: + file_.write(data) + except TypeError: + file_.write(data.decode(encoding, "replace")) + else: + file_.flush() + os.write(fileno, data) + finally: + # reset the code page to what we had before + if old_cp is not None: + ctypes.windll.kernel32.SetConsoleOutputCP(old_cp) + + +class OptionParser(optparse.OptionParser): + """OptionParser subclass which supports printing Unicode under Windows""" + + def print_help(self, file=None): + print_(self.format_help(), file=file) diff --git a/libs/mutagen/_util.py b/libs/mutagen/_util.py index 2c8e1a56..f05ff454 100644 --- a/libs/mutagen/_util.py +++ b/libs/mutagen/_util.py @@ -1,4 +1,6 @@ -# Copyright 2006 Joe Wreschnig +# -*- coding: utf-8 -*- + +# Copyright (C) 2006 Joe Wreschnig # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -11,10 +13,83 @@ intended for internal use in Mutagen only. """ import struct +import codecs from fnmatch import fnmatchcase +from ._compat import chr_, PY2, iteritems, iterbytes, integer_types, xrange, \ + izip + +class MutagenError(Exception): + """Base class for all custom exceptions in mutagen + + .. versionadded:: 1.25 + """ + + __module__ = "mutagen" + + +def total_ordering(cls): + assert "__eq__" in cls.__dict__ + assert "__lt__" in cls.__dict__ + + cls.__le__ = lambda self, other: self == other or self < other + cls.__gt__ = lambda self, other: not (self == other or self < other) + cls.__ge__ = lambda self, other: not self < other + cls.__ne__ = lambda self, other: not self.__eq__(other) + + return cls + + +def hashable(cls): + """Makes sure the class is hashable. + + Needs a working __eq__ and __hash__ and will add a __ne__. + """ + + # py2 + assert "__hash__" in cls.__dict__ + # py3 + assert cls.__dict__["__hash__"] is not None + assert "__eq__" in cls.__dict__ + + cls.__ne__ = lambda self, other: not self.__eq__(other) + + return cls + + +def enum(cls): + assert cls.__bases__ == (object,) + + d = dict(cls.__dict__) + new_type = type(cls.__name__, (int,), d) + new_type.__module__ = cls.__module__ + + map_ = {} + for key, value in iteritems(d): + if key.upper() == key and isinstance(value, integer_types): + value_instance = new_type(value) + setattr(new_type, key, value_instance) + map_[value] = key + + def str_(self): + if self in map_: + return "%s.%s" % (type(self).__name__, map_[self]) + return "%d" % int(self) + + def repr_(self): + if self in map_: + return "<%s.%s: %d>" % (type(self).__name__, map_[self], int(self)) + return "%d" % int(self) + + setattr(new_type, "__repr__", repr_) + setattr(new_type, "__str__", str_) + + return new_type + + +@total_ordering class DictMixin(object): """Implement the dict API using keys() and __*item__ methods. @@ -33,27 +108,37 @@ class DictMixin(object): def __iter__(self): return iter(self.keys()) - def has_key(self, key): + def __has_key(self, key): try: self[key] except KeyError: return False else: return True - __contains__ = has_key - iterkeys = lambda self: iter(self.keys()) + if PY2: + has_key = __has_key + + __contains__ = __has_key + + if PY2: + iterkeys = lambda self: iter(self.keys()) def values(self): - return map(self.__getitem__, self.keys()) - itervalues = lambda self: iter(self.values()) + return [self[k] for k in self.keys()] + + if PY2: + itervalues = lambda self: iter(self.values()) def items(self): - return zip(self.keys(), self.values()) - iteritems = lambda s: iter(s.items()) + return list(izip(self.keys(), self.values())) + + if PY2: + iteritems = lambda s: iter(s.items()) def clear(self): - map(self.__delitem__, self.keys()) + for key in list(self.keys()): + self.__delitem__(key) def pop(self, key, *args): if len(args) > 1: @@ -69,11 +154,11 @@ class DictMixin(object): return value def popitem(self): - try: - key = self.keys()[0] - return key, self.pop(key) - except IndexError: + for key in self.keys(): + break + else: raise KeyError("dictionary is empty") + return key, self.pop(key) def update(self, other=None, **kwargs): if other is None: @@ -81,7 +166,8 @@ class DictMixin(object): other = {} try: - map(self.__setitem__, other.keys(), other.values()) + for key, value in other.items(): + self.__setitem__(key, value) except AttributeError: for key, value in other: self[key] = value @@ -102,11 +188,11 @@ class DictMixin(object): def __repr__(self): return repr(dict(self.items())) - def __cmp__(self, other): - if other is None: - return 1 - else: - return cmp(dict(self.items()), other) + def __eq__(self, other): + return dict(self.items()) == other + + def __lt__(self, other): + return dict(self.items()) < other __hash__ = object.__hash__ @@ -132,98 +218,79 @@ class DictProxy(DictMixin): return self.__dict.keys() +def _fill_cdata(cls): + """Add struct pack/unpack functions""" + + funcs = {} + for key, name in [("b", "char"), ("h", "short"), + ("i", "int"), ("q", "longlong")]: + for echar, esuffix in [("<", "le"), (">", "be")]: + esuffix = "_" + esuffix + for unsigned in [True, False]: + s = struct.Struct(echar + (key.upper() if unsigned else key)) + get_wrapper = lambda f: lambda *a, **k: f(*a, **k)[0] + unpack = get_wrapper(s.unpack) + unpack_from = get_wrapper(s.unpack_from) + + def get_unpack_from(s): + def unpack_from(data, offset=0): + return s.unpack_from(data, offset)[0], offset + s.size + return unpack_from + + unpack_from = get_unpack_from(s) + pack = s.pack + + prefix = "u" if unsigned else "" + if s.size == 1: + esuffix = "" + bits = str(s.size * 8) + funcs["%s%s%s" % (prefix, name, esuffix)] = unpack + funcs["%sint%s%s" % (prefix, bits, esuffix)] = unpack + funcs["%s%s%s_from" % (prefix, name, esuffix)] = unpack_from + funcs["%sint%s%s_from" % (prefix, bits, esuffix)] = unpack_from + funcs["to_%s%s%s" % (prefix, name, esuffix)] = pack + funcs["to_%sint%s%s" % (prefix, bits, esuffix)] = pack + + for key, func in iteritems(funcs): + setattr(cls, key, staticmethod(func)) + + class cdata(object): - """C character buffer to Python numeric type conversions.""" + """C character buffer to Python numeric type conversions. + + For each size/sign/endianness: + uint32_le(data)/to_uint32_le(num)/uint32_le_from(data, offset=0) + """ from struct import error error = error - short_le = staticmethod(lambda data: struct.unpack('h', data)[0]) - ushort_be = staticmethod(lambda data: struct.unpack('>H', data)[0]) - - int_le = staticmethod(lambda data: struct.unpack('i', data)[0]) - uint_be = staticmethod(lambda data: struct.unpack('>I', data)[0]) - - longlong_le = staticmethod(lambda data: struct.unpack('q', data)[0]) - ulonglong_be = staticmethod(lambda data: struct.unpack('>Q', data)[0]) - - to_short_le = staticmethod(lambda data: struct.pack('h', data)) - to_ushort_be = staticmethod(lambda data: struct.pack('>H', data)) - - to_int_le = staticmethod(lambda data: struct.pack('i', data)) - to_uint_be = staticmethod(lambda data: struct.pack('>I', data)) - - to_longlong_le = staticmethod(lambda data: struct.pack('q', data)) - to_ulonglong_be = staticmethod(lambda data: struct.pack('>Q', data)) - - bitswap = ''.join([chr(sum([((val >> i) & 1) << (7-i) for i in range(8)])) - for val in range(256)]) - del(i) - del(val) + bitswap = b''.join( + chr_(sum(((val >> i) & 1) << (7 - i) for i in xrange(8))) + for val in xrange(256)) test_bit = staticmethod(lambda value, n: bool((value >> n) & 1)) -def lock(fileobj): - """Lock a file object 'safely'. +_fill_cdata(cdata) - That means a failure to lock because the platform doesn't - support fcntl or filesystem locks is not considered a - failure. This call does block. - Returns whether or not the lock was successful, or - raises an exception in more extreme circumstances (full - lock table, invalid file). +def get_size(fileobj): + """Returns the size of the file object. The position when passed in will + be preserved if no error occurs. + + In case of an error raises IOError. """ + old_pos = fileobj.tell() try: - import fcntl - except ImportError: - return False - else: - try: - fcntl.lockf(fileobj, fcntl.LOCK_EX) - except IOError: - # FIXME: There's possibly a lot of complicated - # logic that needs to go here in case the IOError - # is EACCES or EAGAIN. - return False - else: - return True + fileobj.seek(0, 2) + return fileobj.tell() + finally: + fileobj.seek(old_pos, 0) -def unlock(fileobj): - """Unlock a file object. - - Don't call this on a file object unless a call to lock() - returned true. - """ - - # If this fails there's a mismatched lock/unlock pair, - # so we definitely don't want to ignore errors. - import fcntl - fcntl.lockf(fileobj, fcntl.LOCK_UN) - - -def insert_bytes(fobj, size, offset, BUFFER_SIZE=2**16): +def insert_bytes(fobj, size, offset, BUFFER_SIZE=2 ** 16): """Insert size bytes of empty space starting at offset. fobj must be an open file object, open rb+ or @@ -233,59 +300,55 @@ def insert_bytes(fobj, size, offset, BUFFER_SIZE=2**16): assert 0 < size assert 0 <= offset - locked = False + fobj.seek(0, 2) filesize = fobj.tell() movesize = filesize - offset - fobj.write('\x00' * size) + fobj.write(b'\x00' * size) fobj.flush() + try: + import mmap + file_map = mmap.mmap(fobj.fileno(), filesize + size) try: - import mmap - map = mmap.mmap(fobj.fileno(), filesize + size) - try: - map.move(offset + size, offset, movesize) - finally: - map.close() - except (ValueError, EnvironmentError, ImportError): - # handle broken mmap scenarios - locked = lock(fobj) - fobj.truncate(filesize) + file_map.move(offset + size, offset, movesize) + finally: + file_map.close() + except (ValueError, EnvironmentError, ImportError, AttributeError): + # handle broken mmap scenarios, BytesIO() + fobj.truncate(filesize) - fobj.seek(0, 2) - padsize = size - # Don't generate an enormous string if we need to pad - # the file out several megs. - while padsize: - addsize = min(BUFFER_SIZE, padsize) - fobj.write("\x00" * addsize) - padsize -= addsize + fobj.seek(0, 2) + padsize = size + # Don't generate an enormous string if we need to pad + # the file out several megs. + while padsize: + addsize = min(BUFFER_SIZE, padsize) + fobj.write(b"\x00" * addsize) + padsize -= addsize - fobj.seek(filesize, 0) - while movesize: - # At the start of this loop, fobj is pointing at the end - # of the data we need to move, which is of movesize length. - thismove = min(BUFFER_SIZE, movesize) - # Seek back however much we're going to read this frame. - fobj.seek(-thismove, 1) - nextpos = fobj.tell() - # Read it, so we're back at the end. - data = fobj.read(thismove) - # Seek back to where we need to write it. - fobj.seek(-thismove + size, 1) - # Write it. - fobj.write(data) - # And seek back to the end of the unmoved data. - fobj.seek(nextpos) - movesize -= thismove + fobj.seek(filesize, 0) + while movesize: + # At the start of this loop, fobj is pointing at the end + # of the data we need to move, which is of movesize length. + thismove = min(BUFFER_SIZE, movesize) + # Seek back however much we're going to read this frame. + fobj.seek(-thismove, 1) + nextpos = fobj.tell() + # Read it, so we're back at the end. + data = fobj.read(thismove) + # Seek back to where we need to write it. + fobj.seek(-thismove + size, 1) + # Write it. + fobj.write(data) + # And seek back to the end of the unmoved data. + fobj.seek(nextpos) + movesize -= thismove - fobj.flush() - finally: - if locked: - unlock(fobj) + fobj.flush() -def delete_bytes(fobj, size, offset, BUFFER_SIZE=2**16): +def delete_bytes(fobj, size, offset, BUFFER_SIZE=2 ** 16): """Delete size bytes of empty space starting at offset. fobj must be an open file object, open rb+ or @@ -293,57 +356,195 @@ def delete_bytes(fobj, size, offset, BUFFER_SIZE=2**16): falls back to a significantly slower method if mmap fails. """ - locked = False assert 0 < size assert 0 <= offset + fobj.seek(0, 2) filesize = fobj.tell() movesize = filesize - offset - size assert 0 <= movesize - try: - if movesize > 0: - fobj.flush() + + if movesize > 0: + fobj.flush() + try: + import mmap + file_map = mmap.mmap(fobj.fileno(), filesize) try: - import mmap - map = mmap.mmap(fobj.fileno(), filesize) - try: - map.move(offset, offset + size, movesize) - finally: - map.close() - except (ValueError, EnvironmentError, ImportError): - # handle broken mmap scenarios - locked = lock(fobj) + file_map.move(offset, offset + size, movesize) + finally: + file_map.close() + except (ValueError, EnvironmentError, ImportError, AttributeError): + # handle broken mmap scenarios, BytesIO() + fobj.seek(offset + size) + buf = fobj.read(BUFFER_SIZE) + while buf: + fobj.seek(offset) + fobj.write(buf) + offset += len(buf) fobj.seek(offset + size) buf = fobj.read(BUFFER_SIZE) - while buf: - fobj.seek(offset) - fobj.write(buf) - offset += len(buf) - fobj.seek(offset + size) - buf = fobj.read(BUFFER_SIZE) - fobj.truncate(filesize - size) - fobj.flush() - finally: - if locked: - unlock(fobj) + fobj.truncate(filesize - size) + fobj.flush() -def utf8(data): - """Convert a basestring to a valid UTF-8 str.""" +def resize_bytes(fobj, old_size, new_size, offset): + """Resize an area in a file adding and deleting at the end of it. + Does nothing if no resizing is needed. + """ - if isinstance(data, str): - return data.decode("utf-8", "replace").encode("utf-8") - elif isinstance(data, unicode): - return data.encode("utf-8") - else: - raise TypeError("only unicode/str types can be converted to UTF-8") + if new_size < old_size: + delete_size = old_size - new_size + delete_at = offset + new_size + delete_bytes(fobj, delete_size, delete_at) + elif new_size > old_size: + insert_size = new_size - old_size + insert_at = offset + old_size + insert_bytes(fobj, insert_size, insert_at) def dict_match(d, key, default=None): - try: + """Like __getitem__ but works as if the keys() are all filename patterns. + Returns the value of any dict key that matches the passed key. + """ + + if key in d and "[" not in key: return d[key] - except KeyError: - for pattern, value in d.iteritems(): + else: + for pattern, value in iteritems(d): if fnmatchcase(key, pattern): return value return default + + +def decode_terminated(data, encoding, strict=True): + """Returns the decoded data until the first NULL terminator + and all data after it. + + In case the data can't be decoded raises UnicodeError. + In case the encoding is not found raises LookupError. + In case the data isn't null terminated (even if it is encoded correctly) + raises ValueError except if strict is False, then the decoded string + will be returned anyway. + """ + + codec_info = codecs.lookup(encoding) + + # normalize encoding name so we can compare by name + encoding = codec_info.name + + # fast path + if encoding in ("utf-8", "iso8859-1"): + index = data.find(b"\x00") + if index == -1: + # make sure we raise UnicodeError first, like in the slow path + res = data.decode(encoding), b"" + if strict: + raise ValueError("not null terminated") + else: + return res + return data[:index].decode(encoding), data[index + 1:] + + # slow path + decoder = codec_info.incrementaldecoder() + r = [] + for i, b in enumerate(iterbytes(data)): + c = decoder.decode(b) + if c == u"\x00": + return u"".join(r), data[i + 1:] + r.append(c) + else: + # make sure the decoder is finished + r.append(decoder.decode(b"", True)) + if strict: + raise ValueError("not null terminated") + return u"".join(r), b"" + + +class BitReaderError(Exception): + pass + + +class BitReader(object): + + def __init__(self, fileobj): + self._fileobj = fileobj + self._buffer = 0 + self._bits = 0 + self._pos = fileobj.tell() + + def bits(self, count): + """Reads `count` bits and returns an uint, MSB read first. + + May raise BitReaderError if not enough data could be read or + IOError by the underlying file object. + """ + + if count < 0: + raise ValueError + + if count > self._bits: + n_bytes = (count - self._bits + 7) // 8 + data = self._fileobj.read(n_bytes) + if len(data) != n_bytes: + raise BitReaderError("not enough data") + for b in bytearray(data): + self._buffer = (self._buffer << 8) | b + self._bits += n_bytes * 8 + + self._bits -= count + value = self._buffer >> self._bits + self._buffer &= (1 << self._bits) - 1 + assert self._bits < 8 + return value + + def bytes(self, count): + """Returns a bytearray of length `count`. Works unaligned.""" + + if count < 0: + raise ValueError + + # fast path + if self._bits == 0: + data = self._fileobj.read(count) + if len(data) != count: + raise BitReaderError("not enough data") + return data + + return bytes(bytearray(self.bits(8) for _ in xrange(count))) + + def skip(self, count): + """Skip `count` bits. + + Might raise BitReaderError if there wasn't enough data to skip, + but might also fail on the next bits() instead. + """ + + if count < 0: + raise ValueError + + if count <= self._bits: + self.bits(count) + else: + count -= self.align() + n_bytes = count // 8 + self._fileobj.seek(n_bytes, 1) + count -= n_bytes * 8 + self.bits(count) + + def get_position(self): + """Returns the amount of bits read or skipped so far""" + + return (self._fileobj.tell() - self._pos) * 8 - self._bits + + def align(self): + """Align to the next byte, returns the amount of bits skipped""" + + bits = self._bits + self._buffer = 0 + self._bits = 0 + return bits + + def is_aligned(self): + """If we are currently aligned to bytes and nothing is buffered""" + + return self._bits == 0 diff --git a/libs/mutagen/_vorbis.py b/libs/mutagen/_vorbis.py index 4ee8da4a..17634e06 100644 --- a/libs/mutagen/_vorbis.py +++ b/libs/mutagen/_vorbis.py @@ -1,5 +1,7 @@ -# Vorbis comment support for Mutagen -# Copyright 2005-2006 Joe Wreschnig +# -*- coding: utf-8 -*- + +# Copyright (C) 2005-2006 Joe Wreschnig +# 2013 Christoph Reiter # # This program is free software; you can redistribute it and/or modify # it under the terms of version 2 of the GNU General Public License as @@ -16,9 +18,8 @@ The specification is at http://www.xiph.org/vorbis/doc/v-comment.html. import sys -from cStringIO import StringIO - import mutagen +from ._compat import reraise, BytesIO, text_type, xrange, PY3, PY2 from mutagen._util import DictMixin, cdata @@ -27,13 +28,20 @@ def is_valid_key(key): Valid Vorbis comment keys are printable ASCII between 0x20 (space) and 0x7D ('}'), excluding '='. + + Takes str/unicode in Python 2, unicode in Python 3 """ + + if PY3 and isinstance(key, bytes): + raise TypeError("needs to be str not bytes") + for c in key: if c < " " or c > "}" or c == "=": return False else: return bool(key) + istag = is_valid_key @@ -49,7 +57,7 @@ class VorbisEncodingError(error): pass -class VComment(mutagen.Metadata, list): +class VComment(mutagen.Tags, list): """A Vorbis comment parser, accessor, and renderer. All comment ordering is preserved. A VComment is a list of @@ -60,35 +68,40 @@ class VComment(mutagen.Metadata, list): file-like object, not a filename. Attributes: - vendor -- the stream 'vendor' (i.e. writer); default 'Mutagen' + + * vendor -- the stream 'vendor' (i.e. writer); default 'Mutagen' """ vendor = u"Mutagen " + mutagen.version_string def __init__(self, data=None, *args, **kwargs): + self._size = 0 # Collect the args to pass to load, this lets child classes # override just load and get equivalent magic for the # constructor. if data is not None: - if isinstance(data, str): - data = StringIO(data) + if isinstance(data, bytes): + data = BytesIO(data) elif not hasattr(data, 'read'): - raise TypeError("VComment requires string data or a file-like") + raise TypeError("VComment requires bytes or a file-like") + start = data.tell() self.load(data, *args, **kwargs) + self._size = data.tell() - start def load(self, fileobj, errors='replace', framing=True): """Parse a Vorbis comment from a file-like object. Keyword arguments: - errors: - 'strict', 'replace', or 'ignore'. This affects Unicode decoding - and how other malformed content is interpreted. - framing -- if true, fail if a framing bit is not present + + * errors: + 'strict', 'replace', or 'ignore'. This affects Unicode decoding + and how other malformed content is interpreted. + * framing -- if true, fail if a framing bit is not present Framing bits are required by the Vorbis comment specification, but are not used in FLAC Vorbis comment blocks. - """ + try: vendor_length = cdata.uint_le(fileobj.read(4)) self.vendor = fileobj.read(vendor_length).decode('utf-8', errors) @@ -101,21 +114,25 @@ class VComment(mutagen.Metadata, list): raise error("cannot read %d bytes, too large" % length) try: tag, value = string.split('=', 1) - except ValueError, err: + except ValueError as err: if errors == "ignore": continue elif errors == "replace": tag, value = u"unknown%d" % i, string else: - raise VorbisEncodingError, err, sys.exc_info()[2] + reraise(VorbisEncodingError, err, sys.exc_info()[2]) try: tag = tag.encode('ascii', errors) except UnicodeEncodeError: raise VorbisEncodingError("invalid tag name %r" % tag) else: + # string keys in py3k + if PY3: + tag = tag.decode("ascii") if is_valid_key(tag): self.append((tag, value)) - if framing and not ord(fileobj.read(1)) & 0x01: + + if framing and not bytearray(fileobj.read(1))[0] & 0x01: raise VorbisUnsetFrameError("framing bit was unset") except (cdata.error, TypeError): raise error("file is not a valid Vorbis comment") @@ -126,9 +143,14 @@ class VComment(mutagen.Metadata, list): Check to make sure every key used is a valid Vorbis key, and that every value used is a valid Unicode or UTF-8 string. If any invalid keys or values are found, a ValueError is raised. + + In Python 3 all keys and values have to be a string. """ - if not isinstance(self.vendor, unicode): + if not isinstance(self.vendor, text_type): + if PY3: + raise ValueError("vendor needs to be str") + try: self.vendor.decode('utf-8') except UnicodeDecodeError: @@ -138,19 +160,25 @@ class VComment(mutagen.Metadata, list): try: if not is_valid_key(key): raise ValueError - except: + except TypeError: raise ValueError("%r is not a valid key" % key) - if not isinstance(value, unicode): + + if not isinstance(value, text_type): + if PY3: + raise ValueError("%r needs to be str" % key) + try: - value.encode("utf-8") + value.decode("utf-8") except: raise ValueError("%r is not a valid value" % value) - else: - return True + + return True def clear(self): """Clear all keys from the comment.""" - del(self[:]) + + for i in list(self): + self.remove(i) def write(self, framing=True): """Return a string representation of the data. @@ -159,25 +187,41 @@ class VComment(mutagen.Metadata, list): invalid data may raise a ValueError. Keyword arguments: - framing -- if true, append a framing bit (see load) + + * framing -- if true, append a framing bit (see load) """ self.validate() - f = StringIO() - f.write(cdata.to_uint_le(len(self.vendor.encode('utf-8')))) - f.write(self.vendor.encode('utf-8')) + def _encode(value): + if not isinstance(value, bytes): + return value.encode('utf-8') + return value + + f = BytesIO() + vendor = _encode(self.vendor) + f.write(cdata.to_uint_le(len(vendor))) + f.write(vendor) f.write(cdata.to_uint_le(len(self))) for tag, value in self: - comment = "%s=%s" % (tag, value.encode('utf-8')) + tag = _encode(tag) + value = _encode(value) + comment = tag + b"=" + value f.write(cdata.to_uint_le(len(comment))) f.write(comment) if framing: - f.write("\x01") + f.write(b"\x01") return f.getvalue() def pprint(self): - return "\n".join(["%s=%s" % (k.lower(), v) for k, v in self]) + + def _decode(value): + if not isinstance(value, text_type): + return value.decode('utf-8', 'replace') + return value + + tags = [u"%s=%s" % (_decode(k), _decode(v)) for k, v in self] + return u"\n".join(tags) class VCommentDict(VComment, DictMixin): @@ -199,9 +243,17 @@ class VCommentDict(VComment, DictMixin): This is a copy, so comment['title'].append('a title') will not work. - """ - key = key.lower().encode('ascii') + + # PY3 only + if isinstance(key, slice): + return VComment.__getitem__(self, key) + + if not is_valid_key(key): + raise ValueError + + key = key.lower() + values = [value for (k, value) in self if k.lower() == key] if not values: raise KeyError(key) @@ -210,16 +262,29 @@ class VCommentDict(VComment, DictMixin): def __delitem__(self, key): """Delete all values associated with the key.""" - key = key.lower().encode('ascii') - to_delete = filter(lambda x: x[0].lower() == key, self) + + # PY3 only + if isinstance(key, slice): + return VComment.__delitem__(self, key) + + if not is_valid_key(key): + raise ValueError + + key = key.lower() + to_delete = [x for x in self if x[0].lower() == key] if not to_delete: raise KeyError(key) else: - map(self.remove, to_delete) + for item in to_delete: + self.remove(item) def __contains__(self, key): """Return true if the key has any values.""" - key = key.lower().encode('ascii') + + if not is_valid_key(key): + raise ValueError + + key = key.lower() for k, value in self: if k.lower() == key: return True @@ -232,23 +297,34 @@ class VCommentDict(VComment, DictMixin): Setting a value overwrites all old ones. The value may be a list of Unicode or UTF-8 strings, or a single Unicode or UTF-8 string. - """ - key = key.encode('ascii') + # PY3 only + if isinstance(key, slice): + return VComment.__setitem__(self, key, values) + + if not is_valid_key(key): + raise ValueError + if not isinstance(values, list): values = [values] try: del(self[key]) except KeyError: pass + + if PY2: + key = key.encode('ascii') + for value in values: self.append((key, value)) def keys(self): """Return all keys in the comment.""" - return self and list(set([k.lower() for k, v in self])) + + return list(set([k.lower() for k, v in self])) def as_dict(self): """Return a copy of the comment data in a real dict.""" + return dict([(key, self[key]) for key in self.keys()]) diff --git a/libs/mutagen/aac.py b/libs/mutagen/aac.py new file mode 100644 index 00000000..83968a05 --- /dev/null +++ b/libs/mutagen/aac.py @@ -0,0 +1,410 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014 Christoph Reiter +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of version 2 of the GNU General Public License as +# published by the Free Software Foundation. + +""" +* ADTS - Audio Data Transport Stream +* ADIF - Audio Data Interchange Format +* See ISO/IEC 13818-7 / 14496-03 +""" + +from mutagen import StreamInfo +from mutagen._file import FileType +from mutagen._util import BitReader, BitReaderError, MutagenError +from mutagen._compat import endswith, xrange + + +_FREQS = [ + 96000, 88200, 64000, 48000, + 44100, 32000, 24000, 22050, + 16000, 12000, 11025, 8000, + 7350, +] + + +class _ADTSStream(object): + """Represents a series of frames belonging to the same stream""" + + parsed_frames = 0 + """Number of successfully parsed frames""" + + offset = 0 + """offset in bytes at which the stream starts (the first sync word)""" + + @classmethod + def find_stream(cls, fileobj, max_bytes): + """Returns a possibly valid _ADTSStream or None. + + Args: + max_bytes (int): maximum bytes to read + """ + + r = BitReader(fileobj) + stream = cls(r) + if stream.sync(max_bytes): + stream.offset = (r.get_position() - 12) // 8 + return stream + + def sync(self, max_bytes): + """Find the next sync. + Returns True if found.""" + + # at least 2 bytes for the sync + max_bytes = max(max_bytes, 2) + + r = self._r + r.align() + while max_bytes > 0: + try: + b = r.bytes(1) + if b == b"\xff": + if r.bits(4) == 0xf: + return True + r.align() + max_bytes -= 2 + else: + max_bytes -= 1 + except BitReaderError: + return False + return False + + def __init__(self, r): + """Use _ADTSStream.find_stream to create a stream""" + + self._fixed_header_key = None + self._r = r + self.offset = -1 + self.parsed_frames = 0 + + self._samples = 0 + self._payload = 0 + self._start = r.get_position() / 8 + self._last = self._start + + @property + def bitrate(self): + """Bitrate of the raw aac blocks, excluding framing/crc""" + + assert self.parsed_frames, "no frame parsed yet" + + if self._samples == 0: + return 0 + + return (8 * self._payload * self.frequency) // self._samples + + @property + def samples(self): + """samples so far""" + + assert self.parsed_frames, "no frame parsed yet" + + return self._samples + + @property + def size(self): + """bytes read in the stream so far (including framing)""" + + assert self.parsed_frames, "no frame parsed yet" + + return self._last - self._start + + @property + def channels(self): + """0 means unknown""" + + assert self.parsed_frames, "no frame parsed yet" + + b_index = self._fixed_header_key[6] + if b_index == 7: + return 8 + elif b_index > 7: + return 0 + else: + return b_index + + @property + def frequency(self): + """0 means unknown""" + + assert self.parsed_frames, "no frame parsed yet" + + f_index = self._fixed_header_key[4] + try: + return _FREQS[f_index] + except IndexError: + return 0 + + def parse_frame(self): + """True if parsing was successful. + Fails either because the frame wasn't valid or the stream ended. + """ + + try: + return self._parse_frame() + except BitReaderError: + return False + + def _parse_frame(self): + r = self._r + # start == position of sync word + start = r.get_position() - 12 + + # adts_fixed_header + id_ = r.bits(1) + layer = r.bits(2) + protection_absent = r.bits(1) + + profile = r.bits(2) + sampling_frequency_index = r.bits(4) + private_bit = r.bits(1) + # TODO: if 0 we could parse program_config_element() + channel_configuration = r.bits(3) + original_copy = r.bits(1) + home = r.bits(1) + + # the fixed header has to be the same for every frame in the stream + fixed_header_key = ( + id_, layer, protection_absent, profile, sampling_frequency_index, + private_bit, channel_configuration, original_copy, home, + ) + + if self._fixed_header_key is None: + self._fixed_header_key = fixed_header_key + else: + if self._fixed_header_key != fixed_header_key: + return False + + # adts_variable_header + r.skip(2) # copyright_identification_bit/start + frame_length = r.bits(13) + r.skip(11) # adts_buffer_fullness + nordbif = r.bits(2) + # adts_variable_header end + + crc_overhead = 0 + if not protection_absent: + crc_overhead += (nordbif + 1) * 16 + if nordbif != 0: + crc_overhead *= 2 + + left = (frame_length * 8) - (r.get_position() - start) + if left < 0: + return False + r.skip(left) + assert r.is_aligned() + + self._payload += (left - crc_overhead) / 8 + self._samples += (nordbif + 1) * 1024 + self._last = r.get_position() / 8 + + self.parsed_frames += 1 + return True + + +class ProgramConfigElement(object): + + element_instance_tag = None + object_type = None + sampling_frequency_index = None + channels = None + + def __init__(self, r): + """Reads the program_config_element() + + Raises BitReaderError + """ + + self.element_instance_tag = r.bits(4) + self.object_type = r.bits(2) + self.sampling_frequency_index = r.bits(4) + num_front_channel_elements = r.bits(4) + num_side_channel_elements = r.bits(4) + num_back_channel_elements = r.bits(4) + num_lfe_channel_elements = r.bits(2) + num_assoc_data_elements = r.bits(3) + num_valid_cc_elements = r.bits(4) + + mono_mixdown_present = r.bits(1) + if mono_mixdown_present == 1: + r.skip(4) + stereo_mixdown_present = r.bits(1) + if stereo_mixdown_present == 1: + r.skip(4) + matrix_mixdown_idx_present = r.bits(1) + if matrix_mixdown_idx_present == 1: + r.skip(3) + + elms = num_front_channel_elements + num_side_channel_elements + \ + num_back_channel_elements + channels = 0 + for i in xrange(elms): + channels += 1 + element_is_cpe = r.bits(1) + if element_is_cpe: + channels += 1 + r.skip(4) + channels += num_lfe_channel_elements + self.channels = channels + + r.skip(4 * num_lfe_channel_elements) + r.skip(4 * num_assoc_data_elements) + r.skip(5 * num_valid_cc_elements) + r.align() + comment_field_bytes = r.bits(8) + r.skip(8 * comment_field_bytes) + + +class AACError(MutagenError): + pass + + +class AACInfo(StreamInfo): + """AAC stream information. + + Attributes: + + * channels -- number of audio channels + * length -- file length in seconds, as a float + * sample_rate -- audio sampling rate in Hz + * bitrate -- audio bitrate, in bits per second + + The length of the stream is just a guess and might not be correct. + """ + + channels = 0 + length = 0 + sample_rate = 0 + bitrate = 0 + + def __init__(self, fileobj): + # skip id3v2 header + start_offset = 0 + header = fileobj.read(10) + from mutagen.id3 import BitPaddedInt + if header.startswith(b"ID3"): + size = BitPaddedInt(header[6:]) + start_offset = size + 10 + + fileobj.seek(start_offset) + adif = fileobj.read(4) + if adif == b"ADIF": + self._parse_adif(fileobj) + self._type = "ADIF" + else: + self._parse_adts(fileobj, start_offset) + self._type = "ADTS" + + def _parse_adif(self, fileobj): + r = BitReader(fileobj) + try: + copyright_id_present = r.bits(1) + if copyright_id_present: + r.skip(72) # copyright_id + r.skip(1 + 1) # original_copy, home + bitstream_type = r.bits(1) + self.bitrate = r.bits(23) + npce = r.bits(4) + if bitstream_type == 0: + r.skip(20) # adif_buffer_fullness + + pce = ProgramConfigElement(r) + try: + self.sample_rate = _FREQS[pce.sampling_frequency_index] + except IndexError: + pass + self.channels = pce.channels + + # other pces.. + for i in xrange(npce): + ProgramConfigElement(r) + r.align() + except BitReaderError as e: + raise AACError(e) + + # use bitrate + data size to guess length + start = fileobj.tell() + fileobj.seek(0, 2) + length = fileobj.tell() - start + if self.bitrate != 0: + self.length = (8.0 * length) / self.bitrate + + def _parse_adts(self, fileobj, start_offset): + max_initial_read = 512 + max_resync_read = 10 + max_sync_tries = 10 + + frames_max = 100 + frames_needed = 3 + + # Try up to X times to find a sync word and read up to Y frames. + # If more than Z frames are valid we assume a valid stream + offset = start_offset + for i in xrange(max_sync_tries): + fileobj.seek(offset) + s = _ADTSStream.find_stream(fileobj, max_initial_read) + if s is None: + raise AACError("sync not found") + # start right after the last found offset + offset += s.offset + 1 + + for i in xrange(frames_max): + if not s.parse_frame(): + break + if not s.sync(max_resync_read): + break + + if s.parsed_frames >= frames_needed: + break + else: + raise AACError( + "no valid stream found (only %d frames)" % s.parsed_frames) + + self.sample_rate = s.frequency + self.channels = s.channels + self.bitrate = s.bitrate + + # size from stream start to end of file + fileobj.seek(0, 2) + stream_size = fileobj.tell() - (offset + s.offset) + # approx + self.length = float(s.samples * stream_size) / (s.size * s.frequency) + + def pprint(self): + return u"AAC (%s), %d Hz, %.2f seconds, %d channel(s), %d bps" % ( + self._type, self.sample_rate, self.length, self.channels, + self.bitrate) + + +class AAC(FileType): + """Load ADTS or ADIF streams containing AAC. + + Tagging is not supported. + Use the ID3/APEv2 classes directly instead. + """ + + _mimes = ["audio/x-aac"] + + def load(self, filename): + self.filename = filename + with open(filename, "rb") as h: + self.info = AACInfo(h) + + def add_tags(self): + raise AACError("doesn't support tags") + + @staticmethod + def score(filename, fileobj, header): + filename = filename.lower() + s = endswith(filename, ".aac") or endswith(filename, ".adts") or \ + endswith(filename, ".adif") + s += b"ADIF" in header + return s + + +Open = AAC +error = AACError + +__all__ = ["AAC", "Open"] diff --git a/libs/mutagen/aiff.py b/libs/mutagen/aiff.py new file mode 100644 index 00000000..dc580063 --- /dev/null +++ b/libs/mutagen/aiff.py @@ -0,0 +1,357 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2014 Evan Purkhiser +# 2014 Ben Ockmore +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of version 2 of the GNU General Public License as +# published by the Free Software Foundation. + +"""AIFF audio stream information and tags.""" + +import sys +import struct +from struct import pack + +from ._compat import endswith, text_type, reraise +from mutagen import StreamInfo, FileType + +from mutagen.id3 import ID3 +from mutagen.id3._util import ID3NoHeaderError, error as ID3Error +from mutagen._util import resize_bytes, delete_bytes, MutagenError + +__all__ = ["AIFF", "Open", "delete"] + + +class error(MutagenError, RuntimeError): + pass + + +class InvalidChunk(error, IOError): + pass + + +# based on stdlib's aifc +_HUGE_VAL = 1.79769313486231e+308 + + +def is_valid_chunk_id(id): + assert isinstance(id, text_type) + + return ((len(id) <= 4) and (min(id) >= u' ') and + (max(id) <= u'~')) + + +def read_float(data): # 10 bytes + expon, himant, lomant = struct.unpack('>hLL', data) + sign = 1 + if expon < 0: + sign = -1 + expon = expon + 0x8000 + if expon == himant == lomant == 0: + f = 0.0 + elif expon == 0x7FFF: + f = _HUGE_VAL + else: + expon = expon - 16383 + f = (himant * 0x100000000 + lomant) * pow(2.0, expon - 63) + return sign * f + + +class IFFChunk(object): + """Representation of a single IFF chunk""" + + # Chunk headers are 8 bytes long (4 for ID and 4 for the size) + HEADER_SIZE = 8 + + def __init__(self, fileobj, parent_chunk=None): + self.__fileobj = fileobj + self.parent_chunk = parent_chunk + self.offset = fileobj.tell() + + header = fileobj.read(self.HEADER_SIZE) + if len(header) < self.HEADER_SIZE: + raise InvalidChunk() + + self.id, self.data_size = struct.unpack('>4si', header) + + try: + self.id = self.id.decode('ascii') + except UnicodeDecodeError: + raise InvalidChunk() + + if not is_valid_chunk_id(self.id): + raise InvalidChunk() + + self.size = self.HEADER_SIZE + self.data_size + self.data_offset = fileobj.tell() + + def read(self): + """Read the chunks data""" + + self.__fileobj.seek(self.data_offset) + return self.__fileobj.read(self.data_size) + + def write(self, data): + """Write the chunk data""" + + if len(data) > self.data_size: + raise ValueError + + self.__fileobj.seek(self.data_offset) + self.__fileobj.write(data) + + def delete(self): + """Removes the chunk from the file""" + + delete_bytes(self.__fileobj, self.size, self.offset) + if self.parent_chunk is not None: + self.parent_chunk._update_size( + self.parent_chunk.data_size - self.size) + + def _update_size(self, data_size): + """Update the size of the chunk""" + + self.__fileobj.seek(self.offset + 4) + self.__fileobj.write(pack('>I', data_size)) + if self.parent_chunk is not None: + size_diff = self.data_size - data_size + self.parent_chunk._update_size( + self.parent_chunk.data_size - size_diff) + self.data_size = data_size + self.size = data_size + self.HEADER_SIZE + + def resize(self, new_data_size): + """Resize the file and update the chunk sizes""" + + resize_bytes( + self.__fileobj, self.data_size, new_data_size, self.data_offset) + self._update_size(new_data_size) + + +class IFFFile(object): + """Representation of a IFF file""" + + def __init__(self, fileobj): + self.__fileobj = fileobj + self.__chunks = {} + + # AIFF Files always start with the FORM chunk which contains a 4 byte + # ID before the start of other chunks + fileobj.seek(0) + self.__chunks[u'FORM'] = IFFChunk(fileobj) + + # Skip past the 4 byte FORM id + fileobj.seek(IFFChunk.HEADER_SIZE + 4) + + # Where the next chunk can be located. We need to keep track of this + # since the size indicated in the FORM header may not match up with the + # offset determined from the size of the last chunk in the file + self.__next_offset = fileobj.tell() + + # Load all of the chunks + while True: + try: + chunk = IFFChunk(fileobj, self[u'FORM']) + except InvalidChunk: + break + self.__chunks[chunk.id.strip()] = chunk + + # Calculate the location of the next chunk, + # considering the pad byte + self.__next_offset = chunk.offset + chunk.size + self.__next_offset += self.__next_offset % 2 + fileobj.seek(self.__next_offset) + + def __contains__(self, id_): + """Check if the IFF file contains a specific chunk""" + + assert isinstance(id_, text_type) + + if not is_valid_chunk_id(id_): + raise KeyError("AIFF key must be four ASCII characters.") + + return id_ in self.__chunks + + def __getitem__(self, id_): + """Get a chunk from the IFF file""" + + assert isinstance(id_, text_type) + + if not is_valid_chunk_id(id_): + raise KeyError("AIFF key must be four ASCII characters.") + + try: + return self.__chunks[id_] + except KeyError: + raise KeyError( + "%r has no %r chunk" % (self.__fileobj.name, id_)) + + def __delitem__(self, id_): + """Remove a chunk from the IFF file""" + + assert isinstance(id_, text_type) + + if not is_valid_chunk_id(id_): + raise KeyError("AIFF key must be four ASCII characters.") + + self.__chunks.pop(id_).delete() + + def insert_chunk(self, id_): + """Insert a new chunk at the end of the IFF file""" + + assert isinstance(id_, text_type) + + if not is_valid_chunk_id(id_): + raise KeyError("AIFF key must be four ASCII characters.") + + self.__fileobj.seek(self.__next_offset) + self.__fileobj.write(pack('>4si', id_.ljust(4).encode('ascii'), 0)) + self.__fileobj.seek(self.__next_offset) + chunk = IFFChunk(self.__fileobj, self[u'FORM']) + self[u'FORM']._update_size(self[u'FORM'].data_size + chunk.size) + + self.__chunks[id_] = chunk + self.__next_offset = chunk.offset + chunk.size + + +class AIFFInfo(StreamInfo): + """AIFF audio stream information. + + Information is parsed from the COMM chunk of the AIFF file + + Useful attributes: + + * length -- audio length, in seconds + * bitrate -- audio bitrate, in bits per second + * channels -- The number of audio channels + * sample_rate -- audio sample rate, in Hz + * sample_size -- The audio sample size + """ + + length = 0 + bitrate = 0 + channels = 0 + sample_rate = 0 + + def __init__(self, fileobj): + iff = IFFFile(fileobj) + try: + common_chunk = iff[u'COMM'] + except KeyError as e: + raise error(str(e)) + + data = common_chunk.read() + + info = struct.unpack('>hLh10s', data[:18]) + channels, frame_count, sample_size, sample_rate = info + + self.sample_rate = int(read_float(sample_rate)) + self.sample_size = sample_size + self.channels = channels + self.bitrate = channels * sample_size * self.sample_rate + self.length = frame_count / float(self.sample_rate) + + def pprint(self): + return u"%d channel AIFF @ %d bps, %s Hz, %.2f seconds" % ( + self.channels, self.bitrate, self.sample_rate, self.length) + + +class _IFFID3(ID3): + """A AIFF file with ID3v2 tags""" + + def _pre_load_header(self, fileobj): + try: + fileobj.seek(IFFFile(fileobj)[u'ID3'].data_offset) + except (InvalidChunk, KeyError): + raise ID3NoHeaderError("No ID3 chunk") + + def save(self, filename=None, v2_version=4, v23_sep='/', padding=None): + """Save ID3v2 data to the AIFF file""" + + if filename is None: + filename = self.filename + + # Unlike the parent ID3.save method, we won't save to a blank file + # since we would have to construct a empty AIFF file + with open(filename, 'rb+') as fileobj: + iff_file = IFFFile(fileobj) + + if u'ID3' not in iff_file: + iff_file.insert_chunk(u'ID3') + + chunk = iff_file[u'ID3'] + + try: + data = self._prepare_data( + fileobj, chunk.data_offset, chunk.data_size, v2_version, + v23_sep, padding) + except ID3Error as e: + reraise(error, e, sys.exc_info()[2]) + + new_size = len(data) + new_size += new_size % 2 # pad byte + assert new_size % 2 == 0 + chunk.resize(new_size) + data += (new_size - len(data)) * b'\x00' + assert new_size == len(data) + chunk.write(data) + + def delete(self, filename=None): + """Completely removes the ID3 chunk from the AIFF file""" + + if filename is None: + filename = self.filename + delete(filename) + self.clear() + + +def delete(filename): + """Completely removes the ID3 chunk from the AIFF file""" + + with open(filename, "rb+") as file_: + try: + del IFFFile(file_)[u'ID3'] + except KeyError: + pass + + +class AIFF(FileType): + """An AIFF audio file. + + :ivar info: :class:`AIFFInfo` + :ivar tags: :class:`ID3` + """ + + _mimes = ["audio/aiff", "audio/x-aiff"] + + @staticmethod + def score(filename, fileobj, header): + filename = filename.lower() + + return (header.startswith(b"FORM") * 2 + endswith(filename, b".aif") + + endswith(filename, b".aiff") + endswith(filename, b".aifc")) + + def add_tags(self): + """Add an empty ID3 tag to the file.""" + if self.tags is None: + self.tags = _IFFID3() + else: + raise error("an ID3 tag already exists") + + def load(self, filename, **kwargs): + """Load stream and tag information from a file.""" + self.filename = filename + + try: + self.tags = _IFFID3(filename, **kwargs) + except ID3NoHeaderError: + self.tags = None + except ID3Error as e: + raise error(e) + + with open(filename, "rb") as fileobj: + self.info = AIFFInfo(fileobj) + + +Open = AIFF diff --git a/libs/mutagen/apev2.py b/libs/mutagen/apev2.py index aa1e00e6..3b79aba9 100644 --- a/libs/mutagen/apev2.py +++ b/libs/mutagen/apev2.py @@ -1,6 +1,6 @@ -# An APEv2 tag reader -# -# Copyright 2005 Joe Wreschnig +# -*- coding: utf-8 -*- + +# Copyright (C) 2005 Joe Wreschnig # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -30,30 +30,45 @@ http://wiki.hydrogenaudio.org/index.php?title=APEv2_specification. __all__ = ["APEv2", "APEv2File", "Open", "delete"] +import sys import struct -from cStringIO import StringIO +from collections import MutableSequence -from mutagen import Metadata, FileType -from mutagen._util import DictMixin, cdata, utf8, delete_bytes +from ._compat import (cBytesIO, PY3, text_type, PY2, reraise, swap_to_string, + xrange) +from mutagen import Metadata, FileType, StreamInfo +from mutagen._util import (DictMixin, cdata, delete_bytes, total_ordering, + MutagenError) def is_valid_apev2_key(key): - return (2 <= len(key) <= 255 and min(key) >= ' ' and max(key) <= '~' and - key not in ["OggS", "TAG", "ID3", "MP+"]) + if not isinstance(key, text_type): + if PY3: + raise TypeError("APEv2 key must be str") + + try: + key = key.decode('ascii') + except UnicodeDecodeError: + return False + + # PY26 - Change to set literal syntax (since set is faster than list here) + return ((2 <= len(key) <= 255) and (min(key) >= u' ') and + (max(key) <= u'~') and + (key not in [u"OggS", u"TAG", u"ID3", u"MP+"])) # There are three different kinds of APE tag values. # "0: Item contains text information coded in UTF-8 # 1: Item contains binary information # 2: Item is a locator of external stored information [e.g. URL] # 3: reserved" -TEXT, BINARY, EXTERNAL = range(3) +TEXT, BINARY, EXTERNAL = xrange(3) -HAS_HEADER = 1L << 31 -HAS_NO_FOOTER = 1L << 30 -IS_HEADER = 1L << 29 +HAS_HEADER = 1 << 31 +HAS_NO_FOOTER = 1 << 30 +IS_HEADER = 1 << 29 -class error(IOError): +class error(IOError, MutagenError): pass @@ -89,9 +104,17 @@ class _APEv2Data(object): def __init__(self, fileobj): self.__find_metadata(fileobj) - self.metadata = max(self.header, self.footer) + + if self.header is None: + self.metadata = self.footer + elif self.footer is None: + self.metadata = self.header + else: + self.metadata = max(self.header, self.footer) + if self.metadata is None: return + self.__fill_missing(fileobj) self.__fix_brokenness(fileobj) if self.data is not None: @@ -107,7 +130,7 @@ class _APEv2Data(object): except IOError: fileobj.seek(0, 2) return - if fileobj.read(8) == "APETAGEX": + if fileobj.read(8) == b"APETAGEX": fileobj.seek(-8, 1) self.footer = self.metadata = fileobj.tell() return @@ -115,10 +138,10 @@ class _APEv2Data(object): # Check for an APEv2 tag followed by an ID3v1 tag at the end. try: fileobj.seek(-128, 2) - if fileobj.read(3) == "TAG": + if fileobj.read(3) == b"TAG": fileobj.seek(-35, 1) # "TAG" + header length - if fileobj.read(8) == "APETAGEX": + if fileobj.read(8) == b"APETAGEX": fileobj.seek(-8, 1) self.footer = fileobj.tell() return @@ -127,7 +150,7 @@ class _APEv2Data(object): # (http://www.id3.org/lyrics3200.html) # (header length - "APETAGEX") - "LYRICS200" fileobj.seek(15, 1) - if fileobj.read(9) == 'LYRICS200': + if fileobj.read(9) == b'LYRICS200': fileobj.seek(-15, 1) # "LYRICS200" + size tag try: offset = int(fileobj.read(6)) @@ -135,7 +158,7 @@ class _APEv2Data(object): raise IOError fileobj.seek(-32 - offset - 6, 1) - if fileobj.read(8) == "APETAGEX": + if fileobj.read(8) == b"APETAGEX": fileobj.seek(-8, 1) self.footer = fileobj.tell() return @@ -145,7 +168,7 @@ class _APEv2Data(object): # Check for a tag at the start. fileobj.seek(0, 0) - if fileobj.read(8) == "APETAGEX": + if fileobj.read(8) == b"APETAGEX": self.is_at_start = True self.header = 0 @@ -162,7 +185,7 @@ class _APEv2Data(object): # offset + the size, which includes the footer. self.end = self.data + self.size fileobj.seek(self.end - 32, 0) - if fileobj.read(8) == "APETAGEX": + if fileobj.read(8) == b"APETAGEX": self.footer = self.end - 32 elif self.footer is not None: self.end = self.footer + 32 @@ -194,7 +217,7 @@ class _APEv2Data(object): except IOError: break else: - if fileobj.read(8) == "APETAGEX": + if fileobj.read(8) == b"APETAGEX": fileobj.seek(-8, 1) start = fileobj.tell() else: @@ -202,18 +225,12 @@ class _APEv2Data(object): self.start = start -class APEv2(DictMixin, Metadata): - """A file with an APEv2 tag. - - ID3v1 tags are silently ignored and overwritten. - """ - - filename = None +class _CIDictProxy(DictMixin): def __init__(self, *args, **kwargs): self.__casemap = {} self.__dict = {} - super(APEv2, self).__init__(*args, **kwargs) + super(_CIDictProxy, self).__init__(*args, **kwargs) # Internally all names are stored as lowercase, but the case # they were set with is remembered and used when saving. This # is roughly in line with the standard, which says that keys @@ -221,31 +238,54 @@ class APEv2(DictMixin, Metadata): # not allowed, and recommends case-insensitive # implementations. + def __getitem__(self, key): + return self.__dict[key.lower()] + + def __setitem__(self, key, value): + lower = key.lower() + self.__casemap[lower] = key + self.__dict[lower] = value + + def __delitem__(self, key): + lower = key.lower() + del(self.__casemap[lower]) + del(self.__dict[lower]) + + def keys(self): + return [self.__casemap.get(key, key) for key in self.__dict.keys()] + + +class APEv2(_CIDictProxy, Metadata): + """A file with an APEv2 tag. + + ID3v1 tags are silently ignored and overwritten. + """ + + filename = None + def pprint(self): """Return tag key=value pairs in a human-readable format.""" - items = self.items() - items.sort() - return "\n".join(["%s=%s" % (k, v.pprint()) for k, v in items]) + + items = sorted(self.items()) + return u"\n".join(u"%s=%s" % (k, v.pprint()) for k, v in items) def load(self, filename): """Load tags from a filename.""" + self.filename = filename - fileobj = open(filename, "rb") - try: + with open(filename, "rb") as fileobj: data = _APEv2Data(fileobj) - finally: - fileobj.close() + if data.tag: self.clear() - self.__casemap.clear() self.__parse_tag(data.tag, data.items) else: raise APENoHeaderError("No APE tag found") def __parse_tag(self, tag, count): - fileobj = StringIO(tag) + fileobj = cBytesIO(tag) - for i in range(count): + for i in xrange(count): size_data = fileobj.read(4) # someone writes wrong item counts if not size_data: @@ -259,25 +299,37 @@ class APEv2(DictMixin, Metadata): if kind == 3: raise APEBadItemError("value type must be 0, 1, or 2") key = value = fileobj.read(1) - while key[-1:] != '\x00' and value: + while key[-1:] != b'\x00' and value: value = fileobj.read(1) key += value - if key[-1:] == "\x00": + if key[-1:] == b"\x00": key = key[:-1] + if PY3: + try: + key = key.decode("ascii") + except UnicodeError as err: + reraise(APEBadItemError, err, sys.exc_info()[2]) value = fileobj.read(size) - self[key] = APEValue(value, kind) + + value = _get_value_type(kind)._new(value) + + self[key] = value def __getitem__(self, key): if not is_valid_apev2_key(key): raise KeyError("%r is not a valid APEv2 key" % key) - key = key.encode('ascii') - return self.__dict[key.lower()] + if PY2: + key = key.encode('ascii') + + return super(APEv2, self).__getitem__(key) def __delitem__(self, key): if not is_valid_apev2_key(key): raise KeyError("%r is not a valid APEv2 key" % key) - key = key.encode('ascii') - del(self.__dict[key.lower()]) + if PY2: + key = key.encode('ascii') + + super(APEv2, self).__delitem__(key) def __setitem__(self, key, value): """'Magic' value setter. @@ -288,6 +340,9 @@ class APEv2(DictMixin, Metadata): as a list of string/Unicode values. If you pass in a string that is not valid UTF-8, it assumes it is a binary value. + Python 3: all bytes will be assumed to be a byte value, even + if they are valid utf-8. + If you need to force a specific type of value (e.g. binary data that also happens to be valid UTF-8, or an external reference), use the APEValue factory and set the value to the @@ -299,30 +354,40 @@ class APEv2(DictMixin, Metadata): if not is_valid_apev2_key(key): raise KeyError("%r is not a valid APEv2 key" % key) - key = key.encode('ascii') + + if PY2: + key = key.encode('ascii') if not isinstance(value, _APEValue): # let's guess at the content if we're not already a value... - if isinstance(value, unicode): + if isinstance(value, text_type): # unicode? we've got to be text. - value = APEValue(utf8(value), TEXT) + value = APEValue(value, TEXT) elif isinstance(value, list): + items = [] + for v in value: + if not isinstance(v, text_type): + if PY3: + raise TypeError("item in list not str") + v = v.decode("utf-8") + items.append(v) + # list? text. - value = APEValue("\0".join(map(utf8, value)), TEXT) + value = APEValue(u"\0".join(items), TEXT) else: - try: - value.decode("utf-8") - except UnicodeError: - # invalid UTF8 text, probably binary + if PY3: value = APEValue(value, BINARY) else: - # valid UTF8, probably text - value = APEValue(value, TEXT) - self.__casemap[key.lower()] = key - self.__dict[key.lower()] = value + try: + value.decode("utf-8") + except UnicodeError: + # invalid UTF8 text, probably binary + value = APEValue(value, BINARY) + else: + # valid UTF8, probably text + value = APEValue(value, TEXT) - def keys(self): - return [self.__casemap.get(key, key) for key in self.__dict.keys()] + super(APEv2, self).__setitem__(key, value) def save(self, filename=None): """Save changes to a file. @@ -348,41 +413,55 @@ class APEv2(DictMixin, Metadata): fileobj.truncate() fileobj.seek(0, 2) + tags = [] + for key, value in self.items(): + # Packed format for an item: + # 4B: Value length + # 4B: Value type + # Key name + # 1B: Null + # Key value + value_data = value._write() + if not isinstance(key, bytes): + key = key.encode("utf-8") + tag_data = bytearray() + tag_data += struct.pack("<2I", len(value_data), value.kind << 1) + tag_data += key + b"\0" + value_data + tags.append(bytes(tag_data)) + # "APE tags items should be sorted ascending by size... This is # not a MUST, but STRONGLY recommended. Actually the items should # be sorted by importance/byte, but this is not feasible." - tags = [v._internal(k) for k, v in self.items()] - tags.sort(lambda a, b: cmp(len(a), len(b))) + tags.sort(key=len) num_tags = len(tags) - tags = "".join(tags) + tags = b"".join(tags) - header = "APETAGEX%s%s" % ( - # version, tag size, item count, flags - struct.pack("<4I", 2000, len(tags) + 32, num_tags, - HAS_HEADER | IS_HEADER), - "\0" * 8) + header = bytearray(b"APETAGEX") + # version, tag size, item count, flags + header += struct.pack("<4I", 2000, len(tags) + 32, num_tags, + HAS_HEADER | IS_HEADER) + header += b"\0" * 8 fileobj.write(header) fileobj.write(tags) - footer = "APETAGEX%s%s" % ( - # version, tag size, item count, flags - struct.pack("<4I", 2000, len(tags) + 32, num_tags, - HAS_HEADER), - "\0" * 8) + footer = bytearray(b"APETAGEX") + footer += struct.pack("<4I", 2000, len(tags) + 32, num_tags, + HAS_HEADER) + footer += b"\0" * 8 + fileobj.write(footer) fileobj.close() def delete(self, filename=None): """Remove tags from a file.""" + filename = filename or self.filename - fileobj = open(filename, "r+b") - try: + with open(filename, "r+b") as fileobj: data = _APEv2Data(fileobj) if data.start is not None and data.size is not None: delete_bytes(fileobj, data.end - data.start, data.start) - finally: - fileobj.close() + self.clear() @@ -391,105 +470,212 @@ Open = APEv2 def delete(filename): """Remove tags from a file.""" + try: APEv2(filename).delete() except APENoHeaderError: pass +def _get_value_type(kind): + """Returns a _APEValue subclass or raises ValueError""" + + if kind == TEXT: + return APETextValue + elif kind == BINARY: + return APEBinaryValue + elif kind == EXTERNAL: + return APEExtValue + raise ValueError("unknown kind %r" % kind) + + def APEValue(value, kind): """APEv2 tag value factory. Use this if you need to specify the value's type manually. Binary and text data are automatically detected by APEv2.__setitem__. """ - if kind == TEXT: - return APETextValue(value, kind) - elif kind == BINARY: - return APEBinaryValue(value, kind) - elif kind == EXTERNAL: - return APEExtValue(value, kind) - else: + + try: + type_ = _get_value_type(kind) + except ValueError: raise ValueError("kind must be TEXT, BINARY, or EXTERNAL") + else: + return type_(value) class _APEValue(object): - def __init__(self, value, kind): - self.kind = kind - self.value = value - def __len__(self): - return len(self.value) + kind = None + value = None - def __str__(self): - return self.value + def __init__(self, value, kind=None): + # kind kwarg is for backwards compat + if kind is not None and kind != self.kind: + raise ValueError + self.value = self._validate(value) - # Packed format for an item: - # 4B: Value length - # 4B: Value type - # Key name - # 1B: Null - # Key value - def _internal(self, key): - return "%s%s\0%s" % ( - struct.pack("<2I", len(self.value), self.kind << 1), - key, self.value) + @classmethod + def _new(cls, data): + instance = cls.__new__(cls) + instance._parse(data) + return instance + + def _parse(self, data): + """Sets value or raises APEBadItemError""" + + raise NotImplementedError + + def _write(self): + """Returns bytes""" + + raise NotImplementedError + + def _validate(self, value): + """Returns validated value or raises TypeError/ValueErrr""" + + raise NotImplementedError def __repr__(self): return "%s(%r, %d)" % (type(self).__name__, self.value, self.kind) -class APETextValue(_APEValue): +@swap_to_string +@total_ordering +class _APEUtf8Value(_APEValue): + + def _parse(self, data): + try: + self.value = data.decode("utf-8") + except UnicodeDecodeError as e: + reraise(APEBadItemError, e, sys.exc_info()[2]) + + def _validate(self, value): + if not isinstance(value, text_type): + if PY3: + raise TypeError("value not str") + else: + value = value.decode("utf-8") + return value + + def _write(self): + return self.value.encode("utf-8") + + def __len__(self): + return len(self.value) + + def __bytes__(self): + return self._write() + + def __eq__(self, other): + return self.value == other + + def __lt__(self, other): + return self.value < other + + def __str__(self): + return self.value + + +class APETextValue(_APEUtf8Value, MutableSequence): """An APEv2 text value. Text values are Unicode/UTF-8 strings. They can be accessed like - strings (with a null seperating the values), or arrays of strings.""" + strings (with a null separating the values), or arrays of strings. + """ - def __unicode__(self): - return unicode(str(self), "utf-8") + kind = TEXT def __iter__(self): """Iterate over the strings of the value (not the characters)""" - return iter(unicode(self).split("\0")) + + return iter(self.value.split(u"\0")) def __getitem__(self, index): - return unicode(self).split("\0")[index] + return self.value.split(u"\0")[index] def __len__(self): - return self.value.count("\0") + 1 - - def __cmp__(self, other): - return cmp(unicode(self), other) - - __hash__ = _APEValue.__hash__ + return self.value.count(u"\0") + 1 def __setitem__(self, index, value): + if not isinstance(value, text_type): + if PY3: + raise TypeError("value not str") + else: + value = value.decode("utf-8") + values = list(self) - values[index] = value.encode("utf-8") - self.value = "\0".join(values).encode("utf-8") + values[index] = value + self.value = u"\0".join(values) + + def insert(self, index, value): + if not isinstance(value, text_type): + if PY3: + raise TypeError("value not str") + else: + value = value.decode("utf-8") + + values = list(self) + values.insert(index, value) + self.value = u"\0".join(values) + + def __delitem__(self, index): + values = list(self) + del values[index] + self.value = u"\0".join(values) def pprint(self): - return " / ".join(self) + return u" / ".join(self) +@swap_to_string +@total_ordering class APEBinaryValue(_APEValue): """An APEv2 binary value.""" + kind = BINARY + + def _parse(self, data): + self.value = data + + def _write(self): + return self.value + + def _validate(self, value): + if not isinstance(value, bytes): + raise TypeError("value not bytes") + return bytes(value) + + def __len__(self): + return len(self.value) + + def __bytes__(self): + return self._write() + + def __eq__(self, other): + return self.value == other + + def __lt__(self, other): + return self.value < other + def pprint(self): - return "[%d bytes]" % len(self) + return u"[%d bytes]" % len(self) -class APEExtValue(_APEValue): +class APEExtValue(_APEUtf8Value): """An APEv2 external value. External values are usually URI or IRI strings. """ + + kind = EXTERNAL + def pprint(self): - return "[External] %s" % unicode(self) + return u"[External] %s" % self.value class APEv2File(FileType): - class _Info(object): + class _Info(StreamInfo): length = 0 bitrate = 0 @@ -498,21 +684,21 @@ class APEv2File(FileType): @staticmethod def pprint(): - return "Unknown format with APEv2 tag." + return u"Unknown format with APEv2 tag." def load(self, filename): self.filename = filename self.info = self._Info(open(filename, "rb")) try: self.tags = APEv2(filename) - except error: + except APENoHeaderError: self.tags = None def add_tags(self): if self.tags is None: self.tags = APEv2() else: - raise ValueError("%r already has tags: %r" % (self, self.tags)) + raise error("%r already has tags: %r" % (self, self.tags)) @staticmethod def score(filename, fileobj, header): @@ -521,5 +707,4 @@ class APEv2File(FileType): except IOError: fileobj.seek(0) footer = fileobj.read() - filename = filename.lower() - return (("APETAGEX" in footer) - header.startswith("ID3")) + return ((b"APETAGEX" in footer) - header.startswith(b"ID3")) diff --git a/libs/mutagen/asf.py b/libs/mutagen/asf.py deleted file mode 100644 index fab5559b..00000000 --- a/libs/mutagen/asf.py +++ /dev/null @@ -1,704 +0,0 @@ -# Copyright 2006-2007 Lukas Lalinsky -# Copyright 2005-2006 Joe Wreschnig -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. - -"""Read and write ASF (Window Media Audio) files.""" - -__all__ = ["ASF", "Open"] - -import struct -from mutagen import FileType, Metadata -from mutagen._util import insert_bytes, delete_bytes, DictMixin - - -class error(IOError): - pass - - -class ASFError(error): - pass - - -class ASFHeaderError(error): - pass - - -class ASFInfo(object): - """ASF stream information.""" - - def __init__(self): - self.length = 0.0 - self.sample_rate = 0 - self.bitrate = 0 - self.channels = 0 - - def pprint(self): - s = "Windows Media Audio %d bps, %s Hz, %d channels, %.2f seconds" % ( - self.bitrate, self.sample_rate, self.channels, self.length) - return s - - -class ASFTags(list, DictMixin, Metadata): - """Dictionary containing ASF attributes.""" - - def pprint(self): - return "\n".join(["%s=%s" % (k, v) for k, v in self]) - - def __getitem__(self, key): - """A list of values for the key. - - This is a copy, so comment['title'].append('a title') will not - work. - - """ - values = [value for (k, value) in self if k == key] - if not values: - raise KeyError(key) - else: - return values - - def __delitem__(self, key): - """Delete all values associated with the key.""" - to_delete = filter(lambda x: x[0] == key, self) - if not to_delete: - raise KeyError(key) - else: - map(self.remove, to_delete) - - def __contains__(self, key): - """Return true if the key has any values.""" - for k, value in self: - if k == key: - return True - else: - return False - - def __setitem__(self, key, values): - """Set a key's value or values. - - Setting a value overwrites all old ones. The value may be a - list of Unicode or UTF-8 strings, or a single Unicode or UTF-8 - string. - - """ - if not isinstance(values, list): - values = [values] - try: - del(self[key]) - except KeyError: - pass - for value in values: - if key in _standard_attribute_names: - value = unicode(value) - elif not isinstance(value, ASFBaseAttribute): - if isinstance(value, basestring): - value = ASFUnicodeAttribute(value) - elif isinstance(value, bool): - value = ASFBoolAttribute(value) - elif isinstance(value, int): - value = ASFDWordAttribute(value) - elif isinstance(value, long): - value = ASFQWordAttribute(value) - self.append((key, value)) - - def keys(self): - """Return all keys in the comment.""" - return self and set(zip(*self)[0]) - - def as_dict(self): - """Return a copy of the comment data in a real dict.""" - d = {} - for key, value in self: - d.setdefault(key, []).append(value) - return d - - -class ASFBaseAttribute(object): - """Generic attribute.""" - TYPE = None - - def __init__(self, value=None, data=None, language=None, - stream=None, **kwargs): - self.language = language - self.stream = stream - if data: - self.value = self.parse(data, **kwargs) - else: - self.value = value - - def data_size(self): - raise NotImplementedError - - def __repr__(self): - name = "%s(%r" % (type(self).__name__, self.value) - if self.language: - name += ", language=%d" % self.language - if self.stream: - name += ", stream=%d" % self.stream - name += ")" - return name - - def render(self, name): - name = name.encode("utf-16-le") + "\x00\x00" - data = self._render() - return (struct.pack(" 0: - texts.append(data[pos:end].decode("utf-16-le").strip("\x00")) - else: - texts.append(None) - pos = end - title, author, copyright, desc, rating = texts - for key, value in dict( - Title=title, - Author=author, - Copyright=copyright, - Description=desc, - Rating=rating - ).items(): - if value is not None: - asf.tags[key] = value - - def render(self, asf): - def render_text(name): - value = asf.tags.get(name, []) - if value: - return value[0].encode("utf-16-le") + "\x00\x00" - else: - return "" - texts = map(render_text, _standard_attribute_names) - data = struct.pack(" 0xFFFF or value.TYPE == GUID) - if (value.language is None and value.stream is None and - name not in self.to_extended_content_description and - not library_only): - self.to_extended_content_description[name] = value - elif (value.language is None and value.stream is not None and - name not in self.to_metadata and not library_only): - self.to_metadata[name] = value - else: - self.to_metadata_library.append((name, value)) - - # Add missing objects - if not self.content_description_obj: - self.content_description_obj = \ - ContentDescriptionObject() - self.objects.append(self.content_description_obj) - if not self.extended_content_description_obj: - self.extended_content_description_obj = \ - ExtendedContentDescriptionObject() - self.objects.append(self.extended_content_description_obj) - if not self.header_extension_obj: - self.header_extension_obj = \ - HeaderExtensionObject() - self.objects.append(self.header_extension_obj) - if not self.metadata_obj: - self.metadata_obj = \ - MetadataObject() - self.header_extension_obj.objects.append(self.metadata_obj) - if not self.metadata_library_obj: - self.metadata_library_obj = \ - MetadataLibraryObject() - self.header_extension_obj.objects.append(self.metadata_library_obj) - - # Render the header - data = "".join([obj.render(self) for obj in self.objects]) - data = (HeaderObject.GUID + - struct.pack(" self.size: - insert_bytes(fileobj, size - self.size, self.size) - if size < self.size: - delete_bytes(fileobj, self.size - size, 0) - fileobj.seek(0) - fileobj.write(data) - finally: - fileobj.close() - - self.size = size - self.num_objects = len(self.objects) - - def __read_file(self, fileobj): - header = fileobj.read(30) - if len(header) != 30 or header[:16] != HeaderObject.GUID: - raise ASFHeaderError("Not an ASF file.") - - self.extended_content_description_obj = None - self.content_description_obj = None - self.header_extension_obj = None - self.metadata_obj = None - self.metadata_library_obj = None - - self.size, self.num_objects = struct.unpack(" 0xFFFF or value.TYPE == GUID) + can_cont_desc = value.TYPE == UNICODE + + if library_only or value.language is not None: + self.to_metadata_library.append((name, value)) + elif value.stream is not None: + if name not in self.to_metadata: + self.to_metadata[name] = value + else: + self.to_metadata_library.append((name, value)) + elif name in ContentDescriptionObject.NAMES: + if name not in self.to_content_description and can_cont_desc: + self.to_content_description[name] = value + else: + self.to_metadata_library.append((name, value)) + else: + if name not in self.to_extended_content_description: + self.to_extended_content_description[name] = value + else: + self.to_metadata_library.append((name, value)) + + # Add missing objects + header = self._header + if header.get_child(ContentDescriptionObject.GUID) is None: + header.objects.append(ContentDescriptionObject()) + if header.get_child(ExtendedContentDescriptionObject.GUID) is None: + header.objects.append(ExtendedContentDescriptionObject()) + header_ext = header.get_child(HeaderExtensionObject.GUID) + if header_ext is None: + header_ext = HeaderExtensionObject() + header.objects.append(header_ext) + if header_ext.get_child(MetadataObject.GUID) is None: + header_ext.objects.append(MetadataObject()) + if header_ext.get_child(MetadataLibraryObject.GUID) is None: + header_ext.objects.append(MetadataLibraryObject()) + + # Render to file + with open(self.filename, "rb+") as fileobj: + old_size = header.parse_size(fileobj)[0] + data = header.render_full(self, fileobj, old_size, padding) + size = len(data) + resize_bytes(fileobj, old_size, size, 0) + fileobj.seek(0) + fileobj.write(data) + + def add_tags(self): + raise ASFError + + def delete(self, filename=None): + + if filename is not None and filename != self.filename: + raise ValueError("saving to another file not supported atm") + + self.tags.clear() + self.save(padding=lambda x: 0) + + @staticmethod + def score(filename, fileobj, header): + return header.startswith(HeaderObject.GUID) * 2 + +Open = ASF diff --git a/libs/mutagen/asf/_attrs.py b/libs/mutagen/asf/_attrs.py new file mode 100644 index 00000000..4621c9fa --- /dev/null +++ b/libs/mutagen/asf/_attrs.py @@ -0,0 +1,438 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2005-2006 Joe Wreschnig +# Copyright (C) 2006-2007 Lukas Lalinsky +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +import sys +import struct + +from mutagen._compat import swap_to_string, text_type, PY2, reraise +from mutagen._util import total_ordering + +from ._util import ASFError + + +class ASFBaseAttribute(object): + """Generic attribute.""" + + TYPE = None + + _TYPES = {} + + value = None + """The Python value of this attribute (type depends on the class)""" + + language = None + """Language""" + + stream = None + """Stream""" + + def __init__(self, value=None, data=None, language=None, + stream=None, **kwargs): + self.language = language + self.stream = stream + if data: + self.value = self.parse(data, **kwargs) + else: + if value is None: + # we used to support not passing any args and instead assign + # them later, keep that working.. + self.value = None + else: + self.value = self._validate(value) + + @classmethod + def _register(cls, other): + cls._TYPES[other.TYPE] = other + return other + + @classmethod + def _get_type(cls, type_): + """Raises KeyError""" + + return cls._TYPES[type_] + + def _validate(self, value): + """Raises TypeError or ValueError in case the user supplied value + isn't valid. + """ + + return value + + def data_size(self): + raise NotImplementedError + + def __repr__(self): + name = "%s(%r" % (type(self).__name__, self.value) + if self.language: + name += ", language=%d" % self.language + if self.stream: + name += ", stream=%d" % self.stream + name += ")" + return name + + def render(self, name): + name = name.encode("utf-16-le") + b"\x00\x00" + data = self._render() + return (struct.pack("" % ( + type(self).__name__, bytes2guid(self.GUID), self.objects) + + def pprint(self): + l = [] + l.append("%s(%s)" % (type(self).__name__, bytes2guid(self.GUID))) + for o in self.objects: + for e in o.pprint().splitlines(): + l.append(" " + e) + return "\n".join(l) + + +class UnknownObject(BaseObject): + """Unknown ASF object.""" + + def __init__(self, guid): + super(UnknownObject, self).__init__() + assert isinstance(guid, bytes) + self.GUID = guid + + +@BaseObject._register +class HeaderObject(BaseObject): + """ASF header.""" + + GUID = guid2bytes("75B22630-668E-11CF-A6D9-00AA0062CE6C") + + @classmethod + def parse_full(cls, asf, fileobj): + """Raises ASFHeaderError""" + + header = cls() + + remaining_header, num_objects = cls.parse_size(fileobj) + remaining_header -= 30 + + for i in xrange(num_objects): + obj_header_size = 24 + if remaining_header < obj_header_size: + raise ASFHeaderError("invalid header size") + data = fileobj.read(obj_header_size) + if len(data) != obj_header_size: + raise ASFHeaderError("truncated") + remaining_header -= obj_header_size + + guid, size = struct.unpack("<16sQ", data) + obj = BaseObject._get_object(guid) + + payload_size = size - obj_header_size + if remaining_header < payload_size: + raise ASFHeaderError("invalid object size") + remaining_header -= payload_size + + try: + data = fileobj.read(payload_size) + except OverflowError: + # read doesn't take 64bit values + raise ASFHeaderError("invalid header size") + if len(data) != payload_size: + raise ASFHeaderError("truncated") + + obj.parse(asf, data) + header.objects.append(obj) + + return header + + @classmethod + def parse_size(cls, fileobj): + """Returns (size, num_objects) + + Raises ASFHeaderError + """ + + header = fileobj.read(30) + if len(header) != 30 or header[:16] != HeaderObject.GUID: + raise ASFHeaderError("Not an ASF file.") + + return struct.unpack("= 0 + info = PaddingInfo(available - needed_size, content_size) + + # add padding + padding = info._get_padding(padding_func) + padding_obj.parse(asf, b"\x00" * padding) + data += padding_obj.render(asf) + num_objects += 1 + + data = (HeaderObject.GUID + + struct.pack(" 0: + texts.append(data[pos:end].decode("utf-16-le").strip(u"\x00")) + else: + texts.append(None) + pos = end + + for key, value in izip(self.NAMES, texts): + if value is not None: + value = ASFUnicodeAttribute(value=value) + asf._tags.setdefault(self.GUID, []).append((key, value)) + + def render(self, asf): + def render_text(name): + value = asf.to_content_description.get(name) + if value is not None: + return text_type(value).encode("utf-16-le") + b"\x00\x00" + else: + return b"" + + texts = [render_text(x) for x in self.NAMES] + data = struct.pack("= 0 + asf.info.length = max((length / 10000000.0) - (preroll / 1000.0), 0.0) + + +@BaseObject._register +class StreamPropertiesObject(BaseObject): + """Stream properties.""" + + GUID = guid2bytes("B7DC0791-A9B7-11CF-8EE6-00C00C205365") + + def parse(self, asf, data): + super(StreamPropertiesObject, self).parse(asf, data) + channels, sample_rate, bitrate = struct.unpack("H", int(s[19:23], 16)), + p(">Q", int(s[24:], 16))[2:], + ]) + + +def bytes2guid(s): + """Converts a serialized GUID to a text GUID""" + + assert isinstance(s, bytes) + + u = struct.unpack + v = [] + v.extend(u("HQ", s[8:10] + b"\x00\x00" + s[10:])) + return "%08X-%04X-%04X-%04X-%012X" % tuple(v) + + +# Names from http://windows.microsoft.com/en-za/windows7/c00d10d1-[0-9A-F]{1,4} +CODECS = { + 0x0000: u"Unknown Wave Format", + 0x0001: u"Microsoft PCM Format", + 0x0002: u"Microsoft ADPCM Format", + 0x0003: u"IEEE Float", + 0x0004: u"Compaq Computer VSELP", + 0x0005: u"IBM CVSD", + 0x0006: u"Microsoft CCITT A-Law", + 0x0007: u"Microsoft CCITT u-Law", + 0x0008: u"Microsoft DTS", + 0x0009: u"Microsoft DRM", + 0x000A: u"Windows Media Audio 9 Voice", + 0x000B: u"Windows Media Audio 10 Voice", + 0x000C: u"OGG Vorbis", + 0x000D: u"FLAC", + 0x000E: u"MOT AMR", + 0x000F: u"Nice Systems IMBE", + 0x0010: u"OKI ADPCM", + 0x0011: u"Intel IMA ADPCM", + 0x0012: u"Videologic MediaSpace ADPCM", + 0x0013: u"Sierra Semiconductor ADPCM", + 0x0014: u"Antex Electronics G.723 ADPCM", + 0x0015: u"DSP Solutions DIGISTD", + 0x0016: u"DSP Solutions DIGIFIX", + 0x0017: u"Dialogic OKI ADPCM", + 0x0018: u"MediaVision ADPCM", + 0x0019: u"Hewlett-Packard CU codec", + 0x001A: u"Hewlett-Packard Dynamic Voice", + 0x0020: u"Yamaha ADPCM", + 0x0021: u"Speech Compression SONARC", + 0x0022: u"DSP Group True Speech", + 0x0023: u"Echo Speech EchoSC1", + 0x0024: u"Ahead Inc. Audiofile AF36", + 0x0025: u"Audio Processing Technology APTX", + 0x0026: u"Ahead Inc. AudioFile AF10", + 0x0027: u"Aculab Prosody 1612", + 0x0028: u"Merging Technologies S.A. LRC", + 0x0030: u"Dolby Labs AC2", + 0x0031: u"Microsoft GSM 6.10", + 0x0032: u"Microsoft MSNAudio", + 0x0033: u"Antex Electronics ADPCME", + 0x0034: u"Control Resources VQLPC", + 0x0035: u"DSP Solutions Digireal", + 0x0036: u"DSP Solutions DigiADPCM", + 0x0037: u"Control Resources CR10", + 0x0038: u"Natural MicroSystems VBXADPCM", + 0x0039: u"Crystal Semiconductor IMA ADPCM", + 0x003A: u"Echo Speech EchoSC3", + 0x003B: u"Rockwell ADPCM", + 0x003C: u"Rockwell DigiTalk", + 0x003D: u"Xebec Multimedia Solutions", + 0x0040: u"Antex Electronics G.721 ADPCM", + 0x0041: u"Antex Electronics G.728 CELP", + 0x0042: u"Intel G.723", + 0x0043: u"Intel G.723.1", + 0x0044: u"Intel G.729 Audio", + 0x0045: u"Sharp G.726 Audio", + 0x0050: u"Microsoft MPEG-1", + 0x0052: u"InSoft RT24", + 0x0053: u"InSoft PAC", + 0x0055: u"MP3 - MPEG Layer III", + 0x0059: u"Lucent G.723", + 0x0060: u"Cirrus Logic", + 0x0061: u"ESS Technology ESPCM", + 0x0062: u"Voxware File-Mode", + 0x0063: u"Canopus Atrac", + 0x0064: u"APICOM G.726 ADPCM", + 0x0065: u"APICOM G.722 ADPCM", + 0x0066: u"Microsoft DSAT", + 0x0067: u"Microsoft DSAT Display", + 0x0069: u"Voxware Byte Aligned", + 0x0070: u"Voxware AC8", + 0x0071: u"Voxware AC10", + 0x0072: u"Voxware AC16", + 0x0073: u"Voxware AC20", + 0x0074: u"Voxware RT24 MetaVoice", + 0x0075: u"Voxware RT29 MetaSound", + 0x0076: u"Voxware RT29HW", + 0x0077: u"Voxware VR12", + 0x0078: u"Voxware VR18", + 0x0079: u"Voxware TQ40", + 0x007A: u"Voxware SC3", + 0x007B: u"Voxware SC3", + 0x0080: u"Softsound", + 0x0081: u"Voxware TQ60", + 0x0082: u"Microsoft MSRT24", + 0x0083: u"AT&T Labs G.729A", + 0x0084: u"Motion Pixels MVI MV12", + 0x0085: u"DataFusion Systems G.726", + 0x0086: u"DataFusion Systems GSM610", + 0x0088: u"Iterated Systems ISIAudio", + 0x0089: u"Onlive", + 0x008A: u"Multitude FT SX20", + 0x008B: u"Infocom ITS ACM G.721", + 0x008C: u"Convedia G.729", + 0x008D: u"Congruency Audio", + 0x0091: u"Siemens Business Communications SBC24", + 0x0092: u"Sonic Foundry Dolby AC3 SPDIF", + 0x0093: u"MediaSonic G.723", + 0x0094: u"Aculab Prosody 8KBPS", + 0x0097: u"ZyXEL ADPCM", + 0x0098: u"Philips LPCBB", + 0x0099: u"Studer Professional Audio AG Packed", + 0x00A0: u"Malden Electronics PHONYTALK", + 0x00A1: u"Racal Recorder GSM", + 0x00A2: u"Racal Recorder G720.a", + 0x00A3: u"Racal Recorder G723.1", + 0x00A4: u"Racal Recorder Tetra ACELP", + 0x00B0: u"NEC AAC", + 0x00FF: u"CoreAAC Audio", + 0x0100: u"Rhetorex ADPCM", + 0x0101: u"BeCubed Software IRAT", + 0x0111: u"Vivo G.723", + 0x0112: u"Vivo Siren", + 0x0120: u"Philips CELP", + 0x0121: u"Philips Grundig", + 0x0123: u"Digital G.723", + 0x0125: u"Sanyo ADPCM", + 0x0130: u"Sipro Lab Telecom ACELP.net", + 0x0131: u"Sipro Lab Telecom ACELP.4800", + 0x0132: u"Sipro Lab Telecom ACELP.8V3", + 0x0133: u"Sipro Lab Telecom ACELP.G.729", + 0x0134: u"Sipro Lab Telecom ACELP.G.729A", + 0x0135: u"Sipro Lab Telecom ACELP.KELVIN", + 0x0136: u"VoiceAge AMR", + 0x0140: u"Dictaphone G.726 ADPCM", + 0x0141: u"Dictaphone CELP68", + 0x0142: u"Dictaphone CELP54", + 0x0150: u"Qualcomm PUREVOICE", + 0x0151: u"Qualcomm HALFRATE", + 0x0155: u"Ring Zero Systems TUBGSM", + 0x0160: u"Windows Media Audio Standard", + 0x0161: u"Windows Media Audio 9 Standard", + 0x0162: u"Windows Media Audio 9 Professional", + 0x0163: u"Windows Media Audio 9 Lossless", + 0x0164: u"Windows Media Audio Pro over SPDIF", + 0x0170: u"Unisys NAP ADPCM", + 0x0171: u"Unisys NAP ULAW", + 0x0172: u"Unisys NAP ALAW", + 0x0173: u"Unisys NAP 16K", + 0x0174: u"Sycom ACM SYC008", + 0x0175: u"Sycom ACM SYC701 G725", + 0x0176: u"Sycom ACM SYC701 CELP54", + 0x0177: u"Sycom ACM SYC701 CELP68", + 0x0178: u"Knowledge Adventure ADPCM", + 0x0180: u"Fraunhofer IIS MPEG-2 AAC", + 0x0190: u"Digital Theater Systems DTS", + 0x0200: u"Creative Labs ADPCM", + 0x0202: u"Creative Labs FastSpeech8", + 0x0203: u"Creative Labs FastSpeech10", + 0x0210: u"UHER informatic GmbH ADPCM", + 0x0215: u"Ulead DV Audio", + 0x0216: u"Ulead DV Audio", + 0x0220: u"Quarterdeck", + 0x0230: u"I-link Worldwide ILINK VC", + 0x0240: u"Aureal Semiconductor RAW SPORT", + 0x0249: u"Generic Passthru", + 0x0250: u"Interactive Products HSX", + 0x0251: u"Interactive Products RPELP", + 0x0260: u"Consistent Software CS2", + 0x0270: u"Sony SCX", + 0x0271: u"Sony SCY", + 0x0272: u"Sony ATRAC3", + 0x0273: u"Sony SPC", + 0x0280: u"Telum Audio", + 0x0281: u"Telum IA Audio", + 0x0285: u"Norcom Voice Systems ADPCM", + 0x0300: u"Fujitsu TOWNS SND", + 0x0350: u"Micronas SC4 Speech", + 0x0351: u"Micronas CELP833", + 0x0400: u"Brooktree BTV Digital", + 0x0401: u"Intel Music Coder", + 0x0402: u"Intel Audio", + 0x0450: u"QDesign Music", + 0x0500: u"On2 AVC0 Audio", + 0x0501: u"On2 AVC1 Audio", + 0x0680: u"AT&T Labs VME VMPCM", + 0x0681: u"AT&T Labs TPC", + 0x08AE: u"ClearJump Lightwave Lossless", + 0x1000: u"Olivetti GSM", + 0x1001: u"Olivetti ADPCM", + 0x1002: u"Olivetti CELP", + 0x1003: u"Olivetti SBC", + 0x1004: u"Olivetti OPR", + 0x1100: u"Lernout & Hauspie", + 0x1101: u"Lernout & Hauspie CELP", + 0x1102: u"Lernout & Hauspie SBC8", + 0x1103: u"Lernout & Hauspie SBC12", + 0x1104: u"Lernout & Hauspie SBC16", + 0x1400: u"Norris Communication", + 0x1401: u"ISIAudio", + 0x1500: u"AT&T Labs Soundspace Music Compression", + 0x1600: u"Microsoft MPEG ADTS AAC", + 0x1601: u"Microsoft MPEG RAW AAC", + 0x1608: u"Nokia MPEG ADTS AAC", + 0x1609: u"Nokia MPEG RAW AAC", + 0x181C: u"VoxWare MetaVoice RT24", + 0x1971: u"Sonic Foundry Lossless", + 0x1979: u"Innings Telecom ADPCM", + 0x1FC4: u"NTCSoft ALF2CD ACM", + 0x2000: u"Dolby AC3", + 0x2001: u"DTS", + 0x4143: u"Divio AAC", + 0x4201: u"Nokia Adaptive Multi-Rate", + 0x4243: u"Divio G.726", + 0x4261: u"ITU-T H.261", + 0x4263: u"ITU-T H.263", + 0x4264: u"ITU-T H.264", + 0x674F: u"Ogg Vorbis Mode 1", + 0x6750: u"Ogg Vorbis Mode 2", + 0x6751: u"Ogg Vorbis Mode 3", + 0x676F: u"Ogg Vorbis Mode 1+", + 0x6770: u"Ogg Vorbis Mode 2+", + 0x6771: u"Ogg Vorbis Mode 3+", + 0x7000: u"3COM NBX Audio", + 0x706D: u"FAAD AAC Audio", + 0x77A1: u"True Audio Lossless Audio", + 0x7A21: u"GSM-AMR CBR 3GPP Audio", + 0x7A22: u"GSM-AMR VBR 3GPP Audio", + 0xA100: u"Comverse Infosys G723.1", + 0xA101: u"Comverse Infosys AVQSBC", + 0xA102: u"Comverse Infosys SBC", + 0xA103: u"Symbol Technologies G729a", + 0xA104: u"VoiceAge AMR WB", + 0xA105: u"Ingenient Technologies G.726", + 0xA106: u"ISO/MPEG-4 Advanced Audio Coding (AAC)", + 0xA107: u"Encore Software Ltd's G.726", + 0xA108: u"ZOLL Medical Corporation ASAO", + 0xA109: u"Speex Voice", + 0xA10A: u"Vianix MASC Speech Compression", + 0xA10B: u"Windows Media 9 Spectrum Analyzer Output", + 0xA10C: u"Media Foundation Spectrum Analyzer Output", + 0xA10D: u"GSM 6.10 (Full-Rate) Speech", + 0xA10E: u"GSM 6.20 (Half-Rate) Speech", + 0xA10F: u"GSM 6.60 (Enchanced Full-Rate) Speech", + 0xA110: u"GSM 6.90 (Adaptive Multi-Rate) Speech", + 0xA111: u"GSM Adaptive Multi-Rate WideBand Speech", + 0xA112: u"Polycom G.722", + 0xA113: u"Polycom G.728", + 0xA114: u"Polycom G.729a", + 0xA115: u"Polycom Siren", + 0xA116: u"Global IP Sound ILBC", + 0xA117: u"Radio Time Time Shifted Radio", + 0xA118: u"Nice Systems ACA", + 0xA119: u"Nice Systems ADPCM", + 0xA11A: u"Vocord Group ITU-T G.721", + 0xA11B: u"Vocord Group ITU-T G.726", + 0xA11C: u"Vocord Group ITU-T G.722.1", + 0xA11D: u"Vocord Group ITU-T G.728", + 0xA11E: u"Vocord Group ITU-T G.729", + 0xA11F: u"Vocord Group ITU-T G.729a", + 0xA120: u"Vocord Group ITU-T G.723.1", + 0xA121: u"Vocord Group LBC", + 0xA122: u"Nice G.728", + 0xA123: u"France Telecom G.729 ACM Audio", + 0xA124: u"CODIAN Audio", + 0xCC12: u"Intel YUV12 Codec", + 0xCFCC: u"Digital Processing Systems Perception Motion JPEG", + 0xD261: u"DEC H.261", + 0xD263: u"DEC H.263", + 0xFFFE: u"Extensible Wave Format", + 0xFFFF: u"Unregistered", +} diff --git a/libs/mutagen/easyid3.py b/libs/mutagen/easyid3.py index e69a5453..f8dd2de0 100644 --- a/libs/mutagen/easyid3.py +++ b/libs/mutagen/easyid3.py @@ -1,5 +1,6 @@ -# Simpler (but far more limited) API for ID3 editing -# Copyright 2006 Joe Wreschnig +# -*- coding: utf-8 -*- + +# Copyright (C) 2006 Joe Wreschnig # # This program is free software; you can redistribute it and/or modify # it under the terms of version 2 of the GNU General Public License as @@ -13,6 +14,7 @@ more like Vorbis or APEv2 tags. import mutagen.id3 +from ._compat import iteritems, text_type, PY2 from mutagen import Metadata from mutagen._util import DictMixin, dict_match from mutagen.id3 import ID3, error, delete, ID3FileType @@ -154,6 +156,8 @@ class EasyID3(DictMixin, Metadata): for v in value: if v and max(v) > u'\x7f': enc = 3 + break + id3.add(mutagen.id3.TXXX(encoding=enc, text=value, desc=desc)) else: frame.text = value @@ -171,8 +175,10 @@ class EasyID3(DictMixin, Metadata): load = property(lambda s: s.__id3.load, lambda s, v: setattr(s.__id3, 'load', v)) - save = property(lambda s: s.__id3.save, - lambda s, v: setattr(s.__id3, 'save', v)) + def save(self, *args, **kwargs): + # ignore v2_version until we support 2.3 here + kwargs.pop("v2_version", None) + self.__id3.save(*args, **kwargs) delete = property(lambda s: s.__id3.delete, lambda s, v: setattr(s.__id3, 'delete', v)) @@ -193,8 +199,12 @@ class EasyID3(DictMixin, Metadata): def __setitem__(self, key, value): key = key.lower() - if isinstance(value, basestring): - value = [value] + if PY2: + if isinstance(value, basestring): + value = [value] + else: + if isinstance(value, text_type): + value = [value] func = dict_match(self.Set, key, self.SetFallback) if func is not None: return func(self.__id3, key, value) @@ -263,6 +273,18 @@ def date_delete(id3, key): del(id3["TDRC"]) +def original_date_get(id3, key): + return [stamp.text for stamp in id3["TDOR"].text] + + +def original_date_set(id3, key, value): + id3.add(mutagen.id3.TDOR(encoding=3, text=value)) + + +def original_date_delete(id3, key): + del(id3["TDOR"]) + + def performer_get(id3, key): people = [] wanted_role = key.split(":", 1)[1] @@ -433,7 +455,7 @@ def peakgain_list(id3, key): keys.append("replaygain_%s_peak" % frame.desc) return keys -for frameid, key in { +for frameid, key in iteritems({ "TALB": "album", "TBPM": "bpm", "TCMP": "compilation", # iTunes extension @@ -461,18 +483,20 @@ for frameid, key in { "TSOT": "titlesort", "TSRC": "isrc", "TSST": "discsubtitle", -}.iteritems(): + "TLAN": "language", +}): EasyID3.RegisterTextKey(key, frameid) EasyID3.RegisterKey("genre", genre_get, genre_set, genre_delete) EasyID3.RegisterKey("date", date_get, date_set, date_delete) +EasyID3.RegisterKey("originaldate", original_date_get, original_date_set, + original_date_delete) EasyID3.RegisterKey( "performer:*", performer_get, performer_set, performer_delete, performer_list) EasyID3.RegisterKey("musicbrainz_trackid", musicbrainz_trackid_get, musicbrainz_trackid_set, musicbrainz_trackid_delete) EasyID3.RegisterKey("website", website_get, website_set, website_delete) -EasyID3.RegisterKey("website", website_get, website_set, website_delete) EasyID3.RegisterKey( "replaygain_*_gain", gain_get, gain_set, gain_delete, peakgain_list) EasyID3.RegisterKey("replaygain_*_peak", peak_get, peak_set, peak_delete) @@ -481,7 +505,7 @@ EasyID3.RegisterKey("replaygain_*_peak", peak_get, peak_set, peak_delete) # http://musicbrainz.org/docs/specs/metadata_tags.html # http://bugs.musicbrainz.org/ticket/1383 # http://musicbrainz.org/doc/MusicBrainzTag -for desc, key in { +for desc, key in iteritems({ u"MusicBrainz Artist Id": "musicbrainz_artistid", u"MusicBrainz Album Id": "musicbrainz_albumid", u"MusicBrainz Album Artist Id": "musicbrainz_albumartistid", @@ -495,7 +519,13 @@ for desc, key in { u"ASIN": "asin", u"ALBUMARTISTSORT": "albumartistsort", u"BARCODE": "barcode", -}.iteritems(): + u"CATALOGNUMBER": "catalognumber", + u"MusicBrainz Release Track Id": "musicbrainz_releasetrackid", + u"MusicBrainz Release Group Id": "musicbrainz_releasegroupid", + u"MusicBrainz Work Id": "musicbrainz_workid", + u"Acoustid Fingerprint": "acoustid_fingerprint", + u"Acoustid Id": "acoustid_id", +}): EasyID3.RegisterTXXXKey(key, desc) diff --git a/libs/mutagen/easymp4.py b/libs/mutagen/easymp4.py index 3abacccc..8ad7fd0e 100644 --- a/libs/mutagen/easymp4.py +++ b/libs/mutagen/easymp4.py @@ -1,12 +1,16 @@ -# Copyright 2009 Joe Wreschnig +# -*- coding: utf-8 -*- + +# Copyright (C) 2009 Joe Wreschnig # # This program is free software; you can redistribute it and/or modify # it under the terms of version 2 of the GNU General Public License as # published by the Free Software Foundation. -from mutagen import Metadata -from mutagen._util import DictMixin, dict_match, utf8 +from mutagen import Tags +from mutagen._util import DictMixin, dict_match from mutagen.mp4 import MP4, MP4Tags, error, delete +from ._compat import PY2, text_type, PY3 + __all__ = ["EasyMP4Tags", "EasyMP4", "delete", "error"] @@ -15,14 +19,14 @@ class EasyMP4KeyError(error, KeyError, ValueError): pass -class EasyMP4Tags(DictMixin, Metadata): +class EasyMP4Tags(DictMixin, Tags): """A file with MPEG-4 iTunes metadata. Like Vorbis comments, EasyMP4Tags keys are case-insensitive ASCII strings, and values are a list of Unicode strings (and these lists are always of length 0 or 1). - If you need access to the full MP4 metadata feature set, you should use + If you need access to the full MP4 metadata feature set, you should use MP4, not EasyMP4. """ @@ -36,6 +40,7 @@ class EasyMP4Tags(DictMixin, Metadata): self.load = self.__mp4.load self.save = self.__mp4.save self.delete = self.__mp4.delete + self._padding = self.__mp4._padding filename = property(lambda s: s.__mp4.filename, lambda s, fn: setattr(s.__mp4, 'filename', fn)) @@ -91,16 +96,16 @@ class EasyMP4Tags(DictMixin, Metadata): cls.RegisterKey(key, getter, setter, deleter) @classmethod - def RegisterIntKey(cls, key, atomid, min_value=0, max_value=2**16-1): + def RegisterIntKey(cls, key, atomid, min_value=0, max_value=(2 ** 16) - 1): """Register a scalar integer key. """ def getter(tags, key): - return map(unicode, tags[atomid]) + return list(map(text_type, tags[atomid])) def setter(tags, key, value): clamp = lambda x: int(min(max(min_value, x), max_value)) - tags[atomid] = map(clamp, map(int, value)) + tags[atomid] = [clamp(v) for v in map(int, value)] def deleter(tags, key): del(tags[atomid]) @@ -108,14 +113,15 @@ class EasyMP4Tags(DictMixin, Metadata): cls.RegisterKey(key, getter, setter, deleter) @classmethod - def RegisterIntPairKey(cls, key, atomid, min_value=0, max_value=2**16-1): + def RegisterIntPairKey(cls, key, atomid, min_value=0, + max_value=(2 ** 16) - 1): def getter(tags, key): ret = [] for (track, total) in tags[atomid]: if total: ret.append(u"%d/%d" % (track, total)) else: - ret.append(unicode(track)) + ret.append(text_type(track)) return ret def setter(tags, key, value): @@ -148,13 +154,20 @@ class EasyMP4Tags(DictMixin, Metadata): EasyMP4Tags.RegisterFreeformKey( "musicbrainz_artistid", "MusicBrainz Artist Id") """ - atomid = "----:%s:%s" % (mean, name) + atomid = "----:" + mean + ":" + name def getter(tags, key): return [s.decode("utf-8", "replace") for s in tags[atomid]] def setter(tags, key, value): - tags[atomid] = map(utf8, value) + encoded = [] + for v in value: + if not isinstance(v, text_type): + if PY3: + raise TypeError("%r not str" % v) + v = v.decode("utf-8") + encoded.append(v.encode("utf-8")) + tags[atomid] = encoded def deleter(tags, key): del(tags[atomid]) @@ -171,8 +184,14 @@ class EasyMP4Tags(DictMixin, Metadata): def __setitem__(self, key, value): key = key.lower() - if isinstance(value, basestring): - value = [value] + + if PY2: + if isinstance(value, basestring): + value = [value] + else: + if isinstance(value, text_type): + value = [value] + func = dict_match(self.Set, key) if func is not None: return func(self.__mp4, key, value) diff --git a/libs/mutagen/flac.py b/libs/mutagen/flac.py index f8e014bc..f3cc5ab5 100644 --- a/libs/mutagen/flac.py +++ b/libs/mutagen/flac.py @@ -1,5 +1,6 @@ -# FLAC comment support for Mutagen -# Copyright 2005 Joe Wreschnig +# -*- coding: utf-8 -*- + +# Copyright (C) 2005 Joe Wreschnig # # This program is free software; you can redistribute it and/or modify # it under the terms of version 2 of the GNU General Public License as @@ -22,17 +23,17 @@ http://flac.sourceforge.net/format.html __all__ = ["FLAC", "Open", "delete"] import struct -from cStringIO import StringIO -from _vorbis import VCommentDict -from mutagen import FileType -from mutagen._util import insert_bytes +from ._vorbis import VCommentDict +import mutagen + +from ._compat import cBytesIO, endswith, chr_, xrange +from mutagen._util import resize_bytes, MutagenError, get_size +from mutagen._tags import PaddingInfo from mutagen.id3 import BitPaddedInt -import sys -if sys.version_info >= (2, 6): - from functools import reduce +from functools import reduce -class error(IOError): +class error(IOError, MutagenError): pass @@ -44,10 +45,10 @@ class FLACVorbisError(ValueError, error): pass -def to_int_be(string): +def to_int_be(data): """Convert an arbitrarily-long string to a long using big-endian byte order.""" - return reduce(lambda a, b: (a << 8) + ord(b), string, 0L) + return reduce(lambda a, b: (a << 8) + b, bytearray(data), 0) class StrictFileObject(object): @@ -83,14 +84,23 @@ class MetadataBlock(object): """ _distrust_size = False + """For block types setting this, we don't trust the size field and + use the size of the content instead.""" + + _invalid_overflow_size = -1 + """In case the real size was bigger than what is representable by the + 24 bit size field, we save the wrong specified size here. This can + only be set if _distrust_size is True""" + + _MAX_SIZE = 2 ** 24 - 1 def __init__(self, data): """Parse the given data string or file-like as a metadata block. The metadata header should not be included.""" if data is not None: if not isinstance(data, StrictFileObject): - if isinstance(data, str): - data = StringIO(data) + if isinstance(data, bytes): + data = cBytesIO(data) elif not hasattr(data, 'read'): raise TypeError( "StreamInfo requires string data or a file-like") @@ -103,37 +113,61 @@ class MetadataBlock(object): def write(self): return self.data - @staticmethod - def writeblocks(blocks): - """Render metadata block as a byte string.""" - data = [] - codes = [[block.code, block.write()] for block in blocks] - codes[-1][0] |= 128 - for code, datum in codes: - byte = chr(code) - if len(datum) > 2**24: + @classmethod + def _writeblock(cls, block, is_last=False): + """Returns the block content + header. + + Raises error. + """ + + data = bytearray() + code = (block.code | 128) if is_last else block.code + datum = block.write() + size = len(datum) + if size > cls._MAX_SIZE: + if block._distrust_size and block._invalid_overflow_size != -1: + # The original size of this block was (1) wrong and (2) + # the real size doesn't allow us to save the file + # according to the spec (too big for 24 bit uint). Instead + # simply write back the original wrong size.. at least + # we don't make the file more "broken" as it is. + size = block._invalid_overflow_size + else: raise error("block is too long to write") - length = struct.pack(">I", len(datum))[-3:] - data.append(byte + length + datum) - return "".join(data) + assert not size > cls._MAX_SIZE + length = struct.pack(">I", size)[-3:] + data.append(code) + data += length + data += datum + return data - @staticmethod - def group_padding(blocks): - """Consolidate FLAC padding metadata blocks. + @classmethod + def _writeblocks(cls, blocks, available, cont_size, padding_func): + """Render metadata block as a byte string.""" - The overall size of the rendered blocks does not change, so - this adds several bytes of padding for each merged block.""" - paddings = filter(lambda x: isinstance(x, Padding), blocks) - map(blocks.remove, paddings) - # total padding size is the sum of padding sizes plus 4 bytes - # per removed header. - size = sum([padding.length for padding in paddings]) - padding = Padding() - padding.length = size + 4 * (len(paddings) - 1) - blocks.append(padding) + # write everything except padding + data = bytearray() + for block in blocks: + if isinstance(block, Padding): + continue + data += cls._writeblock(block) + blockssize = len(data) + + # take the padding overhead into account. we always add one + # to make things simple. + padding_block = Padding() + blockssize += len(cls._writeblock(padding_block)) + + # finally add a padding block + info = PaddingInfo(available - blockssize, cont_size) + padding_block.length = min(info._get_padding(padding_func), + cls._MAX_SIZE) + data += cls._writeblock(padding_block, is_last=True) + + return data -class StreamInfo(MetadataBlock): +class StreamInfo(MetadataBlock, mutagen.StreamInfo): """FLAC stream information. This contains information about the audio data in the FLAC file. @@ -188,13 +222,13 @@ class StreamInfo(MetadataBlock): bps_tail = bps_total >> 36 bps_head = (sample_channels_bps & 1) << 4 self.bits_per_sample = int(bps_head + bps_tail + 1) - self.total_samples = bps_total & 0xFFFFFFFFFL + self.total_samples = bps_total & 0xFFFFFFFFF self.length = self.total_samples / float(self.sample_rate) self.md5_signature = to_int_be(data.read(16)) def write(self): - f = StringIO() + f = cBytesIO() f.write(struct.pack(">I", self.min_blocksize)[-2:]) f.write(struct.pack(">I", self.max_blocksize)[-2:]) f.write(struct.pack(">I", self.min_framesize)[-3:]) @@ -206,22 +240,22 @@ class StreamInfo(MetadataBlock): byte = (self.sample_rate & 0xF) << 4 byte += ((self.channels - 1) & 7) << 1 byte += ((self.bits_per_sample - 1) >> 4) & 1 - f.write(chr(byte)) + f.write(chr_(byte)) # 4 bits of bps, 4 of sample count byte = ((self.bits_per_sample - 1) & 0xF) << 4 byte += (self.total_samples >> 32) & 0xF - f.write(chr(byte)) + f.write(chr_(byte)) # last 32 of sample count - f.write(struct.pack(">I", self.total_samples & 0xFFFFFFFFL)) + f.write(struct.pack(">I", self.total_samples & 0xFFFFFFFF)) # MD5 signature sig = self.md5_signature f.write(struct.pack( - ">4I", (sig >> 96) & 0xFFFFFFFFL, (sig >> 64) & 0xFFFFFFFFL, - (sig >> 32) & 0xFFFFFFFFL, sig & 0xFFFFFFFFL)) + ">4I", (sig >> 96) & 0xFFFFFFFF, (sig >> 64) & 0xFFFFFFFF, + (sig >> 32) & 0xFFFFFFFF, sig & 0xFFFFFFFF)) return f.getvalue() def pprint(self): - return "FLAC, %.2f seconds, %d Hz" % (self.length, self.sample_rate) + return u"FLAC, %.2f seconds, %d Hz" % (self.length, self.sample_rate) class SeekPoint(tuple): @@ -284,7 +318,7 @@ class SeekTable(MetadataBlock): sp = data.tryread(self.__SEEKPOINT_SIZE) def write(self): - f = StringIO() + f = cBytesIO() for seekpoint in self.seekpoints: packed = struct.pack( self.__SEEKPOINT_FORMAT, @@ -378,10 +412,10 @@ class CueSheetTrack(object): __hash__ = object.__hash__ def __repr__(self): - return ("<%s number=%r, offset=%d, isrc=%r, type=%r, " - "pre_emphasis=%r, indexes=%r)>") % ( - type(self).__name__, self.track_number, self.start_offset, - self.isrc, self.type, self.pre_emphasis, self.indexes) + return (("<%s number=%r, offset=%d, isrc=%r, type=%r, " + "pre_emphasis=%r, indexes=%r)>") % + (type(self).__name__, self.track_number, self.start_offset, + self.isrc, self.type, self.pre_emphasis, self.indexes)) class CueSheet(MetadataBlock): @@ -409,7 +443,7 @@ class CueSheet(MetadataBlock): code = 5 - media_catalog_number = '' + media_catalog_number = b'' lead_in_samples = 88200 compact_disc = True @@ -432,20 +466,20 @@ class CueSheet(MetadataBlock): header = data.read(self.__CUESHEET_SIZE) media_catalog_number, lead_in_samples, flags, num_tracks = \ struct.unpack(self.__CUESHEET_FORMAT, header) - self.media_catalog_number = media_catalog_number.rstrip('\0') + self.media_catalog_number = media_catalog_number.rstrip(b'\0') self.lead_in_samples = lead_in_samples self.compact_disc = bool(flags & 0x80) self.tracks = [] - for i in range(num_tracks): + for i in xrange(num_tracks): track = data.read(self.__CUESHEET_TRACK_SIZE) start_offset, track_number, isrc_padded, flags, num_indexes = \ struct.unpack(self.__CUESHEET_TRACK_FORMAT, track) - isrc = isrc_padded.rstrip('\0') + isrc = isrc_padded.rstrip(b'\0') type_ = (flags & 0x80) >> 7 pre_emphasis = bool(flags & 0x40) val = CueSheetTrack( track_number, start_offset, isrc, type_, pre_emphasis) - for j in range(num_indexes): + for j in xrange(num_indexes): index = data.read(self.__CUESHEET_TRACKINDEX_SIZE) index_offset, index_number = struct.unpack( self.__CUESHEET_TRACKINDEX_FORMAT, index) @@ -454,7 +488,7 @@ class CueSheet(MetadataBlock): self.tracks.append(val) def write(self): - f = StringIO() + f = cBytesIO() flags = 0 if self.compact_disc: flags |= 0x80 @@ -480,10 +514,10 @@ class CueSheet(MetadataBlock): return f.getvalue() def __repr__(self): - return ("<%s media_catalog_number=%r, lead_in=%r, compact_disc=%r, " - "tracks=%r>") % ( - type(self).__name__, self.media_catalog_number, - self.lead_in_samples, self.compact_disc, self.tracks) + return (("<%s media_catalog_number=%r, lead_in=%r, compact_disc=%r, " + "tracks=%r>") % + (type(self).__name__, self.media_catalog_number, + self.lead_in_samples, self.compact_disc, self.tracks)) class Picture(MetadataBlock): @@ -500,6 +534,21 @@ class Picture(MetadataBlock): * colors -- number of colors for indexed palettes (like GIF), 0 for non-indexed * data -- picture data + + To create a picture from file (in order to add to a FLAC file), + instantiate this object without passing anything to the constructor and + then set the properties manually:: + + p = Picture() + + with open("Folder.jpg", "rb") as f: + pic.data = f.read() + + pic.type = id3.PictureType.COVER_FRONT + pic.mime = u"image/jpeg" + pic.width = 500 + pic.height = 500 + pic.depth = 16 # color depth """ code = 6 @@ -513,7 +562,7 @@ class Picture(MetadataBlock): self.height = 0 self.depth = 0 self.colors = 0 - self.data = '' + self.data = b'' super(Picture, self).__init__(data) def __eq__(self, other): @@ -541,7 +590,7 @@ class Picture(MetadataBlock): self.data = data.read(length) def write(self): - f = StringIO() + f = cBytesIO() mime = self.mime.encode('UTF-8') f.write(struct.pack('>2I', self.type, len(mime))) f.write(mime) @@ -563,13 +612,12 @@ class Padding(MetadataBlock): To avoid rewriting the entire FLAC file when editing comments, metadata is often padded. Padding should occur at the end, and no - more than one padding block should be in any FLAC file. Mutagen - handles this with MetadataBlock.group_padding. + more than one padding block should be in any FLAC file. """ code = 1 - def __init__(self, data=""): + def __init__(self, data=b""): super(Padding, self).__init__(data) def load(self, data): @@ -577,7 +625,7 @@ class Padding(MetadataBlock): def write(self): try: - return "\x00" * self.length + return b"\x00" * self.length # On some 64 bit platforms this won't generate a MemoryError # or OverflowError since you might have enough RAM, but it # still generates a ValueError. On other 64 bit platforms, @@ -596,28 +644,32 @@ class Padding(MetadataBlock): return "<%s (%d bytes)>" % (type(self).__name__, self.length) -class FLAC(FileType): +class FLAC(mutagen.FileType): """A FLAC audio file. Attributes: - * info -- stream information (length, bitrate, sample rate) - * tags -- metadata tags, if any * cuesheet -- CueSheet object, if any * seektable -- SeekTable object, if any * pictures -- list of embedded pictures """ - _mimes = ["audio/x-flac", "application/x-flac"] + _mimes = ["audio/flac", "audio/x-flac", "application/x-flac"] + + info = None + """A `StreamInfo`""" + + tags = None + """A `VCommentDict`""" METADATA_BLOCKS = [StreamInfo, Padding, None, SeekTable, VCFLACDict, CueSheet, Picture] """Known metadata block types, indexed by ID.""" @staticmethod - def score(filename, fileobj, header): - return (header.startswith("fLaC") + - filename.lower().endswith(".flac") * 3) + def score(filename, fileobj, header_data): + return (header_data.startswith(b"fLaC") + + endswith(filename.lower(), ".flac") * 3) def __read_metadata_block(self, fileobj): byte = ord(fileobj.read(1)) @@ -637,10 +689,14 @@ class FLAC(FileType): # so we have to too. Instead of parsing the size # given, parse an actual Vorbis comment, leaving # fileobj in the right position. - # http://code.google.com/p/mutagen/issues/detail?id=52 + # https://github.com/quodlibet/mutagen/issues/52 # ..same for the Picture block: - # http://code.google.com/p/mutagen/issues/detail?id=106 + # https://github.com/quodlibet/mutagen/issues/106 + start = fileobj.tell() block = block_type(fileobj) + real_size = fileobj.tell() - start + if real_size > MetadataBlock._MAX_SIZE: + block._invalid_overflow_size = size else: data = fileobj.read(size) block = block_type(data) @@ -681,12 +737,12 @@ class FLAC(FileType): """ if filename is None: filename = self.filename - for s in list(self.metadata_blocks): - if isinstance(s, VCFLACDict): - self.metadata_blocks.remove(s) - self.tags = None - self.save() - break + + if self.tags is not None: + self.metadata_blocks.remove(self.tags) + self.save(padding=lambda x: 0) + self.metadata_blocks.append(self.tags) + self.tags.clear() vc = property(lambda s: s.tags, doc="Alias for tags; don't use this.") @@ -721,15 +777,17 @@ class FLAC(FileType): def clear_pictures(self): """Delete all pictures from the file.""" - self.metadata_blocks = filter(lambda b: b.code != Picture.code, - self.metadata_blocks) + + blocks = [b for b in self.metadata_blocks if b.code != Picture.code] + self.metadata_blocks = blocks @property def pictures(self): """List of embedded pictures""" - return filter(lambda b: b.code == Picture.code, self.metadata_blocks) - def save(self, filename=None, deleteid3=False): + return [b for b in self.metadata_blocks if b.code == Picture.code] + + def save(self, filename=None, deleteid3=False, padding=None): """Save metadata blocks to a file. If no filename is given, the one most recently loaded is used. @@ -737,46 +795,28 @@ class FLAC(FileType): if filename is None: filename = self.filename - f = open(filename, 'rb+') - - try: - # Ensure we've got padding at the end, and only at the end. - # If adding makes it too large, we'll scale it down later. - self.metadata_blocks.append(Padding('\x00' * 1020)) - MetadataBlock.group_padding(self.metadata_blocks) + with open(filename, 'rb+') as f: header = self.__check_header(f) + audio_offset = self.__find_audio_offset(f) # "fLaC" and maybe ID3 - available = self.__find_audio_offset(f) - header - data = MetadataBlock.writeblocks(self.metadata_blocks) + available = audio_offset - header # Delete ID3v2 if deleteid3 and header > 4: available += header - 4 header = 4 - if len(data) > available: - # If we have too much data, see if we can reduce padding. - padding = self.metadata_blocks[-1] - newlength = padding.length - (len(data) - available) - if newlength > 0: - padding.length = newlength - data = MetadataBlock.writeblocks(self.metadata_blocks) - assert len(data) == available - - elif len(data) < available: - # If we have too little data, increase padding. - self.metadata_blocks[-1].length += (available - len(data)) - data = MetadataBlock.writeblocks(self.metadata_blocks) - assert len(data) == available - - if len(data) != available: - # We couldn't reduce the padding enough. - diff = (len(data) - available) - insert_bytes(f, diff, header) + content_size = get_size(f) - audio_offset + assert content_size >= 0 + data = MetadataBlock._writeblocks( + self.metadata_blocks, available, content_size, padding) + data_size = len(data) + resize_bytes(f, available, data_size, header) f.seek(header - 4) - f.write("fLaC" + data) + f.write(b"fLaC") + f.write(data) # Delete ID3v1 if deleteid3: @@ -785,11 +825,9 @@ class FLAC(FileType): except IOError: pass else: - if f.read(3) == "TAG": + if f.read(3) == b"TAG": f.seek(-128, 2) f.truncate() - finally: - f.close() def __find_audio_offset(self, fileobj): byte = 0x00 @@ -810,14 +848,19 @@ class FLAC(FileType): return fileobj.tell() def __check_header(self, fileobj): + """Returns the offset of the flac block start + (skipping id3 tags if found). The passed fileobj will be advanced to + that offset as well. + """ + size = 4 header = fileobj.read(4) - if header != "fLaC": + if header != b"fLaC": size = None - if header[:3] == "ID3": + if header[:3] == b"ID3": size = 14 + BitPaddedInt(fileobj.read(6)[2:]) fileobj.seek(size - 4) - if fileobj.read(4) != "fLaC": + if fileobj.read(4) != b"fLaC": size = None if size is None: raise FLACNoHeaderError( diff --git a/libs/mutagen/id3.py b/libs/mutagen/id3/__init__.py similarity index 54% rename from libs/mutagen/id3.py rename to libs/mutagen/id3/__init__.py index 27d30e90..11bf54ed 100644 --- a/libs/mutagen/id3.py +++ b/libs/mutagen/id3/__init__.py @@ -1,4 +1,5 @@ -# id3 support for mutagen +# -*- coding: utf-8 -*- + # Copyright (C) 2005 Michael Urman # 2006 Lukas Lalinsky # 2013 Christoph Reiter @@ -32,15 +33,134 @@ interested in the :class:`ID3` class to start with. __all__ = ['ID3', 'ID3FileType', 'Frames', 'Open', 'delete'] import struct +import errno from struct import unpack, pack, error as StructError import mutagen -from mutagen._util import insert_bytes, delete_bytes, DictProxy +from mutagen._util import insert_bytes, delete_bytes, DictProxy, enum +from mutagen._tags import PaddingInfo +from .._compat import chr_, PY3 -from mutagen._id3util import * -from mutagen._id3frames import * -from mutagen._id3specs import * +from ._util import * +from ._frames import * +from ._specs import * + + +@enum +class ID3v1SaveOptions(object): + + REMOVE = 0 + """ID3v1 tags will be removed""" + + UPDATE = 1 + """ID3v1 tags will be updated but not added""" + + CREATE = 2 + """ID3v1 tags will be created and/or updated""" + + +def _fullread(fileobj, size): + """Read a certain number of bytes from the source file. + + Raises ValueError on invalid size input or EOFError/IOError. + """ + + if size < 0: + raise ValueError('Requested bytes (%s) less than zero' % size) + data = fileobj.read(size) + if len(data) != size: + raise EOFError("Not enough data to read") + return data + + +class ID3Header(object): + + _V24 = (2, 4, 0) + _V23 = (2, 3, 0) + _V22 = (2, 2, 0) + _V11 = (1, 1) + + f_unsynch = property(lambda s: bool(s._flags & 0x80)) + f_extended = property(lambda s: bool(s._flags & 0x40)) + f_experimental = property(lambda s: bool(s._flags & 0x20)) + f_footer = property(lambda s: bool(s._flags & 0x10)) + + def __init__(self, fileobj=None): + """Raises ID3NoHeaderError, ID3UnsupportedVersionError or error""" + + if fileobj is None: + # for testing + self._flags = 0 + return + + fn = getattr(fileobj, "name", "") + try: + data = _fullread(fileobj, 10) + except EOFError: + raise ID3NoHeaderError("%s: too small" % fn) + + id3, vmaj, vrev, flags, size = unpack('>3sBBB4s', data) + self._flags = flags + self.size = BitPaddedInt(size) + 10 + self.version = (2, vmaj, vrev) + + if id3 != b'ID3': + raise ID3NoHeaderError("%r doesn't start with an ID3 tag" % fn) + + if vmaj not in [2, 3, 4]: + raise ID3UnsupportedVersionError("%r ID3v2.%d not supported" + % (fn, vmaj)) + + if not BitPaddedInt.has_valid_padding(size): + raise error("Header size not synchsafe") + + if (self.version >= self._V24) and (flags & 0x0f): + raise error( + "%r has invalid flags %#02x" % (fn, flags)) + elif (self._V23 <= self.version < self._V24) and (flags & 0x1f): + raise error( + "%r has invalid flags %#02x" % (fn, flags)) + + if self.f_extended: + try: + extsize_data = _fullread(fileobj, 4) + except EOFError: + raise error("%s: too small" % fn) + + if PY3: + frame_id = extsize_data.decode("ascii", "replace") + else: + frame_id = extsize_data + + if frame_id in Frames: + # Some tagger sets the extended header flag but + # doesn't write an extended header; in this case, the + # ID3 data follows immediately. Since no extended + # header is going to be long enough to actually match + # a frame, and if it's *not* a frame we're going to be + # completely lost anyway, this seems to be the most + # correct check. + # https://github.com/quodlibet/quodlibet/issues/126 + self._flags ^= 0x40 + extsize = 0 + fileobj.seek(-4, 1) + elif self.version >= self._V24: + # "Where the 'Extended header size' is the size of the whole + # extended header, stored as a 32 bit synchsafe integer." + extsize = BitPaddedInt(extsize_data) - 4 + if not BitPaddedInt.has_valid_padding(extsize_data): + raise error( + "Extended header size not synchsafe") + else: + # "Where the 'Extended header size', currently 6 or 10 bytes, + # excludes itself." + extsize = unpack('>L', extsize_data)[0] + + try: + self._extdata = _fullread(fileobj, extsize) + except EOFError: + raise error("%s: too small" % fn) class ID3(DictProxy, mutagen.Metadata): @@ -53,39 +173,53 @@ class ID3(DictProxy, mutagen.Metadata): * size -- the total size of the ID3 tag, including the header """ + __module__ = "mutagen.id3" + PEDANTIC = True - version = (2, 4, 0) + """Deprecated. Doesn't have any effect""" filename = None - size = 0 - __flags = 0 - __readbytes = 0 - __crc = None - __unknown_version = None - - _V24 = (2, 4, 0) - _V23 = (2, 3, 0) - _V22 = (2, 2, 0) - _V11 = (1, 1) def __init__(self, *args, **kwargs): self.unknown_frames = [] + self.__unknown_version = None + self._header = None + self._version = (2, 4, 0) super(ID3, self).__init__(*args, **kwargs) - def __fullread(self, size): - try: - if size < 0: - raise ValueError('Requested bytes (%s) less than zero' % size) - if size > self.__filesize: - raise EOFError('Requested %#x of %#x (%s)' % ( - long(size), long(self.__filesize), self.filename)) - except AttributeError: - pass - data = self.__fileobj.read(size) - if len(data) != size: - raise EOFError - self.__readbytes += size - return data + @property + def version(self): + """ID3 tag version as a tuple (of the loaded file)""" + + if self._header is not None: + return self._header.version + return self._version + + @version.setter + def version(self, value): + self._version = value + + @property + def f_unsynch(self): + if self._header is not None: + return self._header.f_unsynch + return False + + @property + def f_extended(self): + if self._header is not None: + return self._header.f_extended + return False + + @property + def size(self): + if self._header is not None: + return self._header.size + return 0 + + def _pre_load_header(self, fileobj): + # XXX: for aiff to adjust the offset.. + pass def load(self, filename, known_frames=None, translate=True, v2_version=4): """Load tags from a filename. @@ -107,60 +241,53 @@ class ID3(DictProxy, mutagen.Metadata): mutagen.id3.ID3(filename, known_frames=my_frames) """ - if not v2_version in (3, 4): + if v2_version not in (3, 4): raise ValueError("Only 3 and 4 possible for v2_version") - from os.path import getsize - self.filename = filename + self.unknown_frames = [] self.__known_frames = known_frames - self.__fileobj = open(filename, 'rb') - self.__filesize = getsize(filename) - try: + self._header = None + self._padding = 0 # for testing + + with open(filename, 'rb') as fileobj: + self._pre_load_header(fileobj) + try: - self.__load_header() - except EOFError: - self.size = 0 - raise ID3NoHeaderError("%s: too small (%d bytes)" % ( - filename, self.__filesize)) - except (ID3NoHeaderError, ID3UnsupportedVersionError), err: - self.size = 0 - import sys - stack = sys.exc_info()[2] - try: - self.__fileobj.seek(-128, 2) - except EnvironmentError: - raise err, None, stack - else: - frames = ParseID3v1(self.__fileobj.read(128)) - if frames is not None: - self.version = self._V11 - map(self.add, frames.values()) - else: - raise err, None, stack + self._header = ID3Header(fileobj) + except (ID3NoHeaderError, ID3UnsupportedVersionError): + frames, offset = _find_id3v1(fileobj) + if frames is None: + raise + + self.version = ID3Header._V11 + for v in frames.values(): + self.add(v) else: frames = self.__known_frames if frames is None: - if self._V23 <= self.version: + if self.version >= ID3Header._V23: frames = Frames - elif self._V22 <= self.version: + elif self.version >= ID3Header._V22: frames = Frames_2_2 - data = self.__fullread(self.size - 10) + + try: + data = _fullread(fileobj, self.size - 10) + except (ValueError, EOFError, IOError) as e: + raise error(e) + for frame in self.__read_frames(data, frames=frames): if isinstance(frame, Frame): self.add(frame) else: self.unknown_frames.append(frame) - self.__unknown_version = self.version - finally: - self.__fileobj.close() - del self.__fileobj - del self.__filesize - if translate: - if v2_version == 3: - self.update_to_v23() - else: - self.update_to_v24() + self.__unknown_version = self.version[:2] + + if translate: + if v2_version == 3: + self.update_to_v23() + else: + self.update_to_v24() def getall(self, key): """Return all frames with a given name (the list may be empty). @@ -188,8 +315,9 @@ class ID3(DictProxy, mutagen.Metadata): del(self[key]) else: key = key + ":" - for k in filter(lambda s: s.startswith(key), self.keys()): - del(self[k]) + for k in list(self.keys()): + if k.startswith(key): + del(self[k]) def setall(self, key, values): """Delete frames of the given type and add frames in 'values'.""" @@ -209,8 +337,7 @@ class ID3(DictProxy, mutagen.Metadata): ``POPM=user@example.org=3 128/255`` """ - frames = list(map(Frame.pprint, self.values())) - frames.sort() + frames = sorted(Frame.pprint(s) for s in self.values()) return "\n".join(frames) def loaded_frame(self, tag): @@ -227,151 +354,88 @@ class ID3(DictProxy, mutagen.Metadata): """Add a frame to the tag.""" return self.loaded_frame(frame) - def __load_header(self): - fn = self.filename - data = self.__fullread(10) - id3, vmaj, vrev, flags, size = unpack('>3sBBB4s', data) - self.__flags = flags - self.size = BitPaddedInt(size) + 10 - self.version = (2, vmaj, vrev) - - if id3 != 'ID3': - raise ID3NoHeaderError("'%s' doesn't start with an ID3 tag" % fn) - if vmaj not in [2, 3, 4]: - raise ID3UnsupportedVersionError("'%s' ID3v2.%d not supported" - % (fn, vmaj)) - - if self.PEDANTIC: - if not BitPaddedInt.has_valid_padding(size): - raise ValueError("Header size not synchsafe") - - if self._V24 <= self.version and (flags & 0x0f): - raise ValueError("'%s' has invalid flags %#02x" % (fn, flags)) - elif self._V23 <= self.version < self._V24 and (flags & 0x1f): - raise ValueError("'%s' has invalid flags %#02x" % (fn, flags)) - - if self.f_extended: - extsize = self.__fullread(4) - if extsize in Frames: - # Some tagger sets the extended header flag but - # doesn't write an extended header; in this case, the - # ID3 data follows immediately. Since no extended - # header is going to be long enough to actually match - # a frame, and if it's *not* a frame we're going to be - # completely lost anyway, this seems to be the most - # correct check. - # http://code.google.com/p/quodlibet/issues/detail?id=126 - self.__flags ^= 0x40 - self.__extsize = 0 - self.__fileobj.seek(-4, 1) - self.__readbytes -= 4 - elif self.version >= self._V24: - # "Where the 'Extended header size' is the size of the whole - # extended header, stored as a 32 bit synchsafe integer." - self.__extsize = BitPaddedInt(extsize) - 4 - if self.PEDANTIC: - if not BitPaddedInt.has_valid_padding(extsize): - raise ValueError("Extended header size not synchsafe") - else: - # "Where the 'Extended header size', currently 6 or 10 bytes, - # excludes itself." - self.__extsize = unpack('>L', extsize)[0] - if self.__extsize: - self.__extdata = self.__fullread(self.__extsize) - else: - self.__extdata = "" - - def __determine_bpi(self, data, frames, EMPTY="\x00" * 10): - if self.version < self._V24: - return int - # have to special case whether to use bitpaddedints here - # spec says to use them, but iTunes has it wrong - - # count number of tags found as BitPaddedInt and how far past - o = 0 - asbpi = 0 - while o < len(data) - 10: - part = data[o:o + 10] - if part == EMPTY: - bpioff = -((len(data) - o) % 10) - break - name, size, flags = unpack('>4sLH', part) - size = BitPaddedInt(size) - o += 10 + size - if name in frames: - asbpi += 1 - else: - bpioff = o - len(data) - - # count number of tags found as int and how far past - o = 0 - asint = 0 - while o < len(data) - 10: - part = data[o:o + 10] - if part == EMPTY: - intoff = -((len(data) - o) % 10) - break - name, size, flags = unpack('>4sLH', part) - o += 10 + size - if name in frames: - asint += 1 - else: - intoff = o - len(data) - - # if more tags as int, or equal and bpi is past and int is not - if asint > asbpi or (asint == asbpi and (bpioff >= 1 and intoff <= 1)): - return int - return BitPaddedInt + def __setitem__(self, key, tag): + if not isinstance(tag, Frame): + raise TypeError("%r not a Frame instance" % tag) + super(ID3, self).__setitem__(key, tag) def __read_frames(self, data, frames): - if self.version < self._V24 and self.f_unsynch: + assert self.version >= ID3Header._V22 + + if self.version < ID3Header._V24 and self.f_unsynch: try: data = unsynch.decode(data) except ValueError: pass - if self._V23 <= self.version: - bpi = self.__determine_bpi(data, frames) + if self.version >= ID3Header._V23: + if self.version < ID3Header._V24: + bpi = int + else: + bpi = _determine_bpi(data, frames) + while data: header = data[:10] try: name, size, flags = unpack('>4sLH', header) except struct.error: return # not enough header - if name.strip('\x00') == '': + if name.strip(b'\x00') == b'': return + size = bpi(size) - framedata = data[10:10+size] - data = data[10+size:] + framedata = data[10:10 + size] + data = data[10 + size:] + self._padding = len(data) if size == 0: continue # drop empty frames + + if PY3: + try: + name = name.decode('ascii') + except UnicodeDecodeError: + continue + try: + # someone writes 2.3 frames with 2.2 names + if name[-1] == "\x00": + tag = Frames_2_2[name[:-1]] + name = tag.__base__.__name__ + tag = frames[name] except KeyError: if is_valid_frame_id(name): yield header + framedata else: try: - yield self.__load_framedata(tag, flags, framedata) + yield tag._fromData(self._header, flags, framedata) except NotImplementedError: yield header + framedata except ID3JunkFrameError: pass - - elif self._V22 <= self.version: + elif self.version >= ID3Header._V22: while data: header = data[0:6] try: name, size = unpack('>3s3s', header) except struct.error: return # not enough header - size, = struct.unpack('>L', '\x00'+size) - if name.strip('\x00') == '': + size, = struct.unpack('>L', b'\x00' + size) + if name.strip(b'\x00') == b'': return - framedata = data[6:6+size] - data = data[6+size:] + + framedata = data[6:6 + size] + data = data[6 + size:] + self._padding = len(data) if size == 0: continue # drop empty frames + + if PY3: + try: + name = name.decode('ascii') + except UnicodeDecodeError: + continue + try: tag = frames[name] except KeyError: @@ -379,146 +443,139 @@ class ID3(DictProxy, mutagen.Metadata): yield header + framedata else: try: - yield self.__load_framedata(tag, 0, framedata) - except NotImplementedError: + yield tag._fromData(self._header, 0, framedata) + except (ID3EncryptionUnsupportedError, + NotImplementedError): yield header + framedata except ID3JunkFrameError: pass - def __load_framedata(self, tag, flags, framedata): - return tag.fromData(self, flags, framedata) - - f_unsynch = property(lambda s: bool(s.__flags & 0x80)) - f_extended = property(lambda s: bool(s.__flags & 0x40)) - f_experimental = property(lambda s: bool(s.__flags & 0x20)) - f_footer = property(lambda s: bool(s.__flags & 0x10)) - - #f_crc = property(lambda s: bool(s.__extflags & 0x8000)) - - def save(self, filename=None, v1=1, v2_version=4, v23_sep='/'): - """Save changes to a file. - - If no filename is given, the one most recently loaded is used. - - Keyword arguments: - v1 -- if 0, ID3v1 tags will be removed - if 1, ID3v1 tags will be updated but not added - if 2, ID3v1 tags will be created and/or updated - v2 -- version of ID3v2 tags (3 or 4). - - By default Mutagen saves ID3v2.4 tags. If you want to save ID3v2.3 - tags, you must call method update_to_v23 before saving the file. - - v23_sep -- the separator used to join multiple text values - if v2_version == 3. Defaults to '/' but if it's None - will be the ID3v2v2.4 null separator. - - The lack of a way to update only an ID3v1 tag is intentional. - """ - + def _prepare_data(self, fileobj, start, available, v2_version, v23_sep, + pad_func): if v2_version == 3: - version = self._V23 + version = ID3Header._V23 elif v2_version == 4: - version = self._V24 + version = ID3Header._V24 else: raise ValueError("Only 3 or 4 allowed for v2_version") # Sort frames by 'importance' order = ["TIT2", "TPE1", "TRCK", "TALB", "TPOS", "TDRC", "TCON"] - order = dict(zip(order, range(len(order)))) + order = dict((b, a) for a, b in enumerate(order)) last = len(order) - frames = self.items() - frames.sort(lambda a, b: cmp(order.get(a[0][:4], last), - order.get(b[0][:4], last))) + frames = sorted(self.items(), + key=lambda a: (order.get(a[0][:4], last), a[0])) framedata = [self.__save_frame(frame, version=version, v23_sep=v23_sep) for (key, frame) in frames] # only write unknown frames if they were loaded from the version # we are saving with or upgraded to it - if self.__unknown_version == version: - framedata.extend([data for data in self.unknown_frames - if len(data) > 10]) + if self.__unknown_version == version[:2]: + framedata.extend(data for data in self.unknown_frames + if len(data) > 10) - if not framedata: - try: - self.delete(filename) - except EnvironmentError, err: - from errno import ENOENT - if err.errno != ENOENT: - raise - return + needed = sum(map(len, framedata)) + 10 - framedata = ''.join(framedata) - framesize = len(framedata) + fileobj.seek(0, 2) + trailing_size = fileobj.tell() - start + + info = PaddingInfo(available - needed, trailing_size) + new_padding = info._get_padding(pad_func) + if new_padding < 0: + raise error("invalid padding") + new_size = needed + new_padding + + new_framesize = BitPaddedInt.to_str(new_size - 10, width=4) + header = pack('>3sBBB4s', b'ID3', v2_version, 0, 0, new_framesize) + + data = bytearray(header) + for frame in framedata: + data += frame + assert new_size >= len(data) + data += (new_size - len(data)) * b'\x00' + assert new_size == len(data) + + return data + + def save(self, filename=None, v1=1, v2_version=4, v23_sep='/', + padding=None): + """Save changes to a file. + + Args: + filename: + Filename to save the tag to. If no filename is given, + the one most recently loaded is used. + v1 (ID3v1SaveOptions): + if 0, ID3v1 tags will be removed. + if 1, ID3v1 tags will be updated but not added. + if 2, ID3v1 tags will be created and/or updated + v2 (int): + version of ID3v2 tags (3 or 4). + v23_sep (str): + the separator used to join multiple text values + if v2_version == 3. Defaults to '/' but if it's None + will be the ID3v2v2.4 null separator. + padding (function): + A function taking a PaddingInfo which should + return the amount of padding to use. If None (default) + will default to something reasonable. + + By default Mutagen saves ID3v2.4 tags. If you want to save ID3v2.3 + tags, you must call method update_to_v23 before saving the file. + + The lack of a way to update only an ID3v1 tag is intentional. + + Can raise id3.error. + """ if filename is None: filename = self.filename + try: f = open(filename, 'rb+') - except IOError, err: + except IOError as err: from errno import ENOENT if err.errno != ENOENT: raise f = open(filename, 'ab') # create, then reopen f = open(filename, 'rb+') + try: - idata = f.read(10) try: - id3, vmaj, vrev, flags, insize = unpack('>3sBBB4s', idata) - except struct.error: - id3, insize = '', 0 - insize = BitPaddedInt(insize) - if id3 != 'ID3': - insize = -10 - - if insize >= framesize: - outsize = insize + header = ID3Header(f) + except ID3NoHeaderError: + old_size = 0 else: - outsize = (framesize + 1023) & ~0x3FF - framedata += '\x00' * (outsize - framesize) + old_size = header.size - framesize = BitPaddedInt.to_str(outsize, width=4) - flags = 0 - header = pack('>3sBBB4s', 'ID3', v2_version, 0, flags, framesize) - data = header + framedata + data = self._prepare_data( + f, 0, old_size, v2_version, v23_sep, padding) + new_size = len(data) - if (insize < outsize): - insert_bytes(f, outsize-insize, insize+10) + if (old_size < new_size): + insert_bytes(f, new_size - old_size, old_size) + elif (old_size > new_size): + delete_bytes(f, old_size - new_size, new_size) f.seek(0) f.write(data) - try: - f.seek(-128, 2) - except IOError, err: - # If the file is too small, that's OK - it just means - # we're certain it doesn't have a v1 tag. - from errno import EINVAL - if err.errno != EINVAL: - # If we failed to see for some other reason, bail out. - raise - # Since we're sure this isn't a v1 tag, don't read it. - f.seek(0, 2) - - data = f.read(128) - try: - idx = data.index("TAG") - except ValueError: - offset = 0 - has_v1 = False - else: - offset = idx - len(data) - has_v1 = True - - f.seek(offset, 2) - if v1 == 1 and has_v1 or v1 == 2: - f.write(MakeID3v1(self)) - else: - f.truncate() + self.__save_v1(f, v1) finally: f.close() + def __save_v1(self, f, v1): + tag, offset = _find_id3v1(f) + has_v1 = tag is not None + + f.seek(offset, 2) + if v1 == ID3v1SaveOptions.UPDATE and has_v1 or \ + v1 == ID3v1SaveOptions.CREATE: + f.write(MakeID3v1(self)) + else: + f.truncate() + def delete(self, filename=None, delete_v1=True, delete_v2=True): """Remove tags from a file. @@ -534,13 +591,14 @@ class ID3(DictProxy, mutagen.Metadata): delete(filename, delete_v1, delete_v2) self.clear() - def __save_frame(self, frame, name=None, version=_V24, v23_sep=None): + def __save_frame(self, frame, name=None, version=ID3Header._V24, + v23_sep=None): flags = 0 - if self.PEDANTIC and isinstance(frame, TextFrame): + if isinstance(frame, TextFrame): if len(str(frame)) == 0: - return '' + return b'' - if version == self._V23: + if version == ID3Header._V23: framev23 = frame._get_v23_frame(sep=v23_sep) framedata = framev23._writeData() else: @@ -551,19 +609,28 @@ class ID3(DictProxy, mutagen.Metadata): # Disabled as this causes iTunes and other programs # to fail to find these frames, which usually includes # e.g. APIC. - #framedata = BitPaddedInt.to_str(usize) + framedata.encode('zlib') - #flags |= Frame.FLAG24_COMPRESS | Frame.FLAG24_DATALEN + # framedata = BitPaddedInt.to_str(usize) + framedata.encode('zlib') + # flags |= Frame.FLAG24_COMPRESS | Frame.FLAG24_DATALEN pass - if version == self._V24: + if version == ID3Header._V24: bits = 7 - elif version == self._V23: + elif version == ID3Header._V23: bits = 8 else: raise ValueError datasize = BitPaddedInt.to_str(len(framedata), width=4, bits=bits) - header = pack('>4s4sH', name or type(frame).__name__, datasize, flags) + + if name is not None: + assert isinstance(name, bytes) + frame_name = name + else: + frame_name = type(frame).__name__ + if PY3: + frame_name = frame_name.encode("ascii") + + header = pack('>4s4sH', frame_name, datasize, flags) return header + framedata def __update_common(self): @@ -573,20 +640,14 @@ class ID3(DictProxy, mutagen.Metadata): # Get rid of "(xx)Foobr" format. self["TCON"].genres = self["TCON"].genres - if self.version < self._V23: - # ID3v2.2 PIC frames are slightly different. - pics = self.getall("APIC") - mimes = {"PNG": "image/png", "JPG": "image/jpeg"} - self.delall("APIC") - for pic in pics: + mimes = {"PNG": "image/png", "JPG": "image/jpeg"} + for pic in self.getall("APIC"): + if pic.mime in mimes: newpic = APIC( - encoding=pic.encoding, mime=mimes.get(pic.mime, pic.mime), + encoding=pic.encoding, mime=mimes[pic.mime], type=pic.type, desc=pic.desc, data=pic.data) self.add(newpic) - # ID3v2.2 LNK frames are just way too different to upgrade. - self.delall("LINK") - def update_to_v24(self): """Convert older tags into an ID3v2.4 tag. @@ -597,28 +658,37 @@ class ID3(DictProxy, mutagen.Metadata): self.__update_common() - if self.__unknown_version == (2, 3, 0): + if self.__unknown_version == (2, 3): # convert unknown 2.3 frames (flags/size) to 2.4 converted = [] for frame in self.unknown_frames: try: name, size, flags = unpack('>4sLH', frame[:10]) - frame = BinaryFrame.fromData(self, flags, frame[10:]) - except (struct.error, error): + except struct.error: continue + + try: + frame = BinaryFrame._fromData( + self._header, flags, frame[10:]) + except (error, NotImplementedError): + continue + converted.append(self.__save_frame(frame, name=name)) self.unknown_frames[:] = converted - self.__unknown_version = (2, 4, 0) + self.__unknown_version = (2, 4) # TDAT, TYER, and TIME have been turned into TDRC. try: - if str(self.get("TYER", "")).strip("\x00"): - date = str(self.pop("TYER")) - if str(self.get("TDAT", "")).strip("\x00"): - dat = str(self.pop("TDAT")) + date = text_type(self.get("TYER", "")) + if date.strip(u"\x00"): + self.pop("TYER") + dat = text_type(self.get("TDAT", "")) + if dat.strip("\x00"): + self.pop("TDAT") date = "%s-%s-%s" % (date, dat[2:], dat[:2]) - if str(self.get("TIME", "")).strip("\x00"): - time = str(self.pop("TIME")) + time = text_type(self.get("TIME", "")) + if time.strip("\x00"): + self.pop("TIME") date += "T%s:%s:00" % (time[:2], time[2:]) if "TDRC" not in self: self.add(TDRC(encoding=0, text=date)) @@ -720,45 +790,142 @@ def delete(filename, delete_v1=True, delete_v2=True): * delete_v2 -- delete any ID3v2 tag """ - f = open(filename, 'rb+') + with open(filename, 'rb+') as f: - if delete_v1: - try: - f.seek(-128, 2) - except IOError: - pass - else: - if f.read(3) == "TAG": - f.seek(-128, 2) + if delete_v1: + tag, offset = _find_id3v1(f) + if tag is not None: + f.seek(offset, 2) f.truncate() - # technically an insize=0 tag is invalid, but we delete it anyway - # (primarily because we used to write it) - if delete_v2: - f.seek(0, 0) - idata = f.read(10) - try: - id3, vmaj, vrev, flags, insize = unpack('>3sBBB4s', idata) - except struct.error: - id3, insize = '', -1 - insize = BitPaddedInt(insize) - if id3 == 'ID3' and insize >= 0: - delete_bytes(f, insize + 10, 0) + # technically an insize=0 tag is invalid, but we delete it anyway + # (primarily because we used to write it) + if delete_v2: + f.seek(0, 0) + idata = f.read(10) + try: + id3, vmaj, vrev, flags, insize = unpack('>3sBBB4s', idata) + except struct.error: + id3, insize = b'', -1 + insize = BitPaddedInt(insize) + if id3 == b'ID3' and insize >= 0: + delete_bytes(f, insize + 10, 0) # support open(filename) as interface Open = ID3 -# ID3v1.1 support. -def ParseID3v1(string): - """Parse an ID3v1 tag, returning a list of ID3v2.4 frames.""" +def _determine_bpi(data, frames, EMPTY=b"\x00" * 10): + """Takes id3v2.4 frame data and determines if ints or bitpaddedints + should be used for parsing. Needed because iTunes used to write + normal ints for frame sizes. + """ + + # count number of tags found as BitPaddedInt and how far past + o = 0 + asbpi = 0 + while o < len(data) - 10: + part = data[o:o + 10] + if part == EMPTY: + bpioff = -((len(data) - o) % 10) + break + name, size, flags = unpack('>4sLH', part) + size = BitPaddedInt(size) + o += 10 + size + if PY3: + try: + name = name.decode("ascii") + except UnicodeDecodeError: + continue + if name in frames: + asbpi += 1 + else: + bpioff = o - len(data) + + # count number of tags found as int and how far past + o = 0 + asint = 0 + while o < len(data) - 10: + part = data[o:o + 10] + if part == EMPTY: + intoff = -((len(data) - o) % 10) + break + name, size, flags = unpack('>4sLH', part) + o += 10 + size + if PY3: + try: + name = name.decode("ascii") + except UnicodeDecodeError: + continue + if name in frames: + asint += 1 + else: + intoff = o - len(data) + + # if more tags as int, or equal and bpi is past and int is not + if asint > asbpi or (asint == asbpi and (bpioff >= 1 and intoff <= 1)): + return int + return BitPaddedInt + + +def _find_id3v1(fileobj): + """Returns a tuple of (id3tag, offset_to_end) or (None, 0) + + offset mainly because we used to write too short tags in some cases and + we need the offset to delete them. + """ + + # id3v1 is always at the end (after apev2) + + extra_read = b"APETAGEX".index(b"TAG") try: - string = string[string.index("TAG"):] + fileobj.seek(-128 - extra_read, 2) + except IOError as e: + if e.errno == errno.EINVAL: + # If the file is too small, might be ok since we wrote too small + # tags at some point. let's see how the parsing goes.. + fileobj.seek(0, 0) + else: + raise + + data = fileobj.read(128 + extra_read) + try: + idx = data.index(b"TAG") + except ValueError: + return (None, 0) + else: + # FIXME: make use of the apev2 parser here + # if TAG is part of APETAGEX assume this is an APEv2 tag + try: + ape_idx = data.index(b"APETAGEX") + except ValueError: + pass + else: + if idx == ape_idx + extra_read: + return (None, 0) + + tag = ParseID3v1(data[idx:]) + if tag is None: + return (None, 0) + + offset = idx - len(data) + return (tag, offset) + + +# ID3v1.1 support. +def ParseID3v1(data): + """Parse an ID3v1 tag, returning a list of ID3v2.4 frames. + + Returns a {frame_name: frame} dict or None. + """ + + try: + data = data[data.index(b"TAG"):] except ValueError: return None - if 128 < len(string) or len(string) < 124: + if 128 < len(data) or len(data) < 124: return None # Issue #69 - Previous versions of Mutagen, when encountering @@ -766,19 +933,19 @@ def ParseID3v1(string): # wrote only the characters available - e.g. "1" or "" - into the # year field. To parse those, reduce the size of the year field. # Amazingly, "0s" works as a struct format string. - unpack_fmt = "3s30s30s30s%ds29sBB" % (len(string) - 124) + unpack_fmt = "3s30s30s30s%ds29sBB" % (len(data) - 124) try: tag, title, artist, album, year, comment, track, genre = unpack( - unpack_fmt, string) + unpack_fmt, data) except StructError: return None - if tag != "TAG": + if tag != b"TAG": return None - def fix(string): - return string.split("\x00")[0].strip().decode('latin1') + def fix(data): + return data.split(b"\x00")[0].strip().decode('latin1') title, artist, album, year, comment = map( fix, [title, artist, album, year, comment]) @@ -797,7 +964,7 @@ def ParseID3v1(string): encoding=0, lang="eng", desc="ID3v1 Comment", text=comment) # Don't read a track number if it looks like the comment was # padded with spaces instead of nulls (thanks, WinAmp). - if track and (track != 32 or string[-3] == '\x00'): + if track and ((track != 32) or (data[-3] == b'\x00'[0])): frames["TRCK"] = TRCK(encoding=0, text=str(track)) if genre != 255: frames["TCON"] = TCON(encoding=0, text=str(genre)) @@ -814,22 +981,22 @@ def MakeID3v1(id3): if v2id in id3: text = id3[v2id].text[0].encode('latin1', 'replace')[:30] else: - text = "" - v1[name] = text + ("\x00" * (30 - len(text))) + text = b"" + v1[name] = text + (b"\x00" * (30 - len(text))) if "COMM" in id3: cmnt = id3["COMM"].text[0].encode('latin1', 'replace')[:28] else: - cmnt = "" - v1["comment"] = cmnt + ("\x00" * (29 - len(cmnt))) + cmnt = b"" + v1["comment"] = cmnt + (b"\x00" * (29 - len(cmnt))) if "TRCK" in id3: try: - v1["track"] = chr(+id3["TRCK"]) + v1["track"] = chr_(+id3["TRCK"]) except ValueError: - v1["track"] = "\x00" + v1["track"] = b"\x00" else: - v1["track"] = "\x00" + v1["track"] = b"\x00" if "TCON" in id3: try: @@ -838,20 +1005,28 @@ def MakeID3v1(id3): pass else: if genre in TCON.GENRES: - v1["genre"] = chr(TCON.GENRES.index(genre)) + v1["genre"] = chr_(TCON.GENRES.index(genre)) if "genre" not in v1: - v1["genre"] = "\xff" + v1["genre"] = b"\xff" if "TDRC" in id3: - year = str(id3["TDRC"]) + year = text_type(id3["TDRC"]).encode('ascii') elif "TYER" in id3: - year = str(id3["TYER"]) + year = text_type(id3["TYER"]).encode('ascii') else: - year = "" - v1["year"] = (year + "\x00\x00\x00\x00")[:4] + year = b"" + v1["year"] = (year + b"\x00\x00\x00\x00")[:4] - return ("TAG%(title)s%(artist)s%(album)s%(year)s%(comment)s" - "%(track)s%(genre)s") % v1 + return ( + b"TAG" + + v1["title"] + + v1["artist"] + + v1["album"] + + v1["year"] + + v1["comment"] + + v1["track"] + + v1["genre"] + ) class ID3FileType(mutagen.FileType): @@ -859,7 +1034,7 @@ class ID3FileType(mutagen.FileType): ID3 = ID3 - class _Info(object): + class _Info(mutagen.StreamInfo): length = 0 def __init__(self, fileobj, offset): @@ -870,8 +1045,8 @@ class ID3FileType(mutagen.FileType): return "Unknown format with ID3 tag" @staticmethod - def score(filename, fileobj, header): - return header.startswith("ID3") + def score(filename, fileobj, header_data): + return header_data.startswith(b"ID3") def add_tags(self, ID3=None): """Add an empty ID3 tag to the file. @@ -903,8 +1078,9 @@ class ID3FileType(mutagen.FileType): self.filename = filename try: self.tags = ID3(filename, **kwargs) - except error: + except ID3NoHeaderError: self.tags = None + if self.tags is not None: try: offset = self.tags.size @@ -912,8 +1088,6 @@ class ID3FileType(mutagen.FileType): offset = None else: offset = None - try: - fileobj = open(filename, "rb") + + with open(filename, "rb") as fileobj: self.info = self._Info(fileobj, offset) - finally: - fileobj.close() diff --git a/libs/mutagen/_id3frames.py b/libs/mutagen/id3/_frames.py similarity index 82% rename from libs/mutagen/_id3frames.py rename to libs/mutagen/id3/_frames.py index c6130f6b..33ecf5cd 100644 --- a/libs/mutagen/_id3frames.py +++ b/libs/mutagen/id3/_frames.py @@ -1,28 +1,35 @@ +# -*- coding: utf-8 -*- + # Copyright (C) 2005 Michael Urman # # This program is free software; you can redistribute it and/or modify # it under the terms of version 2 of the GNU General Public License as # published by the Free Software Foundation. -from zlib import error as zlibError -from warnings import warn +import zlib from struct import unpack -from mutagen._id3util import ( - ID3Warning, ID3JunkFrameError, ID3BadCompressedData, - ID3EncryptionUnsupportedError, ID3BadUnsynchData, unsynch) -from mutagen._id3specs import ( +from ._util import ID3JunkFrameError, ID3EncryptionUnsupportedError, unsynch +from ._specs import ( BinaryDataSpec, StringSpec, Latin1TextSpec, EncodedTextSpec, ByteSpec, EncodingSpec, ASPIIndexSpec, SizedIntegerSpec, IntegerSpec, VolumeAdjustmentsSpec, VolumePeakSpec, VolumeAdjustmentSpec, ChannelSpec, MultiSpec, SynchronizedTextSpec, KeyEventSpec, TimeStampSpec, - EncodedNumericPartTextSpec, EncodedNumericTextSpec) + EncodedNumericPartTextSpec, EncodedNumericTextSpec, SpecError, + PictureTypeSpec) +from .._compat import text_type, string_types, swap_to_string, iteritems, izip def is_valid_frame_id(frame_id): return frame_id.isalnum() and frame_id.isupper() +def _bytes2key(b): + assert isinstance(b, bytes) + + return b.decode("latin1") + + class Frame(object): """Fundamental unit of ID3 data. @@ -52,24 +59,28 @@ class Frame(object): if len(args) == 1 and len(kwargs) == 0 and \ isinstance(args[0], type(self)): other = args[0] - for checker in self._framespec: - try: - val = checker.validate(self, getattr(other, checker.name)) - except ValueError as e: - e.message = "%s: %s" % (checker.name, e.message) - raise - setattr(self, checker.name, val) + # ask the sub class to fill in our data + other._to_other(self) else: - for checker, val in zip(self._framespec, args): - setattr(self, checker.name, checker.validate(self, val)) + for checker, val in izip(self._framespec, args): + setattr(self, checker.name, val) for checker in self._framespec[len(args):]: - try: - validated = checker.validate( - self, kwargs.get(checker.name, None)) - except ValueError as e: - e.message = "%s: %s" % (checker.name, e.message) - raise - setattr(self, checker.name, validated) + setattr(self, checker.name, kwargs.get(checker.name)) + + def __setattr__(self, name, value): + for checker in self._framespec: + if checker.name == name: + self.__dict__[name] = checker.validate(self, value) + return + super(Frame, self).__setattr__(name, value) + + def _to_other(self, other): + # this impl covers subclasses with the same framespec + if other._framespec is not self._framespec: + raise ValueError + + for checker in other._framespec: + setattr(other, checker.name, getattr(self, checker.name)) def _get_v23_frame(self, **kwargs): """Returns a frame copy which is suitable for writing into a v2.3 tag. @@ -104,30 +115,31 @@ class Frame(object): """ kw = [] for attr in self._framespec: - kw.append('%s=%r' % (attr.name, getattr(self, attr.name))) + # so repr works during __init__ + if hasattr(self, attr.name): + kw.append('%s=%r' % (attr.name, getattr(self, attr.name))) return '%s(%s)' % (type(self).__name__, ', '.join(kw)) def _readData(self, data): - odata = data + """Raises ID3JunkFrameError; Returns leftover data""" + for reader in self._framespec: if len(data): try: value, data = reader.read(self, data) - except UnicodeDecodeError: - raise ID3JunkFrameError + except SpecError as e: + raise ID3JunkFrameError(e) else: - raise ID3JunkFrameError + raise ID3JunkFrameError("no data left") setattr(self, reader.name, value) - if data.strip('\x00'): - warn('Leftover data: %s: %r (from %r)' % ( - type(self).__name__, data, odata), - ID3Warning) + + return data def _writeData(self): data = [] for writer in self._framespec: data.append(writer.write(self, getattr(self, writer.name))) - return ''.join(data) + return b''.join(data) def pprint(self): """Return a human-readable representation of the frame.""" @@ -137,10 +149,17 @@ class Frame(object): return "[unrepresentable data]" @classmethod - def fromData(cls, id3, tflags, data): - """Construct this ID3 frame from raw string data.""" + def _fromData(cls, id3, tflags, data): + """Construct this ID3 frame from raw string data. - if id3._V24 <= id3.version: + Raises: + + ID3JunkFrameError in case parsing failed + NotImplementedError in case parsing isn't implemented + ID3EncryptionUnsupportedError in case the frame is encrypted. + """ + + if id3.version >= id3._V24: if tflags & (Frame.FLAG24_COMPRESS | Frame.FLAG24_DATALEN): # The data length int is syncsafe in 2.4 (but not 2.3). # However, we don't actually need the data length int, @@ -151,25 +170,28 @@ class Frame(object): if tflags & Frame.FLAG24_UNSYNCH or id3.f_unsynch: try: data = unsynch.decode(data) - except ValueError, err: - if id3.PEDANTIC: - raise ID3BadUnsynchData('%s: %r' % (err, data)) + except ValueError: + # Some things write synch-unsafe data with either the frame + # or global unsynch flag set. Try to load them as is. + # https://github.com/quodlibet/mutagen/issues/210 + # https://github.com/quodlibet/mutagen/issues/223 + pass if tflags & Frame.FLAG24_ENCRYPT: raise ID3EncryptionUnsupportedError if tflags & Frame.FLAG24_COMPRESS: try: - data = data.decode('zlib') - except zlibError, err: + data = zlib.decompress(data) + except zlib.error as err: # the initial mutagen that went out with QL 0.12 did not # write the 4 bytes of uncompressed size. Compensate. data = datalen_bytes + data try: - data = data.decode('zlib') - except zlibError, err: - if id3.PEDANTIC: - raise ID3BadCompressedData('%s: %r' % (err, data)) + data = zlib.decompress(data) + except zlib.error as err: + raise ID3JunkFrameError( + 'zlib: %s: %r' % (err, data)) - elif id3._V23 <= id3.version: + elif id3.version >= id3._V23: if tflags & Frame.FLAG23_COMPRESS: usize, = unpack('>L', data[:4]) data = data[4:] @@ -177,14 +199,11 @@ class Frame(object): raise ID3EncryptionUnsupportedError if tflags & Frame.FLAG23_COMPRESS: try: - data = data.decode('zlib') - except zlibError, err: - if id3.PEDANTIC: - raise ID3BadCompressedData('%s: %r' % (err, data)) + data = zlib.decompress(data) + except zlib.error as err: + raise ID3JunkFrameError('zlib: %s: %r' % (err, data)) frame = cls() - frame._rawdata = data - frame._flags = tflags frame._readData(data) return frame @@ -205,30 +224,53 @@ class FrameOpt(Frame): super(FrameOpt, self).__init__(*args, **kwargs) for spec in self._optionalspec: if spec.name in kwargs: - validated = spec.validate(self, kwargs[spec.name]) - setattr(self, spec.name, validated) + setattr(self, spec.name, kwargs[spec.name]) else: break + def __setattr__(self, name, value): + for checker in self._optionalspec: + if checker.name == name: + self.__dict__[name] = checker.validate(self, value) + return + super(FrameOpt, self).__setattr__(name, value) + + def _to_other(self, other): + super(FrameOpt, self)._to_other(other) + + # this impl covers subclasses with the same optionalspec + if other._optionalspec is not self._optionalspec: + raise ValueError + + for checker in other._optionalspec: + if hasattr(self, checker.name): + setattr(other, checker.name, getattr(self, checker.name)) + def _readData(self, data): - odata = data + """Raises ID3JunkFrameError; Returns leftover data""" + for reader in self._framespec: if len(data): - value, data = reader.read(self, data) + try: + value, data = reader.read(self, data) + except SpecError as e: + raise ID3JunkFrameError(e) else: - raise ID3JunkFrameError + raise ID3JunkFrameError("no data left") setattr(self, reader.name, value) + if data: for reader in self._optionalspec: if len(data): - value, data = reader.read(self, data) + try: + value, data = reader.read(self, data) + except SpecError as e: + raise ID3JunkFrameError(e) else: break setattr(self, reader.name, value) - if data.strip('\x00'): - warn('Leftover data: %s: %r (from %r)' % ( - type(self).__name__, data, odata), - ID3Warning) + + return data def _writeData(self): data = [] @@ -239,7 +281,7 @@ class FrameOpt(Frame): data.append(writer.write(self, getattr(self, writer.name))) except AttributeError: break - return ''.join(data) + return b''.join(data) def __repr__(self): kw = [] @@ -251,6 +293,7 @@ class FrameOpt(Frame): return '%s(%s)' % (type(self).__name__, ', '.join(kw)) +@swap_to_string class TextFrame(Frame): """Text strings. @@ -271,17 +314,17 @@ class TextFrame(Frame): MultiSpec('text', EncodedTextSpec('text'), sep=u'\u0000'), ] - def __str__(self): - return self.__unicode__().encode('utf-8') + def __bytes__(self): + return text_type(self).encode('utf-8') - def __unicode__(self): + def __str__(self): return u'\u0000'.join(self.text) def __eq__(self, other): - if isinstance(other, str): - return str(self) == other - elif isinstance(other, unicode): - return unicode(self) == other + if isinstance(other, bytes): + return bytes(self) == other + elif isinstance(other, text_type): + return text_type(self) == other return self.text == other __hash__ = Frame.__hash__ @@ -344,6 +387,7 @@ class NumericPartTextFrame(TextFrame): return int(self.text[0].split("/")[0]) +@swap_to_string class TimeStampTextFrame(TextFrame): """A list of time stamps. @@ -356,16 +400,17 @@ class TimeStampTextFrame(TextFrame): MultiSpec('text', TimeStampSpec('stamp'), sep=u','), ] - def __str__(self): - return self.__unicode__().encode('utf-8') + def __bytes__(self): + return text_type(self).encode('utf-8') - def __unicode__(self): - return ','.join([stamp.text for stamp in self.text]) + def __str__(self): + return u','.join([stamp.text for stamp in self.text]) def _pprint(self): - return " / ".join([stamp.text for stamp in self.text]) + return u" / ".join([stamp.text for stamp in self.text]) +@swap_to_string class UrlFrame(Frame): """A frame containing a URL string. @@ -380,10 +425,10 @@ class UrlFrame(Frame): _framespec = [Latin1TextSpec('url')] - def __str__(self): + def __bytes__(self): return self.url.encode('utf-8') - def __unicode__(self): + def __str__(self): return self.url def __eq__(self, other): @@ -446,7 +491,7 @@ class TCON(TextFrame): if genreid: for gid in genreid[1:-1].split(")("): if gid.isdigit() and int(gid) < len(self.GENRES): - gid = unicode(self.GENRES[int(gid)]) + gid = text_type(self.GENRES[int(gid)]) newgenres.append(gid) elif gid == "CR": newgenres.append(u"Cover") @@ -467,12 +512,12 @@ class TCON(TextFrame): return genres def __set_genres(self, genres): - if isinstance(genres, basestring): + if isinstance(genres, string_types): genres = [genres] - self.text = map(self.__decode, genres) + self.text = [self.__decode(g) for g in genres] def __decode(self, value): - if isinstance(value, str): + if isinstance(value, bytes): enc = EncodedTextSpec._encodings[self.encoding][0] return value.decode(enc) else: @@ -869,6 +914,7 @@ class SYTC(Frame): __hash__ = Frame.__hash__ +@swap_to_string class USLT(Frame): """Unsynchronised lyrics/text transcription. @@ -885,12 +931,12 @@ class USLT(Frame): @property def HashKey(self): - return '%s:%s:%r' % (self.FrameID, self.desc, self.lang) + return '%s:%s:%s' % (self.FrameID, self.desc, self.lang) - def __str__(self): + def __bytes__(self): return self.text.encode('utf-8') - def __unicode__(self): + def __str__(self): return self.text def __eq__(self, other): @@ -899,6 +945,7 @@ class USLT(Frame): __hash__ = Frame.__hash__ +@swap_to_string class SYLT(Frame): """Synchronised lyrics/text.""" @@ -913,7 +960,7 @@ class SYLT(Frame): @property def HashKey(self): - return '%s:%s:%r' % (self.FrameID, self.desc, self.lang) + return '%s:%s:%s' % (self.FrameID, self.desc, self.lang) def __eq__(self, other): return str(self) == other @@ -921,7 +968,10 @@ class SYLT(Frame): __hash__ = Frame.__hash__ def __str__(self): - return "".join([text for (text, time) in self.text]).encode('utf-8') + return u"".join(text for (text, time) in self.text) + + def __bytes__(self): + return text_type(self).encode("utf-8") class COMM(TextFrame): @@ -940,10 +990,10 @@ class COMM(TextFrame): @property def HashKey(self): - return '%s:%s:%r' % (self.FrameID, self.desc, self.lang) + return '%s:%s:%s' % (self.FrameID, self.desc, self.lang) def _pprint(self): - return "%s=%r=%s" % (self.desc, self.lang, " / ".join(self.text)) + return "%s=%s=%s" % (self.desc, self.lang, " / ".join(self.text)) class RVA2(Frame): @@ -1063,7 +1113,7 @@ class APIC(Frame): _framespec = [ EncodingSpec('encoding'), Latin1TextSpec('mime'), - ByteSpec('type'), + PictureTypeSpec('type'), EncodedTextSpec('desc'), BinaryDataSpec('data'), ] @@ -1077,6 +1127,12 @@ class APIC(Frame): def HashKey(self): return '%s:%s' % (self.FrameID, self.desc) + def _validate_from_22(self, other, checker): + if checker.name == "mime": + self.mime = other.mime.decode("ascii", "ignore") + else: + super(APIC, self)._validate_from_22(other, checker) + def _pprint(self): return "%s (%s, %d bytes)" % ( self.desc, self.mime, len(self.data)) @@ -1102,7 +1158,7 @@ class PCNT(Frame): return self.count def _pprint(self): - return unicode(self.count) + return text_type(self.count) class POPM(FrameOpt): @@ -1202,6 +1258,7 @@ class RBUF(FrameOpt): return self.size +@swap_to_string class AENC(FrameOpt): """Audio encryption. @@ -1227,10 +1284,10 @@ class AENC(FrameOpt): def HashKey(self): return '%s:%s' % (self.FrameID, self.owner) - def __str__(self): + def __bytes__(self): return self.owner.encode('utf-8') - def __unicode__(self): + def __str__(self): return self.owner def __eq__(self, other): @@ -1259,8 +1316,8 @@ class LINK(FrameOpt): @property def HashKey(self): try: - return "%s:%s:%s:%r" % ( - self.FrameID, self.frameid, self.url, self.data) + return "%s:%s:%s:%s" % ( + self.FrameID, self.frameid, self.url, _bytes2key(self.data)) except AttributeError: return "%s:%s:%s" % (self.FrameID, self.frameid, self.url) @@ -1323,13 +1380,10 @@ class UFID(Frame): __hash__ = Frame.__hash__ def _pprint(self): - isascii = ord(max(self.data)) < 128 - if isascii: - return "%s=%s" % (self.owner, self.data) - else: - return "%s (%d bytes)" % (self.owner, len(self.data)) + return "%s=%r" % (self.owner, self.data) +@swap_to_string class USER(Frame): """Terms of use. @@ -1348,12 +1402,12 @@ class USER(Frame): @property def HashKey(self): - return '%s:%r' % (self.FrameID, self.lang) + return '%s:%s' % (self.FrameID, self.lang) - def __str__(self): + def __bytes__(self): return self.text.encode('utf-8') - def __unicode__(self): + def __str__(self): return self.text def __eq__(self, other): @@ -1365,6 +1419,7 @@ class USER(Frame): return "%r=%s" % (self.lang, self.text) +@swap_to_string class OWNE(Frame): """Ownership frame.""" @@ -1375,10 +1430,10 @@ class OWNE(Frame): EncodedTextSpec('seller'), ] - def __str__(self): + def __bytes__(self): return self.seller.encode('utf-8') - def __unicode__(self): + def __str__(self): return self.seller def __eq__(self, other): @@ -1407,7 +1462,7 @@ class COMR(FrameOpt): @property def HashKey(self): - return '%s:%s' % (self.FrameID, self._writeData()) + return '%s:%s' % (self.FrameID, _bytes2key(self._writeData())) def __eq__(self, other): return self._writeData() == other._writeData() @@ -1415,6 +1470,7 @@ class COMR(FrameOpt): __hash__ = FrameOpt.__hash__ +@swap_to_string class ENCR(Frame): """Encryption method registration. @@ -1432,7 +1488,7 @@ class ENCR(Frame): def HashKey(self): return "%s:%s" % (self.FrameID, self.owner) - def __str__(self): + def __bytes__(self): return self.data def __eq__(self, other): @@ -1441,6 +1497,7 @@ class ENCR(Frame): __hash__ = Frame.__hash__ +@swap_to_string class GRID(FrameOpt): """Group identification registration.""" @@ -1458,10 +1515,10 @@ class GRID(FrameOpt): def __pos__(self): return self.group - def __str__(self): + def __bytes__(self): return self.owner.encode('utf-8') - def __unicode__(self): + def __str__(self): return self.owner def __eq__(self, other): @@ -1470,6 +1527,7 @@ class GRID(FrameOpt): __hash__ = FrameOpt.__hash__ +@swap_to_string class PRIV(Frame): """Private frame.""" @@ -1481,24 +1539,21 @@ class PRIV(Frame): @property def HashKey(self): return '%s:%s:%s' % ( - self.FrameID, self.owner, self.data.decode('latin1')) + self.FrameID, self.owner, _bytes2key(self.data)) - def __str__(self): + def __bytes__(self): return self.data def __eq__(self, other): return self.data == other def _pprint(self): - isascii = ord(max(self.data)) < 128 - if isascii: - return "%s=%s" % (self.owner, self.data) - else: - return "%s (%d bytes)" % (self.owner, len(self.data)) + return "%s=%r" % (self.owner, self.data) __hash__ = Frame.__hash__ +@swap_to_string class SIGN(Frame): """Signature frame.""" @@ -1509,9 +1564,9 @@ class SIGN(Frame): @property def HashKey(self): - return '%s:%c:%s' % (self.FrameID, self.group, self.sig) + return '%s:%s:%s' % (self.FrameID, self.group, _bytes2key(self.sig)) - def __str__(self): + def __bytes__(self): return self.sig def __eq__(self, other): @@ -1557,15 +1612,6 @@ class ASPI(Frame): __hash__ = Frame.__hash__ -Frames = dict([(k, v) for (k, v) in globals().items() - if len(k) == 4 and isinstance(v, type) and - issubclass(v, Frame)]) -"""All supported ID3v2 frames, keyed by frame name.""" - -del(k) -del(v) - - # ID3v2.2 frames class UFI(UFID): "Unique File Identifier" @@ -1779,8 +1825,8 @@ class COM(COMM): "Comment" -#class RVA(RVAD) -#class EQU(EQUA) +# class RVA(RVAD) +# class EQU(EQUA) class REV(RVRB): @@ -1793,9 +1839,24 @@ class PIC(APIC): The 'mime' attribute of an ID3v2.2 attached picture must be either 'PNG' or 'JPG'. """ - _framespec = [EncodingSpec('encoding'), StringSpec('mime', 3), - ByteSpec('type'), EncodedTextSpec('desc'), - BinaryDataSpec('data')] + + _framespec = [ + EncodingSpec('encoding'), + StringSpec('mime', 3), + PictureTypeSpec('type'), + EncodedTextSpec('desc'), + BinaryDataSpec('data') + ] + + def _to_other(self, other): + if not isinstance(other, APIC): + raise TypeError + + other.encoding = self.encoding + other.mime = self.mime + other.type = self.type + other.desc = self.desc + other.data = self.data class GEO(GEOB): @@ -1830,13 +1891,50 @@ class CRA(AENC): class LNK(LINK): """Linked information""" - _framespec = [StringSpec('frameid', 3), Latin1TextSpec('url')] + + _framespec = [ + StringSpec('frameid', 3), + Latin1TextSpec('url') + ] + _optionalspec = [BinaryDataSpec('data')] + def _to_other(self, other): + if not isinstance(other, LINK): + raise TypeError -Frames_2_2 = dict([(k, v) for (k, v) in globals().items() - if len(k) == 3 and isinstance(v, type) and - issubclass(v, Frame)]) + if isinstance(other, LNK): + other.frameid = self.frameid + else: + try: + other.frameid = Frames_2_2[self.frameid].__bases__[0].__name__ + except KeyError: + other.frameid = self.frameid.ljust(4) + other.url = self.url + if hasattr(self, "data"): + other.data = self.data -del k -del v + +Frames = {} +"""All supported ID3v2.3/4 frames, keyed by frame name.""" + + +Frames_2_2 = {} +"""All supported ID3v2.2 frames, keyed by frame name.""" + + +k, v = None, None +for k, v in iteritems(globals()): + if isinstance(v, type) and issubclass(v, Frame): + v.__module__ = "mutagen.id3" + + if len(k) == 3: + Frames_2_2[k] = v + elif len(k) == 4: + Frames[k] = v + +try: + del k + del v +except NameError: + pass diff --git a/libs/mutagen/_id3specs.py b/libs/mutagen/id3/_specs.py similarity index 55% rename from libs/mutagen/_id3specs.py rename to libs/mutagen/id3/_specs.py index 32ef3afe..22e4335b 100644 --- a/libs/mutagen/_id3specs.py +++ b/libs/mutagen/id3/_specs.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + # Copyright (C) 2005 Michael Urman # # This program is free software; you can redistribute it and/or modify @@ -6,12 +8,89 @@ import struct from struct import unpack, pack -from warnings import warn -from mutagen._id3util import ID3JunkFrameError, ID3Warning, BitPaddedInt +from .._compat import text_type, chr_, PY3, swap_to_string, string_types, \ + xrange +from .._util import total_ordering, decode_terminated, enum, izip +from ._util import BitPaddedInt + + +@enum +class PictureType(object): + """Enumeration of image types defined by the ID3 standard for the APIC + frame, but also reused in WMA/FLAC/VorbisComment. + """ + + OTHER = 0 + """Other""" + + FILE_ICON = 1 + """32x32 pixels 'file icon' (PNG only)""" + + OTHER_FILE_ICON = 2 + """Other file icon""" + + COVER_FRONT = 3 + """Cover (front)""" + + COVER_BACK = 4 + """Cover (back)""" + + LEAFLET_PAGE = 5 + """Leaflet page""" + + MEDIA = 6 + """Media (e.g. label side of CD)""" + + LEAD_ARTIST = 7 + """Lead artist/lead performer/soloist""" + + ARTIST = 8 + """Artist/performer""" + + CONDUCTOR = 9 + """Conductor""" + + BAND = 10 + """Band/Orchestra""" + + COMPOSER = 11 + """Composer""" + + LYRICIST = 12 + """Lyricist/text writer""" + + RECORDING_LOCATION = 13 + """Recording Location""" + + DURING_RECORDING = 14 + """During recording""" + + DURING_PERFORMANCE = 15 + """During performance""" + + SCREEN_CAPTURE = 16 + """Movie/video screen capture""" + + FISH = 17 + """A bright coloured fish""" + + ILLUSTRATION = 18 + """Illustration""" + + BAND_LOGOTYPE = 19 + """Band/artist logotype""" + + PUBLISHER_LOGOTYPE = 20 + """Publisher/Studio logotype""" + + +class SpecError(Exception): + pass class Spec(object): + def __init__(self, name): self.name = name @@ -25,23 +104,49 @@ class Spec(object): return value + def read(self, frame, data): + """Returns the (value, left_data) or raises SpecError""" + + raise NotImplementedError + + def write(self, frame, value): + raise NotImplementedError + + def validate(self, frame, value): + """Returns the validated data or raises ValueError/TypeError""" + + raise NotImplementedError + class ByteSpec(Spec): def read(self, frame, data): - return ord(data[0]), data[1:] + return bytearray(data)[0], data[1:] def write(self, frame, value): - return chr(value) + return chr_(value) def validate(self, frame, value): if value is not None: - chr(value) + chr_(value) + return value + + +class PictureTypeSpec(ByteSpec): + + def read(self, frame, data): + value, data = ByteSpec.read(self, frame, data) + return PictureType(value), data + + def validate(self, frame, value): + value = ByteSpec.validate(self, frame, value) + if value is not None: + return PictureType(value) return value class IntegerSpec(Spec): def read(self, frame, data): - return int(BitPaddedInt(data, bits=8)), '' + return int(BitPaddedInt(data, bits=8)), b'' def write(self, frame, value): return BitPaddedInt.to_str(value, bits=8, width=-1) @@ -64,98 +169,148 @@ class SizedIntegerSpec(Spec): return value +@enum +class Encoding(object): + """Text Encoding""" + + LATIN1 = 0 + """ISO-8859-1""" + + UTF16 = 1 + """UTF-16 with BOM""" + + UTF16BE = 2 + """UTF-16BE without BOM""" + + UTF8 = 3 + """UTF-8""" + + class EncodingSpec(ByteSpec): + def read(self, frame, data): enc, data = super(EncodingSpec, self).read(frame, data) - if enc < 16: - return enc, data - else: - return 0, chr(enc)+data + if enc not in (Encoding.LATIN1, Encoding.UTF16, Encoding.UTF16BE, + Encoding.UTF8): + raise SpecError('Invalid Encoding: %r' % enc) + return Encoding(enc), data def validate(self, frame, value): - if 0 <= value <= 3: - return value if value is None: return None - raise ValueError('Invalid Encoding: %r' % value) + if value not in (Encoding.LATIN1, Encoding.UTF16, Encoding.UTF16BE, + Encoding.UTF8): + raise ValueError('Invalid Encoding: %r' % value) + return Encoding(value) def _validate23(self, frame, value, **kwargs): # only 0, 1 are valid in v2.3, default to utf-16 - return min(1, value) + if value not in (Encoding.LATIN1, Encoding.UTF16): + value = Encoding.UTF16 + return value class StringSpec(Spec): + """A fixed size ASCII only payload.""" + def __init__(self, name, length): super(StringSpec, self).__init__(name) self.len = length def read(s, frame, data): - return data[:s.len], data[s.len:] + chunk = data[:s.len] + try: + ascii = chunk.decode("ascii") + except UnicodeDecodeError: + raise SpecError("not ascii") + else: + if PY3: + chunk = ascii + + return chunk, data[s.len:] def write(s, frame, value): if value is None: - return '\x00' * s.len + return b'\x00' * s.len else: - return (str(value) + '\x00' * s.len)[:s.len] + if PY3: + value = value.encode("ascii") + return (bytes(value) + b'\x00' * s.len)[:s.len] def validate(s, frame, value): if value is None: return None - if isinstance(value, basestring) and len(value) == s.len: + + if PY3: + if not isinstance(value, str): + raise TypeError("%s has to be str" % s.name) + value.encode("ascii") + else: + if not isinstance(value, bytes): + value = value.encode("ascii") + + if len(value) == s.len: return value + raise ValueError('Invalid StringSpec[%d] data: %r' % (s.len, value)) class BinaryDataSpec(Spec): def read(self, frame, data): - return data, '' + return data, b'' def write(self, frame, value): - return str(value) + if value is None: + return b"" + if isinstance(value, bytes): + return value + value = text_type(value).encode("ascii") + return value def validate(self, frame, value): - return str(value) + if value is None: + return None + + if isinstance(value, bytes): + return value + elif PY3: + raise TypeError("%s has to be bytes" % self.name) + + value = text_type(value).encode("ascii") + return value class EncodedTextSpec(Spec): - # Okay, seriously. This is private and defined explicitly and - # completely by the ID3 specification. You can't just add - # encodings here however you want. - _encodings = ( - ('latin1', '\x00'), - ('utf16', '\x00\x00'), - ('utf_16_be', '\x00\x00'), - ('utf8', '\x00') - ) + + _encodings = { + Encoding.LATIN1: ('latin1', b'\x00'), + Encoding.UTF16: ('utf16', b'\x00\x00'), + Encoding.UTF16BE: ('utf_16_be', b'\x00\x00'), + Encoding.UTF8: ('utf8', b'\x00'), + } def read(self, frame, data): enc, term = self._encodings[frame.encoding] - ret = '' - if len(term) == 1: - if term in data: - data, ret = data.split(term, 1) - else: - offset = -1 - try: - while True: - offset = data.index(term, offset+1) - if offset & 1: - continue - data, ret = data[0:offset], data[offset+2:] - break - except ValueError: - pass + try: + # allow missing termination + return decode_terminated(data, enc, strict=False) + except ValueError: + # utf-16 termination with missing BOM, or single NULL + if not data[:len(term)].strip(b"\x00"): + return u"", data[len(term):] - if len(data) < len(term): - return u'', ret - return data.decode(enc), ret + # utf-16 data with single NULL, see issue 169 + try: + return decode_terminated(data + b"\x00", enc) + except ValueError: + raise SpecError("Decoding error") def write(self, frame, value): enc, term = self._encodings[frame.encoding] return value.encode(enc) + term def validate(self, frame, value): - return unicode(value) + return text_type(value) class MultiSpec(Spec): @@ -184,28 +339,28 @@ class MultiSpec(Spec): data.append(self.specs[0].write(frame, v)) else: for record in value: - for v, s in zip(record, self.specs): + for v, s in izip(record, self.specs): data.append(s.write(frame, v)) - return ''.join(data) + return b''.join(data) def validate(self, frame, value): if value is None: return [] - if self.sep and isinstance(value, basestring): + if self.sep and isinstance(value, string_types): value = value.split(self.sep) if isinstance(value, list): if len(self.specs) == 1: return [self.specs[0].validate(frame, v) for v in value] else: return [ - [s.validate(frame, v) for (v, s) in zip(val, self.specs)] + [s.validate(frame, v) for (v, s) in izip(val, self.specs)] for val in value] raise ValueError('Invalid MultiSpec data: %r' % value) def _validate23(self, frame, value, **kwargs): if len(self.specs) != 1: return [[s._validate23(frame, v, **kwargs) - for (v, s) in zip(val, self.specs)] + for (v, s) in izip(val, self.specs)] for val in value] spec = self.specs[0] @@ -232,19 +387,21 @@ class EncodedNumericPartTextSpec(EncodedTextSpec): class Latin1TextSpec(EncodedTextSpec): def read(self, frame, data): - if '\x00' in data: - data, ret = data.split('\x00', 1) + if b'\x00' in data: + data, ret = data.split(b'\x00', 1) else: - ret = '' + ret = b'' return data.decode('latin1'), ret def write(self, data, value): - return value.encode('latin1') + '\x00' + return value.encode('latin1') + b'\x00' def validate(self, frame, value): - return unicode(value) + return text_type(value) +@swap_to_string +@total_ordering class ID3TimeStamp(object): """A time stamp in ID3v2 format. @@ -261,6 +418,11 @@ class ID3TimeStamp(object): def __init__(self, text): if isinstance(text, ID3TimeStamp): text = text.text + elif not isinstance(text, text_type): + if PY3: + raise TypeError("not a str") + text = text.decode("utf-8") + self.text = text __formats = ['%04d'] + ['%02d'] * 5 @@ -270,7 +432,9 @@ class ID3TimeStamp(object): parts = [self.year, self.month, self.day, self.hour, self.minute, self.second] pieces = [] - for i, part in enumerate(iter(iter(parts).next, None)): + for i, part in enumerate(parts): + if part is None: + break pieces.append(self.__formats[i] % part + self.__seps[i]) return u''.join(pieces)[:-1] @@ -289,11 +453,17 @@ class ID3TimeStamp(object): def __str__(self): return self.text + def __bytes__(self): + return self.text.encode("utf-8") + def __repr__(self): return repr(self.text) - def __cmp__(self, other): - return cmp(self.text, other.text) + def __eq__(self, other): + return self.text == other.text + + def __lt__(self, other): + return self.text < other.text __hash__ = object.__hash__ @@ -319,22 +489,26 @@ class TimeStampSpec(EncodedTextSpec): class ChannelSpec(ByteSpec): (OTHER, MASTER, FRONTRIGHT, FRONTLEFT, BACKRIGHT, BACKLEFT, FRONTCENTRE, - BACKCENTRE, SUBWOOFER) = range(9) + BACKCENTRE, SUBWOOFER) = xrange(9) class VolumeAdjustmentSpec(Spec): def read(self, frame, data): value, = unpack('>h', data[0:2]) - return value/512.0, data[2:] + return value / 512.0, data[2:] def write(self, frame, value): - return pack('>h', int(round(value * 512))) + number = int(round(value * 512)) + # pack only fails in 2.7, do it manually in 2.6 + if not -32768 <= number <= 32767: + raise SpecError("not in range") + return pack('>h', number) def validate(self, frame, value): if value is not None: try: self.write(frame, value) - except struct.error: + except SpecError: raise ValueError("out of range") return value @@ -343,27 +517,32 @@ class VolumePeakSpec(Spec): def read(self, frame, data): # http://bugs.xmms.org/attachment.cgi?id=113&action=view peak = 0 - bits = ord(data[0]) - bytes = min(4, (bits + 7) >> 3) + data_array = bytearray(data) + bits = data_array[0] + vol_bytes = min(4, (bits + 7) >> 3) # not enough frame data - if bytes + 1 > len(data): - raise ID3JunkFrameError - shift = ((8 - (bits & 7)) & 7) + (4 - bytes) * 8 - for i in range(1, bytes+1): + if vol_bytes + 1 > len(data): + raise SpecError("not enough frame data") + shift = ((8 - (bits & 7)) & 7) + (4 - vol_bytes) * 8 + for i in xrange(1, vol_bytes + 1): peak *= 256 - peak += ord(data[i]) + peak += data_array[i] peak *= 2 ** shift - return (float(peak) / (2**31-1)), data[1+bytes:] + return (float(peak) / (2 ** 31 - 1)), data[1 + vol_bytes:] def write(self, frame, value): + number = int(round(value * 32768)) + # pack only fails in 2.7, do it manually in 2.6 + if not 0 <= number <= 65535: + raise SpecError("not in range") # always write as 16 bits for sanity. - return "\x10" + pack('>H', int(round(value * 32768))) + return b"\x10" + pack('>H', number) def validate(self, frame, value): if value is not None: try: self.write(frame, value) - except struct.error: + except SpecError: raise ValueError("out of range") return value @@ -373,26 +552,26 @@ class SynchronizedTextSpec(EncodedTextSpec): texts = [] encoding, term = self._encodings[frame.encoding] while data: - l = len(term) try: - value_idx = data.index(term) + value, data = decode_terminated(data, encoding) except ValueError: - raise ID3JunkFrameError - value = data[:value_idx].decode(encoding) - if len(data) < value_idx + l + 4: - raise ID3JunkFrameError - time, = struct.unpack(">I", data[value_idx+l:value_idx+l+4]) + raise SpecError("decoding error") + + if len(data) < 4: + raise SpecError("not enough data") + time, = struct.unpack(">I", data[:4]) + texts.append((value, time)) - data = data[value_idx+l+4:] - return texts, "" + data = data[4:] + return texts, b"" def write(self, frame, value): data = [] encoding, term = self._encodings[frame.encoding] - for text, time in frame.text: + for text, time in value: text = text.encode(encoding) + term data.append(text + struct.pack(">I", time)) - return "".join(data) + return b"".join(data) def validate(self, frame, value): return value @@ -407,7 +586,7 @@ class KeyEventSpec(Spec): return events, data def write(self, frame, value): - return "".join([struct.pack(">bI", *event) for event in value]) + return b"".join(struct.pack(">bI", *event) for event in value) def validate(self, frame, value): return value @@ -423,14 +602,13 @@ class VolumeAdjustmentsSpec(Spec): freq /= 2.0 adj /= 512.0 adjustments[freq] = adj - adjustments = adjustments.items() - adjustments.sort() + adjustments = sorted(adjustments.items()) return adjustments, data def write(self, frame, value): value.sort() - return "".join([struct.pack(">Hh", int(freq * 2), int(adj * 512)) - for (freq, adj) in value]) + return b"".join(struct.pack(">Hh", int(freq * 2), int(adj * 512)) + for (freq, adj) in value) def validate(self, frame, value): return value @@ -445,12 +623,14 @@ class ASPIIndexSpec(Spec): format = "B" size = 1 else: - warn("invalid bit count in ASPI (%d)" % frame.b, ID3Warning) - return [], data + raise SpecError("invalid bit count in ASPI (%d)" % frame.b) indexes = data[:frame.N * size] data = data[frame.N * size:] - return list(struct.unpack(">" + format * frame.N, indexes)), data + try: + return list(struct.unpack(">" + format * frame.N, indexes)), data + except struct.error as e: + raise SpecError(e) def write(self, frame, values): if frame.b == 16: @@ -458,8 +638,11 @@ class ASPIIndexSpec(Spec): elif frame.b == 8: format = "B" else: - raise ValueError("frame.b must be 8 or 16") - return struct.pack(">" + format * frame.N, *values) + raise SpecError("frame.b must be 8 or 16") + try: + return struct.pack(">" + format * frame.N, *values) + except struct.error as e: + raise SpecError(e) def validate(self, frame, values): return values diff --git a/libs/mutagen/_id3util.py b/libs/mutagen/id3/_util.py similarity index 64% rename from libs/mutagen/_id3util.py rename to libs/mutagen/id3/_util.py index de82e36a..29f7241d 100644 --- a/libs/mutagen/_id3util.py +++ b/libs/mutagen/id3/_util.py @@ -1,12 +1,18 @@ +# -*- coding: utf-8 -*- + # Copyright (C) 2005 Michael Urman # 2013 Christoph Reiter +# 2014 Ben Ockmore # # This program is free software; you can redistribute it and/or modify # it under the terms of version 2 of the GNU General Public License as # published by the Free Software Foundation. +from .._compat import long_, integer_types, PY3 +from .._util import MutagenError -class error(Exception): + +class error(MutagenError): pass @@ -14,18 +20,6 @@ class ID3NoHeaderError(error, ValueError): pass -class ID3BadUnsynchData(error, ValueError): - pass - - -class ID3BadCompressedData(error, ValueError): - pass - - -class ID3TagError(error, ValueError): - pass - - class ID3UnsupportedVersionError(error, NotImplementedError): pass @@ -38,50 +32,29 @@ class ID3JunkFrameError(error, ValueError): pass -class ID3Warning(error, UserWarning): - pass - - class unsynch(object): @staticmethod def decode(value): - output = [] - safe = True - append = output.append - for val in value: - if safe: - append(val) - safe = val != '\xFF' - else: - if val >= '\xE0': - raise ValueError('invalid sync-safe string') - elif val != '\x00': - append(val) - safe = True - if not safe: + fragments = bytearray(value).split(b'\xff') + if len(fragments) > 1 and not fragments[-1]: raise ValueError('string ended unsafe') - return ''.join(output) + + for f in fragments[1:]: + if (not f) or (f[0] >= 0xE0): + raise ValueError('invalid sync-safe string') + + if f[0] == 0x00: + del f[0] + + return bytes(bytearray(b'\xff').join(fragments)) @staticmethod def encode(value): - output = [] - safe = True - append = output.append - for val in value: - if safe: - append(val) - if val == '\xFF': - safe = False - elif val == '\x00' or val >= '\xE0': - append('\x00') - append(val) - safe = val != '\xFF' - else: - append(val) - safe = True - if not safe: - append('\x00') - return ''.join(output) + fragments = bytearray(value).split(b'\xff') + for f in fragments[1:]: + if (not f) or (f[0] >= 0xE0) or (f[0] == 0x00): + f.insert(0, 0x00) + return bytes(bytearray(b'\xff').join(fragments)) class _BitPaddedMixin(object): @@ -111,11 +84,11 @@ class _BitPaddedMixin(object): while value: append(value & mask) value >>= bits - bytes_ = bytes_.ljust(minwidth, "\x00") + bytes_ = bytes_.ljust(minwidth, b"\x00") if bigendian: bytes_.reverse() - return str(bytes_) + return bytes(bytes_) @staticmethod def has_valid_padding(value, bits=7): @@ -125,14 +98,14 @@ class _BitPaddedMixin(object): mask = (((1 << (8 - bits)) - 1) << bits) - if isinstance(value, (int, long)): + if isinstance(value, integer_types): while value: if value & mask: return False value >>= 8 - elif isinstance(value, str): - for byte in value: - if ord(byte) & mask: + elif isinstance(value, bytes): + for byte in bytearray(value): + if byte & mask: return False else: raise TypeError @@ -148,29 +121,47 @@ class BitPaddedInt(int, _BitPaddedMixin): numeric_value = 0 shift = 0 - if isinstance(value, (int, long)): + if isinstance(value, integer_types): while value: numeric_value += (value & mask) << shift value >>= 8 shift += bits - elif isinstance(value, str): + elif isinstance(value, bytes): if bigendian: value = reversed(value) - for byte in value: - numeric_value += (ord(byte) & mask) << shift + for byte in bytearray(value): + numeric_value += (byte & mask) << shift shift += bits else: raise TypeError - if isinstance(numeric_value, long): - self = long.__new__(BitPaddedLong, numeric_value) - else: + if isinstance(numeric_value, int): self = int.__new__(BitPaddedInt, numeric_value) + else: + self = long_.__new__(BitPaddedLong, numeric_value) self.bits = bits self.bigendian = bigendian return self +if PY3: + BitPaddedLong = BitPaddedInt +else: + class BitPaddedLong(long_, _BitPaddedMixin): + pass -class BitPaddedLong(long, _BitPaddedMixin): - pass + +class ID3BadUnsynchData(error, ValueError): + """Deprecated""" + + +class ID3BadCompressedData(error, ValueError): + """Deprecated""" + + +class ID3TagError(error, ValueError): + """Deprecated""" + + +class ID3Warning(error, UserWarning): + """Deprecated""" diff --git a/libs/mutagen/m4a.py b/libs/mutagen/m4a.py index 64b89679..3ed148c5 100644 --- a/libs/mutagen/m4a.py +++ b/libs/mutagen/m4a.py @@ -1,36 +1,27 @@ +# -*- coding: utf-8 -*- # Copyright 2006 Joe Wreschnig # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -"""Read and write MPEG-4 audio files with iTunes metadata. - -This module will read MPEG-4 audio information and metadata, -as found in Apple's M4A (aka MP4, M4B, M4P) files. - -There is no official specification for this format. The source code -for TagLib, FAAD, and various MPEG specifications at -http://developer.apple.com/documentation/QuickTime/QTFF/, -http://www.geocities.com/xhelmboyx/quicktime/formats/mp4-layout.txt, -and http://wiki.multimedia.cx/index.php?title=Apple_QuickTime were all -consulted. - -This module does not support 64 bit atom sizes, and so will not -work on metadata over 4GB. +""" +since 1.9: mutagen.m4a is deprecated; use mutagen.mp4 instead. +since 1.31: mutagen.m4a will no longer work; any operation that could fail + will fail now. """ -import struct -import sys +import warnings -from cStringIO import StringIO +from mutagen import FileType, Tags, StreamInfo +from ._util import DictProxy, MutagenError -from mutagen import FileType, Metadata -from mutagen._constants import GENRES -from mutagen._util import cdata, insert_bytes, delete_bytes, DictProxy +warnings.warn( + "mutagen.m4a is deprecated; use mutagen.mp4 instead.", + DeprecationWarning) -class error(IOError): +class error(IOError, MutagenError): pass @@ -46,493 +37,65 @@ class M4AMetadataValueError(ValueError, M4AMetadataError): pass -import warnings -warnings.warn( - "mutagen.m4a is deprecated; use mutagen.mp4 instead.", DeprecationWarning) - - -# This is not an exhaustive list of container atoms, but just the -# ones this module needs to peek inside. -_CONTAINERS = ["moov", "udta", "trak", "mdia", "meta", "ilst", - "stbl", "minf", "stsd"] -_SKIP_SIZE = {"meta": 4} - __all__ = ['M4A', 'Open', 'delete', 'M4ACover'] -class M4ACover(str): - """A cover artwork. +class M4ACover(bytes): - Attributes: - imageformat -- format of the image (either FORMAT_JPEG or FORMAT_PNG) - """ FORMAT_JPEG = 0x0D FORMAT_PNG = 0x0E def __new__(cls, data, imageformat=None): - self = str.__new__(cls, data) + self = bytes.__new__(cls, data) if imageformat is None: imageformat = M4ACover.FORMAT_JPEG self.imageformat = imageformat - try: - self.format - except AttributeError: - self.format = imageformat return self -class Atom(object): - """An individual atom. - - Attributes: - children -- list child atoms (or None for non-container atoms) - length -- length of this atom, including length and name - name -- four byte name of the atom, as a str - offset -- location in the constructor-given fileobj of this atom - - This structure should only be used internally by Mutagen. - """ - - children = None - - def __init__(self, fileobj): - self.offset = fileobj.tell() - self.length, self.name = struct.unpack(">I4s", fileobj.read(8)) - if self.length == 1: - raise error("64 bit atom sizes are not supported") - elif self.length < 8: - return - - if self.name in _CONTAINERS: - self.children = [] - fileobj.seek(_SKIP_SIZE.get(self.name, 0), 1) - while fileobj.tell() < self.offset + self.length: - self.children.append(Atom(fileobj)) - else: - fileobj.seek(self.offset + self.length, 0) - - @staticmethod - def render(name, data): - """Render raw atom data.""" - # this raises OverflowError if Py_ssize_t can't handle the atom data - size = len(data) + 8 - if size <= 0xFFFFFFFF: - return struct.pack(">I4s", size, name) + data - else: - return struct.pack(">I4sQ", 1, name, size + 8) + data - - def __getitem__(self, remaining): - """Look up a child atom, potentially recursively. - - e.g. atom['udta', 'meta'] => - """ - if not remaining: - return self - elif self.children is None: - raise KeyError("%r is not a container" % self.name) - for child in self.children: - if child.name == remaining[0]: - return child[remaining[1:]] - else: - raise KeyError("%r not found" % remaining[0]) - - def __repr__(self): - klass = self.__class__.__name__ - if self.children is None: - return "<%s name=%r length=%r offset=%r>" % ( - klass, self.name, self.length, self.offset) - else: - children = "\n".join([" " + line for child in self.children - for line in repr(child).splitlines()]) - return "<%s name=%r length=%r offset=%r\n%s>" % ( - klass, self.name, self.length, self.offset, children) - - -class Atoms(object): - """Root atoms in a given file. - - Attributes: - atoms -- a list of top-level atoms as Atom objects - - This structure should only be used internally by Mutagen. - """ - def __init__(self, fileobj): - self.atoms = [] - fileobj.seek(0, 2) - end = fileobj.tell() - fileobj.seek(0) - while fileobj.tell() < end: - self.atoms.append(Atom(fileobj)) - - def path(self, *names): - """Look up and return the complete path of an atom. - - For example, atoms.path('moov', 'udta', 'meta') will return a - list of three atoms, corresponding to the moov, udta, and meta - atoms. - """ - path = [self] - for name in names: - path.append(path[-1][name, ]) - return path[1:] - - def __getitem__(self, names): - """Look up a child atom. - - 'names' may be a list of atoms (['moov', 'udta']) or a string - specifying the complete path ('moov.udta'). - """ - if isinstance(names, basestring): - names = names.split(".") - for child in self.atoms: - if child.name == names[0]: - return child[names[1:]] - else: - raise KeyError("%s not found" % names[0]) - - def __repr__(self): - return "\n".join([repr(child) for child in self.atoms]) - - -class M4ATags(DictProxy, Metadata): - """Dictionary containing Apple iTunes metadata list key/values. - - Keys are four byte identifiers, except for freeform ('----') - keys. Values are usually unicode strings, but some atoms have a - special structure: - cpil -- boolean - trkn, disk -- tuple of 16 bit ints (current, total) - tmpo -- 16 bit int - covr -- list of M4ACover objects (which are tagged strs) - gnre -- not supported. Use '\\xa9gen' instead. - - The freeform '----' frames use a key in the format '----:mean:name' - where 'mean' is usually 'com.apple.iTunes' and 'name' is a unique - identifier for this frame. The value is a str, but is probably - text that can be decoded as UTF-8. - - M4A tag data cannot exist outside of the structure of an M4A file, - so this class should not be manually instantiated. - - Unknown non-text tags are removed. - """ +class M4ATags(DictProxy, Tags): def load(self, atoms, fileobj): - try: - ilst = atoms["moov.udta.meta.ilst"] - except KeyError, key: - raise M4AMetadataError(key) - for atom in ilst.children: - fileobj.seek(atom.offset + 8) - data = fileobj.read(atom.length - 8) - parse = self.__atoms.get(atom.name, (M4ATags.__parse_text,))[0] - parse(self, atom, data) - - @staticmethod - def __key_sort(item1, item2): - (key1, v1) = item1 - (key2, v2) = item2 - # iTunes always writes the tags in order of "relevance", try - # to copy it as closely as possible. - order = ["\xa9nam", "\xa9ART", "\xa9wrt", "\xa9alb", - "\xa9gen", "gnre", "trkn", "disk", - "\xa9day", "cpil", "tmpo", "\xa9too", - "----", "covr", "\xa9lyr"] - order = dict(zip(order, range(len(order)))) - last = len(order) - # If there's no key-based way to distinguish, order by length. - # If there's still no way, go by string comparison on the - # values, so we at least have something determinstic. - return (cmp(order.get(key1[:4], last), order.get(key2[:4], last)) or - cmp(len(v1), len(v2)) or cmp(v1, v2)) + raise error("deprecated") def save(self, filename): - """Save the metadata to the given filename.""" - values = [] - items = self.items() - items.sort(self.__key_sort) - for key, value in items: - render = self.__atoms.get( - key[:4], (None, M4ATags.__render_text))[1] - values.append(render(self, key, value)) - data = Atom.render("ilst", "".join(values)) - - # Find the old atoms. - fileobj = open(filename, "rb+") - try: - atoms = Atoms(fileobj) - - moov = atoms["moov"] - - if moov != atoms.atoms[-1]: - # "Free" the old moov block. Something in the mdat - # block is not happy when its offset changes and it - # won't play back. So, rather than try to figure that - # out, just move the moov atom to the end of the file. - offset = self.__move_moov(fileobj, moov) - else: - offset = 0 - - try: - path = atoms.path("moov", "udta", "meta", "ilst") - except KeyError: - self.__save_new(fileobj, atoms, data, offset) - else: - self.__save_existing(fileobj, atoms, path, data, offset) - finally: - fileobj.close() - - def __move_moov(self, fileobj, moov): - fileobj.seek(moov.offset) - data = fileobj.read(moov.length) - fileobj.seek(moov.offset) - free = Atom.render("free", "\x00" * (moov.length - 8)) - fileobj.write(free) - fileobj.seek(0, 2) - # Figure out how far we have to shift all our successive - # seek calls, relative to what the atoms say. - old_end = fileobj.tell() - fileobj.write(data) - return old_end - moov.offset - - def __save_new(self, fileobj, atoms, ilst, offset): - hdlr = Atom.render("hdlr", "\x00" * 8 + "mdirappl" + "\x00" * 9) - meta = Atom.render("meta", "\x00\x00\x00\x00" + hdlr + ilst) - moov, udta = atoms.path("moov", "udta") - insert_bytes(fileobj, len(meta), udta.offset + offset + 8) - fileobj.seek(udta.offset + offset + 8) - fileobj.write(meta) - self.__update_parents(fileobj, [moov, udta], len(meta), offset) - - def __save_existing(self, fileobj, atoms, path, data, offset): - # Replace the old ilst atom. - ilst = path.pop() - delta = len(data) - ilst.length - fileobj.seek(ilst.offset + offset) - if delta > 0: - insert_bytes(fileobj, delta, ilst.offset + offset) - elif delta < 0: - delete_bytes(fileobj, -delta, ilst.offset + offset) - fileobj.seek(ilst.offset + offset) - fileobj.write(data) - self.__update_parents(fileobj, path, delta, offset) - - def __update_parents(self, fileobj, path, delta, offset): - # Update all parent atoms with the new size. - for atom in path: - fileobj.seek(atom.offset + offset) - size = cdata.uint_be(fileobj.read(4)) + delta - fileobj.seek(atom.offset + offset) - fileobj.write(cdata.to_uint_be(size)) - - def __render_data(self, key, flags, data): - data = struct.pack(">2I", flags, 0) + data - return Atom.render(key, Atom.render("data", data)) - - def __parse_freeform(self, atom, data): - try: - fileobj = StringIO(data) - mean_length = cdata.uint_be(fileobj.read(4)) - # skip over 8 bytes of atom name, flags - mean = fileobj.read(mean_length - 4)[8:] - name_length = cdata.uint_be(fileobj.read(4)) - name = fileobj.read(name_length - 4)[8:] - value_length = cdata.uint_be(fileobj.read(4)) - # Name, flags, and reserved bytes - value = fileobj.read(value_length - 4)[12:] - except struct.error: - # Some ---- atoms have no data atom, I have no clue why - # they actually end up in the file. - pass - else: - self["%s:%s:%s" % (atom.name, mean, name)] = value - - def __render_freeform(self, key, value): - dummy, mean, name = key.split(":", 2) - mean = struct.pack(">I4sI", len(mean) + 12, "mean", 0) + mean - name = struct.pack(">I4sI", len(name) + 12, "name", 0) + name - value = struct.pack(">I4s2I", len(value) + 16, "data", 0x1, 0) + value - final = mean + name + value - return Atom.render("----", final) - - def __parse_pair(self, atom, data): - self[atom.name] = struct.unpack(">2H", data[18:22]) - - def __render_pair(self, key, value): - track, total = value - if 0 <= track < 1 << 16 and 0 <= total < 1 << 16: - data = struct.pack(">4H", 0, track, total, 0) - return self.__render_data(key, 0, data) - else: - raise M4AMetadataValueError("invalid numeric pair %r" % (value,)) - - def __render_pair_no_trailing(self, key, value): - track, total = value - if 0 <= track < 1 << 16 and 0 <= total < 1 << 16: - data = struct.pack(">3H", 0, track, total) - return self.__render_data(key, 0, data) - else: - raise M4AMetadataValueError("invalid numeric pair %r" % (value,)) - - def __parse_genre(self, atom, data): - # Translate to a freeform genre. - genre = cdata.short_be(data[16:18]) - if "\xa9gen" not in self: - try: - self["\xa9gen"] = GENRES[genre - 1] - except IndexError: - pass - - def __parse_tempo(self, atom, data): - self[atom.name] = cdata.short_be(data[16:18]) - - def __render_tempo(self, key, value): - if 0 <= value < 1 << 16: - return self.__render_data(key, 0x15, cdata.to_ushort_be(value)) - else: - raise M4AMetadataValueError("invalid short integer %r" % value) - - def __parse_compilation(self, atom, data): - try: - self[atom.name] = bool(ord(data[16:17])) - except TypeError: - self[atom.name] = False - - def __render_compilation(self, key, value): - return self.__render_data(key, 0x15, chr(bool(value))) - - def __parse_cover(self, atom, data): - length, name, imageformat = struct.unpack(">I4sI", data[:12]) - if name != "data": - raise M4AMetadataError( - "unexpected atom %r inside 'covr'" % name) - if imageformat not in (M4ACover.FORMAT_JPEG, M4ACover.FORMAT_PNG): - imageformat = M4ACover.FORMAT_JPEG - self[atom.name] = M4ACover(data[16:length], imageformat) - - def __render_cover(self, key, value): - try: - imageformat = value.imageformat - except AttributeError: - imageformat = M4ACover.FORMAT_JPEG - data = Atom.render("data", struct.pack(">2I", imageformat, 0) + value) - return Atom.render(key, data) - - def __parse_text(self, atom, data): - flags = cdata.uint_be(data[8:12]) - if flags == 1: - self[atom.name] = data[16:].decode('utf-8', 'replace') - - def __render_text(self, key, value): - return self.__render_data(key, 0x1, value.encode('utf-8')) + raise error("deprecated") def delete(self, filename): - self.clear() - self.save(filename) - - __atoms = { - "----": (__parse_freeform, __render_freeform), - "trkn": (__parse_pair, __render_pair), - "disk": (__parse_pair, __render_pair_no_trailing), - "gnre": (__parse_genre, None), - "tmpo": (__parse_tempo, __render_tempo), - "cpil": (__parse_compilation, __render_compilation), - "covr": (__parse_cover, __render_cover), - } + raise error("deprecated") def pprint(self): - values = [] - for key, value in self.iteritems(): - key = key.decode('latin1') - try: - values.append("%s=%s" % (key, value)) - except UnicodeDecodeError: - values.append("%s=[%d bytes of data]" % (key, len(value))) - return "\n".join(values) + return u"" -class M4AInfo(object): - """MPEG-4 stream information. - - Attributes: - bitrate -- bitrate in bits per second, as an int - length -- file length in seconds, as a float - """ +class M4AInfo(StreamInfo): bitrate = 0 def __init__(self, atoms, fileobj): - hdlr = atoms["moov.trak.mdia.hdlr"] - fileobj.seek(hdlr.offset) - if "soun" not in fileobj.read(hdlr.length): - raise M4AStreamInfoError("track has no audio data") - - mdhd = atoms["moov.trak.mdia.mdhd"] - fileobj.seek(mdhd.offset) - data = fileobj.read(mdhd.length) - if ord(data[8]) == 0: - offset = 20 - fmt = ">2I" - else: - offset = 28 - fmt = ">IQ" - end = offset + struct.calcsize(fmt) - unit, length = struct.unpack(fmt, data[offset:end]) - self.length = float(length) / unit - - try: - atom = atoms["moov.trak.mdia.minf.stbl.stsd"] - fileobj.seek(atom.offset) - data = fileobj.read(atom.length) - self.bitrate = cdata.uint_be(data[-17:-13]) - except (ValueError, KeyError): - # Bitrate values are optional. - pass + raise error("deprecated") def pprint(self): - return "MPEG-4 audio, %.2f seconds, %d bps" % ( - self.length, self.bitrate) + return u"" class M4A(FileType): - """An MPEG-4 audio file, probably containing AAC. - - If more than one track is present in the file, the first is used. - Only audio ('soun') tracks will be read. - """ _mimes = ["audio/mp4", "audio/x-m4a", "audio/mpeg4", "audio/aac"] def load(self, filename): - self.filename = filename - fileobj = open(filename, "rb") - try: - atoms = Atoms(fileobj) - try: - self.info = M4AInfo(atoms, fileobj) - except StandardError, err: - raise M4AStreamInfoError, err, sys.exc_info()[2] - try: - self.tags = M4ATags(atoms, fileobj) - except M4AMetadataError: - self.tags = None - except StandardError, err: - raise M4AMetadataError, err, sys.exc_info()[2] - finally: - fileobj.close() + raise error("deprecated") def add_tags(self): self.tags = M4ATags() @staticmethod def score(filename, fileobj, header): - return ("ftyp" in header) + ("mp4" in header) + return 0 Open = M4A def delete(filename): - """Remove tags from a file.""" - - M4A(filename).delete() + raise error("deprecated") diff --git a/libs/mutagen/monkeysaudio.py b/libs/mutagen/monkeysaudio.py index 355749b9..0e29273f 100644 --- a/libs/mutagen/monkeysaudio.py +++ b/libs/mutagen/monkeysaudio.py @@ -1,6 +1,6 @@ -# A Monkey's Audio (APE) reader/tagger -# -# Copyright 2006 Lukas Lalinsky +# -*- coding: utf-8 -*- + +# Copyright (C) 2006 Lukas Lalinsky # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -18,6 +18,8 @@ __all__ = ["MonkeysAudio", "Open", "delete"] import struct +from ._compat import endswith +from mutagen import StreamInfo from mutagen.apev2 import APEv2File, error, delete from mutagen._util import cdata @@ -26,7 +28,7 @@ class MonkeysAudioHeaderError(error): pass -class MonkeysAudioInfo(object): +class MonkeysAudioInfo(StreamInfo): """Monkey's Audio stream information. Attributes: @@ -40,7 +42,7 @@ class MonkeysAudioInfo(object): def __init__(self, fileobj): header = fileobj.read(76) - if len(header) != 76 or not header.startswith("MAC "): + if len(header) != 76 or not header.startswith(b"MAC "): raise MonkeysAudioHeaderError("not a Monkey's Audio file") self.version = cdata.ushort_le(header[4:6]) if self.version >= 3980: @@ -62,13 +64,13 @@ class MonkeysAudioInfo(object): blocks_per_frame = 9216 self.version /= 1000.0 self.length = 0.0 - if self.sample_rate != 0 and total_frames > 0: + if (self.sample_rate != 0) and (total_frames > 0): total_blocks = ((total_frames - 1) * blocks_per_frame + final_frame_blocks) self.length = float(total_blocks) / self.sample_rate def pprint(self): - return "Monkey's Audio %.2f, %.2f seconds, %d Hz" % ( + return u"Monkey's Audio %.2f, %.2f seconds, %d Hz" % ( self.version, self.length, self.sample_rate) @@ -78,7 +80,7 @@ class MonkeysAudio(APEv2File): @staticmethod def score(filename, fileobj, header): - return header.startswith("MAC ") + filename.lower().endswith(".ape") + return header.startswith(b"MAC ") + endswith(filename.lower(), ".ape") Open = MonkeysAudio diff --git a/libs/mutagen/mp3.py b/libs/mutagen/mp3.py index 2426610b..afb600cf 100644 --- a/libs/mutagen/mp3.py +++ b/libs/mutagen/mp3.py @@ -1,5 +1,6 @@ -# MP3 stream header information support for Mutagen. -# Copyright 2006 Joe Wreschnig +# -*- coding: utf-8 -*- + +# Copyright (C) 2006 Joe Wreschnig # # This program is free software; you can redistribute it and/or modify # it under the terms of version 2 of the GNU General Public License as @@ -10,12 +11,16 @@ import os import struct +from ._compat import endswith, xrange +from ._mp3util import XingHeader, XingHeaderError, VBRIHeader, VBRIHeaderError +from mutagen import StreamInfo +from mutagen._util import MutagenError, enum from mutagen.id3 import ID3FileType, BitPaddedInt, delete __all__ = ["MP3", "Open", "delete", "MP3"] -class error(RuntimeError): +class error(RuntimeError, MutagenError): pass @@ -27,11 +32,50 @@ class InvalidMPEGHeader(error, IOError): pass +@enum +class BitrateMode(object): + + UNKNOWN = 0 + """Probably a CBR file, but not sure""" + + CBR = 1 + """Constant Bitrate""" + + VBR = 2 + """Variable Bitrate""" + + ABR = 3 + """Average Bitrate (a variant of VBR)""" + + +def _guess_xing_bitrate_mode(xing): + + if xing.lame_header: + lame = xing.lame_header + if lame.vbr_method in (1, 8): + return BitrateMode.CBR + elif lame.vbr_method in (2, 9): + return BitrateMode.ABR + elif lame.vbr_method in (3, 4, 5, 6): + return BitrateMode.VBR + # everything else undefined, continue guessing + + # info tags get only written by lame for cbr files + if xing.is_info: + return BitrateMode.CBR + + # older lame and non-lame with some variant of vbr + if xing.vbr_scale != -1 or xing.lame_version: + return BitrateMode.VBR + + return BitrateMode.UNKNOWN + + # Mode values. -STEREO, JOINTSTEREO, DUALCHANNEL, MONO = range(4) +STEREO, JOINTSTEREO, DUALCHANNEL, MONO = xrange(4) -class MPEGInfo(object): +class MPEGInfo(StreamInfo): """MPEG audio stream information Parse information about an MPEG audio file. This also reads the @@ -43,8 +87,18 @@ class MPEGInfo(object): Useful attributes: * length -- audio length, in seconds + * channels -- number of audio channels * bitrate -- audio bitrate, in bits per second * sketchy -- if true, the file may not be valid MPEG audio + * encoder_info -- a string containing encoder name and possibly version. + In case a lame tag is present this will start with + ``"LAME "``, if unknown it is empty, otherwise the + text format is undefined. + * bitrate_mode -- a :class:`BitrateMode` + + * track_gain -- replaygain track gain (89db) or None + * track_peak -- replaygain track peak or None + * album_gain -- replaygain album gain (89db) or None Useless attributes: @@ -58,19 +112,20 @@ class MPEGInfo(object): # Map (version, layer) tuples to bitrates. __BITRATE = { - (1, 1): range(0, 480, 32), + (1, 1): [0, 32, 64, 96, 128, 160, 192, 224, + 256, 288, 320, 352, 384, 416, 448], (1, 2): [0, 32, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 384], (1, 3): [0, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320], (2, 1): [0, 32, 48, 56, 64, 80, 96, 112, 128, 144, 160, 176, 192, 224, 256], - (2, 2): [0, 8, 16, 24, 32, 40, 48, 56, 64, + (2, 2): [0, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160], } __BITRATE[(2, 3)] = __BITRATE[(2, 2)] - for i in range(1, 4): + for i in xrange(1, 4): __BITRATE[(2.5, i)] = __BITRATE[(2, i)] # Map version to sample rates. @@ -81,6 +136,9 @@ class MPEGInfo(object): } sketchy = False + encoder_info = u"" + bitrate_mode = BitrateMode.UNKNOWN + track_gain = track_peak = album_gain = album_peak = None def __init__(self, fileobj, offset=None): """Parse MPEG stream information from a file-like object. @@ -104,9 +162,9 @@ class MPEGInfo(object): try: id3, insize = struct.unpack('>3sxxx4s', idata) except struct.error: - id3, insize = '', 0 + id3, insize = b'', 0 insize = BitPaddedInt(insize) - if id3 == 'ID3' and insize > 0: + if id3 == b'ID3' and insize > 0: offset = insize + 10 else: offset = 0 @@ -138,11 +196,11 @@ class MPEGInfo(object): # is assuming the offset didn't lie. data = fileobj.read(32768) - frame_1 = data.find("\xff") - while 0 <= frame_1 <= len(data) - 4: + frame_1 = data.find(b"\xff") + while 0 <= frame_1 <= (len(data) - 4): frame_data = struct.unpack(">I", data[frame_1:frame_1 + 4])[0] - if (frame_data >> 16) & 0xE0 != 0xE0: - frame_1 = data.find("\xff", frame_1 + 2) + if ((frame_data >> 16) & 0xE0) != 0xE0: + frame_1 = data.find(b"\xff", frame_1 + 2) else: version = (frame_data >> 19) & 0x3 layer = (frame_data >> 17) & 0x3 @@ -150,20 +208,22 @@ class MPEGInfo(object): bitrate = (frame_data >> 12) & 0xF sample_rate = (frame_data >> 10) & 0x3 padding = (frame_data >> 9) & 0x1 - #private = (frame_data >> 8) & 0x1 + # private = (frame_data >> 8) & 0x1 self.mode = (frame_data >> 6) & 0x3 - #mode_extension = (frame_data >> 4) & 0x3 - #copyright = (frame_data >> 3) & 0x1 - #original = (frame_data >> 2) & 0x1 - #emphasis = (frame_data >> 0) & 0x3 + # mode_extension = (frame_data >> 4) & 0x3 + # copyright = (frame_data >> 3) & 0x1 + # original = (frame_data >> 2) & 0x1 + # emphasis = (frame_data >> 0) & 0x3 if (version == 1 or layer == 0 or sample_rate == 0x3 or bitrate == 0 or bitrate == 0xF): - frame_1 = data.find("\xff", frame_1 + 2) + frame_1 = data.find(b"\xff", frame_1 + 2) else: break else: raise HeaderNotFoundError("can't sync to an MPEG frame") + self.channels = 1 if self.mode == MONO else 2 + # There is a serious problem here, which is that many flags # in an MPEG header are backwards. self.version = [2.5, None, 2, 1][version] @@ -176,17 +236,18 @@ class MPEGInfo(object): self.sample_rate = self.__RATES[self.version][sample_rate] if self.layer == 1: - frame_length = (12 * self.bitrate / self.sample_rate + padding) * 4 + frame_length = ( + (12 * self.bitrate // self.sample_rate) + padding) * 4 frame_size = 384 elif self.version >= 2 and self.layer == 3: - frame_length = 72 * self.bitrate / self.sample_rate + padding + frame_length = (72 * self.bitrate // self.sample_rate) + padding frame_size = 576 else: - frame_length = 144 * self.bitrate / self.sample_rate + padding + frame_length = (144 * self.bitrate // self.sample_rate) + padding frame_size = 1152 if check_second: - possible = frame_1 + frame_length + possible = int(frame_1 + frame_length) if possible > len(data) + 4: raise HeaderNotFoundError("can't sync to second MPEG frame") try: @@ -194,51 +255,70 @@ class MPEGInfo(object): ">H", data[possible:possible + 2])[0] except struct.error: raise HeaderNotFoundError("can't sync to second MPEG frame") - if frame_data & 0xFFE0 != 0xFFE0: + if (frame_data & 0xFFE0) != 0xFFE0: raise HeaderNotFoundError("can't sync to second MPEG frame") self.length = 8 * real_size / float(self.bitrate) # Try to find/parse the Xing header, which trumps the above length # and bitrate calculation. - fileobj.seek(offset, 0) - data = fileobj.read(32768) + + if self.layer != 3: + return + + # Xing + xing_offset = XingHeader.get_offset(self) + fileobj.seek(offset + frame_1 + xing_offset, 0) try: - xing = data[:-4].index("Xing") - except ValueError: - # Try to find/parse the VBRI header, which trumps the above length - # calculation. - try: - vbri = data[:-24].index("VBRI") - except ValueError: - pass - else: - # If a VBRI header was found, this is definitely MPEG audio. - self.sketchy = False - vbri_version = struct.unpack('>H', data[vbri + 4:vbri + 6])[0] - if vbri_version == 1: - frame_count = struct.unpack( - '>I', data[vbri + 14:vbri + 18])[0] - samples = float(frame_size * frame_count) - self.length = (samples / self.sample_rate) or self.length + xing = XingHeader(fileobj) + except XingHeaderError: + pass else: - # If a Xing header was found, this is definitely MPEG audio. + lame = xing.lame_header self.sketchy = False - flags = struct.unpack('>I', data[xing + 4:xing + 8])[0] - if flags & 0x1: - frame_count = struct.unpack('>I', data[xing + 8:xing + 12])[0] - samples = float(frame_size * frame_count) - self.length = (samples / self.sample_rate) or self.length - if flags & 0x2: - bytes = struct.unpack('>I', data[xing + 12:xing + 16])[0] - self.bitrate = int((bytes * 8) // self.length) + self.bitrate_mode = _guess_xing_bitrate_mode(xing) + if xing.frames != -1: + samples = frame_size * xing.frames + if lame is not None: + samples -= lame.encoder_delay_start + samples -= lame.encoder_padding_end + self.length = float(samples) / self.sample_rate + if xing.bytes != -1 and self.length: + self.bitrate = int((xing.bytes * 8) / self.length) + if xing.lame_version: + self.encoder_info = u"LAME %s" % xing.lame_version + if lame is not None: + self.track_gain = lame.track_gain_adjustment + self.track_peak = lame.track_peak + self.album_gain = lame.album_gain_adjustment + return + + # VBRI + vbri_offset = VBRIHeader.get_offset(self) + fileobj.seek(offset + frame_1 + vbri_offset, 0) + try: + vbri = VBRIHeader(fileobj) + except VBRIHeaderError: + pass + else: + self.bitrate_mode = BitrateMode.VBR + self.encoder_info = u"FhG" + self.sketchy = False + self.length = float(frame_size * vbri.frames) / self.sample_rate + if self.length: + self.bitrate = int((vbri.bytes * 8) / self.length) def pprint(self): - s = "MPEG %s layer %d, %d bps, %s Hz, %.2f seconds" % ( - self.version, self.layer, self.bitrate, self.sample_rate, - self.length) + info = str(self.bitrate_mode).split(".", 1)[-1] + if self.bitrate_mode == BitrateMode.UNKNOWN: + info = u"CBR?" + if self.encoder_info: + info += ", %s" % self.encoder_info + s = u"MPEG %s layer %d, %d bps (%s), %s Hz, %d chn, %.2f seconds" % ( + self.version, self.layer, self.bitrate, info, + self.sample_rate, self.channels, self.length) if self.sketchy: - s += " (sketchy)" + s += u" (sketchy)" return s @@ -250,15 +330,22 @@ class MP3(ID3FileType): """ _Info = MPEGInfo - _mimes = ["audio/mp3", "audio/x-mp3", "audio/mpeg", "audio/mpg", - "audio/x-mpeg"] + + _mimes = ["audio/mpeg", "audio/mpg", "audio/x-mpeg"] + + @property + def mime(self): + l = self.info.layer + return ["audio/mp%d" % l, "audio/x-mp%d" % l] + super(MP3, self).mime @staticmethod - def score(filename, fileobj, header): + def score(filename, fileobj, header_data): filename = filename.lower() - return (header.startswith("ID3") * 2 + filename.endswith(".mp3") + - filename.endswith(".mp2") + filename.endswith(".mpg") + - filename.endswith(".mpeg")) + + return (header_data.startswith(b"ID3") * 2 + + endswith(filename, b".mp3") + + endswith(filename, b".mp2") + endswith(filename, b".mpg") + + endswith(filename, b".mpeg")) Open = MP3 diff --git a/libs/mutagen/mp4.py b/libs/mutagen/mp4.py deleted file mode 100644 index 984a38c4..00000000 --- a/libs/mutagen/mp4.py +++ /dev/null @@ -1,822 +0,0 @@ -# Copyright 2006 Joe Wreschnig -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. - -"""Read and write MPEG-4 audio files with iTunes metadata. - -This module will read MPEG-4 audio information and metadata, -as found in Apple's MP4 (aka M4A, M4B, M4P) files. - -There is no official specification for this format. The source code -for TagLib, FAAD, and various MPEG specifications at - -* http://developer.apple.com/documentation/QuickTime/QTFF/ -* http://www.geocities.com/xhelmboyx/quicktime/formats/mp4-layout.txt -* http://standards.iso.org/ittf/PubliclyAvailableStandards/\ -c041828_ISO_IEC_14496-12_2005(E).zip -* http://wiki.multimedia.cx/index.php?title=Apple_QuickTime - -were all consulted. -""" - -import struct -import sys - -from mutagen import FileType, Metadata -from mutagen._constants import GENRES -from mutagen._util import cdata, insert_bytes, DictProxy, utf8 - - -class error(IOError): - pass - - -class MP4MetadataError(error): - pass - - -class MP4StreamInfoError(error): - pass - - -class MP4MetadataValueError(ValueError, MP4MetadataError): - pass - - -# This is not an exhaustive list of container atoms, but just the -# ones this module needs to peek inside. -_CONTAINERS = ["moov", "udta", "trak", "mdia", "meta", "ilst", - "stbl", "minf", "moof", "traf"] -_SKIP_SIZE = {"meta": 4} - -__all__ = ['MP4', 'Open', 'delete', 'MP4Cover', 'MP4FreeForm'] - - -class MP4Cover(str): - """A cover artwork. - - Attributes: - - * imageformat -- format of the image (either FORMAT_JPEG or FORMAT_PNG) - """ - FORMAT_JPEG = 0x0D - FORMAT_PNG = 0x0E - - def __new__(cls, data, *args, **kwargs): - return str.__new__(cls, data) - - def __init__(self, data, imageformat=FORMAT_JPEG): - self.imageformat = imageformat - try: - self.format - except AttributeError: - self.format = imageformat - - -class MP4FreeForm(str): - """A freeform value. - - Attributes: - - * dataformat -- format of the data (either FORMAT_TEXT or FORMAT_DATA) - """ - - FORMAT_DATA = 0x0 - FORMAT_TEXT = 0x1 - - def __new__(cls, data, *args, **kwargs): - return str.__new__(cls, data) - - def __init__(self, data, dataformat=FORMAT_TEXT): - self.dataformat = dataformat - - -class Atom(object): - """An individual atom. - - Attributes: - children -- list child atoms (or None for non-container atoms) - length -- length of this atom, including length and name - name -- four byte name of the atom, as a str - offset -- location in the constructor-given fileobj of this atom - - This structure should only be used internally by Mutagen. - """ - - children = None - - def __init__(self, fileobj, level=0): - self.offset = fileobj.tell() - self.length, self.name = struct.unpack(">I4s", fileobj.read(8)) - if self.length == 1: - self.length, = struct.unpack(">Q", fileobj.read(8)) - if self.length < 16: - raise MP4MetadataError( - "64 bit atom length can only be 16 and higher") - elif self.length == 0: - if level != 0: - raise MP4MetadataError( - "only a top-level atom can have zero length") - # Only the last atom is supposed to have a zero-length, meaning it - # extends to the end of file. - fileobj.seek(0, 2) - self.length = fileobj.tell() - self.offset - fileobj.seek(self.offset + 8, 0) - elif self.length < 8: - raise MP4MetadataError( - "atom length can only be 0, 1 or 8 and higher") - - if self.name in _CONTAINERS: - self.children = [] - fileobj.seek(_SKIP_SIZE.get(self.name, 0), 1) - while fileobj.tell() < self.offset + self.length: - self.children.append(Atom(fileobj, level + 1)) - else: - fileobj.seek(self.offset + self.length, 0) - - @staticmethod - def render(name, data): - """Render raw atom data.""" - # this raises OverflowError if Py_ssize_t can't handle the atom data - size = len(data) + 8 - if size <= 0xFFFFFFFF: - return struct.pack(">I4s", size, name) + data - else: - return struct.pack(">I4sQ", 1, name, size + 8) + data - - def findall(self, name, recursive=False): - """Recursively find all child atoms by specified name.""" - if self.children is not None: - for child in self.children: - if child.name == name: - yield child - if recursive: - for atom in child.findall(name, True): - yield atom - - def __getitem__(self, remaining): - """Look up a child atom, potentially recursively. - - e.g. atom['udta', 'meta'] => - """ - if not remaining: - return self - elif self.children is None: - raise KeyError("%r is not a container" % self.name) - for child in self.children: - if child.name == remaining[0]: - return child[remaining[1:]] - else: - raise KeyError("%r not found" % remaining[0]) - - def __repr__(self): - klass = self.__class__.__name__ - if self.children is None: - return "<%s name=%r length=%r offset=%r>" % ( - klass, self.name, self.length, self.offset) - else: - children = "\n".join([" " + line for child in self.children - for line in repr(child).splitlines()]) - return "<%s name=%r length=%r offset=%r\n%s>" % ( - klass, self.name, self.length, self.offset, children) - - -class Atoms(object): - """Root atoms in a given file. - - Attributes: - atoms -- a list of top-level atoms as Atom objects - - This structure should only be used internally by Mutagen. - """ - - def __init__(self, fileobj): - self.atoms = [] - fileobj.seek(0, 2) - end = fileobj.tell() - fileobj.seek(0) - while fileobj.tell() + 8 <= end: - self.atoms.append(Atom(fileobj)) - - def path(self, *names): - """Look up and return the complete path of an atom. - - For example, atoms.path('moov', 'udta', 'meta') will return a - list of three atoms, corresponding to the moov, udta, and meta - atoms. - """ - - path = [self] - for name in names: - path.append(path[-1][name, ]) - return path[1:] - - def __contains__(self, names): - try: - self[names] - except KeyError: - return False - return True - - def __getitem__(self, names): - """Look up a child atom. - - 'names' may be a list of atoms (['moov', 'udta']) or a string - specifying the complete path ('moov.udta'). - """ - - if isinstance(names, basestring): - names = names.split(".") - for child in self.atoms: - if child.name == names[0]: - return child[names[1:]] - else: - raise KeyError("%s not found" % names[0]) - - def __repr__(self): - return "\n".join([repr(child) for child in self.atoms]) - - -class MP4Tags(DictProxy, Metadata): - r"""Dictionary containing Apple iTunes metadata list key/values. - - Keys are four byte identifiers, except for freeform ('----') - keys. Values are usually unicode strings, but some atoms have a - special structure: - - Text values (multiple values per key are supported): - - * '\\xa9nam' -- track title - * '\\xa9alb' -- album - * '\\xa9ART' -- artist - * 'aART' -- album artist - * '\\xa9wrt' -- composer - * '\\xa9day' -- year - * '\\xa9cmt' -- comment - * 'desc' -- description (usually used in podcasts) - * 'purd' -- purchase date - * '\\xa9grp' -- grouping - * '\\xa9gen' -- genre - * '\\xa9lyr' -- lyrics - * 'purl' -- podcast URL - * 'egid' -- podcast episode GUID - * 'catg' -- podcast category - * 'keyw' -- podcast keywords - * '\\xa9too' -- encoded by - * 'cprt' -- copyright - * 'soal' -- album sort order - * 'soaa' -- album artist sort order - * 'soar' -- artist sort order - * 'sonm' -- title sort order - * 'soco' -- composer sort order - * 'sosn' -- show sort order - * 'tvsh' -- show name - - Boolean values: - - * 'cpil' -- part of a compilation - * 'pgap' -- part of a gapless album - * 'pcst' -- podcast (iTunes reads this only on import) - - Tuples of ints (multiple values per key are supported): - - * 'trkn' -- track number, total tracks - * 'disk' -- disc number, total discs - - Others: - - * 'tmpo' -- tempo/BPM, 16 bit int - * 'covr' -- cover artwork, list of MP4Cover objects (which are - tagged strs) - * 'gnre' -- ID3v1 genre. Not supported, use '\\xa9gen' instead. - - The freeform '----' frames use a key in the format '----:mean:name' - where 'mean' is usually 'com.apple.iTunes' and 'name' is a unique - identifier for this frame. The value is a str, but is probably - text that can be decoded as UTF-8. Multiple values per key are - supported. - - MP4 tag data cannot exist outside of the structure of an MP4 file, - so this class should not be manually instantiated. - - Unknown non-text tags are removed. - """ - - def load(self, atoms, fileobj): - try: - ilst = atoms["moov.udta.meta.ilst"] - except KeyError, key: - raise MP4MetadataError(key) - for atom in ilst.children: - fileobj.seek(atom.offset + 8) - data = fileobj.read(atom.length - 8) - if len(data) != atom.length - 8: - raise MP4MetadataError("Not enough data") - - if atom.name in self.__atoms: - info = self.__atoms[atom.name] - info[0](self, atom, data, *info[2:]) - else: - # unknown atom, try as text and skip if it fails - # FIXME: keep them somehow - try: - self.__parse_text(atom, data) - except MP4MetadataError: - continue - - @classmethod - def _can_load(cls, atoms): - return "moov.udta.meta.ilst" in atoms - - @staticmethod - def __key_sort(item1, item2): - (key1, v1) = item1 - (key2, v2) = item2 - # iTunes always writes the tags in order of "relevance", try - # to copy it as closely as possible. - order = ["\xa9nam", "\xa9ART", "\xa9wrt", "\xa9alb", - "\xa9gen", "gnre", "trkn", "disk", - "\xa9day", "cpil", "pgap", "pcst", "tmpo", - "\xa9too", "----", "covr", "\xa9lyr"] - order = dict(zip(order, range(len(order)))) - last = len(order) - # If there's no key-based way to distinguish, order by length. - # If there's still no way, go by string comparison on the - # values, so we at least have something determinstic. - return (cmp(order.get(key1[:4], last), order.get(key2[:4], last)) or - cmp(len(v1), len(v2)) or cmp(v1, v2)) - - def save(self, filename): - """Save the metadata to the given filename.""" - values = [] - items = self.items() - items.sort(self.__key_sort) - for key, value in items: - info = self.__atoms.get(key[:4], (None, type(self).__render_text)) - try: - values.append(info[1](self, key, value, *info[2:])) - except (TypeError, ValueError), s: - raise MP4MetadataValueError, s, sys.exc_info()[2] - data = Atom.render("ilst", "".join(values)) - - # Find the old atoms. - fileobj = open(filename, "rb+") - try: - atoms = Atoms(fileobj) - try: - path = atoms.path("moov", "udta", "meta", "ilst") - except KeyError: - self.__save_new(fileobj, atoms, data) - else: - self.__save_existing(fileobj, atoms, path, data) - finally: - fileobj.close() - - def __pad_ilst(self, data, length=None): - if length is None: - length = ((len(data) + 1023) & ~1023) - len(data) - return Atom.render("free", "\x00" * length) - - def __save_new(self, fileobj, atoms, ilst): - hdlr = Atom.render("hdlr", "\x00" * 8 + "mdirappl" + "\x00" * 9) - meta = Atom.render( - "meta", "\x00\x00\x00\x00" + hdlr + ilst + self.__pad_ilst(ilst)) - try: - path = atoms.path("moov", "udta") - except KeyError: - # moov.udta not found -- create one - path = atoms.path("moov") - meta = Atom.render("udta", meta) - offset = path[-1].offset + 8 - insert_bytes(fileobj, len(meta), offset) - fileobj.seek(offset) - fileobj.write(meta) - self.__update_parents(fileobj, path, len(meta)) - self.__update_offsets(fileobj, atoms, len(meta), offset) - - def __save_existing(self, fileobj, atoms, path, data): - # Replace the old ilst atom. - ilst = path.pop() - offset = ilst.offset - length = ilst.length - - # Check for padding "free" atoms - meta = path[-1] - index = meta.children.index(ilst) - try: - prev = meta.children[index-1] - if prev.name == "free": - offset = prev.offset - length += prev.length - except IndexError: - pass - try: - next = meta.children[index+1] - if next.name == "free": - length += next.length - except IndexError: - pass - - delta = len(data) - length - if delta > 0 or (delta < 0 and delta > -8): - data += self.__pad_ilst(data) - delta = len(data) - length - insert_bytes(fileobj, delta, offset) - elif delta < 0: - data += self.__pad_ilst(data, -delta - 8) - delta = 0 - - fileobj.seek(offset) - fileobj.write(data) - self.__update_parents(fileobj, path, delta) - self.__update_offsets(fileobj, atoms, delta, offset) - - def __update_parents(self, fileobj, path, delta): - """Update all parent atoms with the new size.""" - for atom in path: - fileobj.seek(atom.offset) - size = cdata.uint_be(fileobj.read(4)) - if size == 1: # 64bit - # skip name (4B) and read size (8B) - size = cdata.ulonglong_be(fileobj.read(12)[4:]) - fileobj.seek(atom.offset + 8) - fileobj.write(cdata.to_ulonglong_be(size + delta)) - else: # 32bit - fileobj.seek(atom.offset) - fileobj.write(cdata.to_uint_be(size + delta)) - - def __update_offset_table(self, fileobj, fmt, atom, delta, offset): - """Update offset table in the specified atom.""" - if atom.offset > offset: - atom.offset += delta - fileobj.seek(atom.offset + 12) - data = fileobj.read(atom.length - 12) - fmt = fmt % cdata.uint_be(data[:4]) - offsets = struct.unpack(fmt, data[4:]) - offsets = [o + (0, delta)[offset < o] for o in offsets] - fileobj.seek(atom.offset + 16) - fileobj.write(struct.pack(fmt, *offsets)) - - def __update_tfhd(self, fileobj, atom, delta, offset): - if atom.offset > offset: - atom.offset += delta - fileobj.seek(atom.offset + 9) - data = fileobj.read(atom.length - 9) - flags = cdata.uint_be("\x00" + data[:3]) - if flags & 1: - o = cdata.ulonglong_be(data[7:15]) - if o > offset: - o += delta - fileobj.seek(atom.offset + 16) - fileobj.write(cdata.to_ulonglong_be(o)) - - def __update_offsets(self, fileobj, atoms, delta, offset): - """Update offset tables in all 'stco' and 'co64' atoms.""" - if delta == 0: - return - moov = atoms["moov"] - for atom in moov.findall('stco', True): - self.__update_offset_table(fileobj, ">%dI", atom, delta, offset) - for atom in moov.findall('co64', True): - self.__update_offset_table(fileobj, ">%dQ", atom, delta, offset) - try: - for atom in atoms["moof"].findall('tfhd', True): - self.__update_tfhd(fileobj, atom, delta, offset) - except KeyError: - pass - - def __parse_data(self, atom, data): - pos = 0 - while pos < atom.length - 8: - length, name, flags = struct.unpack(">I4sI", data[pos:pos+12]) - if name != "data": - raise MP4MetadataError( - "unexpected atom %r inside %r" % (name, atom.name)) - yield flags, data[pos+16:pos+length] - pos += length - - def __render_data(self, key, flags, value): - return Atom.render(key, "".join([ - Atom.render("data", struct.pack(">2I", flags, 0) + data) - for data in value])) - - def __parse_freeform(self, atom, data): - length = cdata.uint_be(data[:4]) - mean = data[12:length] - pos = length - length = cdata.uint_be(data[pos:pos+4]) - name = data[pos+12:pos+length] - pos += length - value = [] - while pos < atom.length - 8: - length, atom_name = struct.unpack(">I4s", data[pos:pos+8]) - if atom_name != "data": - raise MP4MetadataError( - "unexpected atom %r inside %r" % (atom_name, atom.name)) - - version = ord(data[pos+8]) - if version != 0: - raise MP4MetadataError("Unsupported version: %r" % version) - - flags = struct.unpack(">I", "\x00" + data[pos+9:pos+12])[0] - value.append(MP4FreeForm(data[pos+16:pos+length], - dataformat=flags)) - pos += length - if value: - self["%s:%s:%s" % (atom.name, mean, name)] = value - - def __render_freeform(self, key, value): - dummy, mean, name = key.split(":", 2) - mean = struct.pack(">I4sI", len(mean) + 12, "mean", 0) + mean - name = struct.pack(">I4sI", len(name) + 12, "name", 0) + name - if isinstance(value, basestring): - value = [value] - data = "" - for v in value: - flags = MP4FreeForm.FORMAT_TEXT - if isinstance(v, MP4FreeForm): - flags = v.dataformat - data += struct.pack(">I4s2I", len(v) + 16, "data", flags, 0) - data += v - return Atom.render("----", mean + name + data) - - def __parse_pair(self, atom, data): - self[atom.name] = [struct.unpack(">2H", d[2:6]) for - flags, d in self.__parse_data(atom, data)] - - def __render_pair(self, key, value): - data = [] - for (track, total) in value: - if 0 <= track < 1 << 16 and 0 <= total < 1 << 16: - data.append(struct.pack(">4H", 0, track, total, 0)) - else: - raise MP4MetadataValueError( - "invalid numeric pair %r" % ((track, total),)) - return self.__render_data(key, 0, data) - - def __render_pair_no_trailing(self, key, value): - data = [] - for (track, total) in value: - if 0 <= track < 1 << 16 and 0 <= total < 1 << 16: - data.append(struct.pack(">3H", 0, track, total)) - else: - raise MP4MetadataValueError( - "invalid numeric pair %r" % ((track, total),)) - return self.__render_data(key, 0, data) - - def __parse_genre(self, atom, data): - # Translate to a freeform genre. - genre = cdata.short_be(data[16:18]) - if "\xa9gen" not in self: - try: - self["\xa9gen"] = [GENRES[genre - 1]] - except IndexError: - pass - - def __parse_tempo(self, atom, data): - self[atom.name] = [cdata.ushort_be(value[1]) for - value in self.__parse_data(atom, data)] - - def __render_tempo(self, key, value): - try: - if len(value) == 0: - return self.__render_data(key, 0x15, "") - - if min(value) < 0 or max(value) >= 2**16: - raise MP4MetadataValueError( - "invalid 16 bit integers: %r" % value) - except TypeError: - raise MP4MetadataValueError( - "tmpo must be a list of 16 bit integers") - - values = map(cdata.to_ushort_be, value) - return self.__render_data(key, 0x15, values) - - def __parse_bool(self, atom, data): - try: - self[atom.name] = bool(ord(data[16:17])) - except TypeError: - self[atom.name] = False - - def __render_bool(self, key, value): - return self.__render_data(key, 0x15, [chr(bool(value))]) - - def __parse_cover(self, atom, data): - self[atom.name] = [] - pos = 0 - while pos < atom.length - 8: - length, name, imageformat = struct.unpack(">I4sI", - data[pos:pos+12]) - if name != "data": - if name == "name": - pos += length - continue - raise MP4MetadataError( - "unexpected atom %r inside 'covr'" % name) - if imageformat not in (MP4Cover.FORMAT_JPEG, MP4Cover.FORMAT_PNG): - imageformat = MP4Cover.FORMAT_JPEG - cover = MP4Cover(data[pos+16:pos+length], imageformat) - self[atom.name].append(cover) - pos += length - - def __render_cover(self, key, value): - atom_data = [] - for cover in value: - try: - imageformat = cover.imageformat - except AttributeError: - imageformat = MP4Cover.FORMAT_JPEG - atom_data.append(Atom.render( - "data", struct.pack(">2I", imageformat, 0) + cover)) - return Atom.render(key, "".join(atom_data)) - - def __parse_text(self, atom, data, expected_flags=1): - value = [text.decode('utf-8', 'replace') for flags, text - in self.__parse_data(atom, data) - if flags == expected_flags] - if value: - self[atom.name] = value - - def __render_text(self, key, value, flags=1): - if isinstance(value, basestring): - value = [value] - return self.__render_data( - key, flags, map(utf8, value)) - - def delete(self, filename): - """Remove the metadata from the given filename.""" - - self.clear() - self.save(filename) - - __atoms = { - "----": (__parse_freeform, __render_freeform), - "trkn": (__parse_pair, __render_pair), - "disk": (__parse_pair, __render_pair_no_trailing), - "gnre": (__parse_genre, None), - "tmpo": (__parse_tempo, __render_tempo), - "cpil": (__parse_bool, __render_bool), - "pgap": (__parse_bool, __render_bool), - "pcst": (__parse_bool, __render_bool), - "covr": (__parse_cover, __render_cover), - "purl": (__parse_text, __render_text, 0), - "egid": (__parse_text, __render_text, 0), - } - - # the text atoms we know about which should make loading fail if parsing - # any of them fails - for name in ["\xa9nam", "\xa9alb", "\xa9ART", "aART", "\xa9wrt", "\xa9day", - "\xa9cmt", "desc", "purd", "\xa9grp", "\xa9gen", "\xa9lyr", - "catg", "keyw", "\xa9too", "cprt", "soal", "soaa", "soar", - "sonm", "soco", "sosn", "tvsh"]: - __atoms[name] = (__parse_text, __render_text) - - def pprint(self): - values = [] - for key, value in self.iteritems(): - key = key.decode('latin1') - if key == "covr": - values.append("%s=%s" % (key, ", ".join( - ["[%d bytes of data]" % len(data) for data in value]))) - elif isinstance(value, list): - values.append("%s=%s" % (key, " / ".join(map(unicode, value)))) - else: - values.append("%s=%s" % (key, value)) - return "\n".join(values) - - -class MP4Info(object): - """MPEG-4 stream information. - - Attributes: - - * bitrate -- bitrate in bits per second, as an int - * length -- file length in seconds, as a float - * channels -- number of audio channels - * sample_rate -- audio sampling rate in Hz - * bits_per_sample -- bits per sample - """ - - bitrate = 0 - channels = 0 - sample_rate = 0 - bits_per_sample = 0 - - def __init__(self, atoms, fileobj): - for trak in list(atoms["moov"].findall("trak")): - hdlr = trak["mdia", "hdlr"] - fileobj.seek(hdlr.offset) - data = fileobj.read(hdlr.length) - if data[16:20] == "soun": - break - else: - raise MP4StreamInfoError("track has no audio data") - - mdhd = trak["mdia", "mdhd"] - fileobj.seek(mdhd.offset) - data = fileobj.read(mdhd.length) - if ord(data[8]) == 0: - offset = 20 - fmt = ">2I" - else: - offset = 28 - fmt = ">IQ" - end = offset + struct.calcsize(fmt) - unit, length = struct.unpack(fmt, data[offset:end]) - self.length = float(length) / unit - - try: - atom = trak["mdia", "minf", "stbl", "stsd"] - fileobj.seek(atom.offset) - data = fileobj.read(atom.length) - if data[20:24] == "mp4a": - length = cdata.uint_be(data[16:20]) - (self.channels, self.bits_per_sample, _, - self.sample_rate) = struct.unpack(">3HI", data[40:50]) - # ES descriptor type - if data[56:60] == "esds" and ord(data[64:65]) == 0x03: - pos = 65 - # skip extended descriptor type tag, length, ES ID - # and stream priority - if data[pos:pos+3] == "\x80\x80\x80": - pos += 3 - pos += 4 - # decoder config descriptor type - if ord(data[pos]) == 0x04: - pos += 1 - # skip extended descriptor type tag, length, - # object type ID, stream type, buffer size - # and maximum bitrate - if data[pos:pos+3] == "\x80\x80\x80": - pos += 3 - pos += 10 - # average bitrate - self.bitrate = cdata.uint_be(data[pos:pos+4]) - except (ValueError, KeyError): - # stsd atoms are optional - pass - - def pprint(self): - return "MPEG-4 audio, %.2f seconds, %d bps" % ( - self.length, self.bitrate) - - -class MP4(FileType): - """An MPEG-4 audio file, probably containing AAC. - - If more than one track is present in the file, the first is used. - Only audio ('soun') tracks will be read. - - :ivar info: :class:`MP4Info` - :ivar tags: :class:`MP4Tags` - """ - - MP4Tags = MP4Tags - - _mimes = ["audio/mp4", "audio/x-m4a", "audio/mpeg4", "audio/aac"] - - def load(self, filename): - self.filename = filename - fileobj = open(filename, "rb") - try: - atoms = Atoms(fileobj) - - # ftyp is always the first atom in a valid MP4 file - if not atoms.atoms or atoms.atoms[0].name != "ftyp": - raise error("Not a MP4 file") - - try: - self.info = MP4Info(atoms, fileobj) - except StandardError, err: - raise MP4StreamInfoError, err, sys.exc_info()[2] - - if not MP4Tags._can_load(atoms): - self.tags = None - else: - try: - self.tags = self.MP4Tags(atoms, fileobj) - except StandardError, err: - raise MP4MetadataError, err, sys.exc_info()[2] - finally: - fileobj.close() - - def add_tags(self): - if self.tags is None: - self.tags = self.MP4Tags() - else: - raise error("an MP4 tag already exists") - - @staticmethod - def score(filename, fileobj, header): - return ("ftyp" in header) + ("mp4" in header) - - -Open = MP4 - - -def delete(filename): - """Remove tags from a file.""" - - MP4(filename).delete() diff --git a/libs/mutagen/mp4/__init__.py b/libs/mutagen/mp4/__init__.py new file mode 100644 index 00000000..e3c16a7f --- /dev/null +++ b/libs/mutagen/mp4/__init__.py @@ -0,0 +1,1023 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2006 Joe Wreschnig +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +"""Read and write MPEG-4 audio files with iTunes metadata. + +This module will read MPEG-4 audio information and metadata, +as found in Apple's MP4 (aka M4A, M4B, M4P) files. + +There is no official specification for this format. The source code +for TagLib, FAAD, and various MPEG specifications at + +* http://developer.apple.com/documentation/QuickTime/QTFF/ +* http://www.geocities.com/xhelmboyx/quicktime/formats/mp4-layout.txt +* http://standards.iso.org/ittf/PubliclyAvailableStandards/\ +c041828_ISO_IEC_14496-12_2005(E).zip +* http://wiki.multimedia.cx/index.php?title=Apple_QuickTime + +were all consulted. +""" + +import struct +import sys + +from mutagen import FileType, Tags, StreamInfo, PaddingInfo +from mutagen._constants import GENRES +from mutagen._util import (cdata, insert_bytes, DictProxy, MutagenError, + hashable, enum, get_size, resize_bytes) +from mutagen._compat import (reraise, PY2, string_types, text_type, chr_, + iteritems, PY3, cBytesIO, izip, xrange) +from ._atom import Atoms, Atom, AtomError +from ._util import parse_full_atom +from ._as_entry import AudioSampleEntry, ASEntryError + + +class error(IOError, MutagenError): + pass + + +class MP4MetadataError(error): + pass + + +class MP4StreamInfoError(error): + pass + + +class MP4MetadataValueError(ValueError, MP4MetadataError): + pass + + +__all__ = ['MP4', 'Open', 'delete', 'MP4Cover', 'MP4FreeForm', 'AtomDataType'] + + +@enum +class AtomDataType(object): + """Enum for `dataformat` attribute of MP4FreeForm. + + .. versionadded:: 1.25 + """ + + IMPLICIT = 0 + """for use with tags for which no type needs to be indicated because + only one type is allowed""" + + UTF8 = 1 + """without any count or null terminator""" + + UTF16 = 2 + """also known as UTF-16BE""" + + SJIS = 3 + """deprecated unless it is needed for special Japanese characters""" + + HTML = 6 + """the HTML file header specifies which HTML version""" + + XML = 7 + """the XML header must identify the DTD or schemas""" + + UUID = 8 + """also known as GUID; stored as 16 bytes in binary (valid as an ID)""" + + ISRC = 9 + """stored as UTF-8 text (valid as an ID)""" + + MI3P = 10 + """stored as UTF-8 text (valid as an ID)""" + + GIF = 12 + """(deprecated) a GIF image""" + + JPEG = 13 + """a JPEG image""" + + PNG = 14 + """PNG image""" + + URL = 15 + """absolute, in UTF-8 characters""" + + DURATION = 16 + """in milliseconds, 32-bit integer""" + + DATETIME = 17 + """in UTC, counting seconds since midnight, January 1, 1904; + 32 or 64-bits""" + + GENRES = 18 + """a list of enumerated values""" + + INTEGER = 21 + """a signed big-endian integer with length one of { 1,2,3,4,8 } bytes""" + + RIAA_PA = 24 + """RIAA parental advisory; { -1=no, 1=yes, 0=unspecified }, + 8-bit ingteger""" + + UPC = 25 + """Universal Product Code, in text UTF-8 format (valid as an ID)""" + + BMP = 27 + """Windows bitmap image""" + + +@hashable +class MP4Cover(bytes): + """A cover artwork. + + Attributes: + + * imageformat -- format of the image (either FORMAT_JPEG or FORMAT_PNG) + """ + + FORMAT_JPEG = AtomDataType.JPEG + FORMAT_PNG = AtomDataType.PNG + + def __new__(cls, data, *args, **kwargs): + return bytes.__new__(cls, data) + + def __init__(self, data, imageformat=FORMAT_JPEG): + self.imageformat = imageformat + + __hash__ = bytes.__hash__ + + def __eq__(self, other): + if not isinstance(other, MP4Cover): + return bytes(self) == other + + return (bytes(self) == bytes(other) and + self.imageformat == other.imageformat) + + def __ne__(self, other): + return not self.__eq__(other) + + def __repr__(self): + return "%s(%r, %r)" % ( + type(self).__name__, bytes(self), + AtomDataType(self.imageformat)) + + +@hashable +class MP4FreeForm(bytes): + """A freeform value. + + Attributes: + + * dataformat -- format of the data (see AtomDataType) + """ + + FORMAT_DATA = AtomDataType.IMPLICIT # deprecated + FORMAT_TEXT = AtomDataType.UTF8 # deprecated + + def __new__(cls, data, *args, **kwargs): + return bytes.__new__(cls, data) + + def __init__(self, data, dataformat=AtomDataType.UTF8, version=0): + self.dataformat = dataformat + self.version = version + + __hash__ = bytes.__hash__ + + def __eq__(self, other): + if not isinstance(other, MP4FreeForm): + return bytes(self) == other + + return (bytes(self) == bytes(other) and + self.dataformat == other.dataformat and + self.version == other.version) + + def __ne__(self, other): + return not self.__eq__(other) + + def __repr__(self): + return "%s(%r, %r)" % ( + type(self).__name__, bytes(self), + AtomDataType(self.dataformat)) + + + +def _name2key(name): + if PY2: + return name + return name.decode("latin-1") + + +def _key2name(key): + if PY2: + return key + return key.encode("latin-1") + + +def _find_padding(atom_path): + # Check for padding "free" atom + # XXX: we only use them if they are adjacent to ilst, and only one. + # and there also is a top level free atom which we could use maybe..? + + meta, ilst = atom_path[-2:] + assert meta.name == b"meta" and ilst.name == b"ilst" + index = meta.children.index(ilst) + try: + prev = meta.children[index - 1] + if prev.name == b"free": + return prev + except IndexError: + pass + + try: + next_ = meta.children[index + 1] + if next_.name == b"free": + return next_ + except IndexError: + pass + + +def _item_sort_key(key, value): + # iTunes always writes the tags in order of "relevance", try + # to copy it as closely as possible. + order = ["\xa9nam", "\xa9ART", "\xa9wrt", "\xa9alb", + "\xa9gen", "gnre", "trkn", "disk", + "\xa9day", "cpil", "pgap", "pcst", "tmpo", + "\xa9too", "----", "covr", "\xa9lyr"] + order = dict(izip(order, xrange(len(order)))) + last = len(order) + # If there's no key-based way to distinguish, order by length. + # If there's still no way, go by string comparison on the + # values, so we at least have something determinstic. + return (order.get(key[:4], last), len(repr(value)), repr(value)) + + +class MP4Tags(DictProxy, Tags): + r"""Dictionary containing Apple iTunes metadata list key/values. + + Keys are four byte identifiers, except for freeform ('----') + keys. Values are usually unicode strings, but some atoms have a + special structure: + + Text values (multiple values per key are supported): + + * '\\xa9nam' -- track title + * '\\xa9alb' -- album + * '\\xa9ART' -- artist + * 'aART' -- album artist + * '\\xa9wrt' -- composer + * '\\xa9day' -- year + * '\\xa9cmt' -- comment + * 'desc' -- description (usually used in podcasts) + * 'purd' -- purchase date + * '\\xa9grp' -- grouping + * '\\xa9gen' -- genre + * '\\xa9lyr' -- lyrics + * 'purl' -- podcast URL + * 'egid' -- podcast episode GUID + * 'catg' -- podcast category + * 'keyw' -- podcast keywords + * '\\xa9too' -- encoded by + * 'cprt' -- copyright + * 'soal' -- album sort order + * 'soaa' -- album artist sort order + * 'soar' -- artist sort order + * 'sonm' -- title sort order + * 'soco' -- composer sort order + * 'sosn' -- show sort order + * 'tvsh' -- show name + + Boolean values: + + * 'cpil' -- part of a compilation + * 'pgap' -- part of a gapless album + * 'pcst' -- podcast (iTunes reads this only on import) + + Tuples of ints (multiple values per key are supported): + + * 'trkn' -- track number, total tracks + * 'disk' -- disc number, total discs + + Others: + + * 'tmpo' -- tempo/BPM, 16 bit int + * 'covr' -- cover artwork, list of MP4Cover objects (which are + tagged strs) + * 'gnre' -- ID3v1 genre. Not supported, use '\\xa9gen' instead. + + The freeform '----' frames use a key in the format '----:mean:name' + where 'mean' is usually 'com.apple.iTunes' and 'name' is a unique + identifier for this frame. The value is a str, but is probably + text that can be decoded as UTF-8. Multiple values per key are + supported. + + MP4 tag data cannot exist outside of the structure of an MP4 file, + so this class should not be manually instantiated. + + Unknown non-text tags and tags that failed to parse will be written + back as is. + """ + + def __init__(self, *args, **kwargs): + self._failed_atoms = {} + super(MP4Tags, self).__init__() + if args or kwargs: + self.load(*args, **kwargs) + + def load(self, atoms, fileobj): + try: + path = atoms.path(b"moov", b"udta", b"meta", b"ilst") + except KeyError as key: + raise MP4MetadataError(key) + + free = _find_padding(path) + self._padding = free.datalength if free is not None else 0 + + ilst = path[-1] + for atom in ilst.children: + ok, data = atom.read(fileobj) + if not ok: + raise MP4MetadataError("Not enough data") + + try: + if atom.name in self.__atoms: + info = self.__atoms[atom.name] + info[0](self, atom, data) + else: + # unknown atom, try as text + self.__parse_text(atom, data, implicit=False) + except MP4MetadataError: + # parsing failed, save them so we can write them back + key = _name2key(atom.name) + self._failed_atoms.setdefault(key, []).append(data) + + def __setitem__(self, key, value): + if not isinstance(key, str): + raise TypeError("key has to be str") + self._render(key, value) + super(MP4Tags, self).__setitem__(key, value) + + @classmethod + def _can_load(cls, atoms): + return b"moov.udta.meta.ilst" in atoms + + def _render(self, key, value): + atom_name = _key2name(key)[:4] + if atom_name in self.__atoms: + render_func = self.__atoms[atom_name][1] + else: + render_func = type(self).__render_text + + return render_func(self, key, value) + + def save(self, filename, padding=None): + """Save the metadata to the given filename.""" + + values = [] + items = sorted(self.items(), key=lambda kv: _item_sort_key(*kv)) + for key, value in items: + try: + values.append(self._render(key, value)) + except (TypeError, ValueError) as s: + reraise(MP4MetadataValueError, s, sys.exc_info()[2]) + + for key, failed in iteritems(self._failed_atoms): + # don't write atoms back if we have added a new one with + # the same name, this excludes freeform which can have + # multiple atoms with the same key (most parsers seem to be able + # to handle that) + if key in self: + assert _key2name(key) != b"----" + continue + for data in failed: + values.append(Atom.render(_key2name(key), data)) + + data = Atom.render(b"ilst", b"".join(values)) + + # Find the old atoms. + with open(filename, "rb+") as fileobj: + try: + atoms = Atoms(fileobj) + except AtomError as err: + reraise(error, err, sys.exc_info()[2]) + + self.__save(fileobj, atoms, data, padding) + + def __save(self, fileobj, atoms, data, padding): + try: + path = atoms.path(b"moov", b"udta", b"meta", b"ilst") + except KeyError: + self.__save_new(fileobj, atoms, data, padding) + else: + self.__save_existing(fileobj, atoms, path, data, padding) + + def __save_new(self, fileobj, atoms, ilst_data, padding_func): + hdlr = Atom.render(b"hdlr", b"\x00" * 8 + b"mdirappl" + b"\x00" * 9) + meta_data = b"\x00\x00\x00\x00" + hdlr + ilst_data + + try: + path = atoms.path(b"moov", b"udta") + except KeyError: + path = atoms.path(b"moov") + + offset = path[-1]._dataoffset + + # ignoring some atom overhead... but we don't have padding left anyway + # and padding_size is guaranteed to be less than zero + content_size = get_size(fileobj) - offset + padding_size = -len(meta_data) + assert padding_size < 0 + info = PaddingInfo(padding_size, content_size) + new_padding = info._get_padding(padding_func) + new_padding = min(0xFFFFFFFF, new_padding) + + free = Atom.render(b"free", b"\x00" * new_padding) + meta = Atom.render(b"meta", meta_data + free) + if path[-1].name != b"udta": + # moov.udta not found -- create one + data = Atom.render(b"udta", meta) + else: + data = meta + + insert_bytes(fileobj, len(data), offset) + fileobj.seek(offset) + fileobj.write(data) + self.__update_parents(fileobj, path, len(data)) + self.__update_offsets(fileobj, atoms, len(data), offset) + + def __save_existing(self, fileobj, atoms, path, ilst_data, padding_func): + # Replace the old ilst atom. + ilst = path[-1] + offset = ilst.offset + length = ilst.length + + # Use adjacent free atom if there is one + free = _find_padding(path) + if free is not None: + offset = min(offset, free.offset) + length += free.length + + # Always add a padding atom to make things easier + padding_overhead = len(Atom.render(b"free", b"")) + content_size = get_size(fileobj) - (offset + length) + padding_size = length - (len(ilst_data) + padding_overhead) + info = PaddingInfo(padding_size, content_size) + new_padding = info._get_padding(padding_func) + # Limit padding size so we can be sure the free atom overhead is as we + # calculated above (see Atom.render) + new_padding = min(0xFFFFFFFF, new_padding) + + ilst_data += Atom.render(b"free", b"\x00" * new_padding) + + resize_bytes(fileobj, length, len(ilst_data), offset) + delta = len(ilst_data) - length + + fileobj.seek(offset) + fileobj.write(ilst_data) + self.__update_parents(fileobj, path[:-1], delta) + self.__update_offsets(fileobj, atoms, delta, offset) + + def __update_parents(self, fileobj, path, delta): + """Update all parent atoms with the new size.""" + + if delta == 0: + return + + for atom in path: + fileobj.seek(atom.offset) + size = cdata.uint_be(fileobj.read(4)) + if size == 1: # 64bit + # skip name (4B) and read size (8B) + size = cdata.ulonglong_be(fileobj.read(12)[4:]) + fileobj.seek(atom.offset + 8) + fileobj.write(cdata.to_ulonglong_be(size + delta)) + else: # 32bit + fileobj.seek(atom.offset) + fileobj.write(cdata.to_uint_be(size + delta)) + + def __update_offset_table(self, fileobj, fmt, atom, delta, offset): + """Update offset table in the specified atom.""" + if atom.offset > offset: + atom.offset += delta + fileobj.seek(atom.offset + 12) + data = fileobj.read(atom.length - 12) + fmt = fmt % cdata.uint_be(data[:4]) + offsets = struct.unpack(fmt, data[4:]) + offsets = [o + (0, delta)[offset < o] for o in offsets] + fileobj.seek(atom.offset + 16) + fileobj.write(struct.pack(fmt, *offsets)) + + def __update_tfhd(self, fileobj, atom, delta, offset): + if atom.offset > offset: + atom.offset += delta + fileobj.seek(atom.offset + 9) + data = fileobj.read(atom.length - 9) + flags = cdata.uint_be(b"\x00" + data[:3]) + if flags & 1: + o = cdata.ulonglong_be(data[7:15]) + if o > offset: + o += delta + fileobj.seek(atom.offset + 16) + fileobj.write(cdata.to_ulonglong_be(o)) + + def __update_offsets(self, fileobj, atoms, delta, offset): + """Update offset tables in all 'stco' and 'co64' atoms.""" + if delta == 0: + return + moov = atoms[b"moov"] + for atom in moov.findall(b'stco', True): + self.__update_offset_table(fileobj, ">%dI", atom, delta, offset) + for atom in moov.findall(b'co64', True): + self.__update_offset_table(fileobj, ">%dQ", atom, delta, offset) + try: + for atom in atoms[b"moof"].findall(b'tfhd', True): + self.__update_tfhd(fileobj, atom, delta, offset) + except KeyError: + pass + + def __parse_data(self, atom, data): + pos = 0 + while pos < atom.length - 8: + head = data[pos:pos + 12] + if len(head) != 12: + raise MP4MetadataError("truncated atom % r" % atom.name) + length, name = struct.unpack(">I4s", head[:8]) + version = ord(head[8:9]) + flags = struct.unpack(">I", b"\x00" + head[9:12])[0] + if name != b"data": + raise MP4MetadataError( + "unexpected atom %r inside %r" % (name, atom.name)) + + chunk = data[pos + 16:pos + length] + if len(chunk) != length - 16: + raise MP4MetadataError("truncated atom % r" % atom.name) + yield version, flags, chunk + pos += length + + def __add(self, key, value, single=False): + assert isinstance(key, str) + + if single: + self[key] = value + else: + self.setdefault(key, []).extend(value) + + def __render_data(self, key, version, flags, value): + return Atom.render(_key2name(key), b"".join([ + Atom.render( + b"data", struct.pack(">2I", version << 24 | flags, 0) + data) + for data in value])) + + def __parse_freeform(self, atom, data): + length = cdata.uint_be(data[:4]) + mean = data[12:length] + pos = length + length = cdata.uint_be(data[pos:pos + 4]) + name = data[pos + 12:pos + length] + pos += length + value = [] + while pos < atom.length - 8: + length, atom_name = struct.unpack(">I4s", data[pos:pos + 8]) + if atom_name != b"data": + raise MP4MetadataError( + "unexpected atom %r inside %r" % (atom_name, atom.name)) + + version = ord(data[pos + 8:pos + 8 + 1]) + flags = struct.unpack(">I", b"\x00" + data[pos + 9:pos + 12])[0] + value.append(MP4FreeForm(data[pos + 16:pos + length], + dataformat=flags, version=version)) + pos += length + + key = _name2key(atom.name + b":" + mean + b":" + name) + self.__add(key, value) + + def __render_freeform(self, key, value): + if isinstance(value, bytes): + value = [value] + + dummy, mean, name = _key2name(key).split(b":", 2) + mean = struct.pack(">I4sI", len(mean) + 12, b"mean", 0) + mean + name = struct.pack(">I4sI", len(name) + 12, b"name", 0) + name + + data = b"" + for v in value: + flags = AtomDataType.UTF8 + version = 0 + if isinstance(v, MP4FreeForm): + flags = v.dataformat + version = v.version + + data += struct.pack( + ">I4s2I", len(v) + 16, b"data", version << 24 | flags, 0) + data += v + + return Atom.render(b"----", mean + name + data) + + def __parse_pair(self, atom, data): + key = _name2key(atom.name) + values = [struct.unpack(">2H", d[2:6]) for + version, flags, d in self.__parse_data(atom, data)] + self.__add(key, values) + + def __render_pair(self, key, value): + data = [] + for v in value: + try: + track, total = v + except TypeError: + raise ValueError + if 0 <= track < 1 << 16 and 0 <= total < 1 << 16: + data.append(struct.pack(">4H", 0, track, total, 0)) + else: + raise MP4MetadataValueError( + "invalid numeric pair %r" % ((track, total),)) + return self.__render_data(key, 0, AtomDataType.IMPLICIT, data) + + def __render_pair_no_trailing(self, key, value): + data = [] + for (track, total) in value: + if 0 <= track < 1 << 16 and 0 <= total < 1 << 16: + data.append(struct.pack(">3H", 0, track, total)) + else: + raise MP4MetadataValueError( + "invalid numeric pair %r" % ((track, total),)) + return self.__render_data(key, 0, AtomDataType.IMPLICIT, data) + + def __parse_genre(self, atom, data): + values = [] + for version, flags, data in self.__parse_data(atom, data): + # version = 0, flags = 0 + if len(data) != 2: + raise MP4MetadataValueError("invalid genre") + genre = cdata.short_be(data) + # Translate to a freeform genre. + try: + genre = GENRES[genre - 1] + except IndexError: + # this will make us write it back at least + raise MP4MetadataValueError("unknown genre") + values.append(genre) + key = _name2key(b"\xa9gen") + self.__add(key, values) + + def __parse_tempo(self, atom, data): + values = [] + for version, flags, data in self.__parse_data(atom, data): + # version = 0, flags = 0 or 21 + if len(data) != 2: + raise MP4MetadataValueError("invalid tempo") + values.append(cdata.ushort_be(data)) + key = _name2key(atom.name) + self.__add(key, values) + + def __render_tempo(self, key, value): + try: + if len(value) == 0: + return self.__render_data(key, 0, AtomDataType.INTEGER, b"") + + if (min(value) < 0) or (max(value) >= 2 ** 16): + raise MP4MetadataValueError( + "invalid 16 bit integers: %r" % value) + except TypeError: + raise MP4MetadataValueError( + "tmpo must be a list of 16 bit integers") + + values = [cdata.to_ushort_be(v) for v in value] + return self.__render_data(key, 0, AtomDataType.INTEGER, values) + + def __parse_bool(self, atom, data): + for version, flags, data in self.__parse_data(atom, data): + if len(data) != 1: + raise MP4MetadataValueError("invalid bool") + + value = bool(ord(data)) + key = _name2key(atom.name) + self.__add(key, value, single=True) + + def __render_bool(self, key, value): + return self.__render_data( + key, 0, AtomDataType.INTEGER, [chr_(bool(value))]) + + def __parse_cover(self, atom, data): + values = [] + pos = 0 + while pos < atom.length - 8: + length, name, imageformat = struct.unpack(">I4sI", + data[pos:pos + 12]) + if name != b"data": + if name == b"name": + pos += length + continue + raise MP4MetadataError( + "unexpected atom %r inside 'covr'" % name) + if imageformat not in (MP4Cover.FORMAT_JPEG, MP4Cover.FORMAT_PNG): + # Sometimes AtomDataType.IMPLICIT or simply wrong. + # In all cases it was jpeg, so default to it + imageformat = MP4Cover.FORMAT_JPEG + cover = MP4Cover(data[pos + 16:pos + length], imageformat) + values.append(cover) + pos += length + + key = _name2key(atom.name) + self.__add(key, values) + + def __render_cover(self, key, value): + atom_data = [] + for cover in value: + try: + imageformat = cover.imageformat + except AttributeError: + imageformat = MP4Cover.FORMAT_JPEG + atom_data.append(Atom.render( + b"data", struct.pack(">2I", imageformat, 0) + cover)) + return Atom.render(_key2name(key), b"".join(atom_data)) + + def __parse_text(self, atom, data, implicit=True): + # implicit = False, for parsing unknown atoms only take utf8 ones. + # For known ones we can assume the implicit are utf8 too. + values = [] + for version, flags, atom_data in self.__parse_data(atom, data): + if implicit: + if flags not in (AtomDataType.IMPLICIT, AtomDataType.UTF8): + raise MP4MetadataError( + "Unknown atom type %r for %r" % (flags, atom.name)) + else: + if flags != AtomDataType.UTF8: + raise MP4MetadataError( + "%r is not text, ignore" % atom.name) + + try: + text = atom_data.decode("utf-8") + except UnicodeDecodeError as e: + raise MP4MetadataError("%s: %s" % (_name2key(atom.name), e)) + + values.append(text) + + key = _name2key(atom.name) + self.__add(key, values) + + def __render_text(self, key, value, flags=AtomDataType.UTF8): + if isinstance(value, string_types): + value = [value] + + encoded = [] + for v in value: + if not isinstance(v, text_type): + if PY3: + raise TypeError("%r not str" % v) + try: + v = v.decode("utf-8") + except (AttributeError, UnicodeDecodeError) as e: + raise TypeError(e) + encoded.append(v.encode("utf-8")) + + return self.__render_data(key, 0, flags, encoded) + + def delete(self, filename): + """Remove the metadata from the given filename.""" + + self._failed_atoms.clear() + self.clear() + self.save(filename, padding=lambda x: 0) + + __atoms = { + b"----": (__parse_freeform, __render_freeform), + b"trkn": (__parse_pair, __render_pair), + b"disk": (__parse_pair, __render_pair_no_trailing), + b"gnre": (__parse_genre, None), + b"tmpo": (__parse_tempo, __render_tempo), + b"cpil": (__parse_bool, __render_bool), + b"pgap": (__parse_bool, __render_bool), + b"pcst": (__parse_bool, __render_bool), + b"covr": (__parse_cover, __render_cover), + b"purl": (__parse_text, __render_text), + b"egid": (__parse_text, __render_text), + } + + # these allow implicit flags and parse as text + for name in [b"\xa9nam", b"\xa9alb", b"\xa9ART", b"aART", b"\xa9wrt", + b"\xa9day", b"\xa9cmt", b"desc", b"purd", b"\xa9grp", + b"\xa9gen", b"\xa9lyr", b"catg", b"keyw", b"\xa9too", + b"cprt", b"soal", b"soaa", b"soar", b"sonm", b"soco", + b"sosn", b"tvsh"]: + __atoms[name] = (__parse_text, __render_text) + + def pprint(self): + + def to_line(key, value): + assert isinstance(key, text_type) + if isinstance(value, text_type): + return u"%s=%s" % (key, value) + return u"%s=%r" % (key, value) + + values = [] + for key, value in sorted(iteritems(self)): + if not isinstance(key, text_type): + key = key.decode("latin-1") + if key == "covr": + values.append(u"%s=%s" % (key, u", ".join( + [u"[%d bytes of data]" % len(data) for data in value]))) + elif isinstance(value, list): + for v in value: + values.append(to_line(key, v)) + else: + values.append(to_line(key, value)) + return u"\n".join(values) + + +class MP4Info(StreamInfo): + """MPEG-4 stream information. + + Attributes: + + * bitrate -- bitrate in bits per second, as an int + * length -- file length in seconds, as a float + * channels -- number of audio channels + * sample_rate -- audio sampling rate in Hz + * bits_per_sample -- bits per sample + * codec (string): + * if starting with ``"mp4a"`` uses an mp4a audio codec + (see the codec parameter in rfc6381 for details e.g. ``"mp4a.40.2"``) + * for everything else see a list of possible values at + http://www.mp4ra.org/codecs.html + + e.g. ``"mp4a"``, ``"alac"``, ``"mp4a.40.2"``, ``"ac-3"`` etc. + * codec_description (string): + Name of the codec used (ALAC, AAC LC, AC-3...). Values might change in + the future, use for display purposes only. + """ + + bitrate = 0 + channels = 0 + sample_rate = 0 + bits_per_sample = 0 + codec = u"" + codec_name = u"" + + def __init__(self, atoms, fileobj): + try: + moov = atoms[b"moov"] + except KeyError: + raise MP4StreamInfoError("not a MP4 file") + + for trak in moov.findall(b"trak"): + hdlr = trak[b"mdia", b"hdlr"] + ok, data = hdlr.read(fileobj) + if not ok: + raise MP4StreamInfoError("Not enough data") + if data[8:12] == b"soun": + break + else: + raise MP4StreamInfoError("track has no audio data") + + mdhd = trak[b"mdia", b"mdhd"] + ok, data = mdhd.read(fileobj) + if not ok: + raise MP4StreamInfoError("Not enough data") + + try: + version, flags, data = parse_full_atom(data) + except ValueError as e: + raise MP4StreamInfoError(e) + + if version == 0: + offset = 8 + fmt = ">2I" + elif version == 1: + offset = 16 + fmt = ">IQ" + else: + raise MP4StreamInfoError("Unknown mdhd version %d" % version) + + end = offset + struct.calcsize(fmt) + unit, length = struct.unpack(fmt, data[offset:end]) + try: + self.length = float(length) / unit + except ZeroDivisionError: + self.length = 0 + + try: + atom = trak[b"mdia", b"minf", b"stbl", b"stsd"] + except KeyError: + pass + else: + self._parse_stsd(atom, fileobj) + + def _parse_stsd(self, atom, fileobj): + """Sets channels, bits_per_sample, sample_rate and optionally bitrate. + + Can raise MP4StreamInfoError. + """ + + assert atom.name == b"stsd" + + ok, data = atom.read(fileobj) + if not ok: + raise MP4StreamInfoError("Invalid stsd") + + try: + version, flags, data = parse_full_atom(data) + except ValueError as e: + raise MP4StreamInfoError(e) + + if version != 0: + raise MP4StreamInfoError("Unsupported stsd version") + + try: + num_entries, offset = cdata.uint32_be_from(data, 0) + except cdata.error as e: + raise MP4StreamInfoError(e) + + if num_entries == 0: + return + + # look at the first entry if there is one + entry_fileobj = cBytesIO(data[offset:]) + try: + entry_atom = Atom(entry_fileobj) + except AtomError as e: + raise MP4StreamInfoError(e) + + try: + entry = AudioSampleEntry(entry_atom, entry_fileobj) + except ASEntryError as e: + raise MP4StreamInfoError(e) + else: + self.channels = entry.channels + self.bits_per_sample = entry.sample_size + self.sample_rate = entry.sample_rate + self.bitrate = entry.bitrate + self.codec = entry.codec + self.codec_description = entry.codec_description + + def pprint(self): + return "MPEG-4 audio (%s), %.2f seconds, %d bps" % ( + self.codec_description, self.length, self.bitrate) + + +class MP4(FileType): + """An MPEG-4 audio file, probably containing AAC. + + If more than one track is present in the file, the first is used. + Only audio ('soun') tracks will be read. + + :ivar info: :class:`MP4Info` + :ivar tags: :class:`MP4Tags` + """ + + MP4Tags = MP4Tags + + _mimes = ["audio/mp4", "audio/x-m4a", "audio/mpeg4", "audio/aac"] + + def load(self, filename): + self.filename = filename + with open(filename, "rb") as fileobj: + try: + atoms = Atoms(fileobj) + except AtomError as err: + reraise(error, err, sys.exc_info()[2]) + + try: + self.info = MP4Info(atoms, fileobj) + except error: + raise + except Exception as err: + reraise(MP4StreamInfoError, err, sys.exc_info()[2]) + + if not MP4Tags._can_load(atoms): + self.tags = None + self._padding = 0 + else: + try: + self.tags = self.MP4Tags(atoms, fileobj) + except error: + raise + except Exception as err: + reraise(MP4MetadataError, err, sys.exc_info()[2]) + else: + self._padding = self.tags._padding + + def save(self, filename=None, padding=None): + super(MP4, self).save(filename, padding=padding) + + def delete(self, filename=None): + super(MP4, self).delete(filename) + + def add_tags(self): + if self.tags is None: + self.tags = self.MP4Tags() + else: + raise error("an MP4 tag already exists") + + @staticmethod + def score(filename, fileobj, header_data): + return (b"ftyp" in header_data) + (b"mp4" in header_data) + + +Open = MP4 + + +def delete(filename): + """Remove tags from a file.""" + + MP4(filename).delete() diff --git a/libs/mutagen/mp4/_as_entry.py b/libs/mutagen/mp4/_as_entry.py new file mode 100644 index 00000000..306d5720 --- /dev/null +++ b/libs/mutagen/mp4/_as_entry.py @@ -0,0 +1,542 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014 Christoph Reiter +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from mutagen._compat import cBytesIO, xrange +from mutagen.aac import ProgramConfigElement +from mutagen._util import BitReader, BitReaderError, cdata +from mutagen._compat import text_type +from ._util import parse_full_atom +from ._atom import Atom, AtomError + + +class ASEntryError(Exception): + pass + + +class AudioSampleEntry(object): + """Parses an AudioSampleEntry atom. + + Private API. + + Attrs: + channels (int): number of channels + sample_size (int): sample size in bits + sample_rate (int): sample rate in Hz + bitrate (int): bits per second (0 means unknown) + codec (string): + audio codec, either 'mp4a[.*][.*]' (rfc6381) or 'alac' + codec_description (string): descriptive codec name e.g. "AAC LC+SBR" + + Can raise ASEntryError. + """ + + channels = 0 + sample_size = 0 + sample_rate = 0 + bitrate = 0 + codec = None + codec_description = None + + def __init__(self, atom, fileobj): + ok, data = atom.read(fileobj) + if not ok: + raise ASEntryError("too short %r atom" % atom.name) + + fileobj = cBytesIO(data) + r = BitReader(fileobj) + + try: + # SampleEntry + r.skip(6 * 8) # reserved + r.skip(2 * 8) # data_ref_index + + # AudioSampleEntry + r.skip(8 * 8) # reserved + self.channels = r.bits(16) + self.sample_size = r.bits(16) + r.skip(2 * 8) # pre_defined + r.skip(2 * 8) # reserved + self.sample_rate = r.bits(32) >> 16 + except BitReaderError as e: + raise ASEntryError(e) + + assert r.is_aligned() + + try: + extra = Atom(fileobj) + except AtomError as e: + raise ASEntryError(e) + + self.codec = atom.name.decode("latin-1") + self.codec_description = None + + if atom.name == b"mp4a" and extra.name == b"esds": + self._parse_esds(extra, fileobj) + elif atom.name == b"alac" and extra.name == b"alac": + self._parse_alac(extra, fileobj) + elif atom.name == b"ac-3" and extra.name == b"dac3": + self._parse_dac3(extra, fileobj) + + if self.codec_description is None: + self.codec_description = self.codec.upper() + + def _parse_dac3(self, atom, fileobj): + # ETSI TS 102 366 + + assert atom.name == b"dac3" + + ok, data = atom.read(fileobj) + if not ok: + raise ASEntryError("truncated %s atom" % atom.name) + fileobj = cBytesIO(data) + r = BitReader(fileobj) + + # sample_rate in AudioSampleEntry covers values in + # fscod2 and not just fscod, so ignore fscod here. + try: + r.skip(2 + 5 + 3) # fscod, bsid, bsmod + acmod = r.bits(3) + lfeon = r.bits(1) + bit_rate_code = r.bits(5) + r.skip(5) # reserved + except BitReaderError as e: + raise ASEntryError(e) + + self.channels = [2, 1, 2, 3, 3, 4, 4, 5][acmod] + lfeon + + try: + self.bitrate = [ + 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, + 224, 256, 320, 384, 448, 512, 576, 640][bit_rate_code] * 1000 + except IndexError: + pass + + def _parse_alac(self, atom, fileobj): + # https://alac.macosforge.org/trac/browser/trunk/ + # ALACMagicCookieDescription.txt + + assert atom.name == b"alac" + + ok, data = atom.read(fileobj) + if not ok: + raise ASEntryError("truncated %s atom" % atom.name) + + try: + version, flags, data = parse_full_atom(data) + except ValueError as e: + raise ASEntryError(e) + + if version != 0: + raise ASEntryError("Unsupported version %d" % version) + + fileobj = cBytesIO(data) + r = BitReader(fileobj) + + try: + # for some files the AudioSampleEntry values default to 44100/2chan + # and the real info is in the alac cookie, so prefer it + r.skip(32) # frameLength + compatibleVersion = r.bits(8) + if compatibleVersion != 0: + return + self.sample_size = r.bits(8) + r.skip(8 + 8 + 8) + self.channels = r.bits(8) + r.skip(16 + 32) + self.bitrate = r.bits(32) + self.sample_rate = r.bits(32) + except BitReaderError as e: + raise ASEntryError(e) + + def _parse_esds(self, esds, fileobj): + assert esds.name == b"esds" + + ok, data = esds.read(fileobj) + if not ok: + raise ASEntryError("truncated %s atom" % esds.name) + + try: + version, flags, data = parse_full_atom(data) + except ValueError as e: + raise ASEntryError(e) + + if version != 0: + raise ASEntryError("Unsupported version %d" % version) + + fileobj = cBytesIO(data) + r = BitReader(fileobj) + + try: + tag = r.bits(8) + if tag != ES_Descriptor.TAG: + raise ASEntryError("unexpected descriptor: %d" % tag) + assert r.is_aligned() + except BitReaderError as e: + raise ASEntryError(e) + + try: + decSpecificInfo = ES_Descriptor.parse(fileobj) + except DescriptorError as e: + raise ASEntryError(e) + dec_conf_desc = decSpecificInfo.decConfigDescr + + self.bitrate = dec_conf_desc.avgBitrate + self.codec += dec_conf_desc.codec_param + self.codec_description = dec_conf_desc.codec_desc + + decSpecificInfo = dec_conf_desc.decSpecificInfo + if decSpecificInfo is not None: + if decSpecificInfo.channels != 0: + self.channels = decSpecificInfo.channels + + if decSpecificInfo.sample_rate != 0: + self.sample_rate = decSpecificInfo.sample_rate + + +class DescriptorError(Exception): + pass + + +class BaseDescriptor(object): + + TAG = None + + @classmethod + def _parse_desc_length_file(cls, fileobj): + """May raise ValueError""" + + value = 0 + for i in xrange(4): + try: + b = cdata.uint8(fileobj.read(1)) + except cdata.error as e: + raise ValueError(e) + value = (value << 7) | (b & 0x7f) + if not b >> 7: + break + else: + raise ValueError("invalid descriptor length") + + return value + + @classmethod + def parse(cls, fileobj): + """Returns a parsed instance of the called type. + The file position is right after the descriptor after this returns. + + Raises DescriptorError + """ + + try: + length = cls._parse_desc_length_file(fileobj) + except ValueError as e: + raise DescriptorError(e) + pos = fileobj.tell() + instance = cls(fileobj, length) + left = length - (fileobj.tell() - pos) + if left < 0: + raise DescriptorError("descriptor parsing read too much data") + fileobj.seek(left, 1) + return instance + + +class ES_Descriptor(BaseDescriptor): + + TAG = 0x3 + + def __init__(self, fileobj, length): + """Raises DescriptorError""" + + r = BitReader(fileobj) + try: + self.ES_ID = r.bits(16) + self.streamDependenceFlag = r.bits(1) + self.URL_Flag = r.bits(1) + self.OCRstreamFlag = r.bits(1) + self.streamPriority = r.bits(5) + if self.streamDependenceFlag: + self.dependsOn_ES_ID = r.bits(16) + if self.URL_Flag: + URLlength = r.bits(8) + self.URLstring = r.bytes(URLlength) + if self.OCRstreamFlag: + self.OCR_ES_Id = r.bits(16) + + tag = r.bits(8) + except BitReaderError as e: + raise DescriptorError(e) + + if tag != DecoderConfigDescriptor.TAG: + raise DescriptorError("unexpected DecoderConfigDescrTag %d" % tag) + + assert r.is_aligned() + self.decConfigDescr = DecoderConfigDescriptor.parse(fileobj) + + +class DecoderConfigDescriptor(BaseDescriptor): + + TAG = 0x4 + + decSpecificInfo = None + """A DecoderSpecificInfo, optional""" + + def __init__(self, fileobj, length): + """Raises DescriptorError""" + + r = BitReader(fileobj) + + try: + self.objectTypeIndication = r.bits(8) + self.streamType = r.bits(6) + self.upStream = r.bits(1) + self.reserved = r.bits(1) + self.bufferSizeDB = r.bits(24) + self.maxBitrate = r.bits(32) + self.avgBitrate = r.bits(32) + + if (self.objectTypeIndication, self.streamType) != (0x40, 0x5): + return + + # all from here is optional + if length * 8 == r.get_position(): + return + + tag = r.bits(8) + except BitReaderError as e: + raise DescriptorError(e) + + if tag == DecoderSpecificInfo.TAG: + assert r.is_aligned() + self.decSpecificInfo = DecoderSpecificInfo.parse(fileobj) + + @property + def codec_param(self): + """string""" + + param = u".%X" % self.objectTypeIndication + info = self.decSpecificInfo + if info is not None: + param += u".%d" % info.audioObjectType + return param + + @property + def codec_desc(self): + """string or None""" + + info = self.decSpecificInfo + desc = None + if info is not None: + desc = info.description + return desc + + +class DecoderSpecificInfo(BaseDescriptor): + + TAG = 0x5 + + _TYPE_NAMES = [ + None, "AAC MAIN", "AAC LC", "AAC SSR", "AAC LTP", "SBR", + "AAC scalable", "TwinVQ", "CELP", "HVXC", None, None, "TTSI", + "Main synthetic", "Wavetable synthesis", "General MIDI", + "Algorithmic Synthesis and Audio FX", "ER AAC LC", None, "ER AAC LTP", + "ER AAC scalable", "ER Twin VQ", "ER BSAC", "ER AAC LD", "ER CELP", + "ER HVXC", "ER HILN", "ER Parametric", "SSC", "PS", "MPEG Surround", + None, "Layer-1", "Layer-2", "Layer-3", "DST", "ALS", "SLS", + "SLS non-core", "ER AAC ELD", "SMR Simple", "SMR Main", "USAC", + "SAOC", "LD MPEG Surround", "USAC" + ] + + _FREQS = [ + 96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, 16000, + 12000, 11025, 8000, 7350, + ] + + @property + def description(self): + """string or None if unknown""" + + name = None + try: + name = self._TYPE_NAMES[self.audioObjectType] + except IndexError: + pass + if name is None: + return + if self.sbrPresentFlag == 1: + name += "+SBR" + if self.psPresentFlag == 1: + name += "+PS" + return text_type(name) + + @property + def sample_rate(self): + """0 means unknown""" + + if self.sbrPresentFlag == 1: + return self.extensionSamplingFrequency + elif self.sbrPresentFlag == 0: + return self.samplingFrequency + else: + # these are all types that support SBR + aot_can_sbr = (1, 2, 3, 4, 6, 17, 19, 20, 22) + if self.audioObjectType not in aot_can_sbr: + return self.samplingFrequency + # there shouldn't be SBR for > 48KHz + if self.samplingFrequency > 24000: + return self.samplingFrequency + # either samplingFrequency or samplingFrequency * 2 + return 0 + + @property + def channels(self): + """channel count or 0 for unknown""" + + # from ProgramConfigElement() + if hasattr(self, "pce_channels"): + return self.pce_channels + + conf = getattr( + self, "extensionChannelConfiguration", self.channelConfiguration) + + if conf == 1: + if self.psPresentFlag == -1: + return 0 + elif self.psPresentFlag == 1: + return 2 + else: + return 1 + elif conf == 7: + return 8 + elif conf > 7: + return 0 + else: + return conf + + def _get_audio_object_type(self, r): + """Raises BitReaderError""" + + audioObjectType = r.bits(5) + if audioObjectType == 31: + audioObjectTypeExt = r.bits(6) + audioObjectType = 32 + audioObjectTypeExt + return audioObjectType + + def _get_sampling_freq(self, r): + """Raises BitReaderError""" + + samplingFrequencyIndex = r.bits(4) + if samplingFrequencyIndex == 0xf: + samplingFrequency = r.bits(24) + else: + try: + samplingFrequency = self._FREQS[samplingFrequencyIndex] + except IndexError: + samplingFrequency = 0 + return samplingFrequency + + def __init__(self, fileobj, length): + """Raises DescriptorError""" + + r = BitReader(fileobj) + try: + self._parse(r, length) + except BitReaderError as e: + raise DescriptorError(e) + + def _parse(self, r, length): + """Raises BitReaderError""" + + def bits_left(): + return length * 8 - r.get_position() + + self.audioObjectType = self._get_audio_object_type(r) + self.samplingFrequency = self._get_sampling_freq(r) + self.channelConfiguration = r.bits(4) + + self.sbrPresentFlag = -1 + self.psPresentFlag = -1 + if self.audioObjectType in (5, 29): + self.extensionAudioObjectType = 5 + self.sbrPresentFlag = 1 + if self.audioObjectType == 29: + self.psPresentFlag = 1 + self.extensionSamplingFrequency = self._get_sampling_freq(r) + self.audioObjectType = self._get_audio_object_type(r) + if self.audioObjectType == 22: + self.extensionChannelConfiguration = r.bits(4) + else: + self.extensionAudioObjectType = 0 + + if self.audioObjectType in (1, 2, 3, 4, 6, 7, 17, 19, 20, 21, 22, 23): + try: + GASpecificConfig(r, self) + except NotImplementedError: + # unsupported, (warn?) + return + else: + # unsupported + return + + if self.audioObjectType in ( + 17, 19, 20, 21, 22, 23, 24, 25, 26, 27, 39): + epConfig = r.bits(2) + if epConfig in (2, 3): + # unsupported + return + + if self.extensionAudioObjectType != 5 and bits_left() >= 16: + syncExtensionType = r.bits(11) + if syncExtensionType == 0x2b7: + self.extensionAudioObjectType = self._get_audio_object_type(r) + + if self.extensionAudioObjectType == 5: + self.sbrPresentFlag = r.bits(1) + if self.sbrPresentFlag == 1: + self.extensionSamplingFrequency = \ + self._get_sampling_freq(r) + if bits_left() >= 12: + syncExtensionType = r.bits(11) + if syncExtensionType == 0x548: + self.psPresentFlag = r.bits(1) + + if self.extensionAudioObjectType == 22: + self.sbrPresentFlag = r.bits(1) + if self.sbrPresentFlag == 1: + self.extensionSamplingFrequency = \ + self._get_sampling_freq(r) + self.extensionChannelConfiguration = r.bits(4) + + +def GASpecificConfig(r, info): + """Reads GASpecificConfig which is needed to get the data after that + (there is no length defined to skip it) and to read program_config_element + which can contain channel counts. + + May raise BitReaderError on error or + NotImplementedError if some reserved data was set. + """ + + assert isinstance(info, DecoderSpecificInfo) + + r.skip(1) # frameLengthFlag + dependsOnCoreCoder = r.bits(1) + if dependsOnCoreCoder: + r.skip(14) + extensionFlag = r.bits(1) + if not info.channelConfiguration: + pce = ProgramConfigElement(r) + info.pce_channels = pce.channels + if info.audioObjectType == 6 or info.audioObjectType == 20: + r.skip(3) + if extensionFlag: + if info.audioObjectType == 22: + r.skip(5 + 11) + if info.audioObjectType in (17, 19, 20, 23): + r.skip(1 + 1 + 1) + extensionFlag3 = r.bits(1) + if extensionFlag3 != 0: + raise NotImplementedError("extensionFlag3 set") diff --git a/libs/mutagen/mp4/_atom.py b/libs/mutagen/mp4/_atom.py new file mode 100644 index 00000000..f73eb556 --- /dev/null +++ b/libs/mutagen/mp4/_atom.py @@ -0,0 +1,194 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2006 Joe Wreschnig +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +import struct + +from mutagen._compat import PY2 + +# This is not an exhaustive list of container atoms, but just the +# ones this module needs to peek inside. +_CONTAINERS = [b"moov", b"udta", b"trak", b"mdia", b"meta", b"ilst", + b"stbl", b"minf", b"moof", b"traf"] +_SKIP_SIZE = {b"meta": 4} + + +class AtomError(Exception): + pass + + +class Atom(object): + """An individual atom. + + Attributes: + children -- list child atoms (or None for non-container atoms) + length -- length of this atom, including length and name + datalength = -- length of this atom without length, name + name -- four byte name of the atom, as a str + offset -- location in the constructor-given fileobj of this atom + + This structure should only be used internally by Mutagen. + """ + + children = None + + def __init__(self, fileobj, level=0): + """May raise AtomError""" + + self.offset = fileobj.tell() + try: + self.length, self.name = struct.unpack(">I4s", fileobj.read(8)) + except struct.error: + raise AtomError("truncated data") + self._dataoffset = self.offset + 8 + if self.length == 1: + try: + self.length, = struct.unpack(">Q", fileobj.read(8)) + except struct.error: + raise AtomError("truncated data") + self._dataoffset += 8 + if self.length < 16: + raise AtomError( + "64 bit atom length can only be 16 and higher") + elif self.length == 0: + if level != 0: + raise AtomError( + "only a top-level atom can have zero length") + # Only the last atom is supposed to have a zero-length, meaning it + # extends to the end of file. + fileobj.seek(0, 2) + self.length = fileobj.tell() - self.offset + fileobj.seek(self.offset + 8, 0) + elif self.length < 8: + raise AtomError( + "atom length can only be 0, 1 or 8 and higher") + + if self.name in _CONTAINERS: + self.children = [] + fileobj.seek(_SKIP_SIZE.get(self.name, 0), 1) + while fileobj.tell() < self.offset + self.length: + self.children.append(Atom(fileobj, level + 1)) + else: + fileobj.seek(self.offset + self.length, 0) + + @property + def datalength(self): + return self.length - (self._dataoffset - self.offset) + + def read(self, fileobj): + """Return if all data could be read and the atom payload""" + + fileobj.seek(self._dataoffset, 0) + data = fileobj.read(self.datalength) + return len(data) == self.datalength, data + + @staticmethod + def render(name, data): + """Render raw atom data.""" + # this raises OverflowError if Py_ssize_t can't handle the atom data + size = len(data) + 8 + if size <= 0xFFFFFFFF: + return struct.pack(">I4s", size, name) + data + else: + return struct.pack(">I4sQ", 1, name, size + 8) + data + + def findall(self, name, recursive=False): + """Recursively find all child atoms by specified name.""" + if self.children is not None: + for child in self.children: + if child.name == name: + yield child + if recursive: + for atom in child.findall(name, True): + yield atom + + def __getitem__(self, remaining): + """Look up a child atom, potentially recursively. + + e.g. atom['udta', 'meta'] => + """ + if not remaining: + return self + elif self.children is None: + raise KeyError("%r is not a container" % self.name) + for child in self.children: + if child.name == remaining[0]: + return child[remaining[1:]] + else: + raise KeyError("%r not found" % remaining[0]) + + def __repr__(self): + cls = self.__class__.__name__ + if self.children is None: + return "<%s name=%r length=%r offset=%r>" % ( + cls, self.name, self.length, self.offset) + else: + children = "\n".join([" " + line for child in self.children + for line in repr(child).splitlines()]) + return "<%s name=%r length=%r offset=%r\n%s>" % ( + cls, self.name, self.length, self.offset, children) + + +class Atoms(object): + """Root atoms in a given file. + + Attributes: + atoms -- a list of top-level atoms as Atom objects + + This structure should only be used internally by Mutagen. + """ + + def __init__(self, fileobj): + self.atoms = [] + fileobj.seek(0, 2) + end = fileobj.tell() + fileobj.seek(0) + while fileobj.tell() + 8 <= end: + self.atoms.append(Atom(fileobj)) + + def path(self, *names): + """Look up and return the complete path of an atom. + + For example, atoms.path('moov', 'udta', 'meta') will return a + list of three atoms, corresponding to the moov, udta, and meta + atoms. + """ + + path = [self] + for name in names: + path.append(path[-1][name, ]) + return path[1:] + + def __contains__(self, names): + try: + self[names] + except KeyError: + return False + return True + + def __getitem__(self, names): + """Look up a child atom. + + 'names' may be a list of atoms (['moov', 'udta']) or a string + specifying the complete path ('moov.udta'). + """ + + if PY2: + if isinstance(names, basestring): + names = names.split(b".") + else: + if isinstance(names, bytes): + names = names.split(b".") + + for child in self.atoms: + if child.name == names[0]: + return child[names[1:]] + else: + raise KeyError("%r not found" % names[0]) + + def __repr__(self): + return "\n".join([repr(child) for child in self.atoms]) diff --git a/libs/mutagen/mp4/_util.py b/libs/mutagen/mp4/_util.py new file mode 100644 index 00000000..9583334a --- /dev/null +++ b/libs/mutagen/mp4/_util.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014 Christoph Reiter +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from mutagen._util import cdata + + +def parse_full_atom(data): + """Some atoms are versioned. Split them up in (version, flags, payload). + Can raise ValueError. + """ + + if len(data) < 4: + raise ValueError("not enough data") + + version = ord(data[0:1]) + flags = cdata.uint_be(b"\x00" + data[1:4]) + return version, flags, data[4:] diff --git a/libs/mutagen/musepack.py b/libs/mutagen/musepack.py index 9804deb3..7880958b 100644 --- a/libs/mutagen/musepack.py +++ b/libs/mutagen/musepack.py @@ -1,7 +1,7 @@ -# A Musepack reader/tagger -# -# Copyright 2006 Lukas Lalinsky -# Copyright 2012 Christoph Reiter +# -*- coding: utf-8 -*- + +# Copyright (C) 2006 Lukas Lalinsky +# Copyright (C) 2012 Christoph Reiter # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -19,6 +19,8 @@ __all__ = ["Musepack", "Open", "delete"] import struct +from ._compat import endswith, xrange +from mutagen import StreamInfo from mutagen.apev2 import APEv2File, error, delete from mutagen.id3 import BitPaddedInt from mutagen._util import cdata @@ -46,8 +48,9 @@ def _parse_sv8_int(fileobj, limit=9): c = fileobj.read(1) if len(c) != 1: raise EOFError - num = (num << 7) | (ord(c) & 0x7F) - if not ord(c) & 0x80: + c = bytearray(c) + num = (num << 7) | (c[0] & 0x7F) + if not c[0] & 0x80: return num, i + 1 if limit > 0: raise ValueError @@ -63,7 +66,7 @@ def _calc_sv8_peak(peak): return (10 ** (peak / (256.0 * 20.0)) / 65535.0) -class MusepackInfo(object): +class MusepackInfo(StreamInfo): """Musepack stream information. Attributes: @@ -91,7 +94,7 @@ class MusepackInfo(object): raise MusepackHeaderError("not a Musepack file") # Skip ID3v2 tags - if header[:3] == "ID3": + if header[:3] == b"ID3": header = fileobj.read(6) if len(header) != 6: raise MusepackHeaderError("not a Musepack file") @@ -101,7 +104,7 @@ class MusepackInfo(object): if len(header) != 4: raise MusepackHeaderError("not a Musepack file") - if header.startswith("MPCK"): + if header.startswith(b"MPCK"): self.__parse_sv8(fileobj) else: self.__parse_sv467(fileobj) @@ -111,29 +114,31 @@ class MusepackInfo(object): self.bitrate = int(round(fileobj.tell() * 8 / self.length)) def __parse_sv8(self, fileobj): - #SV8 http://trac.musepack.net/trac/wiki/SV8Specification + # SV8 http://trac.musepack.net/trac/wiki/SV8Specification key_size = 2 - mandatory_packets = ["SH", "RG"] + mandatory_packets = [b"SH", b"RG"] def check_frame_key(key): - if len(frame_type) != key_size or not 'AA' <= frame_type <= 'ZZ': + if ((len(frame_type) != key_size) or + (not b'AA' <= frame_type <= b'ZZ')): raise MusepackHeaderError("Invalid frame key.") frame_type = fileobj.read(key_size) check_frame_key(frame_type) - while frame_type not in ("AP", "SE") and mandatory_packets: + while frame_type not in (b"AP", b"SE") and mandatory_packets: try: frame_size, slen = _parse_sv8_int(fileobj) except (EOFError, ValueError): raise MusepackHeaderError("Invalid packet size.") data_size = frame_size - key_size - slen + # packets can be at maximum data_size big and are padded with zeros - if frame_type == "SH": + if frame_type == b"SH": mandatory_packets.remove(frame_type) self.__parse_stream_header(fileobj, data_size) - elif frame_type == "RG": + elif frame_type == b"RG": mandatory_packets.remove(frame_type) self.__parse_replaygain_packet(fileobj, data_size) else: @@ -143,37 +148,43 @@ class MusepackInfo(object): check_frame_key(frame_type) if mandatory_packets: - raise MusepackHeaderError("Missing mandatory packets: %s." - % ", ".join(mandatory_packets)) + raise MusepackHeaderError("Missing mandatory packets: %s." % + ", ".join(map(repr, mandatory_packets))) self.length = float(self.samples) / self.sample_rate self.bitrate = 0 def __parse_stream_header(self, fileobj, data_size): + # skip CRC fileobj.seek(4, 1) + remaining_size = data_size - 4 + try: - self.version = ord(fileobj.read(1)) + self.version = bytearray(fileobj.read(1))[0] except TypeError: raise MusepackHeaderError("SH packet ended unexpectedly.") + + remaining_size -= 1 + try: samples, l1 = _parse_sv8_int(fileobj) samples_skip, l2 = _parse_sv8_int(fileobj) except (EOFError, ValueError): raise MusepackHeaderError( "SH packet: Invalid sample counts.") - left_size = data_size - 5 - l1 - l2 - if left_size != 2: - raise MusepackHeaderError("Invalid SH packet size.") - data = fileobj.read(left_size) - if len(data) != left_size: - raise MusepackHeaderError("SH packet ended unexpectedly.") - self.sample_rate = RATES[ord(data[-2]) >> 5] - self.channels = (ord(data[-1]) >> 4) + 1 + self.samples = samples - samples_skip + remaining_size -= l1 + l2 + + data = fileobj.read(remaining_size) + if len(data) != remaining_size: + raise MusepackHeaderError("SH packet ended unexpectedly.") + self.sample_rate = RATES[bytearray(data)[0] >> 5] + self.channels = (bytearray(data)[1] >> 4) + 1 def __parse_replaygain_packet(self, fileobj, data_size): data = fileobj.read(data_size) - if data_size != 9: + if data_size < 9: raise MusepackHeaderError("Invalid RG packet size.") if len(data) != data_size: raise MusepackHeaderError("RG packet ended unexpectedly.") @@ -197,8 +208,8 @@ class MusepackInfo(object): raise MusepackHeaderError("not a Musepack file") # SV7 - if header.startswith("MP+"): - self.version = ord(header[3]) & 0xF + if header.startswith(b"MP+"): + self.version = bytearray(header)[3] & 0xF if self.version < 7: raise MusepackHeaderError("not a Musepack file") frames = cdata.uint_le(header[4:8]) @@ -235,12 +246,12 @@ class MusepackInfo(object): def pprint(self): rg_data = [] if hasattr(self, "title_gain"): - rg_data.append("%+0.2f (title)" % self.title_gain) + rg_data.append(u"%+0.2f (title)" % self.title_gain) if hasattr(self, "album_gain"): - rg_data.append("%+0.2f (album)" % self.album_gain) + rg_data.append(u"%+0.2f (album)" % self.album_gain) rg_data = (rg_data and ", Gain: " + ", ".join(rg_data)) or "" - return "Musepack SV%d, %.2f seconds, %d Hz, %d bps%s" % ( + return u"Musepack SV%d, %.2f seconds, %d Hz, %d bps%s" % ( self.version, self.length, self.sample_rate, self.bitrate, rg_data) @@ -250,8 +261,10 @@ class Musepack(APEv2File): @staticmethod def score(filename, fileobj, header): - return (header.startswith("MP+") + header.startswith("MPCK") + - filename.lower().endswith(".mpc")) + filename = filename.lower() + + return (header.startswith(b"MP+") + header.startswith(b"MPCK") + + endswith(filename, b".mpc")) Open = Musepack diff --git a/libs/mutagen/ogg.py b/libs/mutagen/ogg.py index 657eb7f7..9961a966 100644 --- a/libs/mutagen/ogg.py +++ b/libs/mutagen/ogg.py @@ -1,4 +1,6 @@ -# Copyright 2006 Joe Wreschnig +# -*- coding: utf-8 -*- + +# Copyright (C) 2006 Joe Wreschnig # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -18,13 +20,12 @@ import struct import sys import zlib -from cStringIO import StringIO - from mutagen import FileType -from mutagen._util import cdata, insert_bytes, delete_bytes +from mutagen._util import cdata, resize_bytes, MutagenError +from ._compat import cBytesIO, reraise, chr_, izip, xrange -class error(IOError): +class error(IOError, MutagenError): """Ogg stream parsing errors.""" pass @@ -59,7 +60,7 @@ class OggPage(object): version = 0 __type_flags = 0 - position = 0L + position = 0 serial = 0 sequence = 0 offset = None @@ -78,15 +79,15 @@ class OggPage(object): raise EOFError try: - (oggs, self.version, self.__type_flags, self.position, - self.serial, self.sequence, crc, segments) = struct.unpack( - "<4sBBqIIiB", header) + (oggs, self.version, self.__type_flags, + self.position, self.serial, self.sequence, + crc, segments) = struct.unpack("<4sBBqIIiB", header) except struct.error: raise error("unable to read full header; got %r" % header) - if oggs != "OggS": + if oggs != b"OggS": raise error("read %r, expected %r, at 0x%x" % ( - oggs, "OggS", fileobj.tell() - 27)) + oggs, b"OggS", fileobj.tell() - 27)) if self.version != 0: raise error("version %r unsupported" % self.version) @@ -96,7 +97,7 @@ class OggPage(object): lacing_bytes = fileobj.read(segments) if len(lacing_bytes) != segments: raise error("unable to read %r lacing bytes" % segments) - for c in map(ord, lacing_bytes): + for c in bytearray(lacing_bytes): total += c if c < 255: lacings.append(total) @@ -105,8 +106,8 @@ class OggPage(object): lacings.append(total) self.complete = False - self.packets = map(fileobj.read, lacings) - if map(len, self.packets) != lacings: + self.packets = [fileobj.read(l) for l in lacings] + if [len(p) for p in self.packets] != lacings: raise error("unable to read full data") def __eq__(self, other): @@ -134,21 +135,21 @@ class OggPage(object): """ data = [ - struct.pack("<4sBBqIIi", "OggS", self.version, self.__type_flags, + struct.pack("<4sBBqIIi", b"OggS", self.version, self.__type_flags, self.position, self.serial, self.sequence, 0) ] lacing_data = [] for datum in self.packets: quot, rem = divmod(len(datum), 255) - lacing_data.append("\xff" * quot + chr(rem)) - lacing_data = "".join(lacing_data) - if not self.complete and lacing_data.endswith("\x00"): + lacing_data.append(b"\xff" * quot + chr_(rem)) + lacing_data = b"".join(lacing_data) + if not self.complete and lacing_data.endswith(b"\x00"): lacing_data = lacing_data[:-1] - data.append(chr(len(lacing_data))) + data.append(chr_(len(lacing_data))) data.append(lacing_data) data.extend(self.packets) - data = "".join(data) + data = b"".join(data) # Python's CRC is swapped relative to Ogg's needs. # crc32 returns uint prior to py2.6 on some platforms, so force uint @@ -196,8 +197,8 @@ class OggPage(object): lambda self, v: self.__set_flag(2, v), doc="This is the last page of a logical bitstream.") - @classmethod - def renumber(klass, fileobj, serial, start): + @staticmethod + def renumber(fileobj, serial, start): """Renumber pages belonging to a specified logical stream. fileobj must be opened with mode r+b or w+b. @@ -235,8 +236,8 @@ class OggPage(object): fileobj.seek(page.offset + page.size, 0) number += 1 - @classmethod - def to_packets(klass, pages, strict=False): + @staticmethod + def to_packets(pages, strict=False): """Construct a list of packet data from a list of Ogg pages. If strict is true, the first page must start a new packet, @@ -253,7 +254,7 @@ class OggPage(object): if not pages[-1].complete: raise ValueError("last packet does not complete") elif pages and pages[0].continued: - packets.append([""]) + packets.append([b""]) for page in pages: if serial != page.serial: @@ -267,13 +268,46 @@ class OggPage(object): packets[-1].append(page.packets[0]) else: packets.append([page.packets[0]]) - packets.extend([[p] for p in page.packets[1:]]) + packets.extend([p] for p in page.packets[1:]) - return ["".join(p) for p in packets] + return [b"".join(p) for p in packets] @classmethod - def from_packets(klass, packets, sequence=0, - default_size=4096, wiggle_room=2048): + def _from_packets_try_preserve(cls, packets, old_pages): + """Like from_packets but in case the size and number of the packets + is the same as in the given pages the layout of the pages will + be copied (the page size and number will match). + + If the packets don't match this behaves like:: + + OggPage.from_packets(packets, sequence=old_pages[0].sequence) + """ + + old_packets = cls.to_packets(old_pages) + + if [len(p) for p in packets] != [len(p) for p in old_packets]: + # doesn't match, fall back + return cls.from_packets(packets, old_pages[0].sequence) + + new_data = b"".join(packets) + new_pages = [] + for old in old_pages: + new = OggPage() + new.sequence = old.sequence + new.complete = old.complete + new.continued = old.continued + new.position = old.position + for p in old.packets: + data, new_data = new_data[:len(p)], new_data[len(p):] + new.packets.append(data) + new_pages.append(new) + assert not new_data + + return new_pages + + @staticmethod + def from_packets(packets, sequence=0, default_size=4096, + wiggle_room=2048): """Construct a list of Ogg pages from a list of packet data. The algorithm will generate pages of approximately @@ -300,7 +334,7 @@ class OggPage(object): page.sequence = sequence for packet in packets: - page.packets.append("") + page.packets.append(b"") while packet: data, packet = packet[:chunk_size], packet[chunk_size:] if page.size < default_size and len(page.packets) < 255: @@ -314,7 +348,7 @@ class OggPage(object): if page.packets[-1]: page.complete = False if len(page.packets) == 1: - page.position = -1L + page.position = -1 else: page.packets.pop(-1) pages.append(page) @@ -325,7 +359,7 @@ class OggPage(object): if len(packet) < wiggle_room: page.packets[-1] += packet - packet = "" + packet = b"" if page.packets: pages.append(page) @@ -333,7 +367,7 @@ class OggPage(object): return pages @classmethod - def replace(klass, fileobj, old_pages, new_pages): + def replace(cls, fileobj, old_pages, new_pages): """Replace old_pages with new_pages within fileobj. old_pages must have come from reading fileobj originally. @@ -345,9 +379,13 @@ class OggPage(object): such, it must be opened r+b or w+b. """ + if not len(old_pages) or not len(new_pages): + raise ValueError("empty pages list not allowed") + # Number the new pages starting from the first old page. first = old_pages[0].sequence - for page, seq in zip(new_pages, range(first, first + len(new_pages))): + for page, seq in izip(new_pages, + xrange(first, first + len(new_pages))): page.sequence = seq page.serial = old_pages[0].serial @@ -359,26 +397,30 @@ class OggPage(object): new_pages[-1].last = old_pages[-1].last new_pages[-1].complete = old_pages[-1].complete if not new_pages[-1].complete and len(new_pages[-1].packets) == 1: - new_pages[-1].position = -1L + new_pages[-1].position = -1 - new_data = "".join(map(klass.write, new_pages)) + new_data = [cls.write(p) for p in new_pages] - # Make room in the file for the new data. - delta = len(new_data) - fileobj.seek(old_pages[0].offset, 0) - insert_bytes(fileobj, delta, old_pages[0].offset) - fileobj.seek(old_pages[0].offset, 0) - fileobj.write(new_data) - new_data_end = old_pages[0].offset + delta + # Add dummy data or merge the remaining data together so multiple + # new pages replace an old one + pages_diff = len(old_pages) - len(new_data) + if pages_diff > 0: + new_data.extend([b""] * pages_diff) + elif pages_diff < 0: + new_data[pages_diff - 1:] = [b"".join(new_data[pages_diff - 1:])] - # Go through the old pages and delete them. Since we shifted - # the data down the file, we need to adjust their offsets. We - # also need to go backwards, so we don't adjust the deltas of - # the other pages. - old_pages.reverse() - for old_page in old_pages: - adj_offset = old_page.offset + delta - delete_bytes(fileobj, old_page.size, adj_offset) + # Replace pages one by one. If the sizes match no resize happens. + offset_adjust = 0 + new_data_end = None + assert len(old_pages) == len(new_data) + for old_page, data in izip(old_pages, new_data): + offset = old_page.offset + offset_adjust + data_size = len(data) + resize_bytes(fileobj, old_page.size, data_size, offset) + fileobj.seek(offset, 0) + fileobj.write(data) + new_data_end = offset + data_size + offset_adjust += (data_size - old_page.size) # Finally, if there's any discrepency in length, we need to # renumber the pages for the logical stream. @@ -386,10 +428,10 @@ class OggPage(object): fileobj.seek(new_data_end, 0) serial = new_pages[-1].serial sequence = new_pages[-1].sequence + 1 - klass.renumber(fileobj, serial, sequence) + cls.renumber(fileobj, serial, sequence) - @classmethod - def find_last(klass, fileobj, serial): + @staticmethod + def find_last(fileobj, serial): """Find the last page of the stream 'serial'. If the file is not multiplexed this function is fast. If it is, @@ -401,19 +443,19 @@ class OggPage(object): # For non-muxed streams, look at the last page. try: - fileobj.seek(-256*256, 2) + fileobj.seek(-256 * 256, 2) except IOError: # The file is less than 64k in length. fileobj.seek(0) data = fileobj.read() try: - index = data.rindex("OggS") + index = data.rindex(b"OggS") except ValueError: raise error("unable to find final Ogg header") - stringobj = StringIO(data[index:]) + bytesobj = cBytesIO(data[index:]) best_page = None try: - page = OggPage(stringobj) + page = OggPage(bytesobj) except error: pass else: @@ -453,18 +495,15 @@ class OggFileType(FileType): """Load file information from a filename.""" self.filename = filename - fileobj = open(filename, "rb") - try: + with open(filename, "rb") as fileobj: try: self.info = self._Info(fileobj) self.tags = self._Tags(fileobj, self.info) self.info._post_tags(fileobj) - except error, e: - raise self._Error, e, sys.exc_info()[2] + except error as e: + reraise(self._Error, e, sys.exc_info()[2]) except EOFError: - raise self._Error, "no appropriate stream found" - finally: - fileobj.close() + raise self._Error("no appropriate stream found") def delete(self, filename=None): """Remove tags from a file. @@ -476,18 +515,20 @@ class OggFileType(FileType): filename = self.filename self.tags.clear() - fileobj = open(filename, "rb+") - try: + # TODO: we should delegate the deletion to the subclass and not through + # _inject. + with open(filename, "rb+") as fileobj: try: - self.tags._inject(fileobj) - except error, e: - raise self._Error, e, sys.exc_info()[2] + self.tags._inject(fileobj, lambda x: 0) + except error as e: + reraise(self._Error, e, sys.exc_info()[2]) except EOFError: - raise self._Error, "no appropriate stream found" - finally: - fileobj.close() + raise self._Error("no appropriate stream found") - def save(self, filename=None): + def add_tags(self): + raise self._Error + + def save(self, filename=None, padding=None): """Save a tag to a file. If no filename is given, the one most recently loaded is used. @@ -498,10 +539,10 @@ class OggFileType(FileType): fileobj = open(filename, "rb+") try: try: - self.tags._inject(fileobj) - except error, e: - raise self._Error, e, sys.exc_info()[2] + self.tags._inject(fileobj, padding) + except error as e: + reraise(self._Error, e, sys.exc_info()[2]) except EOFError: - raise self._Error, "no appropriate stream found" + raise self._Error("no appropriate stream found") finally: fileobj.close() diff --git a/libs/mutagen/oggflac.py b/libs/mutagen/oggflac.py index 14ecec00..b86226ca 100644 --- a/libs/mutagen/oggflac.py +++ b/libs/mutagen/oggflac.py @@ -1,6 +1,6 @@ -# Ogg FLAC support. -# -# Copyright 2006 Joe Wreschnig +# -*- coding: utf-8 -*- + +# Copyright (C) 2006 Joe Wreschnig # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -19,9 +19,11 @@ __all__ = ["OggFLAC", "Open", "delete"] import struct -from cStringIO import StringIO +from ._compat import cBytesIO -from mutagen.flac import StreamInfo, VCFLACDict, StrictFileObject +from mutagen import StreamInfo +from mutagen.flac import StreamInfo as FLACStreamInfo, error as FLACError +from mutagen._vorbis import VCommentDict from mutagen.ogg import OggPage, OggFileType, error as OggError @@ -34,31 +36,24 @@ class OggFLACHeaderError(error): class OggFLACStreamInfo(StreamInfo): - """Ogg FLAC general header and stream info. + """Ogg FLAC stream info.""" - This encompasses the Ogg wrapper for the FLAC STREAMINFO metadata - block, as well as the Ogg codec setup that precedes it. + length = 0 + """File length in seconds, as a float""" - Attributes (in addition to StreamInfo's): + channels = 0 + """Number of channels""" - * packets -- number of metadata packets - * serial -- Ogg logical stream serial number - """ + sample_rate = 0 + """Sample rate in Hz""" - packets = 0 - serial = 0 - - def load(self, data): - # Ogg expects file objects that don't raise on read - if isinstance(data, StrictFileObject): - data = data._fileobj - - page = OggPage(data) - while not page.packets[0].startswith("\x7FFLAC"): - page = OggPage(data) + def __init__(self, fileobj): + page = OggPage(fileobj) + while not page.packets[0].startswith(b"\x7FFLAC"): + page = OggPage(fileobj) major, minor, self.packets, flac = struct.unpack( ">BBH4s", page.packets[0][5:13]) - if flac != "fLaC": + if flac != b"fLaC": raise OggFLACHeaderError("invalid FLAC marker (%r)" % flac) elif (major, minor) != (1, 0): raise OggFLACHeaderError( @@ -66,8 +61,16 @@ class OggFLACStreamInfo(StreamInfo): self.serial = page.serial # Skip over the block header. - stringobj = StrictFileObject(StringIO(page.packets[0][17:])) - super(OggFLACStreamInfo, self).load(stringobj) + stringobj = cBytesIO(page.packets[0][17:]) + + try: + flac_info = FLACStreamInfo(stringobj) + except FLACError as e: + raise OggFLACHeaderError(e) + + for attr in ["min_blocksize", "max_blocksize", "sample_rate", + "channels", "bits_per_sample", "total_samples", "length"]: + setattr(self, attr, getattr(flac_info, attr)) def _post_tags(self, fileobj): if self.length: @@ -76,31 +79,33 @@ class OggFLACStreamInfo(StreamInfo): self.length = page.position / float(self.sample_rate) def pprint(self): - return "Ogg " + super(OggFLACStreamInfo, self).pprint() + return u"Ogg FLAC, %.2f seconds, %d Hz" % ( + self.length, self.sample_rate) -class OggFLACVComment(VCFLACDict): - def load(self, data, info, errors='replace'): +class OggFLACVComment(VCommentDict): + + def __init__(self, fileobj, info): # data should be pointing at the start of an Ogg page, after # the first FLAC page. pages = [] complete = False while not complete: - page = OggPage(data) + page = OggPage(fileobj) if page.serial == info.serial: pages.append(page) complete = page.complete or (len(page.packets) > 1) - comment = StringIO(OggPage.to_packets(pages)[0][4:]) - super(OggFLACVComment, self).load(comment, errors=errors) + comment = cBytesIO(OggPage.to_packets(pages)[0][4:]) + super(OggFLACVComment, self).__init__(comment, framing=False) - def _inject(self, fileobj): + def _inject(self, fileobj, padding_func): """Write tag data into the FLAC Vorbis comment packet/page.""" # Ogg FLAC has no convenient data marker like Vorbis, but the # second packet - and second page - must be the comment data. fileobj.seek(0) page = OggPage(fileobj) - while not page.packets[0].startswith("\x7FFLAC"): + while not page.packets[0].startswith(b"\x7FFLAC"): page = OggPage(fileobj) first_page = page @@ -116,8 +121,8 @@ class OggFLACVComment(VCFLACDict): packets = OggPage.to_packets(old_pages, strict=False) # Set the new comment block. - data = self.write() - data = packets[0][0] + struct.pack(">I", len(data))[-3:] + data + data = self.write(framing=False) + data = packets[0][:1] + struct.pack(">I", len(data))[-3:] + data packets[0] = data new_pages = OggPage.from_packets(packets, old_pages[0].sequence) @@ -132,10 +137,19 @@ class OggFLAC(OggFileType): _Error = OggFLACHeaderError _mimes = ["audio/x-oggflac"] + info = None + """A `OggFLACStreamInfo`""" + + tags = None + """A `VCommentDict`""" + + def save(self, filename=None): + return super(OggFLAC, self).save(filename) + @staticmethod def score(filename, fileobj, header): - return (header.startswith("OggS") * ( - ("FLAC" in header) + ("fLaC" in header))) + return (header.startswith(b"OggS") * ( + (b"FLAC" in header) + (b"fLaC" in header))) Open = OggFLAC diff --git a/libs/mutagen/oggopus.py b/libs/mutagen/oggopus.py index 6de44391..7154e479 100644 --- a/libs/mutagen/oggopus.py +++ b/libs/mutagen/oggopus.py @@ -1,4 +1,6 @@ -# Copyright 2012 Christoph Reiter +# -*- coding: utf-8 -*- + +# Copyright (C) 2012, 2013 Christoph Reiter # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -16,6 +18,10 @@ __all__ = ["OggOpus", "Open", "delete"] import struct +from mutagen import StreamInfo +from mutagen._compat import BytesIO +from mutagen._util import get_size +from mutagen._tags import PaddingInfo from mutagen._vorbis import VCommentDict from mutagen.ogg import OggPage, OggFileType, error as OggError @@ -28,20 +34,18 @@ class OggOpusHeaderError(error): pass -class OggOpusInfo(object): - """Ogg Opus stream information. - - Attributes: - - * length - file length in seconds, as a float - * channels - number of channels - """ +class OggOpusInfo(StreamInfo): + """Ogg Opus stream information.""" length = 0 + """File length in seconds, as a float""" + + channels = 0 + """Number of channels""" def __init__(self, fileobj): page = OggPage(fileobj) - while not page.packets[0].startswith("OpusHead"): + while not page.packets[0].startswith(b"OpusHead"): page = OggPage(fileobj) self.serial = page.serial @@ -56,7 +60,7 @@ class OggOpusInfo(object): self.__pre_skip = pre_skip # only the higher 4 bits change on incombatible changes - major, minor = version >> 4, version & 0xF + major = version >> 4 if major != 0: raise OggOpusHeaderError("version %r unsupported" % major) @@ -65,7 +69,7 @@ class OggOpusInfo(object): self.length = (page.position - self.__pre_skip) / float(48000) def pprint(self): - return "Ogg Opus, %.2f seconds" % (self.length) + return u"Ogg Opus, %.2f seconds" % (self.length) class OggOpusVComment(VCommentDict): @@ -74,8 +78,8 @@ class OggOpusVComment(VCommentDict): def __get_comment_pages(self, fileobj, info): # find the first tags page with the right serial page = OggPage(fileobj) - while info.serial != page.serial or \ - not page.packets[0].startswith("OpusTags"): + while ((info.serial != page.serial) or + not page.packets[0].startswith(b"OpusTags")): page = OggPage(fileobj) # get all comment pages @@ -90,16 +94,39 @@ class OggOpusVComment(VCommentDict): def __init__(self, fileobj, info): pages = self.__get_comment_pages(fileobj, info) data = OggPage.to_packets(pages)[0][8:] # Strip OpusTags - super(OggOpusVComment, self).__init__(data, framing=False) + fileobj = BytesIO(data) + super(OggOpusVComment, self).__init__(fileobj, framing=False) + self._padding = len(data) - self._size - def _inject(self, fileobj): + # in case the LSB of the first byte after v-comment is 1, preserve the + # following data + padding_flag = fileobj.read(1) + if padding_flag and ord(padding_flag) & 0x1: + self._pad_data = padding_flag + fileobj.read() + self._padding = 0 # we have to preserve, so no padding + else: + self._pad_data = b"" + + def _inject(self, fileobj, padding_func): fileobj.seek(0) info = OggOpusInfo(fileobj) old_pages = self.__get_comment_pages(fileobj, info) packets = OggPage.to_packets(old_pages) - packets[0] = "OpusTags" + self.write(framing=False) - new_pages = OggPage.from_packets(packets, old_pages[0].sequence) + vcomment_data = b"OpusTags" + self.write(framing=False) + + if self._pad_data: + # if we have padding data to preserver we can't add more padding + # as long as we don't know the structure of what follows + packets[0] = vcomment_data + self._pad_data + else: + content_size = get_size(fileobj) - len(packets[0]) # approx + padding_left = len(packets[0]) - len(vcomment_data) + info = PaddingInfo(padding_left, content_size) + new_padding = info._get_padding(padding_func) + packets[0] = vcomment_data + b"\x00" * new_padding + + new_pages = OggPage._from_packets_try_preserve(packets, old_pages) OggPage.replace(fileobj, old_pages, new_pages) @@ -111,9 +138,15 @@ class OggOpus(OggFileType): _Error = OggOpusHeaderError _mimes = ["audio/ogg", "audio/ogg; codecs=opus"] + info = None + """A `OggOpusInfo`""" + + tags = None + """A `VCommentDict`""" + @staticmethod def score(filename, fileobj, header): - return (header.startswith("OggS") * ("OpusHead" in header)) + return (header.startswith(b"OggS") * (b"OpusHead" in header)) Open = OggOpus diff --git a/libs/mutagen/oggspeex.py b/libs/mutagen/oggspeex.py index 4f208521..9b16930b 100644 --- a/libs/mutagen/oggspeex.py +++ b/libs/mutagen/oggspeex.py @@ -1,5 +1,5 @@ -# Ogg Speex support. -# +# -*- coding: utf-8 -*- + # Copyright 2006 Joe Wreschnig # # This program is free software; you can redistribute it and/or modify @@ -19,9 +19,11 @@ http://lists.xiph.org/pipermail/speex-dev/2006-July/004676.html. __all__ = ["OggSpeex", "Open", "delete"] +from mutagen import StreamInfo from mutagen._vorbis import VCommentDict from mutagen.ogg import OggPage, OggFileType, error as OggError -from mutagen._util import cdata +from mutagen._util import cdata, get_size +from mutagen._tags import PaddingInfo class error(OggError): @@ -32,24 +34,25 @@ class OggSpeexHeaderError(error): pass -class OggSpeexInfo(object): - """Ogg Speex stream information. +class OggSpeexInfo(StreamInfo): + """Ogg Speex stream information.""" - Attributes: + length = 0 + """file length in seconds, as a float""" - * bitrate - nominal bitrate in bits per second - * channels - number of channels - * length - file length in seconds, as a float + channels = 0 + """number of channels""" + + bitrate = 0 + """nominal bitrate in bits per second. The reference encoder does not set the bitrate; in this case, the bitrate will be 0. """ - length = 0 - def __init__(self, fileobj): page = OggPage(fileobj) - while not page.packets[0].startswith("Speex "): + while not page.packets[0].startswith(b"Speex "): page = OggPage(fileobj) if not page.first: raise OggSpeexHeaderError( @@ -64,7 +67,7 @@ class OggSpeexInfo(object): self.length = page.position / float(self.sample_rate) def pprint(self): - return "Ogg Speex, %.2f seconds" % self.length + return u"Ogg Speex, %.2f seconds" % self.length class OggSpeexVComment(VCommentDict): @@ -78,10 +81,11 @@ class OggSpeexVComment(VCommentDict): if page.serial == info.serial: pages.append(page) complete = page.complete or (len(page.packets) > 1) - data = OggPage.to_packets(pages)[0] + "\x01" + data = OggPage.to_packets(pages)[0] super(OggSpeexVComment, self).__init__(data, framing=False) + self._padding = len(data) - self._size - def _inject(self, fileobj): + def _inject(self, fileobj, padding_func): """Write tag data into the Speex comment packet/page.""" fileobj.seek(0) @@ -89,7 +93,7 @@ class OggSpeexVComment(VCommentDict): # Find the first header page, with the stream info. # Use it to get the serial number. page = OggPage(fileobj) - while not page.packets[0].startswith("Speex "): + while not page.packets[0].startswith(b"Speex "): page = OggPage(fileobj) # Look for the next page with that serial number, it'll start @@ -108,10 +112,17 @@ class OggSpeexVComment(VCommentDict): packets = OggPage.to_packets(old_pages, strict=False) - # Set the new comment packet. - packets[0] = self.write(framing=False) + content_size = get_size(fileobj) - len(packets[0]) # approx + vcomment_data = self.write(framing=False) + padding_left = len(packets[0]) - len(vcomment_data) - new_pages = OggPage.from_packets(packets, old_pages[0].sequence) + info = PaddingInfo(padding_left, content_size) + new_padding = info._get_padding(padding_func) + + # Set the new comment packet. + packets[0] = vcomment_data + b"\x00" * new_padding + + new_pages = OggPage._from_packets_try_preserve(packets, old_pages) OggPage.replace(fileobj, old_pages, new_pages) @@ -123,9 +134,15 @@ class OggSpeex(OggFileType): _Error = OggSpeexHeaderError _mimes = ["audio/x-speex"] + info = None + """A `OggSpeexInfo`""" + + tags = None + """A `VCommentDict`""" + @staticmethod def score(filename, fileobj, header): - return (header.startswith("OggS") * ("Speex " in header)) + return (header.startswith(b"OggS") * (b"Speex " in header)) Open = OggSpeex diff --git a/libs/mutagen/oggtheora.py b/libs/mutagen/oggtheora.py index edf221a7..122e7d4b 100644 --- a/libs/mutagen/oggtheora.py +++ b/libs/mutagen/oggtheora.py @@ -1,5 +1,5 @@ -# Ogg Theora support. -# +# -*- coding: utf-8 -*- + # Copyright 2006 Joe Wreschnig # # This program is free software; you can redistribute it and/or modify @@ -18,8 +18,10 @@ __all__ = ["OggTheora", "Open", "delete"] import struct +from mutagen import StreamInfo from mutagen._vorbis import VCommentDict -from mutagen._util import cdata +from mutagen._util import cdata, get_size +from mutagen._tags import PaddingInfo from mutagen.ogg import OggPage, OggFileType, error as OggError @@ -31,20 +33,21 @@ class OggTheoraHeaderError(error): pass -class OggTheoraInfo(object): - """Ogg Theora stream information. - - Attributes: - - * length - file length in seconds, as a float - * fps - video frames per second, as a float - """ +class OggTheoraInfo(StreamInfo): + """Ogg Theora stream information.""" length = 0 + """File length in seconds, as a float""" + + fps = 0 + """Video frames per second, as a float""" + + bitrate = 0 + """Bitrate in bps (int)""" def __init__(self, fileobj): page = OggPage(fileobj) - while not page.packets[0].startswith("\x80theora"): + while not page.packets[0].startswith(b"\x80theora"): page = OggPage(fileobj) if not page.first: raise OggTheoraHeaderError( @@ -56,7 +59,7 @@ class OggTheoraInfo(object): "found Theora version %d.%d != 3.2" % (vmaj, vmin)) fps_num, fps_den = struct.unpack(">2I", data[22:30]) self.fps = fps_num / float(fps_den) - self.bitrate = cdata.uint_be("\x00" + data[37:40]) + self.bitrate = cdata.uint_be(b"\x00" + data[37:40]) self.granule_shift = (cdata.ushort_be(data[40:42]) >> 5) & 0x1F self.serial = page.serial @@ -68,7 +71,8 @@ class OggTheoraInfo(object): self.length = frames / float(self.fps) def pprint(self): - return "Ogg Theora, %.2f seconds, %d bps" % (self.length, self.bitrate) + return u"Ogg Theora, %.2f seconds, %d bps" % (self.length, + self.bitrate) class OggTheoraCommentDict(VCommentDict): @@ -83,14 +87,15 @@ class OggTheoraCommentDict(VCommentDict): pages.append(page) complete = page.complete or (len(page.packets) > 1) data = OggPage.to_packets(pages)[0][7:] - super(OggTheoraCommentDict, self).__init__(data + "\x01") + super(OggTheoraCommentDict, self).__init__(data, framing=False) + self._padding = len(data) - self._size - def _inject(self, fileobj): + def _inject(self, fileobj, padding_func): """Write tag data into the Theora comment packet/page.""" fileobj.seek(0) page = OggPage(fileobj) - while not page.packets[0].startswith("\x81theora"): + while not page.packets[0].startswith(b"\x81theora"): page = OggPage(fileobj) old_pages = [page] @@ -101,9 +106,16 @@ class OggTheoraCommentDict(VCommentDict): packets = OggPage.to_packets(old_pages, strict=False) - packets[0] = "\x81theora" + self.write(framing=False) + content_size = get_size(fileobj) - len(packets[0]) # approx + vcomment_data = b"\x81theora" + self.write(framing=False) + padding_left = len(packets[0]) - len(vcomment_data) - new_pages = OggPage.from_packets(packets, old_pages[0].sequence) + info = PaddingInfo(padding_left, content_size) + new_padding = info._get_padding(padding_func) + + packets[0] = vcomment_data + b"\x00" * new_padding + + new_pages = OggPage._from_packets_try_preserve(packets, old_pages) OggPage.replace(fileobj, old_pages, new_pages) @@ -115,10 +127,16 @@ class OggTheora(OggFileType): _Error = OggTheoraHeaderError _mimes = ["video/x-theora"] + info = None + """A `OggTheoraInfo`""" + + tags = None + """A `VCommentDict`""" + @staticmethod def score(filename, fileobj, header): - return (header.startswith("OggS") * - (("\x80theora" in header) + ("\x81theora" in header))) + return (header.startswith(b"OggS") * + ((b"\x80theora" in header) + (b"\x81theora" in header)) * 2) Open = OggTheora diff --git a/libs/mutagen/oggvorbis.py b/libs/mutagen/oggvorbis.py index 509fd966..b058a0c1 100644 --- a/libs/mutagen/oggvorbis.py +++ b/libs/mutagen/oggvorbis.py @@ -1,5 +1,5 @@ -# Ogg Vorbis support. -# +# -*- coding: utf-8 -*- + # Copyright 2006 Joe Wreschnig # # This program is free software; you can redistribute it and/or modify @@ -19,7 +19,10 @@ __all__ = ["OggVorbis", "Open", "delete"] import struct +from mutagen import StreamInfo from mutagen._vorbis import VCommentDict +from mutagen._util import get_size +from mutagen._tags import PaddingInfo from mutagen.ogg import OggPage, OggFileType, error as OggError @@ -31,20 +34,24 @@ class OggVorbisHeaderError(error): pass -class OggVorbisInfo(object): - """Ogg Vorbis stream information. - - Attributes: - - * length - file length in seconds, as a float - * bitrate - nominal ('average') bitrate in bits per second, as an int - """ +class OggVorbisInfo(StreamInfo): + """Ogg Vorbis stream information.""" length = 0 + """File length in seconds, as a float""" + + channels = 0 + """Number of channels""" + + bitrate = 0 + """Nominal ('average') bitrate in bits per second, as an int""" + + sample_rate = 0 + """Sample rate in Hz""" def __init__(self, fileobj): page = OggPage(fileobj) - while not page.packets[0].startswith("\x01vorbis"): + while not page.packets[0].startswith(b"\x01vorbis"): page = OggPage(fileobj) if not page.first: raise OggVorbisHeaderError( @@ -73,7 +80,8 @@ class OggVorbisInfo(object): self.length = page.position / float(self.sample_rate) def pprint(self): - return "Ogg Vorbis, %.2f seconds, %d bps" % (self.length, self.bitrate) + return u"Ogg Vorbis, %.2f seconds, %d bps" % ( + self.length, self.bitrate) class OggVCommentDict(VCommentDict): @@ -89,15 +97,16 @@ class OggVCommentDict(VCommentDict): complete = page.complete or (len(page.packets) > 1) data = OggPage.to_packets(pages)[0][7:] # Strip off "\x03vorbis". super(OggVCommentDict, self).__init__(data) + self._padding = len(data) - self._size - def _inject(self, fileobj): + def _inject(self, fileobj, padding_func): """Write tag data into the Vorbis comment packet/page.""" # Find the old pages in the file; we'll need to remove them, # plus grab any stray setup packet data out of them. fileobj.seek(0) page = OggPage(fileobj) - while not page.packets[0].startswith("\x03vorbis"): + while not page.packets[0].startswith(b"\x03vorbis"): page = OggPage(fileobj) old_pages = [page] @@ -108,10 +117,17 @@ class OggVCommentDict(VCommentDict): packets = OggPage.to_packets(old_pages, strict=False) - # Set the new comment packet. - packets[0] = "\x03vorbis" + self.write() + content_size = get_size(fileobj) - len(packets[0]) # approx + vcomment_data = b"\x03vorbis" + self.write() + padding_left = len(packets[0]) - len(vcomment_data) - new_pages = OggPage.from_packets(packets, old_pages[0].sequence) + info = PaddingInfo(padding_left, content_size) + new_padding = info._get_padding(padding_func) + + # Set the new comment packet. + packets[0] = vcomment_data + b"\x00" * new_padding + + new_pages = OggPage._from_packets_try_preserve(packets, old_pages) OggPage.replace(fileobj, old_pages, new_pages) @@ -123,9 +139,15 @@ class OggVorbis(OggFileType): _Error = OggVorbisHeaderError _mimes = ["audio/vorbis", "audio/x-vorbis"] + info = None + """A `OggVorbisInfo`""" + + tags = None + """A `VCommentDict`""" + @staticmethod def score(filename, fileobj, header): - return (header.startswith("OggS") * ("\x01vorbis" in header)) + return (header.startswith(b"OggS") * (b"\x01vorbis" in header)) Open = OggVorbis diff --git a/libs/mutagen/optimfrog.py b/libs/mutagen/optimfrog.py index 24a87af8..0d85a818 100644 --- a/libs/mutagen/optimfrog.py +++ b/libs/mutagen/optimfrog.py @@ -1,6 +1,6 @@ -# OptimFROG reader/tagger -# -# Copyright 2006 Lukas Lalinsky +# -*- coding: utf-8 -*- + +# Copyright (C) 2006 Lukas Lalinsky # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -22,6 +22,8 @@ __all__ = ["OptimFROG", "Open", "delete"] import struct +from ._compat import endswith +from mutagen import StreamInfo from mutagen.apev2 import APEv2File, error, delete @@ -29,7 +31,7 @@ class OptimFROGHeaderError(error): pass -class OptimFROGInfo(object): +class OptimFROGInfo(StreamInfo): """OptimFROG stream information. Attributes: @@ -41,7 +43,7 @@ class OptimFROGInfo(object): def __init__(self, fileobj): header = fileobj.read(76) - if (len(header) != 76 or not header.startswith("OFR ") or + if (len(header) != 76 or not header.startswith(b"OFR ") or struct.unpack("I", b"\x00" + bytes(data))[0] + tempos.append((deltasum, TEMPO, tempo)) + off += num + elif event_type in (0xF0, 0xF7): + val, off = _var_int(chunk, off) + off += val + else: + if event_type < 0x80: + # if < 0x80 take the type from the previous midi event + off += 1 + event_type = status + elif event_type < 0xF0: + off += 2 + status = event_type + else: + raise SMFError("invalid event") + + if event_type >> 4 in (0xD, 0xC): + off -= 1 + + events.append((deltasum, MIDI, delta)) + + return events, tempos + + +def _read_midi_length(fileobj): + """Returns the duration in seconds. Can raise all kind of errors...""" + + TEMPO, MIDI = range(2) + + def read_chunk(fileobj): + info = fileobj.read(8) + if len(info) != 8: + raise SMFError("truncated") + chunklen = struct.unpack(">I", info[4:])[0] + data = fileobj.read(chunklen) + if len(data) != chunklen: + raise SMFError("truncated") + return info[:4], data + + identifier, chunk = read_chunk(fileobj) + if identifier != b"MThd": + raise SMFError("Not a MIDI file") + + if len(chunk) != 6: + raise SMFError("truncated") + + format_, ntracks, tickdiv = struct.unpack(">HHH", chunk) + if format_ > 1: + raise SMFError("Not supported format %d" % format_) + + if tickdiv >> 15: + # fps = (-(tickdiv >> 8)) & 0xFF + # subres = tickdiv & 0xFF + # never saw one of those + raise SMFError("Not supported timing interval") + + # get a list of events and tempo changes for each track + tracks = [] + first_tempos = None + for tracknum in xrange(ntracks): + identifier, chunk = read_chunk(fileobj) + if identifier != b"MTrk": + continue + events, tempos = _read_track(chunk) + + # In case of format == 1, copy the first tempo list to all tracks + first_tempos = first_tempos or tempos + if format_ == 1: + tempos = list(first_tempos) + events += tempos + events.sort() + tracks.append(events) + + # calculate the duration of each track + durations = [] + for events in tracks: + tempo = 500000 + parts = [] + deltasum = 0 + for (dummy, type_, data) in events: + if type_ == TEMPO: + parts.append((deltasum, tempo)) + tempo = data + deltasum = 0 + else: + deltasum += data + parts.append((deltasum, tempo)) + + duration = 0 + for (deltasum, tempo) in parts: + quarter, tpq = deltasum / float(tickdiv), tempo + duration += (quarter * tpq) + duration /= 10 ** 6 + + durations.append(duration) + + # return the longest one + return max(durations) + + +class SMFInfo(StreamInfo): + + def __init__(self, fileobj): + """Raises SMFError""" + + self.length = _read_midi_length(fileobj) + """Length in seconds""" + + def pprint(self): + return u"SMF, %.2f seconds" % self.length + + +class SMF(FileType): + """Standard MIDI File (SMF)""" + + _mimes = ["audio/midi", "audio/x-midi"] + + def load(self, filename): + self.filename = filename + try: + with open(filename, "rb") as h: + self.info = SMFInfo(h) + except IOError as e: + raise SMFError(e) + + def add_tags(self): + raise SMFError("doesn't support tags") + + @staticmethod + def score(filename, fileobj, header): + filename = filename.lower() + return header.startswith(b"MThd") and ( + endswith(filename, ".mid") or endswith(filename, ".midi")) + + +Open = SMF +error = SMFError + +__all__ = ["SMF"] diff --git a/libs/mutagen/trueaudio.py b/libs/mutagen/trueaudio.py index 264d13a8..1c8d56c4 100644 --- a/libs/mutagen/trueaudio.py +++ b/libs/mutagen/trueaudio.py @@ -1,5 +1,6 @@ -# True Audio support for Mutagen -# Copyright 2006 Joe Wreschnig +# -*- coding: utf-8 -*- + +# Copyright (C) 2006 Joe Wreschnig # # This program is free software; you can redistribute it and/or modify # it under the terms of version 2 of the GNU General Public License as @@ -16,11 +17,13 @@ True Audio files use ID3 tags. __all__ = ["TrueAudio", "Open", "delete", "EasyTrueAudio"] +from ._compat import endswith +from mutagen import StreamInfo from mutagen.id3 import ID3FileType, delete -from mutagen._util import cdata +from mutagen._util import cdata, MutagenError -class error(RuntimeError): +class error(RuntimeError, MutagenError): pass @@ -28,7 +31,7 @@ class TrueAudioHeaderError(error, IOError): pass -class TrueAudioInfo(object): +class TrueAudioInfo(StreamInfo): """True Audio stream information. Attributes: @@ -40,14 +43,14 @@ class TrueAudioInfo(object): def __init__(self, fileobj, offset): fileobj.seek(offset or 0) header = fileobj.read(18) - if len(header) != 18 or not header.startswith("TTA"): + if len(header) != 18 or not header.startswith(b"TTA"): raise TrueAudioHeaderError("TTA header not found") self.sample_rate = cdata.int_le(header[10:14]) samples = cdata.uint_le(header[14:18]) self.length = float(samples) / self.sample_rate def pprint(self): - return "True Audio, %.2f seconds, %d Hz." % ( + return u"True Audio, %.2f seconds, %d Hz." % ( self.length, self.sample_rate) @@ -63,8 +66,8 @@ class TrueAudio(ID3FileType): @staticmethod def score(filename, fileobj, header): - return (header.startswith("ID3") + header.startswith("TTA") + - filename.lower().endswith(".tta") * 2) + return (header.startswith(b"ID3") + header.startswith(b"TTA") + + endswith(filename.lower(), b".tta") * 2) Open = TrueAudio diff --git a/libs/mutagen/wavpack.py b/libs/mutagen/wavpack.py index 1a2db818..80710f6d 100644 --- a/libs/mutagen/wavpack.py +++ b/libs/mutagen/wavpack.py @@ -1,6 +1,7 @@ -# A WavPack reader/tagger -# +# -*- coding: utf-8 -*- + # Copyright 2006 Joe Wreschnig +# 2014 Christoph Reiter # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -9,11 +10,16 @@ """WavPack reading and writing. WavPack is a lossless format that uses APEv2 tags. Read -http://www.wavpack.com/ for more information. + +* http://www.wavpack.com/ +* http://www.wavpack.com/file_format.txt + +for more information. """ __all__ = ["WavPack", "Open", "delete"] +from mutagen import StreamInfo from mutagen.apev2 import APEv2File, error, delete from mutagen._util import cdata @@ -25,7 +31,46 @@ RATES = [6000, 8000, 9600, 11025, 12000, 16000, 22050, 24000, 32000, 44100, 48000, 64000, 88200, 96000, 192000] -class WavPackInfo(object): +class _WavPackHeader(object): + + def __init__(self, block_size, version, track_no, index_no, total_samples, + block_index, block_samples, flags, crc): + + self.block_size = block_size + self.version = version + self.track_no = track_no + self.index_no = index_no + self.total_samples = total_samples + self.block_index = block_index + self.block_samples = block_samples + self.flags = flags + self.crc = crc + + @classmethod + def from_fileobj(cls, fileobj): + """A new _WavPackHeader or raises WavPackHeaderError""" + + header = fileobj.read(32) + if len(header) != 32 or not header.startswith(b"wvpk"): + raise WavPackHeaderError("not a WavPack header: %r" % header) + + block_size = cdata.uint_le(header[4:8]) + version = cdata.ushort_le(header[8:10]) + track_no = ord(header[10:11]) + index_no = ord(header[11:12]) + samples = cdata.uint_le(header[12:16]) + if samples == 2 ** 32 - 1: + samples = -1 + block_index = cdata.uint_le(header[16:20]) + block_samples = cdata.uint_le(header[20:24]) + flags = cdata.uint_le(header[24:28]) + crc = cdata.uint_le(header[28:32]) + + return _WavPackHeader(block_size, version, track_no, index_no, + samples, block_index, block_samples, flags, crc) + + +class WavPackInfo(StreamInfo): """WavPack stream information. Attributes: @@ -37,18 +82,35 @@ class WavPackInfo(object): """ def __init__(self, fileobj): - header = fileobj.read(28) - if len(header) != 28 or not header.startswith("wvpk"): + try: + header = _WavPackHeader.from_fileobj(fileobj) + except WavPackHeaderError: raise WavPackHeaderError("not a WavPack file") - samples = cdata.uint_le(header[12:16]) - flags = cdata.uint_le(header[24:28]) - self.version = cdata.short_le(header[8:10]) - self.channels = bool(flags & 4) or 2 - self.sample_rate = RATES[(flags >> 23) & 0xF] + + self.version = header.version + self.channels = bool(header.flags & 4) or 2 + self.sample_rate = RATES[(header.flags >> 23) & 0xF] + + if header.total_samples == -1 or header.block_index != 0: + # TODO: we could make this faster by using the tag size + # and search backwards for the last block, then do + # last.block_index + last.block_samples - initial.block_index + samples = header.block_samples + while 1: + fileobj.seek(header.block_size - 32 + 8, 1) + try: + header = _WavPackHeader.from_fileobj(fileobj) + except WavPackHeaderError: + break + samples += header.block_samples + else: + samples = header.total_samples + self.length = float(samples) / self.sample_rate def pprint(self): - return "WavPack, %.2f seconds, %d Hz" % (self.length, self.sample_rate) + return u"WavPack, %.2f seconds, %d Hz" % (self.length, + self.sample_rate) class WavPack(APEv2File): @@ -57,7 +119,7 @@ class WavPack(APEv2File): @staticmethod def score(filename, fileobj, header): - return header.startswith("wvpk") * 2 + return header.startswith(b"wvpk") * 2 Open = WavPack diff --git a/libs/unidecode/__init__.py b/libs/unidecode/__init__.py index 82eb5a3f..3b68de4c 100644 --- a/libs/unidecode/__init__.py +++ b/libs/unidecode/__init__.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +# vi:tabstop=4:expandtab:sw=4 """Transliterate Unicode text into plain 7-bit ASCII. Example usage: @@ -18,19 +19,53 @@ from sys import version_info Cache = {} -def unidecode(string): + +def _warn_if_not_unicode(string): + if version_info[0] < 3 and not isinstance(string, unicode): + warnings.warn( "Argument %r is not an unicode object. " + "Passing an encoded string will likely have " + "unexpected results." % (type(string),), + RuntimeWarning, 2) + + +def unidecode_expect_ascii(string): + """Transliterate an Unicode object into an ASCII string + + >>> unidecode(u"\u5317\u4EB0") + "Bei Jing " + + This function first tries to convert the string using ASCII codec. + If it fails (because of non-ASCII characters), it falls back to + transliteration using the character tables. + + This is approx. five times faster if the string only contains ASCII + characters, but slightly slower than using unidecode directly if non-ASCII + chars are present. + """ + + _warn_if_not_unicode(string) + try: + bytestring = string.encode('ASCII') + except UnicodeEncodeError: + return _unidecode(string) + if version_info[0] >= 3: + return string + else: + return bytestring + +def unidecode_expect_nonascii(string): """Transliterate an Unicode object into an ASCII string >>> unidecode(u"\u5317\u4EB0") "Bei Jing " """ - if version_info[0] < 3 and not isinstance(string, unicode): - warnings.warn( "Argument %r is not an unicode object. " - "Passing an encoded string will likely have " - "unexpected results." % (type(string),), - RuntimeWarning, 2) + _warn_if_not_unicode(string) + return _unidecode(string) +unidecode = unidecode_expect_ascii + +def _unidecode(string): retval = [] for char in string: @@ -43,6 +78,11 @@ def unidecode(string): if codepoint > 0xeffff: continue # Characters in Private Use Area and above are ignored + if 0xd800 <= codepoint <= 0xdfff: + warnings.warn( "Surrogate character %r will be ignored. " + "You might be using a narrow Python build." % (char,), + RuntimeWarning, 2) + section = codepoint >> 8 # Chop off the last two hex digits position = codepoint % 256 # Last two hex digits @@ -50,7 +90,7 @@ def unidecode(string): table = Cache[section] except KeyError: try: - mod = __import__('unidecode.x%03x'%(section), [], [], ['data']) + mod = __import__('unidecode.x%03x'%(section), globals(), locals(), ['data']) except ImportError: Cache[section] = None continue # No match: ignore this character and carry on. diff --git a/libs/unidecode/util.py b/libs/unidecode/util.py new file mode 100644 index 00000000..477280d1 --- /dev/null +++ b/libs/unidecode/util.py @@ -0,0 +1,58 @@ +# vim:ts=4 sw=4 expandtab softtabstop=4 +from __future__ import print_function +import optparse +import locale +import os +import sys +import warnings + +from unidecode import unidecode + +PY3 = sys.version_info[0] >= 3 + +def fatal(msg): + sys.stderr.write(msg + "\n") + sys.exit(1) + +def main(): + default_encoding = locale.getpreferredencoding() + + parser = optparse.OptionParser('%prog [options] [FILE]', + description="Transliterate Unicode text into ASCII. FILE is path to file to transliterate. " + "Standard input is used if FILE is omitted and -c is not specified.") + parser.add_option('-e', '--encoding', metavar='ENCODING', default=default_encoding, + help='Specify an encoding (default is %s)' % (default_encoding,)) + parser.add_option('-c', metavar='TEXT', dest='text', + help='Transliterate TEXT instead of FILE') + + options, args = parser.parse_args() + + encoding = options.encoding + + if args: + if options.text: + fatal("Can't use both FILE and -c option") + else: + with open(args[0], 'rb') as f: + stream = f.read() + elif options.text: + if PY3: + stream = os.fsencode(options.text) + else: + stream = options.text + # add a newline to the string if it comes from the + # command line so that the result is printed nicely + # on the console. + stream += '\n'.encode('ascii') + else: + if PY3: + stream = sys.stdin.buffer.read() + else: + stream = sys.stdin.read() + + try: + stream = stream.decode(encoding) + except UnicodeDecodeError as e: + fatal('Unable to decode input: %s, start: %d, end: %d' % (e.reason, e.start, e.end)) + + sys.stdout.write(unidecode(stream)) diff --git a/libs/unidecode/x000.py b/libs/unidecode/x000.py index 6821df47..c3f8f515 100644 --- a/libs/unidecode/x000.py +++ b/libs/unidecode/x000.py @@ -1,132 +1,15 @@ data = ( -'\x00', # 0x00 -'\x01', # 0x01 -'\x02', # 0x02 -'\x03', # 0x03 -'\x04', # 0x04 -'\x05', # 0x05 -'\x06', # 0x06 -'\x07', # 0x07 -'\x08', # 0x08 -'\x09', # 0x09 -'\x0a', # 0x0a -'\x0b', # 0x0b -'\x0c', # 0x0c -'\x0d', # 0x0d -'\x0e', # 0x0e -'\x0f', # 0x0f -'\x10', # 0x10 -'\x11', # 0x11 -'\x12', # 0x12 -'\x13', # 0x13 -'\x14', # 0x14 -'\x15', # 0x15 -'\x16', # 0x16 -'\x17', # 0x17 -'\x18', # 0x18 -'\x19', # 0x19 -'\x1a', # 0x1a -'\x1b', # 0x1b -'\x1c', # 0x1c -'\x1d', # 0x1d -'\x1e', # 0x1e -'\x1f', # 0x1f -' ', # 0x20 -'!', # 0x21 -'"', # 0x22 -'#', # 0x23 -'$', # 0x24 -'%', # 0x25 -'&', # 0x26 -'\'', # 0x27 -'(', # 0x28 -')', # 0x29 -'*', # 0x2a -'+', # 0x2b -',', # 0x2c -'-', # 0x2d -'.', # 0x2e -'/', # 0x2f -'0', # 0x30 -'1', # 0x31 -'2', # 0x32 -'3', # 0x33 -'4', # 0x34 -'5', # 0x35 -'6', # 0x36 -'7', # 0x37 -'8', # 0x38 -'9', # 0x39 -':', # 0x3a -';', # 0x3b -'<', # 0x3c -'=', # 0x3d -'>', # 0x3e -'?', # 0x3f -'@', # 0x40 -'A', # 0x41 -'B', # 0x42 -'C', # 0x43 -'D', # 0x44 -'E', # 0x45 -'F', # 0x46 -'G', # 0x47 -'H', # 0x48 -'I', # 0x49 -'J', # 0x4a -'K', # 0x4b -'L', # 0x4c -'M', # 0x4d -'N', # 0x4e -'O', # 0x4f -'P', # 0x50 -'Q', # 0x51 -'R', # 0x52 -'S', # 0x53 -'T', # 0x54 -'U', # 0x55 -'V', # 0x56 -'W', # 0x57 -'X', # 0x58 -'Y', # 0x59 -'Z', # 0x5a -']', # 0x5b -'\\', # 0x5c -']', # 0x5d -'^', # 0x5e -'_', # 0x5f -'`', # 0x60 -'a', # 0x61 -'b', # 0x62 -'c', # 0x63 -'d', # 0x64 -'e', # 0x65 -'f', # 0x66 -'g', # 0x67 -'h', # 0x68 -'i', # 0x69 -'j', # 0x6a -'k', # 0x6b -'l', # 0x6c -'m', # 0x6d -'n', # 0x6e -'o', # 0x6f -'p', # 0x70 -'q', # 0x71 -'r', # 0x72 -'s', # 0x73 -'t', # 0x74 -'u', # 0x75 -'v', # 0x76 -'w', # 0x77 -'x', # 0x78 -'y', # 0x79 -'z', # 0x7a -'{', # 0x7b -'|', # 0x7c -'}', # 0x7d -'~', # 0x7e -'', # 0x7f +# Code points u+007f and below are equivalent to ASCII and are handled by a +# special case in the code. Hence they are not present in this table. +'', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', +'', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', +'', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', +'', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', +'', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', +'', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', +'', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', +'', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', + '', # 0x80 '', # 0x81 '', # 0x82 @@ -162,7 +45,10 @@ data = ( ' ', # 0xa0 '!', # 0xa1 'C/', # 0xa2 + +# Not "GBP" - Pound Sign is used for more than just British Pounds. 'PS', # 0xa3 + '$?', # 0xa4 'Y=', # 0xa5 '|', # 0xa6 @@ -177,8 +63,11 @@ data = ( '-', # 0xaf 'deg', # 0xb0 '+-', # 0xb1 + +# These might be combined with other superscript digits (u+2070 - u+2079) '2', # 0xb2 '3', # 0xb3 + '\'', # 0xb4 'u', # 0xb5 'P', # 0xb6 @@ -195,7 +84,10 @@ data = ( 'A', # 0xc1 'A', # 0xc2 'A', # 0xc3 + +# Not "AE" - used in languages other than German 'A', # 0xc4 + 'A', # 0xc5 'AE', # 0xc6 'C', # 0xc7 @@ -213,13 +105,19 @@ data = ( 'O', # 0xd3 'O', # 0xd4 'O', # 0xd5 + +# Not "OE" - used in languages other than German 'O', # 0xd6 + 'x', # 0xd7 'O', # 0xd8 'U', # 0xd9 'U', # 0xda 'U', # 0xdb + +# Not "UE" - used in languages other than German 'U', # 0xdc + 'Y', # 0xdd 'Th', # 0xde 'ss', # 0xdf @@ -227,7 +125,10 @@ data = ( 'a', # 0xe1 'a', # 0xe2 'a', # 0xe3 + +# Not "ae" - used in languages other than German 'a', # 0xe4 + 'a', # 0xe5 'ae', # 0xe6 'c', # 0xe7 @@ -245,13 +146,19 @@ data = ( 'o', # 0xf3 'o', # 0xf4 'o', # 0xf5 + +# Not "oe" - used in languages other than German 'o', # 0xf6 + '/', # 0xf7 'o', # 0xf8 'u', # 0xf9 'u', # 0xfa 'u', # 0xfb + +# Not "ue" - used in languages other than German 'u', # 0xfc + 'y', # 0xfd 'th', # 0xfe 'y', # 0xff diff --git a/libs/unidecode/x020.py b/libs/unidecode/x020.py index f67264c8..b6494730 100644 --- a/libs/unidecode/x020.py +++ b/libs/unidecode/x020.py @@ -171,7 +171,7 @@ data = ( 'W', # 0xa9 'NS', # 0xaa 'D', # 0xab -'EU', # 0xac +'EUR', # 0xac 'K', # 0xad 'T', # 0xae 'Dr', # 0xaf diff --git a/libs/unidecode/x021.py b/libs/unidecode/x021.py index fcb651ba..067d9bdc 100644 --- a/libs/unidecode/x021.py +++ b/libs/unidecode/x021.py @@ -1,7 +1,7 @@ data = ( '', # 0x00 '', # 0x01 -'', # 0x02 +'C', # 0x02 '', # 0x03 '', # 0x04 '', # 0x05 @@ -12,7 +12,7 @@ data = ( '', # 0x0a '', # 0x0b '', # 0x0c -'', # 0x0d +'H', # 0x0d '', # 0x0e '', # 0x0f '', # 0x10 @@ -20,22 +20,22 @@ data = ( '', # 0x12 '', # 0x13 '', # 0x14 -'', # 0x15 +'N', # 0x15 '', # 0x16 '', # 0x17 '', # 0x18 -'', # 0x19 -'', # 0x1a +'P', # 0x19 +'Q', # 0x1a '', # 0x1b '', # 0x1c -'', # 0x1d +'R', # 0x1d '', # 0x1e '', # 0x1f '(sm)', # 0x20 'TEL', # 0x21 '(tm)', # 0x22 '', # 0x23 -'', # 0x24 +'Z', # 0x24 '', # 0x25 '', # 0x26 '', # 0x27 @@ -45,12 +45,12 @@ data = ( 'A', # 0x2b '', # 0x2c '', # 0x2d -'', # 0x2e -'', # 0x2f -'', # 0x30 -'', # 0x31 +'e', # 0x2e +'e', # 0x2f +'E', # 0x30 +'F', # 0x31 'F', # 0x32 -'', # 0x33 +'M', # 0x33 '', # 0x34 '', # 0x35 '', # 0x36 @@ -59,20 +59,20 @@ data = ( '', # 0x39 '', # 0x3a 'FAX', # 0x3b -'[?]', # 0x3c -'[?]', # 0x3d -'[?]', # 0x3e -'[?]', # 0x3f +'', # 0x3c +'', # 0x3d +'', # 0x3e +'', # 0x3f '[?]', # 0x40 '[?]', # 0x41 '[?]', # 0x42 '[?]', # 0x43 '[?]', # 0x44 -'[?]', # 0x45 -'[?]', # 0x46 -'[?]', # 0x47 -'[?]', # 0x48 -'[?]', # 0x49 +'D', # 0x45 +'d', # 0x46 +'e', # 0x47 +'i', # 0x48 +'j', # 0x49 '[?]', # 0x4a '[?]', # 0x4b '[?]', # 0x4c diff --git a/libs/unidecode/x04e.py b/libs/unidecode/x04e.py index e346f67b..b472b855 100644 --- a/libs/unidecode/x04e.py +++ b/libs/unidecode/x04e.py @@ -1,5 +1,5 @@ data = ( -'[?] ', # 0x00 +'Yi ', # 0x00 'Ding ', # 0x01 'Kao ', # 0x02 'Qi ', # 0x03