github/nzbToMedia
1
0
Fork 0
mirror of https://github.com/clinton-hall/nzbToMedia.git synced 2025-08-21 13:53:15 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fixes issues #344, #314

Browse source 
Added in code to place single files or groups of files that are not contained in a folder into there own folder or grouped folder based on parsing of the filenames to extract details required to determin correcting folder naming.
This commit is contained in:
echel0n 2014-04-23 19:50:51 -07:00
commit f7e56b979b
258 changed files with 73367 additions and 174 deletions
Download patch file Download diff file Expand all files Collapse all files

4
DeleteSamples.py
View file

@ -32,8 +32,6 @@
import os
import sys
import nzbtomedia
from nzbtomedia.nzbToMediaUtil import joinPath
def is_sample(filePath, inputName, maxSampleSize, SampleIDs):
# 200 MB in bytes
@ -103,7 +101,7 @@ if os.environ.has_key('NZBOP_SCRIPTDIR') and not os.environ['NZBOP_VERSION'][0:5
for dirpath, dirnames, filenames in os.walk(os.environ['NZBPP_DIRECTORY']):
for file in filenames:
filePath = joinPath(dirpath, file)
filePath = os.path.join(dirpath, file)
fileName, fileExtension = os.path.splitext(file)
if fileExtension in mediaContainer: # If the file is a video file

3
ResetDateTime.py
View file

@ -15,7 +15,6 @@
import os
import sys
import nzbtomedia
from nzbtomedia.nzbToMediaUtil import joinPath
if os.environ.has_key('NZBOP_SCRIPTDIR') and not os.environ['NZBOP_VERSION'][0:5] < '11.0':
print "Script triggered from NZBGet (11.0 or later)."
@ -67,7 +66,7 @@ if os.environ.has_key('NZBOP_SCRIPTDIR') and not os.environ['NZBOP_VERSION'][0:5
directory = os.path.normpath(os.environ['NZBPP_DIRECTORY'])
for dirpath, dirnames, filenames in os.walk(directory):
for file in filenames:
filepath = joinPath(dirpath, file)
filepath = os.path.join(dirpath, file)
print "reseting datetime for file", filepath
try:
os.utime(filepath, None)

35
TorrentToMedia.py
View file

@ -4,7 +4,6 @@ import os
import time
import shutil
import sys
import platform
import nzbtomedia
from subprocess import Popen
@ -55,7 +54,7 @@ def processTorrent(inputDirectory, inputName, inputCategory, inputHash, inputID,
if inputCategory == "":
inputCategory = "UNCAT"
outputDestination = os.path.normpath(nzbtomedia.joinPath(nzbtomedia. OUTPUTDIRECTORY, inputCategory, nzbtomedia.sanitizeFileName(inputName)))
outputDestination = os.path.normpath(nzbtomedia.os.path.join(nzbtomedia.OUTPUTDIRECTORY, inputCategory, nzbtomedia.sanitizeName(inputName)))
logger.info("Output directory set to: %s" % (outputDestination))
@ -74,27 +73,25 @@ def processTorrent(inputDirectory, inputName, inputCategory, inputHash, inputID,
if nzbtomedia.CFG["HeadPhones"][inputCategory]:
nzbtomedia.NOFLATTEN.extend(nzbtomedia.CFG["HeadPhones"].sections) # Make sure we preserve folder structure for HeadPhones.
outputDestinationMaster = outputDestination # Save the original, so we can change this within the loop below, and reset afterwards.
now = datetime.datetime.now()
inputFiles = nzbtomedia.listMediaFiles(inputDirectory)
logger.debug("Found %s files in %s" % (str(len(inputFiles)), inputDirectory))
for inputFile in inputFiles:
fileDirPath = os.path.dirname(inputFile)
filePath = os.path.dirname(inputFile)
fileName, fileExt = os.path.splitext(os.path.basename(inputFile))
fullFileName = os.path.basename(inputFile)
targetFile = nzbtomedia.os.path.join(outputDestination, fullFileName)
if inputCategory in nzbtomedia.NOFLATTEN:
if not fileDirPath == outputDestinationMaster:
outputDestination = nzbtomedia.joinPath(outputDestinationMaster, fileDirPath) # join this extra directory to output.
logger.debug("Setting outputDestination to %s to preserve folder structure" % (outputDestination))
targetDirectory = nzbtomedia.joinPath(outputDestination, fullFileName)
if not os.path.basename(filePath) in outputDestination:
targetFile = nzbtomedia.os.path.join(nzbtomedia.os.path.join(outputDestination, os.path.basename(filePath)), fullFileName)
logger.debug("Setting outputDestination to %s to preserve folder structure" % (os.path.dirname(targetFile)))
if root == 1:
if not foundFile:
logger.debug("Looking for %s in: %s" % (inputName, fullFileName))
if (nzbtomedia.sanitizeFileName(inputName) in nzbtomedia.sanitizeFileName(fullFileName)) or (nzbtomedia.sanitizeFileName(fileName) in nzbtomedia.sanitizeFileName(inputName)):
if (nzbtomedia.sanitizeName(inputName) in nzbtomedia.sanitizeName(fullFileName)) or (nzbtomedia.sanitizeName(fileName) in nzbtomedia.sanitizeName(inputName)):
foundFile = True
logger.debug("Found file %s that matches Torrent Name %s" % (fullFileName, inputName))
else:
@ -115,19 +112,14 @@ def processTorrent(inputDirectory, inputName, inputCategory, inputHash, inputID,
if Torrent_NoLink == 0:
try:
nzbtomedia.copy_link(inputFile, targetDirectory, nzbtomedia.USELINK, outputDestination)
copy_list.append([inputFile, nzbtomedia.joinPath(outputDestination, fullFileName)])
nzbtomedia.copy_link(inputFile, targetFile, nzbtomedia.USELINK)
nzbtomedia.rmReadOnly(targetFile)
except:
logger.error("Failed to link file: %s" % (fullFileName))
outputDestination = outputDestinationMaster # Reset here.
logger.error("Failed to link: %s to %s" % (inputFile, targetFile))
if not inputCategory in nzbtomedia.NOFLATTEN: #don't flatten hp in case multi cd albums, and we need to copy this back later.
nzbtomedia.flatten(outputDestination)
if platform.system() == 'Windows': # remove Read Only flag from files in Windows.
nzbtomedia.remove_read_only(outputDestination)
if nzbtomedia.CFG[section][inputCategory]['extract'] == 1:
logger.debug('Checking for archives to extract in directory: %s' % (outputDestination))
nzbtomedia.extractFiles(outputDestination)
@ -148,7 +140,6 @@ def processTorrent(inputDirectory, inputName, inputCategory, inputHash, inputID,
else:
logger.warning("Found no media files in %s" % outputDestination)
# Only these sections can handling failed downloads so make sure everything else gets through without the check for failed
if not nzbtomedia.CFG['CouchPotato','SickBeard','NzbDrone'][inputCategory]:
status = 0
@ -208,7 +199,7 @@ def external_script(outputDestination, torrentName, torrentLabel):
for dirpath, dirnames, filenames in os.walk(outputDestination):
for file in filenames:
filePath = nzbtomedia.joinPath(dirpath, file)
filePath = nzbtomedia.os.path.join(dirpath, file)
fileName, fileExtension = os.path.splitext(file)
if fileExtension in nzbtomedia.USER_SCRIPT_MEDIAEXTENSIONS or "ALL" in nzbtomedia.USER_SCRIPT_MEDIAEXTENSIONS:
@ -261,7 +252,7 @@ def external_script(outputDestination, torrentName, torrentLabel):
num_files_new = 0
for dirpath, dirnames, filenames in os.walk(outputDestination):
for file in filenames:
filePath = nzbtomedia.joinPath(dirpath, file)
filePath = nzbtomedia.os.path.join(dirpath, file)
fileName, fileExtension = os.path.splitext(file)
if fileExtension in nzbtomedia.USER_SCRIPT_MEDIAEXTENSIONS or nzbtomedia.USER_SCRIPT_MEDIAEXTENSIONS == "ALL":
@ -306,7 +297,7 @@ def main(args):
for section, subsection in nzbtomedia.SUBSECTIONS.items():
for category in subsection:
if nzbtomedia.CFG[section][category].isenabled():
dirNames = nzbtomedia.get_dirnames(section, category)
dirNames = nzbtomedia.getDirs(section, category)
for dirName in dirNames:
clientAgent = 'manual'
inputHash = None

26
libs/beets/__init__.py Normal file
View file

@ -0,0 +1,26 @@
# This file is part of beets.
# Copyright 2013, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
# This particular version has been slightly modified to work with headphones
# https://github.com/rembo10/headphones
__version__ = '1.3.4'
__author__ = 'Adrian Sampson <adrian@radbox.org>'
import beets.library
from beets.util import confit
Library = beets.library.Library
config = confit.LazyConfig('beets', __name__)

247
libs/beets/autotag/__init__.py Normal file
View file

@ -0,0 +1,247 @@
# This file is part of beets.
# Copyright 2013, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Facilities for automatically determining files' correct metadata.
"""
import os
import logging
import re
from beets import library, mediafile, config
from beets.util import sorted_walk, ancestry, displayable_path
# Parts of external interface.
from .hooks import AlbumInfo, TrackInfo, AlbumMatch, TrackMatch
from .match import tag_item, tag_album
from .match import recommendation
# Global logger.
log = logging.getLogger('beets')
# Constants for directory walker.
MULTIDISC_MARKERS = (r'dis[ck]', r'cd')
MULTIDISC_PAT_FMT = r'^(.*%s[\W_]*)\d'
# Additional utilities for the main interface.
def albums_in_dir(path):
"""Recursively searches the given directory and returns an iterable
of (paths, items) where paths is a list of directories and items is
a list of Items that is probably an album. Specifically, any folder
containing any media files is an album.
"""
collapse_pat = collapse_paths = collapse_items = None
for root, dirs, files in sorted_walk(path,
ignore=config['ignore'].as_str_seq(),
logger=log):
# Get a list of items in the directory.
items = []
for filename in files:
try:
i = library.Item.from_path(os.path.join(root, filename))
except mediafile.FileTypeError:
pass
except mediafile.UnreadableFileError:
log.warn(u'unreadable file: {0}'.format(
displayable_path(filename))
)
else:
items.append(i)
# If we're currently collapsing the constituent directories in a
# multi-disc album, check whether we should continue collapsing
# and add the current directory. If so, just add the directory
# and move on to the next directory. If not, stop collapsing.
if collapse_paths:
if (not collapse_pat and collapse_paths[0] in ancestry(root)) or \
(collapse_pat and
collapse_pat.match(os.path.basename(root))):
# Still collapsing.
collapse_paths.append(root)
collapse_items += items
continue
else:
# Collapse finished. Yield the collapsed directory and
# proceed to process the current one.
if collapse_items:
yield collapse_paths, collapse_items
collapse_pat = collapse_paths = collapse_items = None
# Check whether this directory looks like the *first* directory
# in a multi-disc sequence. There are two indicators: the file
# is named like part of a multi-disc sequence (e.g., "Title Disc
# 1") or it contains no items but only directories that are
# named in this way.
start_collapsing = False
for marker in MULTIDISC_MARKERS:
marker_pat = re.compile(MULTIDISC_PAT_FMT % marker, re.I)
match = marker_pat.match(os.path.basename(root))
# Is this directory the root of a nested multi-disc album?
if dirs and not items:
# Check whether all subdirectories have the same prefix.
start_collapsing = True
subdir_pat = None
for subdir in dirs:
# The first directory dictates the pattern for
# the remaining directories.
if not subdir_pat:
match = marker_pat.match(subdir)
if match:
subdir_pat = re.compile(r'^%s\d' %
re.escape(match.group(1)), re.I)
else:
start_collapsing = False
break
# Subsequent directories must match the pattern.
elif not subdir_pat.match(subdir):
start_collapsing = False
break
# If all subdirectories match, don't check other
# markers.
if start_collapsing:
break
# Is this directory the first in a flattened multi-disc album?
elif match:
start_collapsing = True
# Set the current pattern to match directories with the same
# prefix as this one, followed by a digit.
collapse_pat = re.compile(r'^%s\d' %
re.escape(match.group(1)), re.I)
break
# If either of the above heuristics indicated that this is the
# beginning of a multi-disc album, initialize the collapsed
# directory and item lists and check the next directory.
if start_collapsing:
# Start collapsing; continue to the next iteration.
collapse_paths = [root]
collapse_items = items
continue
# If it's nonempty, yield it.
if items:
yield [root], items
# Clear out any unfinished collapse.
if collapse_paths and collapse_items:
yield collapse_paths, collapse_items
def apply_item_metadata(item, track_info):
"""Set an item's metadata from its matched TrackInfo object.
"""
item.artist = track_info.artist
item.artist_sort = track_info.artist_sort
item.artist_credit = track_info.artist_credit
item.title = track_info.title
item.mb_trackid = track_info.track_id
if track_info.artist_id:
item.mb_artistid = track_info.artist_id
# At the moment, the other metadata is left intact (including album
# and track number). Perhaps these should be emptied?
def apply_metadata(album_info, mapping):
"""Set the items' metadata to match an AlbumInfo object using a
mapping from Items to TrackInfo objects.
"""
for item, track_info in mapping.iteritems():
# Album, artist, track count.
if track_info.artist:
item.artist = track_info.artist
else:
item.artist = album_info.artist
item.albumartist = album_info.artist
item.album = album_info.album
# Artist sort and credit names.
item.artist_sort = track_info.artist_sort or album_info.artist_sort
item.artist_credit = track_info.artist_credit or \
album_info.artist_credit
item.albumartist_sort = album_info.artist_sort
item.albumartist_credit = album_info.artist_credit
# Release date.
for prefix in '', 'original_':
if config['original_date'] and not prefix:
# Ignore specific release date.
continue
for suffix in 'year', 'month', 'day':
key = prefix + suffix
value = getattr(album_info, key) or 0
# If we don't even have a year, apply nothing.
if suffix == 'year' and not value:
break
# Otherwise, set the fetched value (or 0 for the month
# and day if not available).
item[key] = value
# If we're using original release date for both fields,
# also set item.year = info.original_year, etc.
if config['original_date']:
item[suffix] = value
# Title.
item.title = track_info.title
if config['per_disc_numbering']:
item.track = track_info.medium_index or track_info.index
item.tracktotal = track_info.medium_total or len(album_info.tracks)
else:
item.track = track_info.index
item.tracktotal = len(album_info.tracks)
# Disc and disc count.
item.disc = track_info.medium
item.disctotal = album_info.mediums
# MusicBrainz IDs.
item.mb_trackid = track_info.track_id
item.mb_albumid = album_info.album_id
if track_info.artist_id:
item.mb_artistid = track_info.artist_id
else:
item.mb_artistid = album_info.artist_id
item.mb_albumartistid = album_info.artist_id
item.mb_releasegroupid = album_info.releasegroup_id
# Compilation flag.
item.comp = album_info.va
# Miscellaneous metadata.
for field in ('albumtype',
'label',
'asin',
'catalognum',
'script',
'language',
'country',
'albumstatus',
'media',
'albumdisambig'):
value = getattr(album_info, field)
if value is not None:
item[field] = value
if track_info.disctitle is not None:
item.disctitle = track_info.disctitle
# Headphones seal of approval
item.comments = 'tagged by headphones/beets'

545
libs/beets/autotag/hooks.py Normal file
View file

@ -0,0 +1,545 @@
# This file is part of beets.
# Copyright 2013, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Glue between metadata sources and the matching logic."""
import logging
from collections import namedtuple
import re
from beets import plugins
from beets import config
from beets.autotag import mb
from beets.util import levenshtein
from unidecode import unidecode
log = logging.getLogger('beets')
# Classes used to represent candidate options.
class AlbumInfo(object):
"""Describes a canonical release that may be used to match a release
in the library. Consists of these data members:
- ``album``: the release title
- ``album_id``: MusicBrainz ID; UUID fragment only
- ``artist``: name of the release's primary artist
- ``artist_id``
- ``tracks``: list of TrackInfo objects making up the release
- ``asin``: Amazon ASIN
- ``albumtype``: string describing the kind of release
- ``va``: boolean: whether the release has "various artists"
- ``year``: release year
- ``month``: release month
- ``day``: release day
- ``label``: music label responsible for the release
- ``mediums``: the number of discs in this release
- ``artist_sort``: name of the release's artist for sorting
- ``releasegroup_id``: MBID for the album's release group
- ``catalognum``: the label's catalog number for the release
- ``script``: character set used for metadata
- ``language``: human language of the metadata
- ``country``: the release country
- ``albumstatus``: MusicBrainz release status (Official, etc.)
- ``media``: delivery mechanism (Vinyl, etc.)
- ``albumdisambig``: MusicBrainz release disambiguation comment
- ``artist_credit``: Release-specific artist name
- ``data_source``: The original data source (MusicBrainz, Discogs, etc.)
- ``data_url``: The data source release URL.
The fields up through ``tracks`` are required. The others are
optional and may be None.
"""
def __init__(self, album, album_id, artist, artist_id, tracks, asin=None,
albumtype=None, va=False, year=None, month=None, day=None,
label=None, mediums=None, artist_sort=None,
releasegroup_id=None, catalognum=None, script=None,
language=None, country=None, albumstatus=None, media=None,
albumdisambig=None, artist_credit=None, original_year=None,
original_month=None, original_day=None, data_source=None,
data_url=None):
self.album = album
self.album_id = album_id
self.artist = artist
self.artist_id = artist_id
self.tracks = tracks
self.asin = asin
self.albumtype = albumtype
self.va = va
self.year = year
self.month = month
self.day = day
self.label = label
self.mediums = mediums
self.artist_sort = artist_sort
self.releasegroup_id = releasegroup_id
self.catalognum = catalognum
self.script = script
self.language = language
self.country = country
self.albumstatus = albumstatus
self.media = media
self.albumdisambig = albumdisambig
self.artist_credit = artist_credit
self.original_year = original_year
self.original_month = original_month
self.original_day = original_day
self.data_source = data_source
self.data_url = data_url
# Work around a bug in python-musicbrainz-ngs that causes some
# strings to be bytes rather than Unicode.
# https://github.com/alastair/python-musicbrainz-ngs/issues/85
def decode(self, codec='utf8'):
"""Ensure that all string attributes on this object, and the
constituent `TrackInfo` objects, are decoded to Unicode.
"""
for fld in ['album', 'artist', 'albumtype', 'label', 'artist_sort',
'catalognum', 'script', 'language', 'country',
'albumstatus', 'albumdisambig', 'artist_credit', 'media']:
value = getattr(self, fld)
if isinstance(value, str):
setattr(self, fld, value.decode(codec, 'ignore'))
if self.tracks:
for track in self.tracks:
track.decode(codec)
class TrackInfo(object):
"""Describes a canonical track present on a release. Appears as part
of an AlbumInfo's ``tracks`` list. Consists of these data members:
- ``title``: name of the track
- ``track_id``: MusicBrainz ID; UUID fragment only
- ``artist``: individual track artist name
- ``artist_id``
- ``length``: float: duration of the track in seconds
- ``index``: position on the entire release
- ``medium``: the disc number this track appears on in the album
- ``medium_index``: the track's position on the disc
- ``medium_total``: the number of tracks on the item's disc
- ``artist_sort``: name of the track artist for sorting
- ``disctitle``: name of the individual medium (subtitle)
- ``artist_credit``: Recording-specific artist name
Only ``title`` and ``track_id`` are required. The rest of the fields
may be None. The indices ``index``, ``medium``, and ``medium_index``
are all 1-based.
"""
def __init__(self, title, track_id, artist=None, artist_id=None,
length=None, index=None, medium=None, medium_index=None,
medium_total=None, artist_sort=None, disctitle=None,
artist_credit=None, data_source=None, data_url=None):
self.title = title
self.track_id = track_id
self.artist = artist
self.artist_id = artist_id
self.length = length
self.index = index
self.medium = medium
self.medium_index = medium_index
self.medium_total = medium_total
self.artist_sort = artist_sort
self.disctitle = disctitle
self.artist_credit = artist_credit
self.data_source = data_source
self.data_url = data_url
# As above, work around a bug in python-musicbrainz-ngs.
def decode(self, codec='utf8'):
"""Ensure that all string attributes on this object are decoded
to Unicode.
"""
for fld in ['title', 'artist', 'medium', 'artist_sort', 'disctitle',
'artist_credit']:
value = getattr(self, fld)
if isinstance(value, str):
setattr(self, fld, value.decode(codec, 'ignore'))
# Candidate distance scoring.
# Parameters for string distance function.
# Words that can be moved to the end of a string using a comma.
SD_END_WORDS = ['the', 'a', 'an']
# Reduced weights for certain portions of the string.
SD_PATTERNS = [
(r'^the ', 0.1),
(r'[\[\(]?(ep|single)[\]\)]?', 0.0),
(r'[\[\(]?(featuring|feat|ft)[\. :].+', 0.1),
(r'\(.*?\)', 0.3),
(r'\[.*?\]', 0.3),
(r'(, )?(pt\.|part) .+', 0.2),
]
# Replacements to use before testing distance.
SD_REPLACE = [
(r'&', 'and'),
]
def _string_dist_basic(str1, str2):
"""Basic edit distance between two strings, ignoring
non-alphanumeric characters and case. Comparisons are based on a
transliteration/lowering to ASCII characters. Normalized by string
length.
"""
str1 = unidecode(str1)
str2 = unidecode(str2)
str1 = re.sub(r'[^a-z0-9]', '', str1.lower())
str2 = re.sub(r'[^a-z0-9]', '', str2.lower())
if not str1 and not str2:
return 0.0
return levenshtein(str1, str2) / float(max(len(str1), len(str2)))
def string_dist(str1, str2):
"""Gives an "intuitive" edit distance between two strings. This is
an edit distance, normalized by the string length, with a number of
tweaks that reflect intuition about text.
"""
if str1 == None and str2 == None: return 0.0
if str1 == None or str2 == None: return 1.0
str1 = str1.lower()
str2 = str2.lower()
# Don't penalize strings that move certain words to the end. For
# example, "the something" should be considered equal to
# "something, the".
for word in SD_END_WORDS:
if str1.endswith(', %s' % word):
str1 = '%s %s' % (word, str1[:-len(word)-2])
if str2.endswith(', %s' % word):
str2 = '%s %s' % (word, str2[:-len(word)-2])
# Perform a couple of basic normalizing substitutions.
for pat, repl in SD_REPLACE:
str1 = re.sub(pat, repl, str1)
str2 = re.sub(pat, repl, str2)
# Change the weight for certain string portions matched by a set
# of regular expressions. We gradually change the strings and build
# up penalties associated with parts of the string that were
# deleted.
base_dist = _string_dist_basic(str1, str2)
penalty = 0.0
for pat, weight in SD_PATTERNS:
# Get strings that drop the pattern.
case_str1 = re.sub(pat, '', str1)
case_str2 = re.sub(pat, '', str2)
if case_str1 != str1 or case_str2 != str2:
# If the pattern was present (i.e., it is deleted in the
# the current case), recalculate the distances for the
# modified strings.
case_dist = _string_dist_basic(case_str1, case_str2)
case_delta = max(0.0, base_dist - case_dist)
if case_delta == 0.0:
continue
# Shift our baseline strings down (to avoid rematching the
# same part of the string) and add a scaled distance
# amount to the penalties.
str1 = case_str1
str2 = case_str2
base_dist = case_dist
penalty += weight * case_delta
return base_dist + penalty
class Distance(object):
"""Keeps track of multiple distance penalties. Provides a single
weighted distance for all penalties as well as a weighted distance
for each individual penalty.
"""
def __init__(self):
self._penalties = {}
weights_view = config['match']['distance_weights']
self._weights = {}
for key in weights_view.keys():
self._weights[key] = weights_view[key].as_number()
# Access the components and their aggregates.
@property
def distance(self):
"""Return a weighted and normalized distance across all
penalties.
"""
dist_max = self.max_distance
if dist_max:
return self.raw_distance / self.max_distance
return 0.0
@property
def max_distance(self):
"""Return the maximum distance penalty (normalization factor).
"""
dist_max = 0.0
for key, penalty in self._penalties.iteritems():
dist_max += len(penalty) * self._weights[key]
return dist_max
@property
def raw_distance(self):
"""Return the raw (denormalized) distance.
"""
dist_raw = 0.0
for key, penalty in self._penalties.iteritems():
dist_raw += sum(penalty) * self._weights[key]
return dist_raw
def items(self):
"""Return a list of (key, dist) pairs, with `dist` being the
weighted distance, sorted from highest to lowest. Does not
include penalties with a zero value.
"""
list_ = []
for key in self._penalties:
dist = self[key]
if dist:
list_.append((key, dist))
# Convert distance into a negative float we can sort items in
# ascending order (for keys, when the penalty is equal) and
# still get the items with the biggest distance first.
return sorted(list_, key=lambda (key, dist): (0-dist, key))
# Behave like a float.
def __cmp__(self, other):
return cmp(self.distance, other)
def __float__(self):
return self.distance
def __sub__(self, other):
return self.distance - other
def __rsub__(self, other):
return other - self.distance
# Behave like a dict.
def __getitem__(self, key):
"""Returns the weighted distance for a named penalty.
"""
dist = sum(self._penalties[key]) * self._weights[key]
dist_max = self.max_distance
if dist_max:
return dist / dist_max
return 0.0
def __iter__(self):
return iter(self.items())
def __len__(self):
return len(self.items())
def keys(self):
return [key for key, _ in self.items()]
def update(self, dist):
"""Adds all the distance penalties from `dist`.
"""
if not isinstance(dist, Distance):
raise ValueError(
'`dist` must be a Distance object. It is: %r' % dist)
for key, penalties in dist._penalties.iteritems():
self._penalties.setdefault(key, []).extend(penalties)
# Adding components.
def _eq(self, value1, value2):
"""Returns True if `value1` is equal to `value2`. `value1` may
be a compiled regular expression, in which case it will be
matched against `value2`.
"""
if isinstance(value1, re._pattern_type):
return bool(value1.match(value2))
return value1 == value2
def add(self, key, dist):
"""Adds a distance penalty. `key` must correspond with a
configured weight setting. `dist` must be a float between 0.0
and 1.0, and will be added to any existing distance penalties
for the same key.
"""
if not 0.0 <= dist <= 1.0:
raise ValueError(
'`dist` must be between 0.0 and 1.0. It is: %r' % dist)
self._penalties.setdefault(key, []).append(dist)
def add_equality(self, key, value, options):
"""Adds a distance penalty of 1.0 if `value` doesn't match any
of the values in `options`. If an option is a compiled regular
expression, it will be considered equal if it matches against
`value`.
"""
if not isinstance(options, (list, tuple)):
options = [options]
for opt in options:
if self._eq(opt, value):
dist = 0.0
break
else:
dist = 1.0
self.add(key, dist)
def add_expr(self, key, expr):
"""Adds a distance penalty of 1.0 if `expr` evaluates to True,
or 0.0.
"""
if expr:
self.add(key, 1.0)
else:
self.add(key, 0.0)
def add_number(self, key, number1, number2):
"""Adds a distance penalty of 1.0 for each number of difference
between `number1` and `number2`, or 0.0 when there is no
difference. Use this when there is no upper limit on the
difference between the two numbers.
"""
diff = abs(number1 - number2)
if diff:
for i in range(diff):
self.add(key, 1.0)
else:
self.add(key, 0.0)
def add_priority(self, key, value, options):
"""Adds a distance penalty that corresponds to the position at
which `value` appears in `options`. A distance penalty of 0.0
for the first option, or 1.0 if there is no matching option. If
an option is a compiled regular expression, it will be
considered equal if it matches against `value`.
"""
if not isinstance(options, (list, tuple)):
options = [options]
unit = 1.0 / (len(options) or 1)
for i, opt in enumerate(options):
if self._eq(opt, value):
dist = i * unit
break
else:
dist = 1.0
self.add(key, dist)
def add_ratio(self, key, number1, number2):
"""Adds a distance penalty for `number1` as a ratio of `number2`.
`number1` is bound at 0 and `number2`.
"""
number = float(max(min(number1, number2), 0))
if number2:
dist = number / number2
else:
dist = 0.0
self.add(key, dist)
def add_string(self, key, str1, str2):
"""Adds a distance penalty based on the edit distance between
`str1` and `str2`.
"""
dist = string_dist(str1, str2)
self.add(key, dist)
# Structures that compose all the information for a candidate match.
AlbumMatch = namedtuple('AlbumMatch', ['distance', 'info', 'mapping',
'extra_items', 'extra_tracks'])
TrackMatch = namedtuple('TrackMatch', ['distance', 'info'])
# Aggregation of sources.
def album_for_mbid(release_id):
"""Get an AlbumInfo object for a MusicBrainz release ID. Return None
if the ID is not found.
"""
try:
return mb.album_for_id(release_id)
except mb.MusicBrainzAPIError as exc:
exc.log(log)
def track_for_mbid(recording_id):
"""Get a TrackInfo object for a MusicBrainz recording ID. Return None
if the ID is not found.
"""
try:
return mb.track_for_id(recording_id)
except mb.MusicBrainzAPIError as exc:
exc.log(log)
def albums_for_id(album_id):
"""Get a list of albums for an ID."""
candidates = [album_for_mbid(album_id)]
candidates.extend(plugins.album_for_id(album_id))
return filter(None, candidates)
def tracks_for_id(track_id):
"""Get a list of tracks for an ID."""
candidates = [track_for_mbid(track_id)]
candidates.extend(plugins.track_for_id(track_id))
return filter(None, candidates)
def album_candidates(items, artist, album, va_likely):
"""Search for album matches. ``items`` is a list of Item objects
that make up the album. ``artist`` and ``album`` are the respective
names (strings), which may be derived from the item list or may be
entered by the user. ``va_likely`` is a boolean indicating whether
the album is likely to be a "various artists" release.
"""
out = []
# Base candidates if we have album and artist to match.
if artist and album:
try:
out.extend(mb.match_album(artist, album, len(items)))
except mb.MusicBrainzAPIError as exc:
exc.log(log)
# Also add VA matches from MusicBrainz where appropriate.
if va_likely and album:
try:
out.extend(mb.match_album(None, album, len(items)))
except mb.MusicBrainzAPIError as exc:
exc.log(log)
# Candidates from plugins.
out.extend(plugins.candidates(items, artist, album, va_likely))
return out
def item_candidates(item, artist, title):
"""Search for item matches. ``item`` is the Item to be matched.
``artist`` and ``title`` are strings and either reflect the item or
are specified by the user.
"""
out = []
# MusicBrainz candidates.
if artist and title:
try:
out.extend(mb.match_track(artist, title))
except mb.MusicBrainzAPIError as exc:
exc.log(log)
# Plugin candidates.
out.extend(plugins.item_candidates(item, artist, title))
return out

456
libs/beets/autotag/match.py Normal file
View file

@ -0,0 +1,456 @@
# This file is part of beets.
# Copyright 2013, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Matches existing metadata with canonical information to identify
releases and tracks.
"""
from __future__ import division
import datetime
import logging
import re
from munkres import Munkres
from beets import plugins
from beets import config
from beets.util import plurality
from beets.util.enumeration import enum
from beets.autotag import hooks
# Recommendation enumeration.
recommendation = enum('none', 'low', 'medium', 'strong', name='recommendation')
# Artist signals that indicate "various artists". These are used at the
# album level to determine whether a given release is likely a VA
# release and also on the track level to to remove the penalty for
# differing artists.
VA_ARTISTS = (u'', u'various artists', u'various', u'va', u'unknown')
# Global logger.
log = logging.getLogger('beets')
# Primary matching functionality.
def current_metadata(items):
"""Extract the likely current metadata for an album given a list of its
items. Return two dictionaries:
- The most common value for each field.
- Whether each field's value was unanimous (values are booleans).
"""
assert items # Must be nonempty.
likelies = {}
consensus = {}
fields = ['artist', 'album', 'albumartist', 'year', 'disctotal',
'mb_albumid', 'label', 'catalognum', 'country', 'media',
'albumdisambig']
for key in fields:
values = [getattr(item, key) for item in items if item]
likelies[key], freq = plurality(values)
consensus[key] = (freq == len(values))
# If there's an album artist consensus, use this for the artist.
if consensus['albumartist'] and likelies['albumartist']:
likelies['artist'] = likelies['albumartist']
return likelies, consensus
def assign_items(items, tracks):
"""Given a list of Items and a list of TrackInfo objects, find the
best mapping between them. Returns a mapping from Items to TrackInfo
objects, a set of extra Items, and a set of extra TrackInfo
objects. These "extra" objects occur when there is an unequal number
of objects of the two types.
"""
# Construct the cost matrix.
costs = []
for item in items:
row = []
for i, track in enumerate(tracks):
row.append(track_distance(item, track))
costs.append(row)
# Find a minimum-cost bipartite matching.
matching = Munkres().compute(costs)
# Produce the output matching.
mapping = dict((items[i], tracks[j]) for (i, j) in matching)
extra_items = list(set(items) - set(mapping.keys()))
extra_items.sort(key=lambda i: (i.disc, i.track, i.title))
extra_tracks = list(set(tracks) - set(mapping.values()))
extra_tracks.sort(key=lambda t: (t.index, t.title))
return mapping, extra_items, extra_tracks
def track_index_changed(item, track_info):
"""Returns True if the item and track info index is different. Tolerates
per disc and per release numbering.
"""
return item.track not in (track_info.medium_index, track_info.index)
def track_distance(item, track_info, incl_artist=False):
"""Determines the significance of a track metadata change. Returns a
Distance object. `incl_artist` indicates that a distance component should
be included for the track artist (i.e., for various-artist releases).
"""
dist = hooks.Distance()
# Length.
if track_info.length:
diff = abs(item.length - track_info.length) - \
config['match']['track_length_grace'].as_number()
dist.add_ratio('track_length', diff,
config['match']['track_length_max'].as_number())
# Title.
dist.add_string('track_title', item.title, track_info.title)
# Artist. Only check if there is actually an artist in the track data.
if incl_artist and track_info.artist and \
item.artist.lower() not in VA_ARTISTS:
dist.add_string('track_artist', item.artist, track_info.artist)
# Track index.
if track_info.index and item.track:
dist.add_expr('track_index', track_index_changed(item, track_info))
# Track ID.
if item.mb_trackid:
dist.add_expr('track_id', item.mb_trackid != track_info.track_id)
# Plugins.
dist.update(plugins.track_distance(item, track_info))
return dist
def distance(items, album_info, mapping):
"""Determines how "significant" an album metadata change would be.
Returns a Distance object. `album_info` is an AlbumInfo object
reflecting the album to be compared. `items` is a sequence of all
Item objects that will be matched (order is not important).
`mapping` is a dictionary mapping Items to TrackInfo objects; the
keys are a subset of `items` and the values are a subset of
`album_info.tracks`.
"""
likelies, _ = current_metadata(items)
dist = hooks.Distance()
# Artist, if not various.
if not album_info.va:
dist.add_string('artist', likelies['artist'], album_info.artist)
# Album.
dist.add_string('album', likelies['album'], album_info.album)
# Current or preferred media.
if album_info.media:
# Preferred media options.
patterns = config['match']['preferred']['media'].as_str_seq()
options = [re.compile(r'(\d+x)?(%s)' % pat, re.I) for pat in patterns]
if options:
dist.add_priority('media', album_info.media, options)
# Current media.
elif likelies['media']:
dist.add_equality('media', album_info.media, likelies['media'])
# Mediums.
if likelies['disctotal'] and album_info.mediums:
dist.add_number('mediums', likelies['disctotal'], album_info.mediums)
# Prefer earliest release.
if album_info.year and config['match']['preferred']['original_year']:
# Assume 1889 (earliest first gramophone discs) if we don't know the
# original year.
original = album_info.original_year or 1889
diff = abs(album_info.year - original)
diff_max = abs(datetime.date.today().year - original)
dist.add_ratio('year', diff, diff_max)
# Year.
elif likelies['year'] and album_info.year:
if likelies['year'] in (album_info.year, album_info.original_year):
# No penalty for matching release or original year.
dist.add('year', 0.0)
elif album_info.original_year:
# Prefer matchest closest to the release year.
diff = abs(likelies['year'] - album_info.year)
diff_max = abs(datetime.date.today().year -
album_info.original_year)
dist.add_ratio('year', diff, diff_max)
else:
# Full penalty when there is no original year.
dist.add('year', 1.0)
# Preferred countries.
patterns = config['match']['preferred']['countries'].as_str_seq()
options = [re.compile(pat, re.I) for pat in patterns]
if album_info.country and options:
dist.add_priority('country', album_info.country, options)
# Country.
elif likelies['country'] and album_info.country:
dist.add_string('country', likelies['country'], album_info.country)
# Label.
if likelies['label'] and album_info.label:
dist.add_string('label', likelies['label'], album_info.label)
# Catalog number.
if likelies['catalognum'] and album_info.catalognum:
dist.add_string('catalognum', likelies['catalognum'],
album_info.catalognum)
# Disambiguation.
if likelies['albumdisambig'] and album_info.albumdisambig:
dist.add_string('albumdisambig', likelies['albumdisambig'],
album_info.albumdisambig)
# Album ID.
if likelies['mb_albumid']:
dist.add_equality('album_id', likelies['mb_albumid'],
album_info.album_id)
# Tracks.
dist.tracks = {}
for item, track in mapping.iteritems():
dist.tracks[track] = track_distance(item, track, album_info.va)
dist.add('tracks', dist.tracks[track].distance)
# Missing tracks.
for i in range(len(album_info.tracks) - len(mapping)):
dist.add('missing_tracks', 1.0)
# Unmatched tracks.
for i in range(len(items) - len(mapping)):
dist.add('unmatched_tracks', 1.0)
# Plugins.
dist.update(plugins.album_distance(items, album_info, mapping))
return dist
def match_by_id(items):
"""If the items are tagged with a MusicBrainz album ID, returns an
AlbumInfo object for the corresponding album. Otherwise, returns
None.
"""
# Is there a consensus on the MB album ID?
albumids = [item.mb_albumid for item in items if item.mb_albumid]
if not albumids:
log.debug('No album IDs found.')
return None
# If all album IDs are equal, look up the album.
if bool(reduce(lambda x,y: x if x==y else (), albumids)):
albumid = albumids[0]
log.debug('Searching for discovered album ID: ' + albumid)
return hooks.album_for_mbid(albumid)
else:
log.debug('No album ID consensus.')
def _recommendation(results):
"""Given a sorted list of AlbumMatch or TrackMatch objects, return a
recommendation based on the results' distances.
If the recommendation is higher than the configured maximum for
an applied penalty, the recommendation will be downgraded to the
configured maximum for that penalty.
"""
if not results:
# No candidates: no recommendation.
return recommendation.none
# Basic distance thresholding.
min_dist = results[0].distance
if min_dist < config['match']['strong_rec_thresh'].as_number():
# Strong recommendation level.
rec = recommendation.strong
elif min_dist <= config['match']['medium_rec_thresh'].as_number():
# Medium recommendation level.
rec = recommendation.medium
elif len(results) == 1:
# Only a single candidate.
rec = recommendation.low
elif results[1].distance - min_dist >= \
config['match']['rec_gap_thresh'].as_number():
# Gap between first two candidates is large.
rec = recommendation.low
else:
# No conclusion. Return immediately. Can't be downgraded any further.
return recommendation.none
# Downgrade to the max rec if it is lower than the current rec for an
# applied penalty.
keys = set(min_dist.keys())
if isinstance(results[0], hooks.AlbumMatch):
for track_dist in min_dist.tracks.values():
keys.update(track_dist.keys())
max_rec_view = config['match']['max_rec']
for key in keys:
if key in max_rec_view.keys():
max_rec = max_rec_view[key].as_choice({
'strong': recommendation.strong,
'medium': recommendation.medium,
'low': recommendation.low,
'none': recommendation.none,
})
rec = min(rec, max_rec)
return rec
def _add_candidate(items, results, info):
"""Given a candidate AlbumInfo object, attempt to add the candidate
to the output dictionary of AlbumMatch objects. This involves
checking the track count, ordering the items, checking for
duplicates, and calculating the distance.
"""
log.debug('Candidate: %s - %s' % (info.artist, info.album))
# Don't duplicate.
if info.album_id in results:
log.debug('Duplicate.')
return
# Find mapping between the items and the track info.
mapping, extra_items, extra_tracks = assign_items(items, info.tracks)
# Get the change distance.
dist = distance(items, info, mapping)
# Skip matches with ignored penalties.
penalties = [key for _, key in dist]
for penalty in config['match']['ignored'].as_str_seq():
if penalty in penalties:
log.debug('Ignored. Penalty: %s' % penalty)
return
log.debug('Success. Distance: %f' % dist)
results[info.album_id] = hooks.AlbumMatch(dist, info, mapping,
extra_items, extra_tracks)
def tag_album(items, search_artist=None, search_album=None,
search_id=None):
"""Bundles together the functionality used to infer tags for a
set of items comprised by an album. Returns everything relevant:
- The current artist.
- The current album.
- A list of AlbumMatch objects. The candidates are sorted by
distance (i.e., best match first).
- A recommendation.
If search_artist and search_album or search_id are provided, then
they are used as search terms in place of the current metadata.
"""
# Get current metadata.
likelies, consensus = current_metadata(items)
cur_artist = likelies['artist']
cur_album = likelies['album']
log.debug('Tagging %s - %s' % (cur_artist, cur_album))
# The output result (distance, AlbumInfo) tuples (keyed by MB album
# ID).
candidates = {}
# Search by explicit ID.
if search_id is not None:
log.debug('Searching for album ID: ' + search_id)
search_cands = hooks.albums_for_id(search_id)
# Use existing metadata or text search.
else:
# Try search based on current ID.
id_info = match_by_id(items)
if id_info:
_add_candidate(items, candidates, id_info)
rec = _recommendation(candidates.values())
log.debug('Album ID match recommendation is ' + str(rec))
if candidates and not config['import']['timid']:
# If we have a very good MBID match, return immediately.
# Otherwise, this match will compete against metadata-based
# matches.
if rec == recommendation.strong:
log.debug('ID match.')
return cur_artist, cur_album, candidates.values(), rec
# Search terms.
if not (search_artist and search_album):
# No explicit search terms -- use current metadata.
search_artist, search_album = cur_artist, cur_album
log.debug(u'Search terms: %s - %s' % (search_artist, search_album))
# Is this album likely to be a "various artist" release?
va_likely = ((not consensus['artist']) or
(search_artist.lower() in VA_ARTISTS) or
any(item.comp for item in items))
log.debug(u'Album might be VA: %s' % str(va_likely))
# Get the results from the data sources.
search_cands = hooks.album_candidates(items, search_artist,
search_album, va_likely)
log.debug(u'Evaluating %i candidates.' % len(search_cands))
for info in search_cands:
_add_candidate(items, candidates, info)
# Sort and get the recommendation.
candidates = sorted(candidates.itervalues())
rec = _recommendation(candidates)
return cur_artist, cur_album, candidates, rec
def tag_item(item, search_artist=None, search_title=None,
search_id=None):
"""Attempts to find metadata for a single track. Returns a
`(candidates, recommendation)` pair where `candidates` is a list of
TrackMatch objects. `search_artist` and `search_title` may be used
to override the current metadata for the purposes of the MusicBrainz
title; likewise `search_id`.
"""
# Holds candidates found so far: keys are MBIDs; values are
# (distance, TrackInfo) pairs.
candidates = {}
# First, try matching by MusicBrainz ID.
trackid = search_id or item.mb_trackid
if trackid:
log.debug('Searching for track ID: ' + trackid)
for track_info in hooks.tracks_for_id(trackid):
dist = track_distance(item, track_info, incl_artist=True)
candidates[track_info.track_id] = \
hooks.TrackMatch(dist, track_info)
# If this is a good match, then don't keep searching.
rec = _recommendation(candidates.values())
if rec == recommendation.strong and not config['import']['timid']:
log.debug('Track ID match.')
return candidates.values(), rec
# If we're searching by ID, don't proceed.
if search_id is not None:
if candidates:
return candidates.values(), rec
else:
return [], recommendation.none
# Search terms.
if not (search_artist and search_title):
search_artist, search_title = item.artist, item.title
log.debug(u'Item search terms: %s - %s' % (search_artist, search_title))
# Get and evaluate candidate metadata.
for track_info in hooks.item_candidates(item, search_artist, search_title):
dist = track_distance(item, track_info, incl_artist=True)
candidates[track_info.track_id] = hooks.TrackMatch(dist, track_info)
# Sort by distance and return with recommendation.
log.debug('Found %i candidates.' % len(candidates))
candidates = sorted(candidates.itervalues())
rec = _recommendation(candidates)
return candidates, rec

390
libs/beets/autotag/mb.py Normal file
View file

@ -0,0 +1,390 @@
# This file is part of beets.
# Copyright 2013, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Searches for albums in the MusicBrainz database.
"""
import logging
import musicbrainzngs
import re
import traceback
from urlparse import urljoin
import beets.autotag.hooks
import beets
from beets import util
from beets import config
SEARCH_LIMIT = 5
VARIOUS_ARTISTS_ID = '89ad4ac3-39f7-470e-963a-56509c546377'
BASE_URL = 'http://musicbrainz.org/'
musicbrainzngs.set_useragent('beets', beets.__version__,
'http://beets.radbox.org/')
class MusicBrainzAPIError(util.HumanReadableException):
"""An error while talking to MusicBrainz. The `query` field is the
parameter to the action and may have any type.
"""
def __init__(self, reason, verb, query, tb=None):
self.query = query
super(MusicBrainzAPIError, self).__init__(reason, verb, tb)
def get_message(self):
return u'"{0}" in {1} with query {2}'.format(
self._reasonstr(), self.verb, repr(self.query)
)
log = logging.getLogger('beets')
RELEASE_INCLUDES = ['artists', 'media', 'recordings', 'release-groups',
'labels', 'artist-credits', 'aliases']
TRACK_INCLUDES = ['artists', 'aliases']
def track_url(trackid):
return urljoin(BASE_URL, 'recording/' + trackid)
def album_url(albumid):
return urljoin(BASE_URL, 'release/' + albumid)
def configure():
"""Set up the python-musicbrainz-ngs module according to settings
from the beets configuration. This should be called at startup.
"""
musicbrainzngs.set_hostname(config['musicbrainz']['host'].get(unicode))
musicbrainzngs.set_rate_limit(
config['musicbrainz']['ratelimit_interval'].as_number(),
config['musicbrainz']['ratelimit'].get(int),
)
def _preferred_alias(aliases):
"""Given an list of alias structures for an artist credit, select
and return the user's preferred alias alias or None if no matching
alias is found.
"""
if not aliases:
return
# Only consider aliases that have locales set.
aliases = [a for a in aliases if 'locale' in a]
# Search configured locales in order.
for locale in config['import']['languages'].as_str_seq():
# Find matching primary aliases for this locale.
matches = [a for a in aliases if a['locale'] == locale and 'primary' in a]
# Skip to the next locale if we have no matches
if not matches:
continue
return matches[0]
def _flatten_artist_credit(credit):
"""Given a list representing an ``artist-credit`` block, flatten the
data into a triple of joined artist name strings: canonical, sort, and
credit.
"""
artist_parts = []
artist_sort_parts = []
artist_credit_parts = []
for el in credit:
if isinstance(el, basestring):
# Join phrase.
artist_parts.append(el)
artist_credit_parts.append(el)
artist_sort_parts.append(el)
else:
alias = _preferred_alias(el['artist'].get('alias-list', ()))
# An artist.
if alias:
cur_artist_name = alias['alias']
else:
cur_artist_name = el['artist']['name']
artist_parts.append(cur_artist_name)
# Artist sort name.
if alias:
artist_sort_parts.append(alias['sort-name'])
elif 'sort-name' in el['artist']:
artist_sort_parts.append(el['artist']['sort-name'])
else:
artist_sort_parts.append(cur_artist_name)
# Artist credit.
if 'name' in el:
artist_credit_parts.append(el['name'])
else:
artist_credit_parts.append(cur_artist_name)
return (
''.join(artist_parts),
''.join(artist_sort_parts),
''.join(artist_credit_parts),
)
def track_info(recording, index=None, medium=None, medium_index=None,
medium_total=None):
"""Translates a MusicBrainz recording result dictionary into a beets
``TrackInfo`` object. Three parameters are optional and are used
only for tracks that appear on releases (non-singletons): ``index``,
the overall track number; ``medium``, the disc number;
``medium_index``, the track's index on its medium; ``medium_total``,
the number of tracks on the medium. Each number is a 1-based index.
"""
info = beets.autotag.hooks.TrackInfo(
recording['title'],
recording['id'],
index=index,
medium=medium,
medium_index=medium_index,
medium_total=medium_total,
data_url=track_url(recording['id']),
)
if recording.get('artist-credit'):
# Get the artist names.
info.artist, info.artist_sort, info.artist_credit = \
_flatten_artist_credit(recording['artist-credit'])
# Get the ID and sort name of the first artist.
artist = recording['artist-credit'][0]['artist']
info.artist_id = artist['id']
if recording.get('length'):
info.length = int(recording['length']) / (1000.0)
info.decode()
return info
def _set_date_str(info, date_str, original=False):
"""Given a (possibly partial) YYYY-MM-DD string and an AlbumInfo
object, set the object's release date fields appropriately. If
`original`, then set the original_year, etc., fields.
"""
if date_str:
date_parts = date_str.split('-')
for key in ('year', 'month', 'day'):
if date_parts:
date_part = date_parts.pop(0)
try:
date_num = int(date_part)
except ValueError:
continue
if original:
key = 'original_' + key
setattr(info, key, date_num)
def album_info(release):
"""Takes a MusicBrainz release result dictionary and returns a beets
AlbumInfo object containing the interesting data about that release.
"""
# Get artist name using join phrases.
artist_name, artist_sort_name, artist_credit_name = \
_flatten_artist_credit(release['artist-credit'])
# Basic info.
track_infos = []
index = 0
for medium in release['medium-list']:
disctitle = medium.get('title')
for track in medium['track-list']:
# Basic information from the recording.
index += 1
ti = track_info(
track['recording'],
index,
int(medium['position']),
int(track['position']),
len(medium['track-list']),
)
ti.disctitle = disctitle
# Prefer track data, where present, over recording data.
if track.get('title'):
ti.title = track['title']
if track.get('artist-credit'):
# Get the artist names.
ti.artist, ti.artist_sort, ti.artist_credit = \
_flatten_artist_credit(track['artist-credit'])
ti.artist_id = track['artist-credit'][0]['artist']['id']
if track.get('length'):
ti.length = int(track['length']) / (1000.0)
track_infos.append(ti)
info = beets.autotag.hooks.AlbumInfo(
release['title'],
release['id'],
artist_name,
release['artist-credit'][0]['artist']['id'],
track_infos,
mediums=len(release['medium-list']),
artist_sort=artist_sort_name,
artist_credit=artist_credit_name,
data_source='MusicBrainz',
data_url=album_url(release['id']),
)
info.va = info.artist_id == VARIOUS_ARTISTS_ID
info.asin = release.get('asin')
info.releasegroup_id = release['release-group']['id']
info.country = release.get('country')
info.albumstatus = release.get('status')
# Build up the disambiguation string from the release group and release.
disambig = []
if release['release-group'].get('disambiguation'):
disambig.append(release['release-group'].get('disambiguation'))
if release.get('disambiguation'):
disambig.append(release.get('disambiguation'))
info.albumdisambig = u', '.join(disambig)
# Release type not always populated.
if 'type' in release['release-group']:
reltype = release['release-group']['type']
if reltype:
info.albumtype = reltype.lower()
# Release dates.
release_date = release.get('date')
release_group_date = release['release-group'].get('first-release-date')
if not release_date:
# Fall back if release-specific date is not available.
release_date = release_group_date
_set_date_str(info, release_date, False)
_set_date_str(info, release_group_date, True)
# Label name.
if release.get('label-info-list'):
label_info = release['label-info-list'][0]
if label_info.get('label'):
label = label_info['label']['name']
if label != '[no label]':
info.label = label
info.catalognum = label_info.get('catalog-number')
# Text representation data.
if release.get('text-representation'):
rep = release['text-representation']
info.script = rep.get('script')
info.language = rep.get('language')
# Media (format).
if release['medium-list']:
first_medium = release['medium-list'][0]
info.media = first_medium.get('format')
info.decode()
return info
def match_album(artist, album, tracks=None, limit=SEARCH_LIMIT):
"""Searches for a single album ("release" in MusicBrainz parlance)
and returns an iterator over AlbumInfo objects. May raise a
MusicBrainzAPIError.
The query consists of an artist name, an album name, and,
optionally, a number of tracks on the album.
"""
# Build search criteria.
criteria = {'release': album.lower()}
if artist is not None:
criteria['artist'] = artist.lower()
else:
# Various Artists search.
criteria['arid'] = VARIOUS_ARTISTS_ID
if tracks is not None:
criteria['tracks'] = str(tracks)
# Abort if we have no search terms.
if not any(criteria.itervalues()):
return
try:
res = musicbrainzngs.search_releases(limit=limit, **criteria)
except musicbrainzngs.MusicBrainzError as exc:
raise MusicBrainzAPIError(exc, 'release search', criteria,
traceback.format_exc())
for release in res['release-list']:
# The search result is missing some data (namely, the tracks),
# so we just use the ID and fetch the rest of the information.
albuminfo = album_for_id(release['id'])
if albuminfo is not None:
yield albuminfo
def match_track(artist, title, limit=SEARCH_LIMIT):
"""Searches for a single track and returns an iterable of TrackInfo
objects. May raise a MusicBrainzAPIError.
"""
criteria = {
'artist': artist.lower(),
'recording': title.lower(),
}
if not any(criteria.itervalues()):
return
try:
res = musicbrainzngs.search_recordings(limit=limit, **criteria)
except musicbrainzngs.MusicBrainzError as exc:
raise MusicBrainzAPIError(exc, 'recording search', criteria,
traceback.format_exc())
for recording in res['recording-list']:
yield track_info(recording)
def _parse_id(s):
"""Search for a MusicBrainz ID in the given string and return it. If
no ID can be found, return None.
"""
# Find the first thing that looks like a UUID/MBID.
match = re.search('[a-f0-9]{8}(-[a-f0-9]{4}){3}-[a-f0-9]{12}', s)
if match:
return match.group()
def album_for_id(albumid):
"""Fetches an album by its MusicBrainz ID and returns an AlbumInfo
object or None if the album is not found. May raise a
MusicBrainzAPIError.
"""
albumid = _parse_id(albumid)
if not albumid:
log.error('Invalid MBID.')
return
try:
res = musicbrainzngs.get_release_by_id(albumid,
RELEASE_INCLUDES)
except musicbrainzngs.ResponseError:
log.debug('Album ID match failed.')
return None
except musicbrainzngs.MusicBrainzError as exc:
raise MusicBrainzAPIError(exc, 'get release by ID', albumid,
traceback.format_exc())
return album_info(res['release'])
def track_for_id(trackid):
"""Fetches a track by its MusicBrainz ID. Returns a TrackInfo object
or None if no track is found. May raise a MusicBrainzAPIError.
"""
trackid = _parse_id(trackid)
if not trackid:
log.error('Invalid MBID.')
return
try:
res = musicbrainzngs.get_recording_by_id(trackid, TRACK_INCLUDES)
except musicbrainzngs.ResponseError:
log.debug('Track ID match failed.')
return None
except musicbrainzngs.MusicBrainzError as exc:
raise MusicBrainzAPIError(exc, 'get recording by ID', trackid,
traceback.format_exc())
return track_info(res['recording'])

102
libs/beets/config_default.yaml Normal file
View file

@ -0,0 +1,102 @@
library: library.db
directory: ~/Music
import:
write: yes
copy: yes
move: no
delete: no
resume: ask
incremental: no
quiet_fallback: skip
none_rec_action: ask
timid: no
log:
autotag: yes
quiet: no
singletons: no
default_action: apply
languages: []
detail: no
flat: no
group_albums: no
clutter: ["Thumbs.DB", ".DS_Store"]
ignore: [".*", "*~", "System Volume Information"]
replace:
'[\\/]': _
'^\.': _
'[\x00-\x1f]': _
'[<>:"\?\*\|]': _
'\.$': _
'\s+$': ''
'^\s+': ''
path_sep_replace: _
art_filename: cover
max_filename_length: 0
plugins: []
pluginpath: []
threaded: yes
color: yes
timeout: 5.0
per_disc_numbering: no
verbose: no
terminal_encoding: utf8
original_date: no
id3v23: no
ui:
terminal_width: 80
length_diff_thresh: 10.0
list_format_item: $artist - $album - $title
list_format_album: $albumartist - $album
time_format: '%Y-%m-%d %H:%M:%S'
paths:
default: $albumartist/$album%aunique{}/$track $title
singleton: Non-Album/$artist/$title
comp: Compilations/$album%aunique{}/$track $title
statefile: state.pickle
musicbrainz:
host: musicbrainz.org
ratelimit: 1
ratelimit_interval: 1.0
match:
strong_rec_thresh: 0.04
medium_rec_thresh: 0.25
rec_gap_thresh: 0.25
max_rec:
missing_tracks: medium
unmatched_tracks: medium
distance_weights:
source: 2.0
artist: 3.0
album: 3.0
media: 1.0
mediums: 1.0
year: 1.0
country: 0.5
label: 0.5
catalognum: 0.5
albumdisambig: 0.5
album_id: 5.0
tracks: 2.0
missing_tracks: 0.9
unmatched_tracks: 0.6
track_title: 3.0
track_artist: 2.0
track_index: 1.0
track_length: 2.0
track_id: 5.0
preferred:
countries: []
media: []
original_year: no
ignored: []
track_length_grace: 10
track_length_max: 30

20
libs/beets/dbcore/__init__.py Normal file
View file

@ -0,0 +1,20 @@
# This file is part of beets.
# Copyright 2014, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""DBCore is an abstract database package that forms the basis for beets'
Library.
"""
from .db import Model, Database
from .query import Query, FieldQuery, MatchQuery, AndQuery, OrQuery
from .types import Type

727
libs/beets/dbcore/db.py Normal file
View file

@ -0,0 +1,727 @@
# This file is part of beets.
# Copyright 2014, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""The central Model and Database constructs for DBCore.
"""
import time
import os
from collections import defaultdict
import threading
import sqlite3
import contextlib
import beets
from beets.util.functemplate import Template
from .query import MatchQuery
# Abstract base for model classes.
class Model(object):
"""An abstract object representing an object in the database. Model
objects act like dictionaries (i.e., the allow subscript access like
``obj['field']``). The same field set is available via attribute
access as a shortcut (i.e., ``obj.field``). Three kinds of attributes are
available:
* **Fixed attributes** come from a predetermined list of field
names. These fields correspond to SQLite table columns and are
thus fast to read, write, and query.
* **Flexible attributes** are free-form and do not need to be listed
ahead of time.
* **Computed attributes** are read-only fields computed by a getter
function provided by a plugin.
Access to all three field types is uniform: ``obj.field`` works the
same regardless of whether ``field`` is fixed, flexible, or
computed.
Model objects can optionally be associated with a `Library` object,
in which case they can be loaded and stored from the database. Dirty
flags are used to track which fields need to be stored.
"""
# Abstract components (to be provided by subclasses).
_table = None
"""The main SQLite table name.
"""
_flex_table = None
"""The flex field SQLite table name.
"""
_fields = {}
"""A mapping indicating available "fixed" fields on this type. The
keys are field names and the values are Type objects.
"""
_bytes_keys = ()
"""Keys whose values should be stored as raw bytes blobs rather than
strings.
"""
_search_fields = ()
"""The fields that should be queried by default by unqualified query
terms.
"""
@classmethod
def _getters(cls):
"""Return a mapping from field names to getter functions.
"""
# We could cache this if it becomes a performance problem to
# gather the getter mapping every time.
raise NotImplementedError()
def _template_funcs(self):
"""Return a mapping from function names to text-transformer
functions.
"""
# As above: we could consider caching this result.
raise NotImplementedError()
# Basic operation.
def __init__(self, db=None, **values):
"""Create a new object with an optional Database association and
initial field values.
"""
self._db = db
self._dirty = set()
self._values_fixed = {}
self._values_flex = {}
# Initial contents.
self.update(values)
self.clear_dirty()
def __repr__(self):
return '{0}({1})'.format(
type(self).__name__,
', '.join('{0}={1!r}'.format(k, v) for k, v in dict(self).items()),
)
def clear_dirty(self):
"""Mark all fields as *clean* (i.e., not needing to be stored to
the database).
"""
self._dirty = set()
def _check_db(self, need_id=True):
"""Ensure that this object is associated with a database row: it
has a reference to a database (`_db`) and an id. A ValueError
exception is raised otherwise.
"""
if not self._db:
raise ValueError('{0} has no database'.format(type(self).__name__))
if need_id and not self.id:
raise ValueError('{0} has no id'.format(type(self).__name__))
# Essential field accessors.
def __getitem__(self, key):
"""Get the value for a field. Raise a KeyError if the field is
not available.
"""
getters = self._getters()
if key in getters: # Computed.
return getters[key](self)
elif key in self._fields: # Fixed.
return self._values_fixed.get(key)
elif key in self._values_flex: # Flexible.
return self._values_flex[key]
else:
raise KeyError(key)
def __setitem__(self, key, value):
"""Assign the value for a field.
"""
source = self._values_fixed if key in self._fields \
else self._values_flex
old_value = source.get(key)
source[key] = value
if old_value != value:
self._dirty.add(key)
def __delitem__(self, key):
"""Remove a flexible attribute from the model.
"""
if key in self._values_flex: # Flexible.
del self._values_flex[key]
self._dirty.add(key) # Mark for dropping on store.
elif key in self._getters(): # Computed.
raise KeyError('computed field {0} cannot be deleted'.format(key))
elif key in self._fields: # Fixed.
raise KeyError('fixed field {0} cannot be deleted'.format(key))
else:
raise KeyError('no such field {0}'.format(key))
def keys(self, computed=False):
"""Get a list of available field names for this object. The
`computed` parameter controls whether computed (plugin-provided)
fields are included in the key list.
"""
base_keys = list(self._fields) + self._values_flex.keys()
if computed:
return base_keys + self._getters().keys()
else:
return base_keys
# Act like a dictionary.
def update(self, values):
"""Assign all values in the given dict.
"""
for key, value in values.items():
self[key] = value
def items(self):
"""Iterate over (key, value) pairs that this object contains.
Computed fields are not included.
"""
for key in self:
yield key, self[key]
def get(self, key, default=None):
"""Get the value for a given key or `default` if it does not
exist.
"""
if key in self:
return self[key]
else:
return default
def __contains__(self, key):
"""Determine whether `key` is an attribute on this object.
"""
return key in self.keys(True)
def __iter__(self):
"""Iterate over the available field names (excluding computed
fields).
"""
return iter(self.keys())
# Convenient attribute access.
def __getattr__(self, key):
if key.startswith('_'):
raise AttributeError('model has no attribute {0!r}'.format(key))
else:
try:
return self[key]
except KeyError:
raise AttributeError('no such field {0!r}'.format(key))
def __setattr__(self, key, value):
if key.startswith('_'):
super(Model, self).__setattr__(key, value)
else:
self[key] = value
def __delattr__(self, key):
if key.startswith('_'):
super(Model, self).__delattr__(key)
else:
del self[key]
# Database interaction (CRUD methods).
def store(self):
"""Save the object's metadata into the library database.
"""
self._check_db()
# Build assignments for query.
assignments = ''
subvars = []
for key in self._fields:
if key != 'id' and key in self._dirty:
self._dirty.remove(key)
assignments += key + '=?,'
value = self[key]
# Wrap path strings in buffers so they get stored
# "in the raw".
if key in self._bytes_keys and isinstance(value, str):
value = buffer(value)
subvars.append(value)
assignments = assignments[:-1] # Knock off last ,
with self._db.transaction() as tx:
# Main table update.
if assignments:
query = 'UPDATE {0} SET {1} WHERE id=?'.format(
self._table, assignments
)
subvars.append(self.id)
tx.mutate(query, subvars)
# Modified/added flexible attributes.
for key, value in self._values_flex.items():
if key in self._dirty:
self._dirty.remove(key)
tx.mutate(
'INSERT INTO {0} '
'(entity_id, key, value) '
'VALUES (?, ?, ?);'.format(self._flex_table),
(self.id, key, value),
)
# Deleted flexible attributes.
for key in self._dirty:
tx.mutate(
'DELETE FROM {0} '
'WHERE entity_id=? AND key=?'.format(self._flex_table),
(self.id, key)
)
self.clear_dirty()
def load(self):
"""Refresh the object's metadata from the library database.
"""
self._check_db()
stored_obj = self._db._get(type(self), self.id)
assert stored_obj is not None, "object {0} not in DB".format(self.id)
self.update(dict(stored_obj))
self.clear_dirty()
def remove(self):
"""Remove the object's associated rows from the database.
"""
self._check_db()
with self._db.transaction() as tx:
tx.mutate(
'DELETE FROM {0} WHERE id=?'.format(self._table),
(self.id,)
)
tx.mutate(
'DELETE FROM {0} WHERE entity_id=?'.format(self._flex_table),
(self.id,)
)
def add(self, db=None):
"""Add the object to the library database. This object must be
associated with a database; you can provide one via the `db`
parameter or use the currently associated database.
The object's `id` and `added` fields are set along with any
current field values.
"""
if db:
self._db = db
self._check_db(False)
with self._db.transaction() as tx:
new_id = tx.mutate(
'INSERT INTO {0} DEFAULT VALUES'.format(self._table)
)
self.id = new_id
self.added = time.time()
# Mark every non-null field as dirty and store.
for key in self:
if self[key] is not None:
self._dirty.add(key)
self.store()
# Formatting and templating.
@classmethod
def _format(cls, key, value, for_path=False):
"""Format a value as the given field for this model.
"""
# Format the value as a string according to its type, if any.
if key in cls._fields:
value = cls._fields[key].format(value)
# Formatting must result in a string. To deal with
# Python2isms, implicitly convert ASCII strings.
assert isinstance(value, basestring), \
u'field formatter must produce strings'
if isinstance(value, bytes):
value = value.decode('utf8', 'ignore')
elif not isinstance(value, unicode):
# Fallback formatter. Convert to unicode at all cost.
if value is None:
value = u''
elif isinstance(value, basestring):
if isinstance(value, bytes):
value = value.decode('utf8', 'ignore')
else:
value = unicode(value)
if for_path:
sep_repl = beets.config['path_sep_replace'].get(unicode)
for sep in (os.path.sep, os.path.altsep):
if sep:
value = value.replace(sep, sep_repl)
return value
def _get_formatted(self, key, for_path=False):
"""Get a field value formatted as a string (`unicode` object)
for display to the user. If `for_path` is true, then the value
will be sanitized for inclusion in a pathname (i.e., path
separators will be removed from the value).
"""
return self._format(key, self.get(key), for_path)
def _formatted_mapping(self, for_path=False):
"""Get a mapping containing all values on this object formatted
as human-readable strings.
"""
# In the future, this could be made "lazy" to avoid computing
# fields unnecessarily.
out = {}
for key in self.keys(True):
out[key] = self._get_formatted(key, for_path)
return out
def evaluate_template(self, template, for_path=False):
"""Evaluate a template (a string or a `Template` object) using
the object's fields. If `for_path` is true, then no new path
separators will be added to the template.
"""
# Build value mapping.
mapping = self._formatted_mapping(for_path)
# Get template functions.
funcs = self._template_funcs()
# Perform substitution.
if isinstance(template, basestring):
template = Template(template)
return template.substitute(mapping, funcs)
# Parsing.
@classmethod
def _parse(cls, key, string):
"""Parse a string as a value for the given key.
"""
if not isinstance(string, basestring):
raise TypeError("_parse() argument must be a string")
typ = cls._fields.get(key)
if typ:
return typ.parse(string)
else:
# Fall back to unparsed string.
return string
# Database controller and supporting interfaces.
class Results(object):
"""An item query result set. Iterating over the collection lazily
constructs LibModel objects that reflect database rows.
"""
def __init__(self, model_class, rows, db, query=None):
"""Create a result set that will construct objects of type
`model_class`, which should be a subclass of `LibModel`, out of
the query result mapping in `rows`. The new objects are
associated with the database `db`. If `query` is provided, it is
used as a predicate to filter the results for a "slow query" that
cannot be evaluated by the database directly.
"""
self.model_class = model_class
self.rows = rows
self.db = db
self.query = query
def __iter__(self):
"""Construct Python objects for all rows that pass the query
predicate.
"""
for row in self.rows:
# Get the flexible attributes for the object.
with self.db.transaction() as tx:
flex_rows = tx.query(
'SELECT * FROM {0} WHERE entity_id=?'.format(
self.model_class._flex_table
),
(row['id'],)
)
values = dict(row)
values.update(
dict((row['key'], row['value']) for row in flex_rows)
)
# Construct the Python object and yield it if it passes the
# predicate.
obj = self.model_class(self.db, **values)
if not self.query or self.query.match(obj):
yield obj
def __len__(self):
"""Get the number of matching objects.
"""
if self.query:
# A slow query. Fall back to testing every object.
count = 0
for obj in self:
count += 1
return count
else:
# A fast query. Just count the rows.
return len(self.rows)
def __nonzero__(self):
"""Does this result contain any objects?
"""
return bool(len(self))
def __getitem__(self, n):
"""Get the nth item in this result set. This is inefficient: all
items up to n are materialized and thrown away.
"""
it = iter(self)
try:
for i in range(n):
it.next()
return it.next()
except StopIteration:
raise IndexError('result index {0} out of range'.format(n))
def get(self):
"""Return the first matching object, or None if no objects
match.
"""
it = iter(self)
try:
return it.next()
except StopIteration:
return None
class Transaction(object):
"""A context manager for safe, concurrent access to the database.
All SQL commands should be executed through a transaction.
"""
def __init__(self, db):
self.db = db
def __enter__(self):
"""Begin a transaction. This transaction may be created while
another is active in a different thread.
"""
with self.db._tx_stack() as stack:
first = not stack
stack.append(self)
if first:
# Beginning a "root" transaction, which corresponds to an
# SQLite transaction.
self.db._db_lock.acquire()
return self
def __exit__(self, exc_type, exc_value, traceback):
"""Complete a transaction. This must be the most recently
entered but not yet exited transaction. If it is the last active
transaction, the database updates are committed.
"""
with self.db._tx_stack() as stack:
assert stack.pop() is self
empty = not stack
if empty:
# Ending a "root" transaction. End the SQLite transaction.
self.db._connection().commit()
self.db._db_lock.release()
def query(self, statement, subvals=()):
"""Execute an SQL statement with substitution values and return
a list of rows from the database.
"""
cursor = self.db._connection().execute(statement, subvals)
return cursor.fetchall()
def mutate(self, statement, subvals=()):
"""Execute an SQL statement with substitution values and return
the row ID of the last affected row.
"""
cursor = self.db._connection().execute(statement, subvals)
return cursor.lastrowid
def script(self, statements):
"""Execute a string containing multiple SQL statements."""
self.db._connection().executescript(statements)
class Database(object):
"""A container for Model objects that wraps an SQLite database as
the backend.
"""
_models = ()
"""The Model subclasses representing tables in this database.
"""
def __init__(self, path):
self.path = path
self._connections = {}
self._tx_stacks = defaultdict(list)
# A lock to protect the _connections and _tx_stacks maps, which
# both map thread IDs to private resources.
self._shared_map_lock = threading.Lock()
# A lock to protect access to the database itself. SQLite does
# allow multiple threads to access the database at the same
# time, but many users were experiencing crashes related to this
# capability: where SQLite was compiled without HAVE_USLEEP, its
# backoff algorithm in the case of contention was causing
# whole-second sleeps (!) that would trigger its internal
# timeout. Using this lock ensures only one SQLite transaction
# is active at a time.
self._db_lock = threading.Lock()
# Set up database schema.
for model_cls in self._models:
self._make_table(model_cls._table, model_cls._fields)
self._make_attribute_table(model_cls._flex_table)
# Primitive access control: connections and transactions.
def _connection(self):
"""Get a SQLite connection object to the underlying database.
One connection object is created per thread.
"""
thread_id = threading.current_thread().ident
with self._shared_map_lock:
if thread_id in self._connections:
return self._connections[thread_id]
else:
# Make a new connection.
conn = sqlite3.connect(
self.path,
timeout=beets.config['timeout'].as_number(),
)
# Access SELECT results like dictionaries.
conn.row_factory = sqlite3.Row
self._connections[thread_id] = conn
return conn
@contextlib.contextmanager
def _tx_stack(self):
"""A context manager providing access to the current thread's
transaction stack. The context manager synchronizes access to
the stack map. Transactions should never migrate across threads.
"""
thread_id = threading.current_thread().ident
with self._shared_map_lock:
yield self._tx_stacks[thread_id]
def transaction(self):
"""Get a :class:`Transaction` object for interacting directly
with the underlying SQLite database.
"""
return Transaction(self)
# Schema setup and migration.
def _make_table(self, table, fields):
"""Set up the schema of the database. `fields` is a mapping
from field names to `Type`s. Columns are added if necessary.
"""
# Get current schema.
with self.transaction() as tx:
rows = tx.query('PRAGMA table_info(%s)' % table)
current_fields = set([row[1] for row in rows])
field_names = set(fields.keys())
if current_fields.issuperset(field_names):
# Table exists and has all the required columns.
return
if not current_fields:
# No table exists.
columns = []
for name, typ in fields.items():
columns.append('{0} {1}'.format(name, typ.sql))
setup_sql = 'CREATE TABLE {0} ({1});\n'.format(table,
', '.join(columns))
else:
# Table exists does not match the field set.
setup_sql = ''
for name, typ in fields.items():
if name in current_fields:
continue
setup_sql += 'ALTER TABLE {0} ADD COLUMN {1} {2};\n'.format(
table, name, typ.sql
)
with self.transaction() as tx:
tx.script(setup_sql)
def _make_attribute_table(self, flex_table):
"""Create a table and associated index for flexible attributes
for the given entity (if they don't exist).
"""
with self.transaction() as tx:
tx.script("""
CREATE TABLE IF NOT EXISTS {0} (
id INTEGER PRIMARY KEY,
entity_id INTEGER,
key TEXT,
value TEXT,
UNIQUE(entity_id, key) ON CONFLICT REPLACE);
CREATE INDEX IF NOT EXISTS {0}_by_entity
ON {0} (entity_id);
""".format(flex_table))
# Querying.
def _fetch(self, model_cls, query, order_by=None):
"""Fetch the objects of type `model_cls` matching the given
query. The query may be given as a string, string sequence, a
Query object, or None (to fetch everything). If provided,
`order_by` is a SQLite ORDER BY clause for sorting.
"""
where, subvals = query.clause()
sql = "SELECT * FROM {0} WHERE {1}".format(
model_cls._table,
where or '1',
)
if order_by:
sql += " ORDER BY {0}".format(order_by)
with self.transaction() as tx:
rows = tx.query(sql, subvals)
return Results(model_cls, rows, self, None if where else query)
def _get(self, model_cls, id):
"""Get a Model object by its id or None if the id does not
exist.
"""
return self._fetch(model_cls, MatchQuery('id', id)).get()

494
libs/beets/dbcore/query.py Normal file
View file

@ -0,0 +1,494 @@
# This file is part of beets.
# Copyright 2014, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""The Query type hierarchy for DBCore.
"""
import re
from beets import util
from datetime import datetime, timedelta
class Query(object):
"""An abstract class representing a query into the item database.
"""
def clause(self):
"""Generate an SQLite expression implementing the query.
Return a clause string, a sequence of substitution values for
the clause, and a Query object representing the "remainder"
Returns (clause, subvals) where clause is a valid sqlite
WHERE clause implementing the query and subvals is a list of
items to be substituted for ?s in the clause.
"""
return None, ()
def match(self, item):
"""Check whether this query matches a given Item. Can be used to
perform queries on arbitrary sets of Items.
"""
raise NotImplementedError
class FieldQuery(Query):
"""An abstract query that searches in a specific field for a
pattern. Subclasses must provide a `value_match` class method, which
determines whether a certain pattern string matches a certain value
string. Subclasses may also provide `col_clause` to implement the
same matching functionality in SQLite.
"""
def __init__(self, field, pattern, fast=True):
self.field = field
self.pattern = pattern
self.fast = fast
def col_clause(self):
return None, ()
def clause(self):
if self.fast:
return self.col_clause()
else:
# Matching a flexattr. This is a slow query.
return None, ()
@classmethod
def value_match(cls, pattern, value):
"""Determine whether the value matches the pattern. Both
arguments are strings.
"""
raise NotImplementedError()
def match(self, item):
return self.value_match(self.pattern, item.get(self.field))
class MatchQuery(FieldQuery):
"""A query that looks for exact matches in an item field."""
def col_clause(self):
return self.field + " = ?", [self.pattern]
@classmethod
def value_match(cls, pattern, value):
return pattern == value
class StringFieldQuery(FieldQuery):
"""A FieldQuery that converts values to strings before matching
them.
"""
@classmethod
def value_match(cls, pattern, value):
"""Determine whether the value matches the pattern. The value
may have any type.
"""
return cls.string_match(pattern, util.as_string(value))
@classmethod
def string_match(cls, pattern, value):
"""Determine whether the value matches the pattern. Both
arguments are strings. Subclasses implement this method.
"""
raise NotImplementedError()
class SubstringQuery(StringFieldQuery):
"""A query that matches a substring in a specific item field."""
def col_clause(self):
search = '%' + (self.pattern.replace('\\','\\\\').replace('%','\\%')
.replace('_','\\_')) + '%'
clause = self.field + " like ? escape '\\'"
subvals = [search]
return clause, subvals
@classmethod
def string_match(cls, pattern, value):
return pattern.lower() in value.lower()
class RegexpQuery(StringFieldQuery):
"""A query that matches a regular expression in a specific item
field.
"""
@classmethod
def string_match(cls, pattern, value):
try:
res = re.search(pattern, value)
except re.error:
# Invalid regular expression.
return False
return res is not None
class BooleanQuery(MatchQuery):
"""Matches a boolean field. Pattern should either be a boolean or a
string reflecting a boolean.
"""
def __init__(self, field, pattern, fast=True):
super(BooleanQuery, self).__init__(field, pattern, fast)
if isinstance(pattern, basestring):
self.pattern = util.str2bool(pattern)
self.pattern = int(self.pattern)
class BytesQuery(MatchQuery):
"""Match a raw bytes field (i.e., a path). This is a necessary hack
to work around the `sqlite3` module's desire to treat `str` and
`unicode` equivalently in Python 2. Always use this query instead of
`MatchQuery` when matching on BLOB values.
"""
def __init__(self, field, pattern):
super(BytesQuery, self).__init__(field, pattern)
# Use a buffer representation of the pattern for SQLite
# matching. This instructs SQLite to treat the blob as binary
# rather than encoded Unicode.
if isinstance(self.pattern, basestring):
# Implicitly coerce Unicode strings to their bytes
# equivalents.
if isinstance(self.pattern, unicode):
self.pattern = self.pattern.encode('utf8')
self.buf_pattern = buffer(self.pattern)
elif isinstance(self.pattern, buffer):
self.buf_pattern = self.pattern
self.pattern = bytes(self.pattern)
def col_clause(self):
return self.field + " = ?", [self.buf_pattern]
class NumericQuery(FieldQuery):
"""Matches numeric fields. A syntax using Ruby-style range ellipses
(``..``) lets users specify one- or two-sided ranges. For example,
``year:2001..`` finds music released since the turn of the century.
"""
def _convert(self, s):
"""Convert a string to a numeric type (float or int). If the
string cannot be converted, return None.
"""
# This is really just a bit of fun premature optimization.
try:
return int(s)
except ValueError:
try:
return float(s)
except ValueError:
return None
def __init__(self, field, pattern, fast=True):
super(NumericQuery, self).__init__(field, pattern, fast)
parts = pattern.split('..', 1)
if len(parts) == 1:
# No range.
self.point = self._convert(parts[0])
self.rangemin = None
self.rangemax = None
else:
# One- or two-sided range.
self.point = None
self.rangemin = self._convert(parts[0])
self.rangemax = self._convert(parts[1])
def match(self, item):
value = getattr(item, self.field)
if isinstance(value, basestring):
value = self._convert(value)
if self.point is not None:
return value == self.point
else:
if self.rangemin is not None and value < self.rangemin:
return False
if self.rangemax is not None and value > self.rangemax:
return False
return True
def col_clause(self):
if self.point is not None:
return self.field + '=?', (self.point,)
else:
if self.rangemin is not None and self.rangemax is not None:
return (u'{0} >= ? AND {0} <= ?'.format(self.field),
(self.rangemin, self.rangemax))
elif self.rangemin is not None:
return u'{0} >= ?'.format(self.field), (self.rangemin,)
elif self.rangemax is not None:
return u'{0} <= ?'.format(self.field), (self.rangemax,)
else:
return '1', ()
class CollectionQuery(Query):
"""An abstract query class that aggregates other queries. Can be
indexed like a list to access the sub-queries.
"""
def __init__(self, subqueries=()):
self.subqueries = subqueries
# Act like a sequence.
def __len__(self):
return len(self.subqueries)
def __getitem__(self, key):
return self.subqueries[key]
def __iter__(self):
return iter(self.subqueries)
def __contains__(self, item):
return item in self.subqueries
def clause_with_joiner(self, joiner):
"""Returns a clause created by joining together the clauses of
all subqueries with the string joiner (padded by spaces).
"""
clause_parts = []
subvals = []
for subq in self.subqueries:
subq_clause, subq_subvals = subq.clause()
if not subq_clause:
# Fall back to slow query.
return None, ()
clause_parts.append('(' + subq_clause + ')')
subvals += subq_subvals
clause = (' ' + joiner + ' ').join(clause_parts)
return clause, subvals
class AnyFieldQuery(CollectionQuery):
"""A query that matches if a given FieldQuery subclass matches in
any field. The individual field query class is provided to the
constructor.
"""
def __init__(self, pattern, fields, cls):
self.pattern = pattern
self.fields = fields
self.query_class = cls
subqueries = []
for field in self.fields:
subqueries.append(cls(field, pattern, True))
super(AnyFieldQuery, self).__init__(subqueries)
def clause(self):
return self.clause_with_joiner('or')
def match(self, item):
for subq in self.subqueries:
if subq.match(item):
return True
return False
class MutableCollectionQuery(CollectionQuery):
"""A collection query whose subqueries may be modified after the
query is initialized.
"""
def __setitem__(self, key, value):
self.subqueries[key] = value
def __delitem__(self, key):
del self.subqueries[key]
class AndQuery(MutableCollectionQuery):
"""A conjunction of a list of other queries."""
def clause(self):
return self.clause_with_joiner('and')
def match(self, item):
return all([q.match(item) for q in self.subqueries])
class OrQuery(MutableCollectionQuery):
"""A conjunction of a list of other queries."""
def clause(self):
return self.clause_with_joiner('or')
def match(self, item):
return any([q.match(item) for q in self.subqueries])
class TrueQuery(Query):
"""A query that always matches."""
def clause(self):
return '1', ()
def match(self, item):
return True
class FalseQuery(Query):
"""A query that never matches."""
def clause(self):
return '0', ()
def match(self, item):
return False
# Time/date queries.
def _to_epoch_time(date):
"""Convert a `datetime` object to an integer number of seconds since
the (local) Unix epoch.
"""
epoch = datetime.fromtimestamp(0)
delta = date - epoch
try:
return int(delta.total_seconds())
except AttributeError:
# datetime.timedelta.total_seconds() is not available on Python 2.6
return delta.seconds + delta.days * 24 * 3600
def _parse_periods(pattern):
"""Parse a string containing two dates separated by two dots (..).
Return a pair of `Period` objects.
"""
parts = pattern.split('..', 1)
if len(parts) == 1:
instant = Period.parse(parts[0])
return (instant, instant)
else:
start = Period.parse(parts[0])
end = Period.parse(parts[1])
return (start, end)
class Period(object):
"""A period of time given by a date, time and precision.
Example: 2014-01-01 10:50:30 with precision 'month' represents all
instants of time during January 2014.
"""
precisions = ('year', 'month', 'day')
date_formats = ('%Y', '%Y-%m', '%Y-%m-%d')
def __init__(self, date, precision):
"""Create a period with the given date (a `datetime` object) and
precision (a string, one of "year", "month", or "day").
"""
if precision not in Period.precisions:
raise ValueError('Invalid precision ' + str(precision))
self.date = date
self.precision = precision
@classmethod
def parse(cls, string):
"""Parse a date and return a `Period` object or `None` if the
string is empty.
"""
if not string:
return None
ordinal = string.count('-')
if ordinal >= len(cls.date_formats):
raise ValueError('date is not in one of the formats '
+ ', '.join(cls.date_formats))
date_format = cls.date_formats[ordinal]
date = datetime.strptime(string, date_format)
precision = cls.precisions[ordinal]
return cls(date, precision)
def open_right_endpoint(self):
"""Based on the precision, convert the period to a precise
`datetime` for use as a right endpoint in a right-open interval.
"""
precision = self.precision
date = self.date
if 'year' == self.precision:
return date.replace(year=date.year + 1, month=1)
elif 'month' == precision:
if (date.month < 12):
return date.replace(month=date.month + 1)
else:
return date.replace(year=date.year + 1, month=1)
elif 'day' == precision:
return date + timedelta(days=1)
else:
raise ValueError('unhandled precision ' + str(precision))
class DateInterval(object):
"""A closed-open interval of dates.
A left endpoint of None means since the beginning of time.
A right endpoint of None means towards infinity.
"""
def __init__(self, start, end):
if start is not None and end is not None and not start < end:
raise ValueError("start date {0} is not before end date {1}"
.format(start, end))
self.start = start
self.end = end
@classmethod
def from_periods(cls, start, end):
"""Create an interval with two Periods as the endpoints.
"""
end_date = end.open_right_endpoint() if end is not None else None
start_date = start.date if start is not None else None
return cls(start_date, end_date)
def contains(self, date):
if self.start is not None and date < self.start:
return False
if self.end is not None and date >= self.end:
return False
return True
def __str__(self):
return'[{0}, {1})'.format(self.start, self.end)
class DateQuery(FieldQuery):
"""Matches date fields stored as seconds since Unix epoch time.
Dates can be specified as ``year-month-day`` strings where only year
is mandatory.
The value of a date field can be matched against a date interval by
using an ellipsis interval syntax similar to that of NumericQuery.
"""
def __init__(self, field, pattern, fast=True):
super(DateQuery, self).__init__(field, pattern, fast)
start, end = _parse_periods(pattern)
self.interval = DateInterval.from_periods(start, end)
def match(self, item):
timestamp = float(item[self.field])
date = datetime.utcfromtimestamp(timestamp)
return self.interval.contains(date)
_clause_tmpl = "{0} {1} ?"
def col_clause(self):
clause_parts = []
subvals = []
if self.interval.start:
clause_parts.append(self._clause_tmpl.format(self.field, ">="))
subvals.append(_to_epoch_time(self.interval.start))
if self.interval.end:
clause_parts.append(self._clause_tmpl.format(self.field, "<"))
subvals.append(_to_epoch_time(self.interval.end))
if clause_parts:
# One- or two-sided interval.
clause = ' AND '.join(clause_parts)
else:
# Match any date.
clause = '1'
return clause, subvals

140
libs/beets/dbcore/types.py Normal file
View file

@ -0,0 +1,140 @@
# This file is part of beets.
# Copyright 2014, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Representation of type information for DBCore model fields.
"""
from . import query
from beets.util import str2bool
# Abstract base.
class Type(object):
"""An object encapsulating the type of a model field. Includes
information about how to store the value in the database, query,
format, and parse a given field.
"""
sql = None
"""The SQLite column type for the value.
"""
query = None
"""The `Query` subclass to be used when querying the field.
"""
def format(self, value):
"""Given a value of this type, produce a Unicode string
representing the value. This is used in template evaluation.
"""
raise NotImplementedError()
def parse(self, string):
"""Parse a (possibly human-written) string and return the
indicated value of this type.
"""
raise NotImplementedError()
# Reusable types.
class Integer(Type):
"""A basic integer type.
"""
sql = u'INTEGER'
query = query.NumericQuery
def format(self, value):
return unicode(value or 0)
def parse(self, string):
try:
return int(string)
except ValueError:
return 0
class PaddedInt(Integer):
"""An integer field that is formatted with a given number of digits,
padded with zeroes.
"""
def __init__(self, digits):
self.digits = digits
def format(self, value):
return u'{0:0{1}d}'.format(value or 0, self.digits)
class ScaledInt(Integer):
"""An integer whose formatting operation scales the number by a
constant and adds a suffix. Good for units with large magnitudes.
"""
def __init__(self, unit, suffix=u''):
self.unit = unit
self.suffix = suffix
def format(self, value):
return u'{0}{1}'.format((value or 0) // self.unit, self.suffix)
class Id(Integer):
"""An integer used as the row key for a SQLite table.
"""
sql = u'INTEGER PRIMARY KEY'
class Float(Type):
"""A basic floating-point type.
"""
sql = u'REAL'
query = query.NumericQuery
def format(self, value):
return u'{0:.1f}'.format(value or 0.0)
def parse(self, string):
try:
return float(string)
except ValueError:
return 0.0
class String(Type):
"""A Unicode string type.
"""
sql = u'TEXT'
query = query.SubstringQuery
def format(self, value):
return unicode(value) if value else u''
def parse(self, string):
return string
class Boolean(Type):
"""A boolean type.
"""
sql = u'INTEGER'
query = query.BooleanQuery
def format(self, value):
return unicode(bool(value))
def parse(self, string):
return str2bool(string)

1030
libs/beets/importer.py Normal file
View file

File diff suppressed because it is too large Load diff

1216
libs/beets/library.py Normal file
View file

File diff suppressed because it is too large Load diff

1672
libs/beets/mediafile.py Normal file
View file

File diff suppressed because it is too large Load diff

359
libs/beets/plugins.py Normal file
View file

@ -0,0 +1,359 @@
# This file is part of beets.
# Copyright 2013, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Support for beets plugins."""
import logging
import traceback
from collections import defaultdict
import beets
from beets import mediafile
PLUGIN_NAMESPACE = 'beetsplug'
# Plugins using the Last.fm API can share the same API key.
LASTFM_KEY = '2dc3914abf35f0d9c92d97d8f8e42b43'
# Global logger.
log = logging.getLogger('beets')
# Managing the plugins themselves.
class BeetsPlugin(object):
"""The base class for all beets plugins. Plugins provide
functionality by defining a subclass of BeetsPlugin and overriding
the abstract methods defined here.
"""
def __init__(self, name=None):
"""Perform one-time plugin setup.
"""
_add_media_fields(self.item_fields())
self.import_stages = []
self.name = name or self.__module__.split('.')[-1]
self.config = beets.config[self.name]
if not self.template_funcs:
self.template_funcs = {}
if not self.template_fields:
self.template_fields = {}
if not self.album_template_fields:
self.album_template_fields = {}
def commands(self):
"""Should return a list of beets.ui.Subcommand objects for
commands that should be added to beets' CLI.
"""
return ()
def queries(self):
"""Should return a dict mapping prefixes to Query subclasses.
"""
return {}
def track_distance(self, item, info):
"""Should return a Distance object to be added to the
distance for every track comparison.
"""
return beets.autotag.hooks.Distance()
def album_distance(self, items, album_info, mapping):
"""Should return a Distance object to be added to the
distance for every album-level comparison.
"""
return beets.autotag.hooks.Distance()
def candidates(self, items, artist, album, va_likely):
"""Should return a sequence of AlbumInfo objects that match the
album whose items are provided.
"""
return ()
def item_candidates(self, item, artist, title):
"""Should return a sequence of TrackInfo objects that match the
item provided.
"""
return ()
def item_fields(self):
"""Returns field descriptors to be added to the MediaFile class,
in the form of a dictionary whose keys are field names and whose
values are descriptor (e.g., MediaField) instances. The Library
database schema is not (currently) extended.
"""
return {}
def album_for_id(self, album_id):
"""Return an AlbumInfo object or None if no matching release was
found.
"""
return None
def track_for_id(self, track_id):
"""Return a TrackInfo object or None if no matching release was
found.
"""
return None
listeners = None
@classmethod
def register_listener(cls, event, func):
"""Add a function as a listener for the specified event. (An
imperative alternative to the @listen decorator.)
"""
if cls.listeners is None:
cls.listeners = defaultdict(list)
cls.listeners[event].append(func)
@classmethod
def listen(cls, event):
"""Decorator that adds a function as an event handler for the
specified event (as a string). The parameters passed to function
will vary depending on what event occurred.
The function should respond to named parameters.
function(**kwargs) will trap all arguments in a dictionary.
Example:
>>> @MyPlugin.listen("imported")
>>> def importListener(**kwargs):
>>> pass
"""
def helper(func):
if cls.listeners is None:
cls.listeners = defaultdict(list)
cls.listeners[event].append(func)
return func
return helper
template_funcs = None
template_fields = None
album_template_fields = None
@classmethod
def template_func(cls, name):
"""Decorator that registers a path template function. The
function will be invoked as ``%name{}`` from path format
strings.
"""
def helper(func):
if cls.template_funcs is None:
cls.template_funcs = {}
cls.template_funcs[name] = func
return func
return helper
@classmethod
def template_field(cls, name):
"""Decorator that registers a path template field computation.
The value will be referenced as ``$name`` from path format
strings. The function must accept a single parameter, the Item
being formatted.
"""
def helper(func):
if cls.template_fields is None:
cls.template_fields = {}
cls.template_fields[name] = func
return func
return helper
_classes = set()
def load_plugins(names=()):
"""Imports the modules for a sequence of plugin names. Each name
must be the name of a Python module under the "beetsplug" namespace
package in sys.path; the module indicated should contain the
BeetsPlugin subclasses desired.
"""
for name in names:
modname = '%s.%s' % (PLUGIN_NAMESPACE, name)
try:
try:
namespace = __import__(modname, None, None)
except ImportError as exc:
# Again, this is hacky:
if exc.args[0].endswith(' ' + name):
log.warn('** plugin %s not found' % name)
else:
raise
else:
for obj in getattr(namespace, name).__dict__.values():
if isinstance(obj, type) and issubclass(obj, BeetsPlugin) \
and obj != BeetsPlugin and obj not in _classes:
_classes.add(obj)
except:
log.warn('** error loading plugin %s' % name)
log.warn(traceback.format_exc())
_instances = {}
def find_plugins():
"""Returns a list of BeetsPlugin subclass instances from all
currently loaded beets plugins. Loads the default plugin set
first.
"""
load_plugins()
plugins = []
for cls in _classes:
# Only instantiate each plugin class once.
if cls not in _instances:
_instances[cls] = cls()
plugins.append(_instances[cls])
return plugins
# Communication with plugins.
def commands():
"""Returns a list of Subcommand objects from all loaded plugins.
"""
out = []
for plugin in find_plugins():
out += plugin.commands()
return out
def queries():
"""Returns a dict mapping prefix strings to Query subclasses all loaded
plugins.
"""
out = {}
for plugin in find_plugins():
out.update(plugin.queries())
return out
def track_distance(item, info):
"""Gets the track distance calculated by all loaded plugins.
Returns a Distance object.
"""
from beets.autotag.hooks import Distance
dist = Distance()
for plugin in find_plugins():
dist.update(plugin.track_distance(item, info))
return dist
def album_distance(items, album_info, mapping):
"""Returns the album distance calculated by plugins."""
from beets.autotag.hooks import Distance
dist = Distance()
for plugin in find_plugins():
dist.update(plugin.album_distance(items, album_info, mapping))
return dist
def candidates(items, artist, album, va_likely):
"""Gets MusicBrainz candidates for an album from each plugin.
"""
out = []
for plugin in find_plugins():
out.extend(plugin.candidates(items, artist, album, va_likely))
return out
def item_candidates(item, artist, title):
"""Gets MusicBrainz candidates for an item from the plugins.
"""
out = []
for plugin in find_plugins():
out.extend(plugin.item_candidates(item, artist, title))
return out
def album_for_id(album_id):
"""Get AlbumInfo objects for a given ID string.
"""
out = []
for plugin in find_plugins():
res = plugin.album_for_id(album_id)
if res:
out.append(res)
return out
def track_for_id(track_id):
"""Get TrackInfo objects for a given ID string.
"""
out = []
for plugin in find_plugins():
res = plugin.track_for_id(track_id)
if res:
out.append(res)
return out
def template_funcs():
"""Get all the template functions declared by plugins as a
dictionary.
"""
funcs = {}
for plugin in find_plugins():
if plugin.template_funcs:
funcs.update(plugin.template_funcs)
return funcs
def _add_media_fields(fields):
"""Adds a {name: descriptor} dictionary of fields to the MediaFile
class. Called during the plugin initialization.
"""
for key, value in fields.iteritems():
setattr(mediafile.MediaFile, key, value)
def import_stages():
"""Get a list of import stage functions defined by plugins."""
stages = []
for plugin in find_plugins():
if hasattr(plugin, 'import_stages'):
stages += plugin.import_stages
return stages
# New-style (lazy) plugin-provided fields.
def item_field_getters():
"""Get a dictionary mapping field names to unary functions that
compute the field's value.
"""
funcs = {}
for plugin in find_plugins():
if plugin.template_fields:
funcs.update(plugin.template_fields)
return funcs
def album_field_getters():
"""As above, for album fields.
"""
funcs = {}
for plugin in find_plugins():
if plugin.album_template_fields:
funcs.update(plugin.album_template_fields)
return funcs
# Event dispatch.
def event_handlers():
"""Find all event handlers from plugins as a dictionary mapping
event names to sequences of callables.
"""
all_handlers = defaultdict(list)
for plugin in find_plugins():
if plugin.listeners:
for event, handlers in plugin.listeners.items():
all_handlers[event] += handlers
return all_handlers
def send(event, **arguments):
"""Sends an event to all assigned event listeners. Event is the
name of the event to send, all other named arguments go to the
event handler(s).
Returns a list of return values from the handlers.
"""
log.debug('Sending event: %s' % event)
return [handler(**arguments) for handler in event_handlers()[event]]

980
libs/beets/ui/__init__.py Normal file
View file

@ -0,0 +1,980 @@
# This file is part of beets.
# Copyright 2014, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""This module contains all of the core logic for beets' command-line
interface. To invoke the CLI, just call beets.ui.main(). The actual
CLI commands are implemented in the ui.commands module.
"""
from __future__ import print_function
import locale
import optparse
import textwrap
import sys
from difflib import SequenceMatcher
import logging
import sqlite3
import errno
import re
import struct
import traceback
from beets import library
from beets import plugins
from beets import util
from beets.util.functemplate import Template
from beets import config
from beets.util import confit
from beets.autotag import mb
# On Windows platforms, use colorama to support "ANSI" terminal colors.
if sys.platform == 'win32':
try:
import colorama
except ImportError:
pass
else:
colorama.init()
# Constants.
PF_KEY_QUERIES = {
'comp': 'comp:true',
'singleton': 'singleton:true',
}
# UI exception. Commands should throw this in order to display
# nonrecoverable errors to the user.
class UserError(Exception):
pass
# Main logger.
log = logging.getLogger('beets')
# Utilities.
def _encoding():
"""Tries to guess the encoding used by the terminal."""
# Configured override?
encoding = config['terminal_encoding'].get()
if encoding:
return encoding
# Determine from locale settings.
try:
return locale.getdefaultlocale()[1] or 'utf8'
except ValueError:
# Invalid locale environment variable setting. To avoid
# failing entirely for no good reason, assume UTF-8.
return 'utf8'
def decargs(arglist):
"""Given a list of command-line argument bytestrings, attempts to
decode them to Unicode strings.
"""
return [s.decode(_encoding()) for s in arglist]
def print_(*strings):
"""Like print, but rather than raising an error when a character
is not in the terminal's encoding's character set, just silently
replaces it.
"""
if strings:
if isinstance(strings[0], unicode):
txt = u' '.join(strings)
else:
txt = ' '.join(strings)
else:
txt = u''
if isinstance(txt, unicode):
txt = txt.encode(_encoding(), 'replace')
print(txt)
def input_(prompt=None):
"""Like `raw_input`, but decodes the result to a Unicode string.
Raises a UserError if stdin is not available. The prompt is sent to
stdout rather than stderr. A printed between the prompt and the
input cursor.
"""
# raw_input incorrectly sends prompts to stderr, not stdout, so we
# use print() explicitly to display prompts.
# http://bugs.python.org/issue1927
if prompt:
if isinstance(prompt, unicode):
prompt = prompt.encode(_encoding(), 'replace')
print(prompt, end=' ')
try:
resp = raw_input()
except EOFError:
raise UserError('stdin stream ended while input required')
return resp.decode(sys.stdin.encoding or 'utf8', 'ignore')
def input_options(options, require=False, prompt=None, fallback_prompt=None,
numrange=None, default=None, max_width=72):
"""Prompts a user for input. The sequence of `options` defines the
choices the user has. A single-letter shortcut is inferred for each
option; the user's choice is returned as that single, lower-case
letter. The options should be provided as lower-case strings unless
a particular shortcut is desired; in that case, only that letter
should be capitalized.
By default, the first option is the default. `default` can be provided to
override this. If `require` is provided, then there is no default. The
prompt and fallback prompt are also inferred but can be overridden.
If numrange is provided, it is a pair of `(high, low)` (both ints)
indicating that, in addition to `options`, the user may enter an
integer in that inclusive range.
`max_width` specifies the maximum number of columns in the
automatically generated prompt string.
"""
# Assign single letters to each option. Also capitalize the options
# to indicate the letter.
letters = {}
display_letters = []
capitalized = []
first = True
for option in options:
# Is a letter already capitalized?
for letter in option:
if letter.isalpha() and letter.upper() == letter:
found_letter = letter
break
else:
# Infer a letter.
for letter in option:
if not letter.isalpha():
continue # Don't use punctuation.
if letter not in letters:
found_letter = letter
break
else:
raise ValueError('no unambiguous lettering found')
letters[found_letter.lower()] = option
index = option.index(found_letter)
# Mark the option's shortcut letter for display.
if not require and ((default is None and not numrange and first) or
(isinstance(default, basestring) and
found_letter.lower() == default.lower())):
# The first option is the default; mark it.
show_letter = '[%s]' % found_letter.upper()
is_default = True
else:
show_letter = found_letter.upper()
is_default = False
# Colorize the letter shortcut.
show_letter = colorize('turquoise' if is_default else 'blue',
show_letter)
# Insert the highlighted letter back into the word.
capitalized.append(
option[:index] + show_letter + option[index + 1:]
)
display_letters.append(found_letter.upper())
first = False
# The default is just the first option if unspecified.
if require:
default = None
elif default is None:
if numrange:
default = numrange[0]
else:
default = display_letters[0].lower()
# Make a prompt if one is not provided.
if not prompt:
prompt_parts = []
prompt_part_lengths = []
if numrange:
if isinstance(default, int):
default_name = str(default)
default_name = colorize('turquoise', default_name)
tmpl = '# selection (default %s)'
prompt_parts.append(tmpl % default_name)
prompt_part_lengths.append(len(tmpl % str(default)))
else:
prompt_parts.append('# selection')
prompt_part_lengths.append(len(prompt_parts[-1]))
prompt_parts += capitalized
prompt_part_lengths += [len(s) for s in options]
# Wrap the query text.
prompt = ''
line_length = 0
for i, (part, length) in enumerate(zip(prompt_parts,
prompt_part_lengths)):
# Add punctuation.
if i == len(prompt_parts) - 1:
part += '?'
else:
part += ','
length += 1
# Choose either the current line or the beginning of the next.
if line_length + length + 1 > max_width:
prompt += '\n'
line_length = 0
if line_length != 0:
# Not the beginning of the line; need a space.
part = ' ' + part
length += 1
prompt += part
line_length += length
# Make a fallback prompt too. This is displayed if the user enters
# something that is not recognized.
if not fallback_prompt:
fallback_prompt = 'Enter one of '
if numrange:
fallback_prompt += '%i-%i, ' % numrange
fallback_prompt += ', '.join(display_letters) + ':'
resp = input_(prompt)
while True:
resp = resp.strip().lower()
# Try default option.
if default is not None and not resp:
resp = default
# Try an integer input if available.
if numrange:
try:
resp = int(resp)
except ValueError:
pass
else:
low, high = numrange
if low <= resp <= high:
return resp
else:
resp = None
# Try a normal letter input.
if resp:
resp = resp[0]
if resp in letters:
return resp
# Prompt for new input.
resp = input_(fallback_prompt)
def input_yn(prompt, require=False):
"""Prompts the user for a "yes" or "no" response. The default is
"yes" unless `require` is `True`, in which case there is no default.
"""
sel = input_options(
('y', 'n'), require, prompt, 'Enter Y or N:'
)
return sel == 'y'
def human_bytes(size):
"""Formats size, a number of bytes, in a human-readable way."""
suffices = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB', 'HB']
for suffix in suffices:
if size < 1024:
return "%3.1f %s" % (size, suffix)
size /= 1024.0
return "big"
def human_seconds(interval):
"""Formats interval, a number of seconds, as a human-readable time
interval using English words.
"""
units = [
(1, 'second'),
(60, 'minute'),
(60, 'hour'),
(24, 'day'),
(7, 'week'),
(52, 'year'),
(10, 'decade'),
]
for i in range(len(units) - 1):
increment, suffix = units[i]
next_increment, _ = units[i + 1]
interval /= float(increment)
if interval < next_increment:
break
else:
# Last unit.
increment, suffix = units[-1]
interval /= float(increment)
return "%3.1f %ss" % (interval, suffix)
def human_seconds_short(interval):
"""Formats a number of seconds as a short human-readable M:SS
string.
"""
interval = int(interval)
return u'%i:%02i' % (interval // 60, interval % 60)
# ANSI terminal colorization code heavily inspired by pygments:
# http://dev.pocoo.org/hg/pygments-main/file/b2deea5b5030/pygments/console.py
# (pygments is by Tim Hatch, Armin Ronacher, et al.)
COLOR_ESCAPE = "\x1b["
DARK_COLORS = ["black", "darkred", "darkgreen", "brown", "darkblue",
"purple", "teal", "lightgray"]
LIGHT_COLORS = ["darkgray", "red", "green", "yellow", "blue",
"fuchsia", "turquoise", "white"]
RESET_COLOR = COLOR_ESCAPE + "39;49;00m"
def _colorize(color, text):
"""Returns a string that prints the given text in the given color
in a terminal that is ANSI color-aware. The color must be something
in DARK_COLORS or LIGHT_COLORS.
"""
if color in DARK_COLORS:
escape = COLOR_ESCAPE + "%im" % (DARK_COLORS.index(color) + 30)
elif color in LIGHT_COLORS:
escape = COLOR_ESCAPE + "%i;01m" % (LIGHT_COLORS.index(color) + 30)
else:
raise ValueError('no such color %s', color)
return escape + text + RESET_COLOR
def colorize(color, text):
"""Colorize text if colored output is enabled. (Like _colorize but
conditional.)
"""
if config['color']:
return _colorize(color, text)
else:
return text
def _colordiff(a, b, highlight='red', minor_highlight='lightgray'):
"""Given two values, return the same pair of strings except with
their differences highlighted in the specified color. Strings are
highlighted intelligently to show differences; other values are
stringified and highlighted in their entirety.
"""
if not isinstance(a, basestring) or not isinstance(b, basestring):
# Non-strings: use ordinary equality.
a = unicode(a)
b = unicode(b)
if a == b:
return a, b
else:
return colorize(highlight, a), colorize(highlight, b)
if isinstance(a, bytes) or isinstance(b, bytes):
# A path field.
a = util.displayable_path(a)
b = util.displayable_path(b)
a_out = []
b_out = []
matcher = SequenceMatcher(lambda x: False, a, b)
for op, a_start, a_end, b_start, b_end in matcher.get_opcodes():
if op == 'equal':
# In both strings.
a_out.append(a[a_start:a_end])
b_out.append(b[b_start:b_end])
elif op == 'insert':
# Right only.
b_out.append(colorize(highlight, b[b_start:b_end]))
elif op == 'delete':
# Left only.
a_out.append(colorize(highlight, a[a_start:a_end]))
elif op == 'replace':
# Right and left differ. Colorise with second highlight if
# it's just a case change.
if a[a_start:a_end].lower() != b[b_start:b_end].lower():
color = highlight
else:
color = minor_highlight
a_out.append(colorize(color, a[a_start:a_end]))
b_out.append(colorize(color, b[b_start:b_end]))
else:
assert(False)
return u''.join(a_out), u''.join(b_out)
def colordiff(a, b, highlight='red'):
"""Colorize differences between two values if color is enabled.
(Like _colordiff but conditional.)
"""
if config['color']:
return _colordiff(a, b, highlight)
else:
return unicode(a), unicode(b)
def color_diff_suffix(a, b, highlight='red'):
"""Colorize the differing suffix between two strings."""
a, b = unicode(a), unicode(b)
if not config['color']:
return a, b
# Fast path.
if a == b:
return a, b
# Find the longest common prefix.
first_diff = None
for i in range(min(len(a), len(b))):
if a[i] != b[i]:
first_diff = i
break
else:
first_diff = min(len(a), len(b))
# Colorize from the first difference on.
return a[:first_diff] + colorize(highlight, a[first_diff:]), \
b[:first_diff] + colorize(highlight, b[first_diff:])
def get_path_formats(subview=None):
"""Get the configuration's path formats as a list of query/template
pairs.
"""
path_formats = []
subview = subview or config['paths']
for query, view in subview.items():
query = PF_KEY_QUERIES.get(query, query) # Expand common queries.
path_formats.append((query, Template(view.get(unicode))))
return path_formats
def get_replacements():
"""Confit validation function that reads regex/string pairs.
"""
replacements = []
for pattern, repl in config['replace'].get(dict).items():
repl = repl or ''
try:
replacements.append((re.compile(pattern), repl))
except re.error:
raise UserError(
u'malformed regular expression in replace: {0}'.format(
pattern
)
)
return replacements
def get_plugin_paths():
"""Get the list of search paths for plugins from the config file.
The value for "pluginpath" may be a single string or a list of
strings.
"""
pluginpaths = config['pluginpath'].get()
if isinstance(pluginpaths, basestring):
pluginpaths = [pluginpaths]
if not isinstance(pluginpaths, list):
raise confit.ConfigTypeError(
u'pluginpath must be string or a list of strings'
)
return map(util.normpath, pluginpaths)
def _pick_format(album, fmt=None):
"""Pick a format string for printing Album or Item objects,
falling back to config options and defaults.
"""
if fmt:
return fmt
if album:
return config['list_format_album'].get(unicode)
else:
return config['list_format_item'].get(unicode)
def print_obj(obj, lib, fmt=None):
"""Print an Album or Item object. If `fmt` is specified, use that
format string. Otherwise, use the configured template.
"""
album = isinstance(obj, library.Album)
fmt = _pick_format(album, fmt)
if isinstance(fmt, Template):
template = fmt
else:
template = Template(fmt)
print_(obj.evaluate_template(template))
def term_width():
"""Get the width (columns) of the terminal."""
fallback = config['ui']['terminal_width'].get(int)
# The fcntl and termios modules are not available on non-Unix
# platforms, so we fall back to a constant.
try:
import fcntl
import termios
except ImportError:
return fallback
try:
buf = fcntl.ioctl(0, termios.TIOCGWINSZ, ' ' * 4)
except IOError:
return fallback
try:
height, width = struct.unpack('hh', buf)
except struct.error:
return fallback
return width
FLOAT_EPSILON = 0.01
def _field_diff(field, old, new):
"""Given two Model objects, format their values for `field` and
highlight changes among them. Return a human-readable string. If the
value has not changed, return None instead.
"""
oldval = old.get(field)
newval = new.get(field)
# If no change, abort.
if isinstance(oldval, float) and isinstance(newval, float) and \
abs(oldval - newval) < FLOAT_EPSILON:
return None
elif oldval == newval:
return None
# Get formatted values for output.
oldstr = old._get_formatted(field)
newstr = new._get_formatted(field)
# For strings, highlight changes. For others, colorize the whole
# thing.
if isinstance(oldval, basestring):
oldstr, newstr = colordiff(oldval, newval)
else:
oldstr, newstr = colorize('red', oldstr), colorize('red', newstr)
return u'{0} -> {1}'.format(oldstr, newstr)
def show_model_changes(new, old=None, fields=None, always=False):
"""Given a Model object, print a list of changes from its pristine
version stored in the database. Return a boolean indicating whether
any changes were found.
`old` may be the "original" object to avoid using the pristine
version from the database. `fields` may be a list of fields to
restrict the detection to. `always` indicates whether the object is
always identified, regardless of whether any changes are present.
"""
old = old or new._db._get(type(new), new.id)
# Build up lines showing changed fields.
changes = []
for field in old:
# Subset of the fields. Never show mtime.
if field == 'mtime' or (fields and field not in fields):
continue
# Detect and show difference for this field.
line = _field_diff(field, old, new)
if line:
changes.append(u' {0}: {1}'.format(field, line))
# New fields.
for field in set(new) - set(old):
changes.append(u' {0}: {1}'.format(
field,
colorize('red', new._get_formatted(field))
))
# Print changes.
if changes or always:
print_obj(old, old._db)
if changes:
print_(u'\n'.join(changes))
return bool(changes)
# Subcommand parsing infrastructure.
# This is a fairly generic subcommand parser for optparse. It is
# maintained externally here:
# http://gist.github.com/462717
# There you will also find a better description of the code and a more
# succinct example program.
class Subcommand(object):
"""A subcommand of a root command-line application that may be
invoked by a SubcommandOptionParser.
"""
def __init__(self, name, parser=None, help='', aliases=(), hide=False):
"""Creates a new subcommand. name is the primary way to invoke
the subcommand; aliases are alternate names. parser is an
OptionParser responsible for parsing the subcommand's options.
help is a short description of the command. If no parser is
given, it defaults to a new, empty OptionParser.
"""
self.name = name
self.parser = parser or optparse.OptionParser()
self.aliases = aliases
self.help = help
self.hide = hide
class SubcommandsOptionParser(optparse.OptionParser):
"""A variant of OptionParser that parses subcommands and their
arguments.
"""
# A singleton command used to give help on other subcommands.
_HelpSubcommand = Subcommand('help', optparse.OptionParser(),
help='give detailed help on a specific sub-command',
aliases=('?',))
def __init__(self, *args, **kwargs):
"""Create a new subcommand-aware option parser. All of the
options to OptionParser.__init__ are supported in addition
to subcommands, a sequence of Subcommand objects.
"""
# The subcommand array, with the help command included.
self.subcommands = list(kwargs.pop('subcommands', []))
self.subcommands.append(self._HelpSubcommand)
# A more helpful default usage.
if 'usage' not in kwargs:
kwargs['usage'] = """
%prog COMMAND [ARGS...]
%prog help COMMAND"""
# Super constructor.
optparse.OptionParser.__init__(self, *args, **kwargs)
# Adjust the help-visible name of each subcommand.
for subcommand in self.subcommands:
subcommand.parser.prog = '%s %s' % \
(self.get_prog_name(), subcommand.name)
# Our root parser needs to stop on the first unrecognized argument.
self.disable_interspersed_args()
def add_subcommand(self, cmd):
"""Adds a Subcommand object to the parser's list of commands.
"""
self.subcommands.append(cmd)
# Add the list of subcommands to the help message.
def format_help(self, formatter=None):
# Get the original help message, to which we will append.
out = optparse.OptionParser.format_help(self, formatter)
if formatter is None:
formatter = self.formatter
# Subcommands header.
result = ["\n"]
result.append(formatter.format_heading('Commands'))
formatter.indent()
# Generate the display names (including aliases).
# Also determine the help position.
disp_names = []
help_position = 0
subcommands = [c for c in self.subcommands if not c.hide]
for subcommand in subcommands:
name = subcommand.name
if subcommand.aliases:
name += ' (%s)' % ', '.join(subcommand.aliases)
disp_names.append(name)
# Set the help position based on the max width.
proposed_help_position = len(name) + formatter.current_indent + 2
if proposed_help_position <= formatter.max_help_position:
help_position = max(help_position, proposed_help_position)
# Add each subcommand to the output.
for subcommand, name in zip(subcommands, disp_names):
# Lifted directly from optparse.py.
name_width = help_position - formatter.current_indent - 2
if len(name) > name_width:
name = "%*s%s\n" % (formatter.current_indent, "", name)
indent_first = help_position
else:
name = "%*s%-*s " % (formatter.current_indent, "",
name_width, name)
indent_first = 0
result.append(name)
help_width = formatter.width - help_position
help_lines = textwrap.wrap(subcommand.help, help_width)
result.append("%*s%s\n" % (indent_first, "", help_lines[0]))
result.extend(["%*s%s\n" % (help_position, "", line)
for line in help_lines[1:]])
formatter.dedent()
# Concatenate the original help message with the subcommand
# list.
return out + "".join(result)
def _subcommand_for_name(self, name):
"""Return the subcommand in self.subcommands matching the
given name. The name may either be the name of a subcommand or
an alias. If no subcommand matches, returns None.
"""
for subcommand in self.subcommands:
if name == subcommand.name or \
name in subcommand.aliases:
return subcommand
return None
def parse_args(self, a=None, v=None):
"""Like OptionParser.parse_args, but returns these four items:
- options: the options passed to the root parser
- subcommand: the Subcommand object that was invoked
- suboptions: the options passed to the subcommand parser
- subargs: the positional arguments passed to the subcommand
"""
options, args = optparse.OptionParser.parse_args(self, a, v)
subcommand, suboptions, subargs = self._parse_sub(args)
return options, subcommand, suboptions, subargs
def _parse_sub(self, args):
"""Given the `args` left unused by a typical OptionParser
`parse_args`, return the invoked subcommand, the subcommand
options, and the subcommand arguments.
"""
if not args:
# No command given.
self.print_help()
self.exit()
else:
cmdname = args.pop(0)
subcommand = self._subcommand_for_name(cmdname)
if not subcommand:
self.error('unknown command ' + cmdname)
suboptions, subargs = subcommand.parser.parse_args(args)
if subcommand is self._HelpSubcommand:
if subargs:
# particular
cmdname = subargs[0]
helpcommand = self._subcommand_for_name(cmdname)
if not helpcommand:
self.error('no command named {0}'.format(cmdname))
helpcommand.parser.print_help()
self.exit()
else:
# general
self.print_help()
self.exit()
return subcommand, suboptions, subargs
optparse.Option.ALWAYS_TYPED_ACTIONS += ('callback',)
def vararg_callback(option, opt_str, value, parser):
"""Callback for an option with variable arguments.
Manually collect arguments right of a callback-action
option (ie. with action="callback"), and add the resulting
list to the destination var.
Usage:
parser.add_option("-c", "--callback", dest="vararg_attr",
action="callback", callback=vararg_callback)
Details:
http://docs.python.org/2/library/optparse.html#callback-example-6-variable
-arguments
"""
value = [value]
def floatable(str):
try:
float(str)
return True
except ValueError:
return False
for arg in parser.rargs:
# stop on --foo like options
if arg[:2] == "--" and len(arg) > 2:
break
# stop on -a, but not on -3 or -3.0
if arg[:1] == "-" and len(arg) > 1 and not floatable(arg):
break
value.append(arg)
del parser.rargs[:len(value) - 1]
setattr(parser.values, option.dest, value)
# The main entry point and bootstrapping.
def _load_plugins():
"""Load the plugins specified in the configuration.
"""
# Add plugin paths.
import beetsplug
beetsplug.__path__ = get_plugin_paths() + beetsplug.__path__
# For backwards compatibility.
sys.path += get_plugin_paths()
# Load requested plugins.
plugins.load_plugins(config['plugins'].as_str_seq())
plugins.send("pluginload")
def _configure(args):
"""Parse the command line, load configuration files (including
loading any indicated plugins), and return the invoked subcomand,
the subcommand options, and the subcommand arguments.
"""
# Temporary: Migrate from 1.0-style configuration.
from beets.ui import migrate
migrate.automigrate()
# Get the default subcommands.
from beets.ui.commands import default_commands
# Construct the root parser.
commands = list(default_commands)
commands.append(migrate.migrate_cmd) # Temporary.
parser = SubcommandsOptionParser(subcommands=commands)
parser.add_option('-l', '--library', dest='library',
help='library database file to use')
parser.add_option('-d', '--directory', dest='directory',
help="destination music directory")
parser.add_option('-v', '--verbose', dest='verbose', action='store_true',
help='print debugging information')
parser.add_option('-c', '--config', dest='config',
help='path to configuration file')
# Parse the command-line!
options, args = optparse.OptionParser.parse_args(parser, args)
# Add any additional config files specified with --config. This
# special handling lets specified plugins get loaded before we
# finish parsing the command line.
if getattr(options, 'config', None) is not None:
config_path = options.config
del options.config
config.set_file(config_path)
config.set_args(options)
# Now add the plugin commands to the parser.
_load_plugins()
for cmd in plugins.commands():
parser.add_subcommand(cmd)
# Parse the remainder of the command line with loaded plugins.
return parser._parse_sub(args)
def _raw_main(args):
"""A helper function for `main` without top-level exception
handling.
"""
subcommand, suboptions, subargs = _configure(args)
# Open library file.
dbpath = config['library'].as_filename()
try:
lib = library.Library(
dbpath,
config['directory'].as_filename(),
get_path_formats(),
get_replacements(),
)
except sqlite3.OperationalError:
raise UserError(u"database file {0} could not be opened".format(
util.displayable_path(dbpath)
))
plugins.send("library_opened", lib=lib)
# Configure the logger.
if config['verbose'].get(bool):
log.setLevel(logging.DEBUG)
else:
log.setLevel(logging.INFO)
log.debug(u'data directory: {0}\n'
u'library database: {1}\n'
u'library directory: {2}'
.format(
util.displayable_path(config.config_dir()),
util.displayable_path(lib.path),
util.displayable_path(lib.directory),
)
)
# Configure the MusicBrainz API.
mb.configure()
# Invoke the subcommand.
subcommand.func(lib, suboptions, subargs)
plugins.send('cli_exit', lib=lib)
def main(args=None):
"""Run the main command-line interface for beets. Includes top-level
exception handlers that print friendly error messages.
"""
try:
_raw_main(args)
except UserError as exc:
message = exc.args[0] if exc.args else None
log.error(u'error: {0}'.format(message))
sys.exit(1)
except util.HumanReadableException as exc:
exc.log(log)
sys.exit(1)
except library.FileOperationError as exc:
# These errors have reasonable human-readable descriptions, but
# we still want to log their tracebacks for debugging.
log.debug(traceback.format_exc())
log.error(exc)
sys.exit(1)
except confit.ConfigError as exc:
log.error(u'configuration error: {0}'.format(exc))
sys.exit(1)
except IOError as exc:
if exc.errno == errno.EPIPE:
# "Broken pipe". End silently.
pass
else:
raise
except KeyboardInterrupt:
# Silently ignore ^C except in verbose mode.
log.debug(traceback.format_exc())

1415
libs/beets/ui/commands.py Normal file
View file

File diff suppressed because it is too large Load diff

162
libs/beets/ui/completion_base.sh Normal file
View file

@ -0,0 +1,162 @@
# This file is part of beets.
# Copyright (c) 2014, Thomas Scholtes.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
# Completion for the `beet` command
# =================================
#
# Load this script to complete beets subcommands, options, and
# queries.
#
# If a beets command is found on the command line it completes filenames and
# the subcommand's options. Otherwise it will complete global options and
# subcommands. If the previous option on the command line expects an argument,
# it also completes filenames or directories. Options are only
# completed if '-' has already been typed on the command line.
#
# Note that completion of plugin commands only works for those plugins
# that were enabled when running `beet completion`. It does not check
# plugins dynamically
#
# Currently, only Bash 3.2 and newer is supported and the
# `bash-completion` package is requied.
#
# TODO
# ----
#
# * There are some issues with arguments that are quoted on the command line.
#
# * Complete arguments for the `--format` option by expanding field variables.
#
# beet ls -f "$tit[TAB]
# beet ls -f "$title
#
# * Support long options with `=`, e.g. `--config=file`. Debian's bash
# completion package can handle this.
#
# Determines the beets subcommand and dispatches the completion
# accordingly.
_beet_dispatch() {
local cur prev cmd=
COMPREPLY=()
_get_comp_words_by_ref -n : cur prev
# Look for the beets subcommand
local arg
for (( i=1; i < COMP_CWORD; i++ )); do
arg="${COMP_WORDS[i]}"
if _list_include_item "${opts___global}" $arg; then
((i++))
elif [[ "$arg" != -* ]]; then
cmd="$arg"
break
fi
done
# Replace command shortcuts
if [[ -n $cmd ]] && _list_include_item "$aliases" "$cmd"; then
eval "cmd=\$alias__$cmd"
fi
case $cmd in
help)
COMPREPLY+=( $(compgen -W "$commands" -- $cur) )
;;
list|remove|move|update|write|stats)
_beet_complete_query
;;
"")
_beet_complete_global
;;
*)
_beet_complete
;;
esac
}
# Adds option and file completion to COMPREPLY for the subcommand $cmd
_beet_complete() {
if [[ $cur == -* ]]; then
local opts flags completions
eval "opts=\$opts__$cmd"
eval "flags=\$flags__$cmd"
completions="${flags___common} ${opts} ${flags}"
COMPREPLY+=( $(compgen -W "$completions" -- $cur) )
else
_filedir
fi
}
# Add global options and subcommands to the completion
_beet_complete_global() {
case $prev in
-h|--help)
# Complete commands
COMPREPLY+=( $(compgen -W "$commands" -- $cur) )
return
;;
-l|--library|-c|--config)
# Filename completion
_filedir
return
;;
-d|--directory)
# Directory completion
_filedir -d
return
;;
esac
if [[ $cur == -* ]]; then
local completions="$opts___global $flags___global"
COMPREPLY+=( $(compgen -W "$completions" -- $cur) )
elif [[ -n $cur ]] && _list_include_item "$aliases" "$cur"; then
local cmd
eval "cmd=\$alias__$cur"
COMPREPLY+=( "$cmd" )
else
COMPREPLY+=( $(compgen -W "$commands" -- $cur) )
fi
}
_beet_complete_query() {
local opts
eval "opts=\$opts__$cmd"
if [[ $cur == -* ]] || _list_include_item "$opts" "$prev"; then
_beet_complete
elif [[ $cur != \'* && $cur != \"* &&
$cur != *:* ]]; then
# Do not complete quoted queries or those who already have a field
# set.
compopt -o nospace
COMPREPLY+=( $(compgen -S : -W "$fields" -- $cur) )
return 0
fi
}
# Returns true if the space separated list $1 includes $2
_list_include_item() {
[[ " $1 " == *[[:space:]]$2[[:space:]]* ]]
}
# This is where beets dynamically adds the _beet function. This
# function sets the variables $flags, $opts, $commands, and $aliases.
complete -o filenames -F _beet beet

401
libs/beets/ui/migrate.py Normal file
View file

@ -0,0 +1,401 @@
# This file is part of beets.
# Copyright 2013, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Conversion from legacy (pre-1.1) configuration to Confit/YAML
configuration.
"""
import os
import ConfigParser
import codecs
import yaml
import logging
import time
import itertools
import re
import beets
from beets import util
from beets import ui
from beets.util import confit
CONFIG_PATH_VAR = 'BEETSCONFIG'
DEFAULT_CONFIG_FILENAME_UNIX = '.beetsconfig'
DEFAULT_CONFIG_FILENAME_WINDOWS = 'beetsconfig.ini'
DEFAULT_LIBRARY_FILENAME_UNIX = '.beetsmusic.blb'
DEFAULT_LIBRARY_FILENAME_WINDOWS = 'beetsmusic.blb'
WINDOWS_BASEDIR = os.environ.get('APPDATA') or '~'
OLD_CONFIG_SUFFIX = '.old'
PLUGIN_NAMES = {
'rdm': 'random',
'fuzzy_search': 'fuzzy',
}
AUTO_KEYS = ('automatic', 'autofetch', 'autoembed', 'autoscrub')
IMPORTFEEDS_PREFIX = 'feeds_'
CONFIG_MIGRATED_MESSAGE = u"""
You appear to be upgrading from beets 1.0 (or earlier) to 1.1. Your
configuration file has been migrated automatically to:
{newconfig}
Edit this file to configure beets. You might want to remove your
old-style ".beetsconfig" file now. See the documentation for more
details on the new configuration system:
http://beets.readthedocs.org/page/reference/config.html
""".strip()
DB_MIGRATED_MESSAGE = u'Your database file has also been copied to:\n{newdb}'
YAML_COMMENT = '# Automatically migrated from legacy .beetsconfig.\n\n'
log = logging.getLogger('beets')
# An itertools recipe.
def grouper(n, iterable):
args = [iter(iterable)] * n
return itertools.izip_longest(*args)
def _displace(fn):
"""Move a file aside using a timestamp suffix so a new file can be
put in its place.
"""
util.move(
fn,
u'{0}.old.{1}'.format(fn, int(time.time())),
True
)
def default_paths():
"""Produces the appropriate default config and library database
paths for the current system. On Unix, this is always in ~. On
Windows, tries ~ first and then $APPDATA for the config and library
files (for backwards compatibility).
"""
windows = os.path.__name__ == 'ntpath'
if windows:
windata = os.environ.get('APPDATA') or '~'
# Shorthand for joining paths.
def exp(*vals):
return os.path.expanduser(os.path.join(*vals))
config = exp('~', DEFAULT_CONFIG_FILENAME_UNIX)
if windows and not os.path.exists(config):
config = exp(windata, DEFAULT_CONFIG_FILENAME_WINDOWS)
libpath = exp('~', DEFAULT_LIBRARY_FILENAME_UNIX)
if windows and not os.path.exists(libpath):
libpath = exp(windata, DEFAULT_LIBRARY_FILENAME_WINDOWS)
return config, libpath
def get_config():
"""Using the same logic as beets 1.0, locate and read the
.beetsconfig file. Return a ConfigParser instance or None if no
config is found.
"""
default_config, default_libpath = default_paths()
if CONFIG_PATH_VAR in os.environ:
configpath = os.path.expanduser(os.environ[CONFIG_PATH_VAR])
else:
configpath = default_config
config = ConfigParser.SafeConfigParser()
if os.path.exists(util.syspath(configpath)):
with codecs.open(configpath, 'r', encoding='utf-8') as f:
config.readfp(f)
return config, configpath
else:
return None, configpath
def flatten_config(config):
"""Given a ConfigParser, flatten the values into a dict-of-dicts
representation where each section gets its own dictionary of values.
"""
out = confit.OrderedDict()
for section in config.sections():
sec_dict = out[section] = confit.OrderedDict()
for option in config.options(section):
sec_dict[option] = config.get(section, option, True)
return out
def transform_value(value):
"""Given a string read as the value of a config option, return a
massaged version of that value (possibly with a different type).
"""
# Booleans.
if value.lower() in ('false', 'no', 'off'):
return False
elif value.lower() in ('true', 'yes', 'on'):
return True
# Integers.
try:
return int(value)
except ValueError:
pass
# Floats.
try:
return float(value)
except ValueError:
pass
return value
def transform_data(data):
"""Given a dict-of-dicts representation of legacy config data, tweak
the data into a new form. This new form is suitable for dumping as
YAML.
"""
out = confit.OrderedDict()
for section, pairs in data.items():
if section == 'beets':
# The "main" section. In the new config system, these values
# are in the "root": no section at all.
for key, value in pairs.items():
value = transform_value(value)
if key.startswith('import_'):
# Importer config is now under an "import:" key.
if 'import' not in out:
out['import'] = confit.OrderedDict()
out['import'][key[7:]] = value
elif key == 'plugins':
# Renamed plugins.
plugins = value.split()
new_plugins = [PLUGIN_NAMES.get(p, p) for p in plugins]
out['plugins'] = ' '.join(new_plugins)
elif key == 'replace':
# YAMLy representation for character replacements.
replacements = confit.OrderedDict()
for pat, repl in grouper(2, value.split()):
if repl == '<strip>':
repl = ''
replacements[pat] = repl
out['replace'] = replacements
elif key == 'pluginpath':
# Used to be a colon-separated string. Now a list.
out['pluginpath'] = value.split(':')
else:
out[key] = value
elif pairs:
# Other sections (plugins, etc).
sec_out = out[section] = confit.OrderedDict()
for key, value in pairs.items():
# Standardized "auto" option.
if key in AUTO_KEYS:
key = 'auto'
# Unnecessary : hack in queries.
if section == 'paths':
key = key.replace('_', ':')
# Changed option names for importfeeds plugin.
if section == 'importfeeds':
if key.startswith(IMPORTFEEDS_PREFIX):
key = key[len(IMPORTFEEDS_PREFIX):]
sec_out[key] = transform_value(value)
return out
class Dumper(yaml.SafeDumper):
"""A PyYAML Dumper that represents OrderedDicts as ordinary mappings
(in order, of course).
"""
# From http://pyyaml.org/attachment/ticket/161/use_ordered_dict.py
def represent_mapping(self, tag, mapping, flow_style=None):
value = []
node = yaml.MappingNode(tag, value, flow_style=flow_style)
if self.alias_key is not None:
self.represented_objects[self.alias_key] = node
best_style = True
if hasattr(mapping, 'items'):
mapping = list(mapping.items())
for item_key, item_value in mapping:
node_key = self.represent_data(item_key)
node_value = self.represent_data(item_value)
if not (isinstance(node_key, yaml.ScalarNode) and \
not node_key.style):
best_style = False
if not (isinstance(node_value, yaml.ScalarNode) and \
not node_value.style):
best_style = False
value.append((node_key, node_value))
if flow_style is None:
if self.default_flow_style is not None:
node.flow_style = self.default_flow_style
else:
node.flow_style = best_style
return node
Dumper.add_representer(confit.OrderedDict, Dumper.represent_dict)
def migrate_config(replace=False):
"""Migrate a legacy beetsconfig file to a new-style config.yaml file
in an appropriate place. If `replace` is enabled, then any existing
config.yaml will be moved aside. Otherwise, the process is aborted
when the file exists.
"""
# Load legacy configuration data, if any.
config, configpath = get_config()
if not config:
log.debug(u'no config file found at {0}'.format(
util.displayable_path(configpath)
))
return
# Get the new configuration file path and possibly move it out of
# the way.
destfn = os.path.join(beets.config.config_dir(), confit.CONFIG_FILENAME)
if os.path.exists(destfn):
if replace:
log.debug(u'moving old config aside: {0}'.format(
util.displayable_path(destfn)
))
_displace(destfn)
else:
# File exists and we won't replace it. We're done.
return
log.debug(u'migrating config file {0}'.format(
util.displayable_path(configpath)
))
# Convert the configuration to a data structure ready to be dumped
# as the new Confit file.
data = transform_data(flatten_config(config))
# Encode result as YAML.
yaml_out = yaml.dump(
data,
Dumper=Dumper,
default_flow_style=False,
indent=4,
width=1000,
)
# A ridiculous little hack to add some whitespace between "sections"
# in the YAML output. I hope this doesn't break any YAML syntax.
yaml_out = re.sub(r'(\n\w+:\n [^-\s])', '\n\\1', yaml_out)
yaml_out = YAML_COMMENT + yaml_out
# Write the data to the new config destination.
log.debug(u'writing migrated config to {0}'.format(
util.displayable_path(destfn)
))
with open(destfn, 'w') as f:
f.write(yaml_out)
return destfn
def migrate_db(replace=False):
"""Copy the beets library database file to the new location (e.g.,
from ~/.beetsmusic.blb to ~/.config/beets/library.db).
"""
_, srcfn = default_paths()
destfn = beets.config['library'].as_filename()
if not os.path.exists(srcfn) or srcfn == destfn:
# Old DB does not exist or we're configured to point to the same
# database. Do nothing.
return
if os.path.exists(destfn):
if replace:
log.debug(u'moving old database aside: {0}'.format(
util.displayable_path(destfn)
))
_displace(destfn)
else:
return
log.debug(u'copying database from {0} to {1}'.format(
util.displayable_path(srcfn), util.displayable_path(destfn)
))
util.copy(srcfn, destfn)
return destfn
def migrate_state(replace=False):
"""Copy the beets runtime state file from the old path (i.e.,
~/.beetsstate) to the new path (i.e., ~/.config/beets/state.pickle).
"""
srcfn = os.path.expanduser(os.path.join('~', '.beetsstate'))
if not os.path.exists(srcfn):
return
destfn = beets.config['statefile'].as_filename()
if os.path.exists(destfn):
if replace:
_displace(destfn)
else:
return
log.debug(u'copying state file from {0} to {1}'.format(
util.displayable_path(srcfn), util.displayable_path(destfn)
))
util.copy(srcfn, destfn)
return destfn
# Automatic migration when beets starts.
def automigrate():
"""Migrate the configuration, database, and state files. If any
migration occurs, print out a notice with some helpful next steps.
"""
config_fn = migrate_config()
db_fn = migrate_db()
migrate_state()
if config_fn:
ui.print_(ui.colorize('fuchsia', u'MIGRATED CONFIGURATION'))
ui.print_(CONFIG_MIGRATED_MESSAGE.format(
newconfig=util.displayable_path(config_fn))
)
if db_fn:
ui.print_(DB_MIGRATED_MESSAGE.format(
newdb=util.displayable_path(db_fn)
))
ui.input_(ui.colorize('fuchsia', u'Press ENTER to continue:'))
ui.print_()
# CLI command for explicit migration.
migrate_cmd = ui.Subcommand('migrate', help='convert legacy config')
def migrate_func(lib, opts, args):
"""Explicit command for migrating files. Existing files in each
destination are moved aside.
"""
config_fn = migrate_config(replace=True)
if config_fn:
log.info(u'Migrated configuration to: {0}'.format(
util.displayable_path(config_fn)
))
db_fn = migrate_db(replace=True)
if db_fn:
log.info(u'Migrated library database to: {0}'.format(
util.displayable_path(db_fn)
))
state_fn = migrate_state(replace=True)
if state_fn:
log.info(u'Migrated state file to: {0}'.format(
util.displayable_path(state_fn)
))
migrate_cmd.func = migrate_func

618
libs/beets/util/__init__.py Normal file
View file

@ -0,0 +1,618 @@
# This file is part of beets.
# Copyright 2013, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Miscellaneous utility functions."""
from __future__ import division
import os
import sys
import re
import shutil
import fnmatch
from collections import defaultdict
import traceback
import subprocess
MAX_FILENAME_LENGTH = 200
WINDOWS_MAGIC_PREFIX = u'\\\\?\\'
class HumanReadableException(Exception):
"""An Exception that can include a human-readable error message to
be logged without a traceback. Can preserve a traceback for
debugging purposes as well.
Has at least two fields: `reason`, the underlying exception or a
string describing the problem; and `verb`, the action being
performed during the error.
If `tb` is provided, it is a string containing a traceback for the
associated exception. (Note that this is not necessary in Python 3.x
and should be removed when we make the transition.)
"""
error_kind = 'Error' # Human-readable description of error type.
def __init__(self, reason, verb, tb=None):
self.reason = reason
self.verb = verb
self.tb = tb
super(HumanReadableException, self).__init__(self.get_message())
def _gerund(self):
"""Generate a (likely) gerund form of the English verb.
"""
if ' ' in self.verb:
return self.verb
gerund = self.verb[:-1] if self.verb.endswith('e') else self.verb
gerund += 'ing'
return gerund
def _reasonstr(self):
"""Get the reason as a string."""
if isinstance(self.reason, unicode):
return self.reason
elif isinstance(self.reason, basestring): # Byte string.
return self.reason.decode('utf8', 'ignore')
elif hasattr(self.reason, 'strerror'): # i.e., EnvironmentError
return self.reason.strerror
else:
return u'"{0}"'.format(unicode(self.reason))
def get_message(self):
"""Create the human-readable description of the error, sans
introduction.
"""
raise NotImplementedError
def log(self, logger):
"""Log to the provided `logger` a human-readable message as an
error and a verbose traceback as a debug message.
"""
if self.tb:
logger.debug(self.tb)
logger.error(u'{0}: {1}'.format(self.error_kind, self.args[0]))
class FilesystemError(HumanReadableException):
"""An error that occurred while performing a filesystem manipulation
via a function in this module. The `paths` field is a sequence of
pathnames involved in the operation.
"""
def __init__(self, reason, verb, paths, tb=None):
self.paths = paths
super(FilesystemError, self).__init__(reason, verb, tb)
def get_message(self):
# Use a nicer English phrasing for some specific verbs.
if self.verb in ('move', 'copy', 'rename'):
clause = u'while {0} {1} to {2}'.format(
self._gerund(),
displayable_path(self.paths[0]),
displayable_path(self.paths[1])
)
elif self.verb in ('delete', 'write', 'create', 'read'):
clause = u'while {0} {1}'.format(
self._gerund(),
displayable_path(self.paths[0])
)
else:
clause = u'during {0} of paths {1}'.format(
self.verb, u', '.join(displayable_path(p) for p in self.paths)
)
return u'{0} {1}'.format(self._reasonstr(), clause)
def normpath(path):
"""Provide the canonical form of the path suitable for storing in
the database.
"""
path = syspath(path, prefix=False)
path = os.path.normpath(os.path.abspath(os.path.expanduser(path)))
return bytestring_path(path)
def ancestry(path):
"""Return a list consisting of path's parent directory, its
grandparent, and so on. For instance:
>>> ancestry('/a/b/c')
['/', '/a', '/a/b']
The argument should *not* be the result of a call to `syspath`.
"""
out = []
last_path = None
while path:
path = os.path.dirname(path)
if path == last_path:
break
last_path = path
if path: # don't yield ''
out.insert(0, path)
return out
def sorted_walk(path, ignore=(), logger=None):
"""Like `os.walk`, but yields things in case-insensitive sorted,
breadth-first order. Directory and file names matching any glob
pattern in `ignore` are skipped. If `logger` is provided, then
warning messages are logged there when a directory cannot be listed.
"""
# Make sure the path isn't a Unicode string.
path = bytestring_path(path)
# Get all the directories and files at this level.
try:
contents = os.listdir(syspath(path))
except OSError as exc:
if logger:
logger.warn(u'could not list directory {0}: {1}'.format(
displayable_path(path), exc.strerror
))
return
dirs = []
files = []
for base in contents:
base = bytestring_path(base)
# Skip ignored filenames.
skip = False
for pat in ignore:
if fnmatch.fnmatch(base, pat):
skip = True
break
if skip:
continue
# Add to output as either a file or a directory.
cur = os.path.join(path, base)
if os.path.isdir(syspath(cur)):
dirs.append(base)
else:
files.append(base)
# Sort lists (case-insensitive) and yield the current level.
dirs.sort(key=bytes.lower)
files.sort(key=bytes.lower)
yield (path, dirs, files)
# Recurse into directories.
for base in dirs:
cur = os.path.join(path, base)
# yield from sorted_walk(...)
for res in sorted_walk(cur, ignore, logger):
yield res
def mkdirall(path):
"""Make all the enclosing directories of path (like mkdir -p on the
parent).
"""
for ancestor in ancestry(path):
if not os.path.isdir(syspath(ancestor)):
try:
os.mkdir(syspath(ancestor))
except (OSError, IOError) as exc:
raise FilesystemError(exc, 'create', (ancestor,),
traceback.format_exc())
def fnmatch_all(names, patterns):
"""Determine whether all strings in `names` match at least one of
the `patterns`, which should be shell glob expressions.
"""
for name in names:
matches = False
for pattern in patterns:
matches = fnmatch.fnmatch(name, pattern)
if matches:
break
if not matches:
return False
return True
def prune_dirs(path, root=None, clutter=('.DS_Store', 'Thumbs.db')):
"""If path is an empty directory, then remove it. Recursively remove
path's ancestry up to root (which is never removed) where there are
empty directories. If path is not contained in root, then nothing is
removed. Glob patterns in clutter are ignored when determining
emptiness. If root is not provided, then only path may be removed
(i.e., no recursive removal).
"""
path = normpath(path)
if root is not None:
root = normpath(root)
ancestors = ancestry(path)
if root is None:
# Only remove the top directory.
ancestors = []
elif root in ancestors:
# Only remove directories below the root.
ancestors = ancestors[ancestors.index(root)+1:]
else:
# Remove nothing.
return
# Traverse upward from path.
ancestors.append(path)
ancestors.reverse()
for directory in ancestors:
directory = syspath(directory)
if not os.path.exists(directory):
# Directory gone already.
continue
if fnmatch_all(os.listdir(directory), clutter):
# Directory contains only clutter (or nothing).
try:
shutil.rmtree(directory)
except OSError:
break
else:
break
def components(path):
"""Return a list of the path components in path. For instance:
>>> components('/a/b/c')
['a', 'b', 'c']
The argument should *not* be the result of a call to `syspath`.
"""
comps = []
ances = ancestry(path)
for anc in ances:
comp = os.path.basename(anc)
if comp:
comps.append(comp)
else: # root
comps.append(anc)
last = os.path.basename(path)
if last:
comps.append(last)
return comps
def _fsencoding():
"""Get the system's filesystem encoding. On Windows, this is always
UTF-8 (not MBCS).
"""
encoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
if encoding == 'mbcs':
# On Windows, a broken encoding known to Python as "MBCS" is
# used for the filesystem. However, we only use the Unicode API
# for Windows paths, so the encoding is actually immaterial so
# we can avoid dealing with this nastiness. We arbitrarily
# choose UTF-8.
encoding = 'utf8'
return encoding
def bytestring_path(path):
"""Given a path, which is either a str or a unicode, returns a str
path (ensuring that we never deal with Unicode pathnames).
"""
# Pass through bytestrings.
if isinstance(path, str):
return path
# On Windows, remove the magic prefix added by `syspath`. This makes
# ``bytestring_path(syspath(X)) == X``, i.e., we can safely
# round-trip through `syspath`.
if os.path.__name__ == 'ntpath' and path.startswith(WINDOWS_MAGIC_PREFIX):
path = path[len(WINDOWS_MAGIC_PREFIX):]
# Try to encode with default encodings, but fall back to UTF8.
try:
return path.encode(_fsencoding())
except (UnicodeError, LookupError):
return path.encode('utf8')
def displayable_path(path, separator=u'; '):
"""Attempts to decode a bytestring path to a unicode object for the
purpose of displaying it to the user. If the `path` argument is a
list or a tuple, the elements are joined with `separator`.
"""
if isinstance(path, (list, tuple)):
return separator.join(displayable_path(p) for p in path)
elif isinstance(path, unicode):
return path
elif not isinstance(path, str):
# A non-string object: just get its unicode representation.
return unicode(path)
try:
return path.decode(_fsencoding(), 'ignore')
except (UnicodeError, LookupError):
return path.decode('utf8', 'ignore')
def syspath(path, prefix=True):
"""Convert a path for use by the operating system. In particular,
paths on Windows must receive a magic prefix and must be converted
to Unicode before they are sent to the OS. To disable the magic
prefix on Windows, set `prefix` to False---but only do this if you
*really* know what you're doing.
"""
# Don't do anything if we're not on windows
if os.path.__name__ != 'ntpath':
return path
if not isinstance(path, unicode):
# Beets currently represents Windows paths internally with UTF-8
# arbitrarily. But earlier versions used MBCS because it is
# reported as the FS encoding by Windows. Try both.
try:
path = path.decode('utf8')
except UnicodeError:
# The encoding should always be MBCS, Windows' broken
# Unicode representation.
encoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
path = path.decode(encoding, 'replace')
# Add the magic prefix if it isn't already there
if prefix and not path.startswith(WINDOWS_MAGIC_PREFIX):
path = WINDOWS_MAGIC_PREFIX + path
return path
def samefile(p1, p2):
"""Safer equality for paths."""
return shutil._samefile(syspath(p1), syspath(p2))
def remove(path, soft=True):
"""Remove the file. If `soft`, then no error will be raised if the
file does not exist.
"""
path = syspath(path)
if soft and not os.path.exists(path):
return
try:
os.remove(path)
except (OSError, IOError) as exc:
raise FilesystemError(exc, 'delete', (path,), traceback.format_exc())
def copy(path, dest, replace=False):
"""Copy a plain file. Permissions are not copied. If `dest` already
exists, raises a FilesystemError unless `replace` is True. Has no
effect if `path` is the same as `dest`. Paths are translated to
system paths before the syscall.
"""
if samefile(path, dest):
return
path = syspath(path)
dest = syspath(dest)
if not replace and os.path.exists(dest):
raise FilesystemError('file exists', 'copy', (path, dest))
try:
shutil.copyfile(path, dest)
except (OSError, IOError) as exc:
raise FilesystemError(exc, 'copy', (path, dest),
traceback.format_exc())
def move(path, dest, replace=False):
"""Rename a file. `dest` may not be a directory. If `dest` already
exists, raises an OSError unless `replace` is True. Has no effect if
`path` is the same as `dest`. If the paths are on different
filesystems (or the rename otherwise fails), a copy is attempted
instead, in which case metadata will *not* be preserved. Paths are
translated to system paths.
"""
if samefile(path, dest):
return
path = syspath(path)
dest = syspath(dest)
if os.path.exists(dest) and not replace:
raise FilesystemError('file exists', 'rename', (path, dest),
traceback.format_exc())
# First, try renaming the file.
try:
os.rename(path, dest)
except OSError:
# Otherwise, copy and delete the original.
try:
shutil.copyfile(path, dest)
os.remove(path)
except (OSError, IOError) as exc:
raise FilesystemError(exc, 'move', (path, dest),
traceback.format_exc())
def unique_path(path):
"""Returns a version of ``path`` that does not exist on the
filesystem. Specifically, if ``path` itself already exists, then
something unique is appended to the path.
"""
if not os.path.exists(syspath(path)):
return path
base, ext = os.path.splitext(path)
match = re.search(r'\.(\d)+$', base)
if match:
num = int(match.group(1))
base = base[:match.start()]
else:
num = 0
while True:
num += 1
new_path = '%s.%i%s' % (base, num, ext)
if not os.path.exists(new_path):
return new_path
# Note: The Windows "reserved characters" are, of course, allowed on
# Unix. They are forbidden here because they cause problems on Samba
# shares, which are sufficiently common as to cause frequent problems.
# http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247.aspx
CHAR_REPLACE = [
(re.compile(ur'[\\/]'), u'_'), # / and \ -- forbidden everywhere.
(re.compile(ur'^\.'), u'_'), # Leading dot (hidden files on Unix).
(re.compile(ur'[\x00-\x1f]'), u''), # Control characters.
(re.compile(ur'[<>:"\?\*\|]'), u'_'), # Windows "reserved characters".
(re.compile(ur'\.$'), u'_'), # Trailing dots.
(re.compile(ur'\s+$'), u''), # Trailing whitespace.
]
def sanitize_path(path, replacements=None):
"""Takes a path (as a Unicode string) and makes sure that it is
legal. Returns a new path. Only works with fragments; won't work
reliably on Windows when a path begins with a drive letter. Path
separators (including altsep!) should already be cleaned from the
path components. If replacements is specified, it is used *instead*
of the default set of replacements; it must be a list of (compiled
regex, replacement string) pairs.
"""
replacements = replacements or CHAR_REPLACE
comps = components(path)
if not comps:
return ''
for i, comp in enumerate(comps):
for regex, repl in replacements:
comp = regex.sub(repl, comp)
comps[i] = comp
return os.path.join(*comps)
def truncate_path(path, length=MAX_FILENAME_LENGTH):
"""Given a bytestring path or a Unicode path fragment, truncate the
components to a legal length. In the last component, the extension
is preserved.
"""
comps = components(path)
out = [c[:length] for c in comps]
base, ext = os.path.splitext(comps[-1])
if ext:
# Last component has an extension.
base = base[:length - len(ext)]
out[-1] = base + ext
return os.path.join(*out)
def str2bool(value):
"""Returns a boolean reflecting a human-entered string."""
if value.lower() in ('yes', '1', 'true', 't', 'y'):
return True
else:
return False
def as_string(value):
"""Convert a value to a Unicode object for matching with a query.
None becomes the empty string. Bytestrings are silently decoded.
"""
if value is None:
return u''
elif isinstance(value, buffer):
return str(value).decode('utf8', 'ignore')
elif isinstance(value, str):
return value.decode('utf8', 'ignore')
else:
return unicode(value)
def levenshtein(s1, s2):
"""A nice DP edit distance implementation from Wikibooks:
http://en.wikibooks.org/wiki/Algorithm_implementation/Strings/
Levenshtein_distance#Python
"""
if len(s1) < len(s2):
return levenshtein(s2, s1)
if not s1:
return len(s2)
previous_row = xrange(len(s2) + 1)
for i, c1 in enumerate(s1):
current_row = [i + 1]
for j, c2 in enumerate(s2):
insertions = previous_row[j + 1] + 1
deletions = current_row[j] + 1
substitutions = previous_row[j] + (c1 != c2)
current_row.append(min(insertions, deletions, substitutions))
previous_row = current_row
return previous_row[-1]
def plurality(objs):
"""Given a sequence of comparable objects, returns the object that
is most common in the set and the frequency of that object. The
sequence must contain at least one object.
"""
# Calculate frequencies.
freqs = defaultdict(int)
for obj in objs:
freqs[obj] += 1
if not freqs:
raise ValueError('sequence must be non-empty')
# Find object with maximum frequency.
max_freq = 0
res = None
for obj, freq in freqs.items():
if freq > max_freq:
max_freq = freq
res = obj
return res, max_freq
def cpu_count():
"""Return the number of hardware thread contexts (cores or SMT
threads) in the system.
"""
# Adapted from the soundconverter project:
# https://github.com/kassoulet/soundconverter
if sys.platform == 'win32':
try:
num = int(os.environ['NUMBER_OF_PROCESSORS'])
except (ValueError, KeyError):
num = 0
elif sys.platform == 'darwin':
try:
num = int(os.popen('sysctl -n hw.ncpu').read())
except ValueError:
num = 0
else:
try:
num = os.sysconf('SC_NPROCESSORS_ONLN')
except (ValueError, OSError, AttributeError):
num = 0
if num >= 1:
return num
else:
return 1
def command_output(cmd):
"""Wraps the `subprocess` module to invoke a command (given as a
list of arguments starting with the command name) and collect
stdout. The stderr stream is ignored. May raise
`subprocess.CalledProcessError` or an `OSError`.
This replaces `subprocess.check_output`, which isn't available in
Python 2.6 and which can have problems if lots of output is sent to
stderr.
"""
with open(os.devnull, 'w') as devnull:
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=devnull)
stdout, _ = proc.communicate()
if proc.returncode:
raise subprocess.CalledProcessError(proc.returncode, cmd)
return stdout
def max_filename_length(path, limit=MAX_FILENAME_LENGTH):
"""Attempt to determine the maximum filename length for the
filesystem containing `path`. If the value is greater than `limit`,
then `limit` is used instead (to prevent errors when a filesystem
misreports its capacity). If it cannot be determined (e.g., on
Windows), return `limit`.
"""
if hasattr(os, 'statvfs'):
try:
res = os.statvfs(path)
except OSError:
return limit
return min(res[9], limit)
else:
return limit

188
libs/beets/util/artresizer.py Normal file
View file

@ -0,0 +1,188 @@
# This file is part of beets.
# Copyright 2013, Fabrice Laporte
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Abstraction layer to resize images using PIL, ImageMagick, or a
public resizing proxy if neither is available.
"""
import urllib
import subprocess
import os
from tempfile import NamedTemporaryFile
import logging
from beets import util
# Resizing methods
PIL = 1
IMAGEMAGICK = 2
WEBPROXY = 3
PROXY_URL = 'http://images.weserv.nl/'
log = logging.getLogger('beets')
def resize_url(url, maxwidth):
"""Return a proxied image URL that resizes the original image to
maxwidth (preserving aspect ratio).
"""
return '{0}?{1}'.format(PROXY_URL, urllib.urlencode({
'url': url.replace('http://',''),
'w': str(maxwidth),
}))
def temp_file_for(path):
"""Return an unused filename with the same extension as the
specified path.
"""
ext = os.path.splitext(path)[1]
with NamedTemporaryFile(suffix=ext, delete=False) as f:
return f.name
def pil_resize(maxwidth, path_in, path_out=None):
"""Resize using Python Imaging Library (PIL). Return the output path
of resized image.
"""
path_out = path_out or temp_file_for(path_in)
from PIL import Image
log.debug(u'artresizer: PIL resizing {0} to {1}'.format(
util.displayable_path(path_in), util.displayable_path(path_out)
))
try:
im = Image.open(util.syspath(path_in))
size = maxwidth, maxwidth
im.thumbnail(size, Image.ANTIALIAS)
im.save(path_out)
return path_out
except IOError:
log.error(u"PIL cannot create thumbnail for '{0}'".format(
util.displayable_path(path_in)
))
return path_in
def im_resize(maxwidth, path_in, path_out=None):
"""Resize using ImageMagick's ``convert`` tool.
tool. Return the output path of resized image.
"""
path_out = path_out or temp_file_for(path_in)
log.debug(u'artresizer: ImageMagick resizing {0} to {1}'.format(
util.displayable_path(path_in), util.displayable_path(path_out)
))
# "-resize widthxheight>" shrinks images with dimension(s) larger
# than the corresponding width and/or height dimension(s). The >
# "only shrink" flag is prefixed by ^ escape char for Windows
# compatibility.
try:
util.command_output([
'convert', util.syspath(path_in),
'-resize', '{0}x^>'.format(maxwidth), path_out
])
except subprocess.CalledProcessError:
log.warn(u'artresizer: IM convert failed for {0}'.format(
util.displayable_path(path_in)
))
return path_in
return path_out
BACKEND_FUNCS = {
PIL: pil_resize,
IMAGEMAGICK: im_resize,
}
class Shareable(type):
"""A pseudo-singleton metaclass that allows both shared and
non-shared instances. The ``MyClass.shared`` property holds a
lazily-created shared instance of ``MyClass`` while calling
``MyClass()`` to construct a new object works as usual.
"""
def __init__(cls, name, bases, dict):
super(Shareable, cls).__init__(name, bases, dict)
cls._instance = None
@property
def shared(cls):
if cls._instance is None:
cls._instance = cls()
return cls._instance
class ArtResizer(object):
"""A singleton class that performs image resizes.
"""
__metaclass__ = Shareable
def __init__(self, method=None):
"""Create a resizer object for the given method or, if none is
specified, with an inferred method.
"""
self.method = method or self._guess_method()
log.debug(u"artresizer: method is {0}".format(self.method))
def resize(self, maxwidth, path_in, path_out=None):
"""Manipulate an image file according to the method, returning a
new path. For PIL or IMAGEMAGIC methods, resizes the image to a
temporary file. For WEBPROXY, returns `path_in` unmodified.
"""
if self.local:
func = BACKEND_FUNCS[self.method]
return func(maxwidth, path_in, path_out)
else:
return path_in
def proxy_url(self, maxwidth, url):
"""Modifies an image URL according the method, returning a new
URL. For WEBPROXY, a URL on the proxy server is returned.
Otherwise, the URL is returned unmodified.
"""
if self.local:
return url
else:
return resize_url(url, maxwidth)
@property
def local(self):
"""A boolean indicating whether the resizing method is performed
locally (i.e., PIL or IMAGEMAGICK).
"""
return self.method in BACKEND_FUNCS
@staticmethod
def _guess_method():
"""Determine which resizing method to use. Returns PIL,
IMAGEMAGICK, or WEBPROXY depending on available dependencies.
"""
# Try importing PIL.
try:
__import__('PIL', fromlist=['Image'])
return PIL
except ImportError:
pass
# Try invoking ImageMagick's "convert".
try:
out = util.command_output(['convert', '--version'])
if 'imagemagick' in out.lower():
# system32/convert.exe may be interfering
return IMAGEMAGICK
except (subprocess.CalledProcessError, OSError):
pass
# Fall back to Web proxy method.
return WEBPROXY

617
libs/beets/util/bluelet.py Normal file
View file

@ -0,0 +1,617 @@
"""Extremely simple pure-Python implementation of coroutine-style
asynchronous socket I/O. Inspired by, but inferior to, Eventlet.
Bluelet can also be thought of as a less-terrible replacement for
asyncore.
Bluelet: easy concurrency without all the messy parallelism.
"""
import socket
import select
import sys
import types
import errno
import traceback
import time
import collections
# A little bit of "six" (Python 2/3 compatibility): cope with PEP 3109 syntax
# changes.
PY3 = sys.version_info[0] == 3
if PY3:
def _reraise(typ, exc, tb):
raise exc.with_traceback(tb)
else:
exec("""
def _reraise(typ, exc, tb):
raise typ, exc, tb
""")
# Basic events used for thread scheduling.
class Event(object):
"""Just a base class identifying Bluelet events. An event is an
object yielded from a Bluelet thread coroutine to suspend operation
and communicate with the scheduler.
"""
pass
class WaitableEvent(Event):
"""A waitable event is one encapsulating an action that can be
waited for using a select() call. That is, it's an event with an
associated file descriptor.
"""
def waitables(self):
"""Return "waitable" objects to pass to select(). Should return
three iterables for input readiness, output readiness, and
exceptional conditions (i.e., the three lists passed to
select()).
"""
return (), (), ()
def fire(self):
"""Called when an associated file descriptor becomes ready
(i.e., is returned from a select() call).
"""
pass
class ValueEvent(Event):
"""An event that does nothing but return a fixed value."""
def __init__(self, value):
self.value = value
class ExceptionEvent(Event):
"""Raise an exception at the yield point. Used internally."""
def __init__(self, exc_info):
self.exc_info = exc_info
class SpawnEvent(Event):
"""Add a new coroutine thread to the scheduler."""
def __init__(self, coro):
self.spawned = coro
class JoinEvent(Event):
"""Suspend the thread until the specified child thread has
completed.
"""
def __init__(self, child):
self.child = child
class KillEvent(Event):
"""Unschedule a child thread."""
def __init__(self, child):
self.child = child
class DelegationEvent(Event):
"""Suspend execution of the current thread, start a new thread and,
once the child thread finished, return control to the parent
thread.
"""
def __init__(self, coro):
self.spawned = coro
class ReturnEvent(Event):
"""Return a value the current thread's delegator at the point of
delegation. Ends the current (delegate) thread.
"""
def __init__(self, value):
self.value = value
class SleepEvent(WaitableEvent):
"""Suspend the thread for a given duration.
"""
def __init__(self, duration):
self.wakeup_time = time.time() + duration
def time_left(self):
return max(self.wakeup_time - time.time(), 0.0)
class ReadEvent(WaitableEvent):
"""Reads from a file-like object."""
def __init__(self, fd, bufsize):
self.fd = fd
self.bufsize = bufsize
def waitables(self):
return (self.fd,), (), ()
def fire(self):
return self.fd.read(self.bufsize)
class WriteEvent(WaitableEvent):
"""Writes to a file-like object."""
def __init__(self, fd, data):
self.fd = fd
self.data = data
def waitable(self):
return (), (self.fd,), ()
def fire(self):
self.fd.write(self.data)
# Core logic for executing and scheduling threads.
def _event_select(events):
"""Perform a select() over all the Events provided, returning the
ones ready to be fired. Only WaitableEvents (including SleepEvents)
matter here; all other events are ignored (and thus postponed).
"""
# Gather waitables and wakeup times.
waitable_to_event = {}
rlist, wlist, xlist = [], [], []
earliest_wakeup = None
for event in events:
if isinstance(event, SleepEvent):
if not earliest_wakeup:
earliest_wakeup = event.wakeup_time
else:
earliest_wakeup = min(earliest_wakeup, event.wakeup_time)
elif isinstance(event, WaitableEvent):
r, w, x = event.waitables()
rlist += r
wlist += w
xlist += x
for waitable in r:
waitable_to_event[('r', waitable)] = event
for waitable in w:
waitable_to_event[('w', waitable)] = event
for waitable in x:
waitable_to_event[('x', waitable)] = event
# If we have a any sleeping threads, determine how long to sleep.
if earliest_wakeup:
timeout = max(earliest_wakeup - time.time(), 0.0)
else:
timeout = None
# Perform select() if we have any waitables.
if rlist or wlist or xlist:
rready, wready, xready = select.select(rlist, wlist, xlist, timeout)
else:
rready, wready, xready = (), (), ()
if timeout:
time.sleep(timeout)
# Gather ready events corresponding to the ready waitables.
ready_events = set()
for ready in rready:
ready_events.add(waitable_to_event[('r', ready)])
for ready in wready:
ready_events.add(waitable_to_event[('w', ready)])
for ready in xready:
ready_events.add(waitable_to_event[('x', ready)])
# Gather any finished sleeps.
for event in events:
if isinstance(event, SleepEvent) and event.time_left() == 0.0:
ready_events.add(event)
return ready_events
class ThreadException(Exception):
def __init__(self, coro, exc_info):
self.coro = coro
self.exc_info = exc_info
def reraise(self):
_reraise(self.exc_info[0], self.exc_info[1], self.exc_info[2])
SUSPENDED = Event() # Special sentinel placeholder for suspended threads.
class Delegated(Event):
"""Placeholder indicating that a thread has delegated execution to a
different thread.
"""
def __init__(self, child):
self.child = child
def run(root_coro):
"""Schedules a coroutine, running it to completion. This
encapsulates the Bluelet scheduler, which the root coroutine can
add to by spawning new coroutines.
"""
# The "threads" dictionary keeps track of all the currently-
# executing and suspended coroutines. It maps coroutines to their
# currently "blocking" event. The event value may be SUSPENDED if
# the coroutine is waiting on some other condition: namely, a
# delegated coroutine or a joined coroutine. In this case, the
# coroutine should *also* appear as a value in one of the below
# dictionaries `delegators` or `joiners`.
threads = {root_coro: ValueEvent(None)}
# Maps child coroutines to delegating parents.
delegators = {}
# Maps child coroutines to joining (exit-waiting) parents.
joiners = collections.defaultdict(list)
def complete_thread(coro, return_value):
"""Remove a coroutine from the scheduling pool, awaking
delegators and joiners as necessary and returning the specified
value to any delegating parent.
"""
del threads[coro]
# Resume delegator.
if coro in delegators:
threads[delegators[coro]] = ValueEvent(return_value)
del delegators[coro]
# Resume joiners.
if coro in joiners:
for parent in joiners[coro]:
threads[parent] = ValueEvent(None)
del joiners[coro]
def advance_thread(coro, value, is_exc=False):
"""After an event is fired, run a given coroutine associated with
it in the threads dict until it yields again. If the coroutine
exits, then the thread is removed from the pool. If the coroutine
raises an exception, it is reraised in a ThreadException. If
is_exc is True, then the value must be an exc_info tuple and the
exception is thrown into the coroutine.
"""
try:
if is_exc:
next_event = coro.throw(*value)
else:
next_event = coro.send(value)
except StopIteration:
# Thread is done.
complete_thread(coro, None)
except:
# Thread raised some other exception.
del threads[coro]
raise ThreadException(coro, sys.exc_info())
else:
if isinstance(next_event, types.GeneratorType):
# Automatically invoke sub-coroutines. (Shorthand for
# explicit bluelet.call().)
next_event = DelegationEvent(next_event)
threads[coro] = next_event
def kill_thread(coro):
"""Unschedule this thread and its (recursive) delegates.
"""
# Collect all coroutines in the delegation stack.
coros = [coro]
while isinstance(threads[coro], Delegated):
coro = threads[coro].child
coros.append(coro)
# Complete each coroutine from the top to the bottom of the
# stack.
for coro in reversed(coros):
complete_thread(coro, None)
# Continue advancing threads until root thread exits.
exit_te = None
while threads:
try:
# Look for events that can be run immediately. Continue
# running immediate events until nothing is ready.
while True:
have_ready = False
for coro, event in list(threads.items()):
if isinstance(event, SpawnEvent):
threads[event.spawned] = ValueEvent(None) # Spawn.
advance_thread(coro, None)
have_ready = True
elif isinstance(event, ValueEvent):
advance_thread(coro, event.value)
have_ready = True
elif isinstance(event, ExceptionEvent):
advance_thread(coro, event.exc_info, True)
have_ready = True
elif isinstance(event, DelegationEvent):
threads[coro] = Delegated(event.spawned) # Suspend.
threads[event.spawned] = ValueEvent(None) # Spawn.
delegators[event.spawned] = coro
have_ready = True
elif isinstance(event, ReturnEvent):
# Thread is done.
complete_thread(coro, event.value)
have_ready = True
elif isinstance(event, JoinEvent):
threads[coro] = SUSPENDED # Suspend.
joiners[event.child].append(coro)
have_ready = True
elif isinstance(event, KillEvent):
threads[coro] = ValueEvent(None)
kill_thread(event.child)
have_ready = True
# Only start the select when nothing else is ready.
if not have_ready:
break
# Wait and fire.
event2coro = dict((v,k) for k,v in threads.items())
for event in _event_select(threads.values()):
# Run the IO operation, but catch socket errors.
try:
value = event.fire()
except socket.error as exc:
if isinstance(exc.args, tuple) and \
exc.args[0] == errno.EPIPE:
# Broken pipe. Remote host disconnected.
pass
else:
traceback.print_exc()
# Abort the coroutine.
threads[event2coro[event]] = ReturnEvent(None)
else:
advance_thread(event2coro[event], value)
except ThreadException as te:
# Exception raised from inside a thread.
event = ExceptionEvent(te.exc_info)
if te.coro in delegators:
# The thread is a delegate. Raise exception in its
# delegator.
threads[delegators[te.coro]] = event
del delegators[te.coro]
else:
# The thread is root-level. Raise in client code.
exit_te = te
break
except:
# For instance, KeyboardInterrupt during select(). Raise
# into root thread and terminate others.
threads = {root_coro: ExceptionEvent(sys.exc_info())}
# If any threads still remain, kill them.
for coro in threads:
coro.close()
# If we're exiting with an exception, raise it in the client.
if exit_te:
exit_te.reraise()
# Sockets and their associated events.
class SocketClosedError(Exception):
pass
class Listener(object):
"""A socket wrapper object for listening sockets.
"""
def __init__(self, host, port):
"""Create a listening socket on the given hostname and port.
"""
self._closed = False
self.host = host
self.port = port
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.sock.bind((host, port))
self.sock.listen(5)
def accept(self):
"""An event that waits for a connection on the listening socket.
When a connection is made, the event returns a Connection
object.
"""
if self._closed:
raise SocketClosedError()
return AcceptEvent(self)
def close(self):
"""Immediately close the listening socket. (Not an event.)
"""
self._closed = True
self.sock.close()
class Connection(object):
"""A socket wrapper object for connected sockets.
"""
def __init__(self, sock, addr):
self.sock = sock
self.addr = addr
self._buf = b''
self._closed = False
def close(self):
"""Close the connection."""
self._closed = True
self.sock.close()
def recv(self, size):
"""Read at most size bytes of data from the socket."""
if self._closed:
raise SocketClosedError()
if self._buf:
# We already have data read previously.
out = self._buf[:size]
self._buf = self._buf[size:]
return ValueEvent(out)
else:
return ReceiveEvent(self, size)
def send(self, data):
"""Sends data on the socket, returning the number of bytes
successfully sent.
"""
if self._closed:
raise SocketClosedError()
return SendEvent(self, data)
def sendall(self, data):
"""Send all of data on the socket."""
if self._closed:
raise SocketClosedError()
return SendEvent(self, data, True)
def readline(self, terminator=b"\n", bufsize=1024):
"""Reads a line (delimited by terminator) from the socket."""
if self._closed:
raise SocketClosedError()
while True:
if terminator in self._buf:
line, self._buf = self._buf.split(terminator, 1)
line += terminator
yield ReturnEvent(line)
break
data = yield ReceiveEvent(self, bufsize)
if data:
self._buf += data
else:
line = self._buf
self._buf = b''
yield ReturnEvent(line)
break
class AcceptEvent(WaitableEvent):
"""An event for Listener objects (listening sockets) that suspends
execution until the socket gets a connection.
"""
def __init__(self, listener):
self.listener = listener
def waitables(self):
return (self.listener.sock,), (), ()
def fire(self):
sock, addr = self.listener.sock.accept()
return Connection(sock, addr)
class ReceiveEvent(WaitableEvent):
"""An event for Connection objects (connected sockets) for
asynchronously reading data.
"""
def __init__(self, conn, bufsize):
self.conn = conn
self.bufsize = bufsize
def waitables(self):
return (self.conn.sock,), (), ()
def fire(self):
return self.conn.sock.recv(self.bufsize)
class SendEvent(WaitableEvent):
"""An event for Connection objects (connected sockets) for
asynchronously writing data.
"""
def __init__(self, conn, data, sendall=False):
self.conn = conn
self.data = data
self.sendall = sendall
def waitables(self):
return (), (self.conn.sock,), ()
def fire(self):
if self.sendall:
return self.conn.sock.sendall(self.data)
else:
return self.conn.sock.send(self.data)
# Public interface for threads; each returns an event object that
# can immediately be "yield"ed.
def null():
"""Event: yield to the scheduler without doing anything special.
"""
return ValueEvent(None)
def spawn(coro):
"""Event: add another coroutine to the scheduler. Both the parent
and child coroutines run concurrently.
"""
if not isinstance(coro, types.GeneratorType):
raise ValueError('%s is not a coroutine' % str(coro))
return SpawnEvent(coro)
def call(coro):
"""Event: delegate to another coroutine. The current coroutine
is resumed once the sub-coroutine finishes. If the sub-coroutine
returns a value using end(), then this event returns that value.
"""
if not isinstance(coro, types.GeneratorType):
raise ValueError('%s is not a coroutine' % str(coro))
return DelegationEvent(coro)
def end(value=None):
"""Event: ends the coroutine and returns a value to its
delegator.
"""
return ReturnEvent(value)
def read(fd, bufsize=None):
"""Event: read from a file descriptor asynchronously."""
if bufsize is None:
# Read all.
def reader():
buf = []
while True:
data = yield read(fd, 1024)
if not data:
break
buf.append(data)
yield ReturnEvent(''.join(buf))
return DelegationEvent(reader())
else:
return ReadEvent(fd, bufsize)
def write(fd, data):
"""Event: write to a file descriptor asynchronously."""
return WriteEvent(fd, data)
def connect(host, port):
"""Event: connect to a network address and return a Connection
object for communicating on the socket.
"""
addr = (host, port)
sock = socket.create_connection(addr)
return ValueEvent(Connection(sock, addr))
def sleep(duration):
"""Event: suspend the thread for ``duration`` seconds.
"""
return SleepEvent(duration)
def join(coro):
"""Suspend the thread until another, previously `spawn`ed thread
completes.
"""
return JoinEvent(coro)
def kill(coro):
"""Halt the execution of a different `spawn`ed thread.
"""
return KillEvent(coro)
# Convenience function for running socket servers.
def server(host, port, func):
"""A coroutine that runs a network server. Host and port specify the
listening address. func should be a coroutine that takes a single
parameter, a Connection object. The coroutine is invoked for every
incoming connection on the listening socket.
"""
def handler(conn):
try:
yield func(conn)
finally:
conn.close()
listener = Listener(host, port)
try:
while True:
conn = yield listener.accept()
yield spawn(handler(conn))
except KeyboardInterrupt:
pass
finally:
listener.close()

900
libs/beets/util/confit.py Normal file
View file

@ -0,0 +1,900 @@
# This file is part of Confit.
# Copyright 2014, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Worry-free YAML configuration files.
"""
from __future__ import unicode_literals
import platform
import os
import pkgutil
import sys
import yaml
import types
try:
from collections import OrderedDict
except ImportError:
from ordereddict import OrderedDict
UNIX_DIR_VAR = 'XDG_CONFIG_HOME'
UNIX_DIR_FALLBACK = '~/.config'
WINDOWS_DIR_VAR = 'APPDATA'
WINDOWS_DIR_FALLBACK = '~\\AppData\\Roaming'
MAC_DIR = '~/Library/Application Support'
CONFIG_FILENAME = 'config.yaml'
DEFAULT_FILENAME = 'config_default.yaml'
ROOT_NAME = 'root'
YAML_TAB_PROBLEM = "found character '\\t' that cannot start any token"
# Utilities.
PY3 = sys.version_info[0] == 3
STRING = str if PY3 else unicode
BASESTRING = str if PY3 else basestring
NUMERIC_TYPES = (int, float) if PY3 else (int, float, long)
TYPE_TYPES = (type,) if PY3 else (type, types.ClassType)
def iter_first(sequence):
"""Get the first element from an iterable or raise a ValueError if
the iterator generates no values.
"""
it = iter(sequence)
try:
if PY3:
return next(it)
else:
return it.next()
except StopIteration:
raise ValueError()
# Exceptions.
class ConfigError(Exception):
"""Base class for exceptions raised when querying a configuration.
"""
class NotFoundError(ConfigError):
"""A requested value could not be found in the configuration trees.
"""
class ConfigTypeError(ConfigError, TypeError):
"""The value in the configuration did not match the expected type.
"""
class ConfigValueError(ConfigError, ValueError):
"""The value in the configuration is illegal."""
class ConfigReadError(ConfigError):
"""A configuration file could not be read."""
def __init__(self, filename, reason=None):
self.filename = filename
self.reason = reason
message = 'file {0} could not be read'.format(filename)
if isinstance(reason, yaml.scanner.ScannerError) and \
reason.problem == YAML_TAB_PROBLEM:
# Special-case error message for tab indentation in YAML markup.
message += ': found tab character at line {0}, column {1}'.format(
reason.problem_mark.line + 1,
reason.problem_mark.column + 1,
)
elif reason:
# Generic error message uses exception's message.
message += ': {0}'.format(reason)
super(ConfigReadError, self).__init__(message)
# Views and sources.
class ConfigSource(dict):
"""A dictionary augmented with metadata about the source of the
configuration.
"""
def __init__(self, value, filename=None, default=False):
super(ConfigSource, self).__init__(value)
if filename is not None and not isinstance(filename, BASESTRING):
raise TypeError('filename must be a string or None')
self.filename = filename
self.default = default
def __repr__(self):
return 'ConfigSource({0}, {1}, {2})'.format(
super(ConfigSource, self).__repr__(),
repr(self.filename),
repr(self.default)
)
@classmethod
def of(self, value):
"""Given either a dictionary or a `ConfigSource` object, return
a `ConfigSource` object. This lets a function accept either type
of object as an argument.
"""
if isinstance(value, ConfigSource):
return value
elif isinstance(value, dict):
return ConfigSource(value)
else:
raise TypeError('source value must be a dict')
class ConfigView(object):
"""A configuration "view" is a query into a program's configuration
data. A view represents a hypothetical location in the configuration
tree; to extract the data from the location, a client typically
calls the ``view.get()`` method. The client can access children in
the tree (subviews) by subscripting the parent view (i.e.,
``view[key]``).
"""
name = None
"""The name of the view, depicting the path taken through the
configuration in Python-like syntax (e.g., ``foo['bar'][42]``).
"""
def resolve(self):
"""The core (internal) data retrieval method. Generates (value,
source) pairs for each source that contains a value for this
view. May raise ConfigTypeError if a type error occurs while
traversing a source.
"""
raise NotImplementedError
def first(self):
"""Return a (value, source) pair for the first object found for
this view. This amounts to the first element returned by
`resolve`. If no values are available, a NotFoundError is
raised.
"""
pairs = self.resolve()
try:
return iter_first(pairs)
except ValueError:
raise NotFoundError("{0} not found".format(self.name))
def exists(self):
"""Determine whether the view has a setting in any source.
"""
try:
self.first()
except NotFoundError:
return False
return True
def add(self, value):
"""Set the *default* value for this configuration view. The
specified value is added as the lowest-priority configuration
data source.
"""
raise NotImplementedError
def set(self, value):
"""*Override* the value for this configuration view. The
specified value is added as the highest-priority configuration
data source.
"""
raise NotImplementedError
def root(self):
"""The RootView object from which this view is descended.
"""
raise NotImplementedError
def __repr__(self):
return '<ConfigView: %s>' % self.name
def __getitem__(self, key):
"""Get a subview of this view."""
return Subview(self, key)
def __setitem__(self, key, value):
"""Create an overlay source to assign a given key under this
view.
"""
self.set({key: value})
def set_args(self, namespace):
"""Overlay parsed command-line arguments, generated by a library
like argparse or optparse, onto this view's value.
"""
args = {}
for key, value in namespace.__dict__.items():
if value is not None: # Avoid unset options.
args[key] = value
self.set(args)
# Magical conversions. These special methods make it possible to use
# View objects somewhat transparently in certain circumstances. For
# example, rather than using ``view.get(bool)``, it's possible to
# just say ``bool(view)`` or use ``view`` in a conditional.
def __str__(self):
"""Gets the value for this view as a byte string."""
return str(self.get())
def __unicode__(self):
"""Gets the value for this view as a unicode string. (Python 2
only.)
"""
return unicode(self.get())
def __nonzero__(self):
"""Gets the value for this view as a boolean. (Python 2 only.)
"""
return self.__bool__()
def __bool__(self):
"""Gets the value for this view as a boolean. (Python 3 only.)
"""
return bool(self.get())
# Dictionary emulation methods.
def keys(self):
"""Returns a list containing all the keys available as subviews
of the current views. This enumerates all the keys in *all*
dictionaries matching the current view, in contrast to
``view.get(dict).keys()``, which gets all the keys for the
*first* dict matching the view. If the object for this view in
any source is not a dict, then a ConfigTypeError is raised. The
keys are ordered according to how they appear in each source.
"""
keys = []
for dic, _ in self.resolve():
try:
cur_keys = dic.keys()
except AttributeError:
raise ConfigTypeError(
'{0} must be a dict, not {1}'.format(
self.name, type(dic).__name__
)
)
for key in cur_keys:
if key not in keys:
keys.append(key)
return keys
def items(self):
"""Iterates over (key, subview) pairs contained in dictionaries
from *all* sources at this view. If the object for this view in
any source is not a dict, then a ConfigTypeError is raised.
"""
for key in self.keys():
yield key, self[key]
def values(self):
"""Iterates over all the subviews contained in dictionaries from
*all* sources at this view. If the object for this view in any
source is not a dict, then a ConfigTypeError is raised.
"""
for key in self.keys():
yield self[key]
# List/sequence emulation.
def all_contents(self):
"""Iterates over all subviews from collections at this view from
*all* sources. If the object for this view in any source is not
iterable, then a ConfigTypeError is raised. This method is
intended to be used when the view indicates a list; this method
will concatenate the contents of the list from all sources.
"""
for collection, _ in self.resolve():
try:
it = iter(collection)
except TypeError:
raise ConfigTypeError(
'{0} must be an iterable, not {1}'.format(
self.name, type(collection).__name__
)
)
for value in it:
yield value
# Validation and conversion.
def get(self, typ=None):
"""Returns the canonical value for the view, checked against the
passed-in type. If the value is not an instance of the given
type, a ConfigTypeError is raised. May also raise a
NotFoundError.
"""
value, _ = self.first()
if typ is not None:
if not isinstance(typ, TYPE_TYPES):
raise TypeError('argument to get() must be a type')
if not isinstance(value, typ):
raise ConfigTypeError(
"{0} must be of type {1}, not {2}".format(
self.name, typ.__name__, type(value).__name__
)
)
return value
def as_filename(self):
"""Get a string as a normalized as an absolute, tilde-free path.
Relative paths are relative to the configuration directory (see
the `config_dir` method) if they come from a file. Otherwise,
they are relative to the current working directory. This helps
attain the expected behavior when using command-line options.
"""
path, source = self.first()
if not isinstance(path, BASESTRING):
raise ConfigTypeError('{0} must be a filename, not {1}'.format(
self.name, type(path).__name__
))
path = os.path.expanduser(STRING(path))
if not os.path.isabs(path) and source.filename:
# From defaults: relative to the app's directory.
path = os.path.join(self.root().config_dir(), path)
return os.path.abspath(path)
def as_choice(self, choices):
"""Ensure that the value is among a collection of choices and
return it. If `choices` is a dictionary, then return the
corresponding value rather than the value itself (the key).
"""
value = self.get()
if value not in choices:
raise ConfigValueError(
'{0} must be one of {1}, not {2}'.format(
self.name, repr(list(choices)), repr(value)
)
)
if isinstance(choices, dict):
return choices[value]
else:
return value
def as_number(self):
"""Ensure that a value is of numeric type."""
value = self.get()
if isinstance(value, NUMERIC_TYPES):
return value
raise ConfigTypeError(
'{0} must be numeric, not {1}'.format(
self.name, type(value).__name__
)
)
def as_str_seq(self):
"""Get the value as a list of strings. The underlying configured
value can be a sequence or a single string. In the latter case,
the string is treated as a white-space separated list of words.
"""
value = self.get()
if isinstance(value, bytes):
value = value.decode('utf8', 'ignore')
if isinstance(value, STRING):
return value.split()
else:
try:
return list(value)
except TypeError:
raise ConfigTypeError(
'{0} must be a whitespace-separated string or '
'a list'.format(self.name)
)
def flatten(self):
"""Create a hierarchy of OrderedDicts containing the data from
this view, recursively reifying all views to get their
represented values.
"""
od = OrderedDict()
for key, view in self.items():
try:
od[key] = view.flatten()
except ConfigTypeError:
od[key] = view.get()
return od
class RootView(ConfigView):
"""The base of a view hierarchy. This view keeps track of the
sources that may be accessed by subviews.
"""
def __init__(self, sources):
"""Create a configuration hierarchy for a list of sources. At
least one source must be provided. The first source in the list
has the highest priority.
"""
self.sources = list(sources)
self.name = ROOT_NAME
def add(self, obj):
self.sources.append(ConfigSource.of(obj))
def set(self, value):
self.sources.insert(0, ConfigSource.of(value))
def resolve(self):
return ((dict(s), s) for s in self.sources)
def clear(self):
"""Remove all sources from this configuration."""
del self.sources[:]
def root(self):
return self
class Subview(ConfigView):
"""A subview accessed via a subscript of a parent view."""
def __init__(self, parent, key):
"""Make a subview of a parent view for a given subscript key.
"""
self.parent = parent
self.key = key
# Choose a human-readable name for this view.
if isinstance(self.parent, RootView):
self.name = ''
else:
self.name = self.parent.name
if not isinstance(self.key, int):
self.name += '.'
if isinstance(self.key, int):
self.name += '#{0}'.format(self.key)
elif isinstance(self.key, BASESTRING):
self.name += '{0}'.format(self.key)
else:
self.name += '{0}'.format(repr(self.key))
def resolve(self):
for collection, source in self.parent.resolve():
try:
value = collection[self.key]
except IndexError:
# List index out of bounds.
continue
except KeyError:
# Dict key does not exist.
continue
except TypeError:
# Not subscriptable.
raise ConfigTypeError(
"{0} must be a collection, not {1}".format(
self.parent.name, type(collection).__name__
)
)
yield value, source
def set(self, value):
self.parent.set({self.key: value})
def add(self, value):
self.parent.add({self.key: value})
def root(self):
return self.parent.root()
# Config file paths, including platform-specific paths and in-package
# defaults.
# Based on get_root_path from Flask by Armin Ronacher.
def _package_path(name):
"""Returns the path to the package containing the named module or
None if the path could not be identified (e.g., if
``name == "__main__"``).
"""
loader = pkgutil.get_loader(name)
if loader is None or name == '__main__':
return None
if hasattr(loader, 'get_filename'):
filepath = loader.get_filename(name)
else:
# Fall back to importing the specified module.
__import__(name)
filepath = sys.modules[name].__file__
return os.path.dirname(os.path.abspath(filepath))
def config_dirs():
"""Return a platform-specific list of candidates for user
configuration directories on the system.
The candidates are in order of priority, from highest to lowest. The
last element is the "fallback" location to be used when no
higher-priority config file exists.
"""
paths = []
if platform.system() == 'Darwin':
paths.append(MAC_DIR)
paths.append(UNIX_DIR_FALLBACK)
if UNIX_DIR_VAR in os.environ:
paths.append(os.environ[UNIX_DIR_VAR])
elif platform.system() == 'Windows':
paths.append(WINDOWS_DIR_FALLBACK)
if WINDOWS_DIR_VAR in os.environ:
paths.append(os.environ[WINDOWS_DIR_VAR])
else:
# Assume Unix.
paths.append(UNIX_DIR_FALLBACK)
if UNIX_DIR_VAR in os.environ:
paths.append(os.environ[UNIX_DIR_VAR])
# Expand and deduplicate paths.
out = []
for path in paths:
path = os.path.abspath(os.path.expanduser(path))
if path not in out:
out.append(path)
return out
# YAML loading.
class Loader(yaml.SafeLoader):
"""A customized YAML loader. This loader deviates from the official
YAML spec in a few convenient ways:
- All strings as are Unicode objects.
- All maps are OrderedDicts.
- Strings can begin with % without quotation.
"""
# All strings should be Unicode objects, regardless of contents.
def _construct_unicode(self, node):
return self.construct_scalar(node)
# Use ordered dictionaries for every YAML map.
# From https://gist.github.com/844388
def construct_yaml_map(self, node):
data = OrderedDict()
yield data
value = self.construct_mapping(node)
data.update(value)
def construct_mapping(self, node, deep=False):
if isinstance(node, yaml.MappingNode):
self.flatten_mapping(node)
else:
raise yaml.constructor.ConstructorError(
None, None,
'expected a mapping node, but found %s' % node.id,
node.start_mark
)
mapping = OrderedDict()
for key_node, value_node in node.value:
key = self.construct_object(key_node, deep=deep)
try:
hash(key)
except TypeError as exc:
raise yaml.constructor.ConstructorError(
'while constructing a mapping',
node.start_mark, 'found unacceptable key (%s)' % exc,
key_node.start_mark
)
value = self.construct_object(value_node, deep=deep)
mapping[key] = value
return mapping
# Allow bare strings to begin with %. Directives are still detected.
def check_plain(self):
plain = super(Loader, self).check_plain()
return plain or self.peek() == '%'
Loader.add_constructor('tag:yaml.org,2002:str', Loader._construct_unicode)
Loader.add_constructor('tag:yaml.org,2002:map', Loader.construct_yaml_map)
Loader.add_constructor('tag:yaml.org,2002:omap', Loader.construct_yaml_map)
def load_yaml(filename):
"""Read a YAML document from a file. If the file cannot be read or
parsed, a ConfigReadError is raised.
"""
try:
with open(filename, 'r') as f:
return yaml.load(f, Loader=Loader)
except (IOError, yaml.error.YAMLError) as exc:
raise ConfigReadError(filename, exc)
# YAML dumping.
class Dumper(yaml.SafeDumper):
"""A PyYAML Dumper that represents OrderedDicts as ordinary mappings
(in order, of course).
"""
# From http://pyyaml.org/attachment/ticket/161/use_ordered_dict.py
def represent_mapping(self, tag, mapping, flow_style=None):
value = []
node = yaml.MappingNode(tag, value, flow_style=flow_style)
if self.alias_key is not None:
self.represented_objects[self.alias_key] = node
best_style = False
if hasattr(mapping, 'items'):
mapping = list(mapping.items())
for item_key, item_value in mapping:
node_key = self.represent_data(item_key)
node_value = self.represent_data(item_value)
if not (isinstance(node_key, yaml.ScalarNode)
and not node_key.style):
best_style = False
if not (isinstance(node_value, yaml.ScalarNode)
and not node_value.style):
best_style = False
value.append((node_key, node_value))
if flow_style is None:
if self.default_flow_style is not None:
node.flow_style = self.default_flow_style
else:
node.flow_style = best_style
return node
def represent_list(self, data):
"""If a list has less than 4 items, represent it in inline style
(i.e. comma separated, within square brackets).
"""
node = super(Dumper, self).represent_list(data)
length = len(data)
if self.default_flow_style is None and length < 4:
node.flow_style = True
elif self.default_flow_style is None:
node.flow_style = False
return node
def represent_bool(self, data):
"""Represent bool as 'yes' or 'no' instead of 'true' or 'false'.
"""
if data:
value = 'yes'
else:
value = 'no'
return self.represent_scalar('tag:yaml.org,2002:bool', value)
def represent_none(self, data):
"""Represent a None value with nothing instead of 'none'.
"""
return self.represent_scalar('tag:yaml.org,2002:null', '')
Dumper.add_representer(OrderedDict, Dumper.represent_dict)
Dumper.add_representer(bool, Dumper.represent_bool)
Dumper.add_representer(type(None), Dumper.represent_none)
Dumper.add_representer(list, Dumper.represent_list)
def restore_yaml_comments(data, default_data):
"""Scan default_data for comments (we include empty lines in our
definition of comments) and place them before the same keys in data.
Only works with comments that are on one or more own lines, i.e.
not next to a yaml mapping.
"""
comment_map = dict()
default_lines = iter(default_data.splitlines())
for line in default_lines:
if not line:
comment = "\n"
elif line.startswith("#"):
comment = "{0}\n".format(line)
else:
continue
while True:
line = next(default_lines)
if line and not line.startswith("#"):
break
comment += "{0}\n".format(line)
key = line.split(':')[0].strip()
comment_map[key] = comment
out_lines = iter(data.splitlines())
out_data = ""
for line in out_lines:
key = line.split(':')[0].strip()
if key in comment_map:
out_data += comment_map[key]
out_data += "{0}\n".format(line)
return out_data
# Main interface.
class Configuration(RootView):
def __init__(self, appname, modname=None, read=True):
"""Create a configuration object by reading the
automatically-discovered config files for the application for a
given name. If `modname` is specified, it should be the import
name of a module whose package will be searched for a default
config file. (Otherwise, no defaults are used.) Pass `False` for
`read` to disable automatic reading of all discovered
configuration files. Use this when creating a configuration
object at module load time and then call the `read` method
later.
"""
super(Configuration, self).__init__([])
self.appname = appname
self.modname = modname
self._env_var = '{0}DIR'.format(self.appname.upper())
if read:
self.read()
def user_config_path(self):
"""Points to the location of the user configuration.
The file may not exist.
"""
return os.path.join(self.config_dir(), CONFIG_FILENAME)
def _add_user_source(self):
"""Add the configuration options from the YAML file in the
user's configuration directory (given by `config_dir`) if it
exists.
"""
filename = self.user_config_path()
if os.path.isfile(filename):
self.add(ConfigSource(load_yaml(filename) or {}, filename))
def _add_default_source(self):
"""Add the package's default configuration settings. This looks
for a YAML file located inside the package for the module
`modname` if it was given.
"""
if self.modname:
pkg_path = _package_path(self.modname)
if pkg_path:
filename = os.path.join(pkg_path, DEFAULT_FILENAME)
if os.path.isfile(filename):
self.add(ConfigSource(load_yaml(filename), filename, True))
def read(self, user=True, defaults=True):
"""Find and read the files for this configuration and set them
as the sources for this configuration. To disable either
discovered user configuration files or the in-package defaults,
set `user` or `defaults` to `False`.
"""
if user:
self._add_user_source()
if defaults:
self._add_default_source()
def config_dir(self):
"""Get the path to the user configuration directory. The
directory is guaranteed to exist as a postcondition (one may be
created if none exist).
If the application's ``...DIR`` environment variable is set, it
is used as the configuration directory. Otherwise,
platform-specific standard configuration locations are searched
for a ``config.yaml`` file. If no configuration file is found, a
fallback path is used.
"""
# If environment variable is set, use it.
if self._env_var in os.environ:
appdir = os.environ[self._env_var]
appdir = os.path.abspath(os.path.expanduser(appdir))
if os.path.isfile(appdir):
raise ConfigError('{0} must be a directory'.format(
self._env_var
))
else:
# Search platform-specific locations. If no config file is
# found, fall back to the final directory in the list.
for confdir in config_dirs():
appdir = os.path.join(confdir, self.appname)
if os.path.isfile(os.path.join(appdir, CONFIG_FILENAME)):
break
# Ensure that the directory exists.
if not os.path.isdir(appdir):
os.makedirs(appdir)
return appdir
def set_file(self, filename):
"""Parses the file as YAML and inserts it into the configuration
sources with highest priority.
"""
filename = os.path.abspath(filename)
self.set(ConfigSource(load_yaml(filename), filename))
def dump(self, full=True):
"""Dump the Configuration object to a YAML file.
The order of the keys is determined from the default
configuration file. All keys not in the default configuration
will be appended to the end of the file.
:param filename: The file to dump the configuration to, or None
if the YAML string should be returned instead
:type filename: unicode
:param full: Dump settings that don't differ from the defaults
as well
"""
if full:
out_dict = self.flatten()
else:
# Exclude defaults when flattening.
sources = [s for s in self.sources if not s.default]
out_dict = RootView(sources).flatten()
yaml_out = yaml.dump(out_dict, Dumper=Dumper,
default_flow_style=None, indent=4,
width=1000)
# Restore comments to the YAML text.
default_source = None
for source in self.sources:
if source.default:
default_source = source
break
if default_source:
with open(default_source.filename, 'r') as fp:
default_data = fp.read()
yaml_out = restore_yaml_comments(yaml_out, default_data)
return yaml_out
class LazyConfig(Configuration):
"""A Configuration at reads files on demand when it is first
accessed. This is appropriate for using as a global config object at
the module level.
"""
def __init__(self, appname, modname=None):
super(LazyConfig, self).__init__(appname, modname, False)
self._materialized = False # Have we read the files yet?
self._lazy_prefix = [] # Pre-materialization calls to set().
self._lazy_suffix = [] # Calls to add().
def read(self, user=True, defaults=True):
self._materialized = True
super(LazyConfig, self).read(user, defaults)
def resolve(self):
if not self._materialized:
# Read files and unspool buffers.
self.read()
self.sources += self._lazy_suffix
self.sources[:0] = self._lazy_prefix
return super(LazyConfig, self).resolve()
def add(self, value):
super(LazyConfig, self).add(value)
if not self._materialized:
# Buffer additions to end.
self._lazy_suffix += self.sources
del self.sources[:]
def set(self, value):
super(LazyConfig, self).set(value)
if not self._materialized:
# Buffer additions to beginning.
self._lazy_prefix[:0] = self.sources
del self.sources[:]
def clear(self):
"""Remove all sources from this configuration."""
del self.sources[:]
self._lazy_suffix = []
self._lazy_prefix = []

178
libs/beets/util/enumeration.py Normal file
View file

@ -0,0 +1,178 @@
# This file is part of beets.
# Copyright 2013, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""A metaclass for enumerated types that really are types.
You can create enumerations with `enum(values, [name])` and they work
how you would expect them to.
>>> from enumeration import enum
>>> Direction = enum('north east south west', name='Direction')
>>> Direction.west
Direction.west
>>> Direction.west == Direction.west
True
>>> Direction.west == Direction.east
False
>>> isinstance(Direction.west, Direction)
True
>>> Direction[3]
Direction.west
>>> Direction['west']
Direction.west
>>> Direction.west.name
'west'
>>> Direction.north < Direction.west
True
Enumerations are classes; their instances represent the possible values
of the enumeration. Because Python classes must have names, you may
provide a `name` parameter to `enum`; if you don't, a meaningless one
will be chosen for you.
"""
import random
class Enumeration(type):
"""A metaclass whose classes are enumerations.
The `values` attribute of the class is used to populate the
enumeration. Values may either be a list of enumerated names or a
string containing a space-separated list of names. When the class
is created, it is instantiated for each name value in `values`.
Each such instance is the name of the enumerated item as the sole
argument.
The `Enumerated` class is a good choice for a superclass.
"""
def __init__(cls, name, bases, dic):
super(Enumeration, cls).__init__(name, bases, dic)
if 'values' not in dic:
# Do nothing if no values are provided (i.e., with
# Enumerated itself).
return
# May be called with a single string, in which case we split on
# whitespace for convenience.
values = dic['values']
if isinstance(values, basestring):
values = values.split()
# Create the Enumerated instances for each value. We have to use
# super's __setattr__ here because we disallow setattr below.
super(Enumeration, cls).__setattr__('_items_dict', {})
super(Enumeration, cls).__setattr__('_items_list', [])
for value in values:
item = cls(value, len(cls._items_list))
cls._items_dict[value] = item
cls._items_list.append(item)
def __getattr__(cls, key):
try:
return cls._items_dict[key]
except KeyError:
raise AttributeError("enumeration '" + cls.__name__ +
"' has no item '" + key + "'")
def __setattr__(cls, key, val):
raise TypeError("enumerations do not support attribute assignment")
def __getitem__(cls, key):
if isinstance(key, int):
return cls._items_list[key]
else:
return getattr(cls, key)
def __len__(cls):
return len(cls._items_list)
def __iter__(cls):
return iter(cls._items_list)
def __nonzero__(cls):
# Ensures that __len__ doesn't get called before __init__ by
# pydoc.
return True
class Enumerated(object):
"""An item in an enumeration.
Contains instance methods inherited by enumerated objects. The
metaclass is preset to `Enumeration` for your convenience.
Instance attributes:
name -- The name of the item.
index -- The index of the item in its enumeration.
>>> from enumeration import Enumerated
>>> class Garment(Enumerated):
... values = 'hat glove belt poncho lederhosen suspenders'
... def wear(self):
... print('now wearing a ' + self.name)
...
>>> Garment.poncho.wear()
now wearing a poncho
"""
__metaclass__ = Enumeration
def __init__(self, name, index):
self.name = name
self.index = index
def __str__(self):
return type(self).__name__ + '.' + self.name
def __repr__(self):
return str(self)
def __cmp__(self, other):
if type(self) is type(other):
# Note that we're assuming that the items are direct
# instances of the same Enumeration (i.e., no fancy
# subclassing), which is probably okay.
return cmp(self.index, other.index)
else:
return NotImplemented
def enum(*values, **kwargs):
"""Shorthand for creating a new Enumeration class.
Call with enumeration values as a list, a space-delimited string, or
just an argument list. To give the class a name, pass it as the
`name` keyword argument. Otherwise, a name will be chosen for you.
The following are all equivalent:
enum('pinkie ring middle index thumb')
enum('pinkie', 'ring', 'middle', 'index', 'thumb')
enum(['pinkie', 'ring', 'middle', 'index', 'thumb'])
"""
if ('name' not in kwargs) or kwargs['name'] is None:
# Create a probably-unique name. It doesn't really have to be
# unique, but getting distinct names each time helps with
# identification in debugging.
name = 'Enumeration' + hex(random.randint(0,0xfffffff))[2:].upper()
else:
name = kwargs['name']
if len(values) == 1:
# If there's only one value, we have a couple of alternate calling
# styles.
if isinstance(values[0], basestring) or hasattr(values[0], '__iter__'):
values = values[0]
return type(name, (Enumerated,), {'values': values})

561
libs/beets/util/functemplate.py Normal file
View file

@ -0,0 +1,561 @@
# This file is part of beets.
# Copyright 2013, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""This module implements a string formatter based on the standard PEP
292 string.Template class extended with function calls. Variables, as
with string.Template, are indicated with $ and functions are delimited
with %.
This module assumes that everything is Unicode: the template and the
substitution values. Bytestrings are not supported. Also, the templates
always behave like the ``safe_substitute`` method in the standard
library: unknown symbols are left intact.
This is sort of like a tiny, horrible degeneration of a real templating
engine like Jinja2 or Mustache.
"""
from __future__ import print_function
import re
import ast
import dis
import types
SYMBOL_DELIM = u'$'
FUNC_DELIM = u'%'
GROUP_OPEN = u'{'
GROUP_CLOSE = u'}'
ARG_SEP = u','
ESCAPE_CHAR = u'$'
VARIABLE_PREFIX = '__var_'
FUNCTION_PREFIX = '__func_'
class Environment(object):
"""Contains the values and functions to be substituted into a
template.
"""
def __init__(self, values, functions):
self.values = values
self.functions = functions
# Code generation helpers.
def ex_lvalue(name):
"""A variable load expression."""
return ast.Name(name, ast.Store())
def ex_rvalue(name):
"""A variable store expression."""
return ast.Name(name, ast.Load())
def ex_literal(val):
"""An int, float, long, bool, string, or None literal with the given
value.
"""
if val is None:
return ast.Name('None', ast.Load())
elif isinstance(val, (int, float, long)):
return ast.Num(val)
elif isinstance(val, bool):
return ast.Name(str(val), ast.Load())
elif isinstance(val, basestring):
return ast.Str(val)
raise TypeError('no literal for {0}'.format(type(val)))
def ex_varassign(name, expr):
"""Assign an expression into a single variable. The expression may
either be an `ast.expr` object or a value to be used as a literal.
"""
if not isinstance(expr, ast.expr):
expr = ex_literal(expr)
return ast.Assign([ex_lvalue(name)], expr)
def ex_call(func, args):
"""A function-call expression with only positional parameters. The
function may be an expression or the name of a function. Each
argument may be an expression or a value to be used as a literal.
"""
if isinstance(func, basestring):
func = ex_rvalue(func)
args = list(args)
for i in range(len(args)):
if not isinstance(args[i], ast.expr):
args[i] = ex_literal(args[i])
return ast.Call(func, args, [], None, None)
def compile_func(arg_names, statements, name='_the_func', debug=False):
"""Compile a list of statements as the body of a function and return
the resulting Python function. If `debug`, then print out the
bytecode of the compiled function.
"""
func_def = ast.FunctionDef(
name,
ast.arguments(
[ast.Name(n, ast.Param()) for n in arg_names],
None, None,
[ex_literal(None) for _ in arg_names],
),
statements,
[],
)
mod = ast.Module([func_def])
ast.fix_missing_locations(mod)
prog = compile(mod, '<generated>', 'exec')
# Debug: show bytecode.
if debug:
dis.dis(prog)
for const in prog.co_consts:
if isinstance(const, types.CodeType):
dis.dis(const)
the_locals = {}
exec prog in {}, the_locals
return the_locals[name]
# AST nodes for the template language.
class Symbol(object):
"""A variable-substitution symbol in a template."""
def __init__(self, ident, original):
self.ident = ident
self.original = original
def __repr__(self):
return u'Symbol(%s)' % repr(self.ident)
def evaluate(self, env):
"""Evaluate the symbol in the environment, returning a Unicode
string.
"""
if self.ident in env.values:
# Substitute for a value.
return env.values[self.ident]
else:
# Keep original text.
return self.original
def translate(self):
"""Compile the variable lookup."""
expr = ex_rvalue(VARIABLE_PREFIX + self.ident.encode('utf8'))
return [expr], set([self.ident.encode('utf8')]), set()
class Call(object):
"""A function call in a template."""
def __init__(self, ident, args, original):
self.ident = ident
self.args = args
self.original = original
def __repr__(self):
return u'Call(%s, %s, %s)' % (repr(self.ident), repr(self.args),
repr(self.original))
def evaluate(self, env):
"""Evaluate the function call in the environment, returning a
Unicode string.
"""
if self.ident in env.functions:
arg_vals = [expr.evaluate(env) for expr in self.args]
try:
out = env.functions[self.ident](*arg_vals)
except Exception as exc:
# Function raised exception! Maybe inlining the name of
# the exception will help debug.
return u'<%s>' % unicode(exc)
return unicode(out)
else:
return self.original
def translate(self):
"""Compile the function call."""
varnames = set()
funcnames = set([self.ident.encode('utf8')])
arg_exprs = []
for arg in self.args:
subexprs, subvars, subfuncs = arg.translate()
varnames.update(subvars)
funcnames.update(subfuncs)
# Create a subexpression that joins the result components of
# the arguments.
arg_exprs.append(ex_call(
ast.Attribute(ex_literal(u''), 'join', ast.Load()),
[ex_call(
'map',
[
ex_rvalue('unicode'),
ast.List(subexprs, ast.Load()),
]
)],
))
subexpr_call = ex_call(
FUNCTION_PREFIX + self.ident.encode('utf8'),
arg_exprs
)
return [subexpr_call], varnames, funcnames
class Expression(object):
"""Top-level template construct: contains a list of text blobs,
Symbols, and Calls.
"""
def __init__(self, parts):
self.parts = parts
def __repr__(self):
return u'Expression(%s)' % (repr(self.parts))
def evaluate(self, env):
"""Evaluate the entire expression in the environment, returning
a Unicode string.
"""
out = []
for part in self.parts:
if isinstance(part, basestring):
out.append(part)
else:
out.append(part.evaluate(env))
return u''.join(map(unicode, out))
def translate(self):
"""Compile the expression to a list of Python AST expressions, a
set of variable names used, and a set of function names.
"""
expressions = []
varnames = set()
funcnames = set()
for part in self.parts:
if isinstance(part, basestring):
expressions.append(ex_literal(part))
else:
e, v, f = part.translate()
expressions.extend(e)
varnames.update(v)
funcnames.update(f)
return expressions, varnames, funcnames
# Parser.
class ParseError(Exception):
pass
class Parser(object):
"""Parses a template expression string. Instantiate the class with
the template source and call ``parse_expression``. The ``pos`` field
will indicate the character after the expression finished and
``parts`` will contain a list of Unicode strings, Symbols, and Calls
reflecting the concatenated portions of the expression.
This is a terrible, ad-hoc parser implementation based on a
left-to-right scan with no lexing step to speak of; it's probably
both inefficient and incorrect. Maybe this should eventually be
replaced with a real, accepted parsing technique (PEG, parser
generator, etc.).
"""
def __init__(self, string):
self.string = string
self.pos = 0
self.parts = []
# Common parsing resources.
special_chars = (SYMBOL_DELIM, FUNC_DELIM, GROUP_OPEN, GROUP_CLOSE,
ARG_SEP, ESCAPE_CHAR)
special_char_re = re.compile(ur'[%s]|$' %
u''.join(re.escape(c) for c in special_chars))
def parse_expression(self):
"""Parse a template expression starting at ``pos``. Resulting
components (Unicode strings, Symbols, and Calls) are added to
the ``parts`` field, a list. The ``pos`` field is updated to be
the next character after the expression.
"""
text_parts = []
while self.pos < len(self.string):
char = self.string[self.pos]
if char not in self.special_chars:
# A non-special character. Skip to the next special
# character, treating the interstice as literal text.
next_pos = (
self.special_char_re.search(self.string[self.pos:]).start()
+ self.pos
)
text_parts.append(self.string[self.pos:next_pos])
self.pos = next_pos
continue
if self.pos == len(self.string) - 1:
# The last character can never begin a structure, so we
# just interpret it as a literal character (unless it
# terminates the expression, as with , and }).
if char not in (GROUP_CLOSE, ARG_SEP):
text_parts.append(char)
self.pos += 1
break
next_char = self.string[self.pos + 1]
if char == ESCAPE_CHAR and next_char in \
(SYMBOL_DELIM, FUNC_DELIM, GROUP_CLOSE, ARG_SEP):
# An escaped special character ($$, $}, etc.). Note that
# ${ is not an escape sequence: this is ambiguous with
# the start of a symbol and it's not necessary (just
# using { suffices in all cases).
text_parts.append(next_char)
self.pos += 2 # Skip the next character.
continue
# Shift all characters collected so far into a single string.
if text_parts:
self.parts.append(u''.join(text_parts))
text_parts = []
if char == SYMBOL_DELIM:
# Parse a symbol.
self.parse_symbol()
elif char == FUNC_DELIM:
# Parse a function call.
self.parse_call()
elif char in (GROUP_CLOSE, ARG_SEP):
# Template terminated.
break
elif char == GROUP_OPEN:
# Start of a group has no meaning hear; just pass
# through the character.
text_parts.append(char)
self.pos += 1
else:
assert False
# If any parsed characters remain, shift them into a string.
if text_parts:
self.parts.append(u''.join(text_parts))
def parse_symbol(self):
"""Parse a variable reference (like ``$foo`` or ``${foo}``)
starting at ``pos``. Possibly appends a Symbol object (or,
failing that, text) to the ``parts`` field and updates ``pos``.
The character at ``pos`` must, as a precondition, be ``$``.
"""
assert self.pos < len(self.string)
assert self.string[self.pos] == SYMBOL_DELIM
if self.pos == len(self.string) - 1:
# Last character.
self.parts.append(SYMBOL_DELIM)
self.pos += 1
return
next_char = self.string[self.pos + 1]
start_pos = self.pos
self.pos += 1
if next_char == GROUP_OPEN:
# A symbol like ${this}.
self.pos += 1 # Skip opening.
closer = self.string.find(GROUP_CLOSE, self.pos)
if closer == -1 or closer == self.pos:
# No closing brace found or identifier is empty.
self.parts.append(self.string[start_pos:self.pos])
else:
# Closer found.
ident = self.string[self.pos:closer]
self.pos = closer + 1
self.parts.append(Symbol(ident,
self.string[start_pos:self.pos]))
else:
# A bare-word symbol.
ident = self._parse_ident()
if ident:
# Found a real symbol.
self.parts.append(Symbol(ident,
self.string[start_pos:self.pos]))
else:
# A standalone $.
self.parts.append(SYMBOL_DELIM)
def parse_call(self):
"""Parse a function call (like ``%foo{bar,baz}``) starting at
``pos``. Possibly appends a Call object to ``parts`` and update
``pos``. The character at ``pos`` must be ``%``.
"""
assert self.pos < len(self.string)
assert self.string[self.pos] == FUNC_DELIM
start_pos = self.pos
self.pos += 1
ident = self._parse_ident()
if not ident:
# No function name.
self.parts.append(FUNC_DELIM)
return
if self.pos >= len(self.string):
# Identifier terminates string.
self.parts.append(self.string[start_pos:self.pos])
return
if self.string[self.pos] != GROUP_OPEN:
# Argument list not opened.
self.parts.append(self.string[start_pos:self.pos])
return
# Skip past opening brace and try to parse an argument list.
self.pos += 1
args = self.parse_argument_list()
if self.pos >= len(self.string) or \
self.string[self.pos] != GROUP_CLOSE:
# Arguments unclosed.
self.parts.append(self.string[start_pos:self.pos])
return
self.pos += 1 # Move past closing brace.
self.parts.append(Call(ident, args, self.string[start_pos:self.pos]))
def parse_argument_list(self):
"""Parse a list of arguments starting at ``pos``, returning a
list of Expression objects. Does not modify ``parts``. Should
leave ``pos`` pointing to a } character or the end of the
string.
"""
# Try to parse a subexpression in a subparser.
expressions = []
while self.pos < len(self.string):
subparser = Parser(self.string[self.pos:])
subparser.parse_expression()
# Extract and advance past the parsed expression.
expressions.append(Expression(subparser.parts))
self.pos += subparser.pos
if self.pos >= len(self.string) or \
self.string[self.pos] == GROUP_CLOSE:
# Argument list terminated by EOF or closing brace.
break
# Only other way to terminate an expression is with ,.
# Continue to the next argument.
assert self.string[self.pos] == ARG_SEP
self.pos += 1
return expressions
def _parse_ident(self):
"""Parse an identifier and return it (possibly an empty string).
Updates ``pos``.
"""
remainder = self.string[self.pos:]
ident = re.match(ur'\w*', remainder).group(0)
self.pos += len(ident)
return ident
def _parse(template):
"""Parse a top-level template string Expression. Any extraneous text
is considered literal text.
"""
parser = Parser(template)
parser.parse_expression()
parts = parser.parts
remainder = parser.string[parser.pos:]
if remainder:
parts.append(remainder)
return Expression(parts)
# External interface.
class Template(object):
"""A string template, including text, Symbols, and Calls.
"""
def __init__(self, template):
self.expr = _parse(template)
self.original = template
self.compiled = self.translate()
def __eq__(self, other):
return self.original == other.original
def interpret(self, values={}, functions={}):
"""Like `substitute`, but forces the interpreter (rather than
the compiled version) to be used. The interpreter includes
exception-handling code for missing variables and buggy template
functions but is much slower.
"""
return self.expr.evaluate(Environment(values, functions))
def substitute(self, values={}, functions={}):
"""Evaluate the template given the values and functions.
"""
try:
res = self.compiled(values, functions)
except: # Handle any exceptions thrown by compiled version.
res = self.interpret(values, functions)
return res
def translate(self):
"""Compile the template to a Python function."""
expressions, varnames, funcnames = self.expr.translate()
argnames = []
for varname in varnames:
argnames.append(VARIABLE_PREFIX.encode('utf8') + varname)
for funcname in funcnames:
argnames.append(FUNCTION_PREFIX.encode('utf8') + funcname)
func = compile_func(
argnames,
[ast.Return(ast.List(expressions, ast.Load()))],
)
def wrapper_func(values={}, functions={}):
args = {}
for varname in varnames:
args[VARIABLE_PREFIX + varname] = values[varname]
for funcname in funcnames:
args[FUNCTION_PREFIX + funcname] = functions[funcname]
parts = func(**args)
return u''.join(parts)
return wrapper_func
# Performance tests.
if __name__ == '__main__':
import timeit
_tmpl = Template(u'foo $bar %baz{foozle $bar barzle} $bar')
_vars = {'bar': 'qux'}
_funcs = {'baz': unicode.upper}
interp_time = timeit.timeit('_tmpl.interpret(_vars, _funcs)',
'from __main__ import _tmpl, _vars, _funcs',
number=10000)
print(interp_time)
comp_time = timeit.timeit('_tmpl.substitute(_vars, _funcs)',
'from __main__ import _tmpl, _vars, _funcs',
number=10000)
print(comp_time)
print('Speedup:', interp_time / comp_time)

454
libs/beets/util/pipeline.py Normal file
View file

@ -0,0 +1,454 @@
# This file is part of beets.
# Copyright 2013, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Simple but robust implementation of generator/coroutine-based
pipelines in Python. The pipelines may be run either sequentially
(single-threaded) or in parallel (one thread per pipeline stage).
This implementation supports pipeline bubbles (indications that the
processing for a certain item should abort). To use them, yield the
BUBBLE constant from any stage coroutine except the last.
In the parallel case, the implementation transparently handles thread
shutdown when the processing is complete and when a stage raises an
exception. KeyboardInterrupts (^C) are also handled.
When running a parallel pipeline, it is also possible to use
multiple coroutines for the same pipeline stage; this lets you speed
up a bottleneck stage by dividing its work among multiple threads.
To do so, pass an iterable of coroutines to the Pipeline constructor
in place of any single coroutine.
"""
from __future__ import print_function
import Queue
from threading import Thread, Lock
import sys
import types
BUBBLE = '__PIPELINE_BUBBLE__'
POISON = '__PIPELINE_POISON__'
DEFAULT_QUEUE_SIZE = 16
def _invalidate_queue(q, val=None, sync=True):
"""Breaks a Queue such that it never blocks, always has size 1,
and has no maximum size. get()ing from the queue returns `val`,
which defaults to None. `sync` controls whether a lock is
required (because it's not reentrant!).
"""
def _qsize(len=len):
return 1
def _put(item):
pass
def _get():
return val
if sync:
q.mutex.acquire()
try:
q.maxsize = 0
q._qsize = _qsize
q._put = _put
q._get = _get
q.not_empty.notifyAll()
q.not_full.notifyAll()
finally:
if sync:
q.mutex.release()
class CountedQueue(Queue.Queue):
"""A queue that keeps track of the number of threads that are
still feeding into it. The queue is poisoned when all threads are
finished with the queue.
"""
def __init__(self, maxsize=0):
Queue.Queue.__init__(self, maxsize)
self.nthreads = 0
self.poisoned = False
def acquire(self):
"""Indicate that a thread will start putting into this queue.
Should not be called after the queue is already poisoned.
"""
with self.mutex:
assert not self.poisoned
assert self.nthreads >= 0
self.nthreads += 1
def release(self):
"""Indicate that a thread that was putting into this queue has
exited. If this is the last thread using the queue, the queue
is poisoned.
"""
with self.mutex:
self.nthreads -= 1
assert self.nthreads >= 0
if self.nthreads == 0:
# All threads are done adding to this queue. Poison it
# when it becomes empty.
self.poisoned = True
# Replacement _get invalidates when no items remain.
_old_get = self._get
def _get():
out = _old_get()
if not self.queue:
_invalidate_queue(self, POISON, False)
return out
if self.queue:
# Items remain.
self._get = _get
else:
# No items. Invalidate immediately.
_invalidate_queue(self, POISON, False)
class MultiMessage(object):
"""A message yielded by a pipeline stage encapsulating multiple
values to be sent to the next stage.
"""
def __init__(self, messages):
self.messages = messages
def multiple(messages):
"""Yield multiple([message, ..]) from a pipeline stage to send
multiple values to the next pipeline stage.
"""
return MultiMessage(messages)
def _allmsgs(obj):
"""Returns a list of all the messages encapsulated in obj. If obj
is a MultiMessage, returns its enclosed messages. If obj is BUBBLE,
returns an empty list. Otherwise, returns a list containing obj.
"""
if isinstance(obj, MultiMessage):
return obj.messages
elif obj == BUBBLE:
return []
else:
return [obj]
class PipelineThread(Thread):
"""Abstract base class for pipeline-stage threads."""
def __init__(self, all_threads):
super(PipelineThread, self).__init__()
self.abort_lock = Lock()
self.abort_flag = False
self.all_threads = all_threads
self.exc_info = None
def abort(self):
"""Shut down the thread at the next chance possible.
"""
with self.abort_lock:
self.abort_flag = True
# Ensure that we are not blocking on a queue read or write.
if hasattr(self, 'in_queue'):
_invalidate_queue(self.in_queue, POISON)
if hasattr(self, 'out_queue'):
_invalidate_queue(self.out_queue, POISON)
def abort_all(self, exc_info):
"""Abort all other threads in the system for an exception.
"""
self.exc_info = exc_info
for thread in self.all_threads:
thread.abort()
class FirstPipelineThread(PipelineThread):
"""The thread running the first stage in a parallel pipeline setup.
The coroutine should just be a generator.
"""
def __init__(self, coro, out_queue, all_threads):
super(FirstPipelineThread, self).__init__(all_threads)
self.coro = coro
self.out_queue = out_queue
self.out_queue.acquire()
self.abort_lock = Lock()
self.abort_flag = False
def run(self):
try:
while True:
with self.abort_lock:
if self.abort_flag:
return
# Get the value from the generator.
try:
msg = self.coro.next()
except StopIteration:
break
# Send messages to the next stage.
for msg in _allmsgs(msg):
with self.abort_lock:
if self.abort_flag:
return
self.out_queue.put(msg)
except:
self.abort_all(sys.exc_info())
return
# Generator finished; shut down the pipeline.
self.out_queue.release()
class MiddlePipelineThread(PipelineThread):
"""A thread running any stage in the pipeline except the first or
last.
"""
def __init__(self, coro, in_queue, out_queue, all_threads):
super(MiddlePipelineThread, self).__init__(all_threads)
self.coro = coro
self.in_queue = in_queue
self.out_queue = out_queue
self.out_queue.acquire()
def run(self):
try:
# Prime the coroutine.
self.coro.next()
while True:
with self.abort_lock:
if self.abort_flag:
return
# Get the message from the previous stage.
msg = self.in_queue.get()
if msg is POISON:
break
with self.abort_lock:
if self.abort_flag:
return
# Invoke the current stage.
out = self.coro.send(msg)
# Send messages to next stage.
for msg in _allmsgs(out):
with self.abort_lock:
if self.abort_flag:
return
self.out_queue.put(msg)
except:
self.abort_all(sys.exc_info())
return
# Pipeline is shutting down normally.
self.out_queue.release()
class LastPipelineThread(PipelineThread):
"""A thread running the last stage in a pipeline. The coroutine
should yield nothing.
"""
def __init__(self, coro, in_queue, all_threads):
super(LastPipelineThread, self).__init__(all_threads)
self.coro = coro
self.in_queue = in_queue
def run(self):
# Prime the coroutine.
self.coro.next()
try:
while True:
with self.abort_lock:
if self.abort_flag:
return
# Get the message from the previous stage.
msg = self.in_queue.get()
if msg is POISON:
break
with self.abort_lock:
if self.abort_flag:
return
# Send to consumer.
self.coro.send(msg)
except:
self.abort_all(sys.exc_info())
return
class Pipeline(object):
"""Represents a staged pattern of work. Each stage in the pipeline
is a coroutine that receives messages from the previous stage and
yields messages to be sent to the next stage.
"""
def __init__(self, stages):
"""Makes a new pipeline from a list of coroutines. There must
be at least two stages.
"""
if len(stages) < 2:
raise ValueError('pipeline must have at least two stages')
self.stages = []
for stage in stages:
if isinstance(stage, (list, tuple)):
self.stages.append(stage)
else:
# Default to one thread per stage.
self.stages.append((stage,))
def run_sequential(self):
"""Run the pipeline sequentially in the current thread. The
stages are run one after the other. Only the first coroutine
in each stage is used.
"""
list(self.pull())
def run_parallel(self, queue_size=DEFAULT_QUEUE_SIZE):
"""Run the pipeline in parallel using one thread per stage. The
messages between the stages are stored in queues of the given
size.
"""
queues = [CountedQueue(queue_size) for i in range(len(self.stages)-1)]
threads = []
# Set up first stage.
for coro in self.stages[0]:
threads.append(FirstPipelineThread(coro, queues[0], threads))
# Middle stages.
for i in range(1, len(self.stages)-1):
for coro in self.stages[i]:
threads.append(MiddlePipelineThread(
coro, queues[i-1], queues[i], threads
))
# Last stage.
for coro in self.stages[-1]:
threads.append(
LastPipelineThread(coro, queues[-1], threads)
)
# Start threads.
for thread in threads:
thread.start()
# Wait for termination. The final thread lasts the longest.
try:
# Using a timeout allows us to receive KeyboardInterrupt
# exceptions during the join().
while threads[-1].isAlive():
threads[-1].join(1)
except:
# Stop all the threads immediately.
for thread in threads:
thread.abort()
raise
finally:
# Make completely sure that all the threads have finished
# before we return. They should already be either finished,
# in normal operation, or aborted, in case of an exception.
for thread in threads[:-1]:
thread.join()
for thread in threads:
exc_info = thread.exc_info
if exc_info:
# Make the exception appear as it was raised originally.
raise exc_info[0], exc_info[1], exc_info[2]
def pull(self):
"""Yield elements from the end of the pipeline. Runs the stages
sequentially until the last yields some messages. Each of the messages
is then yielded by ``pulled.next()``. If the pipeline has a consumer,
that is the last stage does not yield any messages, then pull will not
yield any messages. Only the first coroutine in each stage is used
"""
coros = [stage[0] for stage in self.stages]
# "Prime" the coroutines.
for coro in coros[1:]:
coro.next()
# Begin the pipeline.
for out in coros[0]:
msgs = _allmsgs(out)
for coro in coros[1:]:
next_msgs = []
for msg in msgs:
out = coro.send(msg)
next_msgs.extend(_allmsgs(out))
msgs = next_msgs
for msg in msgs:
yield msg
# Smoke test.
if __name__ == '__main__':
import time
# Test a normally-terminating pipeline both in sequence and
# in parallel.
def produce():
for i in range(5):
print('generating %i' % i)
time.sleep(1)
yield i
def work():
num = yield
while True:
print('processing %i' % num)
time.sleep(2)
num = yield num*2
def consume():
while True:
num = yield
time.sleep(1)
print('received %i' % num)
ts_start = time.time()
Pipeline([produce(), work(), consume()]).run_sequential()
ts_seq = time.time()
Pipeline([produce(), work(), consume()]).run_parallel()
ts_par = time.time()
Pipeline([produce(), (work(), work()), consume()]).run_parallel()
ts_end = time.time()
print('Sequential time:', ts_seq - ts_start)
print('Parallel time:', ts_par - ts_seq)
print('Multiply-parallel time:', ts_end - ts_par)
print()
# Test a pipeline that raises an exception.
def exc_produce():
for i in range(10):
print('generating %i' % i)
time.sleep(1)
yield i
def exc_work():
num = yield
while True:
print('processing %i' % num)
time.sleep(3)
if num == 3:
raise Exception()
num = yield num * 2
def exc_consume():
while True:
num = yield
#if num == 4:
# raise Exception()
print('received %i' % num)
Pipeline([exc_produce(), exc_work(), exc_consume()]).run_parallel(1)

48
libs/beets/vfs.py Normal file
View file

@ -0,0 +1,48 @@
# This file is part of beets.
# Copyright 2013, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""A simple utility for constructing filesystem-like trees from beets
libraries.
"""
from collections import namedtuple
from beets import util
Node = namedtuple('Node', ['files', 'dirs'])
def _insert(node, path, itemid):
"""Insert an item into a virtual filesystem node."""
if len(path) == 1:
# Last component. Insert file.
node.files[path[0]] = itemid
else:
# In a directory.
dirname = path[0]
rest = path[1:]
if dirname not in node.dirs:
node.dirs[dirname] = Node({}, {})
_insert(node.dirs[dirname], rest, itemid)
def libtree(lib):
"""Generates a filesystem-like directory tree for the files
contained in `lib`. Filesystem nodes are (files, dirs) named
tuples in which both components are dictionaries. The first
maps filenames to Item ids. The second maps directory names to
child node tuples.
"""
root = Node({}, {})
for item in lib.items():
dest = item.destination(fragment=True)
parts = util.components(dest)
_insert(root, parts, item.id)
return root

263
libs/mutagen/__init__.py Normal file
View file

@ -0,0 +1,263 @@
# mutagen aims to be an all purpose media tagging library
# Copyright (C) 2005 Michael Urman
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of version 2 of the GNU General Public License as
# published by the Free Software Foundation.
"""Mutagen aims to be an all purpose tagging library.
::
import mutagen.[format]
metadata = mutagen.[format].Open(filename)
`metadata` acts like a dictionary of tags in the file. Tags are generally a
list of string-like values, but may have additional methods available
depending on tag or format. They may also be entirely different objects
for certain keys, again depending on format.
"""
version = (1, 22)
"""Version tuple."""
version_string = ".".join(map(str, version))
"""Version string."""
import warnings
import mutagen._util
class Metadata(object):
"""An abstract dict-like object.
Metadata is the base class for many of the tag objects in Mutagen.
"""
def __init__(self, *args, **kwargs):
if args or kwargs:
self.load(*args, **kwargs)
def load(self, *args, **kwargs):
raise NotImplementedError
def save(self, filename=None):
"""Save changes to a file."""
raise NotImplementedError
def delete(self, filename=None):
"""Remove tags from a file."""
raise NotImplementedError
class FileType(mutagen._util.DictMixin):
"""An abstract object wrapping tags and audio stream information.
Attributes:
* info -- stream information (length, bitrate, sample rate)
* tags -- metadata tags, if any
Each file format has different potential tags and stream
information.
FileTypes implement an interface very similar to Metadata; the
dict interface, save, load, and delete calls on a FileType call
the appropriate methods on its tag data.
"""
info = None
tags = None
filename = None
_mimes = ["application/octet-stream"]
def __init__(self, filename=None, *args, **kwargs):
if filename is None:
warnings.warn("FileType constructor requires a filename",
DeprecationWarning)
else:
self.load(filename, *args, **kwargs)
def load(self, filename, *args, **kwargs):
raise NotImplementedError
def __getitem__(self, key):
"""Look up a metadata tag key.
If the file has no tags at all, a KeyError is raised.
"""
if self.tags is None:
raise KeyError(key)
else:
return self.tags[key]
def __setitem__(self, key, value):
"""Set a metadata tag.
If the file has no tags, an appropriate format is added (but
not written until save is called).
"""
if self.tags is None:
self.add_tags()
self.tags[key] = value
def __delitem__(self, key):
"""Delete a metadata tag key.
If the file has no tags at all, a KeyError is raised.
"""
if self.tags is None:
raise KeyError(key)
else:
del(self.tags[key])
def keys(self):
"""Return a list of keys in the metadata tag.
If the file has no tags at all, an empty list is returned.
"""
if self.tags is None:
return []
else:
return self.tags.keys()
def delete(self, filename=None):
"""Remove tags from a file."""
if self.tags is not None:
if filename is None:
filename = self.filename
else:
warnings.warn(
"delete(filename=...) is deprecated, reload the file",
DeprecationWarning)
return self.tags.delete(filename)
def save(self, filename=None, **kwargs):
"""Save metadata tags."""
if filename is None:
filename = self.filename
else:
warnings.warn(
"save(filename=...) is deprecated, reload the file",
DeprecationWarning)
if self.tags is not None:
return self.tags.save(filename, **kwargs)
else:
raise ValueError("no tags in file")
def pprint(self):
"""Print stream information and comment key=value pairs."""
stream = "%s (%s)" % (self.info.pprint(), self.mime[0])
try:
tags = self.tags.pprint()
except AttributeError:
return stream
else:
return stream + ((tags and "\n" + tags) or "")
def add_tags(self):
"""Adds new tags to the file.
Raises if tags already exist.
"""
raise NotImplementedError
@property
def mime(self):
"""A list of mime types"""
mimes = []
for Kind in type(self).__mro__:
for mime in getattr(Kind, '_mimes', []):
if mime not in mimes:
mimes.append(mime)
return mimes
@staticmethod
def score(filename, fileobj, header):
raise NotImplementedError
def File(filename, options=None, easy=False):
"""Guess the type of the file and try to open it.
The file type is decided by several things, such as the first 128
bytes (which usually contains a file type identifier), the
filename extension, and the presence of existing tags.
If no appropriate type could be found, None is returned.
:param options: Sequence of :class:`FileType` implementations, defaults to
all included ones.
:param easy: If the easy wrappers should be returnd if available.
For example :class:`EasyMP3 <mp3.EasyMP3>` instead
of :class:`MP3 <mp3.MP3>`.
"""
if options is None:
from mutagen.asf import ASF
from mutagen.apev2 import APEv2File
from mutagen.flac import FLAC
if easy:
from mutagen.easyid3 import EasyID3FileType as ID3FileType
else:
from mutagen.id3 import ID3FileType
if easy:
from mutagen.mp3 import EasyMP3 as MP3
else:
from mutagen.mp3 import MP3
from mutagen.oggflac import OggFLAC
from mutagen.oggspeex import OggSpeex
from mutagen.oggtheora import OggTheora
from mutagen.oggvorbis import OggVorbis
from mutagen.oggopus import OggOpus
if easy:
from mutagen.trueaudio import EasyTrueAudio as TrueAudio
else:
from mutagen.trueaudio import TrueAudio
from mutagen.wavpack import WavPack
if easy:
from mutagen.easymp4 import EasyMP4 as MP4
else:
from mutagen.mp4 import MP4
from mutagen.musepack import Musepack
from mutagen.monkeysaudio import MonkeysAudio
from mutagen.optimfrog import OptimFROG
options = [MP3, TrueAudio, OggTheora, OggSpeex, OggVorbis, OggFLAC,
FLAC, APEv2File, MP4, ID3FileType, WavPack, Musepack,
MonkeysAudio, OptimFROG, ASF, OggOpus]
if not options:
return None
fileobj = open(filename, "rb")
try:
header = fileobj.read(128)
# Sort by name after score. Otherwise import order affects
# Kind sort order, which affects treatment of things with
# equals scores.
results = [(Kind.score(filename, fileobj, header), Kind.__name__)
for Kind in options]
finally:
fileobj.close()
results = zip(results, options)
results.sort()
(score, name), Kind = results[-1]
if score > 0:
return Kind(filename)
else:
return None

197
libs/mutagen/_constants.py Normal file
View file

@ -0,0 +1,197 @@
"""Constants used by Mutagen."""
GENRES = [
u"Blues",
u"Classic Rock",
u"Country",
u"Dance",
u"Disco",
u"Funk",
u"Grunge",
u"Hip-Hop",
u"Jazz",
u"Metal",
u"New Age",
u"Oldies",
u"Other",
u"Pop",
u"R&B",
u"Rap",
u"Reggae",
u"Rock",
u"Techno",
u"Industrial",
u"Alternative",
u"Ska",
u"Death Metal",
u"Pranks",
u"Soundtrack",
u"Euro-Techno",
u"Ambient",
u"Trip-Hop",
u"Vocal",
u"Jazz+Funk",
u"Fusion",
u"Trance",
u"Classical",
u"Instrumental",
u"Acid",
u"House",
u"Game",
u"Sound Clip",
u"Gospel",
u"Noise",
u"Alt. Rock",
u"Bass",
u"Soul",
u"Punk",
u"Space",
u"Meditative",
u"Instrumental Pop",
u"Instrumental Rock",
u"Ethnic",
u"Gothic",
u"Darkwave",
u"Techno-Industrial",
u"Electronic",
u"Pop-Folk",
u"Eurodance",
u"Dream",
u"Southern Rock",
u"Comedy",
u"Cult",
u"Gangsta Rap",
u"Top 40",
u"Christian Rap",
u"Pop/Funk",
u"Jungle",
u"Native American",
u"Cabaret",
u"New Wave",
u"Psychedelic",
u"Rave",
u"Showtunes",
u"Trailer",
u"Lo-Fi",
u"Tribal",
u"Acid Punk",
u"Acid Jazz",
u"Polka",
u"Retro",
u"Musical",
u"Rock & Roll",
u"Hard Rock",
u"Folk",
u"Folk-Rock",
u"National Folk",
u"Swing",
u"Fast-Fusion",
u"Bebop",
u"Latin",
u"Revival",
u"Celtic",
u"Bluegrass",
u"Avantgarde",
u"Gothic Rock",
u"Progressive Rock",
u"Psychedelic Rock",
u"Symphonic Rock",
u"Slow Rock",
u"Big Band",
u"Chorus",
u"Easy Listening",
u"Acoustic",
u"Humour",
u"Speech",
u"Chanson",
u"Opera",
u"Chamber Music",
u"Sonata",
u"Symphony",
u"Booty Bass",
u"Primus",
u"Porn Groove",
u"Satire",
u"Slow Jam",
u"Club",
u"Tango",
u"Samba",
u"Folklore",
u"Ballad",
u"Power Ballad",
u"Rhythmic Soul",
u"Freestyle",
u"Duet",
u"Punk Rock",
u"Drum Solo",
u"A Cappella",
u"Euro-House",
u"Dance Hall",
u"Goa",
u"Drum & Bass",
u"Club-House",
u"Hardcore",
u"Terror",
u"Indie",
u"BritPop",
u"Afro-Punk",
u"Polsk Punk",
u"Beat",
u"Christian Gangsta Rap",
u"Heavy Metal",
u"Black Metal",
u"Crossover",
u"Contemporary Christian",
u"Christian Rock",
u"Merengue",
u"Salsa",
u"Thrash Metal",
u"Anime",
u"JPop",
u"Synthpop",
u"Abstract",
u"Art Rock",
u"Baroque",
u"Bhangra",
u"Big Beat",
u"Breakbeat",
u"Chillout",
u"Downtempo",
u"Dub",
u"EBM",
u"Eclectic",
u"Electro",
u"Electroclash",
u"Emo",
u"Experimental",
u"Garage",
u"Global",
u"IDM",
u"Illbient",
u"Industro-Goth",
u"Jam Band",
u"Krautrock",
u"Leftfield",
u"Lounge",
u"Math Rock",
u"New Romantic",
u"Nu-Breakz",
u"Post-Punk",
u"Post-Rock",
u"Psytrance",
u"Shoegaze",
u"Space Rock",
u"Trop Rock",
u"World Music",
u"Neoclassical",
u"Audiobook",
u"Audio Theatre",
u"Neue Deutsche Welle",
u"Podcast",
u"Indie Rock",
u"G-Funk",
u"Dubstep",
u"Garage Rock",
u"Psybient",
]
"""The ID3v1 genre list."""

1842
libs/mutagen/_id3frames.py Normal file
View file

File diff suppressed because it is too large Load diff

465
libs/mutagen/_id3specs.py Normal file
View file

@ -0,0 +1,465 @@
# Copyright (C) 2005 Michael Urman
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of version 2 of the GNU General Public License as
# published by the Free Software Foundation.
import struct
from struct import unpack, pack
from warnings import warn
from mutagen._id3util import ID3JunkFrameError, ID3Warning, BitPaddedInt
class Spec(object):
def __init__(self, name):
self.name = name
def __hash__(self):
raise TypeError("Spec objects are unhashable")
def _validate23(self, frame, value, **kwargs):
"""Return a possibly modified value which, if written,
results in valid id3v2.3 data.
"""
return value
class ByteSpec(Spec):
def read(self, frame, data):
return ord(data[0]), data[1:]
def write(self, frame, value):
return chr(value)
def validate(self, frame, value):
if value is not None:
chr(value)
return value
class IntegerSpec(Spec):
def read(self, frame, data):
return int(BitPaddedInt(data, bits=8)), ''
def write(self, frame, value):
return BitPaddedInt.to_str(value, bits=8, width=-1)
def validate(self, frame, value):
return value
class SizedIntegerSpec(Spec):
def __init__(self, name, size):
self.name, self.__sz = name, size
def read(self, frame, data):
return int(BitPaddedInt(data[:self.__sz], bits=8)), data[self.__sz:]
def write(self, frame, value):
return BitPaddedInt.to_str(value, bits=8, width=self.__sz)
def validate(self, frame, value):
return value
class EncodingSpec(ByteSpec):
def read(self, frame, data):
enc, data = super(EncodingSpec, self).read(frame, data)
if enc < 16:
return enc, data
else:
return 0, chr(enc)+data
def validate(self, frame, value):
if 0 <= value <= 3:
return value
if value is None:
return None
raise ValueError('Invalid Encoding: %r' % value)
def _validate23(self, frame, value, **kwargs):
# only 0, 1 are valid in v2.3, default to utf-16
return min(1, value)
class StringSpec(Spec):
def __init__(self, name, length):
super(StringSpec, self).__init__(name)
self.len = length
def read(s, frame, data):
return data[:s.len], data[s.len:]
def write(s, frame, value):
if value is None:
return '\x00' * s.len
else:
return (str(value) + '\x00' * s.len)[:s.len]
def validate(s, frame, value):
if value is None:
return None
if isinstance(value, basestring) and len(value) == s.len:
return value
raise ValueError('Invalid StringSpec[%d] data: %r' % (s.len, value))
class BinaryDataSpec(Spec):
def read(self, frame, data):
return data, ''
def write(self, frame, value):
return str(value)
def validate(self, frame, value):
return str(value)
class EncodedTextSpec(Spec):
# Okay, seriously. This is private and defined explicitly and
# completely by the ID3 specification. You can't just add
# encodings here however you want.
_encodings = (
('latin1', '\x00'),
('utf16', '\x00\x00'),
('utf_16_be', '\x00\x00'),
('utf8', '\x00')
)
def read(self, frame, data):
enc, term = self._encodings[frame.encoding]
ret = ''
if len(term) == 1:
if term in data:
data, ret = data.split(term, 1)
else:
offset = -1
try:
while True:
offset = data.index(term, offset+1)
if offset & 1:
continue
data, ret = data[0:offset], data[offset+2:]
break
except ValueError:
pass
if len(data) < len(term):
return u'', ret
return data.decode(enc), ret
def write(self, frame, value):
enc, term = self._encodings[frame.encoding]
return value.encode(enc) + term
def validate(self, frame, value):
return unicode(value)
class MultiSpec(Spec):
def __init__(self, name, *specs, **kw):
super(MultiSpec, self).__init__(name)
self.specs = specs
self.sep = kw.get('sep')
def read(self, frame, data):
values = []
while data:
record = []
for spec in self.specs:
value, data = spec.read(frame, data)
record.append(value)
if len(self.specs) != 1:
values.append(record)
else:
values.append(record[0])
return values, data
def write(self, frame, value):
data = []
if len(self.specs) == 1:
for v in value:
data.append(self.specs[0].write(frame, v))
else:
for record in value:
for v, s in zip(record, self.specs):
data.append(s.write(frame, v))
return ''.join(data)
def validate(self, frame, value):
if value is None:
return []
if self.sep and isinstance(value, basestring):
value = value.split(self.sep)
if isinstance(value, list):
if len(self.specs) == 1:
return [self.specs[0].validate(frame, v) for v in value]
else:
return [
[s.validate(frame, v) for (v, s) in zip(val, self.specs)]
for val in value]
raise ValueError('Invalid MultiSpec data: %r' % value)
def _validate23(self, frame, value, **kwargs):
if len(self.specs) != 1:
return [[s._validate23(frame, v, **kwargs)
for (v, s) in zip(val, self.specs)]
for val in value]
spec = self.specs[0]
# Merge single text spec multispecs only.
# (TimeStampSpec beeing the exception, but it's not a valid v2.3 frame)
if not isinstance(spec, EncodedTextSpec) or \
isinstance(spec, TimeStampSpec):
return value
value = [spec._validate23(frame, v, **kwargs) for v in value]
if kwargs.get("sep") is not None:
return [spec.validate(frame, kwargs["sep"].join(value))]
return value
class EncodedNumericTextSpec(EncodedTextSpec):
pass
class EncodedNumericPartTextSpec(EncodedTextSpec):
pass
class Latin1TextSpec(EncodedTextSpec):
def read(self, frame, data):
if '\x00' in data:
data, ret = data.split('\x00', 1)
else:
ret = ''
return data.decode('latin1'), ret
def write(self, data, value):
return value.encode('latin1') + '\x00'
def validate(self, frame, value):
return unicode(value)
class ID3TimeStamp(object):
"""A time stamp in ID3v2 format.
This is a restricted form of the ISO 8601 standard; time stamps
take the form of:
YYYY-MM-DD HH:MM:SS
Or some partial form (YYYY-MM-DD HH, YYYY, etc.).
The 'text' attribute contains the raw text data of the time stamp.
"""
import re
def __init__(self, text):
if isinstance(text, ID3TimeStamp):
text = text.text
self.text = text
__formats = ['%04d'] + ['%02d'] * 5
__seps = ['-', '-', ' ', ':', ':', 'x']
def get_text(self):
parts = [self.year, self.month, self.day,
self.hour, self.minute, self.second]
pieces = []
for i, part in enumerate(iter(iter(parts).next, None)):
pieces.append(self.__formats[i] % part + self.__seps[i])
return u''.join(pieces)[:-1]
def set_text(self, text, splitre=re.compile('[-T:/.]|\s+')):
year, month, day, hour, minute, second = \
splitre.split(text + ':::::')[:6]
for a in 'year month day hour minute second'.split():
try:
v = int(locals()[a])
except ValueError:
v = None
setattr(self, a, v)
text = property(get_text, set_text, doc="ID3v2.4 date and time.")
def __str__(self):
return self.text
def __repr__(self):
return repr(self.text)
def __cmp__(self, other):
return cmp(self.text, other.text)
__hash__ = object.__hash__
def encode(self, *args):
return self.text.encode(*args)
class TimeStampSpec(EncodedTextSpec):
def read(self, frame, data):
value, data = super(TimeStampSpec, self).read(frame, data)
return self.validate(frame, value), data
def write(self, frame, data):
return super(TimeStampSpec, self).write(frame,
data.text.replace(' ', 'T'))
def validate(self, frame, value):
try:
return ID3TimeStamp(value)
except TypeError:
raise ValueError("Invalid ID3TimeStamp: %r" % value)
class ChannelSpec(ByteSpec):
(OTHER, MASTER, FRONTRIGHT, FRONTLEFT, BACKRIGHT, BACKLEFT, FRONTCENTRE,
BACKCENTRE, SUBWOOFER) = range(9)
class VolumeAdjustmentSpec(Spec):
def read(self, frame, data):
value, = unpack('>h', data[0:2])
return value/512.0, data[2:]
def write(self, frame, value):
return pack('>h', int(round(value * 512)))
def validate(self, frame, value):
if value is not None:
try:
self.write(frame, value)
except struct.error:
raise ValueError("out of range")
return value
class VolumePeakSpec(Spec):
def read(self, frame, data):
# http://bugs.xmms.org/attachment.cgi?id=113&action=view
peak = 0
bits = ord(data[0])
bytes = min(4, (bits + 7) >> 3)
# not enough frame data
if bytes + 1 > len(data):
raise ID3JunkFrameError
shift = ((8 - (bits & 7)) & 7) + (4 - bytes) * 8
for i in range(1, bytes+1):
peak *= 256
peak += ord(data[i])
peak *= 2 ** shift
return (float(peak) / (2**31-1)), data[1+bytes:]
def write(self, frame, value):
# always write as 16 bits for sanity.
return "\x10" + pack('>H', int(round(value * 32768)))
def validate(self, frame, value):
if value is not None:
try:
self.write(frame, value)
except struct.error:
raise ValueError("out of range")
return value
class SynchronizedTextSpec(EncodedTextSpec):
def read(self, frame, data):
texts = []
encoding, term = self._encodings[frame.encoding]
while data:
l = len(term)
try:
value_idx = data.index(term)
except ValueError:
raise ID3JunkFrameError
value = data[:value_idx].decode(encoding)
if len(data) < value_idx + l + 4:
raise ID3JunkFrameError
time, = struct.unpack(">I", data[value_idx+l:value_idx+l+4])
texts.append((value, time))
data = data[value_idx+l+4:]
return texts, ""
def write(self, frame, value):
data = []
encoding, term = self._encodings[frame.encoding]
for text, time in frame.text:
text = text.encode(encoding) + term
data.append(text + struct.pack(">I", time))
return "".join(data)
def validate(self, frame, value):
return value
class KeyEventSpec(Spec):
def read(self, frame, data):
events = []
while len(data) >= 5:
events.append(struct.unpack(">bI", data[:5]))
data = data[5:]
return events, data
def write(self, frame, value):
return "".join([struct.pack(">bI", *event) for event in value])
def validate(self, frame, value):
return value
class VolumeAdjustmentsSpec(Spec):
# Not to be confused with VolumeAdjustmentSpec.
def read(self, frame, data):
adjustments = {}
while len(data) >= 4:
freq, adj = struct.unpack(">Hh", data[:4])
data = data[4:]
freq /= 2.0
adj /= 512.0
adjustments[freq] = adj
adjustments = adjustments.items()
adjustments.sort()
return adjustments, data
def write(self, frame, value):
value.sort()
return "".join([struct.pack(">Hh", int(freq * 2), int(adj * 512))
for (freq, adj) in value])
def validate(self, frame, value):
return value
class ASPIIndexSpec(Spec):
def read(self, frame, data):
if frame.b == 16:
format = "H"
size = 2
elif frame.b == 8:
format = "B"
size = 1
else:
warn("invalid bit count in ASPI (%d)" % frame.b, ID3Warning)
return [], data
indexes = data[:frame.N * size]
data = data[frame.N * size:]
return list(struct.unpack(">" + format * frame.N, indexes)), data
def write(self, frame, values):
if frame.b == 16:
format = "H"
elif frame.b == 8:
format = "B"
else:
raise ValueError("frame.b must be 8 or 16")
return struct.pack(">" + format * frame.N, *values)
def validate(self, frame, values):
return values

176
libs/mutagen/_id3util.py Normal file
View file

@ -0,0 +1,176 @@
# Copyright (C) 2005 Michael Urman
# 2013 Christoph Reiter
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of version 2 of the GNU General Public License as
# published by the Free Software Foundation.
class error(Exception):
pass
class ID3NoHeaderError(error, ValueError):
pass
class ID3BadUnsynchData(error, ValueError):
pass
class ID3BadCompressedData(error, ValueError):
pass
class ID3TagError(error, ValueError):
pass
class ID3UnsupportedVersionError(error, NotImplementedError):
pass
class ID3EncryptionUnsupportedError(error, NotImplementedError):
pass
class ID3JunkFrameError(error, ValueError):
pass
class ID3Warning(error, UserWarning):
pass
class unsynch(object):
@staticmethod
def decode(value):
output = []
safe = True
append = output.append
for val in value:
if safe:
append(val)
safe = val != '\xFF'
else:
if val >= '\xE0':
raise ValueError('invalid sync-safe string')
elif val != '\x00':
append(val)
safe = True
if not safe:
raise ValueError('string ended unsafe')
return ''.join(output)
@staticmethod
def encode(value):
output = []
safe = True
append = output.append
for val in value:
if safe:
append(val)
if val == '\xFF':
safe = False
elif val == '\x00' or val >= '\xE0':
append('\x00')
append(val)
safe = val != '\xFF'
else:
append(val)
safe = True
if not safe:
append('\x00')
return ''.join(output)
class _BitPaddedMixin(object):
def as_str(self, width=4, minwidth=4):
return self.to_str(self, self.bits, self.bigendian, width, minwidth)
@staticmethod
def to_str(value, bits=7, bigendian=True, width=4, minwidth=4):
mask = (1 << bits) - 1
if width != -1:
index = 0
bytes_ = bytearray(width)
try:
while value:
bytes_[index] = value & mask
value >>= bits
index += 1
except IndexError:
raise ValueError('Value too wide (>%d bytes)' % width)
else:
# PCNT and POPM use growing integers
# of at least 4 bytes (=minwidth) as counters.
bytes_ = bytearray()
append = bytes_.append
while value:
append(value & mask)
value >>= bits
bytes_ = bytes_.ljust(minwidth, "\x00")
if bigendian:
bytes_.reverse()
return str(bytes_)
@staticmethod
def has_valid_padding(value, bits=7):
"""Whether the padding bits are all zero"""
assert bits <= 8
mask = (((1 << (8 - bits)) - 1) << bits)
if isinstance(value, (int, long)):
while value:
if value & mask:
return False
value >>= 8
elif isinstance(value, str):
for byte in value:
if ord(byte) & mask:
return False
else:
raise TypeError
return True
class BitPaddedInt(int, _BitPaddedMixin):
def __new__(cls, value, bits=7, bigendian=True):
mask = (1 << (bits)) - 1
numeric_value = 0
shift = 0
if isinstance(value, (int, long)):
while value:
numeric_value += (value & mask) << shift
value >>= 8
shift += bits
elif isinstance(value, str):
if bigendian:
value = reversed(value)
for byte in value:
numeric_value += (ord(byte) & mask) << shift
shift += bits
else:
raise TypeError
if isinstance(numeric_value, long):
self = long.__new__(BitPaddedLong, numeric_value)
else:
self = int.__new__(BitPaddedInt, numeric_value)
self.bits = bits
self.bigendian = bigendian
return self
class BitPaddedLong(long, _BitPaddedMixin):
pass

349
libs/mutagen/_util.py Normal file
View file

@ -0,0 +1,349 @@
# Copyright 2006 Joe Wreschnig
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Utility classes for Mutagen.
You should not rely on the interfaces here being stable. They are
intended for internal use in Mutagen only.
"""
import struct
from fnmatch import fnmatchcase
class DictMixin(object):
"""Implement the dict API using keys() and __*item__ methods.
Similar to UserDict.DictMixin, this takes a class that defines
__getitem__, __setitem__, __delitem__, and keys(), and turns it
into a full dict-like object.
UserDict.DictMixin is not suitable for this purpose because it's
an old-style class.
This class is not optimized for very large dictionaries; many
functions have linear memory requirements. I recommend you
override some of these functions if speed is required.
"""
def __iter__(self):
return iter(self.keys())
def has_key(self, key):
try:
self[key]
except KeyError:
return False
else:
return True
__contains__ = has_key
iterkeys = lambda self: iter(self.keys())
def values(self):
return map(self.__getitem__, self.keys())
itervalues = lambda self: iter(self.values())
def items(self):
return zip(self.keys(), self.values())
iteritems = lambda s: iter(s.items())
def clear(self):
map(self.__delitem__, self.keys())
def pop(self, key, *args):
if len(args) > 1:
raise TypeError("pop takes at most two arguments")
try:
value = self[key]
except KeyError:
if args:
return args[0]
else:
raise
del(self[key])
return value
def popitem(self):
try:
key = self.keys()[0]
return key, self.pop(key)
except IndexError:
raise KeyError("dictionary is empty")
def update(self, other=None, **kwargs):
if other is None:
self.update(kwargs)
other = {}
try:
map(self.__setitem__, other.keys(), other.values())
except AttributeError:
for key, value in other:
self[key] = value
def setdefault(self, key, default=None):
try:
return self[key]
except KeyError:
self[key] = default
return default
def get(self, key, default=None):
try:
return self[key]
except KeyError:
return default
def __repr__(self):
return repr(dict(self.items()))
def __cmp__(self, other):
if other is None:
return 1
else:
return cmp(dict(self.items()), other)
__hash__ = object.__hash__
def __len__(self):
return len(self.keys())
class DictProxy(DictMixin):
def __init__(self, *args, **kwargs):
self.__dict = {}
super(DictProxy, self).__init__(*args, **kwargs)
def __getitem__(self, key):
return self.__dict[key]
def __setitem__(self, key, value):
self.__dict[key] = value
def __delitem__(self, key):
del(self.__dict[key])
def keys(self):
return self.__dict.keys()
class cdata(object):
"""C character buffer to Python numeric type conversions."""
from struct import error
error = error
short_le = staticmethod(lambda data: struct.unpack('<h', data)[0])
ushort_le = staticmethod(lambda data: struct.unpack('<H', data)[0])
short_be = staticmethod(lambda data: struct.unpack('>h', data)[0])
ushort_be = staticmethod(lambda data: struct.unpack('>H', data)[0])
int_le = staticmethod(lambda data: struct.unpack('<i', data)[0])
uint_le = staticmethod(lambda data: struct.unpack('<I', data)[0])
int_be = staticmethod(lambda data: struct.unpack('>i', data)[0])
uint_be = staticmethod(lambda data: struct.unpack('>I', data)[0])
longlong_le = staticmethod(lambda data: struct.unpack('<q', data)[0])
ulonglong_le = staticmethod(lambda data: struct.unpack('<Q', data)[0])
longlong_be = staticmethod(lambda data: struct.unpack('>q', data)[0])
ulonglong_be = staticmethod(lambda data: struct.unpack('>Q', data)[0])
to_short_le = staticmethod(lambda data: struct.pack('<h', data))
to_ushort_le = staticmethod(lambda data: struct.pack('<H', data))
to_short_be = staticmethod(lambda data: struct.pack('>h', data))
to_ushort_be = staticmethod(lambda data: struct.pack('>H', data))
to_int_le = staticmethod(lambda data: struct.pack('<i', data))
to_uint_le = staticmethod(lambda data: struct.pack('<I', data))
to_int_be = staticmethod(lambda data: struct.pack('>i', data))
to_uint_be = staticmethod(lambda data: struct.pack('>I', data))
to_longlong_le = staticmethod(lambda data: struct.pack('<q', data))
to_ulonglong_le = staticmethod(lambda data: struct.pack('<Q', data))
to_longlong_be = staticmethod(lambda data: struct.pack('>q', data))
to_ulonglong_be = staticmethod(lambda data: struct.pack('>Q', data))
bitswap = ''.join([chr(sum([((val >> i) & 1) << (7-i) for i in range(8)]))
for val in range(256)])
del(i)
del(val)
test_bit = staticmethod(lambda value, n: bool((value >> n) & 1))
def lock(fileobj):
"""Lock a file object 'safely'.
That means a failure to lock because the platform doesn't
support fcntl or filesystem locks is not considered a
failure. This call does block.
Returns whether or not the lock was successful, or
raises an exception in more extreme circumstances (full
lock table, invalid file).
"""
try:
import fcntl
except ImportError:
return False
else:
try:
fcntl.lockf(fileobj, fcntl.LOCK_EX)
except IOError:
# FIXME: There's possibly a lot of complicated
# logic that needs to go here in case the IOError
# is EACCES or EAGAIN.
return False
else:
return True
def unlock(fileobj):
"""Unlock a file object.
Don't call this on a file object unless a call to lock()
returned true.
"""
# If this fails there's a mismatched lock/unlock pair,
# so we definitely don't want to ignore errors.
import fcntl
fcntl.lockf(fileobj, fcntl.LOCK_UN)
def insert_bytes(fobj, size, offset, BUFFER_SIZE=2**16):
"""Insert size bytes of empty space starting at offset.
fobj must be an open file object, open rb+ or
equivalent. Mutagen tries to use mmap to resize the file, but
falls back to a significantly slower method if mmap fails.
"""
assert 0 < size
assert 0 <= offset
locked = False
fobj.seek(0, 2)
filesize = fobj.tell()
movesize = filesize - offset
fobj.write('\x00' * size)
fobj.flush()
try:
try:
import mmap
map = mmap.mmap(fobj.fileno(), filesize + size)
try:
map.move(offset + size, offset, movesize)
finally:
map.close()
except (ValueError, EnvironmentError, ImportError):
# handle broken mmap scenarios
locked = lock(fobj)
fobj.truncate(filesize)
fobj.seek(0, 2)
padsize = size
# Don't generate an enormous string if we need to pad
# the file out several megs.
while padsize:
addsize = min(BUFFER_SIZE, padsize)
fobj.write("\x00" * addsize)
padsize -= addsize
fobj.seek(filesize, 0)
while movesize:
# At the start of this loop, fobj is pointing at the end
# of the data we need to move, which is of movesize length.
thismove = min(BUFFER_SIZE, movesize)
# Seek back however much we're going to read this frame.
fobj.seek(-thismove, 1)
nextpos = fobj.tell()
# Read it, so we're back at the end.
data = fobj.read(thismove)
# Seek back to where we need to write it.
fobj.seek(-thismove + size, 1)
# Write it.
fobj.write(data)
# And seek back to the end of the unmoved data.
fobj.seek(nextpos)
movesize -= thismove
fobj.flush()
finally:
if locked:
unlock(fobj)
def delete_bytes(fobj, size, offset, BUFFER_SIZE=2**16):
"""Delete size bytes of empty space starting at offset.
fobj must be an open file object, open rb+ or
equivalent. Mutagen tries to use mmap to resize the file, but
falls back to a significantly slower method if mmap fails.
"""
locked = False
assert 0 < size
assert 0 <= offset
fobj.seek(0, 2)
filesize = fobj.tell()
movesize = filesize - offset - size
assert 0 <= movesize
try:
if movesize > 0:
fobj.flush()
try:
import mmap
map = mmap.mmap(fobj.fileno(), filesize)
try:
map.move(offset, offset + size, movesize)
finally:
map.close()
except (ValueError, EnvironmentError, ImportError):
# handle broken mmap scenarios
locked = lock(fobj)
fobj.seek(offset + size)
buf = fobj.read(BUFFER_SIZE)
while buf:
fobj.seek(offset)
fobj.write(buf)
offset += len(buf)
fobj.seek(offset + size)
buf = fobj.read(BUFFER_SIZE)
fobj.truncate(filesize - size)
fobj.flush()
finally:
if locked:
unlock(fobj)
def utf8(data):
"""Convert a basestring to a valid UTF-8 str."""
if isinstance(data, str):
return data.decode("utf-8", "replace").encode("utf-8")
elif isinstance(data, unicode):
return data.encode("utf-8")
else:
raise TypeError("only unicode/str types can be converted to UTF-8")
def dict_match(d, key, default=None):
try:
return d[key]
except KeyError:
for pattern, value in d.iteritems():
if fnmatchcase(key, pattern):
return value
return default

254
libs/mutagen/_vorbis.py Normal file
View file

@ -0,0 +1,254 @@
# Vorbis comment support for Mutagen
# Copyright 2005-2006 Joe Wreschnig
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of version 2 of the GNU General Public License as
# published by the Free Software Foundation.
"""Read and write Vorbis comment data.
Vorbis comments are freeform key/value pairs; keys are
case-insensitive ASCII and values are Unicode strings. A key may have
multiple values.
The specification is at http://www.xiph.org/vorbis/doc/v-comment.html.
"""
import sys
from cStringIO import StringIO
import mutagen
from mutagen._util import DictMixin, cdata
def is_valid_key(key):
"""Return true if a string is a valid Vorbis comment key.
Valid Vorbis comment keys are printable ASCII between 0x20 (space)
and 0x7D ('}'), excluding '='.
"""
for c in key:
if c < " " or c > "}" or c == "=":
return False
else:
return bool(key)
istag = is_valid_key
class error(IOError):
pass
class VorbisUnsetFrameError(error):
pass
class VorbisEncodingError(error):
pass
class VComment(mutagen.Metadata, list):
"""A Vorbis comment parser, accessor, and renderer.
All comment ordering is preserved. A VComment is a list of
key/value pairs, and so any Python list method can be used on it.
Vorbis comments are always wrapped in something like an Ogg Vorbis
bitstream or a FLAC metadata block, so this loads string data or a
file-like object, not a filename.
Attributes:
vendor -- the stream 'vendor' (i.e. writer); default 'Mutagen'
"""
vendor = u"Mutagen " + mutagen.version_string
def __init__(self, data=None, *args, **kwargs):
# Collect the args to pass to load, this lets child classes
# override just load and get equivalent magic for the
# constructor.
if data is not None:
if isinstance(data, str):
data = StringIO(data)
elif not hasattr(data, 'read'):
raise TypeError("VComment requires string data or a file-like")
self.load(data, *args, **kwargs)
def load(self, fileobj, errors='replace', framing=True):
"""Parse a Vorbis comment from a file-like object.
Keyword arguments:
errors:
'strict', 'replace', or 'ignore'. This affects Unicode decoding
and how other malformed content is interpreted.
framing -- if true, fail if a framing bit is not present
Framing bits are required by the Vorbis comment specification,
but are not used in FLAC Vorbis comment blocks.
"""
try:
vendor_length = cdata.uint_le(fileobj.read(4))
self.vendor = fileobj.read(vendor_length).decode('utf-8', errors)
count = cdata.uint_le(fileobj.read(4))
for i in xrange(count):
length = cdata.uint_le(fileobj.read(4))
try:
string = fileobj.read(length).decode('utf-8', errors)
except (OverflowError, MemoryError):
raise error("cannot read %d bytes, too large" % length)
try:
tag, value = string.split('=', 1)
except ValueError, err:
if errors == "ignore":
continue
elif errors == "replace":
tag, value = u"unknown%d" % i, string
else:
raise VorbisEncodingError, err, sys.exc_info()[2]
try:
tag = tag.encode('ascii', errors)
except UnicodeEncodeError:
raise VorbisEncodingError("invalid tag name %r" % tag)
else:
if is_valid_key(tag):
self.append((tag, value))
if framing and not ord(fileobj.read(1)) & 0x01:
raise VorbisUnsetFrameError("framing bit was unset")
except (cdata.error, TypeError):
raise error("file is not a valid Vorbis comment")
def validate(self):
"""Validate keys and values.
Check to make sure every key used is a valid Vorbis key, and
that every value used is a valid Unicode or UTF-8 string. If
any invalid keys or values are found, a ValueError is raised.
"""
if not isinstance(self.vendor, unicode):
try:
self.vendor.decode('utf-8')
except UnicodeDecodeError:
raise ValueError
for key, value in self:
try:
if not is_valid_key(key):
raise ValueError
except:
raise ValueError("%r is not a valid key" % key)
if not isinstance(value, unicode):
try:
value.encode("utf-8")
except:
raise ValueError("%r is not a valid value" % value)
else:
return True
def clear(self):
"""Clear all keys from the comment."""
del(self[:])
def write(self, framing=True):
"""Return a string representation of the data.
Validation is always performed, so calling this function on
invalid data may raise a ValueError.
Keyword arguments:
framing -- if true, append a framing bit (see load)
"""
self.validate()
f = StringIO()
f.write(cdata.to_uint_le(len(self.vendor.encode('utf-8'))))
f.write(self.vendor.encode('utf-8'))
f.write(cdata.to_uint_le(len(self)))
for tag, value in self:
comment = "%s=%s" % (tag, value.encode('utf-8'))
f.write(cdata.to_uint_le(len(comment)))
f.write(comment)
if framing:
f.write("\x01")
return f.getvalue()
def pprint(self):
return "\n".join(["%s=%s" % (k.lower(), v) for k, v in self])
class VCommentDict(VComment, DictMixin):
"""A VComment that looks like a dictionary.
This object differs from a dictionary in two ways. First,
len(comment) will still return the number of values, not the
number of keys. Secondly, iterating through the object will
iterate over (key, value) pairs, not keys. Since a key may have
multiple values, the same value may appear multiple times while
iterating.
Since Vorbis comment keys are case-insensitive, all keys are
normalized to lowercase ASCII.
"""
def __getitem__(self, key):
"""A list of values for the key.
This is a copy, so comment['title'].append('a title') will not
work.
"""
key = key.lower().encode('ascii')
values = [value for (k, value) in self if k.lower() == key]
if not values:
raise KeyError(key)
else:
return values
def __delitem__(self, key):
"""Delete all values associated with the key."""
key = key.lower().encode('ascii')
to_delete = filter(lambda x: x[0].lower() == key, self)
if not to_delete:
raise KeyError(key)
else:
map(self.remove, to_delete)
def __contains__(self, key):
"""Return true if the key has any values."""
key = key.lower().encode('ascii')
for k, value in self:
if k.lower() == key:
return True
else:
return False
def __setitem__(self, key, values):
"""Set a key's value or values.
Setting a value overwrites all old ones. The value may be a
list of Unicode or UTF-8 strings, or a single Unicode or UTF-8
string.
"""
key = key.encode('ascii')
if not isinstance(values, list):
values = [values]
try:
del(self[key])
except KeyError:
pass
for value in values:
self.append((key, value))
def keys(self):
"""Return all keys in the comment."""
return self and list(set([k.lower() for k, v in self]))
def as_dict(self):
"""Return a copy of the comment data in a real dict."""
return dict([(key, self[key]) for key in self.keys()])

525
libs/mutagen/apev2.py Normal file
View file

@ -0,0 +1,525 @@
# An APEv2 tag reader
#
# Copyright 2005 Joe Wreschnig
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""APEv2 reading and writing.
The APEv2 format is most commonly used with Musepack files, but is
also the format of choice for WavPack and other formats. Some MP3s
also have APEv2 tags, but this can cause problems with many MP3
decoders and taggers.
APEv2 tags, like Vorbis comments, are freeform key=value pairs. APEv2
keys can be any ASCII string with characters from 0x20 to 0x7E,
between 2 and 255 characters long. Keys are case-sensitive, but
readers are recommended to be case insensitive, and it is forbidden to
multiple keys which differ only in case. Keys are usually stored
title-cased (e.g. 'Artist' rather than 'artist').
APEv2 values are slightly more structured than Vorbis comments; values
are flagged as one of text, binary, or an external reference (usually
a URI).
Based off the format specification found at
http://wiki.hydrogenaudio.org/index.php?title=APEv2_specification.
"""
__all__ = ["APEv2", "APEv2File", "Open", "delete"]
import struct
from cStringIO import StringIO
from mutagen import Metadata, FileType
from mutagen._util import DictMixin, cdata, utf8, delete_bytes
def is_valid_apev2_key(key):
return (2 <= len(key) <= 255 and min(key) >= ' ' and max(key) <= '~' and
key not in ["OggS", "TAG", "ID3", "MP+"])
# There are three different kinds of APE tag values.
# "0: Item contains text information coded in UTF-8
# 1: Item contains binary information
# 2: Item is a locator of external stored information [e.g. URL]
# 3: reserved"
TEXT, BINARY, EXTERNAL = range(3)
HAS_HEADER = 1L << 31
HAS_NO_FOOTER = 1L << 30
IS_HEADER = 1L << 29
class error(IOError):
pass
class APENoHeaderError(error, ValueError):
pass
class APEUnsupportedVersionError(error, ValueError):
pass
class APEBadItemError(error, ValueError):
pass
class _APEv2Data(object):
# Store offsets of the important parts of the file.
start = header = data = footer = end = None
# Footer or header; seek here and read 32 to get version/size/items/flags
metadata = None
# Actual tag data
tag = None
version = None
size = None
items = None
flags = 0
# The tag is at the start rather than the end. A tag at both
# the start and end of the file (i.e. the tag is the whole file)
# is not considered to be at the start.
is_at_start = False
def __init__(self, fileobj):
self.__find_metadata(fileobj)
self.metadata = max(self.header, self.footer)
if self.metadata is None:
return
self.__fill_missing(fileobj)
self.__fix_brokenness(fileobj)
if self.data is not None:
fileobj.seek(self.data)
self.tag = fileobj.read(self.size)
def __find_metadata(self, fileobj):
# Try to find a header or footer.
# Check for a simple footer.
try:
fileobj.seek(-32, 2)
except IOError:
fileobj.seek(0, 2)
return
if fileobj.read(8) == "APETAGEX":
fileobj.seek(-8, 1)
self.footer = self.metadata = fileobj.tell()
return
# Check for an APEv2 tag followed by an ID3v1 tag at the end.
try:
fileobj.seek(-128, 2)
if fileobj.read(3) == "TAG":
fileobj.seek(-35, 1) # "TAG" + header length
if fileobj.read(8) == "APETAGEX":
fileobj.seek(-8, 1)
self.footer = fileobj.tell()
return
# ID3v1 tag at the end, maybe preceded by Lyrics3v2.
# (http://www.id3.org/lyrics3200.html)
# (header length - "APETAGEX") - "LYRICS200"
fileobj.seek(15, 1)
if fileobj.read(9) == 'LYRICS200':
fileobj.seek(-15, 1) # "LYRICS200" + size tag
try:
offset = int(fileobj.read(6))
except ValueError:
raise IOError
fileobj.seek(-32 - offset - 6, 1)
if fileobj.read(8) == "APETAGEX":
fileobj.seek(-8, 1)
self.footer = fileobj.tell()
return
except IOError:
pass
# Check for a tag at the start.
fileobj.seek(0, 0)
if fileobj.read(8) == "APETAGEX":
self.is_at_start = True
self.header = 0
def __fill_missing(self, fileobj):
fileobj.seek(self.metadata + 8)
self.version = fileobj.read(4)
self.size = cdata.uint_le(fileobj.read(4))
self.items = cdata.uint_le(fileobj.read(4))
self.flags = cdata.uint_le(fileobj.read(4))
if self.header is not None:
self.data = self.header + 32
# If we're reading the header, the size is the header
# offset + the size, which includes the footer.
self.end = self.data + self.size
fileobj.seek(self.end - 32, 0)
if fileobj.read(8) == "APETAGEX":
self.footer = self.end - 32
elif self.footer is not None:
self.end = self.footer + 32
self.data = self.end - self.size
if self.flags & HAS_HEADER:
self.header = self.data - 32
else:
self.header = self.data
else:
raise APENoHeaderError("No APE tag found")
# exclude the footer from size
if self.footer is not None:
self.size -= 32
def __fix_brokenness(self, fileobj):
# Fix broken tags written with PyMusepack.
if self.header is not None:
start = self.header
else:
start = self.data
fileobj.seek(start)
while start > 0:
# Clean up broken writing from pre-Mutagen PyMusepack.
# It didn't remove the first 24 bytes of header.
try:
fileobj.seek(-24, 1)
except IOError:
break
else:
if fileobj.read(8) == "APETAGEX":
fileobj.seek(-8, 1)
start = fileobj.tell()
else:
break
self.start = start
class APEv2(DictMixin, Metadata):
"""A file with an APEv2 tag.
ID3v1 tags are silently ignored and overwritten.
"""
filename = None
def __init__(self, *args, **kwargs):
self.__casemap = {}
self.__dict = {}
super(APEv2, self).__init__(*args, **kwargs)
# Internally all names are stored as lowercase, but the case
# they were set with is remembered and used when saving. This
# is roughly in line with the standard, which says that keys
# are case-sensitive but two keys differing only in case are
# not allowed, and recommends case-insensitive
# implementations.
def pprint(self):
"""Return tag key=value pairs in a human-readable format."""
items = self.items()
items.sort()
return "\n".join(["%s=%s" % (k, v.pprint()) for k, v in items])
def load(self, filename):
"""Load tags from a filename."""
self.filename = filename
fileobj = open(filename, "rb")
try:
data = _APEv2Data(fileobj)
finally:
fileobj.close()
if data.tag:
self.clear()
self.__casemap.clear()
self.__parse_tag(data.tag, data.items)
else:
raise APENoHeaderError("No APE tag found")
def __parse_tag(self, tag, count):
fileobj = StringIO(tag)
for i in range(count):
size_data = fileobj.read(4)
# someone writes wrong item counts
if not size_data:
break
size = cdata.uint_le(size_data)
flags = cdata.uint_le(fileobj.read(4))
# Bits 1 and 2 bits are flags, 0-3
# Bit 0 is read/write flag, ignored
kind = (flags & 6) >> 1
if kind == 3:
raise APEBadItemError("value type must be 0, 1, or 2")
key = value = fileobj.read(1)
while key[-1:] != '\x00' and value:
value = fileobj.read(1)
key += value
if key[-1:] == "\x00":
key = key[:-1]
value = fileobj.read(size)
self[key] = APEValue(value, kind)
def __getitem__(self, key):
if not is_valid_apev2_key(key):
raise KeyError("%r is not a valid APEv2 key" % key)
key = key.encode('ascii')
return self.__dict[key.lower()]
def __delitem__(self, key):
if not is_valid_apev2_key(key):
raise KeyError("%r is not a valid APEv2 key" % key)
key = key.encode('ascii')
del(self.__dict[key.lower()])
def __setitem__(self, key, value):
"""'Magic' value setter.
This function tries to guess at what kind of value you want to
store. If you pass in a valid UTF-8 or Unicode string, it
treats it as a text value. If you pass in a list, it treats it
as a list of string/Unicode values. If you pass in a string
that is not valid UTF-8, it assumes it is a binary value.
If you need to force a specific type of value (e.g. binary
data that also happens to be valid UTF-8, or an external
reference), use the APEValue factory and set the value to the
result of that::
from mutagen.apev2 import APEValue, EXTERNAL
tag['Website'] = APEValue('http://example.org', EXTERNAL)
"""
if not is_valid_apev2_key(key):
raise KeyError("%r is not a valid APEv2 key" % key)
key = key.encode('ascii')
if not isinstance(value, _APEValue):
# let's guess at the content if we're not already a value...
if isinstance(value, unicode):
# unicode? we've got to be text.
value = APEValue(utf8(value), TEXT)
elif isinstance(value, list):
# list? text.
value = APEValue("\0".join(map(utf8, value)), TEXT)
else:
try:
value.decode("utf-8")
except UnicodeError:
# invalid UTF8 text, probably binary
value = APEValue(value, BINARY)
else:
# valid UTF8, probably text
value = APEValue(value, TEXT)
self.__casemap[key.lower()] = key
self.__dict[key.lower()] = value
def keys(self):
return [self.__casemap.get(key, key) for key in self.__dict.keys()]
def save(self, filename=None):
"""Save changes to a file.
If no filename is given, the one most recently loaded is used.
Tags are always written at the end of the file, and include
a header and a footer.
"""
filename = filename or self.filename
try:
fileobj = open(filename, "r+b")
except IOError:
fileobj = open(filename, "w+b")
data = _APEv2Data(fileobj)
if data.is_at_start:
delete_bytes(fileobj, data.end - data.start, data.start)
elif data.start is not None:
fileobj.seek(data.start)
# Delete an ID3v1 tag if present, too.
fileobj.truncate()
fileobj.seek(0, 2)
# "APE tags items should be sorted ascending by size... This is
# not a MUST, but STRONGLY recommended. Actually the items should
# be sorted by importance/byte, but this is not feasible."
tags = [v._internal(k) for k, v in self.items()]
tags.sort(lambda a, b: cmp(len(a), len(b)))
num_tags = len(tags)
tags = "".join(tags)
header = "APETAGEX%s%s" % (
# version, tag size, item count, flags
struct.pack("<4I", 2000, len(tags) + 32, num_tags,
HAS_HEADER | IS_HEADER),
"\0" * 8)
fileobj.write(header)
fileobj.write(tags)
footer = "APETAGEX%s%s" % (
# version, tag size, item count, flags
struct.pack("<4I", 2000, len(tags) + 32, num_tags,
HAS_HEADER),
"\0" * 8)
fileobj.write(footer)
fileobj.close()
def delete(self, filename=None):
"""Remove tags from a file."""
filename = filename or self.filename
fileobj = open(filename, "r+b")
try:
data = _APEv2Data(fileobj)
if data.start is not None and data.size is not None:
delete_bytes(fileobj, data.end - data.start, data.start)
finally:
fileobj.close()
self.clear()
Open = APEv2
def delete(filename):
"""Remove tags from a file."""
try:
APEv2(filename).delete()
except APENoHeaderError:
pass
def APEValue(value, kind):
"""APEv2 tag value factory.
Use this if you need to specify the value's type manually. Binary
and text data are automatically detected by APEv2.__setitem__.
"""
if kind == TEXT:
return APETextValue(value, kind)
elif kind == BINARY:
return APEBinaryValue(value, kind)
elif kind == EXTERNAL:
return APEExtValue(value, kind)
else:
raise ValueError("kind must be TEXT, BINARY, or EXTERNAL")
class _APEValue(object):
def __init__(self, value, kind):
self.kind = kind
self.value = value
def __len__(self):
return len(self.value)
def __str__(self):
return self.value
# Packed format for an item:
# 4B: Value length
# 4B: Value type
# Key name
# 1B: Null
# Key value
def _internal(self, key):
return "%s%s\0%s" % (
struct.pack("<2I", len(self.value), self.kind << 1),
key, self.value)
def __repr__(self):
return "%s(%r, %d)" % (type(self).__name__, self.value, self.kind)
class APETextValue(_APEValue):
"""An APEv2 text value.
Text values are Unicode/UTF-8 strings. They can be accessed like
strings (with a null seperating the values), or arrays of strings."""
def __unicode__(self):
return unicode(str(self), "utf-8")
def __iter__(self):
"""Iterate over the strings of the value (not the characters)"""
return iter(unicode(self).split("\0"))
def __getitem__(self, index):
return unicode(self).split("\0")[index]
def __len__(self):
return self.value.count("\0") + 1
def __cmp__(self, other):
return cmp(unicode(self), other)
__hash__ = _APEValue.__hash__
def __setitem__(self, index, value):
values = list(self)
values[index] = value.encode("utf-8")
self.value = "\0".join(values).encode("utf-8")
def pprint(self):
return " / ".join(self)
class APEBinaryValue(_APEValue):
"""An APEv2 binary value."""
def pprint(self):
return "[%d bytes]" % len(self)
class APEExtValue(_APEValue):
"""An APEv2 external value.
External values are usually URI or IRI strings.
"""
def pprint(self):
return "[External] %s" % unicode(self)
class APEv2File(FileType):
class _Info(object):
length = 0
bitrate = 0
def __init__(self, fileobj):
pass
@staticmethod
def pprint():
return "Unknown format with APEv2 tag."
def load(self, filename):
self.filename = filename
self.info = self._Info(open(filename, "rb"))
try:
self.tags = APEv2(filename)
except error:
self.tags = None
def add_tags(self):
if self.tags is None:
self.tags = APEv2()
else:
raise ValueError("%r already has tags: %r" % (self, self.tags))
@staticmethod
def score(filename, fileobj, header):
try:
fileobj.seek(-160, 2)
except IOError:
fileobj.seek(0)
footer = fileobj.read()
filename = filename.lower()
return (("APETAGEX" in footer) - header.startswith("ID3"))

704
libs/mutagen/asf.py Normal file
View file

@ -0,0 +1,704 @@
# Copyright 2006-2007 Lukas Lalinsky
# Copyright 2005-2006 Joe Wreschnig
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Read and write ASF (Window Media Audio) files."""
__all__ = ["ASF", "Open"]
import struct
from mutagen import FileType, Metadata
from mutagen._util import insert_bytes, delete_bytes, DictMixin
class error(IOError):
pass
class ASFError(error):
pass
class ASFHeaderError(error):
pass
class ASFInfo(object):
"""ASF stream information."""
def __init__(self):
self.length = 0.0
self.sample_rate = 0
self.bitrate = 0
self.channels = 0
def pprint(self):
s = "Windows Media Audio %d bps, %s Hz, %d channels, %.2f seconds" % (
self.bitrate, self.sample_rate, self.channels, self.length)
return s
class ASFTags(list, DictMixin, Metadata):
"""Dictionary containing ASF attributes."""
def pprint(self):
return "\n".join(["%s=%s" % (k, v) for k, v in self])
def __getitem__(self, key):
"""A list of values for the key.
This is a copy, so comment['title'].append('a title') will not
work.
"""
values = [value for (k, value) in self if k == key]
if not values:
raise KeyError(key)
else:
return values
def __delitem__(self, key):
"""Delete all values associated with the key."""
to_delete = filter(lambda x: x[0] == key, self)
if not to_delete:
raise KeyError(key)
else:
map(self.remove, to_delete)
def __contains__(self, key):
"""Return true if the key has any values."""
for k, value in self:
if k == key:
return True
else:
return False
def __setitem__(self, key, values):
"""Set a key's value or values.
Setting a value overwrites all old ones. The value may be a
list of Unicode or UTF-8 strings, or a single Unicode or UTF-8
string.
"""
if not isinstance(values, list):
values = [values]
try:
del(self[key])
except KeyError:
pass
for value in values:
if key in _standard_attribute_names:
value = unicode(value)
elif not isinstance(value, ASFBaseAttribute):
if isinstance(value, basestring):
value = ASFUnicodeAttribute(value)
elif isinstance(value, bool):
value = ASFBoolAttribute(value)
elif isinstance(value, int):
value = ASFDWordAttribute(value)
elif isinstance(value, long):
value = ASFQWordAttribute(value)
self.append((key, value))
def keys(self):
"""Return all keys in the comment."""
return self and set(zip(*self)[0])
def as_dict(self):
"""Return a copy of the comment data in a real dict."""
d = {}
for key, value in self:
d.setdefault(key, []).append(value)
return d
class ASFBaseAttribute(object):
"""Generic attribute."""
TYPE = None
def __init__(self, value=None, data=None, language=None,
stream=None, **kwargs):
self.language = language
self.stream = stream
if data:
self.value = self.parse(data, **kwargs)
else:
self.value = value
def data_size(self):
raise NotImplementedError
def __repr__(self):
name = "%s(%r" % (type(self).__name__, self.value)
if self.language:
name += ", language=%d" % self.language
if self.stream:
name += ", stream=%d" % self.stream
name += ")"
return name
def render(self, name):
name = name.encode("utf-16-le") + "\x00\x00"
data = self._render()
return (struct.pack("<H", len(name)) + name +
struct.pack("<HH", self.TYPE, len(data)) + data)
def render_m(self, name):
name = name.encode("utf-16-le") + "\x00\x00"
if self.TYPE == 2:
data = self._render(dword=False)
else:
data = self._render()
return (struct.pack("<HHHHI", 0, self.stream or 0, len(name),
self.TYPE, len(data)) + name + data)
def render_ml(self, name):
name = name.encode("utf-16-le") + "\x00\x00"
if self.TYPE == 2:
data = self._render(dword=False)
else:
data = self._render()
return (struct.pack("<HHHHI", self.language or 0, self.stream or 0,
len(name), self.TYPE, len(data)) + name + data)
class ASFUnicodeAttribute(ASFBaseAttribute):
"""Unicode string attribute."""
TYPE = 0x0000
def parse(self, data):
return data.decode("utf-16-le").strip("\x00")
def _render(self):
return self.value.encode("utf-16-le") + "\x00\x00"
def data_size(self):
return len(self._render())
def __str__(self):
return self.value
def __cmp__(self, other):
return cmp(unicode(self), other)
__hash__ = ASFBaseAttribute.__hash__
class ASFByteArrayAttribute(ASFBaseAttribute):
"""Byte array attribute."""
TYPE = 0x0001
def parse(self, data):
return data
def _render(self):
return self.value
def data_size(self):
return len(self.value)
def __str__(self):
return "[binary data (%s bytes)]" % len(self.value)
def __cmp__(self, other):
return cmp(str(self), other)
__hash__ = ASFBaseAttribute.__hash__
class ASFBoolAttribute(ASFBaseAttribute):
"""Bool attribute."""
TYPE = 0x0002
def parse(self, data, dword=True):
if dword:
return struct.unpack("<I", data)[0] == 1
else:
return struct.unpack("<H", data)[0] == 1
def _render(self, dword=True):
if dword:
return struct.pack("<I", int(self.value))
else:
return struct.pack("<H", int(self.value))
def data_size(self):
return 4
def __bool__(self):
return self.value
def __str__(self):
return str(self.value)
def __cmp__(self, other):
return cmp(bool(self), other)
__hash__ = ASFBaseAttribute.__hash__
class ASFDWordAttribute(ASFBaseAttribute):
"""DWORD attribute."""
TYPE = 0x0003
def parse(self, data):
return struct.unpack("<L", data)[0]
def _render(self):
return struct.pack("<L", self.value)
def data_size(self):
return 4
def __int__(self):
return self.value
def __str__(self):
return str(self.value)
def __cmp__(self, other):
return cmp(int(self), other)
__hash__ = ASFBaseAttribute.__hash__
class ASFQWordAttribute(ASFBaseAttribute):
"""QWORD attribute."""
TYPE = 0x0004
def parse(self, data):
return struct.unpack("<Q", data)[0]
def _render(self):
return struct.pack("<Q", self.value)
def data_size(self):
return 8
def __int__(self):
return self.value
def __str__(self):
return str(self.value)
def __cmp__(self, other):
return cmp(int(self), other)
__hash__ = ASFBaseAttribute.__hash__
class ASFWordAttribute(ASFBaseAttribute):
"""WORD attribute."""
TYPE = 0x0005
def parse(self, data):
return struct.unpack("<H", data)[0]
def _render(self):
return struct.pack("<H", self.value)
def data_size(self):
return 2
def __int__(self):
return self.value
def __str__(self):
return str(self.value)
def __cmp__(self, other):
return cmp(int(self), other)
__hash__ = ASFBaseAttribute.__hash__
class ASFGUIDAttribute(ASFBaseAttribute):
"""GUID attribute."""
TYPE = 0x0006
def parse(self, data):
return data
def _render(self):
return self.value
def data_size(self):
return len(self.value)
def __str__(self):
return self.value
def __cmp__(self, other):
return cmp(str(self), other)
__hash__ = ASFBaseAttribute.__hash__
UNICODE = ASFUnicodeAttribute.TYPE
BYTEARRAY = ASFByteArrayAttribute.TYPE
BOOL = ASFBoolAttribute.TYPE
DWORD = ASFDWordAttribute.TYPE
QWORD = ASFQWordAttribute.TYPE
WORD = ASFWordAttribute.TYPE
GUID = ASFGUIDAttribute.TYPE
def ASFValue(value, kind, **kwargs):
for t, c in _attribute_types.items():
if kind == t:
return c(value=value, **kwargs)
raise ValueError("Unknown value type")
_attribute_types = {
ASFUnicodeAttribute.TYPE: ASFUnicodeAttribute,
ASFByteArrayAttribute.TYPE: ASFByteArrayAttribute,
ASFBoolAttribute.TYPE: ASFBoolAttribute,
ASFDWordAttribute.TYPE: ASFDWordAttribute,
ASFQWordAttribute.TYPE: ASFQWordAttribute,
ASFWordAttribute.TYPE: ASFWordAttribute,
ASFGUIDAttribute.TYPE: ASFGUIDAttribute,
}
_standard_attribute_names = [
"Title",
"Author",
"Copyright",
"Description",
"Rating"
]
class BaseObject(object):
"""Base ASF object."""
GUID = None
def parse(self, asf, data, fileobj, size):
self.data = data
def render(self, asf):
data = self.GUID + struct.pack("<Q", len(self.data) + 24) + self.data
return data
class UnknownObject(BaseObject):
"""Unknown ASF object."""
def __init__(self, guid):
self.GUID = guid
class HeaderObject(object):
"""ASF header."""
GUID = "\x30\x26\xB2\x75\x8E\x66\xCF\x11\xA6\xD9\x00\xAA\x00\x62\xCE\x6C"
class ContentDescriptionObject(BaseObject):
"""Content description."""
GUID = "\x33\x26\xB2\x75\x8E\x66\xCF\x11\xA6\xD9\x00\xAA\x00\x62\xCE\x6C"
def parse(self, asf, data, fileobj, size):
super(ContentDescriptionObject, self).parse(asf, data, fileobj, size)
asf.content_description_obj = self
lengths = struct.unpack("<HHHHH", data[:10])
texts = []
pos = 10
for length in lengths:
end = pos + length
if length > 0:
texts.append(data[pos:end].decode("utf-16-le").strip("\x00"))
else:
texts.append(None)
pos = end
title, author, copyright, desc, rating = texts
for key, value in dict(
Title=title,
Author=author,
Copyright=copyright,
Description=desc,
Rating=rating
).items():
if value is not None:
asf.tags[key] = value
def render(self, asf):
def render_text(name):
value = asf.tags.get(name, [])
if value:
return value[0].encode("utf-16-le") + "\x00\x00"
else:
return ""
texts = map(render_text, _standard_attribute_names)
data = struct.pack("<HHHHH", *map(len, texts)) + "".join(texts)
return self.GUID + struct.pack("<Q", 24 + len(data)) + data
class ExtendedContentDescriptionObject(BaseObject):
"""Extended content description."""
GUID = "\x40\xA4\xD0\xD2\x07\xE3\xD2\x11\x97\xF0\x00\xA0\xC9\x5E\xA8\x50"
def parse(self, asf, data, fileobj, size):
super(ExtendedContentDescriptionObject, self).parse(
asf, data, fileobj, size)
asf.extended_content_description_obj = self
num_attributes, = struct.unpack("<H", data[0:2])
pos = 2
for i in range(num_attributes):
name_length, = struct.unpack("<H", data[pos:pos+2])
pos += 2
name = data[pos:pos+name_length].decode("utf-16-le").strip("\x00")
pos += name_length
value_type, value_length = struct.unpack("<HH", data[pos:pos+4])
pos += 4
value = data[pos:pos+value_length]
pos += value_length
attr = _attribute_types[value_type](data=value)
asf.tags.append((name, attr))
def render(self, asf):
attrs = asf.to_extended_content_description.items()
data = "".join([attr.render(name) for (name, attr) in attrs])
data = struct.pack("<QH", 26 + len(data), len(attrs)) + data
return self.GUID + data
class FilePropertiesObject(BaseObject):
"""File properties."""
GUID = "\xA1\xDC\xAB\x8C\x47\xA9\xCF\x11\x8E\xE4\x00\xC0\x0C\x20\x53\x65"
def parse(self, asf, data, fileobj, size):
super(FilePropertiesObject, self).parse(asf, data, fileobj, size)
length, _, preroll = struct.unpack("<QQQ", data[40:64])
asf.info.length = length / 10000000.0 - preroll / 1000.0
class StreamPropertiesObject(BaseObject):
"""Stream properties."""
GUID = "\x91\x07\xDC\xB7\xB7\xA9\xCF\x11\x8E\xE6\x00\xC0\x0C\x20\x53\x65"
def parse(self, asf, data, fileobj, size):
super(StreamPropertiesObject, self).parse(asf, data, fileobj, size)
channels, sample_rate, bitrate = struct.unpack("<HII", data[56:66])
asf.info.channels = channels
asf.info.sample_rate = sample_rate
asf.info.bitrate = bitrate * 8
class HeaderExtensionObject(BaseObject):
"""Header extension."""
GUID = "\xb5\x03\xbf_.\xa9\xcf\x11\x8e\xe3\x00\xc0\x0c Se"
def parse(self, asf, data, fileobj, size):
super(HeaderExtensionObject, self).parse(asf, data, fileobj, size)
asf.header_extension_obj = self
datasize, = struct.unpack("<I", data[18:22])
datapos = 0
self.objects = []
while datapos < datasize:
guid, size = struct.unpack("<16sQ", data[22+datapos:22+datapos+24])
if guid in _object_types:
obj = _object_types[guid]()
else:
obj = UnknownObject(guid)
obj.parse(asf, data[22+datapos+24:22+datapos+size], fileobj, size)
self.objects.append(obj)
datapos += size
def render(self, asf):
data = "".join([obj.render(asf) for obj in self.objects])
return (self.GUID + struct.pack("<Q", 24 + 16 + 6 + len(data)) +
"\x11\xD2\xD3\xAB\xBA\xA9\xcf\x11" +
"\x8E\xE6\x00\xC0\x0C\x20\x53\x65" +
"\x06\x00" + struct.pack("<I", len(data)) + data)
class MetadataObject(BaseObject):
"""Metadata description."""
GUID = "\xea\xcb\xf8\xc5\xaf[wH\x84g\xaa\x8cD\xfaL\xca"
def parse(self, asf, data, fileobj, size):
super(MetadataObject, self).parse(asf, data, fileobj, size)
asf.metadata_obj = self
num_attributes, = struct.unpack("<H", data[0:2])
pos = 2
for i in range(num_attributes):
(reserved, stream, name_length, value_type,
value_length) = struct.unpack("<HHHHI", data[pos:pos+12])
pos += 12
name = data[pos:pos+name_length].decode("utf-16-le").strip("\x00")
pos += name_length
value = data[pos:pos+value_length]
pos += value_length
args = {'data': value, 'stream': stream}
if value_type == 2:
args['dword'] = False
attr = _attribute_types[value_type](**args)
asf.tags.append((name, attr))
def render(self, asf):
attrs = asf.to_metadata.items()
data = "".join([attr.render_m(name) for (name, attr) in attrs])
return (self.GUID + struct.pack("<QH", 26 + len(data), len(attrs)) +
data)
class MetadataLibraryObject(BaseObject):
"""Metadata library description."""
GUID = "\x94\x1c#D\x98\x94\xd1I\xa1A\x1d\x13NEpT"
def parse(self, asf, data, fileobj, size):
super(MetadataLibraryObject, self).parse(asf, data, fileobj, size)
asf.metadata_library_obj = self
num_attributes, = struct.unpack("<H", data[0:2])
pos = 2
for i in range(num_attributes):
(language, stream, name_length, value_type,
value_length) = struct.unpack("<HHHHI", data[pos:pos+12])
pos += 12
name = data[pos:pos+name_length].decode("utf-16-le").strip("\x00")
pos += name_length
value = data[pos:pos+value_length]
pos += value_length
args = {'data': value, 'language': language, 'stream': stream}
if value_type == 2:
args['dword'] = False
attr = _attribute_types[value_type](**args)
asf.tags.append((name, attr))
def render(self, asf):
attrs = asf.to_metadata_library
data = "".join([attr.render_ml(name) for (name, attr) in attrs])
return (self.GUID + struct.pack("<QH", 26 + len(data), len(attrs)) +
data)
_object_types = {
ExtendedContentDescriptionObject.GUID: ExtendedContentDescriptionObject,
ContentDescriptionObject.GUID: ContentDescriptionObject,
FilePropertiesObject.GUID: FilePropertiesObject,
StreamPropertiesObject.GUID: StreamPropertiesObject,
HeaderExtensionObject.GUID: HeaderExtensionObject,
MetadataLibraryObject.GUID: MetadataLibraryObject,
MetadataObject.GUID: MetadataObject,
}
class ASF(FileType):
"""An ASF file, probably containing WMA or WMV."""
_mimes = ["audio/x-ms-wma", "audio/x-ms-wmv", "video/x-ms-asf",
"audio/x-wma", "video/x-wmv"]
def load(self, filename):
self.filename = filename
fileobj = open(filename, "rb")
try:
self.size = 0
self.size1 = 0
self.size2 = 0
self.offset1 = 0
self.offset2 = 0
self.num_objects = 0
self.info = ASFInfo()
self.tags = ASFTags()
self.__read_file(fileobj)
finally:
fileobj.close()
def save(self):
# Move attributes to the right objects
self.to_extended_content_description = {}
self.to_metadata = {}
self.to_metadata_library = []
for name, value in self.tags:
if name in _standard_attribute_names:
continue
library_only = (value.data_size() > 0xFFFF or value.TYPE == GUID)
if (value.language is None and value.stream is None and
name not in self.to_extended_content_description and
not library_only):
self.to_extended_content_description[name] = value
elif (value.language is None and value.stream is not None and
name not in self.to_metadata and not library_only):
self.to_metadata[name] = value
else:
self.to_metadata_library.append((name, value))
# Add missing objects
if not self.content_description_obj:
self.content_description_obj = \
ContentDescriptionObject()
self.objects.append(self.content_description_obj)
if not self.extended_content_description_obj:
self.extended_content_description_obj = \
ExtendedContentDescriptionObject()
self.objects.append(self.extended_content_description_obj)
if not self.header_extension_obj:
self.header_extension_obj = \
HeaderExtensionObject()
self.objects.append(self.header_extension_obj)
if not self.metadata_obj:
self.metadata_obj = \
MetadataObject()
self.header_extension_obj.objects.append(self.metadata_obj)
if not self.metadata_library_obj:
self.metadata_library_obj = \
MetadataLibraryObject()
self.header_extension_obj.objects.append(self.metadata_library_obj)
# Render the header
data = "".join([obj.render(self) for obj in self.objects])
data = (HeaderObject.GUID +
struct.pack("<QL", len(data) + 30, len(self.objects)) +
"\x01\x02" + data)
fileobj = open(self.filename, "rb+")
try:
size = len(data)
if size > self.size:
insert_bytes(fileobj, size - self.size, self.size)
if size < self.size:
delete_bytes(fileobj, self.size - size, 0)
fileobj.seek(0)
fileobj.write(data)
finally:
fileobj.close()
self.size = size
self.num_objects = len(self.objects)
def __read_file(self, fileobj):
header = fileobj.read(30)
if len(header) != 30 or header[:16] != HeaderObject.GUID:
raise ASFHeaderError("Not an ASF file.")
self.extended_content_description_obj = None
self.content_description_obj = None
self.header_extension_obj = None
self.metadata_obj = None
self.metadata_library_obj = None
self.size, self.num_objects = struct.unpack("<QL", header[16:28])
self.objects = []
for i in range(self.num_objects):
self.__read_object(fileobj)
def __read_object(self, fileobj):
guid, size = struct.unpack("<16sQ", fileobj.read(24))
if guid in _object_types:
obj = _object_types[guid]()
else:
obj = UnknownObject(guid)
data = fileobj.read(size - 24)
obj.parse(self, data, fileobj, size)
self.objects.append(obj)
@staticmethod
def score(filename, fileobj, header):
return header.startswith(HeaderObject.GUID) * 2
Open = ASF

504
libs/mutagen/easyid3.py Normal file
View file

@ -0,0 +1,504 @@
# Simpler (but far more limited) API for ID3 editing
# Copyright 2006 Joe Wreschnig
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of version 2 of the GNU General Public License as
# published by the Free Software Foundation.
"""Easier access to ID3 tags.
EasyID3 is a wrapper around mutagen.id3.ID3 to make ID3 tags appear
more like Vorbis or APEv2 tags.
"""
import mutagen.id3
from mutagen import Metadata
from mutagen._util import DictMixin, dict_match
from mutagen.id3 import ID3, error, delete, ID3FileType
__all__ = ['EasyID3', 'Open', 'delete']
class EasyID3KeyError(KeyError, ValueError, error):
"""Raised when trying to get/set an invalid key.
Subclasses both KeyError and ValueError for API compatibility,
catching KeyError is preferred.
"""
class EasyID3(DictMixin, Metadata):
"""A file with an ID3 tag.
Like Vorbis comments, EasyID3 keys are case-insensitive ASCII
strings. Only a subset of ID3 frames are supported by default. Use
EasyID3.RegisterKey and its wrappers to support more.
You can also set the GetFallback, SetFallback, and DeleteFallback
to generic key getter/setter/deleter functions, which are called
if no specific handler is registered for a key. Additionally,
ListFallback can be used to supply an arbitrary list of extra
keys. These can be set on EasyID3 or on individual instances after
creation.
To use an EasyID3 class with mutagen.mp3.MP3::
from mutagen.mp3 import EasyMP3 as MP3
MP3(filename)
Because many of the attributes are constructed on the fly, things
like the following will not work::
ezid3["performer"].append("Joe")
Instead, you must do::
values = ezid3["performer"]
values.append("Joe")
ezid3["performer"] = values
"""
Set = {}
Get = {}
Delete = {}
List = {}
# For compatibility.
valid_keys = Get
GetFallback = None
SetFallback = None
DeleteFallback = None
ListFallback = None
@classmethod
def RegisterKey(cls, key,
getter=None, setter=None, deleter=None, lister=None):
"""Register a new key mapping.
A key mapping is four functions, a getter, setter, deleter,
and lister. The key may be either a string or a glob pattern.
The getter, deleted, and lister receive an ID3 instance and
the requested key name. The setter also receives the desired
value, which will be a list of strings.
The getter, setter, and deleter are used to implement __getitem__,
__setitem__, and __delitem__.
The lister is used to implement keys(). It should return a
list of keys that are actually in the ID3 instance, provided
by its associated getter.
"""
key = key.lower()
if getter is not None:
cls.Get[key] = getter
if setter is not None:
cls.Set[key] = setter
if deleter is not None:
cls.Delete[key] = deleter
if lister is not None:
cls.List[key] = lister
@classmethod
def RegisterTextKey(cls, key, frameid):
"""Register a text key.
If the key you need to register is a simple one-to-one mapping
of ID3 frame name to EasyID3 key, then you can use this
function::
EasyID3.RegisterTextKey("title", "TIT2")
"""
def getter(id3, key):
return list(id3[frameid])
def setter(id3, key, value):
try:
frame = id3[frameid]
except KeyError:
id3.add(mutagen.id3.Frames[frameid](encoding=3, text=value))
else:
frame.encoding = 3
frame.text = value
def deleter(id3, key):
del(id3[frameid])
cls.RegisterKey(key, getter, setter, deleter)
@classmethod
def RegisterTXXXKey(cls, key, desc):
"""Register a user-defined text frame key.
Some ID3 tags are stored in TXXX frames, which allow a
freeform 'description' which acts as a subkey,
e.g. TXXX:BARCODE.::
EasyID3.RegisterTXXXKey('barcode', 'BARCODE').
"""
frameid = "TXXX:" + desc
def getter(id3, key):
return list(id3[frameid])
def setter(id3, key, value):
try:
frame = id3[frameid]
except KeyError:
enc = 0
# Store 8859-1 if we can, per MusicBrainz spec.
for v in value:
if v and max(v) > u'\x7f':
enc = 3
id3.add(mutagen.id3.TXXX(encoding=enc, text=value, desc=desc))
else:
frame.text = value
def deleter(id3, key):
del(id3[frameid])
cls.RegisterKey(key, getter, setter, deleter)
def __init__(self, filename=None):
self.__id3 = ID3()
if filename is not None:
self.load(filename)
load = property(lambda s: s.__id3.load,
lambda s, v: setattr(s.__id3, 'load', v))
save = property(lambda s: s.__id3.save,
lambda s, v: setattr(s.__id3, 'save', v))
delete = property(lambda s: s.__id3.delete,
lambda s, v: setattr(s.__id3, 'delete', v))
filename = property(lambda s: s.__id3.filename,
lambda s, fn: setattr(s.__id3, 'filename', fn))
size = property(lambda s: s.__id3.size,
lambda s, fn: setattr(s.__id3, 'size', s))
def __getitem__(self, key):
key = key.lower()
func = dict_match(self.Get, key, self.GetFallback)
if func is not None:
return func(self.__id3, key)
else:
raise EasyID3KeyError("%r is not a valid key" % key)
def __setitem__(self, key, value):
key = key.lower()
if isinstance(value, basestring):
value = [value]
func = dict_match(self.Set, key, self.SetFallback)
if func is not None:
return func(self.__id3, key, value)
else:
raise EasyID3KeyError("%r is not a valid key" % key)
def __delitem__(self, key):
key = key.lower()
func = dict_match(self.Delete, key, self.DeleteFallback)
if func is not None:
return func(self.__id3, key)
else:
raise EasyID3KeyError("%r is not a valid key" % key)
def keys(self):
keys = []
for key in self.Get.keys():
if key in self.List:
keys.extend(self.List[key](self.__id3, key))
elif key in self:
keys.append(key)
if self.ListFallback is not None:
keys.extend(self.ListFallback(self.__id3, ""))
return keys
def pprint(self):
"""Print tag key=value pairs."""
strings = []
for key in sorted(self.keys()):
values = self[key]
for value in values:
strings.append("%s=%s" % (key, value))
return "\n".join(strings)
Open = EasyID3
def genre_get(id3, key):
return id3["TCON"].genres
def genre_set(id3, key, value):
try:
frame = id3["TCON"]
except KeyError:
id3.add(mutagen.id3.TCON(encoding=3, text=value))
else:
frame.encoding = 3
frame.genres = value
def genre_delete(id3, key):
del(id3["TCON"])
def date_get(id3, key):
return [stamp.text for stamp in id3["TDRC"].text]
def date_set(id3, key, value):
id3.add(mutagen.id3.TDRC(encoding=3, text=value))
def date_delete(id3, key):
del(id3["TDRC"])
def performer_get(id3, key):
people = []
wanted_role = key.split(":", 1)[1]
try:
mcl = id3["TMCL"]
except KeyError:
raise KeyError(key)
for role, person in mcl.people:
if role == wanted_role:
people.append(person)
if people:
return people
else:
raise KeyError(key)
def performer_set(id3, key, value):
wanted_role = key.split(":", 1)[1]
try:
mcl = id3["TMCL"]
except KeyError:
mcl = mutagen.id3.TMCL(encoding=3, people=[])
id3.add(mcl)
mcl.encoding = 3
people = [p for p in mcl.people if p[0] != wanted_role]
for v in value:
people.append((wanted_role, v))
mcl.people = people
def performer_delete(id3, key):
wanted_role = key.split(":", 1)[1]
try:
mcl = id3["TMCL"]
except KeyError:
raise KeyError(key)
people = [p for p in mcl.people if p[0] != wanted_role]
if people == mcl.people:
raise KeyError(key)
elif people:
mcl.people = people
else:
del(id3["TMCL"])
def performer_list(id3, key):
try:
mcl = id3["TMCL"]
except KeyError:
return []
else:
return list(set("performer:" + p[0] for p in mcl.people))
def musicbrainz_trackid_get(id3, key):
return [id3["UFID:http://musicbrainz.org"].data.decode('ascii')]
def musicbrainz_trackid_set(id3, key, value):
if len(value) != 1:
raise ValueError("only one track ID may be set per song")
value = value[0].encode('ascii')
try:
frame = id3["UFID:http://musicbrainz.org"]
except KeyError:
frame = mutagen.id3.UFID(owner="http://musicbrainz.org", data=value)
id3.add(frame)
else:
frame.data = value
def musicbrainz_trackid_delete(id3, key):
del(id3["UFID:http://musicbrainz.org"])
def website_get(id3, key):
urls = [frame.url for frame in id3.getall("WOAR")]
if urls:
return urls
else:
raise EasyID3KeyError(key)
def website_set(id3, key, value):
id3.delall("WOAR")
for v in value:
id3.add(mutagen.id3.WOAR(url=v))
def website_delete(id3, key):
id3.delall("WOAR")
def gain_get(id3, key):
try:
frame = id3["RVA2:" + key[11:-5]]
except KeyError:
raise EasyID3KeyError(key)
else:
return [u"%+f dB" % frame.gain]
def gain_set(id3, key, value):
if len(value) != 1:
raise ValueError(
"there must be exactly one gain value, not %r.", value)
gain = float(value[0].split()[0])
try:
frame = id3["RVA2:" + key[11:-5]]
except KeyError:
frame = mutagen.id3.RVA2(desc=key[11:-5], gain=0, peak=0, channel=1)
id3.add(frame)
frame.gain = gain
def gain_delete(id3, key):
try:
frame = id3["RVA2:" + key[11:-5]]
except KeyError:
pass
else:
if frame.peak:
frame.gain = 0.0
else:
del(id3["RVA2:" + key[11:-5]])
def peak_get(id3, key):
try:
frame = id3["RVA2:" + key[11:-5]]
except KeyError:
raise EasyID3KeyError(key)
else:
return [u"%f" % frame.peak]
def peak_set(id3, key, value):
if len(value) != 1:
raise ValueError(
"there must be exactly one peak value, not %r.", value)
peak = float(value[0])
if peak >= 2 or peak < 0:
raise ValueError("peak must be => 0 and < 2.")
try:
frame = id3["RVA2:" + key[11:-5]]
except KeyError:
frame = mutagen.id3.RVA2(desc=key[11:-5], gain=0, peak=0, channel=1)
id3.add(frame)
frame.peak = peak
def peak_delete(id3, key):
try:
frame = id3["RVA2:" + key[11:-5]]
except KeyError:
pass
else:
if frame.gain:
frame.peak = 0.0
else:
del(id3["RVA2:" + key[11:-5]])
def peakgain_list(id3, key):
keys = []
for frame in id3.getall("RVA2"):
keys.append("replaygain_%s_gain" % frame.desc)
keys.append("replaygain_%s_peak" % frame.desc)
return keys
for frameid, key in {
"TALB": "album",
"TBPM": "bpm",
"TCMP": "compilation", # iTunes extension
"TCOM": "composer",
"TCOP": "copyright",
"TENC": "encodedby",
"TEXT": "lyricist",
"TLEN": "length",
"TMED": "media",
"TMOO": "mood",
"TIT2": "title",
"TIT3": "version",
"TPE1": "artist",
"TPE2": "performer",
"TPE3": "conductor",
"TPE4": "arranger",
"TPOS": "discnumber",
"TPUB": "organization",
"TRCK": "tracknumber",
"TOLY": "author",
"TSO2": "albumartistsort", # iTunes extension
"TSOA": "albumsort",
"TSOC": "composersort", # iTunes extension
"TSOP": "artistsort",
"TSOT": "titlesort",
"TSRC": "isrc",
"TSST": "discsubtitle",
}.iteritems():
EasyID3.RegisterTextKey(key, frameid)
EasyID3.RegisterKey("genre", genre_get, genre_set, genre_delete)
EasyID3.RegisterKey("date", date_get, date_set, date_delete)
EasyID3.RegisterKey(
"performer:*", performer_get, performer_set, performer_delete,
performer_list)
EasyID3.RegisterKey("musicbrainz_trackid", musicbrainz_trackid_get,
musicbrainz_trackid_set, musicbrainz_trackid_delete)
EasyID3.RegisterKey("website", website_get, website_set, website_delete)
EasyID3.RegisterKey("website", website_get, website_set, website_delete)
EasyID3.RegisterKey(
"replaygain_*_gain", gain_get, gain_set, gain_delete, peakgain_list)
EasyID3.RegisterKey("replaygain_*_peak", peak_get, peak_set, peak_delete)
# At various times, information for this came from
# http://musicbrainz.org/docs/specs/metadata_tags.html
# http://bugs.musicbrainz.org/ticket/1383
# http://musicbrainz.org/doc/MusicBrainzTag
for desc, key in {
u"MusicBrainz Artist Id": "musicbrainz_artistid",
u"MusicBrainz Album Id": "musicbrainz_albumid",
u"MusicBrainz Album Artist Id": "musicbrainz_albumartistid",
u"MusicBrainz TRM Id": "musicbrainz_trmid",
u"MusicIP PUID": "musicip_puid",
u"MusicMagic Fingerprint": "musicip_fingerprint",
u"MusicBrainz Album Status": "musicbrainz_albumstatus",
u"MusicBrainz Album Type": "musicbrainz_albumtype",
u"MusicBrainz Album Release Country": "releasecountry",
u"MusicBrainz Disc Id": "musicbrainz_discid",
u"ASIN": "asin",
u"ALBUMARTISTSORT": "albumartistsort",
u"BARCODE": "barcode",
}.iteritems():
EasyID3.RegisterTXXXKey(key, desc)
class EasyID3FileType(ID3FileType):
"""Like ID3FileType, but uses EasyID3 for tags."""
ID3 = EasyID3

266
libs/mutagen/easymp4.py Normal file
View file

@ -0,0 +1,266 @@
# Copyright 2009 Joe Wreschnig
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of version 2 of the GNU General Public License as
# published by the Free Software Foundation.
from mutagen import Metadata
from mutagen._util import DictMixin, dict_match, utf8
from mutagen.mp4 import MP4, MP4Tags, error, delete
__all__ = ["EasyMP4Tags", "EasyMP4", "delete", "error"]
class EasyMP4KeyError(error, KeyError, ValueError):
pass
class EasyMP4Tags(DictMixin, Metadata):
"""A file with MPEG-4 iTunes metadata.
Like Vorbis comments, EasyMP4Tags keys are case-insensitive ASCII
strings, and values are a list of Unicode strings (and these lists
are always of length 0 or 1).
If you need access to the full MP4 metadata feature set, you should use
MP4, not EasyMP4.
"""
Set = {}
Get = {}
Delete = {}
List = {}
def __init__(self, *args, **kwargs):
self.__mp4 = MP4Tags(*args, **kwargs)
self.load = self.__mp4.load
self.save = self.__mp4.save
self.delete = self.__mp4.delete
filename = property(lambda s: s.__mp4.filename,
lambda s, fn: setattr(s.__mp4, 'filename', fn))
@classmethod
def RegisterKey(cls, key,
getter=None, setter=None, deleter=None, lister=None):
"""Register a new key mapping.
A key mapping is four functions, a getter, setter, deleter,
and lister. The key may be either a string or a glob pattern.
The getter, deleted, and lister receive an MP4Tags instance
and the requested key name. The setter also receives the
desired value, which will be a list of strings.
The getter, setter, and deleter are used to implement __getitem__,
__setitem__, and __delitem__.
The lister is used to implement keys(). It should return a
list of keys that are actually in the MP4 instance, provided
by its associated getter.
"""
key = key.lower()
if getter is not None:
cls.Get[key] = getter
if setter is not None:
cls.Set[key] = setter
if deleter is not None:
cls.Delete[key] = deleter
if lister is not None:
cls.List[key] = lister
@classmethod
def RegisterTextKey(cls, key, atomid):
"""Register a text key.
If the key you need to register is a simple one-to-one mapping
of MP4 atom name to EasyMP4Tags key, then you can use this
function::
EasyMP4Tags.RegisterTextKey("artist", "\xa9ART")
"""
def getter(tags, key):
return tags[atomid]
def setter(tags, key, value):
tags[atomid] = value
def deleter(tags, key):
del(tags[atomid])
cls.RegisterKey(key, getter, setter, deleter)
@classmethod
def RegisterIntKey(cls, key, atomid, min_value=0, max_value=2**16-1):
"""Register a scalar integer key.
"""
def getter(tags, key):
return map(unicode, tags[atomid])
def setter(tags, key, value):
clamp = lambda x: int(min(max(min_value, x), max_value))
tags[atomid] = map(clamp, map(int, value))
def deleter(tags, key):
del(tags[atomid])
cls.RegisterKey(key, getter, setter, deleter)
@classmethod
def RegisterIntPairKey(cls, key, atomid, min_value=0, max_value=2**16-1):
def getter(tags, key):
ret = []
for (track, total) in tags[atomid]:
if total:
ret.append(u"%d/%d" % (track, total))
else:
ret.append(unicode(track))
return ret
def setter(tags, key, value):
clamp = lambda x: int(min(max(min_value, x), max_value))
data = []
for v in value:
try:
tracks, total = v.split("/")
tracks = clamp(int(tracks))
total = clamp(int(total))
except (ValueError, TypeError):
tracks = clamp(int(v))
total = min_value
data.append((tracks, total))
tags[atomid] = data
def deleter(tags, key):
del(tags[atomid])
cls.RegisterKey(key, getter, setter, deleter)
@classmethod
def RegisterFreeformKey(cls, key, name, mean="com.apple.iTunes"):
"""Register a text key.
If the key you need to register is a simple one-to-one mapping
of MP4 freeform atom (----) and name to EasyMP4Tags key, then
you can use this function::
EasyMP4Tags.RegisterFreeformKey(
"musicbrainz_artistid", "MusicBrainz Artist Id")
"""
atomid = "----:%s:%s" % (mean, name)
def getter(tags, key):
return [s.decode("utf-8", "replace") for s in tags[atomid]]
def setter(tags, key, value):
tags[atomid] = map(utf8, value)
def deleter(tags, key):
del(tags[atomid])
cls.RegisterKey(key, getter, setter, deleter)
def __getitem__(self, key):
key = key.lower()
func = dict_match(self.Get, key)
if func is not None:
return func(self.__mp4, key)
else:
raise EasyMP4KeyError("%r is not a valid key" % key)
def __setitem__(self, key, value):
key = key.lower()
if isinstance(value, basestring):
value = [value]
func = dict_match(self.Set, key)
if func is not None:
return func(self.__mp4, key, value)
else:
raise EasyMP4KeyError("%r is not a valid key" % key)
def __delitem__(self, key):
key = key.lower()
func = dict_match(self.Delete, key)
if func is not None:
return func(self.__mp4, key)
else:
raise EasyMP4KeyError("%r is not a valid key" % key)
def keys(self):
keys = []
for key in self.Get.keys():
if key in self.List:
keys.extend(self.List[key](self.__mp4, key))
elif key in self:
keys.append(key)
return keys
def pprint(self):
"""Print tag key=value pairs."""
strings = []
for key in sorted(self.keys()):
values = self[key]
for value in values:
strings.append("%s=%s" % (key, value))
return "\n".join(strings)
for atomid, key in {
'\xa9nam': 'title',
'\xa9alb': 'album',
'\xa9ART': 'artist',
'aART': 'albumartist',
'\xa9day': 'date',
'\xa9cmt': 'comment',
'desc': 'description',
'\xa9grp': 'grouping',
'\xa9gen': 'genre',
'cprt': 'copyright',
'soal': 'albumsort',
'soaa': 'albumartistsort',
'soar': 'artistsort',
'sonm': 'titlesort',
'soco': 'composersort',
}.items():
EasyMP4Tags.RegisterTextKey(key, atomid)
for name, key in {
'MusicBrainz Artist Id': 'musicbrainz_artistid',
'MusicBrainz Track Id': 'musicbrainz_trackid',
'MusicBrainz Album Id': 'musicbrainz_albumid',
'MusicBrainz Album Artist Id': 'musicbrainz_albumartistid',
'MusicIP PUID': 'musicip_puid',
'MusicBrainz Album Status': 'musicbrainz_albumstatus',
'MusicBrainz Album Type': 'musicbrainz_albumtype',
'MusicBrainz Release Country': 'releasecountry',
}.items():
EasyMP4Tags.RegisterFreeformKey(key, name)
for name, key in {
"tmpo": "bpm",
}.items():
EasyMP4Tags.RegisterIntKey(key, name)
for name, key in {
"trkn": "tracknumber",
"disk": "discnumber",
}.items():
EasyMP4Tags.RegisterIntPairKey(key, name)
class EasyMP4(MP4):
"""Like :class:`MP4 <mutagen.mp4.MP4>`,
but uses :class:`EasyMP4Tags` for tags.
:ivar info: :class:`MP4Info <mutagen.mp4.MP4Info>`
:ivar tags: :class:`EasyMP4Tags`
"""
MP4Tags = EasyMP4Tags
Get = EasyMP4Tags.Get
Set = EasyMP4Tags.Set
Delete = EasyMP4Tags.Delete
List = EasyMP4Tags.List
RegisterTextKey = EasyMP4Tags.RegisterTextKey
RegisterKey = EasyMP4Tags.RegisterKey

833
libs/mutagen/flac.py Normal file
View file

@ -0,0 +1,833 @@
# FLAC comment support for Mutagen
# Copyright 2005 Joe Wreschnig
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of version 2 of the GNU General Public License as
# published by the Free Software Foundation.
"""Read and write FLAC Vorbis comments and stream information.
Read more about FLAC at http://flac.sourceforge.net.
FLAC supports arbitrary metadata blocks. The two most interesting ones
are the FLAC stream information block, and the Vorbis comment block;
these are also the only ones Mutagen can currently read.
This module does not handle Ogg FLAC files.
Based off documentation available at
http://flac.sourceforge.net/format.html
"""
__all__ = ["FLAC", "Open", "delete"]
import struct
from cStringIO import StringIO
from _vorbis import VCommentDict
from mutagen import FileType
from mutagen._util import insert_bytes
from mutagen.id3 import BitPaddedInt
import sys
if sys.version_info >= (2, 6):
from functools import reduce
class error(IOError):
pass
class FLACNoHeaderError(error):
pass
class FLACVorbisError(ValueError, error):
pass
def to_int_be(string):
"""Convert an arbitrarily-long string to a long using big-endian
byte order."""
return reduce(lambda a, b: (a << 8) + ord(b), string, 0L)
class StrictFileObject(object):
"""Wraps a file-like object and raises an exception if the requested
amount of data to read isn't returned."""
def __init__(self, fileobj):
self._fileobj = fileobj
for m in ["close", "tell", "seek", "write", "name"]:
if hasattr(fileobj, m):
setattr(self, m, getattr(fileobj, m))
def read(self, size=-1):
data = self._fileobj.read(size)
if size >= 0 and len(data) != size:
raise error("file said %d bytes, read %d bytes" % (
size, len(data)))
return data
def tryread(self, *args):
return self._fileobj.read(*args)
class MetadataBlock(object):
"""A generic block of FLAC metadata.
This class is extended by specific used as an ancestor for more specific
blocks, and also as a container for data blobs of unknown blocks.
Attributes:
* data -- raw binary data for this block
"""
_distrust_size = False
def __init__(self, data):
"""Parse the given data string or file-like as a metadata block.
The metadata header should not be included."""
if data is not None:
if not isinstance(data, StrictFileObject):
if isinstance(data, str):
data = StringIO(data)
elif not hasattr(data, 'read'):
raise TypeError(
"StreamInfo requires string data or a file-like")
data = StrictFileObject(data)
self.load(data)
def load(self, data):
self.data = data.read()
def write(self):
return self.data
@staticmethod
def writeblocks(blocks):
"""Render metadata block as a byte string."""
data = []
codes = [[block.code, block.write()] for block in blocks]
codes[-1][0] |= 128
for code, datum in codes:
byte = chr(code)
if len(datum) > 2**24:
raise error("block is too long to write")
length = struct.pack(">I", len(datum))[-3:]
data.append(byte + length + datum)
return "".join(data)
@staticmethod
def group_padding(blocks):
"""Consolidate FLAC padding metadata blocks.
The overall size of the rendered blocks does not change, so
this adds several bytes of padding for each merged block."""
paddings = filter(lambda x: isinstance(x, Padding), blocks)
map(blocks.remove, paddings)
# total padding size is the sum of padding sizes plus 4 bytes
# per removed header.
size = sum([padding.length for padding in paddings])
padding = Padding()
padding.length = size + 4 * (len(paddings) - 1)
blocks.append(padding)
class StreamInfo(MetadataBlock):
"""FLAC stream information.
This contains information about the audio data in the FLAC file.
Unlike most stream information objects in Mutagen, changes to this
one will rewritten to the file when it is saved. Unless you are
actually changing the audio stream itself, don't change any
attributes of this block.
Attributes:
* min_blocksize -- minimum audio block size
* max_blocksize -- maximum audio block size
* sample_rate -- audio sample rate in Hz
* channels -- audio channels (1 for mono, 2 for stereo)
* bits_per_sample -- bits per sample
* total_samples -- total samples in file
* length -- audio length in seconds
"""
code = 0
def __eq__(self, other):
try:
return (self.min_blocksize == other.min_blocksize and
self.max_blocksize == other.max_blocksize and
self.sample_rate == other.sample_rate and
self.channels == other.channels and
self.bits_per_sample == other.bits_per_sample and
self.total_samples == other.total_samples)
except:
return False
__hash__ = MetadataBlock.__hash__
def load(self, data):
self.min_blocksize = int(to_int_be(data.read(2)))
self.max_blocksize = int(to_int_be(data.read(2)))
self.min_framesize = int(to_int_be(data.read(3)))
self.max_framesize = int(to_int_be(data.read(3)))
# first 16 bits of sample rate
sample_first = to_int_be(data.read(2))
# last 4 bits of sample rate, 3 of channels, first 1 of bits/sample
sample_channels_bps = to_int_be(data.read(1))
# last 4 of bits/sample, 36 of total samples
bps_total = to_int_be(data.read(5))
sample_tail = sample_channels_bps >> 4
self.sample_rate = int((sample_first << 4) + sample_tail)
if not self.sample_rate:
raise error("A sample rate value of 0 is invalid")
self.channels = int(((sample_channels_bps >> 1) & 7) + 1)
bps_tail = bps_total >> 36
bps_head = (sample_channels_bps & 1) << 4
self.bits_per_sample = int(bps_head + bps_tail + 1)
self.total_samples = bps_total & 0xFFFFFFFFFL
self.length = self.total_samples / float(self.sample_rate)
self.md5_signature = to_int_be(data.read(16))
def write(self):
f = StringIO()
f.write(struct.pack(">I", self.min_blocksize)[-2:])
f.write(struct.pack(">I", self.max_blocksize)[-2:])
f.write(struct.pack(">I", self.min_framesize)[-3:])
f.write(struct.pack(">I", self.max_framesize)[-3:])
# first 16 bits of sample rate
f.write(struct.pack(">I", self.sample_rate >> 4)[-2:])
# 4 bits sample, 3 channel, 1 bps
byte = (self.sample_rate & 0xF) << 4
byte += ((self.channels - 1) & 7) << 1
byte += ((self.bits_per_sample - 1) >> 4) & 1
f.write(chr(byte))
# 4 bits of bps, 4 of sample count
byte = ((self.bits_per_sample - 1) & 0xF) << 4
byte += (self.total_samples >> 32) & 0xF
f.write(chr(byte))
# last 32 of sample count
f.write(struct.pack(">I", self.total_samples & 0xFFFFFFFFL))
# MD5 signature
sig = self.md5_signature
f.write(struct.pack(
">4I", (sig >> 96) & 0xFFFFFFFFL, (sig >> 64) & 0xFFFFFFFFL,
(sig >> 32) & 0xFFFFFFFFL, sig & 0xFFFFFFFFL))
return f.getvalue()
def pprint(self):
return "FLAC, %.2f seconds, %d Hz" % (self.length, self.sample_rate)
class SeekPoint(tuple):
"""A single seek point in a FLAC file.
Placeholder seek points have first_sample of 0xFFFFFFFFFFFFFFFFL,
and byte_offset and num_samples undefined. Seek points must be
sorted in ascending order by first_sample number. Seek points must
be unique by first_sample number, except for placeholder
points. Placeholder points must occur last in the table and there
may be any number of them.
Attributes:
* first_sample -- sample number of first sample in the target frame
* byte_offset -- offset from first frame to target frame
* num_samples -- number of samples in target frame
"""
def __new__(cls, first_sample, byte_offset, num_samples):
return super(cls, SeekPoint).__new__(
cls, (first_sample, byte_offset, num_samples))
first_sample = property(lambda self: self[0])
byte_offset = property(lambda self: self[1])
num_samples = property(lambda self: self[2])
class SeekTable(MetadataBlock):
"""Read and write FLAC seek tables.
Attributes:
* seekpoints -- list of SeekPoint objects
"""
__SEEKPOINT_FORMAT = '>QQH'
__SEEKPOINT_SIZE = struct.calcsize(__SEEKPOINT_FORMAT)
code = 3
def __init__(self, data):
self.seekpoints = []
super(SeekTable, self).__init__(data)
def __eq__(self, other):
try:
return (self.seekpoints == other.seekpoints)
except (AttributeError, TypeError):
return False
__hash__ = MetadataBlock.__hash__
def load(self, data):
self.seekpoints = []
sp = data.tryread(self.__SEEKPOINT_SIZE)
while len(sp) == self.__SEEKPOINT_SIZE:
self.seekpoints.append(SeekPoint(
*struct.unpack(self.__SEEKPOINT_FORMAT, sp)))
sp = data.tryread(self.__SEEKPOINT_SIZE)
def write(self):
f = StringIO()
for seekpoint in self.seekpoints:
packed = struct.pack(
self.__SEEKPOINT_FORMAT,
seekpoint.first_sample, seekpoint.byte_offset,
seekpoint.num_samples)
f.write(packed)
return f.getvalue()
def __repr__(self):
return "<%s seekpoints=%r>" % (type(self).__name__, self.seekpoints)
class VCFLACDict(VCommentDict):
"""Read and write FLAC Vorbis comments.
FLACs don't use the framing bit at the end of the comment block.
So this extends VCommentDict to not use the framing bit.
"""
code = 4
_distrust_size = True
def load(self, data, errors='replace', framing=False):
super(VCFLACDict, self).load(data, errors=errors, framing=framing)
def write(self, framing=False):
return super(VCFLACDict, self).write(framing=framing)
class CueSheetTrackIndex(tuple):
"""Index for a track in a cuesheet.
For CD-DA, an index_number of 0 corresponds to the track
pre-gap. The first index in a track must have a number of 0 or 1,
and subsequently, index_numbers must increase by 1. Index_numbers
must be unique within a track. And index_offset must be evenly
divisible by 588 samples.
Attributes:
* index_number -- index point number
* index_offset -- offset in samples from track start
"""
def __new__(cls, index_number, index_offset):
return super(cls, CueSheetTrackIndex).__new__(
cls, (index_number, index_offset))
index_number = property(lambda self: self[0])
index_offset = property(lambda self: self[1])
class CueSheetTrack(object):
"""A track in a cuesheet.
For CD-DA, track_numbers must be 1-99, or 170 for the
lead-out. Track_numbers must be unique within a cue sheet. There
must be atleast one index in every track except the lead-out track
which must have none.
Attributes:
* track_number -- track number
* start_offset -- track offset in samples from start of FLAC stream
* isrc -- ISRC code
* type -- 0 for audio, 1 for digital data
* pre_emphasis -- true if the track is recorded with pre-emphasis
* indexes -- list of CueSheetTrackIndex objects
"""
def __init__(self, track_number, start_offset, isrc='', type_=0,
pre_emphasis=False):
self.track_number = track_number
self.start_offset = start_offset
self.isrc = isrc
self.type = type_
self.pre_emphasis = pre_emphasis
self.indexes = []
def __eq__(self, other):
try:
return (self.track_number == other.track_number and
self.start_offset == other.start_offset and
self.isrc == other.isrc and
self.type == other.type and
self.pre_emphasis == other.pre_emphasis and
self.indexes == other.indexes)
except (AttributeError, TypeError):
return False
__hash__ = object.__hash__
def __repr__(self):
return ("<%s number=%r, offset=%d, isrc=%r, type=%r, "
"pre_emphasis=%r, indexes=%r)>") % (
type(self).__name__, self.track_number, self.start_offset,
self.isrc, self.type, self.pre_emphasis, self.indexes)
class CueSheet(MetadataBlock):
"""Read and write FLAC embedded cue sheets.
Number of tracks should be from 1 to 100. There should always be
exactly one lead-out track and that track must be the last track
in the cue sheet.
Attributes:
* media_catalog_number -- media catalog number in ASCII
* lead_in_samples -- number of lead-in samples
* compact_disc -- true if the cuesheet corresponds to a compact disc
* tracks -- list of CueSheetTrack objects
* lead_out -- lead-out as CueSheetTrack or None if lead-out was not found
"""
__CUESHEET_FORMAT = '>128sQB258xB'
__CUESHEET_SIZE = struct.calcsize(__CUESHEET_FORMAT)
__CUESHEET_TRACK_FORMAT = '>QB12sB13xB'
__CUESHEET_TRACK_SIZE = struct.calcsize(__CUESHEET_TRACK_FORMAT)
__CUESHEET_TRACKINDEX_FORMAT = '>QB3x'
__CUESHEET_TRACKINDEX_SIZE = struct.calcsize(__CUESHEET_TRACKINDEX_FORMAT)
code = 5
media_catalog_number = ''
lead_in_samples = 88200
compact_disc = True
def __init__(self, data):
self.tracks = []
super(CueSheet, self).__init__(data)
def __eq__(self, other):
try:
return (self.media_catalog_number == other.media_catalog_number and
self.lead_in_samples == other.lead_in_samples and
self.compact_disc == other.compact_disc and
self.tracks == other.tracks)
except (AttributeError, TypeError):
return False
__hash__ = MetadataBlock.__hash__
def load(self, data):
header = data.read(self.__CUESHEET_SIZE)
media_catalog_number, lead_in_samples, flags, num_tracks = \
struct.unpack(self.__CUESHEET_FORMAT, header)
self.media_catalog_number = media_catalog_number.rstrip('\0')
self.lead_in_samples = lead_in_samples
self.compact_disc = bool(flags & 0x80)
self.tracks = []
for i in range(num_tracks):
track = data.read(self.__CUESHEET_TRACK_SIZE)
start_offset, track_number, isrc_padded, flags, num_indexes = \
struct.unpack(self.__CUESHEET_TRACK_FORMAT, track)
isrc = isrc_padded.rstrip('\0')
type_ = (flags & 0x80) >> 7
pre_emphasis = bool(flags & 0x40)
val = CueSheetTrack(
track_number, start_offset, isrc, type_, pre_emphasis)
for j in range(num_indexes):
index = data.read(self.__CUESHEET_TRACKINDEX_SIZE)
index_offset, index_number = struct.unpack(
self.__CUESHEET_TRACKINDEX_FORMAT, index)
val.indexes.append(
CueSheetTrackIndex(index_number, index_offset))
self.tracks.append(val)
def write(self):
f = StringIO()
flags = 0
if self.compact_disc:
flags |= 0x80
packed = struct.pack(
self.__CUESHEET_FORMAT, self.media_catalog_number,
self.lead_in_samples, flags, len(self.tracks))
f.write(packed)
for track in self.tracks:
track_flags = 0
track_flags |= (track.type & 1) << 7
if track.pre_emphasis:
track_flags |= 0x40
track_packed = struct.pack(
self.__CUESHEET_TRACK_FORMAT, track.start_offset,
track.track_number, track.isrc, track_flags,
len(track.indexes))
f.write(track_packed)
for index in track.indexes:
index_packed = struct.pack(
self.__CUESHEET_TRACKINDEX_FORMAT,
index.index_offset, index.index_number)
f.write(index_packed)
return f.getvalue()
def __repr__(self):
return ("<%s media_catalog_number=%r, lead_in=%r, compact_disc=%r, "
"tracks=%r>") % (
type(self).__name__, self.media_catalog_number,
self.lead_in_samples, self.compact_disc, self.tracks)
class Picture(MetadataBlock):
"""Read and write FLAC embed pictures.
Attributes:
* type -- picture type (same as types for ID3 APIC frames)
* mime -- MIME type of the picture
* desc -- picture's description
* width -- width in pixels
* height -- height in pixels
* depth -- color depth in bits-per-pixel
* colors -- number of colors for indexed palettes (like GIF),
0 for non-indexed
* data -- picture data
"""
code = 6
_distrust_size = True
def __init__(self, data=None):
self.type = 0
self.mime = u''
self.desc = u''
self.width = 0
self.height = 0
self.depth = 0
self.colors = 0
self.data = ''
super(Picture, self).__init__(data)
def __eq__(self, other):
try:
return (self.type == other.type and
self.mime == other.mime and
self.desc == other.desc and
self.width == other.width and
self.height == other.height and
self.depth == other.depth and
self.colors == other.colors and
self.data == other.data)
except (AttributeError, TypeError):
return False
__hash__ = MetadataBlock.__hash__
def load(self, data):
self.type, length = struct.unpack('>2I', data.read(8))
self.mime = data.read(length).decode('UTF-8', 'replace')
length, = struct.unpack('>I', data.read(4))
self.desc = data.read(length).decode('UTF-8', 'replace')
(self.width, self.height, self.depth,
self.colors, length) = struct.unpack('>5I', data.read(20))
self.data = data.read(length)
def write(self):
f = StringIO()
mime = self.mime.encode('UTF-8')
f.write(struct.pack('>2I', self.type, len(mime)))
f.write(mime)
desc = self.desc.encode('UTF-8')
f.write(struct.pack('>I', len(desc)))
f.write(desc)
f.write(struct.pack('>5I', self.width, self.height, self.depth,
self.colors, len(self.data)))
f.write(self.data)
return f.getvalue()
def __repr__(self):
return "<%s '%s' (%d bytes)>" % (type(self).__name__, self.mime,
len(self.data))
class Padding(MetadataBlock):
"""Empty padding space for metadata blocks.
To avoid rewriting the entire FLAC file when editing comments,
metadata is often padded. Padding should occur at the end, and no
more than one padding block should be in any FLAC file. Mutagen
handles this with MetadataBlock.group_padding.
"""
code = 1
def __init__(self, data=""):
super(Padding, self).__init__(data)
def load(self, data):
self.length = len(data.read())
def write(self):
try:
return "\x00" * self.length
# On some 64 bit platforms this won't generate a MemoryError
# or OverflowError since you might have enough RAM, but it
# still generates a ValueError. On other 64 bit platforms,
# this will still succeed for extremely large values.
# Those should never happen in the real world, and if they
# do, writeblocks will catch it.
except (OverflowError, ValueError, MemoryError):
raise error("cannot write %d bytes" % self.length)
def __eq__(self, other):
return isinstance(other, Padding) and self.length == other.length
__hash__ = MetadataBlock.__hash__
def __repr__(self):
return "<%s (%d bytes)>" % (type(self).__name__, self.length)
class FLAC(FileType):
"""A FLAC audio file.
Attributes:
* info -- stream information (length, bitrate, sample rate)
* tags -- metadata tags, if any
* cuesheet -- CueSheet object, if any
* seektable -- SeekTable object, if any
* pictures -- list of embedded pictures
"""
_mimes = ["audio/x-flac", "application/x-flac"]
METADATA_BLOCKS = [StreamInfo, Padding, None, SeekTable, VCFLACDict,
CueSheet, Picture]
"""Known metadata block types, indexed by ID."""
@staticmethod
def score(filename, fileobj, header):
return (header.startswith("fLaC") +
filename.lower().endswith(".flac") * 3)
def __read_metadata_block(self, fileobj):
byte = ord(fileobj.read(1))
size = to_int_be(fileobj.read(3))
code = byte & 0x7F
last_block = bool(byte & 0x80)
try:
block_type = self.METADATA_BLOCKS[code] or MetadataBlock
except IndexError:
block_type = MetadataBlock
if block_type._distrust_size:
# Some jackass is writing broken Metadata block length
# for Vorbis comment blocks, and the FLAC reference
# implementaton can parse them (mostly by accident),
# so we have to too. Instead of parsing the size
# given, parse an actual Vorbis comment, leaving
# fileobj in the right position.
# http://code.google.com/p/mutagen/issues/detail?id=52
# ..same for the Picture block:
# http://code.google.com/p/mutagen/issues/detail?id=106
block = block_type(fileobj)
else:
data = fileobj.read(size)
block = block_type(data)
block.code = code
if block.code == VCFLACDict.code:
if self.tags is None:
self.tags = block
else:
raise FLACVorbisError("> 1 Vorbis comment block found")
elif block.code == CueSheet.code:
if self.cuesheet is None:
self.cuesheet = block
else:
raise error("> 1 CueSheet block found")
elif block.code == SeekTable.code:
if self.seektable is None:
self.seektable = block
else:
raise error("> 1 SeekTable block found")
self.metadata_blocks.append(block)
return not last_block
def add_tags(self):
"""Add a Vorbis comment block to the file."""
if self.tags is None:
self.tags = VCFLACDict()
self.metadata_blocks.append(self.tags)
else:
raise FLACVorbisError("a Vorbis comment already exists")
add_vorbiscomment = add_tags
def delete(self, filename=None):
"""Remove Vorbis comments from a file.
If no filename is given, the one most recently loaded is used.
"""
if filename is None:
filename = self.filename
for s in list(self.metadata_blocks):
if isinstance(s, VCFLACDict):
self.metadata_blocks.remove(s)
self.tags = None
self.save()
break
vc = property(lambda s: s.tags, doc="Alias for tags; don't use this.")
def load(self, filename):
"""Load file information from a filename."""
self.metadata_blocks = []
self.tags = None
self.cuesheet = None
self.seektable = None
self.filename = filename
fileobj = StrictFileObject(open(filename, "rb"))
try:
self.__check_header(fileobj)
while self.__read_metadata_block(fileobj):
pass
finally:
fileobj.close()
try:
self.metadata_blocks[0].length
except (AttributeError, IndexError):
raise FLACNoHeaderError("Stream info block not found")
@property
def info(self):
return self.metadata_blocks[0]
def add_picture(self, picture):
"""Add a new picture to the file."""
self.metadata_blocks.append(picture)
def clear_pictures(self):
"""Delete all pictures from the file."""
self.metadata_blocks = filter(lambda b: b.code != Picture.code,
self.metadata_blocks)
@property
def pictures(self):
"""List of embedded pictures"""
return filter(lambda b: b.code == Picture.code, self.metadata_blocks)
def save(self, filename=None, deleteid3=False):
"""Save metadata blocks to a file.
If no filename is given, the one most recently loaded is used.
"""
if filename is None:
filename = self.filename
f = open(filename, 'rb+')
try:
# Ensure we've got padding at the end, and only at the end.
# If adding makes it too large, we'll scale it down later.
self.metadata_blocks.append(Padding('\x00' * 1020))
MetadataBlock.group_padding(self.metadata_blocks)
header = self.__check_header(f)
# "fLaC" and maybe ID3
available = self.__find_audio_offset(f) - header
data = MetadataBlock.writeblocks(self.metadata_blocks)
# Delete ID3v2
if deleteid3 and header > 4:
available += header - 4
header = 4
if len(data) > available:
# If we have too much data, see if we can reduce padding.
padding = self.metadata_blocks[-1]
newlength = padding.length - (len(data) - available)
if newlength > 0:
padding.length = newlength
data = MetadataBlock.writeblocks(self.metadata_blocks)
assert len(data) == available
elif len(data) < available:
# If we have too little data, increase padding.
self.metadata_blocks[-1].length += (available - len(data))
data = MetadataBlock.writeblocks(self.metadata_blocks)
assert len(data) == available
if len(data) != available:
# We couldn't reduce the padding enough.
diff = (len(data) - available)
insert_bytes(f, diff, header)
f.seek(header - 4)
f.write("fLaC" + data)
# Delete ID3v1
if deleteid3:
try:
f.seek(-128, 2)
except IOError:
pass
else:
if f.read(3) == "TAG":
f.seek(-128, 2)
f.truncate()
finally:
f.close()
def __find_audio_offset(self, fileobj):
byte = 0x00
while not (byte & 0x80):
byte = ord(fileobj.read(1))
size = to_int_be(fileobj.read(3))
try:
block_type = self.METADATA_BLOCKS[byte & 0x7F]
except IndexError:
block_type = None
if block_type and block_type._distrust_size:
# See comments in read_metadata_block; the size can't
# be trusted for Vorbis comment blocks and Picture block
block_type(fileobj)
else:
fileobj.read(size)
return fileobj.tell()
def __check_header(self, fileobj):
size = 4
header = fileobj.read(4)
if header != "fLaC":
size = None
if header[:3] == "ID3":
size = 14 + BitPaddedInt(fileobj.read(6)[2:])
fileobj.seek(size - 4)
if fileobj.read(4) != "fLaC":
size = None
if size is None:
raise FLACNoHeaderError(
"%r is not a valid FLAC file" % fileobj.name)
return size
Open = FLAC
def delete(filename):
"""Remove tags from a file."""
FLAC(filename).delete()

919
libs/mutagen/id3.py Normal file
View file

@ -0,0 +1,919 @@
# id3 support for mutagen
# Copyright (C) 2005 Michael Urman
# 2006 Lukas Lalinsky
# 2013 Christoph Reiter
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of version 2 of the GNU General Public License as
# published by the Free Software Foundation.
"""ID3v2 reading and writing.
This is based off of the following references:
* http://id3.org/id3v2.4.0-structure
* http://id3.org/id3v2.4.0-frames
* http://id3.org/id3v2.3.0
* http://id3.org/id3v2-00
* http://id3.org/ID3v1
Its largest deviation from the above (versions 2.3 and 2.2) is that it
will not interpret the / characters as a separator, and will almost
always accept null separators to generate multi-valued text frames.
Because ID3 frame structure differs between frame types, each frame is
implemented as a different class (e.g. TIT2 as mutagen.id3.TIT2). Each
frame's documentation contains a list of its attributes.
Since this file's documentation is a little unwieldy, you are probably
interested in the :class:`ID3` class to start with.
"""
__all__ = ['ID3', 'ID3FileType', 'Frames', 'Open', 'delete']
import struct
from struct import unpack, pack, error as StructError
import mutagen
from mutagen._util import insert_bytes, delete_bytes, DictProxy
from mutagen._id3util import *
from mutagen._id3frames import *
from mutagen._id3specs import *
class ID3(DictProxy, mutagen.Metadata):
"""A file with an ID3v2 tag.
Attributes:
* version -- ID3 tag version as a tuple
* unknown_frames -- raw frame data of any unknown frames found
* size -- the total size of the ID3 tag, including the header
"""
PEDANTIC = True
version = (2, 4, 0)
filename = None
size = 0
__flags = 0
__readbytes = 0
__crc = None
__unknown_version = None
_V24 = (2, 4, 0)
_V23 = (2, 3, 0)
_V22 = (2, 2, 0)
_V11 = (1, 1)
def __init__(self, *args, **kwargs):
self.unknown_frames = []
super(ID3, self).__init__(*args, **kwargs)
def __fullread(self, size):
try:
if size < 0:
raise ValueError('Requested bytes (%s) less than zero' % size)
if size > self.__filesize:
raise EOFError('Requested %#x of %#x (%s)' % (
long(size), long(self.__filesize), self.filename))
except AttributeError:
pass
data = self.__fileobj.read(size)
if len(data) != size:
raise EOFError
self.__readbytes += size
return data
def load(self, filename, known_frames=None, translate=True, v2_version=4):
"""Load tags from a filename.
Keyword arguments:
* filename -- filename to load tag data from
* known_frames -- dict mapping frame IDs to Frame objects
* translate -- Update all tags to ID3v2.3/4 internally. If you
intend to save, this must be true or you have to
call update_to_v23() / update_to_v24() manually.
* v2_version -- if update_to_v23 or update_to_v24 get called (3 or 4)
Example of loading a custom frame::
my_frames = dict(mutagen.id3.Frames)
class XMYF(Frame): ...
my_frames["XMYF"] = XMYF
mutagen.id3.ID3(filename, known_frames=my_frames)
"""
if not v2_version in (3, 4):
raise ValueError("Only 3 and 4 possible for v2_version")
from os.path import getsize
self.filename = filename
self.__known_frames = known_frames
self.__fileobj = open(filename, 'rb')
self.__filesize = getsize(filename)
try:
try:
self.__load_header()
except EOFError:
self.size = 0
raise ID3NoHeaderError("%s: too small (%d bytes)" % (
filename, self.__filesize))
except (ID3NoHeaderError, ID3UnsupportedVersionError), err:
self.size = 0
import sys
stack = sys.exc_info()[2]
try:
self.__fileobj.seek(-128, 2)
except EnvironmentError:
raise err, None, stack
else:
frames = ParseID3v1(self.__fileobj.read(128))
if frames is not None:
self.version = self._V11
map(self.add, frames.values())
else:
raise err, None, stack
else:
frames = self.__known_frames
if frames is None:
if self._V23 <= self.version:
frames = Frames
elif self._V22 <= self.version:
frames = Frames_2_2
data = self.__fullread(self.size - 10)
for frame in self.__read_frames(data, frames=frames):
if isinstance(frame, Frame):
self.add(frame)
else:
self.unknown_frames.append(frame)
self.__unknown_version = self.version
finally:
self.__fileobj.close()
del self.__fileobj
del self.__filesize
if translate:
if v2_version == 3:
self.update_to_v23()
else:
self.update_to_v24()
def getall(self, key):
"""Return all frames with a given name (the list may be empty).
This is best explained by examples::
id3.getall('TIT2') == [id3['TIT2']]
id3.getall('TTTT') == []
id3.getall('TXXX') == [TXXX(desc='woo', text='bar'),
TXXX(desc='baz', text='quuuux'), ...]
Since this is based on the frame's HashKey, which is
colon-separated, you can use it to do things like
``getall('COMM:MusicMatch')`` or ``getall('TXXX:QuodLibet:')``.
"""
if key in self:
return [self[key]]
else:
key = key + ":"
return [v for s, v in self.items() if s.startswith(key)]
def delall(self, key):
"""Delete all tags of a given kind; see getall."""
if key in self:
del(self[key])
else:
key = key + ":"
for k in filter(lambda s: s.startswith(key), self.keys()):
del(self[k])
def setall(self, key, values):
"""Delete frames of the given type and add frames in 'values'."""
self.delall(key)
for tag in values:
self[tag.HashKey] = tag
def pprint(self):
"""Return tags in a human-readable format.
"Human-readable" is used loosely here. The format is intended
to mirror that used for Vorbis or APEv2 output, e.g.
``TIT2=My Title``
However, ID3 frames can have multiple keys:
``POPM=user@example.org=3 128/255``
"""
frames = list(map(Frame.pprint, self.values()))
frames.sort()
return "\n".join(frames)
def loaded_frame(self, tag):
"""Deprecated; use the add method."""
# turn 2.2 into 2.3/2.4 tags
if len(type(tag).__name__) == 3:
tag = type(tag).__base__(tag)
self[tag.HashKey] = tag
# add = loaded_frame (and vice versa) break applications that
# expect to be able to override loaded_frame (e.g. Quod Libet),
# as does making loaded_frame call add.
def add(self, frame):
"""Add a frame to the tag."""
return self.loaded_frame(frame)
def __load_header(self):
fn = self.filename
data = self.__fullread(10)
id3, vmaj, vrev, flags, size = unpack('>3sBBB4s', data)
self.__flags = flags
self.size = BitPaddedInt(size) + 10
self.version = (2, vmaj, vrev)
if id3 != 'ID3':
raise ID3NoHeaderError("'%s' doesn't start with an ID3 tag" % fn)
if vmaj not in [2, 3, 4]:
raise ID3UnsupportedVersionError("'%s' ID3v2.%d not supported"
% (fn, vmaj))
if self.PEDANTIC:
if not BitPaddedInt.has_valid_padding(size):
raise ValueError("Header size not synchsafe")
if self._V24 <= self.version and (flags & 0x0f):
raise ValueError("'%s' has invalid flags %#02x" % (fn, flags))
elif self._V23 <= self.version < self._V24 and (flags & 0x1f):
raise ValueError("'%s' has invalid flags %#02x" % (fn, flags))
if self.f_extended:
extsize = self.__fullread(4)
if extsize in Frames:
# Some tagger sets the extended header flag but
# doesn't write an extended header; in this case, the
# ID3 data follows immediately. Since no extended
# header is going to be long enough to actually match
# a frame, and if it's *not* a frame we're going to be
# completely lost anyway, this seems to be the most
# correct check.
# http://code.google.com/p/quodlibet/issues/detail?id=126
self.__flags ^= 0x40
self.__extsize = 0
self.__fileobj.seek(-4, 1)
self.__readbytes -= 4
elif self.version >= self._V24:
# "Where the 'Extended header size' is the size of the whole
# extended header, stored as a 32 bit synchsafe integer."
self.__extsize = BitPaddedInt(extsize) - 4
if self.PEDANTIC:
if not BitPaddedInt.has_valid_padding(extsize):
raise ValueError("Extended header size not synchsafe")
else:
# "Where the 'Extended header size', currently 6 or 10 bytes,
# excludes itself."
self.__extsize = unpack('>L', extsize)[0]
if self.__extsize:
self.__extdata = self.__fullread(self.__extsize)
else:
self.__extdata = ""
def __determine_bpi(self, data, frames, EMPTY="\x00" * 10):
if self.version < self._V24:
return int
# have to special case whether to use bitpaddedints here
# spec says to use them, but iTunes has it wrong
# count number of tags found as BitPaddedInt and how far past
o = 0
asbpi = 0
while o < len(data) - 10:
part = data[o:o + 10]
if part == EMPTY:
bpioff = -((len(data) - o) % 10)
break
name, size, flags = unpack('>4sLH', part)
size = BitPaddedInt(size)
o += 10 + size
if name in frames:
asbpi += 1
else:
bpioff = o - len(data)
# count number of tags found as int and how far past
o = 0
asint = 0
while o < len(data) - 10:
part = data[o:o + 10]
if part == EMPTY:
intoff = -((len(data) - o) % 10)
break
name, size, flags = unpack('>4sLH', part)
o += 10 + size
if name in frames:
asint += 1
else:
intoff = o - len(data)
# if more tags as int, or equal and bpi is past and int is not
if asint > asbpi or (asint == asbpi and (bpioff >= 1 and intoff <= 1)):
return int
return BitPaddedInt
def __read_frames(self, data, frames):
if self.version < self._V24 and self.f_unsynch:
try:
data = unsynch.decode(data)
except ValueError:
pass
if self._V23 <= self.version:
bpi = self.__determine_bpi(data, frames)
while data:
header = data[:10]
try:
name, size, flags = unpack('>4sLH', header)
except struct.error:
return # not enough header
if name.strip('\x00') == '':
return
size = bpi(size)
framedata = data[10:10+size]
data = data[10+size:]
if size == 0:
continue # drop empty frames
try:
tag = frames[name]
except KeyError:
if is_valid_frame_id(name):
yield header + framedata
else:
try:
yield self.__load_framedata(tag, flags, framedata)
except NotImplementedError:
yield header + framedata
except ID3JunkFrameError:
pass
elif self._V22 <= self.version:
while data:
header = data[0:6]
try:
name, size = unpack('>3s3s', header)
except struct.error:
return # not enough header
size, = struct.unpack('>L', '\x00'+size)
if name.strip('\x00') == '':
return
framedata = data[6:6+size]
data = data[6+size:]
if size == 0:
continue # drop empty frames
try:
tag = frames[name]
except KeyError:
if is_valid_frame_id(name):
yield header + framedata
else:
try:
yield self.__load_framedata(tag, 0, framedata)
except NotImplementedError:
yield header + framedata
except ID3JunkFrameError:
pass
def __load_framedata(self, tag, flags, framedata):
return tag.fromData(self, flags, framedata)
f_unsynch = property(lambda s: bool(s.__flags & 0x80))
f_extended = property(lambda s: bool(s.__flags & 0x40))
f_experimental = property(lambda s: bool(s.__flags & 0x20))
f_footer = property(lambda s: bool(s.__flags & 0x10))
#f_crc = property(lambda s: bool(s.__extflags & 0x8000))
def save(self, filename=None, v1=1, v2_version=4, v23_sep='/'):
"""Save changes to a file.
If no filename is given, the one most recently loaded is used.
Keyword arguments:
v1 -- if 0, ID3v1 tags will be removed
if 1, ID3v1 tags will be updated but not added
if 2, ID3v1 tags will be created and/or updated
v2 -- version of ID3v2 tags (3 or 4).
By default Mutagen saves ID3v2.4 tags. If you want to save ID3v2.3
tags, you must call method update_to_v23 before saving the file.
v23_sep -- the separator used to join multiple text values
if v2_version == 3. Defaults to '/' but if it's None
will be the ID3v2v2.4 null separator.
The lack of a way to update only an ID3v1 tag is intentional.
"""
if v2_version == 3:
version = self._V23
elif v2_version == 4:
version = self._V24
else:
raise ValueError("Only 3 or 4 allowed for v2_version")
# Sort frames by 'importance'
order = ["TIT2", "TPE1", "TRCK", "TALB", "TPOS", "TDRC", "TCON"]
order = dict(zip(order, range(len(order))))
last = len(order)
frames = self.items()
frames.sort(lambda a, b: cmp(order.get(a[0][:4], last),
order.get(b[0][:4], last)))
framedata = [self.__save_frame(frame, version=version, v23_sep=v23_sep)
for (key, frame) in frames]
# only write unknown frames if they were loaded from the version
# we are saving with or upgraded to it
if self.__unknown_version == version:
framedata.extend([data for data in self.unknown_frames
if len(data) > 10])
if not framedata:
try:
self.delete(filename)
except EnvironmentError, err:
from errno import ENOENT
if err.errno != ENOENT:
raise
return
framedata = ''.join(framedata)
framesize = len(framedata)
if filename is None:
filename = self.filename
try:
f = open(filename, 'rb+')
except IOError, err:
from errno import ENOENT
if err.errno != ENOENT:
raise
f = open(filename, 'ab') # create, then reopen
f = open(filename, 'rb+')
try:
idata = f.read(10)
try:
id3, vmaj, vrev, flags, insize = unpack('>3sBBB4s', idata)
except struct.error:
id3, insize = '', 0
insize = BitPaddedInt(insize)
if id3 != 'ID3':
insize = -10
if insize >= framesize:
outsize = insize
else:
outsize = (framesize + 1023) & ~0x3FF
framedata += '\x00' * (outsize - framesize)
framesize = BitPaddedInt.to_str(outsize, width=4)
flags = 0
header = pack('>3sBBB4s', 'ID3', v2_version, 0, flags, framesize)
data = header + framedata
if (insize < outsize):
insert_bytes(f, outsize-insize, insize+10)
f.seek(0)
f.write(data)
try:
f.seek(-128, 2)
except IOError, err:
# If the file is too small, that's OK - it just means
# we're certain it doesn't have a v1 tag.
from errno import EINVAL
if err.errno != EINVAL:
# If we failed to see for some other reason, bail out.
raise
# Since we're sure this isn't a v1 tag, don't read it.
f.seek(0, 2)
data = f.read(128)
try:
idx = data.index("TAG")
except ValueError:
offset = 0
has_v1 = False
else:
offset = idx - len(data)
has_v1 = True
f.seek(offset, 2)
if v1 == 1 and has_v1 or v1 == 2:
f.write(MakeID3v1(self))
else:
f.truncate()
finally:
f.close()
def delete(self, filename=None, delete_v1=True, delete_v2=True):
"""Remove tags from a file.
If no filename is given, the one most recently loaded is used.
Keyword arguments:
* delete_v1 -- delete any ID3v1 tag
* delete_v2 -- delete any ID3v2 tag
"""
if filename is None:
filename = self.filename
delete(filename, delete_v1, delete_v2)
self.clear()
def __save_frame(self, frame, name=None, version=_V24, v23_sep=None):
flags = 0
if self.PEDANTIC and isinstance(frame, TextFrame):
if len(str(frame)) == 0:
return ''
if version == self._V23:
framev23 = frame._get_v23_frame(sep=v23_sep)
framedata = framev23._writeData()
else:
framedata = frame._writeData()
usize = len(framedata)
if usize > 2048:
# Disabled as this causes iTunes and other programs
# to fail to find these frames, which usually includes
# e.g. APIC.
#framedata = BitPaddedInt.to_str(usize) + framedata.encode('zlib')
#flags |= Frame.FLAG24_COMPRESS | Frame.FLAG24_DATALEN
pass
if version == self._V24:
bits = 7
elif version == self._V23:
bits = 8
else:
raise ValueError
datasize = BitPaddedInt.to_str(len(framedata), width=4, bits=bits)
header = pack('>4s4sH', name or type(frame).__name__, datasize, flags)
return header + framedata
def __update_common(self):
"""Updates done by both v23 and v24 update"""
if "TCON" in self:
# Get rid of "(xx)Foobr" format.
self["TCON"].genres = self["TCON"].genres
if self.version < self._V23:
# ID3v2.2 PIC frames are slightly different.
pics = self.getall("APIC")
mimes = {"PNG": "image/png", "JPG": "image/jpeg"}
self.delall("APIC")
for pic in pics:
newpic = APIC(
encoding=pic.encoding, mime=mimes.get(pic.mime, pic.mime),
type=pic.type, desc=pic.desc, data=pic.data)
self.add(newpic)
# ID3v2.2 LNK frames are just way too different to upgrade.
self.delall("LINK")
def update_to_v24(self):
"""Convert older tags into an ID3v2.4 tag.
This updates old ID3v2 frames to ID3v2.4 ones (e.g. TYER to
TDRC). If you intend to save tags, you must call this function
at some point; it is called by default when loading the tag.
"""
self.__update_common()
if self.__unknown_version == (2, 3, 0):
# convert unknown 2.3 frames (flags/size) to 2.4
converted = []
for frame in self.unknown_frames:
try:
name, size, flags = unpack('>4sLH', frame[:10])
frame = BinaryFrame.fromData(self, flags, frame[10:])
except (struct.error, error):
continue
converted.append(self.__save_frame(frame, name=name))
self.unknown_frames[:] = converted
self.__unknown_version = (2, 4, 0)
# TDAT, TYER, and TIME have been turned into TDRC.
try:
if str(self.get("TYER", "")).strip("\x00"):
date = str(self.pop("TYER"))
if str(self.get("TDAT", "")).strip("\x00"):
dat = str(self.pop("TDAT"))
date = "%s-%s-%s" % (date, dat[2:], dat[:2])
if str(self.get("TIME", "")).strip("\x00"):
time = str(self.pop("TIME"))
date += "T%s:%s:00" % (time[:2], time[2:])
if "TDRC" not in self:
self.add(TDRC(encoding=0, text=date))
except UnicodeDecodeError:
# Old ID3 tags have *lots* of Unicode problems, so if TYER
# is bad, just chuck the frames.
pass
# TORY can be the first part of a TDOR.
if "TORY" in self:
f = self.pop("TORY")
if "TDOR" not in self:
try:
self.add(TDOR(encoding=0, text=str(f)))
except UnicodeDecodeError:
pass
# IPLS is now TIPL.
if "IPLS" in self:
f = self.pop("IPLS")
if "TIPL" not in self:
self.add(TIPL(encoding=f.encoding, people=f.people))
# These can't be trivially translated to any ID3v2.4 tags, or
# should have been removed already.
for key in ["RVAD", "EQUA", "TRDA", "TSIZ", "TDAT", "TIME", "CRM"]:
if key in self:
del(self[key])
def update_to_v23(self):
"""Convert older (and newer) tags into an ID3v2.3 tag.
This updates incompatible ID3v2 frames to ID3v2.3 ones. If you
intend to save tags as ID3v2.3, you must call this function
at some point.
If you want to to go off spec and include some v2.4 frames
in v2.3, remove them before calling this and add them back afterwards.
"""
self.__update_common()
# we could downgrade unknown v2.4 frames here, but given that
# the main reason to save v2.3 is compatibility and this
# might increase the chance of some parser breaking.. better not
# TMCL, TIPL -> TIPL
if "TIPL" in self or "TMCL" in self:
people = []
if "TIPL" in self:
f = self.pop("TIPL")
people.extend(f.people)
if "TMCL" in self:
f = self.pop("TMCL")
people.extend(f.people)
if "IPLS" not in self:
self.add(IPLS(encoding=f.encoding, people=people))
# TDOR -> TORY
if "TDOR" in self:
f = self.pop("TDOR")
if f.text:
d = f.text[0]
if d.year and "TORY" not in self:
self.add(TORY(encoding=f.encoding, text="%04d" % d.year))
# TDRC -> TYER, TDAT, TIME
if "TDRC" in self:
f = self.pop("TDRC")
if f.text:
d = f.text[0]
if d.year and "TYER" not in self:
self.add(TYER(encoding=f.encoding, text="%04d" % d.year))
if d.month and d.day and "TDAT" not in self:
self.add(TDAT(encoding=f.encoding,
text="%02d%02d" % (d.day, d.month)))
if d.hour and d.minute and "TIME" not in self:
self.add(TIME(encoding=f.encoding,
text="%02d%02d" % (d.hour, d.minute)))
# New frames added in v2.4
v24_frames = [
'ASPI', 'EQU2', 'RVA2', 'SEEK', 'SIGN', 'TDEN', 'TDOR',
'TDRC', 'TDRL', 'TDTG', 'TIPL', 'TMCL', 'TMOO', 'TPRO',
'TSOA', 'TSOP', 'TSOT', 'TSST',
]
for key in v24_frames:
if key in self:
del(self[key])
def delete(filename, delete_v1=True, delete_v2=True):
"""Remove tags from a file.
Keyword arguments:
* delete_v1 -- delete any ID3v1 tag
* delete_v2 -- delete any ID3v2 tag
"""
f = open(filename, 'rb+')
if delete_v1:
try:
f.seek(-128, 2)
except IOError:
pass
else:
if f.read(3) == "TAG":
f.seek(-128, 2)
f.truncate()
# technically an insize=0 tag is invalid, but we delete it anyway
# (primarily because we used to write it)
if delete_v2:
f.seek(0, 0)
idata = f.read(10)
try:
id3, vmaj, vrev, flags, insize = unpack('>3sBBB4s', idata)
except struct.error:
id3, insize = '', -1
insize = BitPaddedInt(insize)
if id3 == 'ID3' and insize >= 0:
delete_bytes(f, insize + 10, 0)
# support open(filename) as interface
Open = ID3
# ID3v1.1 support.
def ParseID3v1(string):
"""Parse an ID3v1 tag, returning a list of ID3v2.4 frames."""
try:
string = string[string.index("TAG"):]
except ValueError:
return None
if 128 < len(string) or len(string) < 124:
return None
# Issue #69 - Previous versions of Mutagen, when encountering
# out-of-spec TDRC and TYER frames of less than four characters,
# wrote only the characters available - e.g. "1" or "" - into the
# year field. To parse those, reduce the size of the year field.
# Amazingly, "0s" works as a struct format string.
unpack_fmt = "3s30s30s30s%ds29sBB" % (len(string) - 124)
try:
tag, title, artist, album, year, comment, track, genre = unpack(
unpack_fmt, string)
except StructError:
return None
if tag != "TAG":
return None
def fix(string):
return string.split("\x00")[0].strip().decode('latin1')
title, artist, album, year, comment = map(
fix, [title, artist, album, year, comment])
frames = {}
if title:
frames["TIT2"] = TIT2(encoding=0, text=title)
if artist:
frames["TPE1"] = TPE1(encoding=0, text=[artist])
if album:
frames["TALB"] = TALB(encoding=0, text=album)
if year:
frames["TDRC"] = TDRC(encoding=0, text=year)
if comment:
frames["COMM"] = COMM(
encoding=0, lang="eng", desc="ID3v1 Comment", text=comment)
# Don't read a track number if it looks like the comment was
# padded with spaces instead of nulls (thanks, WinAmp).
if track and (track != 32 or string[-3] == '\x00'):
frames["TRCK"] = TRCK(encoding=0, text=str(track))
if genre != 255:
frames["TCON"] = TCON(encoding=0, text=str(genre))
return frames
def MakeID3v1(id3):
"""Return an ID3v1.1 tag string from a dict of ID3v2.4 frames."""
v1 = {}
for v2id, name in {"TIT2": "title", "TPE1": "artist",
"TALB": "album"}.items():
if v2id in id3:
text = id3[v2id].text[0].encode('latin1', 'replace')[:30]
else:
text = ""
v1[name] = text + ("\x00" * (30 - len(text)))
if "COMM" in id3:
cmnt = id3["COMM"].text[0].encode('latin1', 'replace')[:28]
else:
cmnt = ""
v1["comment"] = cmnt + ("\x00" * (29 - len(cmnt)))
if "TRCK" in id3:
try:
v1["track"] = chr(+id3["TRCK"])
except ValueError:
v1["track"] = "\x00"
else:
v1["track"] = "\x00"
if "TCON" in id3:
try:
genre = id3["TCON"].genres[0]
except IndexError:
pass
else:
if genre in TCON.GENRES:
v1["genre"] = chr(TCON.GENRES.index(genre))
if "genre" not in v1:
v1["genre"] = "\xff"
if "TDRC" in id3:
year = str(id3["TDRC"])
elif "TYER" in id3:
year = str(id3["TYER"])
else:
year = ""
v1["year"] = (year + "\x00\x00\x00\x00")[:4]
return ("TAG%(title)s%(artist)s%(album)s%(year)s%(comment)s"
"%(track)s%(genre)s") % v1
class ID3FileType(mutagen.FileType):
"""An unknown type of file with ID3 tags."""
ID3 = ID3
class _Info(object):
length = 0
def __init__(self, fileobj, offset):
pass
@staticmethod
def pprint():
return "Unknown format with ID3 tag"
@staticmethod
def score(filename, fileobj, header):
return header.startswith("ID3")
def add_tags(self, ID3=None):
"""Add an empty ID3 tag to the file.
A custom tag reader may be used in instead of the default
mutagen.id3.ID3 object, e.g. an EasyID3 reader.
"""
if ID3 is None:
ID3 = self.ID3
if self.tags is None:
self.ID3 = ID3
self.tags = ID3()
else:
raise error("an ID3 tag already exists")
def load(self, filename, ID3=None, **kwargs):
"""Load stream and tag information from a file.
A custom tag reader may be used in instead of the default
mutagen.id3.ID3 object, e.g. an EasyID3 reader.
"""
if ID3 is None:
ID3 = self.ID3
else:
# If this was initialized with EasyID3, remember that for
# when tags are auto-instantiated in add_tags.
self.ID3 = ID3
self.filename = filename
try:
self.tags = ID3(filename, **kwargs)
except error:
self.tags = None
if self.tags is not None:
try:
offset = self.tags.size
except AttributeError:
offset = None
else:
offset = None
try:
fileobj = open(filename, "rb")
self.info = self._Info(fileobj, offset)
finally:
fileobj.close()

538
libs/mutagen/m4a.py Normal file
View file

@ -0,0 +1,538 @@
# Copyright 2006 Joe Wreschnig
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Read and write MPEG-4 audio files with iTunes metadata.
This module will read MPEG-4 audio information and metadata,
as found in Apple's M4A (aka MP4, M4B, M4P) files.
There is no official specification for this format. The source code
for TagLib, FAAD, and various MPEG specifications at
http://developer.apple.com/documentation/QuickTime/QTFF/,
http://www.geocities.com/xhelmboyx/quicktime/formats/mp4-layout.txt,
and http://wiki.multimedia.cx/index.php?title=Apple_QuickTime were all
consulted.
This module does not support 64 bit atom sizes, and so will not
work on metadata over 4GB.
"""
import struct
import sys
from cStringIO import StringIO
from mutagen import FileType, Metadata
from mutagen._constants import GENRES
from mutagen._util import cdata, insert_bytes, delete_bytes, DictProxy
class error(IOError):
pass
class M4AMetadataError(error):
pass
class M4AStreamInfoError(error):
pass
class M4AMetadataValueError(ValueError, M4AMetadataError):
pass
import warnings
warnings.warn(
"mutagen.m4a is deprecated; use mutagen.mp4 instead.", DeprecationWarning)
# This is not an exhaustive list of container atoms, but just the
# ones this module needs to peek inside.
_CONTAINERS = ["moov", "udta", "trak", "mdia", "meta", "ilst",
"stbl", "minf", "stsd"]
_SKIP_SIZE = {"meta": 4}
__all__ = ['M4A', 'Open', 'delete', 'M4ACover']
class M4ACover(str):
"""A cover artwork.
Attributes:
imageformat -- format of the image (either FORMAT_JPEG or FORMAT_PNG)
"""
FORMAT_JPEG = 0x0D
FORMAT_PNG = 0x0E
def __new__(cls, data, imageformat=None):
self = str.__new__(cls, data)
if imageformat is None:
imageformat = M4ACover.FORMAT_JPEG
self.imageformat = imageformat
try:
self.format
except AttributeError:
self.format = imageformat
return self
class Atom(object):
"""An individual atom.
Attributes:
children -- list child atoms (or None for non-container atoms)
length -- length of this atom, including length and name
name -- four byte name of the atom, as a str
offset -- location in the constructor-given fileobj of this atom
This structure should only be used internally by Mutagen.
"""
children = None
def __init__(self, fileobj):
self.offset = fileobj.tell()
self.length, self.name = struct.unpack(">I4s", fileobj.read(8))
if self.length == 1:
raise error("64 bit atom sizes are not supported")
elif self.length < 8:
return
if self.name in _CONTAINERS:
self.children = []
fileobj.seek(_SKIP_SIZE.get(self.name, 0), 1)
while fileobj.tell() < self.offset + self.length:
self.children.append(Atom(fileobj))
else:
fileobj.seek(self.offset + self.length, 0)
@staticmethod
def render(name, data):
"""Render raw atom data."""
# this raises OverflowError if Py_ssize_t can't handle the atom data
size = len(data) + 8
if size <= 0xFFFFFFFF:
return struct.pack(">I4s", size, name) + data
else:
return struct.pack(">I4sQ", 1, name, size + 8) + data
def __getitem__(self, remaining):
"""Look up a child atom, potentially recursively.
e.g. atom['udta', 'meta'] => <Atom name='meta' ...>
"""
if not remaining:
return self
elif self.children is None:
raise KeyError("%r is not a container" % self.name)
for child in self.children:
if child.name == remaining[0]:
return child[remaining[1:]]
else:
raise KeyError("%r not found" % remaining[0])
def __repr__(self):
klass = self.__class__.__name__
if self.children is None:
return "<%s name=%r length=%r offset=%r>" % (
klass, self.name, self.length, self.offset)
else:
children = "\n".join([" " + line for child in self.children
for line in repr(child).splitlines()])
return "<%s name=%r length=%r offset=%r\n%s>" % (
klass, self.name, self.length, self.offset, children)
class Atoms(object):
"""Root atoms in a given file.
Attributes:
atoms -- a list of top-level atoms as Atom objects
This structure should only be used internally by Mutagen.
"""
def __init__(self, fileobj):
self.atoms = []
fileobj.seek(0, 2)
end = fileobj.tell()
fileobj.seek(0)
while fileobj.tell() < end:
self.atoms.append(Atom(fileobj))
def path(self, *names):
"""Look up and return the complete path of an atom.
For example, atoms.path('moov', 'udta', 'meta') will return a
list of three atoms, corresponding to the moov, udta, and meta
atoms.
"""
path = [self]
for name in names:
path.append(path[-1][name, ])
return path[1:]
def __getitem__(self, names):
"""Look up a child atom.
'names' may be a list of atoms (['moov', 'udta']) or a string
specifying the complete path ('moov.udta').
"""
if isinstance(names, basestring):
names = names.split(".")
for child in self.atoms:
if child.name == names[0]:
return child[names[1:]]
else:
raise KeyError("%s not found" % names[0])
def __repr__(self):
return "\n".join([repr(child) for child in self.atoms])
class M4ATags(DictProxy, Metadata):
"""Dictionary containing Apple iTunes metadata list key/values.
Keys are four byte identifiers, except for freeform ('----')
keys. Values are usually unicode strings, but some atoms have a
special structure:
cpil -- boolean
trkn, disk -- tuple of 16 bit ints (current, total)
tmpo -- 16 bit int
covr -- list of M4ACover objects (which are tagged strs)
gnre -- not supported. Use '\\xa9gen' instead.
The freeform '----' frames use a key in the format '----:mean:name'
where 'mean' is usually 'com.apple.iTunes' and 'name' is a unique
identifier for this frame. The value is a str, but is probably
text that can be decoded as UTF-8.
M4A tag data cannot exist outside of the structure of an M4A file,
so this class should not be manually instantiated.
Unknown non-text tags are removed.
"""
def load(self, atoms, fileobj):
try:
ilst = atoms["moov.udta.meta.ilst"]
except KeyError, key:
raise M4AMetadataError(key)
for atom in ilst.children:
fileobj.seek(atom.offset + 8)
data = fileobj.read(atom.length - 8)
parse = self.__atoms.get(atom.name, (M4ATags.__parse_text,))[0]
parse(self, atom, data)
@staticmethod
def __key_sort(item1, item2):
(key1, v1) = item1
(key2, v2) = item2
# iTunes always writes the tags in order of "relevance", try
# to copy it as closely as possible.
order = ["\xa9nam", "\xa9ART", "\xa9wrt", "\xa9alb",
"\xa9gen", "gnre", "trkn", "disk",
"\xa9day", "cpil", "tmpo", "\xa9too",
"----", "covr", "\xa9lyr"]
order = dict(zip(order, range(len(order))))
last = len(order)
# If there's no key-based way to distinguish, order by length.
# If there's still no way, go by string comparison on the
# values, so we at least have something determinstic.
return (cmp(order.get(key1[:4], last), order.get(key2[:4], last)) or
cmp(len(v1), len(v2)) or cmp(v1, v2))
def save(self, filename):
"""Save the metadata to the given filename."""
values = []
items = self.items()
items.sort(self.__key_sort)
for key, value in items:
render = self.__atoms.get(
key[:4], (None, M4ATags.__render_text))[1]
values.append(render(self, key, value))
data = Atom.render("ilst", "".join(values))
# Find the old atoms.
fileobj = open(filename, "rb+")
try:
atoms = Atoms(fileobj)
moov = atoms["moov"]
if moov != atoms.atoms[-1]:
# "Free" the old moov block. Something in the mdat
# block is not happy when its offset changes and it
# won't play back. So, rather than try to figure that
# out, just move the moov atom to the end of the file.
offset = self.__move_moov(fileobj, moov)
else:
offset = 0
try:
path = atoms.path("moov", "udta", "meta", "ilst")
except KeyError:
self.__save_new(fileobj, atoms, data, offset)
else:
self.__save_existing(fileobj, atoms, path, data, offset)
finally:
fileobj.close()
def __move_moov(self, fileobj, moov):
fileobj.seek(moov.offset)
data = fileobj.read(moov.length)
fileobj.seek(moov.offset)
free = Atom.render("free", "\x00" * (moov.length - 8))
fileobj.write(free)
fileobj.seek(0, 2)
# Figure out how far we have to shift all our successive
# seek calls, relative to what the atoms say.
old_end = fileobj.tell()
fileobj.write(data)
return old_end - moov.offset
def __save_new(self, fileobj, atoms, ilst, offset):
hdlr = Atom.render("hdlr", "\x00" * 8 + "mdirappl" + "\x00" * 9)
meta = Atom.render("meta", "\x00\x00\x00\x00" + hdlr + ilst)
moov, udta = atoms.path("moov", "udta")
insert_bytes(fileobj, len(meta), udta.offset + offset + 8)
fileobj.seek(udta.offset + offset + 8)
fileobj.write(meta)
self.__update_parents(fileobj, [moov, udta], len(meta), offset)
def __save_existing(self, fileobj, atoms, path, data, offset):
# Replace the old ilst atom.
ilst = path.pop()
delta = len(data) - ilst.length
fileobj.seek(ilst.offset + offset)
if delta > 0:
insert_bytes(fileobj, delta, ilst.offset + offset)
elif delta < 0:
delete_bytes(fileobj, -delta, ilst.offset + offset)
fileobj.seek(ilst.offset + offset)
fileobj.write(data)
self.__update_parents(fileobj, path, delta, offset)
def __update_parents(self, fileobj, path, delta, offset):
# Update all parent atoms with the new size.
for atom in path:
fileobj.seek(atom.offset + offset)
size = cdata.uint_be(fileobj.read(4)) + delta
fileobj.seek(atom.offset + offset)
fileobj.write(cdata.to_uint_be(size))
def __render_data(self, key, flags, data):
data = struct.pack(">2I", flags, 0) + data
return Atom.render(key, Atom.render("data", data))
def __parse_freeform(self, atom, data):
try:
fileobj = StringIO(data)
mean_length = cdata.uint_be(fileobj.read(4))
# skip over 8 bytes of atom name, flags
mean = fileobj.read(mean_length - 4)[8:]
name_length = cdata.uint_be(fileobj.read(4))
name = fileobj.read(name_length - 4)[8:]
value_length = cdata.uint_be(fileobj.read(4))
# Name, flags, and reserved bytes
value = fileobj.read(value_length - 4)[12:]
except struct.error:
# Some ---- atoms have no data atom, I have no clue why
# they actually end up in the file.
pass
else:
self["%s:%s:%s" % (atom.name, mean, name)] = value
def __render_freeform(self, key, value):
dummy, mean, name = key.split(":", 2)
mean = struct.pack(">I4sI", len(mean) + 12, "mean", 0) + mean
name = struct.pack(">I4sI", len(name) + 12, "name", 0) + name
value = struct.pack(">I4s2I", len(value) + 16, "data", 0x1, 0) + value
final = mean + name + value
return Atom.render("----", final)
def __parse_pair(self, atom, data):
self[atom.name] = struct.unpack(">2H", data[18:22])
def __render_pair(self, key, value):
track, total = value
if 0 <= track < 1 << 16 and 0 <= total < 1 << 16:
data = struct.pack(">4H", 0, track, total, 0)
return self.__render_data(key, 0, data)
else:
raise M4AMetadataValueError("invalid numeric pair %r" % (value,))
def __render_pair_no_trailing(self, key, value):
track, total = value
if 0 <= track < 1 << 16 and 0 <= total < 1 << 16:
data = struct.pack(">3H", 0, track, total)
return self.__render_data(key, 0, data)
else:
raise M4AMetadataValueError("invalid numeric pair %r" % (value,))
def __parse_genre(self, atom, data):
# Translate to a freeform genre.
genre = cdata.short_be(data[16:18])
if "\xa9gen" not in self:
try:
self["\xa9gen"] = GENRES[genre - 1]
except IndexError:
pass
def __parse_tempo(self, atom, data):
self[atom.name] = cdata.short_be(data[16:18])
def __render_tempo(self, key, value):
if 0 <= value < 1 << 16:
return self.__render_data(key, 0x15, cdata.to_ushort_be(value))
else:
raise M4AMetadataValueError("invalid short integer %r" % value)
def __parse_compilation(self, atom, data):
try:
self[atom.name] = bool(ord(data[16:17]))
except TypeError:
self[atom.name] = False
def __render_compilation(self, key, value):
return self.__render_data(key, 0x15, chr(bool(value)))
def __parse_cover(self, atom, data):
length, name, imageformat = struct.unpack(">I4sI", data[:12])
if name != "data":
raise M4AMetadataError(
"unexpected atom %r inside 'covr'" % name)
if imageformat not in (M4ACover.FORMAT_JPEG, M4ACover.FORMAT_PNG):
imageformat = M4ACover.FORMAT_JPEG
self[atom.name] = M4ACover(data[16:length], imageformat)
def __render_cover(self, key, value):
try:
imageformat = value.imageformat
except AttributeError:
imageformat = M4ACover.FORMAT_JPEG
data = Atom.render("data", struct.pack(">2I", imageformat, 0) + value)
return Atom.render(key, data)
def __parse_text(self, atom, data):
flags = cdata.uint_be(data[8:12])
if flags == 1:
self[atom.name] = data[16:].decode('utf-8', 'replace')
def __render_text(self, key, value):
return self.__render_data(key, 0x1, value.encode('utf-8'))
def delete(self, filename):
self.clear()
self.save(filename)
__atoms = {
"----": (__parse_freeform, __render_freeform),
"trkn": (__parse_pair, __render_pair),
"disk": (__parse_pair, __render_pair_no_trailing),
"gnre": (__parse_genre, None),
"tmpo": (__parse_tempo, __render_tempo),
"cpil": (__parse_compilation, __render_compilation),
"covr": (__parse_cover, __render_cover),
}
def pprint(self):
values = []
for key, value in self.iteritems():
key = key.decode('latin1')
try:
values.append("%s=%s" % (key, value))
except UnicodeDecodeError:
values.append("%s=[%d bytes of data]" % (key, len(value)))
return "\n".join(values)
class M4AInfo(object):
"""MPEG-4 stream information.
Attributes:
bitrate -- bitrate in bits per second, as an int
length -- file length in seconds, as a float
"""
bitrate = 0
def __init__(self, atoms, fileobj):
hdlr = atoms["moov.trak.mdia.hdlr"]
fileobj.seek(hdlr.offset)
if "soun" not in fileobj.read(hdlr.length):
raise M4AStreamInfoError("track has no audio data")
mdhd = atoms["moov.trak.mdia.mdhd"]
fileobj.seek(mdhd.offset)
data = fileobj.read(mdhd.length)
if ord(data[8]) == 0:
offset = 20
fmt = ">2I"
else:
offset = 28
fmt = ">IQ"
end = offset + struct.calcsize(fmt)
unit, length = struct.unpack(fmt, data[offset:end])
self.length = float(length) / unit
try:
atom = atoms["moov.trak.mdia.minf.stbl.stsd"]
fileobj.seek(atom.offset)
data = fileobj.read(atom.length)
self.bitrate = cdata.uint_be(data[-17:-13])
except (ValueError, KeyError):
# Bitrate values are optional.
pass
def pprint(self):
return "MPEG-4 audio, %.2f seconds, %d bps" % (
self.length, self.bitrate)
class M4A(FileType):
"""An MPEG-4 audio file, probably containing AAC.
If more than one track is present in the file, the first is used.
Only audio ('soun') tracks will be read.
"""
_mimes = ["audio/mp4", "audio/x-m4a", "audio/mpeg4", "audio/aac"]
def load(self, filename):
self.filename = filename
fileobj = open(filename, "rb")
try:
atoms = Atoms(fileobj)
try:
self.info = M4AInfo(atoms, fileobj)
except StandardError, err:
raise M4AStreamInfoError, err, sys.exc_info()[2]
try:
self.tags = M4ATags(atoms, fileobj)
except M4AMetadataError:
self.tags = None
except StandardError, err:
raise M4AMetadataError, err, sys.exc_info()[2]
finally:
fileobj.close()
def add_tags(self):
self.tags = M4ATags()
@staticmethod
def score(filename, fileobj, header):
return ("ftyp" in header) + ("mp4" in header)
Open = M4A
def delete(filename):
"""Remove tags from a file."""
M4A(filename).delete()

84
libs/mutagen/monkeysaudio.py Normal file
View file

@ -0,0 +1,84 @@
# A Monkey's Audio (APE) reader/tagger
#
# Copyright 2006 Lukas Lalinsky <lalinsky@gmail.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Monkey's Audio streams with APEv2 tags.
Monkey's Audio is a very efficient lossless audio compressor developed
by Matt Ashland.
For more information, see http://www.monkeysaudio.com/.
"""
__all__ = ["MonkeysAudio", "Open", "delete"]
import struct
from mutagen.apev2 import APEv2File, error, delete
from mutagen._util import cdata
class MonkeysAudioHeaderError(error):
pass
class MonkeysAudioInfo(object):
"""Monkey's Audio stream information.
Attributes:
* channels -- number of audio channels
* length -- file length in seconds, as a float
* sample_rate -- audio sampling rate in Hz
* bits_per_sample -- bits per sample
* version -- Monkey's Audio stream version, as a float (eg: 3.99)
"""
def __init__(self, fileobj):
header = fileobj.read(76)
if len(header) != 76 or not header.startswith("MAC "):
raise MonkeysAudioHeaderError("not a Monkey's Audio file")
self.version = cdata.ushort_le(header[4:6])
if self.version >= 3980:
(blocks_per_frame, final_frame_blocks, total_frames,
self.bits_per_sample, self.channels,
self.sample_rate) = struct.unpack("<IIIHHI", header[56:76])
else:
compression_level = cdata.ushort_le(header[6:8])
self.channels, self.sample_rate = struct.unpack(
"<HI", header[10:16])
total_frames, final_frame_blocks = struct.unpack(
"<II", header[24:32])
if self.version >= 3950:
blocks_per_frame = 73728 * 4
elif self.version >= 3900 or (self.version >= 3800 and
compression_level == 4):
blocks_per_frame = 73728
else:
blocks_per_frame = 9216
self.version /= 1000.0
self.length = 0.0
if self.sample_rate != 0 and total_frames > 0:
total_blocks = ((total_frames - 1) * blocks_per_frame +
final_frame_blocks)
self.length = float(total_blocks) / self.sample_rate
def pprint(self):
return "Monkey's Audio %.2f, %.2f seconds, %d Hz" % (
self.version, self.length, self.sample_rate)
class MonkeysAudio(APEv2File):
_Info = MonkeysAudioInfo
_mimes = ["audio/ape", "audio/x-ape"]
@staticmethod
def score(filename, fileobj, header):
return header.startswith("MAC ") + filename.lower().endswith(".ape")
Open = MonkeysAudio

275
libs/mutagen/mp3.py Normal file
View file

@ -0,0 +1,275 @@
# MP3 stream header information support for Mutagen.
# Copyright 2006 Joe Wreschnig
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of version 2 of the GNU General Public License as
# published by the Free Software Foundation.
"""MPEG audio stream information and tags."""
import os
import struct
from mutagen.id3 import ID3FileType, BitPaddedInt, delete
__all__ = ["MP3", "Open", "delete", "MP3"]
class error(RuntimeError):
pass
class HeaderNotFoundError(error, IOError):
pass
class InvalidMPEGHeader(error, IOError):
pass
# Mode values.
STEREO, JOINTSTEREO, DUALCHANNEL, MONO = range(4)
class MPEGInfo(object):
"""MPEG audio stream information
Parse information about an MPEG audio file. This also reads the
Xing VBR header format.
This code was implemented based on the format documentation at
http://mpgedit.org/mpgedit/mpeg_format/mpeghdr.htm.
Useful attributes:
* length -- audio length, in seconds
* bitrate -- audio bitrate, in bits per second
* sketchy -- if true, the file may not be valid MPEG audio
Useless attributes:
* version -- MPEG version (1, 2, 2.5)
* layer -- 1, 2, or 3
* mode -- One of STEREO, JOINTSTEREO, DUALCHANNEL, or MONO (0-3)
* protected -- whether or not the file is "protected"
* padding -- whether or not audio frames are padded
* sample_rate -- audio sample rate, in Hz
"""
# Map (version, layer) tuples to bitrates.
__BITRATE = {
(1, 1): range(0, 480, 32),
(1, 2): [0, 32, 48, 56, 64, 80, 96, 112, 128,
160, 192, 224, 256, 320, 384],
(1, 3): [0, 32, 40, 48, 56, 64, 80, 96, 112,
128, 160, 192, 224, 256, 320],
(2, 1): [0, 32, 48, 56, 64, 80, 96, 112, 128,
144, 160, 176, 192, 224, 256],
(2, 2): [0, 8, 16, 24, 32, 40, 48, 56, 64,
80, 96, 112, 128, 144, 160],
}
__BITRATE[(2, 3)] = __BITRATE[(2, 2)]
for i in range(1, 4):
__BITRATE[(2.5, i)] = __BITRATE[(2, i)]
# Map version to sample rates.
__RATES = {
1: [44100, 48000, 32000],
2: [22050, 24000, 16000],
2.5: [11025, 12000, 8000]
}
sketchy = False
def __init__(self, fileobj, offset=None):
"""Parse MPEG stream information from a file-like object.
If an offset argument is given, it is used to start looking
for stream information and Xing headers; otherwise, ID3v2 tags
will be skipped automatically. A correct offset can make
loading files significantly faster.
"""
try:
size = os.path.getsize(fileobj.name)
except (IOError, OSError, AttributeError):
fileobj.seek(0, 2)
size = fileobj.tell()
# If we don't get an offset, try to skip an ID3v2 tag.
if offset is None:
fileobj.seek(0, 0)
idata = fileobj.read(10)
try:
id3, insize = struct.unpack('>3sxxx4s', idata)
except struct.error:
id3, insize = '', 0
insize = BitPaddedInt(insize)
if id3 == 'ID3' and insize > 0:
offset = insize + 10
else:
offset = 0
# Try to find two valid headers (meaning, very likely MPEG data)
# at the given offset, 30% through the file, 60% through the file,
# and 90% through the file.
for i in [offset, 0.3 * size, 0.6 * size, 0.9 * size]:
try:
self.__try(fileobj, int(i), size - offset)
except error:
pass
else:
break
# If we can't find any two consecutive frames, try to find just
# one frame back at the original offset given.
else:
self.__try(fileobj, offset, size - offset, False)
self.sketchy = True
def __try(self, fileobj, offset, real_size, check_second=True):
# This is going to be one really long function; bear with it,
# because there's not really a sane point to cut it up.
fileobj.seek(offset, 0)
# We "know" we have an MPEG file if we find two frames that look like
# valid MPEG data. If we can't find them in 32k of reads, something
# is horribly wrong (the longest frame can only be about 4k). This
# is assuming the offset didn't lie.
data = fileobj.read(32768)
frame_1 = data.find("\xff")
while 0 <= frame_1 <= len(data) - 4:
frame_data = struct.unpack(">I", data[frame_1:frame_1 + 4])[0]
if (frame_data >> 16) & 0xE0 != 0xE0:
frame_1 = data.find("\xff", frame_1 + 2)
else:
version = (frame_data >> 19) & 0x3
layer = (frame_data >> 17) & 0x3
protection = (frame_data >> 16) & 0x1
bitrate = (frame_data >> 12) & 0xF
sample_rate = (frame_data >> 10) & 0x3
padding = (frame_data >> 9) & 0x1
#private = (frame_data >> 8) & 0x1
self.mode = (frame_data >> 6) & 0x3
#mode_extension = (frame_data >> 4) & 0x3
#copyright = (frame_data >> 3) & 0x1
#original = (frame_data >> 2) & 0x1
#emphasis = (frame_data >> 0) & 0x3
if (version == 1 or layer == 0 or sample_rate == 0x3 or
bitrate == 0 or bitrate == 0xF):
frame_1 = data.find("\xff", frame_1 + 2)
else:
break
else:
raise HeaderNotFoundError("can't sync to an MPEG frame")
# There is a serious problem here, which is that many flags
# in an MPEG header are backwards.
self.version = [2.5, None, 2, 1][version]
self.layer = 4 - layer
self.protected = not protection
self.padding = bool(padding)
self.bitrate = self.__BITRATE[(self.version, self.layer)][bitrate]
self.bitrate *= 1000
self.sample_rate = self.__RATES[self.version][sample_rate]
if self.layer == 1:
frame_length = (12 * self.bitrate / self.sample_rate + padding) * 4
frame_size = 384
elif self.version >= 2 and self.layer == 3:
frame_length = 72 * self.bitrate / self.sample_rate + padding
frame_size = 576
else:
frame_length = 144 * self.bitrate / self.sample_rate + padding
frame_size = 1152
if check_second:
possible = frame_1 + frame_length
if possible > len(data) + 4:
raise HeaderNotFoundError("can't sync to second MPEG frame")
try:
frame_data = struct.unpack(
">H", data[possible:possible + 2])[0]
except struct.error:
raise HeaderNotFoundError("can't sync to second MPEG frame")
if frame_data & 0xFFE0 != 0xFFE0:
raise HeaderNotFoundError("can't sync to second MPEG frame")
self.length = 8 * real_size / float(self.bitrate)
# Try to find/parse the Xing header, which trumps the above length
# and bitrate calculation.
fileobj.seek(offset, 0)
data = fileobj.read(32768)
try:
xing = data[:-4].index("Xing")
except ValueError:
# Try to find/parse the VBRI header, which trumps the above length
# calculation.
try:
vbri = data[:-24].index("VBRI")
except ValueError:
pass
else:
# If a VBRI header was found, this is definitely MPEG audio.
self.sketchy = False
vbri_version = struct.unpack('>H', data[vbri + 4:vbri + 6])[0]
if vbri_version == 1:
frame_count = struct.unpack(
'>I', data[vbri + 14:vbri + 18])[0]
samples = float(frame_size * frame_count)
self.length = (samples / self.sample_rate) or self.length
else:
# If a Xing header was found, this is definitely MPEG audio.
self.sketchy = False
flags = struct.unpack('>I', data[xing + 4:xing + 8])[0]
if flags & 0x1:
frame_count = struct.unpack('>I', data[xing + 8:xing + 12])[0]
samples = float(frame_size * frame_count)
self.length = (samples / self.sample_rate) or self.length
if flags & 0x2:
bytes = struct.unpack('>I', data[xing + 12:xing + 16])[0]
self.bitrate = int((bytes * 8) // self.length)
def pprint(self):
s = "MPEG %s layer %d, %d bps, %s Hz, %.2f seconds" % (
self.version, self.layer, self.bitrate, self.sample_rate,
self.length)
if self.sketchy:
s += " (sketchy)"
return s
class MP3(ID3FileType):
"""An MPEG audio (usually MPEG-1 Layer 3) file.
:ivar info: :class:`MPEGInfo`
:ivar tags: :class:`ID3 <mutagen.id3.ID3>`
"""
_Info = MPEGInfo
_mimes = ["audio/mp3", "audio/x-mp3", "audio/mpeg", "audio/mpg",
"audio/x-mpeg"]
@staticmethod
def score(filename, fileobj, header):
filename = filename.lower()
return (header.startswith("ID3") * 2 + filename.endswith(".mp3") +
filename.endswith(".mp2") + filename.endswith(".mpg") +
filename.endswith(".mpeg"))
Open = MP3
class EasyMP3(MP3):
"""Like MP3, but uses EasyID3 for tags.
:ivar info: :class:`MPEGInfo`
:ivar tags: :class:`EasyID3 <mutagen.easyid3.EasyID3>`
"""
from mutagen.easyid3 import EasyID3 as ID3
ID3 = ID3

822
libs/mutagen/mp4.py Normal file
View file

@ -0,0 +1,822 @@
# Copyright 2006 Joe Wreschnig
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Read and write MPEG-4 audio files with iTunes metadata.
This module will read MPEG-4 audio information and metadata,
as found in Apple's MP4 (aka M4A, M4B, M4P) files.
There is no official specification for this format. The source code
for TagLib, FAAD, and various MPEG specifications at
* http://developer.apple.com/documentation/QuickTime/QTFF/
* http://www.geocities.com/xhelmboyx/quicktime/formats/mp4-layout.txt
* http://standards.iso.org/ittf/PubliclyAvailableStandards/\
c041828_ISO_IEC_14496-12_2005(E).zip
* http://wiki.multimedia.cx/index.php?title=Apple_QuickTime
were all consulted.
"""
import struct
import sys
from mutagen import FileType, Metadata
from mutagen._constants import GENRES
from mutagen._util import cdata, insert_bytes, DictProxy, utf8
class error(IOError):
pass
class MP4MetadataError(error):
pass
class MP4StreamInfoError(error):
pass
class MP4MetadataValueError(ValueError, MP4MetadataError):
pass
# This is not an exhaustive list of container atoms, but just the
# ones this module needs to peek inside.
_CONTAINERS = ["moov", "udta", "trak", "mdia", "meta", "ilst",
"stbl", "minf", "moof", "traf"]
_SKIP_SIZE = {"meta": 4}
__all__ = ['MP4', 'Open', 'delete', 'MP4Cover', 'MP4FreeForm']
class MP4Cover(str):
"""A cover artwork.
Attributes:
* imageformat -- format of the image (either FORMAT_JPEG or FORMAT_PNG)
"""
FORMAT_JPEG = 0x0D
FORMAT_PNG = 0x0E
def __new__(cls, data, *args, **kwargs):
return str.__new__(cls, data)
def __init__(self, data, imageformat=FORMAT_JPEG):
self.imageformat = imageformat
try:
self.format
except AttributeError:
self.format = imageformat
class MP4FreeForm(str):
"""A freeform value.
Attributes:
* dataformat -- format of the data (either FORMAT_TEXT or FORMAT_DATA)
"""
FORMAT_DATA = 0x0
FORMAT_TEXT = 0x1
def __new__(cls, data, *args, **kwargs):
return str.__new__(cls, data)
def __init__(self, data, dataformat=FORMAT_TEXT):
self.dataformat = dataformat
class Atom(object):
"""An individual atom.
Attributes:
children -- list child atoms (or None for non-container atoms)
length -- length of this atom, including length and name
name -- four byte name of the atom, as a str
offset -- location in the constructor-given fileobj of this atom
This structure should only be used internally by Mutagen.
"""
children = None
def __init__(self, fileobj, level=0):
self.offset = fileobj.tell()
self.length, self.name = struct.unpack(">I4s", fileobj.read(8))
if self.length == 1:
self.length, = struct.unpack(">Q", fileobj.read(8))
if self.length < 16:
raise MP4MetadataError(
"64 bit atom length can only be 16 and higher")
elif self.length == 0:
if level != 0:
raise MP4MetadataError(
"only a top-level atom can have zero length")
# Only the last atom is supposed to have a zero-length, meaning it
# extends to the end of file.
fileobj.seek(0, 2)
self.length = fileobj.tell() - self.offset
fileobj.seek(self.offset + 8, 0)
elif self.length < 8:
raise MP4MetadataError(
"atom length can only be 0, 1 or 8 and higher")
if self.name in _CONTAINERS:
self.children = []
fileobj.seek(_SKIP_SIZE.get(self.name, 0), 1)
while fileobj.tell() < self.offset + self.length:
self.children.append(Atom(fileobj, level + 1))
else:
fileobj.seek(self.offset + self.length, 0)
@staticmethod
def render(name, data):
"""Render raw atom data."""
# this raises OverflowError if Py_ssize_t can't handle the atom data
size = len(data) + 8
if size <= 0xFFFFFFFF:
return struct.pack(">I4s", size, name) + data
else:
return struct.pack(">I4sQ", 1, name, size + 8) + data
def findall(self, name, recursive=False):
"""Recursively find all child atoms by specified name."""
if self.children is not None:
for child in self.children:
if child.name == name:
yield child
if recursive:
for atom in child.findall(name, True):
yield atom
def __getitem__(self, remaining):
"""Look up a child atom, potentially recursively.
e.g. atom['udta', 'meta'] => <Atom name='meta' ...>
"""
if not remaining:
return self
elif self.children is None:
raise KeyError("%r is not a container" % self.name)
for child in self.children:
if child.name == remaining[0]:
return child[remaining[1:]]
else:
raise KeyError("%r not found" % remaining[0])
def __repr__(self):
klass = self.__class__.__name__
if self.children is None:
return "<%s name=%r length=%r offset=%r>" % (
klass, self.name, self.length, self.offset)
else:
children = "\n".join([" " + line for child in self.children
for line in repr(child).splitlines()])
return "<%s name=%r length=%r offset=%r\n%s>" % (
klass, self.name, self.length, self.offset, children)
class Atoms(object):
"""Root atoms in a given file.
Attributes:
atoms -- a list of top-level atoms as Atom objects
This structure should only be used internally by Mutagen.
"""
def __init__(self, fileobj):
self.atoms = []
fileobj.seek(0, 2)
end = fileobj.tell()
fileobj.seek(0)
while fileobj.tell() + 8 <= end:
self.atoms.append(Atom(fileobj))
def path(self, *names):
"""Look up and return the complete path of an atom.
For example, atoms.path('moov', 'udta', 'meta') will return a
list of three atoms, corresponding to the moov, udta, and meta
atoms.
"""
path = [self]
for name in names:
path.append(path[-1][name, ])
return path[1:]
def __contains__(self, names):
try:
self[names]
except KeyError:
return False
return True
def __getitem__(self, names):
"""Look up a child atom.
'names' may be a list of atoms (['moov', 'udta']) or a string
specifying the complete path ('moov.udta').
"""
if isinstance(names, basestring):
names = names.split(".")
for child in self.atoms:
if child.name == names[0]:
return child[names[1:]]
else:
raise KeyError("%s not found" % names[0])
def __repr__(self):
return "\n".join([repr(child) for child in self.atoms])
class MP4Tags(DictProxy, Metadata):
r"""Dictionary containing Apple iTunes metadata list key/values.
Keys are four byte identifiers, except for freeform ('----')
keys. Values are usually unicode strings, but some atoms have a
special structure:
Text values (multiple values per key are supported):
* '\\xa9nam' -- track title
* '\\xa9alb' -- album
* '\\xa9ART' -- artist
* 'aART' -- album artist
* '\\xa9wrt' -- composer
* '\\xa9day' -- year
* '\\xa9cmt' -- comment
* 'desc' -- description (usually used in podcasts)
* 'purd' -- purchase date
* '\\xa9grp' -- grouping
* '\\xa9gen' -- genre
* '\\xa9lyr' -- lyrics
* 'purl' -- podcast URL
* 'egid' -- podcast episode GUID
* 'catg' -- podcast category
* 'keyw' -- podcast keywords
* '\\xa9too' -- encoded by
* 'cprt' -- copyright
* 'soal' -- album sort order
* 'soaa' -- album artist sort order
* 'soar' -- artist sort order
* 'sonm' -- title sort order
* 'soco' -- composer sort order
* 'sosn' -- show sort order
* 'tvsh' -- show name
Boolean values:
* 'cpil' -- part of a compilation
* 'pgap' -- part of a gapless album
* 'pcst' -- podcast (iTunes reads this only on import)
Tuples of ints (multiple values per key are supported):
* 'trkn' -- track number, total tracks
* 'disk' -- disc number, total discs
Others:
* 'tmpo' -- tempo/BPM, 16 bit int
* 'covr' -- cover artwork, list of MP4Cover objects (which are
tagged strs)
* 'gnre' -- ID3v1 genre. Not supported, use '\\xa9gen' instead.
The freeform '----' frames use a key in the format '----:mean:name'
where 'mean' is usually 'com.apple.iTunes' and 'name' is a unique
identifier for this frame. The value is a str, but is probably
text that can be decoded as UTF-8. Multiple values per key are
supported.
MP4 tag data cannot exist outside of the structure of an MP4 file,
so this class should not be manually instantiated.
Unknown non-text tags are removed.
"""
def load(self, atoms, fileobj):
try:
ilst = atoms["moov.udta.meta.ilst"]
except KeyError, key:
raise MP4MetadataError(key)
for atom in ilst.children:
fileobj.seek(atom.offset + 8)
data = fileobj.read(atom.length - 8)
if len(data) != atom.length - 8:
raise MP4MetadataError("Not enough data")
if atom.name in self.__atoms:
info = self.__atoms[atom.name]
info[0](self, atom, data, *info[2:])
else:
# unknown atom, try as text and skip if it fails
# FIXME: keep them somehow
try:
self.__parse_text(atom, data)
except MP4MetadataError:
continue
@classmethod
def _can_load(cls, atoms):
return "moov.udta.meta.ilst" in atoms
@staticmethod
def __key_sort(item1, item2):
(key1, v1) = item1
(key2, v2) = item2
# iTunes always writes the tags in order of "relevance", try
# to copy it as closely as possible.
order = ["\xa9nam", "\xa9ART", "\xa9wrt", "\xa9alb",
"\xa9gen", "gnre", "trkn", "disk",
"\xa9day", "cpil", "pgap", "pcst", "tmpo",
"\xa9too", "----", "covr", "\xa9lyr"]
order = dict(zip(order, range(len(order))))
last = len(order)
# If there's no key-based way to distinguish, order by length.
# If there's still no way, go by string comparison on the
# values, so we at least have something determinstic.
return (cmp(order.get(key1[:4], last), order.get(key2[:4], last)) or
cmp(len(v1), len(v2)) or cmp(v1, v2))
def save(self, filename):
"""Save the metadata to the given filename."""
values = []
items = self.items()
items.sort(self.__key_sort)
for key, value in items:
info = self.__atoms.get(key[:4], (None, type(self).__render_text))
try:
values.append(info[1](self, key, value, *info[2:]))
except (TypeError, ValueError), s:
raise MP4MetadataValueError, s, sys.exc_info()[2]
data = Atom.render("ilst", "".join(values))
# Find the old atoms.
fileobj = open(filename, "rb+")
try:
atoms = Atoms(fileobj)
try:
path = atoms.path("moov", "udta", "meta", "ilst")
except KeyError:
self.__save_new(fileobj, atoms, data)
else:
self.__save_existing(fileobj, atoms, path, data)
finally:
fileobj.close()
def __pad_ilst(self, data, length=None):
if length is None:
length = ((len(data) + 1023) & ~1023) - len(data)
return Atom.render("free", "\x00" * length)
def __save_new(self, fileobj, atoms, ilst):
hdlr = Atom.render("hdlr", "\x00" * 8 + "mdirappl" + "\x00" * 9)
meta = Atom.render(
"meta", "\x00\x00\x00\x00" + hdlr + ilst + self.__pad_ilst(ilst))
try:
path = atoms.path("moov", "udta")
except KeyError:
# moov.udta not found -- create one
path = atoms.path("moov")
meta = Atom.render("udta", meta)
offset = path[-1].offset + 8
insert_bytes(fileobj, len(meta), offset)
fileobj.seek(offset)
fileobj.write(meta)
self.__update_parents(fileobj, path, len(meta))
self.__update_offsets(fileobj, atoms, len(meta), offset)
def __save_existing(self, fileobj, atoms, path, data):
# Replace the old ilst atom.
ilst = path.pop()
offset = ilst.offset
length = ilst.length
# Check for padding "free" atoms
meta = path[-1]
index = meta.children.index(ilst)
try:
prev = meta.children[index-1]
if prev.name == "free":
offset = prev.offset
length += prev.length
except IndexError:
pass
try:
next = meta.children[index+1]
if next.name == "free":
length += next.length
except IndexError:
pass
delta = len(data) - length
if delta > 0 or (delta < 0 and delta > -8):
data += self.__pad_ilst(data)
delta = len(data) - length
insert_bytes(fileobj, delta, offset)
elif delta < 0:
data += self.__pad_ilst(data, -delta - 8)
delta = 0
fileobj.seek(offset)
fileobj.write(data)
self.__update_parents(fileobj, path, delta)
self.__update_offsets(fileobj, atoms, delta, offset)
def __update_parents(self, fileobj, path, delta):
"""Update all parent atoms with the new size."""
for atom in path:
fileobj.seek(atom.offset)
size = cdata.uint_be(fileobj.read(4))
if size == 1: # 64bit
# skip name (4B) and read size (8B)
size = cdata.ulonglong_be(fileobj.read(12)[4:])
fileobj.seek(atom.offset + 8)
fileobj.write(cdata.to_ulonglong_be(size + delta))
else: # 32bit
fileobj.seek(atom.offset)
fileobj.write(cdata.to_uint_be(size + delta))
def __update_offset_table(self, fileobj, fmt, atom, delta, offset):
"""Update offset table in the specified atom."""
if atom.offset > offset:
atom.offset += delta
fileobj.seek(atom.offset + 12)
data = fileobj.read(atom.length - 12)
fmt = fmt % cdata.uint_be(data[:4])
offsets = struct.unpack(fmt, data[4:])
offsets = [o + (0, delta)[offset < o] for o in offsets]
fileobj.seek(atom.offset + 16)
fileobj.write(struct.pack(fmt, *offsets))
def __update_tfhd(self, fileobj, atom, delta, offset):
if atom.offset > offset:
atom.offset += delta
fileobj.seek(atom.offset + 9)
data = fileobj.read(atom.length - 9)
flags = cdata.uint_be("\x00" + data[:3])
if flags & 1:
o = cdata.ulonglong_be(data[7:15])
if o > offset:
o += delta
fileobj.seek(atom.offset + 16)
fileobj.write(cdata.to_ulonglong_be(o))
def __update_offsets(self, fileobj, atoms, delta, offset):
"""Update offset tables in all 'stco' and 'co64' atoms."""
if delta == 0:
return
moov = atoms["moov"]
for atom in moov.findall('stco', True):
self.__update_offset_table(fileobj, ">%dI", atom, delta, offset)
for atom in moov.findall('co64', True):
self.__update_offset_table(fileobj, ">%dQ", atom, delta, offset)
try:
for atom in atoms["moof"].findall('tfhd', True):
self.__update_tfhd(fileobj, atom, delta, offset)
except KeyError:
pass
def __parse_data(self, atom, data):
pos = 0
while pos < atom.length - 8:
length, name, flags = struct.unpack(">I4sI", data[pos:pos+12])
if name != "data":
raise MP4MetadataError(
"unexpected atom %r inside %r" % (name, atom.name))
yield flags, data[pos+16:pos+length]
pos += length
def __render_data(self, key, flags, value):
return Atom.render(key, "".join([
Atom.render("data", struct.pack(">2I", flags, 0) + data)
for data in value]))
def __parse_freeform(self, atom, data):
length = cdata.uint_be(data[:4])
mean = data[12:length]
pos = length
length = cdata.uint_be(data[pos:pos+4])
name = data[pos+12:pos+length]
pos += length
value = []
while pos < atom.length - 8:
length, atom_name = struct.unpack(">I4s", data[pos:pos+8])
if atom_name != "data":
raise MP4MetadataError(
"unexpected atom %r inside %r" % (atom_name, atom.name))
version = ord(data[pos+8])
if version != 0:
raise MP4MetadataError("Unsupported version: %r" % version)
flags = struct.unpack(">I", "\x00" + data[pos+9:pos+12])[0]
value.append(MP4FreeForm(data[pos+16:pos+length],
dataformat=flags))
pos += length
if value:
self["%s:%s:%s" % (atom.name, mean, name)] = value
def __render_freeform(self, key, value):
dummy, mean, name = key.split(":", 2)
mean = struct.pack(">I4sI", len(mean) + 12, "mean", 0) + mean
name = struct.pack(">I4sI", len(name) + 12, "name", 0) + name
if isinstance(value, basestring):
value = [value]
data = ""
for v in value:
flags = MP4FreeForm.FORMAT_TEXT
if isinstance(v, MP4FreeForm):
flags = v.dataformat
data += struct.pack(">I4s2I", len(v) + 16, "data", flags, 0)
data += v
return Atom.render("----", mean + name + data)
def __parse_pair(self, atom, data):
self[atom.name] = [struct.unpack(">2H", d[2:6]) for
flags, d in self.__parse_data(atom, data)]
def __render_pair(self, key, value):
data = []
for (track, total) in value:
if 0 <= track < 1 << 16 and 0 <= total < 1 << 16:
data.append(struct.pack(">4H", 0, track, total, 0))
else:
raise MP4MetadataValueError(
"invalid numeric pair %r" % ((track, total),))
return self.__render_data(key, 0, data)
def __render_pair_no_trailing(self, key, value):
data = []
for (track, total) in value:
if 0 <= track < 1 << 16 and 0 <= total < 1 << 16:
data.append(struct.pack(">3H", 0, track, total))
else:
raise MP4MetadataValueError(
"invalid numeric pair %r" % ((track, total),))
return self.__render_data(key, 0, data)
def __parse_genre(self, atom, data):
# Translate to a freeform genre.
genre = cdata.short_be(data[16:18])
if "\xa9gen" not in self:
try:
self["\xa9gen"] = [GENRES[genre - 1]]
except IndexError:
pass
def __parse_tempo(self, atom, data):
self[atom.name] = [cdata.ushort_be(value[1]) for
value in self.__parse_data(atom, data)]
def __render_tempo(self, key, value):
try:
if len(value) == 0:
return self.__render_data(key, 0x15, "")
if min(value) < 0 or max(value) >= 2**16:
raise MP4MetadataValueError(
"invalid 16 bit integers: %r" % value)
except TypeError:
raise MP4MetadataValueError(
"tmpo must be a list of 16 bit integers")
values = map(cdata.to_ushort_be, value)
return self.__render_data(key, 0x15, values)
def __parse_bool(self, atom, data):
try:
self[atom.name] = bool(ord(data[16:17]))
except TypeError:
self[atom.name] = False
def __render_bool(self, key, value):
return self.__render_data(key, 0x15, [chr(bool(value))])
def __parse_cover(self, atom, data):
self[atom.name] = []
pos = 0
while pos < atom.length - 8:
length, name, imageformat = struct.unpack(">I4sI",
data[pos:pos+12])
if name != "data":
if name == "name":
pos += length
continue
raise MP4MetadataError(
"unexpected atom %r inside 'covr'" % name)
if imageformat not in (MP4Cover.FORMAT_JPEG, MP4Cover.FORMAT_PNG):
imageformat = MP4Cover.FORMAT_JPEG
cover = MP4Cover(data[pos+16:pos+length], imageformat)
self[atom.name].append(cover)
pos += length
def __render_cover(self, key, value):
atom_data = []
for cover in value:
try:
imageformat = cover.imageformat
except AttributeError:
imageformat = MP4Cover.FORMAT_JPEG
atom_data.append(Atom.render(
"data", struct.pack(">2I", imageformat, 0) + cover))
return Atom.render(key, "".join(atom_data))
def __parse_text(self, atom, data, expected_flags=1):
value = [text.decode('utf-8', 'replace') for flags, text
in self.__parse_data(atom, data)
if flags == expected_flags]
if value:
self[atom.name] = value
def __render_text(self, key, value, flags=1):
if isinstance(value, basestring):
value = [value]
return self.__render_data(
key, flags, map(utf8, value))
def delete(self, filename):
"""Remove the metadata from the given filename."""
self.clear()
self.save(filename)
__atoms = {
"----": (__parse_freeform, __render_freeform),
"trkn": (__parse_pair, __render_pair),
"disk": (__parse_pair, __render_pair_no_trailing),
"gnre": (__parse_genre, None),
"tmpo": (__parse_tempo, __render_tempo),
"cpil": (__parse_bool, __render_bool),
"pgap": (__parse_bool, __render_bool),
"pcst": (__parse_bool, __render_bool),
"covr": (__parse_cover, __render_cover),
"purl": (__parse_text, __render_text, 0),
"egid": (__parse_text, __render_text, 0),
}
# the text atoms we know about which should make loading fail if parsing
# any of them fails
for name in ["\xa9nam", "\xa9alb", "\xa9ART", "aART", "\xa9wrt", "\xa9day",
"\xa9cmt", "desc", "purd", "\xa9grp", "\xa9gen", "\xa9lyr",
"catg", "keyw", "\xa9too", "cprt", "soal", "soaa", "soar",
"sonm", "soco", "sosn", "tvsh"]:
__atoms[name] = (__parse_text, __render_text)
def pprint(self):
values = []
for key, value in self.iteritems():
key = key.decode('latin1')
if key == "covr":
values.append("%s=%s" % (key, ", ".join(
["[%d bytes of data]" % len(data) for data in value])))
elif isinstance(value, list):
values.append("%s=%s" % (key, " / ".join(map(unicode, value))))
else:
values.append("%s=%s" % (key, value))
return "\n".join(values)
class MP4Info(object):
"""MPEG-4 stream information.
Attributes:
* bitrate -- bitrate in bits per second, as an int
* length -- file length in seconds, as a float
* channels -- number of audio channels
* sample_rate -- audio sampling rate in Hz
* bits_per_sample -- bits per sample
"""
bitrate = 0
channels = 0
sample_rate = 0
bits_per_sample = 0
def __init__(self, atoms, fileobj):
for trak in list(atoms["moov"].findall("trak")):
hdlr = trak["mdia", "hdlr"]
fileobj.seek(hdlr.offset)
data = fileobj.read(hdlr.length)
if data[16:20] == "soun":
break
else:
raise MP4StreamInfoError("track has no audio data")
mdhd = trak["mdia", "mdhd"]
fileobj.seek(mdhd.offset)
data = fileobj.read(mdhd.length)
if ord(data[8]) == 0:
offset = 20
fmt = ">2I"
else:
offset = 28
fmt = ">IQ"
end = offset + struct.calcsize(fmt)
unit, length = struct.unpack(fmt, data[offset:end])
self.length = float(length) / unit
try:
atom = trak["mdia", "minf", "stbl", "stsd"]
fileobj.seek(atom.offset)
data = fileobj.read(atom.length)
if data[20:24] == "mp4a":
length = cdata.uint_be(data[16:20])
(self.channels, self.bits_per_sample, _,
self.sample_rate) = struct.unpack(">3HI", data[40:50])
# ES descriptor type
if data[56:60] == "esds" and ord(data[64:65]) == 0x03:
pos = 65
# skip extended descriptor type tag, length, ES ID
# and stream priority
if data[pos:pos+3] == "\x80\x80\x80":
pos += 3
pos += 4
# decoder config descriptor type
if ord(data[pos]) == 0x04:
pos += 1
# skip extended descriptor type tag, length,
# object type ID, stream type, buffer size
# and maximum bitrate
if data[pos:pos+3] == "\x80\x80\x80":
pos += 3
pos += 10
# average bitrate
self.bitrate = cdata.uint_be(data[pos:pos+4])
except (ValueError, KeyError):
# stsd atoms are optional
pass
def pprint(self):
return "MPEG-4 audio, %.2f seconds, %d bps" % (
self.length, self.bitrate)
class MP4(FileType):
"""An MPEG-4 audio file, probably containing AAC.
If more than one track is present in the file, the first is used.
Only audio ('soun') tracks will be read.
:ivar info: :class:`MP4Info`
:ivar tags: :class:`MP4Tags`
"""
MP4Tags = MP4Tags
_mimes = ["audio/mp4", "audio/x-m4a", "audio/mpeg4", "audio/aac"]
def load(self, filename):
self.filename = filename
fileobj = open(filename, "rb")
try:
atoms = Atoms(fileobj)
# ftyp is always the first atom in a valid MP4 file
if not atoms.atoms or atoms.atoms[0].name != "ftyp":
raise error("Not a MP4 file")
try:
self.info = MP4Info(atoms, fileobj)
except StandardError, err:
raise MP4StreamInfoError, err, sys.exc_info()[2]
if not MP4Tags._can_load(atoms):
self.tags = None
else:
try:
self.tags = self.MP4Tags(atoms, fileobj)
except StandardError, err:
raise MP4MetadataError, err, sys.exc_info()[2]
finally:
fileobj.close()
def add_tags(self):
if self.tags is None:
self.tags = self.MP4Tags()
else:
raise error("an MP4 tag already exists")
@staticmethod
def score(filename, fileobj, header):
return ("ftyp" in header) + ("mp4" in header)
Open = MP4
def delete(filename):
"""Remove tags from a file."""
MP4(filename).delete()

257
libs/mutagen/musepack.py Normal file
View file

@ -0,0 +1,257 @@
# A Musepack reader/tagger
#
# Copyright 2006 Lukas Lalinsky <lalinsky@gmail.com>
# Copyright 2012 Christoph Reiter <christoph.reiter@gmx.at>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Musepack audio streams with APEv2 tags.
Musepack is an audio format originally based on the MPEG-1 Layer-2
algorithms. Stream versions 4 through 7 are supported.
For more information, see http://www.musepack.net/.
"""
__all__ = ["Musepack", "Open", "delete"]
import struct
from mutagen.apev2 import APEv2File, error, delete
from mutagen.id3 import BitPaddedInt
from mutagen._util import cdata
class MusepackHeaderError(error):
pass
RATES = [44100, 48000, 37800, 32000]
def _parse_sv8_int(fileobj, limit=9):
"""Reads (max limit) bytes from fileobj until the MSB is zero.
All 7 LSB will be merged to a big endian uint.
Raises ValueError in case not MSB is zero, or EOFError in
case the file ended before limit is reached.
Returns (parsed number, number of bytes read)
"""
num = 0
for i in xrange(limit):
c = fileobj.read(1)
if len(c) != 1:
raise EOFError
num = (num << 7) | (ord(c) & 0x7F)
if not ord(c) & 0x80:
return num, i + 1
if limit > 0:
raise ValueError
return 0, 0
def _calc_sv8_gain(gain):
# 64.82 taken from mpcdec
return 64.82 - gain / 256.0
def _calc_sv8_peak(peak):
return (10 ** (peak / (256.0 * 20.0)) / 65535.0)
class MusepackInfo(object):
"""Musepack stream information.
Attributes:
* channels -- number of audio channels
* length -- file length in seconds, as a float
* sample_rate -- audio sampling rate in Hz
* bitrate -- audio bitrate, in bits per second
* version -- Musepack stream version
Optional Attributes:
* title_gain, title_peak -- Replay Gain and peak data for this song
* album_gain, album_peak -- Replay Gain and peak data for this album
These attributes are only available in stream version 7/8. The
gains are a float, +/- some dB. The peaks are a percentage [0..1] of
the maximum amplitude. This means to get a number comparable to
VorbisGain, you must multiply the peak by 2.
"""
def __init__(self, fileobj):
header = fileobj.read(4)
if len(header) != 4:
raise MusepackHeaderError("not a Musepack file")
# Skip ID3v2 tags
if header[:3] == "ID3":
header = fileobj.read(6)
if len(header) != 6:
raise MusepackHeaderError("not a Musepack file")
size = 10 + BitPaddedInt(header[2:6])
fileobj.seek(size)
header = fileobj.read(4)
if len(header) != 4:
raise MusepackHeaderError("not a Musepack file")
if header.startswith("MPCK"):
self.__parse_sv8(fileobj)
else:
self.__parse_sv467(fileobj)
if not self.bitrate and self.length != 0:
fileobj.seek(0, 2)
self.bitrate = int(round(fileobj.tell() * 8 / self.length))
def __parse_sv8(self, fileobj):
#SV8 http://trac.musepack.net/trac/wiki/SV8Specification
key_size = 2
mandatory_packets = ["SH", "RG"]
def check_frame_key(key):
if len(frame_type) != key_size or not 'AA' <= frame_type <= 'ZZ':
raise MusepackHeaderError("Invalid frame key.")
frame_type = fileobj.read(key_size)
check_frame_key(frame_type)
while frame_type not in ("AP", "SE") and mandatory_packets:
try:
frame_size, slen = _parse_sv8_int(fileobj)
except (EOFError, ValueError):
raise MusepackHeaderError("Invalid packet size.")
data_size = frame_size - key_size - slen
if frame_type == "SH":
mandatory_packets.remove(frame_type)
self.__parse_stream_header(fileobj, data_size)
elif frame_type == "RG":
mandatory_packets.remove(frame_type)
self.__parse_replaygain_packet(fileobj, data_size)
else:
fileobj.seek(data_size, 1)
frame_type = fileobj.read(key_size)
check_frame_key(frame_type)
if mandatory_packets:
raise MusepackHeaderError("Missing mandatory packets: %s."
% ", ".join(mandatory_packets))
self.length = float(self.samples) / self.sample_rate
self.bitrate = 0
def __parse_stream_header(self, fileobj, data_size):
fileobj.seek(4, 1)
try:
self.version = ord(fileobj.read(1))
except TypeError:
raise MusepackHeaderError("SH packet ended unexpectedly.")
try:
samples, l1 = _parse_sv8_int(fileobj)
samples_skip, l2 = _parse_sv8_int(fileobj)
except (EOFError, ValueError):
raise MusepackHeaderError(
"SH packet: Invalid sample counts.")
left_size = data_size - 5 - l1 - l2
if left_size != 2:
raise MusepackHeaderError("Invalid SH packet size.")
data = fileobj.read(left_size)
if len(data) != left_size:
raise MusepackHeaderError("SH packet ended unexpectedly.")
self.sample_rate = RATES[ord(data[-2]) >> 5]
self.channels = (ord(data[-1]) >> 4) + 1
self.samples = samples - samples_skip
def __parse_replaygain_packet(self, fileobj, data_size):
data = fileobj.read(data_size)
if data_size != 9:
raise MusepackHeaderError("Invalid RG packet size.")
if len(data) != data_size:
raise MusepackHeaderError("RG packet ended unexpectedly.")
title_gain = cdata.short_be(data[1:3])
title_peak = cdata.short_be(data[3:5])
album_gain = cdata.short_be(data[5:7])
album_peak = cdata.short_be(data[7:9])
if title_gain:
self.title_gain = _calc_sv8_gain(title_gain)
if title_peak:
self.title_peak = _calc_sv8_peak(title_peak)
if album_gain:
self.album_gain = _calc_sv8_gain(album_gain)
if album_peak:
self.album_peak = _calc_sv8_peak(album_peak)
def __parse_sv467(self, fileobj):
fileobj.seek(-4, 1)
header = fileobj.read(32)
if len(header) != 32:
raise MusepackHeaderError("not a Musepack file")
# SV7
if header.startswith("MP+"):
self.version = ord(header[3]) & 0xF
if self.version < 7:
raise MusepackHeaderError("not a Musepack file")
frames = cdata.uint_le(header[4:8])
flags = cdata.uint_le(header[8:12])
self.title_peak, self.title_gain = struct.unpack(
"<Hh", header[12:16])
self.album_peak, self.album_gain = struct.unpack(
"<Hh", header[16:20])
self.title_gain /= 100.0
self.album_gain /= 100.0
self.title_peak /= 65535.0
self.album_peak /= 65535.0
self.sample_rate = RATES[(flags >> 16) & 0x0003]
self.bitrate = 0
# SV4-SV6
else:
header_dword = cdata.uint_le(header[0:4])
self.version = (header_dword >> 11) & 0x03FF
if self.version < 4 or self.version > 6:
raise MusepackHeaderError("not a Musepack file")
self.bitrate = (header_dword >> 23) & 0x01FF
self.sample_rate = 44100
if self.version >= 5:
frames = cdata.uint_le(header[4:8])
else:
frames = cdata.ushort_le(header[6:8])
if self.version < 6:
frames -= 1
self.channels = 2
self.length = float(frames * 1152 - 576) / self.sample_rate
def pprint(self):
rg_data = []
if hasattr(self, "title_gain"):
rg_data.append("%+0.2f (title)" % self.title_gain)
if hasattr(self, "album_gain"):
rg_data.append("%+0.2f (album)" % self.album_gain)
rg_data = (rg_data and ", Gain: " + ", ".join(rg_data)) or ""
return "Musepack SV%d, %.2f seconds, %d Hz, %d bps%s" % (
self.version, self.length, self.sample_rate, self.bitrate, rg_data)
class Musepack(APEv2File):
_Info = MusepackInfo
_mimes = ["audio/x-musepack", "audio/x-mpc"]
@staticmethod
def score(filename, fileobj, header):
return (header.startswith("MP+") + header.startswith("MPCK") +
filename.lower().endswith(".mpc"))
Open = Musepack

507
libs/mutagen/ogg.py Normal file
View file

@ -0,0 +1,507 @@
# Copyright 2006 Joe Wreschnig
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Read and write Ogg bitstreams and pages.
This module reads and writes a subset of the Ogg bitstream format
version 0. It does *not* read or write Ogg Vorbis files! For that,
you should use mutagen.oggvorbis.
This implementation is based on the RFC 3533 standard found at
http://www.xiph.org/ogg/doc/rfc3533.txt.
"""
import struct
import sys
import zlib
from cStringIO import StringIO
from mutagen import FileType
from mutagen._util import cdata, insert_bytes, delete_bytes
class error(IOError):
"""Ogg stream parsing errors."""
pass
class OggPage(object):
"""A single Ogg page (not necessarily a single encoded packet).
A page is a header of 26 bytes, followed by the length of the
data, followed by the data.
The constructor is givin a file-like object pointing to the start
of an Ogg page. After the constructor is finished it is pointing
to the start of the next page.
Attributes:
* version -- stream structure version (currently always 0)
* position -- absolute stream position (default -1)
* serial -- logical stream serial number (default 0)
* sequence -- page sequence number within logical stream (default 0)
* offset -- offset this page was read from (default None)
* complete -- if the last packet on this page is complete (default True)
* packets -- list of raw packet data (default [])
Note that if 'complete' is false, the next page's 'continued'
property must be true (so set both when constructing pages).
If a file-like object is supplied to the constructor, the above
attributes will be filled in based on it.
"""
version = 0
__type_flags = 0
position = 0L
serial = 0
sequence = 0
offset = None
complete = True
def __init__(self, fileobj=None):
self.packets = []
if fileobj is None:
return
self.offset = fileobj.tell()
header = fileobj.read(27)
if len(header) == 0:
raise EOFError
try:
(oggs, self.version, self.__type_flags, self.position,
self.serial, self.sequence, crc, segments) = struct.unpack(
"<4sBBqIIiB", header)
except struct.error:
raise error("unable to read full header; got %r" % header)
if oggs != "OggS":
raise error("read %r, expected %r, at 0x%x" % (
oggs, "OggS", fileobj.tell() - 27))
if self.version != 0:
raise error("version %r unsupported" % self.version)
total = 0
lacings = []
lacing_bytes = fileobj.read(segments)
if len(lacing_bytes) != segments:
raise error("unable to read %r lacing bytes" % segments)
for c in map(ord, lacing_bytes):
total += c
if c < 255:
lacings.append(total)
total = 0
if total:
lacings.append(total)
self.complete = False
self.packets = map(fileobj.read, lacings)
if map(len, self.packets) != lacings:
raise error("unable to read full data")
def __eq__(self, other):
"""Two Ogg pages are the same if they write the same data."""
try:
return (self.write() == other.write())
except AttributeError:
return False
__hash__ = object.__hash__
def __repr__(self):
attrs = ['version', 'position', 'serial', 'sequence', 'offset',
'complete', 'continued', 'first', 'last']
values = ["%s=%r" % (attr, getattr(self, attr)) for attr in attrs]
return "<%s %s, %d bytes in %d packets>" % (
type(self).__name__, " ".join(values), sum(map(len, self.packets)),
len(self.packets))
def write(self):
"""Return a string encoding of the page header and data.
A ValueError is raised if the data is too big to fit in a
single page.
"""
data = [
struct.pack("<4sBBqIIi", "OggS", self.version, self.__type_flags,
self.position, self.serial, self.sequence, 0)
]
lacing_data = []
for datum in self.packets:
quot, rem = divmod(len(datum), 255)
lacing_data.append("\xff" * quot + chr(rem))
lacing_data = "".join(lacing_data)
if not self.complete and lacing_data.endswith("\x00"):
lacing_data = lacing_data[:-1]
data.append(chr(len(lacing_data)))
data.append(lacing_data)
data.extend(self.packets)
data = "".join(data)
# Python's CRC is swapped relative to Ogg's needs.
# crc32 returns uint prior to py2.6 on some platforms, so force uint
crc = (~zlib.crc32(data.translate(cdata.bitswap), -1)) & 0xffffffff
# Although we're using to_uint_be, this actually makes the CRC
# a proper le integer, since Python's CRC is byteswapped.
crc = cdata.to_uint_be(crc).translate(cdata.bitswap)
data = data[:22] + crc + data[26:]
return data
@property
def size(self):
"""Total frame size."""
size = 27 # Initial header size
for datum in self.packets:
quot, rem = divmod(len(datum), 255)
size += quot + 1
if not self.complete and rem == 0:
# Packet contains a multiple of 255 bytes and is not
# terminated, so we don't have a \x00 at the end.
size -= 1
size += sum(map(len, self.packets))
return size
def __set_flag(self, bit, val):
mask = 1 << bit
if val:
self.__type_flags |= mask
else:
self.__type_flags &= ~mask
continued = property(
lambda self: cdata.test_bit(self.__type_flags, 0),
lambda self, v: self.__set_flag(0, v),
doc="The first packet is continued from the previous page.")
first = property(
lambda self: cdata.test_bit(self.__type_flags, 1),
lambda self, v: self.__set_flag(1, v),
doc="This is the first page of a logical bitstream.")
last = property(
lambda self: cdata.test_bit(self.__type_flags, 2),
lambda self, v: self.__set_flag(2, v),
doc="This is the last page of a logical bitstream.")
@classmethod
def renumber(klass, fileobj, serial, start):
"""Renumber pages belonging to a specified logical stream.
fileobj must be opened with mode r+b or w+b.
Starting at page number 'start', renumber all pages belonging
to logical stream 'serial'. Other pages will be ignored.
fileobj must point to the start of a valid Ogg page; any
occuring after it and part of the specified logical stream
will be numbered. No adjustment will be made to the data in
the pages nor the granule position; only the page number, and
so also the CRC.
If an error occurs (e.g. non-Ogg data is found), fileobj will
be left pointing to the place in the stream the error occured,
but the invalid data will be left intact (since this function
does not change the total file size).
"""
number = start
while True:
try:
page = OggPage(fileobj)
except EOFError:
break
else:
if page.serial != serial:
# Wrong stream, skip this page.
continue
# Changing the number can't change the page size,
# so seeking back based on the current size is safe.
fileobj.seek(-page.size, 1)
page.sequence = number
fileobj.write(page.write())
fileobj.seek(page.offset + page.size, 0)
number += 1
@classmethod
def to_packets(klass, pages, strict=False):
"""Construct a list of packet data from a list of Ogg pages.
If strict is true, the first page must start a new packet,
and the last page must end the last packet.
"""
serial = pages[0].serial
sequence = pages[0].sequence
packets = []
if strict:
if pages[0].continued:
raise ValueError("first packet is continued")
if not pages[-1].complete:
raise ValueError("last packet does not complete")
elif pages and pages[0].continued:
packets.append([""])
for page in pages:
if serial != page.serial:
raise ValueError("invalid serial number in %r" % page)
elif sequence != page.sequence:
raise ValueError("bad sequence number in %r" % page)
else:
sequence += 1
if page.continued:
packets[-1].append(page.packets[0])
else:
packets.append([page.packets[0]])
packets.extend([[p] for p in page.packets[1:]])
return ["".join(p) for p in packets]
@classmethod
def from_packets(klass, packets, sequence=0,
default_size=4096, wiggle_room=2048):
"""Construct a list of Ogg pages from a list of packet data.
The algorithm will generate pages of approximately
default_size in size (rounded down to the nearest multiple of
255). However, it will also allow pages to increase to
approximately default_size + wiggle_room if allowing the
wiggle room would finish a packet (only one packet will be
finished in this way per page; if the next packet would fit
into the wiggle room, it still starts on a new page).
This method reduces packet fragmentation when packet sizes are
slightly larger than the default page size, while still
ensuring most pages are of the average size.
Pages are numbered started at 'sequence'; other information is
uninitialized.
"""
chunk_size = (default_size // 255) * 255
pages = []
page = OggPage()
page.sequence = sequence
for packet in packets:
page.packets.append("")
while packet:
data, packet = packet[:chunk_size], packet[chunk_size:]
if page.size < default_size and len(page.packets) < 255:
page.packets[-1] += data
else:
# If we've put any packet data into this page yet,
# we need to mark it incomplete. However, we can
# also have just started this packet on an already
# full page, in which case, just start the new
# page with this packet.
if page.packets[-1]:
page.complete = False
if len(page.packets) == 1:
page.position = -1L
else:
page.packets.pop(-1)
pages.append(page)
page = OggPage()
page.continued = not pages[-1].complete
page.sequence = pages[-1].sequence + 1
page.packets.append(data)
if len(packet) < wiggle_room:
page.packets[-1] += packet
packet = ""
if page.packets:
pages.append(page)
return pages
@classmethod
def replace(klass, fileobj, old_pages, new_pages):
"""Replace old_pages with new_pages within fileobj.
old_pages must have come from reading fileobj originally.
new_pages are assumed to have the 'same' data as old_pages,
and so the serial and sequence numbers will be copied, as will
the flags for the first and last pages.
fileobj will be resized and pages renumbered as necessary. As
such, it must be opened r+b or w+b.
"""
# Number the new pages starting from the first old page.
first = old_pages[0].sequence
for page, seq in zip(new_pages, range(first, first + len(new_pages))):
page.sequence = seq
page.serial = old_pages[0].serial
new_pages[0].first = old_pages[0].first
new_pages[0].last = old_pages[0].last
new_pages[0].continued = old_pages[0].continued
new_pages[-1].first = old_pages[-1].first
new_pages[-1].last = old_pages[-1].last
new_pages[-1].complete = old_pages[-1].complete
if not new_pages[-1].complete and len(new_pages[-1].packets) == 1:
new_pages[-1].position = -1L
new_data = "".join(map(klass.write, new_pages))
# Make room in the file for the new data.
delta = len(new_data)
fileobj.seek(old_pages[0].offset, 0)
insert_bytes(fileobj, delta, old_pages[0].offset)
fileobj.seek(old_pages[0].offset, 0)
fileobj.write(new_data)
new_data_end = old_pages[0].offset + delta
# Go through the old pages and delete them. Since we shifted
# the data down the file, we need to adjust their offsets. We
# also need to go backwards, so we don't adjust the deltas of
# the other pages.
old_pages.reverse()
for old_page in old_pages:
adj_offset = old_page.offset + delta
delete_bytes(fileobj, old_page.size, adj_offset)
# Finally, if there's any discrepency in length, we need to
# renumber the pages for the logical stream.
if len(old_pages) != len(new_pages):
fileobj.seek(new_data_end, 0)
serial = new_pages[-1].serial
sequence = new_pages[-1].sequence + 1
klass.renumber(fileobj, serial, sequence)
@classmethod
def find_last(klass, fileobj, serial):
"""Find the last page of the stream 'serial'.
If the file is not multiplexed this function is fast. If it is,
it must read the whole the stream.
This finds the last page in the actual file object, or the last
page in the stream (with eos set), whichever comes first.
"""
# For non-muxed streams, look at the last page.
try:
fileobj.seek(-256*256, 2)
except IOError:
# The file is less than 64k in length.
fileobj.seek(0)
data = fileobj.read()
try:
index = data.rindex("OggS")
except ValueError:
raise error("unable to find final Ogg header")
stringobj = StringIO(data[index:])
best_page = None
try:
page = OggPage(stringobj)
except error:
pass
else:
if page.serial == serial:
if page.last:
return page
else:
best_page = page
else:
best_page = None
# The stream is muxed, so use the slow way.
fileobj.seek(0)
try:
page = OggPage(fileobj)
while not page.last:
page = OggPage(fileobj)
while page.serial != serial:
page = OggPage(fileobj)
best_page = page
return page
except error:
return best_page
except EOFError:
return best_page
class OggFileType(FileType):
"""An generic Ogg file."""
_Info = None
_Tags = None
_Error = None
_mimes = ["application/ogg", "application/x-ogg"]
def load(self, filename):
"""Load file information from a filename."""
self.filename = filename
fileobj = open(filename, "rb")
try:
try:
self.info = self._Info(fileobj)
self.tags = self._Tags(fileobj, self.info)
self.info._post_tags(fileobj)
except error, e:
raise self._Error, e, sys.exc_info()[2]
except EOFError:
raise self._Error, "no appropriate stream found"
finally:
fileobj.close()
def delete(self, filename=None):
"""Remove tags from a file.
If no filename is given, the one most recently loaded is used.
"""
if filename is None:
filename = self.filename
self.tags.clear()
fileobj = open(filename, "rb+")
try:
try:
self.tags._inject(fileobj)
except error, e:
raise self._Error, e, sys.exc_info()[2]
except EOFError:
raise self._Error, "no appropriate stream found"
finally:
fileobj.close()
def save(self, filename=None):
"""Save a tag to a file.
If no filename is given, the one most recently loaded is used.
"""
if filename is None:
filename = self.filename
fileobj = open(filename, "rb+")
try:
try:
self.tags._inject(fileobj)
except error, e:
raise self._Error, e, sys.exc_info()[2]
except EOFError:
raise self._Error, "no appropriate stream found"
finally:
fileobj.close()

147
libs/mutagen/oggflac.py Normal file
View file

@ -0,0 +1,147 @@
# Ogg FLAC support.
#
# Copyright 2006 Joe Wreschnig
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Read and write Ogg FLAC comments.
This module handles FLAC files wrapped in an Ogg bitstream. The first
FLAC stream found is used. For 'naked' FLACs, see mutagen.flac.
This module is based off the specification at
http://flac.sourceforge.net/ogg_mapping.html.
"""
__all__ = ["OggFLAC", "Open", "delete"]
import struct
from cStringIO import StringIO
from mutagen.flac import StreamInfo, VCFLACDict, StrictFileObject
from mutagen.ogg import OggPage, OggFileType, error as OggError
class error(OggError):
pass
class OggFLACHeaderError(error):
pass
class OggFLACStreamInfo(StreamInfo):
"""Ogg FLAC general header and stream info.
This encompasses the Ogg wrapper for the FLAC STREAMINFO metadata
block, as well as the Ogg codec setup that precedes it.
Attributes (in addition to StreamInfo's):
* packets -- number of metadata packets
* serial -- Ogg logical stream serial number
"""
packets = 0
serial = 0
def load(self, data):
# Ogg expects file objects that don't raise on read
if isinstance(data, StrictFileObject):
data = data._fileobj
page = OggPage(data)
while not page.packets[0].startswith("\x7FFLAC"):
page = OggPage(data)
major, minor, self.packets, flac = struct.unpack(
">BBH4s", page.packets[0][5:13])
if flac != "fLaC":
raise OggFLACHeaderError("invalid FLAC marker (%r)" % flac)
elif (major, minor) != (1, 0):
raise OggFLACHeaderError(
"unknown mapping version: %d.%d" % (major, minor))
self.serial = page.serial
# Skip over the block header.
stringobj = StrictFileObject(StringIO(page.packets[0][17:]))
super(OggFLACStreamInfo, self).load(stringobj)
def _post_tags(self, fileobj):
if self.length:
return
page = OggPage.find_last(fileobj, self.serial)
self.length = page.position / float(self.sample_rate)
def pprint(self):
return "Ogg " + super(OggFLACStreamInfo, self).pprint()
class OggFLACVComment(VCFLACDict):
def load(self, data, info, errors='replace'):
# data should be pointing at the start of an Ogg page, after
# the first FLAC page.
pages = []
complete = False
while not complete:
page = OggPage(data)
if page.serial == info.serial:
pages.append(page)
complete = page.complete or (len(page.packets) > 1)
comment = StringIO(OggPage.to_packets(pages)[0][4:])
super(OggFLACVComment, self).load(comment, errors=errors)
def _inject(self, fileobj):
"""Write tag data into the FLAC Vorbis comment packet/page."""
# Ogg FLAC has no convenient data marker like Vorbis, but the
# second packet - and second page - must be the comment data.
fileobj.seek(0)
page = OggPage(fileobj)
while not page.packets[0].startswith("\x7FFLAC"):
page = OggPage(fileobj)
first_page = page
while not (page.sequence == 1 and page.serial == first_page.serial):
page = OggPage(fileobj)
old_pages = [page]
while not (old_pages[-1].complete or len(old_pages[-1].packets) > 1):
page = OggPage(fileobj)
if page.serial == first_page.serial:
old_pages.append(page)
packets = OggPage.to_packets(old_pages, strict=False)
# Set the new comment block.
data = self.write()
data = packets[0][0] + struct.pack(">I", len(data))[-3:] + data
packets[0] = data
new_pages = OggPage.from_packets(packets, old_pages[0].sequence)
OggPage.replace(fileobj, old_pages, new_pages)
class OggFLAC(OggFileType):
"""An Ogg FLAC file."""
_Info = OggFLACStreamInfo
_Tags = OggFLACVComment
_Error = OggFLACHeaderError
_mimes = ["audio/x-oggflac"]
@staticmethod
def score(filename, fileobj, header):
return (header.startswith("OggS") * (
("FLAC" in header) + ("fLaC" in header)))
Open = OggFLAC
def delete(filename):
"""Remove tags from a file."""
OggFLAC(filename).delete()

125
libs/mutagen/oggopus.py Normal file
View file

@ -0,0 +1,125 @@
# Copyright 2012 Christoph Reiter
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Read and write Ogg Opus comments.
This module handles Opus files wrapped in an Ogg bitstream. The
first Opus stream found is used.
Based on http://tools.ietf.org/html/draft-terriberry-oggopus-01
"""
__all__ = ["OggOpus", "Open", "delete"]
import struct
from mutagen._vorbis import VCommentDict
from mutagen.ogg import OggPage, OggFileType, error as OggError
class error(OggError):
pass
class OggOpusHeaderError(error):
pass
class OggOpusInfo(object):
"""Ogg Opus stream information.
Attributes:
* length - file length in seconds, as a float
* channels - number of channels
"""
length = 0
def __init__(self, fileobj):
page = OggPage(fileobj)
while not page.packets[0].startswith("OpusHead"):
page = OggPage(fileobj)
self.serial = page.serial
if not page.first:
raise OggOpusHeaderError(
"page has ID header, but doesn't start a stream")
(version, self.channels, pre_skip, orig_sample_rate, output_gain,
channel_map) = struct.unpack("<BBHIhB", page.packets[0][8:19])
self.__pre_skip = pre_skip
# only the higher 4 bits change on incombatible changes
major, minor = version >> 4, version & 0xF
if major != 0:
raise OggOpusHeaderError("version %r unsupported" % major)
def _post_tags(self, fileobj):
page = OggPage.find_last(fileobj, self.serial)
self.length = (page.position - self.__pre_skip) / float(48000)
def pprint(self):
return "Ogg Opus, %.2f seconds" % (self.length)
class OggOpusVComment(VCommentDict):
"""Opus comments embedded in an Ogg bitstream."""
def __get_comment_pages(self, fileobj, info):
# find the first tags page with the right serial
page = OggPage(fileobj)
while info.serial != page.serial or \
not page.packets[0].startswith("OpusTags"):
page = OggPage(fileobj)
# get all comment pages
pages = [page]
while not (pages[-1].complete or len(pages[-1].packets) > 1):
page = OggPage(fileobj)
if page.serial == pages[0].serial:
pages.append(page)
return pages
def __init__(self, fileobj, info):
pages = self.__get_comment_pages(fileobj, info)
data = OggPage.to_packets(pages)[0][8:] # Strip OpusTags
super(OggOpusVComment, self).__init__(data, framing=False)
def _inject(self, fileobj):
fileobj.seek(0)
info = OggOpusInfo(fileobj)
old_pages = self.__get_comment_pages(fileobj, info)
packets = OggPage.to_packets(old_pages)
packets[0] = "OpusTags" + self.write(framing=False)
new_pages = OggPage.from_packets(packets, old_pages[0].sequence)
OggPage.replace(fileobj, old_pages, new_pages)
class OggOpus(OggFileType):
"""An Ogg Opus file."""
_Info = OggOpusInfo
_Tags = OggOpusVComment
_Error = OggOpusHeaderError
_mimes = ["audio/ogg", "audio/ogg; codecs=opus"]
@staticmethod
def score(filename, fileobj, header):
return (header.startswith("OggS") * ("OpusHead" in header))
Open = OggOpus
def delete(filename):
"""Remove tags from a file."""
OggOpus(filename).delete()

137
libs/mutagen/oggspeex.py Normal file
View file

@ -0,0 +1,137 @@
# Ogg Speex support.
#
# Copyright 2006 Joe Wreschnig
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Read and write Ogg Speex comments.
This module handles Speex files wrapped in an Ogg bitstream. The
first Speex stream found is used.
Read more about Ogg Speex at http://www.speex.org/. This module is
based on the specification at http://www.speex.org/manual2/node7.html
and clarifications after personal communication with Jean-Marc,
http://lists.xiph.org/pipermail/speex-dev/2006-July/004676.html.
"""
__all__ = ["OggSpeex", "Open", "delete"]
from mutagen._vorbis import VCommentDict
from mutagen.ogg import OggPage, OggFileType, error as OggError
from mutagen._util import cdata
class error(OggError):
pass
class OggSpeexHeaderError(error):
pass
class OggSpeexInfo(object):
"""Ogg Speex stream information.
Attributes:
* bitrate - nominal bitrate in bits per second
* channels - number of channels
* length - file length in seconds, as a float
The reference encoder does not set the bitrate; in this case,
the bitrate will be 0.
"""
length = 0
def __init__(self, fileobj):
page = OggPage(fileobj)
while not page.packets[0].startswith("Speex "):
page = OggPage(fileobj)
if not page.first:
raise OggSpeexHeaderError(
"page has ID header, but doesn't start a stream")
self.sample_rate = cdata.uint_le(page.packets[0][36:40])
self.channels = cdata.uint_le(page.packets[0][48:52])
self.bitrate = max(0, cdata.int_le(page.packets[0][52:56]))
self.serial = page.serial
def _post_tags(self, fileobj):
page = OggPage.find_last(fileobj, self.serial)
self.length = page.position / float(self.sample_rate)
def pprint(self):
return "Ogg Speex, %.2f seconds" % self.length
class OggSpeexVComment(VCommentDict):
"""Speex comments embedded in an Ogg bitstream."""
def __init__(self, fileobj, info):
pages = []
complete = False
while not complete:
page = OggPage(fileobj)
if page.serial == info.serial:
pages.append(page)
complete = page.complete or (len(page.packets) > 1)
data = OggPage.to_packets(pages)[0] + "\x01"
super(OggSpeexVComment, self).__init__(data, framing=False)
def _inject(self, fileobj):
"""Write tag data into the Speex comment packet/page."""
fileobj.seek(0)
# Find the first header page, with the stream info.
# Use it to get the serial number.
page = OggPage(fileobj)
while not page.packets[0].startswith("Speex "):
page = OggPage(fileobj)
# Look for the next page with that serial number, it'll start
# the comment packet.
serial = page.serial
page = OggPage(fileobj)
while page.serial != serial:
page = OggPage(fileobj)
# Then find all the pages with the comment packet.
old_pages = [page]
while not (old_pages[-1].complete or len(old_pages[-1].packets) > 1):
page = OggPage(fileobj)
if page.serial == old_pages[0].serial:
old_pages.append(page)
packets = OggPage.to_packets(old_pages, strict=False)
# Set the new comment packet.
packets[0] = self.write(framing=False)
new_pages = OggPage.from_packets(packets, old_pages[0].sequence)
OggPage.replace(fileobj, old_pages, new_pages)
class OggSpeex(OggFileType):
"""An Ogg Speex file."""
_Info = OggSpeexInfo
_Tags = OggSpeexVComment
_Error = OggSpeexHeaderError
_mimes = ["audio/x-speex"]
@staticmethod
def score(filename, fileobj, header):
return (header.startswith("OggS") * ("Speex " in header))
Open = OggSpeex
def delete(filename):
"""Remove tags from a file."""
OggSpeex(filename).delete()

130
libs/mutagen/oggtheora.py Normal file
View file

@ -0,0 +1,130 @@
# Ogg Theora support.
#
# Copyright 2006 Joe Wreschnig
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Read and write Ogg Theora comments.
This module handles Theora files wrapped in an Ogg bitstream. The
first Theora stream found is used.
Based on the specification at http://theora.org/doc/Theora_I_spec.pdf.
"""
__all__ = ["OggTheora", "Open", "delete"]
import struct
from mutagen._vorbis import VCommentDict
from mutagen._util import cdata
from mutagen.ogg import OggPage, OggFileType, error as OggError
class error(OggError):
pass
class OggTheoraHeaderError(error):
pass
class OggTheoraInfo(object):
"""Ogg Theora stream information.
Attributes:
* length - file length in seconds, as a float
* fps - video frames per second, as a float
"""
length = 0
def __init__(self, fileobj):
page = OggPage(fileobj)
while not page.packets[0].startswith("\x80theora"):
page = OggPage(fileobj)
if not page.first:
raise OggTheoraHeaderError(
"page has ID header, but doesn't start a stream")
data = page.packets[0]
vmaj, vmin = struct.unpack("2B", data[7:9])
if (vmaj, vmin) != (3, 2):
raise OggTheoraHeaderError(
"found Theora version %d.%d != 3.2" % (vmaj, vmin))
fps_num, fps_den = struct.unpack(">2I", data[22:30])
self.fps = fps_num / float(fps_den)
self.bitrate = cdata.uint_be("\x00" + data[37:40])
self.granule_shift = (cdata.ushort_be(data[40:42]) >> 5) & 0x1F
self.serial = page.serial
def _post_tags(self, fileobj):
page = OggPage.find_last(fileobj, self.serial)
position = page.position
mask = (1 << self.granule_shift) - 1
frames = (position >> self.granule_shift) + (position & mask)
self.length = frames / float(self.fps)
def pprint(self):
return "Ogg Theora, %.2f seconds, %d bps" % (self.length, self.bitrate)
class OggTheoraCommentDict(VCommentDict):
"""Theora comments embedded in an Ogg bitstream."""
def __init__(self, fileobj, info):
pages = []
complete = False
while not complete:
page = OggPage(fileobj)
if page.serial == info.serial:
pages.append(page)
complete = page.complete or (len(page.packets) > 1)
data = OggPage.to_packets(pages)[0][7:]
super(OggTheoraCommentDict, self).__init__(data + "\x01")
def _inject(self, fileobj):
"""Write tag data into the Theora comment packet/page."""
fileobj.seek(0)
page = OggPage(fileobj)
while not page.packets[0].startswith("\x81theora"):
page = OggPage(fileobj)
old_pages = [page]
while not (old_pages[-1].complete or len(old_pages[-1].packets) > 1):
page = OggPage(fileobj)
if page.serial == old_pages[0].serial:
old_pages.append(page)
packets = OggPage.to_packets(old_pages, strict=False)
packets[0] = "\x81theora" + self.write(framing=False)
new_pages = OggPage.from_packets(packets, old_pages[0].sequence)
OggPage.replace(fileobj, old_pages, new_pages)
class OggTheora(OggFileType):
"""An Ogg Theora file."""
_Info = OggTheoraInfo
_Tags = OggTheoraCommentDict
_Error = OggTheoraHeaderError
_mimes = ["video/x-theora"]
@staticmethod
def score(filename, fileobj, header):
return (header.startswith("OggS") *
(("\x80theora" in header) + ("\x81theora" in header)))
Open = OggTheora
def delete(filename):
"""Remove tags from a file."""
OggTheora(filename).delete()

137
libs/mutagen/oggvorbis.py Normal file
View file

@ -0,0 +1,137 @@
# Ogg Vorbis support.
#
# Copyright 2006 Joe Wreschnig
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Read and write Ogg Vorbis comments.
This module handles Vorbis files wrapped in an Ogg bitstream. The
first Vorbis stream found is used.
Read more about Ogg Vorbis at http://vorbis.com/. This module is based
on the specification at http://www.xiph.org/vorbis/doc/Vorbis_I_spec.html.
"""
__all__ = ["OggVorbis", "Open", "delete"]
import struct
from mutagen._vorbis import VCommentDict
from mutagen.ogg import OggPage, OggFileType, error as OggError
class error(OggError):
pass
class OggVorbisHeaderError(error):
pass
class OggVorbisInfo(object):
"""Ogg Vorbis stream information.
Attributes:
* length - file length in seconds, as a float
* bitrate - nominal ('average') bitrate in bits per second, as an int
"""
length = 0
def __init__(self, fileobj):
page = OggPage(fileobj)
while not page.packets[0].startswith("\x01vorbis"):
page = OggPage(fileobj)
if not page.first:
raise OggVorbisHeaderError(
"page has ID header, but doesn't start a stream")
(self.channels, self.sample_rate, max_bitrate, nominal_bitrate,
min_bitrate) = struct.unpack("<B4i", page.packets[0][11:28])
self.serial = page.serial
max_bitrate = max(0, max_bitrate)
min_bitrate = max(0, min_bitrate)
nominal_bitrate = max(0, nominal_bitrate)
if nominal_bitrate == 0:
self.bitrate = (max_bitrate + min_bitrate) // 2
elif max_bitrate and max_bitrate < nominal_bitrate:
# If the max bitrate is less than the nominal, we know
# the nominal is wrong.
self.bitrate = max_bitrate
elif min_bitrate > nominal_bitrate:
self.bitrate = min_bitrate
else:
self.bitrate = nominal_bitrate
def _post_tags(self, fileobj):
page = OggPage.find_last(fileobj, self.serial)
self.length = page.position / float(self.sample_rate)
def pprint(self):
return "Ogg Vorbis, %.2f seconds, %d bps" % (self.length, self.bitrate)
class OggVCommentDict(VCommentDict):
"""Vorbis comments embedded in an Ogg bitstream."""
def __init__(self, fileobj, info):
pages = []
complete = False
while not complete:
page = OggPage(fileobj)
if page.serial == info.serial:
pages.append(page)
complete = page.complete or (len(page.packets) > 1)
data = OggPage.to_packets(pages)[0][7:] # Strip off "\x03vorbis".
super(OggVCommentDict, self).__init__(data)
def _inject(self, fileobj):
"""Write tag data into the Vorbis comment packet/page."""
# Find the old pages in the file; we'll need to remove them,
# plus grab any stray setup packet data out of them.
fileobj.seek(0)
page = OggPage(fileobj)
while not page.packets[0].startswith("\x03vorbis"):
page = OggPage(fileobj)
old_pages = [page]
while not (old_pages[-1].complete or len(old_pages[-1].packets) > 1):
page = OggPage(fileobj)
if page.serial == old_pages[0].serial:
old_pages.append(page)
packets = OggPage.to_packets(old_pages, strict=False)
# Set the new comment packet.
packets[0] = "\x03vorbis" + self.write()
new_pages = OggPage.from_packets(packets, old_pages[0].sequence)
OggPage.replace(fileobj, old_pages, new_pages)
class OggVorbis(OggFileType):
"""An Ogg Vorbis file."""
_Info = OggVorbisInfo
_Tags = OggVCommentDict
_Error = OggVorbisHeaderError
_mimes = ["audio/vorbis", "audio/x-vorbis"]
@staticmethod
def score(filename, fileobj, header):
return (header.startswith("OggS") * ("\x01vorbis" in header))
Open = OggVorbis
def delete(filename):
"""Remove tags from a file."""
OggVorbis(filename).delete()

71
libs/mutagen/optimfrog.py Normal file
View file

@ -0,0 +1,71 @@
# OptimFROG reader/tagger
#
# Copyright 2006 Lukas Lalinsky <lalinsky@gmail.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""OptimFROG audio streams with APEv2 tags.
OptimFROG is a lossless audio compression program. Its main goal is to
reduce at maximum the size of audio files, while permitting bit
identical restoration for all input. It is similar with the ZIP
compression, but it is highly specialized to compress audio data.
Only versions 4.5 and higher are supported.
For more information, see http://www.losslessaudio.org/
"""
__all__ = ["OptimFROG", "Open", "delete"]
import struct
from mutagen.apev2 import APEv2File, error, delete
class OptimFROGHeaderError(error):
pass
class OptimFROGInfo(object):
"""OptimFROG stream information.
Attributes:
* channels - number of audio channels
* length - file length in seconds, as a float
* sample_rate - audio sampling rate in Hz
"""
def __init__(self, fileobj):
header = fileobj.read(76)
if (len(header) != 76 or not header.startswith("OFR ") or
struct.unpack("<I", header[4:8])[0] not in [12, 15]):
raise OptimFROGHeaderError("not an OptimFROG file")
(total_samples, total_samples_high, sample_type, self.channels,
self.sample_rate) = struct.unpack("<IHBBI", header[8:20])
total_samples += total_samples_high << 32
self.channels += 1
if self.sample_rate:
self.length = float(total_samples) / (self.channels *
self.sample_rate)
else:
self.length = 0.0
def pprint(self):
return "OptimFROG, %.2f seconds, %d Hz" % (self.length,
self.sample_rate)
class OptimFROG(APEv2File):
_Info = OptimFROGInfo
@staticmethod
def score(filename, fileobj, header):
filename = filename.lower()
return (header.startswith("OFR") + filename.endswith(".ofr") +
filename.endswith(".ofs"))
Open = OptimFROG

81
libs/mutagen/trueaudio.py Normal file
View file

@ -0,0 +1,81 @@
# True Audio support for Mutagen
# Copyright 2006 Joe Wreschnig
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of version 2 of the GNU General Public License as
# published by the Free Software Foundation.
"""True Audio audio stream information and tags.
True Audio is a lossless format designed for real-time encoding and
decoding. This module is based on the documentation at
http://www.true-audio.com/TTA_Lossless_Audio_Codec\_-_Format_Description
True Audio files use ID3 tags.
"""
__all__ = ["TrueAudio", "Open", "delete", "EasyTrueAudio"]
from mutagen.id3 import ID3FileType, delete
from mutagen._util import cdata
class error(RuntimeError):
pass
class TrueAudioHeaderError(error, IOError):
pass
class TrueAudioInfo(object):
"""True Audio stream information.
Attributes:
* length - audio length, in seconds
* sample_rate - audio sample rate, in Hz
"""
def __init__(self, fileobj, offset):
fileobj.seek(offset or 0)
header = fileobj.read(18)
if len(header) != 18 or not header.startswith("TTA"):
raise TrueAudioHeaderError("TTA header not found")
self.sample_rate = cdata.int_le(header[10:14])
samples = cdata.uint_le(header[14:18])
self.length = float(samples) / self.sample_rate
def pprint(self):
return "True Audio, %.2f seconds, %d Hz." % (
self.length, self.sample_rate)
class TrueAudio(ID3FileType):
"""A True Audio file.
:ivar info: :class:`TrueAudioInfo`
:ivar tags: :class:`ID3 <mutagen.id3.ID3>`
"""
_Info = TrueAudioInfo
_mimes = ["audio/x-tta"]
@staticmethod
def score(filename, fileobj, header):
return (header.startswith("ID3") + header.startswith("TTA") +
filename.lower().endswith(".tta") * 2)
Open = TrueAudio
class EasyTrueAudio(TrueAudio):
"""Like MP3, but uses EasyID3 for tags.
:ivar info: :class:`TrueAudioInfo`
:ivar tags: :class:`EasyID3 <mutagen.easyid3.EasyID3>`
"""
from mutagen.easyid3 import EasyID3 as ID3
ID3 = ID3

63
libs/mutagen/wavpack.py Normal file
View file

@ -0,0 +1,63 @@
# A WavPack reader/tagger
#
# Copyright 2006 Joe Wreschnig
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""WavPack reading and writing.
WavPack is a lossless format that uses APEv2 tags. Read
http://www.wavpack.com/ for more information.
"""
__all__ = ["WavPack", "Open", "delete"]
from mutagen.apev2 import APEv2File, error, delete
from mutagen._util import cdata
class WavPackHeaderError(error):
pass
RATES = [6000, 8000, 9600, 11025, 12000, 16000, 22050, 24000, 32000, 44100,
48000, 64000, 88200, 96000, 192000]
class WavPackInfo(object):
"""WavPack stream information.
Attributes:
* channels - number of audio channels (1 or 2)
* length - file length in seconds, as a float
* sample_rate - audio sampling rate in Hz
* version - WavPack stream version
"""
def __init__(self, fileobj):
header = fileobj.read(28)
if len(header) != 28 or not header.startswith("wvpk"):
raise WavPackHeaderError("not a WavPack file")
samples = cdata.uint_le(header[12:16])
flags = cdata.uint_le(header[24:28])
self.version = cdata.short_le(header[8:10])
self.channels = bool(flags & 4) or 2
self.sample_rate = RATES[(flags >> 23) & 0xF]
self.length = float(samples) / self.sample_rate
def pprint(self):
return "WavPack, %.2f seconds, %d Hz" % (self.length, self.sample_rate)
class WavPack(APEv2File):
_Info = WavPackInfo
_mimes = ["audio/x-wavpack"]
@staticmethod
def score(filename, fileobj, header):
return header.startswith("wvpk") * 2
Open = WavPack

63
libs/unidecode/__init__.py Normal file
View file

@ -0,0 +1,63 @@
# -*- coding: utf-8 -*-
"""Transliterate Unicode text into plain 7-bit ASCII.
Example usage:
>>> from unidecode import unidecode:
>>> unidecode(u"\u5317\u4EB0")
"Bei Jing "
The transliteration uses a straightforward map, and doesn't have alternatives
for the same character based on language, position, or anything else.
In Python 3, a standard string object will be returned. If you need bytes, use:
>>> unidecode("Κνωσός").encode("ascii")
b'Knosos'
"""
import warnings
from sys import version_info
Cache = {}
def unidecode(string):
"""Transliterate an Unicode object into an ASCII string
>>> unidecode(u"\u5317\u4EB0")
"Bei Jing "
"""
if version_info[0] < 3 and not isinstance(string, unicode):
warnings.warn( "Argument %r is not an unicode object. "
"Passing an encoded string will likely have "
"unexpected results." % (type(string),),
RuntimeWarning, 2)
retval = []
for char in string:
codepoint = ord(char)
if codepoint < 0x80: # Basic ASCII
retval.append(str(char))
continue
if codepoint > 0xeffff:
continue # Characters in Private Use Area and above are ignored
section = codepoint >> 8 # Chop off the last two hex digits
position = codepoint % 256 # Last two hex digits
try:
table = Cache[section]
except KeyError:
try:
mod = __import__('unidecode.x%03x'%(section), [], [], ['data'])
except ImportError:
Cache[section] = None
continue # No match: ignore this character and carry on.
Cache[section] = table = mod.data
if table and len(table) > position:
retval.append( table[position] )
return ''.join(retval)

258
libs/unidecode/x000.py Normal file
View file

@ -0,0 +1,258 @@
data = (
'\x00', # 0x00
'\x01', # 0x01
'\x02', # 0x02
'\x03', # 0x03
'\x04', # 0x04
'\x05', # 0x05
'\x06', # 0x06
'\x07', # 0x07
'\x08', # 0x08
'\x09', # 0x09
'\x0a', # 0x0a
'\x0b', # 0x0b
'\x0c', # 0x0c
'\x0d', # 0x0d
'\x0e', # 0x0e
'\x0f', # 0x0f
'\x10', # 0x10
'\x11', # 0x11
'\x12', # 0x12
'\x13', # 0x13
'\x14', # 0x14
'\x15', # 0x15
'\x16', # 0x16
'\x17', # 0x17
'\x18', # 0x18
'\x19', # 0x19
'\x1a', # 0x1a
'\x1b', # 0x1b
'\x1c', # 0x1c
'\x1d', # 0x1d
'\x1e', # 0x1e
'\x1f', # 0x1f
' ', # 0x20
'!', # 0x21
'"', # 0x22
'#', # 0x23
'$', # 0x24
'%', # 0x25
'&', # 0x26
'\'', # 0x27
'(', # 0x28
')', # 0x29
'*', # 0x2a
'+', # 0x2b
',', # 0x2c
'-', # 0x2d
'.', # 0x2e
'/', # 0x2f
'0', # 0x30
'1', # 0x31
'2', # 0x32
'3', # 0x33
'4', # 0x34
'5', # 0x35
'6', # 0x36
'7', # 0x37
'8', # 0x38
'9', # 0x39
':', # 0x3a
';', # 0x3b
'<', # 0x3c
'=', # 0x3d
'>', # 0x3e
'?', # 0x3f
'@', # 0x40
'A', # 0x41
'B', # 0x42
'C', # 0x43
'D', # 0x44
'E', # 0x45
'F', # 0x46
'G', # 0x47
'H', # 0x48
'I', # 0x49
'J', # 0x4a
'K', # 0x4b
'L', # 0x4c
'M', # 0x4d
'N', # 0x4e
'O', # 0x4f
'P', # 0x50
'Q', # 0x51
'R', # 0x52
'S', # 0x53
'T', # 0x54
'U', # 0x55
'V', # 0x56
'W', # 0x57
'X', # 0x58
'Y', # 0x59
'Z', # 0x5a
']', # 0x5b
'\\', # 0x5c
']', # 0x5d
'^', # 0x5e
'_', # 0x5f
'`', # 0x60
'a', # 0x61
'b', # 0x62
'c', # 0x63
'd', # 0x64
'e', # 0x65
'f', # 0x66
'g', # 0x67
'h', # 0x68
'i', # 0x69
'j', # 0x6a
'k', # 0x6b
'l', # 0x6c
'm', # 0x6d
'n', # 0x6e
'o', # 0x6f
'p', # 0x70
'q', # 0x71
'r', # 0x72
's', # 0x73
't', # 0x74
'u', # 0x75
'v', # 0x76
'w', # 0x77
'x', # 0x78
'y', # 0x79
'z', # 0x7a
'{', # 0x7b
'|', # 0x7c
'}', # 0x7d
'~', # 0x7e
'', # 0x7f
'', # 0x80
'', # 0x81
'', # 0x82
'', # 0x83
'', # 0x84
'', # 0x85
'', # 0x86
'', # 0x87
'', # 0x88
'', # 0x89
'', # 0x8a
'', # 0x8b
'', # 0x8c
'', # 0x8d
'', # 0x8e
'', # 0x8f
'', # 0x90
'', # 0x91
'', # 0x92
'', # 0x93
'', # 0x94
'', # 0x95
'', # 0x96
'', # 0x97
'', # 0x98
'', # 0x99
'', # 0x9a
'', # 0x9b
'', # 0x9c
'', # 0x9d
'', # 0x9e
'', # 0x9f
' ', # 0xa0
'!', # 0xa1
'C/', # 0xa2
'PS', # 0xa3
'$?', # 0xa4
'Y=', # 0xa5
'|', # 0xa6
'SS', # 0xa7
'"', # 0xa8
'(c)', # 0xa9
'a', # 0xaa
'<<', # 0xab
'!', # 0xac
'', # 0xad
'(r)', # 0xae
'-', # 0xaf
'deg', # 0xb0
'+-', # 0xb1
'2', # 0xb2
'3', # 0xb3
'\'', # 0xb4
'u', # 0xb5
'P', # 0xb6
'*', # 0xb7
',', # 0xb8
'1', # 0xb9
'o', # 0xba
'>>', # 0xbb
'1/4', # 0xbc
'1/2', # 0xbd
'3/4', # 0xbe
'?', # 0xbf
'A', # 0xc0
'A', # 0xc1
'A', # 0xc2
'A', # 0xc3
'A', # 0xc4
'A', # 0xc5
'AE', # 0xc6
'C', # 0xc7
'E', # 0xc8
'E', # 0xc9
'E', # 0xca
'E', # 0xcb
'I', # 0xcc
'I', # 0xcd
'I', # 0xce
'I', # 0xcf
'D', # 0xd0
'N', # 0xd1
'O', # 0xd2
'O', # 0xd3
'O', # 0xd4
'O', # 0xd5
'O', # 0xd6
'x', # 0xd7
'O', # 0xd8
'U', # 0xd9
'U', # 0xda
'U', # 0xdb
'U', # 0xdc
'Y', # 0xdd
'Th', # 0xde
'ss', # 0xdf
'a', # 0xe0
'a', # 0xe1
'a', # 0xe2
'a', # 0xe3
'a', # 0xe4
'a', # 0xe5
'ae', # 0xe6
'c', # 0xe7
'e', # 0xe8
'e', # 0xe9
'e', # 0xea
'e', # 0xeb
'i', # 0xec
'i', # 0xed
'i', # 0xee
'i', # 0xef
'd', # 0xf0
'n', # 0xf1
'o', # 0xf2
'o', # 0xf3
'o', # 0xf4
'o', # 0xf5
'o', # 0xf6
'/', # 0xf7
'o', # 0xf8
'u', # 0xf9
'u', # 0xfa
'u', # 0xfb
'u', # 0xfc
'y', # 0xfd
'th', # 0xfe
'y', # 0xff
)

258
libs/unidecode/x001.py Normal file
View file

@ -0,0 +1,258 @@
data = (
'A', # 0x00
'a', # 0x01
'A', # 0x02
'a', # 0x03
'A', # 0x04
'a', # 0x05
'C', # 0x06
'c', # 0x07
'C', # 0x08
'c', # 0x09
'C', # 0x0a
'c', # 0x0b
'C', # 0x0c
'c', # 0x0d
'D', # 0x0e
'd', # 0x0f
'D', # 0x10
'd', # 0x11
'E', # 0x12
'e', # 0x13
'E', # 0x14
'e', # 0x15
'E', # 0x16
'e', # 0x17
'E', # 0x18
'e', # 0x19
'E', # 0x1a
'e', # 0x1b
'G', # 0x1c
'g', # 0x1d
'G', # 0x1e
'g', # 0x1f
'G', # 0x20
'g', # 0x21
'G', # 0x22
'g', # 0x23
'H', # 0x24
'h', # 0x25
'H', # 0x26
'h', # 0x27
'I', # 0x28
'i', # 0x29
'I', # 0x2a
'i', # 0x2b
'I', # 0x2c
'i', # 0x2d
'I', # 0x2e
'i', # 0x2f
'I', # 0x30
'i', # 0x31
'IJ', # 0x32
'ij', # 0x33
'J', # 0x34
'j', # 0x35
'K', # 0x36
'k', # 0x37
'k', # 0x38
'L', # 0x39
'l', # 0x3a
'L', # 0x3b
'l', # 0x3c
'L', # 0x3d
'l', # 0x3e
'L', # 0x3f
'l', # 0x40
'L', # 0x41
'l', # 0x42
'N', # 0x43
'n', # 0x44
'N', # 0x45
'n', # 0x46
'N', # 0x47
'n', # 0x48
'\'n', # 0x49
'ng', # 0x4a
'NG', # 0x4b
'O', # 0x4c
'o', # 0x4d
'O', # 0x4e
'o', # 0x4f
'O', # 0x50
'o', # 0x51
'OE', # 0x52
'oe', # 0x53
'R', # 0x54
'r', # 0x55
'R', # 0x56
'r', # 0x57
'R', # 0x58
'r', # 0x59
'S', # 0x5a
's', # 0x5b
'S', # 0x5c
's', # 0x5d
'S', # 0x5e
's', # 0x5f
'S', # 0x60
's', # 0x61
'T', # 0x62
't', # 0x63
'T', # 0x64
't', # 0x65
'T', # 0x66
't', # 0x67
'U', # 0x68
'u', # 0x69
'U', # 0x6a
'u', # 0x6b
'U', # 0x6c
'u', # 0x6d
'U', # 0x6e
'u', # 0x6f
'U', # 0x70
'u', # 0x71
'U', # 0x72
'u', # 0x73
'W', # 0x74
'w', # 0x75
'Y', # 0x76
'y', # 0x77
'Y', # 0x78
'Z', # 0x79
'z', # 0x7a
'Z', # 0x7b
'z', # 0x7c
'Z', # 0x7d
'z', # 0x7e
's', # 0x7f
'b', # 0x80
'B', # 0x81
'B', # 0x82
'b', # 0x83
'6', # 0x84
'6', # 0x85
'O', # 0x86
'C', # 0x87
'c', # 0x88
'D', # 0x89
'D', # 0x8a
'D', # 0x8b
'd', # 0x8c
'd', # 0x8d
'3', # 0x8e
'@', # 0x8f
'E', # 0x90
'F', # 0x91
'f', # 0x92
'G', # 0x93
'G', # 0x94
'hv', # 0x95
'I', # 0x96
'I', # 0x97
'K', # 0x98
'k', # 0x99
'l', # 0x9a
'l', # 0x9b
'W', # 0x9c
'N', # 0x9d
'n', # 0x9e
'O', # 0x9f
'O', # 0xa0
'o', # 0xa1
'OI', # 0xa2
'oi', # 0xa3
'P', # 0xa4
'p', # 0xa5
'YR', # 0xa6
'2', # 0xa7
'2', # 0xa8
'SH', # 0xa9
'sh', # 0xaa
't', # 0xab
'T', # 0xac
't', # 0xad
'T', # 0xae
'U', # 0xaf
'u', # 0xb0
'Y', # 0xb1
'V', # 0xb2
'Y', # 0xb3
'y', # 0xb4
'Z', # 0xb5
'z', # 0xb6
'ZH', # 0xb7
'ZH', # 0xb8
'zh', # 0xb9
'zh', # 0xba
'2', # 0xbb
'5', # 0xbc
'5', # 0xbd
'ts', # 0xbe
'w', # 0xbf
'|', # 0xc0
'||', # 0xc1
'|=', # 0xc2
'!', # 0xc3
'DZ', # 0xc4
'Dz', # 0xc5
'dz', # 0xc6
'LJ', # 0xc7
'Lj', # 0xc8
'lj', # 0xc9
'NJ', # 0xca
'Nj', # 0xcb
'nj', # 0xcc
'A', # 0xcd
'a', # 0xce
'I', # 0xcf
'i', # 0xd0
'O', # 0xd1
'o', # 0xd2
'U', # 0xd3
'u', # 0xd4
'U', # 0xd5
'u', # 0xd6
'U', # 0xd7
'u', # 0xd8
'U', # 0xd9
'u', # 0xda
'U', # 0xdb
'u', # 0xdc
'@', # 0xdd
'A', # 0xde
'a', # 0xdf
'A', # 0xe0
'a', # 0xe1
'AE', # 0xe2
'ae', # 0xe3
'G', # 0xe4
'g', # 0xe5
'G', # 0xe6
'g', # 0xe7
'K', # 0xe8
'k', # 0xe9
'O', # 0xea
'o', # 0xeb
'O', # 0xec
'o', # 0xed
'ZH', # 0xee
'zh', # 0xef
'j', # 0xf0
'DZ', # 0xf1
'Dz', # 0xf2
'dz', # 0xf3
'G', # 0xf4
'g', # 0xf5
'HV', # 0xf6
'W', # 0xf7
'N', # 0xf8
'n', # 0xf9
'A', # 0xfa
'a', # 0xfb
'AE', # 0xfc
'ae', # 0xfd
'O', # 0xfe
'o', # 0xff
)

257
libs/unidecode/x002.py Normal file
View file

@ -0,0 +1,257 @@
data = (
'A', # 0x00
'a', # 0x01
'A', # 0x02
'a', # 0x03
'E', # 0x04
'e', # 0x05
'E', # 0x06
'e', # 0x07
'I', # 0x08
'i', # 0x09
'I', # 0x0a
'i', # 0x0b
'O', # 0x0c
'o', # 0x0d
'O', # 0x0e
'o', # 0x0f
'R', # 0x10
'r', # 0x11
'R', # 0x12
'r', # 0x13
'U', # 0x14
'u', # 0x15
'U', # 0x16
'u', # 0x17
'S', # 0x18
's', # 0x19
'T', # 0x1a
't', # 0x1b
'Y', # 0x1c
'y', # 0x1d
'H', # 0x1e
'h', # 0x1f
'N', # 0x20
'd', # 0x21
'OU', # 0x22
'ou', # 0x23
'Z', # 0x24
'z', # 0x25
'A', # 0x26
'a', # 0x27
'E', # 0x28
'e', # 0x29
'O', # 0x2a
'o', # 0x2b
'O', # 0x2c
'o', # 0x2d
'O', # 0x2e
'o', # 0x2f
'O', # 0x30
'o', # 0x31
'Y', # 0x32
'y', # 0x33
'l', # 0x34
'n', # 0x35
't', # 0x36
'j', # 0x37
'db', # 0x38
'qp', # 0x39
'A', # 0x3a
'C', # 0x3b
'c', # 0x3c
'L', # 0x3d
'T', # 0x3e
's', # 0x3f
'z', # 0x40
'[?]', # 0x41
'[?]', # 0x42
'B', # 0x43
'U', # 0x44
'^', # 0x45
'E', # 0x46
'e', # 0x47
'J', # 0x48
'j', # 0x49
'q', # 0x4a
'q', # 0x4b
'R', # 0x4c
'r', # 0x4d
'Y', # 0x4e
'y', # 0x4f
'a', # 0x50
'a', # 0x51
'a', # 0x52
'b', # 0x53
'o', # 0x54
'c', # 0x55
'd', # 0x56
'd', # 0x57
'e', # 0x58
'@', # 0x59
'@', # 0x5a
'e', # 0x5b
'e', # 0x5c
'e', # 0x5d
'e', # 0x5e
'j', # 0x5f
'g', # 0x60
'g', # 0x61
'g', # 0x62
'g', # 0x63
'u', # 0x64
'Y', # 0x65
'h', # 0x66
'h', # 0x67
'i', # 0x68
'i', # 0x69
'I', # 0x6a
'l', # 0x6b
'l', # 0x6c
'l', # 0x6d
'lZ', # 0x6e
'W', # 0x6f
'W', # 0x70
'm', # 0x71
'n', # 0x72
'n', # 0x73
'n', # 0x74
'o', # 0x75
'OE', # 0x76
'O', # 0x77
'F', # 0x78
'r', # 0x79
'r', # 0x7a
'r', # 0x7b
'r', # 0x7c
'r', # 0x7d
'r', # 0x7e
'r', # 0x7f
'R', # 0x80
'R', # 0x81
's', # 0x82
'S', # 0x83
'j', # 0x84
'S', # 0x85
'S', # 0x86
't', # 0x87
't', # 0x88
'u', # 0x89
'U', # 0x8a
'v', # 0x8b
'^', # 0x8c
'w', # 0x8d
'y', # 0x8e
'Y', # 0x8f
'z', # 0x90
'z', # 0x91
'Z', # 0x92
'Z', # 0x93
'?', # 0x94
'?', # 0x95
'?', # 0x96
'C', # 0x97
'@', # 0x98
'B', # 0x99
'E', # 0x9a
'G', # 0x9b
'H', # 0x9c
'j', # 0x9d
'k', # 0x9e
'L', # 0x9f
'q', # 0xa0
'?', # 0xa1
'?', # 0xa2
'dz', # 0xa3
'dZ', # 0xa4
'dz', # 0xa5
'ts', # 0xa6
'tS', # 0xa7
'tC', # 0xa8
'fN', # 0xa9
'ls', # 0xaa
'lz', # 0xab
'WW', # 0xac
']]', # 0xad
'h', # 0xae
'h', # 0xaf
'k', # 0xb0
'h', # 0xb1
'j', # 0xb2
'r', # 0xb3
'r', # 0xb4
'r', # 0xb5
'r', # 0xb6
'w', # 0xb7
'y', # 0xb8
'\'', # 0xb9
'"', # 0xba
'`', # 0xbb
'\'', # 0xbc
'`', # 0xbd
'`', # 0xbe
'\'', # 0xbf
'?', # 0xc0
'?', # 0xc1
'<', # 0xc2
'>', # 0xc3
'^', # 0xc4
'V', # 0xc5
'^', # 0xc6
'V', # 0xc7
'\'', # 0xc8
'-', # 0xc9
'/', # 0xca
'\\', # 0xcb
',', # 0xcc
'_', # 0xcd
'\\', # 0xce
'/', # 0xcf
':', # 0xd0
'.', # 0xd1
'`', # 0xd2
'\'', # 0xd3
'^', # 0xd4
'V', # 0xd5
'+', # 0xd6
'-', # 0xd7
'V', # 0xd8
'.', # 0xd9
'@', # 0xda
',', # 0xdb
'~', # 0xdc
'"', # 0xdd
'R', # 0xde
'X', # 0xdf
'G', # 0xe0
'l', # 0xe1
's', # 0xe2
'x', # 0xe3
'?', # 0xe4
'', # 0xe5
'', # 0xe6
'', # 0xe7
'', # 0xe8
'', # 0xe9
'', # 0xea
'', # 0xeb
'V', # 0xec
'=', # 0xed
'"', # 0xee
'[?]', # 0xef
'[?]', # 0xf0
'[?]', # 0xf1
'[?]', # 0xf2
'[?]', # 0xf3
'[?]', # 0xf4
'[?]', # 0xf5
'[?]', # 0xf6
'[?]', # 0xf7
'[?]', # 0xf8
'[?]', # 0xf9
'[?]', # 0xfa
'[?]', # 0xfb
'[?]', # 0xfc
'[?]', # 0xfd
'[?]', # 0xfe
)

257
libs/unidecode/x003.py Normal file
View file

@ -0,0 +1,257 @@
data = (
'', # 0x00
'', # 0x01
'', # 0x02
'', # 0x03
'', # 0x04
'', # 0x05
'', # 0x06
'', # 0x07
'', # 0x08
'', # 0x09
'', # 0x0a
'', # 0x0b
'', # 0x0c
'', # 0x0d
'', # 0x0e
'', # 0x0f
'', # 0x10
'', # 0x11
'', # 0x12
'', # 0x13
'', # 0x14
'', # 0x15
'', # 0x16
'', # 0x17
'', # 0x18
'', # 0x19
'', # 0x1a
'', # 0x1b
'', # 0x1c
'', # 0x1d
'', # 0x1e
'', # 0x1f
'', # 0x20
'', # 0x21
'', # 0x22
'', # 0x23
'', # 0x24
'', # 0x25
'', # 0x26
'', # 0x27
'', # 0x28
'', # 0x29
'', # 0x2a
'', # 0x2b
'', # 0x2c
'', # 0x2d
'', # 0x2e
'', # 0x2f
'', # 0x30
'', # 0x31
'', # 0x32
'', # 0x33
'', # 0x34
'', # 0x35
'', # 0x36
'', # 0x37
'', # 0x38
'', # 0x39
'', # 0x3a
'', # 0x3b
'', # 0x3c
'', # 0x3d
'', # 0x3e
'', # 0x3f
'', # 0x40
'', # 0x41
'', # 0x42
'', # 0x43
'', # 0x44
'', # 0x45
'', # 0x46
'', # 0x47
'', # 0x48
'', # 0x49
'', # 0x4a
'', # 0x4b
'', # 0x4c
'', # 0x4d
'', # 0x4e
'[?]', # 0x4f
'[?]', # 0x50
'[?]', # 0x51
'[?]', # 0x52
'[?]', # 0x53
'[?]', # 0x54
'[?]', # 0x55
'[?]', # 0x56
'[?]', # 0x57
'[?]', # 0x58
'[?]', # 0x59
'[?]', # 0x5a
'[?]', # 0x5b
'[?]', # 0x5c
'[?]', # 0x5d
'[?]', # 0x5e
'[?]', # 0x5f
'', # 0x60
'', # 0x61
'', # 0x62
'a', # 0x63
'e', # 0x64
'i', # 0x65
'o', # 0x66
'u', # 0x67
'c', # 0x68
'd', # 0x69
'h', # 0x6a
'm', # 0x6b
'r', # 0x6c
't', # 0x6d
'v', # 0x6e
'x', # 0x6f
'[?]', # 0x70
'[?]', # 0x71
'[?]', # 0x72
'[?]', # 0x73
'\'', # 0x74
',', # 0x75
'[?]', # 0x76
'[?]', # 0x77
'[?]', # 0x78
'[?]', # 0x79
'', # 0x7a
'[?]', # 0x7b
'[?]', # 0x7c
'[?]', # 0x7d
'?', # 0x7e
'[?]', # 0x7f
'[?]', # 0x80
'[?]', # 0x81
'[?]', # 0x82
'[?]', # 0x83
'', # 0x84
'', # 0x85
'A', # 0x86
';', # 0x87
'E', # 0x88
'E', # 0x89
'I', # 0x8a
'[?]', # 0x8b
'O', # 0x8c
'[?]', # 0x8d
'U', # 0x8e
'O', # 0x8f
'I', # 0x90
'A', # 0x91
'B', # 0x92
'G', # 0x93
'D', # 0x94
'E', # 0x95
'Z', # 0x96
'E', # 0x97
'Th', # 0x98
'I', # 0x99
'K', # 0x9a
'L', # 0x9b
'M', # 0x9c
'N', # 0x9d
'Ks', # 0x9e
'O', # 0x9f
'P', # 0xa0
'R', # 0xa1
'[?]', # 0xa2
'S', # 0xa3
'T', # 0xa4
'U', # 0xa5
'Ph', # 0xa6
'Kh', # 0xa7
'Ps', # 0xa8
'O', # 0xa9
'I', # 0xaa
'U', # 0xab
'a', # 0xac
'e', # 0xad
'e', # 0xae
'i', # 0xaf
'u', # 0xb0
'a', # 0xb1
'b', # 0xb2
'g', # 0xb3
'd', # 0xb4
'e', # 0xb5
'z', # 0xb6
'e', # 0xb7
'th', # 0xb8
'i', # 0xb9
'k', # 0xba
'l', # 0xbb
'm', # 0xbc
'n', # 0xbd
'x', # 0xbe
'o', # 0xbf
'p', # 0xc0
'r', # 0xc1
's', # 0xc2
's', # 0xc3
't', # 0xc4
'u', # 0xc5
'ph', # 0xc6
'kh', # 0xc7
'ps', # 0xc8
'o', # 0xc9
'i', # 0xca
'u', # 0xcb
'o', # 0xcc
'u', # 0xcd
'o', # 0xce
'[?]', # 0xcf
'b', # 0xd0
'th', # 0xd1
'U', # 0xd2
'U', # 0xd3
'U', # 0xd4
'ph', # 0xd5
'p', # 0xd6
'&', # 0xd7
'[?]', # 0xd8
'[?]', # 0xd9
'St', # 0xda
'st', # 0xdb
'W', # 0xdc
'w', # 0xdd
'Q', # 0xde
'q', # 0xdf
'Sp', # 0xe0
'sp', # 0xe1
'Sh', # 0xe2
'sh', # 0xe3
'F', # 0xe4
'f', # 0xe5
'Kh', # 0xe6
'kh', # 0xe7
'H', # 0xe8
'h', # 0xe9
'G', # 0xea
'g', # 0xeb
'CH', # 0xec
'ch', # 0xed
'Ti', # 0xee
'ti', # 0xef
'k', # 0xf0
'r', # 0xf1
'c', # 0xf2
'j', # 0xf3
'[?]', # 0xf4
'[?]', # 0xf5
'[?]', # 0xf6
'[?]', # 0xf7
'[?]', # 0xf8
'[?]', # 0xf9
'[?]', # 0xfa
'[?]', # 0xfb
'[?]', # 0xfc
'[?]', # 0xfd
'[?]', # 0xfe
)

257
libs/unidecode/x004.py Normal file
View file

@ -0,0 +1,257 @@
data = (
'Ie', # 0x00
'Io', # 0x01
'Dj', # 0x02
'Gj', # 0x03
'Ie', # 0x04
'Dz', # 0x05
'I', # 0x06
'Yi', # 0x07
'J', # 0x08
'Lj', # 0x09
'Nj', # 0x0a
'Tsh', # 0x0b
'Kj', # 0x0c
'I', # 0x0d
'U', # 0x0e
'Dzh', # 0x0f
'A', # 0x10
'B', # 0x11
'V', # 0x12
'G', # 0x13
'D', # 0x14
'E', # 0x15
'Zh', # 0x16
'Z', # 0x17
'I', # 0x18
'I', # 0x19
'K', # 0x1a
'L', # 0x1b
'M', # 0x1c
'N', # 0x1d
'O', # 0x1e
'P', # 0x1f
'R', # 0x20
'S', # 0x21
'T', # 0x22
'U', # 0x23
'F', # 0x24
'Kh', # 0x25
'Ts', # 0x26
'Ch', # 0x27
'Sh', # 0x28
'Shch', # 0x29
'\'', # 0x2a
'Y', # 0x2b
'\'', # 0x2c
'E', # 0x2d
'Iu', # 0x2e
'Ia', # 0x2f
'a', # 0x30
'b', # 0x31
'v', # 0x32
'g', # 0x33
'd', # 0x34
'e', # 0x35
'zh', # 0x36
'z', # 0x37
'i', # 0x38
'i', # 0x39
'k', # 0x3a
'l', # 0x3b
'm', # 0x3c
'n', # 0x3d
'o', # 0x3e
'p', # 0x3f
'r', # 0x40
's', # 0x41
't', # 0x42
'u', # 0x43
'f', # 0x44
'kh', # 0x45
'ts', # 0x46
'ch', # 0x47
'sh', # 0x48
'shch', # 0x49
'\'', # 0x4a
'y', # 0x4b
'\'', # 0x4c
'e', # 0x4d
'iu', # 0x4e
'ia', # 0x4f
'ie', # 0x50
'io', # 0x51
'dj', # 0x52
'gj', # 0x53
'ie', # 0x54
'dz', # 0x55
'i', # 0x56
'yi', # 0x57
'j', # 0x58
'lj', # 0x59
'nj', # 0x5a
'tsh', # 0x5b
'kj', # 0x5c
'i', # 0x5d
'u', # 0x5e
'dzh', # 0x5f
'O', # 0x60
'o', # 0x61
'E', # 0x62
'e', # 0x63
'Ie', # 0x64
'ie', # 0x65
'E', # 0x66
'e', # 0x67
'Ie', # 0x68
'ie', # 0x69
'O', # 0x6a
'o', # 0x6b
'Io', # 0x6c
'io', # 0x6d
'Ks', # 0x6e
'ks', # 0x6f
'Ps', # 0x70
'ps', # 0x71
'F', # 0x72
'f', # 0x73
'Y', # 0x74
'y', # 0x75
'Y', # 0x76
'y', # 0x77
'u', # 0x78
'u', # 0x79
'O', # 0x7a
'o', # 0x7b
'O', # 0x7c
'o', # 0x7d
'Ot', # 0x7e
'ot', # 0x7f
'Q', # 0x80
'q', # 0x81
'*1000*', # 0x82
'', # 0x83
'', # 0x84
'', # 0x85
'', # 0x86
'[?]', # 0x87
'*100.000*', # 0x88
'*1.000.000*', # 0x89
'[?]', # 0x8a
'[?]', # 0x8b
'"', # 0x8c
'"', # 0x8d
'R\'', # 0x8e
'r\'', # 0x8f
'G\'', # 0x90
'g\'', # 0x91
'G\'', # 0x92
'g\'', # 0x93
'G\'', # 0x94
'g\'', # 0x95
'Zh\'', # 0x96
'zh\'', # 0x97
'Z\'', # 0x98
'z\'', # 0x99
'K\'', # 0x9a
'k\'', # 0x9b
'K\'', # 0x9c
'k\'', # 0x9d
'K\'', # 0x9e
'k\'', # 0x9f
'K\'', # 0xa0
'k\'', # 0xa1
'N\'', # 0xa2
'n\'', # 0xa3
'Ng', # 0xa4
'ng', # 0xa5
'P\'', # 0xa6
'p\'', # 0xa7
'Kh', # 0xa8
'kh', # 0xa9
'S\'', # 0xaa
's\'', # 0xab
'T\'', # 0xac
't\'', # 0xad
'U', # 0xae
'u', # 0xaf
'U\'', # 0xb0
'u\'', # 0xb1
'Kh\'', # 0xb2
'kh\'', # 0xb3
'Tts', # 0xb4
'tts', # 0xb5
'Ch\'', # 0xb6
'ch\'', # 0xb7
'Ch\'', # 0xb8
'ch\'', # 0xb9
'H', # 0xba
'h', # 0xbb
'Ch', # 0xbc
'ch', # 0xbd
'Ch\'', # 0xbe
'ch\'', # 0xbf
'`', # 0xc0
'Zh', # 0xc1
'zh', # 0xc2
'K\'', # 0xc3
'k\'', # 0xc4
'[?]', # 0xc5
'[?]', # 0xc6
'N\'', # 0xc7
'n\'', # 0xc8
'[?]', # 0xc9
'[?]', # 0xca
'Ch', # 0xcb
'ch', # 0xcc
'[?]', # 0xcd
'[?]', # 0xce
'[?]', # 0xcf
'a', # 0xd0
'a', # 0xd1
'A', # 0xd2
'a', # 0xd3
'Ae', # 0xd4
'ae', # 0xd5
'Ie', # 0xd6
'ie', # 0xd7
'@', # 0xd8
'@', # 0xd9
'@', # 0xda
'@', # 0xdb
'Zh', # 0xdc
'zh', # 0xdd
'Z', # 0xde
'z', # 0xdf
'Dz', # 0xe0
'dz', # 0xe1
'I', # 0xe2
'i', # 0xe3
'I', # 0xe4
'i', # 0xe5
'O', # 0xe6
'o', # 0xe7
'O', # 0xe8
'o', # 0xe9
'O', # 0xea
'o', # 0xeb
'E', # 0xec
'e', # 0xed
'U', # 0xee
'u', # 0xef
'U', # 0xf0
'u', # 0xf1
'U', # 0xf2
'u', # 0xf3
'Ch', # 0xf4
'ch', # 0xf5
'[?]', # 0xf6
'[?]', # 0xf7
'Y', # 0xf8
'y', # 0xf9
'[?]', # 0xfa
'[?]', # 0xfb
'[?]', # 0xfc
'[?]', # 0xfd
'[?]', # 0xfe
)

257
libs/unidecode/x005.py Normal file
View file

@ -0,0 +1,257 @@
data = (
'[?]', # 0x00
'[?]', # 0x01
'[?]', # 0x02
'[?]', # 0x03
'[?]', # 0x04
'[?]', # 0x05
'[?]', # 0x06
'[?]', # 0x07
'[?]', # 0x08
'[?]', # 0x09
'[?]', # 0x0a
'[?]', # 0x0b
'[?]', # 0x0c
'[?]', # 0x0d
'[?]', # 0x0e
'[?]', # 0x0f
'[?]', # 0x10
'[?]', # 0x11
'[?]', # 0x12
'[?]', # 0x13
'[?]', # 0x14
'[?]', # 0x15
'[?]', # 0x16
'[?]', # 0x17
'[?]', # 0x18
'[?]', # 0x19
'[?]', # 0x1a
'[?]', # 0x1b
'[?]', # 0x1c
'[?]', # 0x1d
'[?]', # 0x1e
'[?]', # 0x1f
'[?]', # 0x20
'[?]', # 0x21
'[?]', # 0x22
'[?]', # 0x23
'[?]', # 0x24
'[?]', # 0x25
'[?]', # 0x26
'[?]', # 0x27
'[?]', # 0x28
'[?]', # 0x29
'[?]', # 0x2a
'[?]', # 0x2b
'[?]', # 0x2c
'[?]', # 0x2d
'[?]', # 0x2e
'[?]', # 0x2f
'[?]', # 0x30
'A', # 0x31
'B', # 0x32
'G', # 0x33
'D', # 0x34
'E', # 0x35
'Z', # 0x36
'E', # 0x37
'E', # 0x38
'T`', # 0x39
'Zh', # 0x3a
'I', # 0x3b
'L', # 0x3c
'Kh', # 0x3d
'Ts', # 0x3e
'K', # 0x3f
'H', # 0x40
'Dz', # 0x41
'Gh', # 0x42
'Ch', # 0x43
'M', # 0x44
'Y', # 0x45
'N', # 0x46
'Sh', # 0x47
'O', # 0x48
'Ch`', # 0x49
'P', # 0x4a
'J', # 0x4b
'Rh', # 0x4c
'S', # 0x4d
'V', # 0x4e
'T', # 0x4f
'R', # 0x50
'Ts`', # 0x51
'W', # 0x52
'P`', # 0x53
'K`', # 0x54
'O', # 0x55
'F', # 0x56
'[?]', # 0x57
'[?]', # 0x58
'<', # 0x59
'\'', # 0x5a
'/', # 0x5b
'!', # 0x5c
',', # 0x5d
'?', # 0x5e
'.', # 0x5f
'[?]', # 0x60
'a', # 0x61
'b', # 0x62
'g', # 0x63
'd', # 0x64
'e', # 0x65
'z', # 0x66
'e', # 0x67
'e', # 0x68
't`', # 0x69
'zh', # 0x6a
'i', # 0x6b
'l', # 0x6c
'kh', # 0x6d
'ts', # 0x6e
'k', # 0x6f
'h', # 0x70
'dz', # 0x71
'gh', # 0x72
'ch', # 0x73
'm', # 0x74
'y', # 0x75
'n', # 0x76
'sh', # 0x77
'o', # 0x78
'ch`', # 0x79
'p', # 0x7a
'j', # 0x7b
'rh', # 0x7c
's', # 0x7d
'v', # 0x7e
't', # 0x7f
'r', # 0x80
'ts`', # 0x81
'w', # 0x82
'p`', # 0x83
'k`', # 0x84
'o', # 0x85
'f', # 0x86
'ew', # 0x87
'[?]', # 0x88
':', # 0x89
'-', # 0x8a
'[?]', # 0x8b
'[?]', # 0x8c
'[?]', # 0x8d
'[?]', # 0x8e
'[?]', # 0x8f
'[?]', # 0x90
'', # 0x91
'', # 0x92
'', # 0x93
'', # 0x94
'', # 0x95
'', # 0x96
'', # 0x97
'', # 0x98
'', # 0x99
'', # 0x9a
'', # 0x9b
'', # 0x9c
'', # 0x9d
'', # 0x9e
'', # 0x9f
'', # 0xa0
'', # 0xa1
'[?]', # 0xa2
'', # 0xa3
'', # 0xa4
'', # 0xa5
'', # 0xa6
'', # 0xa7
'', # 0xa8
'', # 0xa9
'', # 0xaa
'', # 0xab
'', # 0xac
'', # 0xad
'', # 0xae
'', # 0xaf
'@', # 0xb0
'e', # 0xb1
'a', # 0xb2
'o', # 0xb3
'i', # 0xb4
'e', # 0xb5
'e', # 0xb6
'a', # 0xb7
'a', # 0xb8
'o', # 0xb9
'[?]', # 0xba
'u', # 0xbb
'\'', # 0xbc
'', # 0xbd
'', # 0xbe
'', # 0xbf
'|', # 0xc0
'', # 0xc1
'', # 0xc2
':', # 0xc3
'', # 0xc4
'[?]', # 0xc5
'[?]', # 0xc6
'[?]', # 0xc7
'[?]', # 0xc8
'[?]', # 0xc9
'[?]', # 0xca
'[?]', # 0xcb
'[?]', # 0xcc
'[?]', # 0xcd
'[?]', # 0xce
'[?]', # 0xcf
'', # 0xd0
'b', # 0xd1
'g', # 0xd2
'd', # 0xd3
'h', # 0xd4
'v', # 0xd5
'z', # 0xd6
'kh', # 0xd7
't', # 0xd8
'y', # 0xd9
'k', # 0xda
'k', # 0xdb
'l', # 0xdc
'm', # 0xdd
'm', # 0xde
'n', # 0xdf
'n', # 0xe0
's', # 0xe1
'`', # 0xe2
'p', # 0xe3
'p', # 0xe4
'ts', # 0xe5
'ts', # 0xe6
'q', # 0xe7
'r', # 0xe8
'sh', # 0xe9
't', # 0xea
'[?]', # 0xeb
'[?]', # 0xec
'[?]', # 0xed
'[?]', # 0xee
'[?]', # 0xef
'V', # 0xf0
'oy', # 0xf1
'i', # 0xf2
'\'', # 0xf3
'"', # 0xf4
'[?]', # 0xf5
'[?]', # 0xf6
'[?]', # 0xf7
'[?]', # 0xf8
'[?]', # 0xf9
'[?]', # 0xfa
'[?]', # 0xfb
'[?]', # 0xfc
'[?]', # 0xfd
'[?]', # 0xfe
)

257
libs/unidecode/x006.py Normal file
View file

@ -0,0 +1,257 @@
data = (
'[?]', # 0x00
'[?]', # 0x01
'[?]', # 0x02
'[?]', # 0x03
'[?]', # 0x04
'[?]', # 0x05
'[?]', # 0x06
'[?]', # 0x07
'[?]', # 0x08
'[?]', # 0x09
'[?]', # 0x0a
'[?]', # 0x0b
',', # 0x0c
'[?]', # 0x0d
'[?]', # 0x0e
'[?]', # 0x0f
'[?]', # 0x10
'[?]', # 0x11
'[?]', # 0x12
'[?]', # 0x13
'[?]', # 0x14
'[?]', # 0x15
'[?]', # 0x16
'[?]', # 0x17
'[?]', # 0x18
'[?]', # 0x19
'[?]', # 0x1a
';', # 0x1b
'[?]', # 0x1c
'[?]', # 0x1d
'[?]', # 0x1e
'?', # 0x1f
'[?]', # 0x20
'', # 0x21
'a', # 0x22
'\'', # 0x23
'w\'', # 0x24
'', # 0x25
'y\'', # 0x26
'', # 0x27
'b', # 0x28
'@', # 0x29
't', # 0x2a
'th', # 0x2b
'j', # 0x2c
'H', # 0x2d
'kh', # 0x2e
'd', # 0x2f
'dh', # 0x30
'r', # 0x31
'z', # 0x32
's', # 0x33
'sh', # 0x34
'S', # 0x35
'D', # 0x36
'T', # 0x37
'Z', # 0x38
'`', # 0x39
'G', # 0x3a
'[?]', # 0x3b
'[?]', # 0x3c
'[?]', # 0x3d
'[?]', # 0x3e
'[?]', # 0x3f
'', # 0x40
'f', # 0x41
'q', # 0x42
'k', # 0x43
'l', # 0x44
'm', # 0x45
'n', # 0x46
'h', # 0x47
'w', # 0x48
'~', # 0x49
'y', # 0x4a
'an', # 0x4b
'un', # 0x4c
'in', # 0x4d
'a', # 0x4e
'u', # 0x4f
'i', # 0x50
'W', # 0x51
'', # 0x52
'', # 0x53
'\'', # 0x54
'\'', # 0x55
'[?]', # 0x56
'[?]', # 0x57
'[?]', # 0x58
'[?]', # 0x59
'[?]', # 0x5a
'[?]', # 0x5b
'[?]', # 0x5c
'[?]', # 0x5d
'[?]', # 0x5e
'[?]', # 0x5f
'0', # 0x60
'1', # 0x61
'2', # 0x62
'3', # 0x63
'4', # 0x64
'5', # 0x65
'6', # 0x66
'7', # 0x67
'8', # 0x68
'9', # 0x69
'%', # 0x6a
'.', # 0x6b
',', # 0x6c
'*', # 0x6d
'[?]', # 0x6e
'[?]', # 0x6f
'', # 0x70
'\'', # 0x71
'\'', # 0x72
'\'', # 0x73
'', # 0x74
'\'', # 0x75
'\'w', # 0x76
'\'u', # 0x77
'\'y', # 0x78
'tt', # 0x79
'tth', # 0x7a
'b', # 0x7b
't', # 0x7c
'T', # 0x7d
'p', # 0x7e
'th', # 0x7f
'bh', # 0x80
'\'h', # 0x81
'H', # 0x82
'ny', # 0x83
'dy', # 0x84
'H', # 0x85
'ch', # 0x86
'cch', # 0x87
'dd', # 0x88
'D', # 0x89
'D', # 0x8a
'Dt', # 0x8b
'dh', # 0x8c
'ddh', # 0x8d
'd', # 0x8e
'D', # 0x8f
'D', # 0x90
'rr', # 0x91
'R', # 0x92
'R', # 0x93
'R', # 0x94
'R', # 0x95
'R', # 0x96
'R', # 0x97
'j', # 0x98
'R', # 0x99
'S', # 0x9a
'S', # 0x9b
'S', # 0x9c
'S', # 0x9d
'S', # 0x9e
'T', # 0x9f
'GH', # 0xa0
'F', # 0xa1
'F', # 0xa2
'F', # 0xa3
'v', # 0xa4
'f', # 0xa5
'ph', # 0xa6
'Q', # 0xa7
'Q', # 0xa8
'kh', # 0xa9
'k', # 0xaa
'K', # 0xab
'K', # 0xac
'ng', # 0xad
'K', # 0xae
'g', # 0xaf
'G', # 0xb0
'N', # 0xb1
'G', # 0xb2
'G', # 0xb3
'G', # 0xb4
'L', # 0xb5
'L', # 0xb6
'L', # 0xb7
'L', # 0xb8
'N', # 0xb9
'N', # 0xba
'N', # 0xbb
'N', # 0xbc
'N', # 0xbd
'h', # 0xbe
'Ch', # 0xbf
'hy', # 0xc0
'h', # 0xc1
'H', # 0xc2
'@', # 0xc3
'W', # 0xc4
'oe', # 0xc5
'oe', # 0xc6
'u', # 0xc7
'yu', # 0xc8
'yu', # 0xc9
'W', # 0xca
'v', # 0xcb
'y', # 0xcc
'Y', # 0xcd
'Y', # 0xce
'W', # 0xcf
'', # 0xd0
'', # 0xd1
'y', # 0xd2
'y\'', # 0xd3
'.', # 0xd4
'ae', # 0xd5
'', # 0xd6
'', # 0xd7
'', # 0xd8
'', # 0xd9
'', # 0xda
'', # 0xdb
'', # 0xdc
'@', # 0xdd
'#', # 0xde
'', # 0xdf
'', # 0xe0
'', # 0xe1
'', # 0xe2
'', # 0xe3
'', # 0xe4
'', # 0xe5
'', # 0xe6
'', # 0xe7
'', # 0xe8
'^', # 0xe9
'', # 0xea
'', # 0xeb
'', # 0xec
'', # 0xed
'[?]', # 0xee
'[?]', # 0xef
'0', # 0xf0
'1', # 0xf1
'2', # 0xf2
'3', # 0xf3
'4', # 0xf4
'5', # 0xf5
'6', # 0xf6
'7', # 0xf7
'8', # 0xf8
'9', # 0xf9
'Sh', # 0xfa
'D', # 0xfb
'Gh', # 0xfc
'&', # 0xfd
'+m', # 0xfe
)

257
libs/unidecode/x007.py Normal file
View file

@ -0,0 +1,257 @@
data = (
'//', # 0x00
'/', # 0x01
',', # 0x02
'!', # 0x03
'!', # 0x04
'-', # 0x05
',', # 0x06
',', # 0x07
';', # 0x08
'?', # 0x09
'~', # 0x0a
'{', # 0x0b
'}', # 0x0c
'*', # 0x0d
'[?]', # 0x0e
'', # 0x0f
'\'', # 0x10
'', # 0x11
'b', # 0x12
'g', # 0x13
'g', # 0x14
'd', # 0x15
'd', # 0x16
'h', # 0x17
'w', # 0x18
'z', # 0x19
'H', # 0x1a
't', # 0x1b
't', # 0x1c
'y', # 0x1d
'yh', # 0x1e
'k', # 0x1f
'l', # 0x20
'm', # 0x21
'n', # 0x22
's', # 0x23
's', # 0x24
'`', # 0x25
'p', # 0x26
'p', # 0x27
'S', # 0x28
'q', # 0x29
'r', # 0x2a
'sh', # 0x2b
't', # 0x2c
'[?]', # 0x2d
'[?]', # 0x2e
'[?]', # 0x2f
'a', # 0x30
'a', # 0x31
'a', # 0x32
'A', # 0x33
'A', # 0x34
'A', # 0x35
'e', # 0x36
'e', # 0x37
'e', # 0x38
'E', # 0x39
'i', # 0x3a
'i', # 0x3b
'u', # 0x3c
'u', # 0x3d
'u', # 0x3e
'o', # 0x3f
'', # 0x40
'`', # 0x41
'\'', # 0x42
'', # 0x43
'', # 0x44
'X', # 0x45
'Q', # 0x46
'@', # 0x47
'@', # 0x48
'|', # 0x49
'+', # 0x4a
'[?]', # 0x4b
'[?]', # 0x4c
'[?]', # 0x4d
'[?]', # 0x4e
'[?]', # 0x4f
'[?]', # 0x50
'[?]', # 0x51
'[?]', # 0x52
'[?]', # 0x53
'[?]', # 0x54
'[?]', # 0x55
'[?]', # 0x56
'[?]', # 0x57
'[?]', # 0x58
'[?]', # 0x59
'[?]', # 0x5a
'[?]', # 0x5b
'[?]', # 0x5c
'[?]', # 0x5d
'[?]', # 0x5e
'[?]', # 0x5f
'[?]', # 0x60
'[?]', # 0x61
'[?]', # 0x62
'[?]', # 0x63
'[?]', # 0x64
'[?]', # 0x65
'[?]', # 0x66
'[?]', # 0x67
'[?]', # 0x68
'[?]', # 0x69
'[?]', # 0x6a
'[?]', # 0x6b
'[?]', # 0x6c
'[?]', # 0x6d
'[?]', # 0x6e
'[?]', # 0x6f
'[?]', # 0x70
'[?]', # 0x71
'[?]', # 0x72
'[?]', # 0x73
'[?]', # 0x74
'[?]', # 0x75
'[?]', # 0x76
'[?]', # 0x77
'[?]', # 0x78
'[?]', # 0x79
'[?]', # 0x7a
'[?]', # 0x7b
'[?]', # 0x7c
'[?]', # 0x7d
'[?]', # 0x7e
'[?]', # 0x7f
'h', # 0x80
'sh', # 0x81
'n', # 0x82
'r', # 0x83
'b', # 0x84
'L', # 0x85
'k', # 0x86
'\'', # 0x87
'v', # 0x88
'm', # 0x89
'f', # 0x8a
'dh', # 0x8b
'th', # 0x8c
'l', # 0x8d
'g', # 0x8e
'ny', # 0x8f
's', # 0x90
'd', # 0x91
'z', # 0x92
't', # 0x93
'y', # 0x94
'p', # 0x95
'j', # 0x96
'ch', # 0x97
'tt', # 0x98
'hh', # 0x99
'kh', # 0x9a
'th', # 0x9b
'z', # 0x9c
'sh', # 0x9d
's', # 0x9e
'd', # 0x9f
't', # 0xa0
'z', # 0xa1
'`', # 0xa2
'gh', # 0xa3
'q', # 0xa4
'w', # 0xa5
'a', # 0xa6
'aa', # 0xa7
'i', # 0xa8
'ee', # 0xa9
'u', # 0xaa
'oo', # 0xab
'e', # 0xac
'ey', # 0xad
'o', # 0xae
'oa', # 0xaf
'', # 0xb0
'[?]', # 0xb1
'[?]', # 0xb2
'[?]', # 0xb3
'[?]', # 0xb4
'[?]', # 0xb5
'[?]', # 0xb6
'[?]', # 0xb7
'[?]', # 0xb8
'[?]', # 0xb9
'[?]', # 0xba
'[?]', # 0xbb
'[?]', # 0xbc
'[?]', # 0xbd
'[?]', # 0xbe
'[?]', # 0xbf
'[?]', # 0xc0
'[?]', # 0xc1
'[?]', # 0xc2
'[?]', # 0xc3
'[?]', # 0xc4
'[?]', # 0xc5
'[?]', # 0xc6
'[?]', # 0xc7
'[?]', # 0xc8
'[?]', # 0xc9
'[?]', # 0xca
'[?]', # 0xcb
'[?]', # 0xcc
'[?]', # 0xcd
'[?]', # 0xce
'[?]', # 0xcf
'[?]', # 0xd0
'[?]', # 0xd1
'[?]', # 0xd2
'[?]', # 0xd3
'[?]', # 0xd4
'[?]', # 0xd5
'[?]', # 0xd6
'[?]', # 0xd7
'[?]', # 0xd8
'[?]', # 0xd9
'[?]', # 0xda
'[?]', # 0xdb
'[?]', # 0xdc
'[?]', # 0xdd
'[?]', # 0xde
'[?]', # 0xdf
'[?]', # 0xe0
'[?]', # 0xe1
'[?]', # 0xe2
'[?]', # 0xe3
'[?]', # 0xe4
'[?]', # 0xe5
'[?]', # 0xe6
'[?]', # 0xe7
'[?]', # 0xe8
'[?]', # 0xe9
'[?]', # 0xea
'[?]', # 0xeb
'[?]', # 0xec
'[?]', # 0xed
'[?]', # 0xee
'[?]', # 0xef
'[?]', # 0xf0
'[?]', # 0xf1
'[?]', # 0xf2
'[?]', # 0xf3
'[?]', # 0xf4
'[?]', # 0xf5
'[?]', # 0xf6
'[?]', # 0xf7
'[?]', # 0xf8
'[?]', # 0xf9
'[?]', # 0xfa
'[?]', # 0xfb
'[?]', # 0xfc
'[?]', # 0xfd
'[?]', # 0xfe
)

257
libs/unidecode/x009.py Normal file
View file

@ -0,0 +1,257 @@
data = (
'[?]', # 0x00
'N', # 0x01
'N', # 0x02
'H', # 0x03
'[?]', # 0x04
'a', # 0x05
'aa', # 0x06
'i', # 0x07
'ii', # 0x08
'u', # 0x09
'uu', # 0x0a
'R', # 0x0b
'L', # 0x0c
'eN', # 0x0d
'e', # 0x0e
'e', # 0x0f
'ai', # 0x10
'oN', # 0x11
'o', # 0x12
'o', # 0x13
'au', # 0x14
'k', # 0x15
'kh', # 0x16
'g', # 0x17
'gh', # 0x18
'ng', # 0x19
'c', # 0x1a
'ch', # 0x1b
'j', # 0x1c
'jh', # 0x1d
'ny', # 0x1e
'tt', # 0x1f
'tth', # 0x20
'dd', # 0x21
'ddh', # 0x22
'nn', # 0x23
't', # 0x24
'th', # 0x25
'd', # 0x26
'dh', # 0x27
'n', # 0x28
'nnn', # 0x29
'p', # 0x2a
'ph', # 0x2b
'b', # 0x2c
'bh', # 0x2d
'm', # 0x2e
'y', # 0x2f
'r', # 0x30
'rr', # 0x31
'l', # 0x32
'l', # 0x33
'lll', # 0x34
'v', # 0x35
'sh', # 0x36
'ss', # 0x37
's', # 0x38
'h', # 0x39
'[?]', # 0x3a
'[?]', # 0x3b
'\'', # 0x3c
'\'', # 0x3d
'aa', # 0x3e
'i', # 0x3f
'ii', # 0x40
'u', # 0x41
'uu', # 0x42
'R', # 0x43
'RR', # 0x44
'eN', # 0x45
'e', # 0x46
'e', # 0x47
'ai', # 0x48
'oN', # 0x49
'o', # 0x4a
'o', # 0x4b
'au', # 0x4c
'', # 0x4d
'[?]', # 0x4e
'[?]', # 0x4f
'AUM', # 0x50
'\'', # 0x51
'\'', # 0x52
'`', # 0x53
'\'', # 0x54
'[?]', # 0x55
'[?]', # 0x56
'[?]', # 0x57
'q', # 0x58
'khh', # 0x59
'ghh', # 0x5a
'z', # 0x5b
'dddh', # 0x5c
'rh', # 0x5d
'f', # 0x5e
'yy', # 0x5f
'RR', # 0x60
'LL', # 0x61
'L', # 0x62
'LL', # 0x63
' / ', # 0x64
' // ', # 0x65
'0', # 0x66
'1', # 0x67
'2', # 0x68
'3', # 0x69
'4', # 0x6a
'5', # 0x6b
'6', # 0x6c
'7', # 0x6d
'8', # 0x6e
'9', # 0x6f
'.', # 0x70
'[?]', # 0x71
'[?]', # 0x72
'[?]', # 0x73
'[?]', # 0x74
'[?]', # 0x75
'[?]', # 0x76
'[?]', # 0x77
'[?]', # 0x78
'[?]', # 0x79
'[?]', # 0x7a
'[?]', # 0x7b
'[?]', # 0x7c
'[?]', # 0x7d
'[?]', # 0x7e
'[?]', # 0x7f
'[?]', # 0x80
'N', # 0x81
'N', # 0x82
'H', # 0x83
'[?]', # 0x84
'a', # 0x85
'aa', # 0x86
'i', # 0x87
'ii', # 0x88
'u', # 0x89
'uu', # 0x8a
'R', # 0x8b
'RR', # 0x8c
'[?]', # 0x8d
'[?]', # 0x8e
'e', # 0x8f
'ai', # 0x90
'[?]', # 0x91
'[?]', # 0x92
'o', # 0x93
'au', # 0x94
'k', # 0x95
'kh', # 0x96
'g', # 0x97
'gh', # 0x98
'ng', # 0x99
'c', # 0x9a
'ch', # 0x9b
'j', # 0x9c
'jh', # 0x9d
'ny', # 0x9e
'tt', # 0x9f
'tth', # 0xa0
'dd', # 0xa1
'ddh', # 0xa2
'nn', # 0xa3
't', # 0xa4
'th', # 0xa5
'd', # 0xa6
'dh', # 0xa7
'n', # 0xa8
'[?]', # 0xa9
'p', # 0xaa
'ph', # 0xab
'b', # 0xac
'bh', # 0xad
'm', # 0xae
'y', # 0xaf
'r', # 0xb0
'[?]', # 0xb1
'l', # 0xb2
'[?]', # 0xb3
'[?]', # 0xb4
'[?]', # 0xb5
'sh', # 0xb6
'ss', # 0xb7
's', # 0xb8
'h', # 0xb9
'[?]', # 0xba
'[?]', # 0xbb
'\'', # 0xbc
'[?]', # 0xbd
'aa', # 0xbe
'i', # 0xbf
'ii', # 0xc0
'u', # 0xc1
'uu', # 0xc2
'R', # 0xc3
'RR', # 0xc4
'[?]', # 0xc5
'[?]', # 0xc6
'e', # 0xc7
'ai', # 0xc8
'[?]', # 0xc9
'[?]', # 0xca
'o', # 0xcb
'au', # 0xcc
'', # 0xcd
'[?]', # 0xce
'[?]', # 0xcf
'[?]', # 0xd0
'[?]', # 0xd1
'[?]', # 0xd2
'[?]', # 0xd3
'[?]', # 0xd4
'[?]', # 0xd5
'[?]', # 0xd6
'+', # 0xd7
'[?]', # 0xd8
'[?]', # 0xd9
'[?]', # 0xda
'[?]', # 0xdb
'rr', # 0xdc
'rh', # 0xdd
'[?]', # 0xde
'yy', # 0xdf
'RR', # 0xe0
'LL', # 0xe1
'L', # 0xe2
'LL', # 0xe3
'[?]', # 0xe4
'[?]', # 0xe5
'0', # 0xe6
'1', # 0xe7
'2', # 0xe8
'3', # 0xe9
'4', # 0xea
'5', # 0xeb
'6', # 0xec
'7', # 0xed
'8', # 0xee
'9', # 0xef
'r\'', # 0xf0
'r`', # 0xf1
'Rs', # 0xf2
'Rs', # 0xf3
'1/', # 0xf4
'2/', # 0xf5
'3/', # 0xf6
'4/', # 0xf7
' 1 - 1/', # 0xf8
'/16', # 0xf9
'', # 0xfa
'[?]', # 0xfb
'[?]', # 0xfc
'[?]', # 0xfd
'[?]', # 0xfe
)

257
libs/unidecode/x00a.py Normal file
View file

@ -0,0 +1,257 @@
data = (
'[?]', # 0x00
'[?]', # 0x01
'N', # 0x02
'[?]', # 0x03
'[?]', # 0x04
'a', # 0x05
'aa', # 0x06
'i', # 0x07
'ii', # 0x08
'u', # 0x09
'uu', # 0x0a
'[?]', # 0x0b
'[?]', # 0x0c
'[?]', # 0x0d
'[?]', # 0x0e
'ee', # 0x0f
'ai', # 0x10
'[?]', # 0x11
'[?]', # 0x12
'oo', # 0x13
'au', # 0x14
'k', # 0x15
'kh', # 0x16
'g', # 0x17
'gh', # 0x18
'ng', # 0x19
'c', # 0x1a
'ch', # 0x1b
'j', # 0x1c
'jh', # 0x1d
'ny', # 0x1e
'tt', # 0x1f
'tth', # 0x20
'dd', # 0x21
'ddh', # 0x22
'nn', # 0x23
't', # 0x24
'th', # 0x25
'd', # 0x26
'dh', # 0x27
'n', # 0x28
'[?]', # 0x29
'p', # 0x2a
'ph', # 0x2b
'b', # 0x2c
'bb', # 0x2d
'm', # 0x2e
'y', # 0x2f
'r', # 0x30
'[?]', # 0x31
'l', # 0x32
'll', # 0x33
'[?]', # 0x34
'v', # 0x35
'sh', # 0x36
'[?]', # 0x37
's', # 0x38
'h', # 0x39
'[?]', # 0x3a
'[?]', # 0x3b
'\'', # 0x3c
'[?]', # 0x3d
'aa', # 0x3e
'i', # 0x3f
'ii', # 0x40
'u', # 0x41
'uu', # 0x42
'[?]', # 0x43
'[?]', # 0x44
'[?]', # 0x45
'[?]', # 0x46
'ee', # 0x47
'ai', # 0x48
'[?]', # 0x49
'[?]', # 0x4a
'oo', # 0x4b
'au', # 0x4c
'', # 0x4d
'[?]', # 0x4e
'[?]', # 0x4f
'[?]', # 0x50
'[?]', # 0x51
'[?]', # 0x52
'[?]', # 0x53
'[?]', # 0x54
'[?]', # 0x55
'[?]', # 0x56
'[?]', # 0x57
'[?]', # 0x58
'khh', # 0x59
'ghh', # 0x5a
'z', # 0x5b
'rr', # 0x5c
'[?]', # 0x5d
'f', # 0x5e
'[?]', # 0x5f
'[?]', # 0x60
'[?]', # 0x61
'[?]', # 0x62
'[?]', # 0x63
'[?]', # 0x64
'[?]', # 0x65
'0', # 0x66
'1', # 0x67
'2', # 0x68
'3', # 0x69
'4', # 0x6a
'5', # 0x6b
'6', # 0x6c
'7', # 0x6d
'8', # 0x6e
'9', # 0x6f
'N', # 0x70
'H', # 0x71
'', # 0x72
'', # 0x73
'G.E.O.', # 0x74
'[?]', # 0x75
'[?]', # 0x76
'[?]', # 0x77
'[?]', # 0x78
'[?]', # 0x79
'[?]', # 0x7a
'[?]', # 0x7b
'[?]', # 0x7c
'[?]', # 0x7d
'[?]', # 0x7e
'[?]', # 0x7f
'[?]', # 0x80
'N', # 0x81
'N', # 0x82
'H', # 0x83
'[?]', # 0x84
'a', # 0x85
'aa', # 0x86
'i', # 0x87
'ii', # 0x88
'u', # 0x89
'uu', # 0x8a
'R', # 0x8b
'[?]', # 0x8c
'eN', # 0x8d
'[?]', # 0x8e
'e', # 0x8f
'ai', # 0x90
'oN', # 0x91
'[?]', # 0x92
'o', # 0x93
'au', # 0x94
'k', # 0x95
'kh', # 0x96
'g', # 0x97
'gh', # 0x98
'ng', # 0x99
'c', # 0x9a
'ch', # 0x9b
'j', # 0x9c
'jh', # 0x9d
'ny', # 0x9e
'tt', # 0x9f
'tth', # 0xa0
'dd', # 0xa1
'ddh', # 0xa2
'nn', # 0xa3
't', # 0xa4
'th', # 0xa5
'd', # 0xa6
'dh', # 0xa7
'n', # 0xa8
'[?]', # 0xa9
'p', # 0xaa
'ph', # 0xab
'b', # 0xac
'bh', # 0xad
'm', # 0xae
'ya', # 0xaf
'r', # 0xb0
'[?]', # 0xb1
'l', # 0xb2
'll', # 0xb3
'[?]', # 0xb4
'v', # 0xb5
'sh', # 0xb6
'ss', # 0xb7
's', # 0xb8
'h', # 0xb9
'[?]', # 0xba
'[?]', # 0xbb
'\'', # 0xbc
'\'', # 0xbd
'aa', # 0xbe
'i', # 0xbf
'ii', # 0xc0
'u', # 0xc1
'uu', # 0xc2
'R', # 0xc3
'RR', # 0xc4
'eN', # 0xc5
'[?]', # 0xc6
'e', # 0xc7
'ai', # 0xc8
'oN', # 0xc9
'[?]', # 0xca
'o', # 0xcb
'au', # 0xcc
'', # 0xcd
'[?]', # 0xce
'[?]', # 0xcf
'AUM', # 0xd0
'[?]', # 0xd1
'[?]', # 0xd2
'[?]', # 0xd3
'[?]', # 0xd4
'[?]', # 0xd5
'[?]', # 0xd6
'[?]', # 0xd7
'[?]', # 0xd8
'[?]', # 0xd9
'[?]', # 0xda
'[?]', # 0xdb
'[?]', # 0xdc
'[?]', # 0xdd
'[?]', # 0xde
'[?]', # 0xdf
'RR', # 0xe0
'[?]', # 0xe1
'[?]', # 0xe2
'[?]', # 0xe3
'[?]', # 0xe4
'[?]', # 0xe5
'0', # 0xe6
'1', # 0xe7
'2', # 0xe8
'3', # 0xe9
'4', # 0xea
'5', # 0xeb
'6', # 0xec
'7', # 0xed
'8', # 0xee
'9', # 0xef
'[?]', # 0xf0
'[?]', # 0xf1
'[?]', # 0xf2
'[?]', # 0xf3
'[?]', # 0xf4
'[?]', # 0xf5
'[?]', # 0xf6
'[?]', # 0xf7
'[?]', # 0xf8
'[?]', # 0xf9
'[?]', # 0xfa
'[?]', # 0xfb
'[?]', # 0xfc
'[?]', # 0xfd
'[?]', # 0xfe
)

257
libs/unidecode/x00b.py Normal file
View file

@ -0,0 +1,257 @@
data = (
'[?]', # 0x00
'N', # 0x01
'N', # 0x02
'H', # 0x03
'[?]', # 0x04
'a', # 0x05
'aa', # 0x06
'i', # 0x07
'ii', # 0x08
'u', # 0x09
'uu', # 0x0a
'R', # 0x0b
'L', # 0x0c
'[?]', # 0x0d
'[?]', # 0x0e
'e', # 0x0f
'ai', # 0x10
'[?]', # 0x11
'[?]', # 0x12
'o', # 0x13
'au', # 0x14
'k', # 0x15
'kh', # 0x16
'g', # 0x17
'gh', # 0x18
'ng', # 0x19
'c', # 0x1a
'ch', # 0x1b
'j', # 0x1c
'jh', # 0x1d
'ny', # 0x1e
'tt', # 0x1f
'tth', # 0x20
'dd', # 0x21
'ddh', # 0x22
'nn', # 0x23
't', # 0x24
'th', # 0x25
'd', # 0x26
'dh', # 0x27
'n', # 0x28
'[?]', # 0x29
'p', # 0x2a
'ph', # 0x2b
'b', # 0x2c
'bh', # 0x2d
'm', # 0x2e
'y', # 0x2f
'r', # 0x30
'[?]', # 0x31
'l', # 0x32
'll', # 0x33
'[?]', # 0x34
'', # 0x35
'sh', # 0x36
'ss', # 0x37
's', # 0x38
'h', # 0x39
'[?]', # 0x3a
'[?]', # 0x3b
'\'', # 0x3c
'\'', # 0x3d
'aa', # 0x3e
'i', # 0x3f
'ii', # 0x40
'u', # 0x41
'uu', # 0x42
'R', # 0x43
'[?]', # 0x44
'[?]', # 0x45
'[?]', # 0x46
'e', # 0x47
'ai', # 0x48
'[?]', # 0x49
'[?]', # 0x4a
'o', # 0x4b
'au', # 0x4c
'', # 0x4d
'[?]', # 0x4e
'[?]', # 0x4f
'[?]', # 0x50
'[?]', # 0x51
'[?]', # 0x52
'[?]', # 0x53
'[?]', # 0x54
'[?]', # 0x55
'+', # 0x56
'+', # 0x57
'[?]', # 0x58
'[?]', # 0x59
'[?]', # 0x5a
'[?]', # 0x5b
'rr', # 0x5c
'rh', # 0x5d
'[?]', # 0x5e
'yy', # 0x5f
'RR', # 0x60
'LL', # 0x61
'[?]', # 0x62
'[?]', # 0x63
'[?]', # 0x64
'[?]', # 0x65
'0', # 0x66
'1', # 0x67
'2', # 0x68
'3', # 0x69
'4', # 0x6a
'5', # 0x6b
'6', # 0x6c
'7', # 0x6d
'8', # 0x6e
'9', # 0x6f
'', # 0x70
'[?]', # 0x71
'[?]', # 0x72
'[?]', # 0x73
'[?]', # 0x74
'[?]', # 0x75
'[?]', # 0x76
'[?]', # 0x77
'[?]', # 0x78
'[?]', # 0x79
'[?]', # 0x7a
'[?]', # 0x7b
'[?]', # 0x7c
'[?]', # 0x7d
'[?]', # 0x7e
'[?]', # 0x7f
'[?]', # 0x80
'[?]', # 0x81
'N', # 0x82
'H', # 0x83
'[?]', # 0x84
'a', # 0x85
'aa', # 0x86
'i', # 0x87
'ii', # 0x88
'u', # 0x89
'uu', # 0x8a
'[?]', # 0x8b
'[?]', # 0x8c
'[?]', # 0x8d
'e', # 0x8e
'ee', # 0x8f
'ai', # 0x90
'[?]', # 0x91
'o', # 0x92
'oo', # 0x93
'au', # 0x94
'k', # 0x95
'[?]', # 0x96
'[?]', # 0x97
'[?]', # 0x98
'ng', # 0x99
'c', # 0x9a
'[?]', # 0x9b
'j', # 0x9c
'[?]', # 0x9d
'ny', # 0x9e
'tt', # 0x9f
'[?]', # 0xa0
'[?]', # 0xa1
'[?]', # 0xa2
'nn', # 0xa3
't', # 0xa4
'[?]', # 0xa5
'[?]', # 0xa6
'[?]', # 0xa7
'n', # 0xa8
'nnn', # 0xa9
'p', # 0xaa
'[?]', # 0xab
'[?]', # 0xac
'[?]', # 0xad
'm', # 0xae
'y', # 0xaf
'r', # 0xb0
'rr', # 0xb1
'l', # 0xb2
'll', # 0xb3
'lll', # 0xb4
'v', # 0xb5
'[?]', # 0xb6
'ss', # 0xb7
's', # 0xb8
'h', # 0xb9
'[?]', # 0xba
'[?]', # 0xbb
'[?]', # 0xbc
'[?]', # 0xbd
'aa', # 0xbe
'i', # 0xbf
'ii', # 0xc0
'u', # 0xc1
'uu', # 0xc2
'[?]', # 0xc3
'[?]', # 0xc4
'[?]', # 0xc5
'e', # 0xc6
'ee', # 0xc7
'ai', # 0xc8
'[?]', # 0xc9
'o', # 0xca
'oo', # 0xcb
'au', # 0xcc
'', # 0xcd
'[?]', # 0xce
'[?]', # 0xcf
'[?]', # 0xd0
'[?]', # 0xd1
'[?]', # 0xd2
'[?]', # 0xd3
'[?]', # 0xd4
'[?]', # 0xd5
'[?]', # 0xd6
'+', # 0xd7
'[?]', # 0xd8
'[?]', # 0xd9
'[?]', # 0xda
'[?]', # 0xdb
'[?]', # 0xdc
'[?]', # 0xdd
'[?]', # 0xde
'[?]', # 0xdf
'[?]', # 0xe0
'[?]', # 0xe1
'[?]', # 0xe2
'[?]', # 0xe3
'[?]', # 0xe4
'[?]', # 0xe5
'0', # 0xe6
'1', # 0xe7
'2', # 0xe8
'3', # 0xe9
'4', # 0xea
'5', # 0xeb
'6', # 0xec
'7', # 0xed
'8', # 0xee
'9', # 0xef
'+10+', # 0xf0
'+100+', # 0xf1
'+1000+', # 0xf2
'[?]', # 0xf3
'[?]', # 0xf4
'[?]', # 0xf5
'[?]', # 0xf6
'[?]', # 0xf7
'[?]', # 0xf8
'[?]', # 0xf9
'[?]', # 0xfa
'[?]', # 0xfb
'[?]', # 0xfc
'[?]', # 0xfd
'[?]', # 0xfe
)

257
libs/unidecode/x00c.py Normal file
View file

@ -0,0 +1,257 @@
data = (
'[?]', # 0x00
'N', # 0x01
'N', # 0x02
'H', # 0x03
'[?]', # 0x04
'a', # 0x05
'aa', # 0x06
'i', # 0x07
'ii', # 0x08
'u', # 0x09
'uu', # 0x0a
'R', # 0x0b
'L', # 0x0c
'[?]', # 0x0d
'e', # 0x0e
'ee', # 0x0f
'ai', # 0x10
'[?]', # 0x11
'o', # 0x12
'oo', # 0x13
'au', # 0x14
'k', # 0x15
'kh', # 0x16
'g', # 0x17
'gh', # 0x18
'ng', # 0x19
'c', # 0x1a
'ch', # 0x1b
'j', # 0x1c
'jh', # 0x1d
'ny', # 0x1e
'tt', # 0x1f
'tth', # 0x20
'dd', # 0x21
'ddh', # 0x22
'nn', # 0x23
't', # 0x24
'th', # 0x25
'd', # 0x26
'dh', # 0x27
'n', # 0x28
'[?]', # 0x29
'p', # 0x2a
'ph', # 0x2b
'b', # 0x2c
'bh', # 0x2d
'm', # 0x2e
'y', # 0x2f
'r', # 0x30
'rr', # 0x31
'l', # 0x32
'll', # 0x33
'[?]', # 0x34
'v', # 0x35
'sh', # 0x36
'ss', # 0x37
's', # 0x38
'h', # 0x39
'[?]', # 0x3a
'[?]', # 0x3b
'[?]', # 0x3c
'[?]', # 0x3d
'aa', # 0x3e
'i', # 0x3f
'ii', # 0x40
'u', # 0x41
'uu', # 0x42
'R', # 0x43
'RR', # 0x44
'[?]', # 0x45
'e', # 0x46
'ee', # 0x47
'ai', # 0x48
'[?]', # 0x49
'o', # 0x4a
'oo', # 0x4b
'au', # 0x4c
'', # 0x4d
'[?]', # 0x4e
'[?]', # 0x4f
'[?]', # 0x50
'[?]', # 0x51
'[?]', # 0x52
'[?]', # 0x53
'[?]', # 0x54
'+', # 0x55
'+', # 0x56
'[?]', # 0x57
'[?]', # 0x58
'[?]', # 0x59
'[?]', # 0x5a
'[?]', # 0x5b
'[?]', # 0x5c
'[?]', # 0x5d
'[?]', # 0x5e
'[?]', # 0x5f
'RR', # 0x60
'LL', # 0x61
'[?]', # 0x62
'[?]', # 0x63
'[?]', # 0x64
'[?]', # 0x65
'0', # 0x66
'1', # 0x67
'2', # 0x68
'3', # 0x69
'4', # 0x6a
'5', # 0x6b
'6', # 0x6c
'7', # 0x6d
'8', # 0x6e
'9', # 0x6f
'[?]', # 0x70
'[?]', # 0x71
'[?]', # 0x72
'[?]', # 0x73
'[?]', # 0x74
'[?]', # 0x75
'[?]', # 0x76
'[?]', # 0x77
'[?]', # 0x78
'[?]', # 0x79
'[?]', # 0x7a
'[?]', # 0x7b
'[?]', # 0x7c
'[?]', # 0x7d
'[?]', # 0x7e
'[?]', # 0x7f
'[?]', # 0x80
'[?]', # 0x81
'N', # 0x82
'H', # 0x83
'[?]', # 0x84
'a', # 0x85
'aa', # 0x86
'i', # 0x87
'ii', # 0x88
'u', # 0x89
'uu', # 0x8a
'R', # 0x8b
'L', # 0x8c
'[?]', # 0x8d
'e', # 0x8e
'ee', # 0x8f
'ai', # 0x90
'[?]', # 0x91
'o', # 0x92
'oo', # 0x93
'au', # 0x94
'k', # 0x95
'kh', # 0x96
'g', # 0x97
'gh', # 0x98
'ng', # 0x99
'c', # 0x9a
'ch', # 0x9b
'j', # 0x9c
'jh', # 0x9d
'ny', # 0x9e
'tt', # 0x9f
'tth', # 0xa0
'dd', # 0xa1
'ddh', # 0xa2
'nn', # 0xa3
't', # 0xa4
'th', # 0xa5
'd', # 0xa6
'dh', # 0xa7
'n', # 0xa8
'[?]', # 0xa9
'p', # 0xaa
'ph', # 0xab
'b', # 0xac
'bh', # 0xad
'm', # 0xae
'y', # 0xaf
'r', # 0xb0
'rr', # 0xb1
'l', # 0xb2
'll', # 0xb3
'[?]', # 0xb4
'v', # 0xb5
'sh', # 0xb6
'ss', # 0xb7
's', # 0xb8
'h', # 0xb9
'[?]', # 0xba
'[?]', # 0xbb
'[?]', # 0xbc
'[?]', # 0xbd
'aa', # 0xbe
'i', # 0xbf
'ii', # 0xc0
'u', # 0xc1
'uu', # 0xc2
'R', # 0xc3
'RR', # 0xc4
'[?]', # 0xc5
'e', # 0xc6
'ee', # 0xc7
'ai', # 0xc8
'[?]', # 0xc9
'o', # 0xca
'oo', # 0xcb
'au', # 0xcc
'', # 0xcd
'[?]', # 0xce
'[?]', # 0xcf
'[?]', # 0xd0
'[?]', # 0xd1
'[?]', # 0xd2
'[?]', # 0xd3
'[?]', # 0xd4
'+', # 0xd5
'+', # 0xd6
'[?]', # 0xd7
'[?]', # 0xd8
'[?]', # 0xd9
'[?]', # 0xda
'[?]', # 0xdb
'[?]', # 0xdc
'[?]', # 0xdd
'lll', # 0xde
'[?]', # 0xdf
'RR', # 0xe0
'LL', # 0xe1
'[?]', # 0xe2
'[?]', # 0xe3
'[?]', # 0xe4
'[?]', # 0xe5
'0', # 0xe6
'1', # 0xe7
'2', # 0xe8
'3', # 0xe9
'4', # 0xea
'5', # 0xeb
'6', # 0xec
'7', # 0xed
'8', # 0xee
'9', # 0xef
'[?]', # 0xf0
'[?]', # 0xf1
'[?]', # 0xf2
'[?]', # 0xf3
'[?]', # 0xf4
'[?]', # 0xf5
'[?]', # 0xf6
'[?]', # 0xf7
'[?]', # 0xf8
'[?]', # 0xf9
'[?]', # 0xfa
'[?]', # 0xfb
'[?]', # 0xfc
'[?]', # 0xfd
'[?]', # 0xfe
)

257
libs/unidecode/x00d.py Normal file
View file

@ -0,0 +1,257 @@
data = (
'[?]', # 0x00
'[?]', # 0x01
'N', # 0x02
'H', # 0x03
'[?]', # 0x04
'a', # 0x05
'aa', # 0x06
'i', # 0x07
'ii', # 0x08
'u', # 0x09
'uu', # 0x0a
'R', # 0x0b
'L', # 0x0c
'[?]', # 0x0d
'e', # 0x0e
'ee', # 0x0f
'ai', # 0x10
'[?]', # 0x11
'o', # 0x12
'oo', # 0x13
'au', # 0x14
'k', # 0x15
'kh', # 0x16
'g', # 0x17
'gh', # 0x18
'ng', # 0x19
'c', # 0x1a
'ch', # 0x1b
'j', # 0x1c
'jh', # 0x1d
'ny', # 0x1e
'tt', # 0x1f
'tth', # 0x20
'dd', # 0x21
'ddh', # 0x22
'nn', # 0x23
't', # 0x24
'th', # 0x25
'd', # 0x26
'dh', # 0x27
'n', # 0x28
'[?]', # 0x29
'p', # 0x2a
'ph', # 0x2b
'b', # 0x2c
'bh', # 0x2d
'm', # 0x2e
'y', # 0x2f
'r', # 0x30
'rr', # 0x31
'l', # 0x32
'll', # 0x33
'lll', # 0x34
'v', # 0x35
'sh', # 0x36
'ss', # 0x37
's', # 0x38
'h', # 0x39
'[?]', # 0x3a
'[?]', # 0x3b
'[?]', # 0x3c
'[?]', # 0x3d
'aa', # 0x3e
'i', # 0x3f
'ii', # 0x40
'u', # 0x41
'uu', # 0x42
'R', # 0x43
'[?]', # 0x44
'[?]', # 0x45
'e', # 0x46
'ee', # 0x47
'ai', # 0x48
'', # 0x49
'o', # 0x4a
'oo', # 0x4b
'au', # 0x4c
'', # 0x4d
'[?]', # 0x4e
'[?]', # 0x4f
'[?]', # 0x50
'[?]', # 0x51
'[?]', # 0x52
'[?]', # 0x53
'[?]', # 0x54
'[?]', # 0x55
'[?]', # 0x56
'+', # 0x57
'[?]', # 0x58
'[?]', # 0x59
'[?]', # 0x5a
'[?]', # 0x5b
'[?]', # 0x5c
'[?]', # 0x5d
'[?]', # 0x5e
'[?]', # 0x5f
'RR', # 0x60
'LL', # 0x61
'[?]', # 0x62
'[?]', # 0x63
'[?]', # 0x64
'[?]', # 0x65
'0', # 0x66
'1', # 0x67
'2', # 0x68
'3', # 0x69
'4', # 0x6a
'5', # 0x6b
'6', # 0x6c
'7', # 0x6d
'8', # 0x6e
'9', # 0x6f
'[?]', # 0x70
'[?]', # 0x71
'[?]', # 0x72
'[?]', # 0x73
'[?]', # 0x74
'[?]', # 0x75
'[?]', # 0x76
'[?]', # 0x77
'[?]', # 0x78
'[?]', # 0x79
'[?]', # 0x7a
'[?]', # 0x7b
'[?]', # 0x7c
'[?]', # 0x7d
'[?]', # 0x7e
'[?]', # 0x7f
'[?]', # 0x80
'[?]', # 0x81
'N', # 0x82
'H', # 0x83
'[?]', # 0x84
'a', # 0x85
'aa', # 0x86
'ae', # 0x87
'aae', # 0x88
'i', # 0x89
'ii', # 0x8a
'u', # 0x8b
'uu', # 0x8c
'R', # 0x8d
'RR', # 0x8e
'L', # 0x8f
'LL', # 0x90
'e', # 0x91
'ee', # 0x92
'ai', # 0x93
'o', # 0x94
'oo', # 0x95
'au', # 0x96
'[?]', # 0x97
'[?]', # 0x98
'[?]', # 0x99
'k', # 0x9a
'kh', # 0x9b
'g', # 0x9c
'gh', # 0x9d
'ng', # 0x9e
'nng', # 0x9f
'c', # 0xa0
'ch', # 0xa1
'j', # 0xa2
'jh', # 0xa3
'ny', # 0xa4
'jny', # 0xa5
'nyj', # 0xa6
'tt', # 0xa7
'tth', # 0xa8
'dd', # 0xa9
'ddh', # 0xaa
'nn', # 0xab
'nndd', # 0xac
't', # 0xad
'th', # 0xae
'd', # 0xaf
'dh', # 0xb0
'n', # 0xb1
'[?]', # 0xb2
'nd', # 0xb3
'p', # 0xb4
'ph', # 0xb5
'b', # 0xb6
'bh', # 0xb7
'm', # 0xb8
'mb', # 0xb9
'y', # 0xba
'r', # 0xbb
'[?]', # 0xbc
'l', # 0xbd
'[?]', # 0xbe
'[?]', # 0xbf
'v', # 0xc0
'sh', # 0xc1
'ss', # 0xc2
's', # 0xc3
'h', # 0xc4
'll', # 0xc5
'f', # 0xc6
'[?]', # 0xc7
'[?]', # 0xc8
'[?]', # 0xc9
'', # 0xca
'[?]', # 0xcb
'[?]', # 0xcc
'[?]', # 0xcd
'[?]', # 0xce
'aa', # 0xcf
'ae', # 0xd0
'aae', # 0xd1
'i', # 0xd2
'ii', # 0xd3
'u', # 0xd4
'[?]', # 0xd5
'uu', # 0xd6
'[?]', # 0xd7
'R', # 0xd8
'e', # 0xd9
'ee', # 0xda
'ai', # 0xdb
'o', # 0xdc
'oo', # 0xdd
'au', # 0xde
'L', # 0xdf
'[?]', # 0xe0
'[?]', # 0xe1
'[?]', # 0xe2
'[?]', # 0xe3
'[?]', # 0xe4
'[?]', # 0xe5
'[?]', # 0xe6
'[?]', # 0xe7
'[?]', # 0xe8
'[?]', # 0xe9
'[?]', # 0xea
'[?]', # 0xeb
'[?]', # 0xec
'[?]', # 0xed
'[?]', # 0xee
'[?]', # 0xef
'[?]', # 0xf0
'[?]', # 0xf1
'RR', # 0xf2
'LL', # 0xf3
' . ', # 0xf4
'[?]', # 0xf5
'[?]', # 0xf6
'[?]', # 0xf7
'[?]', # 0xf8
'[?]', # 0xf9
'[?]', # 0xfa
'[?]', # 0xfb
'[?]', # 0xfc
'[?]', # 0xfd
'[?]', # 0xfe
)

257
libs/unidecode/x00e.py Normal file
View file

@ -0,0 +1,257 @@
data = (
'[?]', # 0x00
'k', # 0x01
'kh', # 0x02
'kh', # 0x03
'kh', # 0x04
'kh', # 0x05
'kh', # 0x06
'ng', # 0x07
'cch', # 0x08
'ch', # 0x09
'ch', # 0x0a
'ch', # 0x0b
'ch', # 0x0c
'y', # 0x0d
'd', # 0x0e
't', # 0x0f
'th', # 0x10
'th', # 0x11
'th', # 0x12
'n', # 0x13
'd', # 0x14
't', # 0x15
'th', # 0x16
'th', # 0x17
'th', # 0x18
'n', # 0x19
'b', # 0x1a
'p', # 0x1b
'ph', # 0x1c
'f', # 0x1d
'ph', # 0x1e
'f', # 0x1f
'ph', # 0x20
'm', # 0x21
'y', # 0x22
'r', # 0x23
'R', # 0x24
'l', # 0x25
'L', # 0x26
'w', # 0x27
's', # 0x28
's', # 0x29
's', # 0x2a
'h', # 0x2b
'l', # 0x2c
'`', # 0x2d
'h', # 0x2e
'~', # 0x2f
'a', # 0x30
'a', # 0x31
'aa', # 0x32
'am', # 0x33
'i', # 0x34
'ii', # 0x35
'ue', # 0x36
'uue', # 0x37
'u', # 0x38
'uu', # 0x39
'\'', # 0x3a
'[?]', # 0x3b
'[?]', # 0x3c
'[?]', # 0x3d
'[?]', # 0x3e
'Bh.', # 0x3f
'e', # 0x40
'ae', # 0x41
'o', # 0x42
'ai', # 0x43
'ai', # 0x44
'ao', # 0x45
'+', # 0x46
'', # 0x47
'', # 0x48
'', # 0x49
'', # 0x4a
'', # 0x4b
'', # 0x4c
'M', # 0x4d
'', # 0x4e
' * ', # 0x4f
'0', # 0x50
'1', # 0x51
'2', # 0x52
'3', # 0x53
'4', # 0x54
'5', # 0x55
'6', # 0x56
'7', # 0x57
'8', # 0x58
'9', # 0x59
' // ', # 0x5a
' /// ', # 0x5b
'[?]', # 0x5c
'[?]', # 0x5d
'[?]', # 0x5e
'[?]', # 0x5f
'[?]', # 0x60
'[?]', # 0x61
'[?]', # 0x62
'[?]', # 0x63
'[?]', # 0x64
'[?]', # 0x65
'[?]', # 0x66
'[?]', # 0x67
'[?]', # 0x68
'[?]', # 0x69
'[?]', # 0x6a
'[?]', # 0x6b
'[?]', # 0x6c
'[?]', # 0x6d
'[?]', # 0x6e
'[?]', # 0x6f
'[?]', # 0x70
'[?]', # 0x71
'[?]', # 0x72
'[?]', # 0x73
'[?]', # 0x74
'[?]', # 0x75
'[?]', # 0x76
'[?]', # 0x77
'[?]', # 0x78
'[?]', # 0x79
'[?]', # 0x7a
'[?]', # 0x7b
'[?]', # 0x7c
'[?]', # 0x7d
'[?]', # 0x7e
'[?]', # 0x7f
'[?]', # 0x80
'k', # 0x81
'kh', # 0x82
'[?]', # 0x83
'kh', # 0x84
'[?]', # 0x85
'[?]', # 0x86
'ng', # 0x87
'ch', # 0x88
'[?]', # 0x89
's', # 0x8a
'[?]', # 0x8b
'[?]', # 0x8c
'ny', # 0x8d
'[?]', # 0x8e
'[?]', # 0x8f
'[?]', # 0x90
'[?]', # 0x91
'[?]', # 0x92
'[?]', # 0x93
'd', # 0x94
'h', # 0x95
'th', # 0x96
'th', # 0x97
'[?]', # 0x98
'n', # 0x99
'b', # 0x9a
'p', # 0x9b
'ph', # 0x9c
'f', # 0x9d
'ph', # 0x9e
'f', # 0x9f
'[?]', # 0xa0
'm', # 0xa1
'y', # 0xa2
'r', # 0xa3
'[?]', # 0xa4
'l', # 0xa5
'[?]', # 0xa6
'w', # 0xa7
'[?]', # 0xa8
'[?]', # 0xa9
's', # 0xaa
'h', # 0xab
'[?]', # 0xac
'`', # 0xad
'', # 0xae
'~', # 0xaf
'a', # 0xb0
'', # 0xb1
'aa', # 0xb2
'am', # 0xb3
'i', # 0xb4
'ii', # 0xb5
'y', # 0xb6
'yy', # 0xb7
'u', # 0xb8
'uu', # 0xb9
'[?]', # 0xba
'o', # 0xbb
'l', # 0xbc
'ny', # 0xbd
'[?]', # 0xbe
'[?]', # 0xbf
'e', # 0xc0
'ei', # 0xc1
'o', # 0xc2
'ay', # 0xc3
'ai', # 0xc4
'[?]', # 0xc5
'+', # 0xc6
'[?]', # 0xc7
'', # 0xc8
'', # 0xc9
'', # 0xca
'', # 0xcb
'', # 0xcc
'M', # 0xcd
'[?]', # 0xce
'[?]', # 0xcf
'0', # 0xd0
'1', # 0xd1
'2', # 0xd2
'3', # 0xd3
'4', # 0xd4
'5', # 0xd5
'6', # 0xd6
'7', # 0xd7
'8', # 0xd8
'9', # 0xd9
'[?]', # 0xda
'[?]', # 0xdb
'hn', # 0xdc
'hm', # 0xdd
'[?]', # 0xde
'[?]', # 0xdf
'[?]', # 0xe0
'[?]', # 0xe1
'[?]', # 0xe2
'[?]', # 0xe3
'[?]', # 0xe4
'[?]', # 0xe5
'[?]', # 0xe6
'[?]', # 0xe7
'[?]', # 0xe8
'[?]', # 0xe9
'[?]', # 0xea
'[?]', # 0xeb
'[?]', # 0xec
'[?]', # 0xed
'[?]', # 0xee
'[?]', # 0xef
'[?]', # 0xf0
'[?]', # 0xf1
'[?]', # 0xf2
'[?]', # 0xf3
'[?]', # 0xf4
'[?]', # 0xf5
'[?]', # 0xf6
'[?]', # 0xf7
'[?]', # 0xf8
'[?]', # 0xf9
'[?]', # 0xfa
'[?]', # 0xfb
'[?]', # 0xfc
'[?]', # 0xfd
'[?]', # 0xfe
)

257
libs/unidecode/x00f.py Normal file
View file

@ -0,0 +1,257 @@
data = (
'AUM', # 0x00
'', # 0x01
'', # 0x02
'', # 0x03
'', # 0x04
'', # 0x05
'', # 0x06
'', # 0x07
' // ', # 0x08
' * ', # 0x09
'', # 0x0a
'-', # 0x0b
' / ', # 0x0c
' / ', # 0x0d
' // ', # 0x0e
' -/ ', # 0x0f
' +/ ', # 0x10
' X/ ', # 0x11
' /XX/ ', # 0x12
' /X/ ', # 0x13
', ', # 0x14
'', # 0x15
'', # 0x16
'', # 0x17
'', # 0x18
'', # 0x19
'', # 0x1a
'', # 0x1b
'', # 0x1c
'', # 0x1d
'', # 0x1e
'', # 0x1f
'0', # 0x20
'1', # 0x21
'2', # 0x22
'3', # 0x23
'4', # 0x24
'5', # 0x25
'6', # 0x26
'7', # 0x27
'8', # 0x28
'9', # 0x29
'.5', # 0x2a
'1.5', # 0x2b
'2.5', # 0x2c
'3.5', # 0x2d
'4.5', # 0x2e
'5.5', # 0x2f
'6.5', # 0x30
'7.5', # 0x31
'8.5', # 0x32
'-.5', # 0x33
'+', # 0x34
'*', # 0x35
'^', # 0x36
'_', # 0x37
'', # 0x38
'~', # 0x39
'[?]', # 0x3a
']', # 0x3b
'[[', # 0x3c
']]', # 0x3d
'', # 0x3e
'', # 0x3f
'k', # 0x40
'kh', # 0x41
'g', # 0x42
'gh', # 0x43
'ng', # 0x44
'c', # 0x45
'ch', # 0x46
'j', # 0x47
'[?]', # 0x48
'ny', # 0x49
'tt', # 0x4a
'tth', # 0x4b
'dd', # 0x4c
'ddh', # 0x4d
'nn', # 0x4e
't', # 0x4f
'th', # 0x50
'd', # 0x51
'dh', # 0x52
'n', # 0x53
'p', # 0x54
'ph', # 0x55
'b', # 0x56
'bh', # 0x57
'm', # 0x58
'ts', # 0x59
'tsh', # 0x5a
'dz', # 0x5b
'dzh', # 0x5c
'w', # 0x5d
'zh', # 0x5e
'z', # 0x5f
'\'', # 0x60
'y', # 0x61
'r', # 0x62
'l', # 0x63
'sh', # 0x64
'ssh', # 0x65
's', # 0x66
'h', # 0x67
'a', # 0x68
'kss', # 0x69
'r', # 0x6a
'[?]', # 0x6b
'[?]', # 0x6c
'[?]', # 0x6d
'[?]', # 0x6e
'[?]', # 0x6f
'[?]', # 0x70
'aa', # 0x71
'i', # 0x72
'ii', # 0x73
'u', # 0x74
'uu', # 0x75
'R', # 0x76
'RR', # 0x77
'L', # 0x78
'LL', # 0x79
'e', # 0x7a
'ee', # 0x7b
'o', # 0x7c
'oo', # 0x7d
'M', # 0x7e
'H', # 0x7f
'i', # 0x80
'ii', # 0x81
'', # 0x82
'', # 0x83
'', # 0x84
'', # 0x85
'', # 0x86
'', # 0x87
'', # 0x88
'', # 0x89
'', # 0x8a
'', # 0x8b
'[?]', # 0x8c
'[?]', # 0x8d
'[?]', # 0x8e
'[?]', # 0x8f
'k', # 0x90
'kh', # 0x91
'g', # 0x92
'gh', # 0x93
'ng', # 0x94
'c', # 0x95
'ch', # 0x96
'j', # 0x97
'[?]', # 0x98
'ny', # 0x99
'tt', # 0x9a
'tth', # 0x9b
'dd', # 0x9c
'ddh', # 0x9d
'nn', # 0x9e
't', # 0x9f
'th', # 0xa0
'd', # 0xa1
'dh', # 0xa2
'n', # 0xa3
'p', # 0xa4
'ph', # 0xa5
'b', # 0xa6
'bh', # 0xa7
'm', # 0xa8
'ts', # 0xa9
'tsh', # 0xaa
'dz', # 0xab
'dzh', # 0xac
'w', # 0xad
'zh', # 0xae
'z', # 0xaf
'\'', # 0xb0
'y', # 0xb1
'r', # 0xb2
'l', # 0xb3
'sh', # 0xb4
'ss', # 0xb5
's', # 0xb6
'h', # 0xb7
'a', # 0xb8
'kss', # 0xb9
'w', # 0xba
'y', # 0xbb
'r', # 0xbc
'[?]', # 0xbd
'X', # 0xbe
' :X: ', # 0xbf
' /O/ ', # 0xc0
' /o/ ', # 0xc1
' \\o\\ ', # 0xc2
' (O) ', # 0xc3
'', # 0xc4
'', # 0xc5
'', # 0xc6
'', # 0xc7
'', # 0xc8
'', # 0xc9
'', # 0xca
'', # 0xcb
'', # 0xcc
'[?]', # 0xcd
'[?]', # 0xce
'', # 0xcf
'[?]', # 0xd0
'[?]', # 0xd1
'[?]', # 0xd2
'[?]', # 0xd3
'[?]', # 0xd4
'[?]', # 0xd5
'[?]', # 0xd6
'[?]', # 0xd7
'[?]', # 0xd8
'[?]', # 0xd9
'[?]', # 0xda
'[?]', # 0xdb
'[?]', # 0xdc
'[?]', # 0xdd
'[?]', # 0xde
'[?]', # 0xdf
'[?]', # 0xe0
'[?]', # 0xe1
'[?]', # 0xe2
'[?]', # 0xe3
'[?]', # 0xe4
'[?]', # 0xe5
'[?]', # 0xe6
'[?]', # 0xe7
'[?]', # 0xe8
'[?]', # 0xe9
'[?]', # 0xea
'[?]', # 0xeb
'[?]', # 0xec
'[?]', # 0xed
'[?]', # 0xee
'[?]', # 0xef
'[?]', # 0xf0
'[?]', # 0xf1
'[?]', # 0xf2
'[?]', # 0xf3
'[?]', # 0xf4
'[?]', # 0xf5
'[?]', # 0xf6
'[?]', # 0xf7
'[?]', # 0xf8
'[?]', # 0xf9
'[?]', # 0xfa
'[?]', # 0xfb
'[?]', # 0xfc
'[?]', # 0xfd
'[?]', # 0xfe
)

257
libs/unidecode/x010.py Normal file
View file

@ -0,0 +1,257 @@
data = (
'k', # 0x00
'kh', # 0x01
'g', # 0x02
'gh', # 0x03
'ng', # 0x04
'c', # 0x05
'ch', # 0x06
'j', # 0x07
'jh', # 0x08
'ny', # 0x09
'nny', # 0x0a
'tt', # 0x0b
'tth', # 0x0c
'dd', # 0x0d
'ddh', # 0x0e
'nn', # 0x0f
'tt', # 0x10
'th', # 0x11
'd', # 0x12
'dh', # 0x13
'n', # 0x14
'p', # 0x15
'ph', # 0x16
'b', # 0x17
'bh', # 0x18
'm', # 0x19
'y', # 0x1a
'r', # 0x1b
'l', # 0x1c
'w', # 0x1d
's', # 0x1e
'h', # 0x1f
'll', # 0x20
'a', # 0x21
'[?]', # 0x22
'i', # 0x23
'ii', # 0x24
'u', # 0x25
'uu', # 0x26
'e', # 0x27
'[?]', # 0x28
'o', # 0x29
'au', # 0x2a
'[?]', # 0x2b
'aa', # 0x2c
'i', # 0x2d
'ii', # 0x2e
'u', # 0x2f
'uu', # 0x30
'e', # 0x31
'ai', # 0x32
'[?]', # 0x33
'[?]', # 0x34
'[?]', # 0x35
'N', # 0x36
'\'', # 0x37
':', # 0x38
'', # 0x39
'[?]', # 0x3a
'[?]', # 0x3b
'[?]', # 0x3c
'[?]', # 0x3d
'[?]', # 0x3e
'[?]', # 0x3f
'0', # 0x40
'1', # 0x41
'2', # 0x42
'3', # 0x43
'4', # 0x44
'5', # 0x45
'6', # 0x46
'7', # 0x47
'8', # 0x48
'9', # 0x49
' / ', # 0x4a
' // ', # 0x4b
'n*', # 0x4c
'r*', # 0x4d
'l*', # 0x4e
'e*', # 0x4f
'sh', # 0x50
'ss', # 0x51
'R', # 0x52
'RR', # 0x53
'L', # 0x54
'LL', # 0x55
'R', # 0x56
'RR', # 0x57
'L', # 0x58
'LL', # 0x59
'[?]', # 0x5a
'[?]', # 0x5b
'[?]', # 0x5c
'[?]', # 0x5d
'[?]', # 0x5e
'[?]', # 0x5f
'[?]', # 0x60
'[?]', # 0x61
'[?]', # 0x62
'[?]', # 0x63
'[?]', # 0x64
'[?]', # 0x65
'[?]', # 0x66
'[?]', # 0x67
'[?]', # 0x68
'[?]', # 0x69
'[?]', # 0x6a
'[?]', # 0x6b
'[?]', # 0x6c
'[?]', # 0x6d
'[?]', # 0x6e
'[?]', # 0x6f
'[?]', # 0x70
'[?]', # 0x71
'[?]', # 0x72
'[?]', # 0x73
'[?]', # 0x74
'[?]', # 0x75
'[?]', # 0x76
'[?]', # 0x77
'[?]', # 0x78
'[?]', # 0x79
'[?]', # 0x7a
'[?]', # 0x7b
'[?]', # 0x7c
'[?]', # 0x7d
'[?]', # 0x7e
'[?]', # 0x7f
'[?]', # 0x80
'[?]', # 0x81
'[?]', # 0x82
'[?]', # 0x83
'[?]', # 0x84
'[?]', # 0x85
'[?]', # 0x86
'[?]', # 0x87
'[?]', # 0x88
'[?]', # 0x89
'[?]', # 0x8a
'[?]', # 0x8b
'[?]', # 0x8c
'[?]', # 0x8d
'[?]', # 0x8e
'[?]', # 0x8f
'[?]', # 0x90
'[?]', # 0x91
'[?]', # 0x92
'[?]', # 0x93
'[?]', # 0x94
'[?]', # 0x95
'[?]', # 0x96
'[?]', # 0x97
'[?]', # 0x98
'[?]', # 0x99
'[?]', # 0x9a
'[?]', # 0x9b
'[?]', # 0x9c
'[?]', # 0x9d
'[?]', # 0x9e
'[?]', # 0x9f
'A', # 0xa0
'B', # 0xa1
'G', # 0xa2
'D', # 0xa3
'E', # 0xa4
'V', # 0xa5
'Z', # 0xa6
'T`', # 0xa7
'I', # 0xa8
'K', # 0xa9
'L', # 0xaa
'M', # 0xab
'N', # 0xac
'O', # 0xad
'P', # 0xae
'Zh', # 0xaf
'R', # 0xb0
'S', # 0xb1
'T', # 0xb2
'U', # 0xb3
'P`', # 0xb4
'K`', # 0xb5
'G\'', # 0xb6
'Q', # 0xb7
'Sh', # 0xb8
'Ch`', # 0xb9
'C`', # 0xba
'Z\'', # 0xbb
'C', # 0xbc
'Ch', # 0xbd
'X', # 0xbe
'J', # 0xbf
'H', # 0xc0
'E', # 0xc1
'Y', # 0xc2
'W', # 0xc3
'Xh', # 0xc4
'OE', # 0xc5
'[?]', # 0xc6
'[?]', # 0xc7
'[?]', # 0xc8
'[?]', # 0xc9
'[?]', # 0xca
'[?]', # 0xcb
'[?]', # 0xcc
'[?]', # 0xcd
'[?]', # 0xce
'[?]', # 0xcf
'a', # 0xd0
'b', # 0xd1
'g', # 0xd2
'd', # 0xd3
'e', # 0xd4
'v', # 0xd5
'z', # 0xd6
't`', # 0xd7
'i', # 0xd8
'k', # 0xd9
'l', # 0xda
'm', # 0xdb
'n', # 0xdc
'o', # 0xdd
'p', # 0xde
'zh', # 0xdf
'r', # 0xe0
's', # 0xe1
't', # 0xe2
'u', # 0xe3
'p`', # 0xe4
'k`', # 0xe5
'g\'', # 0xe6
'q', # 0xe7
'sh', # 0xe8
'ch`', # 0xe9
'c`', # 0xea
'z\'', # 0xeb
'c', # 0xec
'ch', # 0xed
'x', # 0xee
'j', # 0xef
'h', # 0xf0
'e', # 0xf1
'y', # 0xf2
'w', # 0xf3
'xh', # 0xf4
'oe', # 0xf5
'f', # 0xf6
'[?]', # 0xf7
'[?]', # 0xf8
'[?]', # 0xf9
'[?]', # 0xfa
' // ', # 0xfb
'[?]', # 0xfc
'[?]', # 0xfd
'[?]', # 0xfe
)

257
libs/unidecode/x011.py Normal file
View file

@ -0,0 +1,257 @@
data = (
'g', # 0x00
'gg', # 0x01
'n', # 0x02
'd', # 0x03
'dd', # 0x04
'r', # 0x05
'm', # 0x06
'b', # 0x07
'bb', # 0x08
's', # 0x09
'ss', # 0x0a
'', # 0x0b
'j', # 0x0c
'jj', # 0x0d
'c', # 0x0e
'k', # 0x0f
't', # 0x10
'p', # 0x11
'h', # 0x12
'ng', # 0x13
'nn', # 0x14
'nd', # 0x15
'nb', # 0x16
'dg', # 0x17
'rn', # 0x18
'rr', # 0x19
'rh', # 0x1a
'rN', # 0x1b
'mb', # 0x1c
'mN', # 0x1d
'bg', # 0x1e
'bn', # 0x1f
'', # 0x20
'bs', # 0x21
'bsg', # 0x22
'bst', # 0x23
'bsb', # 0x24
'bss', # 0x25
'bsj', # 0x26
'bj', # 0x27
'bc', # 0x28
'bt', # 0x29
'bp', # 0x2a
'bN', # 0x2b
'bbN', # 0x2c
'sg', # 0x2d
'sn', # 0x2e
'sd', # 0x2f
'sr', # 0x30
'sm', # 0x31
'sb', # 0x32
'sbg', # 0x33
'sss', # 0x34
's', # 0x35
'sj', # 0x36
'sc', # 0x37
'sk', # 0x38
'st', # 0x39
'sp', # 0x3a
'sh', # 0x3b
'', # 0x3c
'', # 0x3d
'', # 0x3e
'', # 0x3f
'Z', # 0x40
'g', # 0x41
'd', # 0x42
'm', # 0x43
'b', # 0x44
's', # 0x45
'Z', # 0x46
'', # 0x47
'j', # 0x48
'c', # 0x49
't', # 0x4a
'p', # 0x4b
'N', # 0x4c
'j', # 0x4d
'', # 0x4e
'', # 0x4f
'', # 0x50
'', # 0x51
'ck', # 0x52
'ch', # 0x53
'', # 0x54
'', # 0x55
'pb', # 0x56
'pN', # 0x57
'hh', # 0x58
'Q', # 0x59
'[?]', # 0x5a
'[?]', # 0x5b
'[?]', # 0x5c
'[?]', # 0x5d
'[?]', # 0x5e
'', # 0x5f
'', # 0x60
'a', # 0x61
'ae', # 0x62
'ya', # 0x63
'yae', # 0x64
'eo', # 0x65
'e', # 0x66
'yeo', # 0x67
'ye', # 0x68
'o', # 0x69
'wa', # 0x6a
'wae', # 0x6b
'oe', # 0x6c
'yo', # 0x6d
'u', # 0x6e
'weo', # 0x6f
'we', # 0x70
'wi', # 0x71
'yu', # 0x72
'eu', # 0x73
'yi', # 0x74
'i', # 0x75
'a-o', # 0x76
'a-u', # 0x77
'ya-o', # 0x78
'ya-yo', # 0x79
'eo-o', # 0x7a
'eo-u', # 0x7b
'eo-eu', # 0x7c
'yeo-o', # 0x7d
'yeo-u', # 0x7e
'o-eo', # 0x7f
'o-e', # 0x80
'o-ye', # 0x81
'o-o', # 0x82
'o-u', # 0x83
'yo-ya', # 0x84
'yo-yae', # 0x85
'yo-yeo', # 0x86
'yo-o', # 0x87
'yo-i', # 0x88
'u-a', # 0x89
'u-ae', # 0x8a
'u-eo-eu', # 0x8b
'u-ye', # 0x8c
'u-u', # 0x8d
'yu-a', # 0x8e
'yu-eo', # 0x8f
'yu-e', # 0x90
'yu-yeo', # 0x91
'yu-ye', # 0x92
'yu-u', # 0x93
'yu-i', # 0x94
'eu-u', # 0x95
'eu-eu', # 0x96
'yi-u', # 0x97
'i-a', # 0x98
'i-ya', # 0x99
'i-o', # 0x9a
'i-u', # 0x9b
'i-eu', # 0x9c
'i-U', # 0x9d
'U', # 0x9e
'U-eo', # 0x9f
'U-u', # 0xa0
'U-i', # 0xa1
'UU', # 0xa2
'[?]', # 0xa3
'[?]', # 0xa4
'[?]', # 0xa5
'[?]', # 0xa6
'[?]', # 0xa7
'g', # 0xa8
'gg', # 0xa9
'gs', # 0xaa
'n', # 0xab
'nj', # 0xac
'nh', # 0xad
'd', # 0xae
'l', # 0xaf
'lg', # 0xb0
'lm', # 0xb1
'lb', # 0xb2
'ls', # 0xb3
'lt', # 0xb4
'lp', # 0xb5
'lh', # 0xb6
'm', # 0xb7
'b', # 0xb8
'bs', # 0xb9
's', # 0xba
'ss', # 0xbb
'ng', # 0xbc
'j', # 0xbd
'c', # 0xbe
'k', # 0xbf
't', # 0xc0
'p', # 0xc1
'h', # 0xc2
'gl', # 0xc3
'gsg', # 0xc4
'ng', # 0xc5
'nd', # 0xc6
'ns', # 0xc7
'nZ', # 0xc8
'nt', # 0xc9
'dg', # 0xca
'tl', # 0xcb
'lgs', # 0xcc
'ln', # 0xcd
'ld', # 0xce
'lth', # 0xcf
'll', # 0xd0
'lmg', # 0xd1
'lms', # 0xd2
'lbs', # 0xd3
'lbh', # 0xd4
'rNp', # 0xd5
'lss', # 0xd6
'lZ', # 0xd7
'lk', # 0xd8
'lQ', # 0xd9
'mg', # 0xda
'ml', # 0xdb
'mb', # 0xdc
'ms', # 0xdd
'mss', # 0xde
'mZ', # 0xdf
'mc', # 0xe0
'mh', # 0xe1
'mN', # 0xe2
'bl', # 0xe3
'bp', # 0xe4
'ph', # 0xe5
'pN', # 0xe6
'sg', # 0xe7
'sd', # 0xe8
'sl', # 0xe9
'sb', # 0xea
'Z', # 0xeb
'g', # 0xec
'ss', # 0xed
'', # 0xee
'kh', # 0xef
'N', # 0xf0
'Ns', # 0xf1
'NZ', # 0xf2
'pb', # 0xf3
'pN', # 0xf4
'hn', # 0xf5
'hl', # 0xf6
'hm', # 0xf7
'hb', # 0xf8
'Q', # 0xf9
'[?]', # 0xfa
'[?]', # 0xfb
'[?]', # 0xfc
'[?]', # 0xfd
'[?]', # 0xfe
)

258
libs/unidecode/x012.py Normal file
View file

@ -0,0 +1,258 @@
data = (
'ha', # 0x00
'hu', # 0x01
'hi', # 0x02
'haa', # 0x03
'hee', # 0x04
'he', # 0x05
'ho', # 0x06
'[?]', # 0x07
'la', # 0x08
'lu', # 0x09
'li', # 0x0a
'laa', # 0x0b
'lee', # 0x0c
'le', # 0x0d
'lo', # 0x0e
'lwa', # 0x0f
'hha', # 0x10
'hhu', # 0x11
'hhi', # 0x12
'hhaa', # 0x13
'hhee', # 0x14
'hhe', # 0x15
'hho', # 0x16
'hhwa', # 0x17
'ma', # 0x18
'mu', # 0x19
'mi', # 0x1a
'maa', # 0x1b
'mee', # 0x1c
'me', # 0x1d
'mo', # 0x1e
'mwa', # 0x1f
'sza', # 0x20
'szu', # 0x21
'szi', # 0x22
'szaa', # 0x23
'szee', # 0x24
'sze', # 0x25
'szo', # 0x26
'szwa', # 0x27
'ra', # 0x28
'ru', # 0x29
'ri', # 0x2a
'raa', # 0x2b
'ree', # 0x2c
're', # 0x2d
'ro', # 0x2e
'rwa', # 0x2f
'sa', # 0x30
'su', # 0x31
'si', # 0x32
'saa', # 0x33
'see', # 0x34
'se', # 0x35
'so', # 0x36
'swa', # 0x37
'sha', # 0x38
'shu', # 0x39
'shi', # 0x3a
'shaa', # 0x3b
'shee', # 0x3c
'she', # 0x3d
'sho', # 0x3e
'shwa', # 0x3f
'qa', # 0x40
'qu', # 0x41
'qi', # 0x42
'qaa', # 0x43
'qee', # 0x44
'qe', # 0x45
'qo', # 0x46
'[?]', # 0x47
'qwa', # 0x48
'[?]', # 0x49
'qwi', # 0x4a
'qwaa', # 0x4b
'qwee', # 0x4c
'qwe', # 0x4d
'[?]', # 0x4e
'[?]', # 0x4f
'qha', # 0x50
'qhu', # 0x51
'qhi', # 0x52
'qhaa', # 0x53
'qhee', # 0x54
'qhe', # 0x55
'qho', # 0x56
'[?]', # 0x57
'qhwa', # 0x58
'[?]', # 0x59
'qhwi', # 0x5a
'qhwaa', # 0x5b
'qhwee', # 0x5c
'qhwe', # 0x5d
'[?]', # 0x5e
'[?]', # 0x5f
'ba', # 0x60
'bu', # 0x61
'bi', # 0x62
'baa', # 0x63
'bee', # 0x64
'be', # 0x65
'bo', # 0x66
'bwa', # 0x67
'va', # 0x68
'vu', # 0x69
'vi', # 0x6a
'vaa', # 0x6b
'vee', # 0x6c
've', # 0x6d
'vo', # 0x6e
'vwa', # 0x6f
'ta', # 0x70
'tu', # 0x71
'ti', # 0x72
'taa', # 0x73
'tee', # 0x74
'te', # 0x75
'to', # 0x76
'twa', # 0x77
'ca', # 0x78
'cu', # 0x79
'ci', # 0x7a
'caa', # 0x7b
'cee', # 0x7c
'ce', # 0x7d
'co', # 0x7e
'cwa', # 0x7f
'xa', # 0x80
'xu', # 0x81
'xi', # 0x82
'xaa', # 0x83
'xee', # 0x84
'xe', # 0x85
'xo', # 0x86
'[?]', # 0x87
'xwa', # 0x88
'[?]', # 0x89
'xwi', # 0x8a
'xwaa', # 0x8b
'xwee', # 0x8c
'xwe', # 0x8d
'[?]', # 0x8e
'[?]', # 0x8f
'na', # 0x90
'nu', # 0x91
'ni', # 0x92
'naa', # 0x93
'nee', # 0x94
'ne', # 0x95
'no', # 0x96
'nwa', # 0x97
'nya', # 0x98
'nyu', # 0x99
'nyi', # 0x9a
'nyaa', # 0x9b
'nyee', # 0x9c
'nye', # 0x9d
'nyo', # 0x9e
'nywa', # 0x9f
'\'a', # 0xa0
'\'u', # 0xa1
'[?]', # 0xa2
'\'aa', # 0xa3
'\'ee', # 0xa4
'\'e', # 0xa5
'\'o', # 0xa6
'\'wa', # 0xa7
'ka', # 0xa8
'ku', # 0xa9
'ki', # 0xaa
'kaa', # 0xab
'kee', # 0xac
'ke', # 0xad
'ko', # 0xae
'[?]', # 0xaf
'kwa', # 0xb0
'[?]', # 0xb1
'kwi', # 0xb2
'kwaa', # 0xb3
'kwee', # 0xb4
'kwe', # 0xb5
'[?]', # 0xb6
'[?]', # 0xb7
'kxa', # 0xb8
'kxu', # 0xb9
'kxi', # 0xba
'kxaa', # 0xbb
'kxee', # 0xbc
'kxe', # 0xbd
'kxo', # 0xbe
'[?]', # 0xbf
'kxwa', # 0xc0
'[?]', # 0xc1
'kxwi', # 0xc2
'kxwaa', # 0xc3
'kxwee', # 0xc4
'kxwe', # 0xc5
'[?]', # 0xc6
'[?]', # 0xc7
'wa', # 0xc8
'wu', # 0xc9
'wi', # 0xca
'waa', # 0xcb
'wee', # 0xcc
'we', # 0xcd
'wo', # 0xce
'[?]', # 0xcf
'`a', # 0xd0
'`u', # 0xd1
'`i', # 0xd2
'`aa', # 0xd3
'`ee', # 0xd4
'`e', # 0xd5
'`o', # 0xd6
'[?]', # 0xd7
'za', # 0xd8
'zu', # 0xd9
'zi', # 0xda
'zaa', # 0xdb
'zee', # 0xdc
'ze', # 0xdd
'zo', # 0xde
'zwa', # 0xdf
'zha', # 0xe0
'zhu', # 0xe1
'zhi', # 0xe2
'zhaa', # 0xe3
'zhee', # 0xe4
'zhe', # 0xe5
'zho', # 0xe6
'zhwa', # 0xe7
'ya', # 0xe8
'yu', # 0xe9
'yi', # 0xea
'yaa', # 0xeb
'yee', # 0xec
'ye', # 0xed
'yo', # 0xee
'[?]', # 0xef
'da', # 0xf0
'du', # 0xf1
'di', # 0xf2
'daa', # 0xf3
'dee', # 0xf4
'de', # 0xf5
'do', # 0xf6
'dwa', # 0xf7
'dda', # 0xf8
'ddu', # 0xf9
'ddi', # 0xfa
'ddaa', # 0xfb
'ddee', # 0xfc
'dde', # 0xfd
'ddo', # 0xfe
'ddwa', # 0xff
)

257
libs/unidecode/x013.py Normal file
View file

@ -0,0 +1,257 @@
data = (
'ja', # 0x00
'ju', # 0x01
'ji', # 0x02
'jaa', # 0x03
'jee', # 0x04
'je', # 0x05
'jo', # 0x06
'jwa', # 0x07
'ga', # 0x08
'gu', # 0x09
'gi', # 0x0a
'gaa', # 0x0b
'gee', # 0x0c
'ge', # 0x0d
'go', # 0x0e
'[?]', # 0x0f
'gwa', # 0x10
'[?]', # 0x11
'gwi', # 0x12
'gwaa', # 0x13
'gwee', # 0x14
'gwe', # 0x15
'[?]', # 0x16
'[?]', # 0x17
'gga', # 0x18
'ggu', # 0x19
'ggi', # 0x1a
'ggaa', # 0x1b
'ggee', # 0x1c
'gge', # 0x1d
'ggo', # 0x1e
'[?]', # 0x1f
'tha', # 0x20
'thu', # 0x21
'thi', # 0x22
'thaa', # 0x23
'thee', # 0x24
'the', # 0x25
'tho', # 0x26
'thwa', # 0x27
'cha', # 0x28
'chu', # 0x29
'chi', # 0x2a
'chaa', # 0x2b
'chee', # 0x2c
'che', # 0x2d
'cho', # 0x2e
'chwa', # 0x2f
'pha', # 0x30
'phu', # 0x31
'phi', # 0x32
'phaa', # 0x33
'phee', # 0x34
'phe', # 0x35
'pho', # 0x36
'phwa', # 0x37
'tsa', # 0x38
'tsu', # 0x39
'tsi', # 0x3a
'tsaa', # 0x3b
'tsee', # 0x3c
'tse', # 0x3d
'tso', # 0x3e
'tswa', # 0x3f
'tza', # 0x40
'tzu', # 0x41
'tzi', # 0x42
'tzaa', # 0x43
'tzee', # 0x44
'tze', # 0x45
'tzo', # 0x46
'[?]', # 0x47
'fa', # 0x48
'fu', # 0x49
'fi', # 0x4a
'faa', # 0x4b
'fee', # 0x4c
'fe', # 0x4d
'fo', # 0x4e
'fwa', # 0x4f
'pa', # 0x50
'pu', # 0x51
'pi', # 0x52
'paa', # 0x53
'pee', # 0x54
'pe', # 0x55
'po', # 0x56
'pwa', # 0x57
'rya', # 0x58
'mya', # 0x59
'fya', # 0x5a
'[?]', # 0x5b
'[?]', # 0x5c
'[?]', # 0x5d
'[?]', # 0x5e
'[?]', # 0x5f
'[?]', # 0x60
' ', # 0x61
'.', # 0x62
',', # 0x63
';', # 0x64
':', # 0x65
':: ', # 0x66
'?', # 0x67
'//', # 0x68
'1', # 0x69
'2', # 0x6a
'3', # 0x6b
'4', # 0x6c
'5', # 0x6d
'6', # 0x6e
'7', # 0x6f
'8', # 0x70
'9', # 0x71
'10+', # 0x72
'20+', # 0x73
'30+', # 0x74
'40+', # 0x75
'50+', # 0x76
'60+', # 0x77
'70+', # 0x78
'80+', # 0x79
'90+', # 0x7a
'100+', # 0x7b
'10,000+', # 0x7c
'[?]', # 0x7d
'[?]', # 0x7e
'[?]', # 0x7f
'[?]', # 0x80
'[?]', # 0x81
'[?]', # 0x82
'[?]', # 0x83
'[?]', # 0x84
'[?]', # 0x85
'[?]', # 0x86
'[?]', # 0x87
'[?]', # 0x88
'[?]', # 0x89
'[?]', # 0x8a
'[?]', # 0x8b
'[?]', # 0x8c
'[?]', # 0x8d
'[?]', # 0x8e
'[?]', # 0x8f
'[?]', # 0x90
'[?]', # 0x91
'[?]', # 0x92
'[?]', # 0x93
'[?]', # 0x94
'[?]', # 0x95
'[?]', # 0x96
'[?]', # 0x97
'[?]', # 0x98
'[?]', # 0x99
'[?]', # 0x9a
'[?]', # 0x9b
'[?]', # 0x9c
'[?]', # 0x9d
'[?]', # 0x9e
'[?]', # 0x9f
'a', # 0xa0
'e', # 0xa1
'i', # 0xa2
'o', # 0xa3
'u', # 0xa4
'v', # 0xa5
'ga', # 0xa6
'ka', # 0xa7
'ge', # 0xa8
'gi', # 0xa9
'go', # 0xaa
'gu', # 0xab
'gv', # 0xac
'ha', # 0xad
'he', # 0xae
'hi', # 0xaf
'ho', # 0xb0
'hu', # 0xb1
'hv', # 0xb2
'la', # 0xb3
'le', # 0xb4
'li', # 0xb5
'lo', # 0xb6
'lu', # 0xb7
'lv', # 0xb8
'ma', # 0xb9
'me', # 0xba
'mi', # 0xbb
'mo', # 0xbc
'mu', # 0xbd
'na', # 0xbe
'hna', # 0xbf
'nah', # 0xc0
'ne', # 0xc1
'ni', # 0xc2
'no', # 0xc3
'nu', # 0xc4
'nv', # 0xc5
'qua', # 0xc6
'que', # 0xc7
'qui', # 0xc8
'quo', # 0xc9
'quu', # 0xca
'quv', # 0xcb
'sa', # 0xcc
's', # 0xcd
'se', # 0xce
'si', # 0xcf
'so', # 0xd0
'su', # 0xd1
'sv', # 0xd2
'da', # 0xd3
'ta', # 0xd4
'de', # 0xd5
'te', # 0xd6
'di', # 0xd7
'ti', # 0xd8
'do', # 0xd9
'du', # 0xda
'dv', # 0xdb
'dla', # 0xdc
'tla', # 0xdd
'tle', # 0xde
'tli', # 0xdf
'tlo', # 0xe0
'tlu', # 0xe1
'tlv', # 0xe2
'tsa', # 0xe3
'tse', # 0xe4
'tsi', # 0xe5
'tso', # 0xe6
'tsu', # 0xe7
'tsv', # 0xe8
'wa', # 0xe9
'we', # 0xea
'wi', # 0xeb
'wo', # 0xec
'wu', # 0xed
'wv', # 0xee
'ya', # 0xef
'ye', # 0xf0
'yi', # 0xf1
'yo', # 0xf2
'yu', # 0xf3
'yv', # 0xf4
'[?]', # 0xf5
'[?]', # 0xf6
'[?]', # 0xf7
'[?]', # 0xf8
'[?]', # 0xf9
'[?]', # 0xfa
'[?]', # 0xfb
'[?]', # 0xfc
'[?]', # 0xfd
'[?]', # 0xfe
)

258
libs/unidecode/x014.py Normal file
View file

@ -0,0 +1,258 @@
data = (
'[?]', # 0x00
'e', # 0x01
'aai', # 0x02
'i', # 0x03
'ii', # 0x04
'o', # 0x05
'oo', # 0x06
'oo', # 0x07
'ee', # 0x08
'i', # 0x09
'a', # 0x0a
'aa', # 0x0b
'we', # 0x0c
'we', # 0x0d
'wi', # 0x0e
'wi', # 0x0f
'wii', # 0x10
'wii', # 0x11
'wo', # 0x12
'wo', # 0x13
'woo', # 0x14
'woo', # 0x15
'woo', # 0x16
'wa', # 0x17
'wa', # 0x18
'waa', # 0x19
'waa', # 0x1a
'waa', # 0x1b
'ai', # 0x1c
'w', # 0x1d
'\'', # 0x1e
't', # 0x1f
'k', # 0x20
'sh', # 0x21
's', # 0x22
'n', # 0x23
'w', # 0x24
'n', # 0x25
'[?]', # 0x26
'w', # 0x27
'c', # 0x28
'?', # 0x29
'l', # 0x2a
'en', # 0x2b
'in', # 0x2c
'on', # 0x2d
'an', # 0x2e
'pe', # 0x2f
'paai', # 0x30
'pi', # 0x31
'pii', # 0x32
'po', # 0x33
'poo', # 0x34
'poo', # 0x35
'hee', # 0x36
'hi', # 0x37
'pa', # 0x38
'paa', # 0x39
'pwe', # 0x3a
'pwe', # 0x3b
'pwi', # 0x3c
'pwi', # 0x3d
'pwii', # 0x3e
'pwii', # 0x3f
'pwo', # 0x40
'pwo', # 0x41
'pwoo', # 0x42
'pwoo', # 0x43
'pwa', # 0x44
'pwa', # 0x45
'pwaa', # 0x46
'pwaa', # 0x47
'pwaa', # 0x48
'p', # 0x49
'p', # 0x4a
'h', # 0x4b
'te', # 0x4c
'taai', # 0x4d
'ti', # 0x4e
'tii', # 0x4f
'to', # 0x50
'too', # 0x51
'too', # 0x52
'dee', # 0x53
'di', # 0x54
'ta', # 0x55
'taa', # 0x56
'twe', # 0x57
'twe', # 0x58
'twi', # 0x59
'twi', # 0x5a
'twii', # 0x5b
'twii', # 0x5c
'two', # 0x5d
'two', # 0x5e
'twoo', # 0x5f
'twoo', # 0x60
'twa', # 0x61
'twa', # 0x62
'twaa', # 0x63
'twaa', # 0x64
'twaa', # 0x65
't', # 0x66
'tte', # 0x67
'tti', # 0x68
'tto', # 0x69
'tta', # 0x6a
'ke', # 0x6b
'kaai', # 0x6c
'ki', # 0x6d
'kii', # 0x6e
'ko', # 0x6f
'koo', # 0x70
'koo', # 0x71
'ka', # 0x72
'kaa', # 0x73
'kwe', # 0x74
'kwe', # 0x75
'kwi', # 0x76
'kwi', # 0x77
'kwii', # 0x78
'kwii', # 0x79
'kwo', # 0x7a
'kwo', # 0x7b
'kwoo', # 0x7c
'kwoo', # 0x7d
'kwa', # 0x7e
'kwa', # 0x7f
'kwaa', # 0x80
'kwaa', # 0x81
'kwaa', # 0x82
'k', # 0x83
'kw', # 0x84
'keh', # 0x85
'kih', # 0x86
'koh', # 0x87
'kah', # 0x88
'ce', # 0x89
'caai', # 0x8a
'ci', # 0x8b
'cii', # 0x8c
'co', # 0x8d
'coo', # 0x8e
'coo', # 0x8f
'ca', # 0x90
'caa', # 0x91
'cwe', # 0x92
'cwe', # 0x93
'cwi', # 0x94
'cwi', # 0x95
'cwii', # 0x96
'cwii', # 0x97
'cwo', # 0x98
'cwo', # 0x99
'cwoo', # 0x9a
'cwoo', # 0x9b
'cwa', # 0x9c
'cwa', # 0x9d
'cwaa', # 0x9e
'cwaa', # 0x9f
'cwaa', # 0xa0
'c', # 0xa1
'th', # 0xa2
'me', # 0xa3
'maai', # 0xa4
'mi', # 0xa5
'mii', # 0xa6
'mo', # 0xa7
'moo', # 0xa8
'moo', # 0xa9
'ma', # 0xaa
'maa', # 0xab
'mwe', # 0xac
'mwe', # 0xad
'mwi', # 0xae
'mwi', # 0xaf
'mwii', # 0xb0
'mwii', # 0xb1
'mwo', # 0xb2
'mwo', # 0xb3
'mwoo', # 0xb4
'mwoo', # 0xb5
'mwa', # 0xb6
'mwa', # 0xb7
'mwaa', # 0xb8
'mwaa', # 0xb9
'mwaa', # 0xba
'm', # 0xbb
'm', # 0xbc
'mh', # 0xbd
'm', # 0xbe
'm', # 0xbf
'ne', # 0xc0
'naai', # 0xc1
'ni', # 0xc2
'nii', # 0xc3
'no', # 0xc4
'noo', # 0xc5
'noo', # 0xc6
'na', # 0xc7
'naa', # 0xc8
'nwe', # 0xc9
'nwe', # 0xca
'nwa', # 0xcb
'nwa', # 0xcc
'nwaa', # 0xcd
'nwaa', # 0xce
'nwaa', # 0xcf
'n', # 0xd0
'ng', # 0xd1
'nh', # 0xd2
'le', # 0xd3
'laai', # 0xd4
'li', # 0xd5
'lii', # 0xd6
'lo', # 0xd7
'loo', # 0xd8
'loo', # 0xd9
'la', # 0xda
'laa', # 0xdb
'lwe', # 0xdc
'lwe', # 0xdd
'lwi', # 0xde
'lwi', # 0xdf
'lwii', # 0xe0
'lwii', # 0xe1
'lwo', # 0xe2
'lwo', # 0xe3
'lwoo', # 0xe4
'lwoo', # 0xe5
'lwa', # 0xe6
'lwa', # 0xe7
'lwaa', # 0xe8
'lwaa', # 0xe9
'l', # 0xea
'l', # 0xeb
'l', # 0xec
'se', # 0xed
'saai', # 0xee
'si', # 0xef
'sii', # 0xf0
'so', # 0xf1
'soo', # 0xf2
'soo', # 0xf3
'sa', # 0xf4
'saa', # 0xf5
'swe', # 0xf6
'swe', # 0xf7
'swi', # 0xf8
'swi', # 0xf9
'swii', # 0xfa
'swii', # 0xfb
'swo', # 0xfc
'swo', # 0xfd
'swoo', # 0xfe
'swoo', # 0xff
)

258
libs/unidecode/x015.py Normal file
View file

@ -0,0 +1,258 @@
data = (
'swa', # 0x00
'swa', # 0x01
'swaa', # 0x02
'swaa', # 0x03
'swaa', # 0x04
's', # 0x05
's', # 0x06
'sw', # 0x07
's', # 0x08
'sk', # 0x09
'skw', # 0x0a
'sW', # 0x0b
'spwa', # 0x0c
'stwa', # 0x0d
'skwa', # 0x0e
'scwa', # 0x0f
'she', # 0x10
'shi', # 0x11
'shii', # 0x12
'sho', # 0x13
'shoo', # 0x14
'sha', # 0x15
'shaa', # 0x16
'shwe', # 0x17
'shwe', # 0x18
'shwi', # 0x19
'shwi', # 0x1a
'shwii', # 0x1b
'shwii', # 0x1c
'shwo', # 0x1d
'shwo', # 0x1e
'shwoo', # 0x1f
'shwoo', # 0x20
'shwa', # 0x21
'shwa', # 0x22
'shwaa', # 0x23
'shwaa', # 0x24
'sh', # 0x25
'ye', # 0x26
'yaai', # 0x27
'yi', # 0x28
'yii', # 0x29
'yo', # 0x2a
'yoo', # 0x2b
'yoo', # 0x2c
'ya', # 0x2d
'yaa', # 0x2e
'ywe', # 0x2f
'ywe', # 0x30
'ywi', # 0x31
'ywi', # 0x32
'ywii', # 0x33
'ywii', # 0x34
'ywo', # 0x35
'ywo', # 0x36
'ywoo', # 0x37
'ywoo', # 0x38
'ywa', # 0x39
'ywa', # 0x3a
'ywaa', # 0x3b
'ywaa', # 0x3c
'ywaa', # 0x3d
'y', # 0x3e
'y', # 0x3f
'y', # 0x40
'yi', # 0x41
're', # 0x42
're', # 0x43
'le', # 0x44
'raai', # 0x45
'ri', # 0x46
'rii', # 0x47
'ro', # 0x48
'roo', # 0x49
'lo', # 0x4a
'ra', # 0x4b
'raa', # 0x4c
'la', # 0x4d
'rwaa', # 0x4e
'rwaa', # 0x4f
'r', # 0x50
'r', # 0x51
'r', # 0x52
'fe', # 0x53
'faai', # 0x54
'fi', # 0x55
'fii', # 0x56
'fo', # 0x57
'foo', # 0x58
'fa', # 0x59
'faa', # 0x5a
'fwaa', # 0x5b
'fwaa', # 0x5c
'f', # 0x5d
'the', # 0x5e
'the', # 0x5f
'thi', # 0x60
'thi', # 0x61
'thii', # 0x62
'thii', # 0x63
'tho', # 0x64
'thoo', # 0x65
'tha', # 0x66
'thaa', # 0x67
'thwaa', # 0x68
'thwaa', # 0x69
'th', # 0x6a
'tthe', # 0x6b
'tthi', # 0x6c
'ttho', # 0x6d
'ttha', # 0x6e
'tth', # 0x6f
'tye', # 0x70
'tyi', # 0x71
'tyo', # 0x72
'tya', # 0x73
'he', # 0x74
'hi', # 0x75
'hii', # 0x76
'ho', # 0x77
'hoo', # 0x78
'ha', # 0x79
'haa', # 0x7a
'h', # 0x7b
'h', # 0x7c
'hk', # 0x7d
'qaai', # 0x7e
'qi', # 0x7f
'qii', # 0x80
'qo', # 0x81
'qoo', # 0x82
'qa', # 0x83
'qaa', # 0x84
'q', # 0x85
'tlhe', # 0x86
'tlhi', # 0x87
'tlho', # 0x88
'tlha', # 0x89
're', # 0x8a
'ri', # 0x8b
'ro', # 0x8c
'ra', # 0x8d
'ngaai', # 0x8e
'ngi', # 0x8f
'ngii', # 0x90
'ngo', # 0x91
'ngoo', # 0x92
'nga', # 0x93
'ngaa', # 0x94
'ng', # 0x95
'nng', # 0x96
'she', # 0x97
'shi', # 0x98
'sho', # 0x99
'sha', # 0x9a
'the', # 0x9b
'thi', # 0x9c
'tho', # 0x9d
'tha', # 0x9e
'th', # 0x9f
'lhi', # 0xa0
'lhii', # 0xa1
'lho', # 0xa2
'lhoo', # 0xa3
'lha', # 0xa4
'lhaa', # 0xa5
'lh', # 0xa6
'the', # 0xa7
'thi', # 0xa8
'thii', # 0xa9
'tho', # 0xaa
'thoo', # 0xab
'tha', # 0xac
'thaa', # 0xad
'th', # 0xae
'b', # 0xaf
'e', # 0xb0
'i', # 0xb1
'o', # 0xb2
'a', # 0xb3
'we', # 0xb4
'wi', # 0xb5
'wo', # 0xb6
'wa', # 0xb7
'ne', # 0xb8
'ni', # 0xb9
'no', # 0xba
'na', # 0xbb
'ke', # 0xbc
'ki', # 0xbd
'ko', # 0xbe
'ka', # 0xbf
'he', # 0xc0
'hi', # 0xc1
'ho', # 0xc2
'ha', # 0xc3
'ghu', # 0xc4
'gho', # 0xc5
'ghe', # 0xc6
'ghee', # 0xc7
'ghi', # 0xc8
'gha', # 0xc9
'ru', # 0xca
'ro', # 0xcb
're', # 0xcc
'ree', # 0xcd
'ri', # 0xce
'ra', # 0xcf
'wu', # 0xd0
'wo', # 0xd1
'we', # 0xd2
'wee', # 0xd3
'wi', # 0xd4
'wa', # 0xd5
'hwu', # 0xd6
'hwo', # 0xd7
'hwe', # 0xd8
'hwee', # 0xd9
'hwi', # 0xda
'hwa', # 0xdb
'thu', # 0xdc
'tho', # 0xdd
'the', # 0xde
'thee', # 0xdf
'thi', # 0xe0
'tha', # 0xe1
'ttu', # 0xe2
'tto', # 0xe3
'tte', # 0xe4
'ttee', # 0xe5
'tti', # 0xe6
'tta', # 0xe7
'pu', # 0xe8
'po', # 0xe9
'pe', # 0xea
'pee', # 0xeb
'pi', # 0xec
'pa', # 0xed
'p', # 0xee
'gu', # 0xef
'go', # 0xf0
'ge', # 0xf1
'gee', # 0xf2
'gi', # 0xf3
'ga', # 0xf4
'khu', # 0xf5
'kho', # 0xf6
'khe', # 0xf7
'khee', # 0xf8
'khi', # 0xf9
'kha', # 0xfa
'kku', # 0xfb
'kko', # 0xfc
'kke', # 0xfd
'kkee', # 0xfe
'kki', # 0xff
)

257
libs/unidecode/x016.py Normal file
View file

@ -0,0 +1,257 @@
data = (
'kka', # 0x00
'kk', # 0x01
'nu', # 0x02
'no', # 0x03
'ne', # 0x04
'nee', # 0x05
'ni', # 0x06
'na', # 0x07
'mu', # 0x08
'mo', # 0x09
'me', # 0x0a
'mee', # 0x0b
'mi', # 0x0c
'ma', # 0x0d
'yu', # 0x0e
'yo', # 0x0f
'ye', # 0x10
'yee', # 0x11
'yi', # 0x12
'ya', # 0x13
'ju', # 0x14
'ju', # 0x15
'jo', # 0x16
'je', # 0x17
'jee', # 0x18
'ji', # 0x19
'ji', # 0x1a
'ja', # 0x1b
'jju', # 0x1c
'jjo', # 0x1d
'jje', # 0x1e
'jjee', # 0x1f
'jji', # 0x20
'jja', # 0x21
'lu', # 0x22
'lo', # 0x23
'le', # 0x24
'lee', # 0x25
'li', # 0x26
'la', # 0x27
'dlu', # 0x28
'dlo', # 0x29
'dle', # 0x2a
'dlee', # 0x2b
'dli', # 0x2c
'dla', # 0x2d
'lhu', # 0x2e
'lho', # 0x2f
'lhe', # 0x30
'lhee', # 0x31
'lhi', # 0x32
'lha', # 0x33
'tlhu', # 0x34
'tlho', # 0x35
'tlhe', # 0x36
'tlhee', # 0x37
'tlhi', # 0x38
'tlha', # 0x39
'tlu', # 0x3a
'tlo', # 0x3b
'tle', # 0x3c
'tlee', # 0x3d
'tli', # 0x3e
'tla', # 0x3f
'zu', # 0x40
'zo', # 0x41
'ze', # 0x42
'zee', # 0x43
'zi', # 0x44
'za', # 0x45
'z', # 0x46
'z', # 0x47
'dzu', # 0x48
'dzo', # 0x49
'dze', # 0x4a
'dzee', # 0x4b
'dzi', # 0x4c
'dza', # 0x4d
'su', # 0x4e
'so', # 0x4f
'se', # 0x50
'see', # 0x51
'si', # 0x52
'sa', # 0x53
'shu', # 0x54
'sho', # 0x55
'she', # 0x56
'shee', # 0x57
'shi', # 0x58
'sha', # 0x59
'sh', # 0x5a
'tsu', # 0x5b
'tso', # 0x5c
'tse', # 0x5d
'tsee', # 0x5e
'tsi', # 0x5f
'tsa', # 0x60
'chu', # 0x61
'cho', # 0x62
'che', # 0x63
'chee', # 0x64
'chi', # 0x65
'cha', # 0x66
'ttsu', # 0x67
'ttso', # 0x68
'ttse', # 0x69
'ttsee', # 0x6a
'ttsi', # 0x6b
'ttsa', # 0x6c
'X', # 0x6d
'.', # 0x6e
'qai', # 0x6f
'ngai', # 0x70
'nngi', # 0x71
'nngii', # 0x72
'nngo', # 0x73
'nngoo', # 0x74
'nnga', # 0x75
'nngaa', # 0x76
'[?]', # 0x77
'[?]', # 0x78
'[?]', # 0x79
'[?]', # 0x7a
'[?]', # 0x7b
'[?]', # 0x7c
'[?]', # 0x7d
'[?]', # 0x7e
'[?]', # 0x7f
' ', # 0x80
'b', # 0x81
'l', # 0x82
'f', # 0x83
's', # 0x84
'n', # 0x85
'h', # 0x86
'd', # 0x87
't', # 0x88
'c', # 0x89
'q', # 0x8a
'm', # 0x8b
'g', # 0x8c
'ng', # 0x8d
'z', # 0x8e
'r', # 0x8f
'a', # 0x90
'o', # 0x91
'u', # 0x92
'e', # 0x93
'i', # 0x94
'ch', # 0x95
'th', # 0x96
'ph', # 0x97
'p', # 0x98
'x', # 0x99
'p', # 0x9a
'<', # 0x9b
'>', # 0x9c
'[?]', # 0x9d
'[?]', # 0x9e
'[?]', # 0x9f
'f', # 0xa0
'v', # 0xa1
'u', # 0xa2
'yr', # 0xa3
'y', # 0xa4
'w', # 0xa5
'th', # 0xa6
'th', # 0xa7
'a', # 0xa8
'o', # 0xa9
'ac', # 0xaa
'ae', # 0xab
'o', # 0xac
'o', # 0xad
'o', # 0xae
'oe', # 0xaf
'on', # 0xb0
'r', # 0xb1
'k', # 0xb2
'c', # 0xb3
'k', # 0xb4
'g', # 0xb5
'ng', # 0xb6
'g', # 0xb7
'g', # 0xb8
'w', # 0xb9
'h', # 0xba
'h', # 0xbb
'h', # 0xbc
'h', # 0xbd
'n', # 0xbe
'n', # 0xbf
'n', # 0xc0
'i', # 0xc1
'e', # 0xc2
'j', # 0xc3
'g', # 0xc4
'ae', # 0xc5
'a', # 0xc6
'eo', # 0xc7
'p', # 0xc8
'z', # 0xc9
's', # 0xca
's', # 0xcb
's', # 0xcc
'c', # 0xcd
'z', # 0xce
't', # 0xcf
't', # 0xd0
'd', # 0xd1
'b', # 0xd2
'b', # 0xd3
'p', # 0xd4
'p', # 0xd5
'e', # 0xd6
'm', # 0xd7
'm', # 0xd8
'm', # 0xd9
'l', # 0xda
'l', # 0xdb
'ng', # 0xdc
'ng', # 0xdd
'd', # 0xde
'o', # 0xdf
'ear', # 0xe0
'ior', # 0xe1
'qu', # 0xe2
'qu', # 0xe3
'qu', # 0xe4
's', # 0xe5
'yr', # 0xe6
'yr', # 0xe7
'yr', # 0xe8
'q', # 0xe9
'x', # 0xea
'.', # 0xeb
':', # 0xec
'+', # 0xed
'17', # 0xee
'18', # 0xef
'19', # 0xf0
'[?]', # 0xf1
'[?]', # 0xf2
'[?]', # 0xf3
'[?]', # 0xf4
'[?]', # 0xf5
'[?]', # 0xf6
'[?]', # 0xf7
'[?]', # 0xf8
'[?]', # 0xf9
'[?]', # 0xfa
'[?]', # 0xfb
'[?]', # 0xfc
'[?]', # 0xfd
'[?]', # 0xfe
)

257
libs/unidecode/x017.py Normal file
View file

@ -0,0 +1,257 @@
data = (
'[?]', # 0x00
'[?]', # 0x01
'[?]', # 0x02
'[?]', # 0x03
'[?]', # 0x04
'[?]', # 0x05
'[?]', # 0x06
'[?]', # 0x07
'[?]', # 0x08
'[?]', # 0x09
'[?]', # 0x0a
'[?]', # 0x0b
'[?]', # 0x0c
'[?]', # 0x0d
'[?]', # 0x0e
'[?]', # 0x0f
'[?]', # 0x10
'[?]', # 0x11
'[?]', # 0x12
'[?]', # 0x13
'[?]', # 0x14
'[?]', # 0x15
'[?]', # 0x16
'[?]', # 0x17
'[?]', # 0x18
'[?]', # 0x19
'[?]', # 0x1a
'[?]', # 0x1b
'[?]', # 0x1c
'[?]', # 0x1d
'[?]', # 0x1e
'[?]', # 0x1f
'[?]', # 0x20
'[?]', # 0x21
'[?]', # 0x22
'[?]', # 0x23
'[?]', # 0x24
'[?]', # 0x25
'[?]', # 0x26
'[?]', # 0x27
'[?]', # 0x28
'[?]', # 0x29
'[?]', # 0x2a
'[?]', # 0x2b
'[?]', # 0x2c
'[?]', # 0x2d
'[?]', # 0x2e
'[?]', # 0x2f
'[?]', # 0x30
'[?]', # 0x31
'[?]', # 0x32
'[?]', # 0x33
'[?]', # 0x34
'[?]', # 0x35
'[?]', # 0x36
'[?]', # 0x37
'[?]', # 0x38
'[?]', # 0x39
'[?]', # 0x3a
'[?]', # 0x3b
'[?]', # 0x3c
'[?]', # 0x3d
'[?]', # 0x3e
'[?]', # 0x3f
'[?]', # 0x40
'[?]', # 0x41
'[?]', # 0x42
'[?]', # 0x43
'[?]', # 0x44
'[?]', # 0x45
'[?]', # 0x46
'[?]', # 0x47
'[?]', # 0x48
'[?]', # 0x49
'[?]', # 0x4a
'[?]', # 0x4b
'[?]', # 0x4c
'[?]', # 0x4d
'[?]', # 0x4e
'[?]', # 0x4f
'[?]', # 0x50
'[?]', # 0x51
'[?]', # 0x52
'[?]', # 0x53
'[?]', # 0x54
'[?]', # 0x55
'[?]', # 0x56
'[?]', # 0x57
'[?]', # 0x58
'[?]', # 0x59
'[?]', # 0x5a
'[?]', # 0x5b
'[?]', # 0x5c
'[?]', # 0x5d
'[?]', # 0x5e
'[?]', # 0x5f
'[?]', # 0x60
'[?]', # 0x61
'[?]', # 0x62
'[?]', # 0x63
'[?]', # 0x64
'[?]', # 0x65
'[?]', # 0x66
'[?]', # 0x67
'[?]', # 0x68
'[?]', # 0x69
'[?]', # 0x6a
'[?]', # 0x6b
'[?]', # 0x6c
'[?]', # 0x6d
'[?]', # 0x6e
'[?]', # 0x6f
'[?]', # 0x70
'[?]', # 0x71
'[?]', # 0x72
'[?]', # 0x73
'[?]', # 0x74
'[?]', # 0x75
'[?]', # 0x76
'[?]', # 0x77
'[?]', # 0x78
'[?]', # 0x79
'[?]', # 0x7a
'[?]', # 0x7b
'[?]', # 0x7c
'[?]', # 0x7d
'[?]', # 0x7e
'[?]', # 0x7f
'k', # 0x80
'kh', # 0x81
'g', # 0x82
'gh', # 0x83
'ng', # 0x84
'c', # 0x85
'ch', # 0x86
'j', # 0x87
'jh', # 0x88
'ny', # 0x89
't', # 0x8a
'tth', # 0x8b
'd', # 0x8c
'ddh', # 0x8d
'nn', # 0x8e
't', # 0x8f
'th', # 0x90
'd', # 0x91
'dh', # 0x92
'n', # 0x93
'p', # 0x94
'ph', # 0x95
'b', # 0x96
'bh', # 0x97
'm', # 0x98
'y', # 0x99
'r', # 0x9a
'l', # 0x9b
'v', # 0x9c
'sh', # 0x9d
'ss', # 0x9e
's', # 0x9f
'h', # 0xa0
'l', # 0xa1
'q', # 0xa2
'a', # 0xa3
'aa', # 0xa4
'i', # 0xa5
'ii', # 0xa6
'u', # 0xa7
'uk', # 0xa8
'uu', # 0xa9
'uuv', # 0xaa
'ry', # 0xab
'ryy', # 0xac
'ly', # 0xad
'lyy', # 0xae
'e', # 0xaf
'ai', # 0xb0
'oo', # 0xb1
'oo', # 0xb2
'au', # 0xb3
'a', # 0xb4
'aa', # 0xb5
'aa', # 0xb6
'i', # 0xb7
'ii', # 0xb8
'y', # 0xb9
'yy', # 0xba
'u', # 0xbb
'uu', # 0xbc
'ua', # 0xbd
'oe', # 0xbe
'ya', # 0xbf
'ie', # 0xc0
'e', # 0xc1
'ae', # 0xc2
'ai', # 0xc3
'oo', # 0xc4
'au', # 0xc5
'M', # 0xc6
'H', # 0xc7
'a`', # 0xc8
'', # 0xc9
'', # 0xca
'', # 0xcb
'r', # 0xcc
'', # 0xcd
'!', # 0xce
'', # 0xcf
'', # 0xd0
'', # 0xd1
'', # 0xd2
'', # 0xd3
'.', # 0xd4
' // ', # 0xd5
':', # 0xd6
'+', # 0xd7
'++', # 0xd8
' * ', # 0xd9
' /// ', # 0xda
'KR', # 0xdb
'\'', # 0xdc
'[?]', # 0xdd
'[?]', # 0xde
'[?]', # 0xdf
'0', # 0xe0
'1', # 0xe1
'2', # 0xe2
'3', # 0xe3
'4', # 0xe4
'5', # 0xe5
'6', # 0xe6
'7', # 0xe7
'8', # 0xe8
'9', # 0xe9
'[?]', # 0xea
'[?]', # 0xeb
'[?]', # 0xec
'[?]', # 0xed
'[?]', # 0xee
'[?]', # 0xef
'[?]', # 0xf0
'[?]', # 0xf1
'[?]', # 0xf2
'[?]', # 0xf3
'[?]', # 0xf4
'[?]', # 0xf5
'[?]', # 0xf6
'[?]', # 0xf7
'[?]', # 0xf8
'[?]', # 0xf9
'[?]', # 0xfa
'[?]', # 0xfb
'[?]', # 0xfc
'[?]', # 0xfd
'[?]', # 0xfe
)

257
libs/unidecode/x018.py Normal file
View file

@ -0,0 +1,257 @@
data = (
' @ ', # 0x00
' ... ', # 0x01
', ', # 0x02
'. ', # 0x03
': ', # 0x04
' // ', # 0x05
'', # 0x06
'-', # 0x07
', ', # 0x08
'. ', # 0x09
'', # 0x0a
'', # 0x0b
'', # 0x0c
'', # 0x0d
'', # 0x0e
'[?]', # 0x0f
'0', # 0x10
'1', # 0x11
'2', # 0x12
'3', # 0x13
'4', # 0x14
'5', # 0x15
'6', # 0x16
'7', # 0x17
'8', # 0x18
'9', # 0x19
'[?]', # 0x1a
'[?]', # 0x1b
'[?]', # 0x1c
'[?]', # 0x1d
'[?]', # 0x1e
'[?]', # 0x1f
'a', # 0x20
'e', # 0x21
'i', # 0x22
'o', # 0x23
'u', # 0x24
'O', # 0x25
'U', # 0x26
'ee', # 0x27
'n', # 0x28
'ng', # 0x29
'b', # 0x2a
'p', # 0x2b
'q', # 0x2c
'g', # 0x2d
'm', # 0x2e
'l', # 0x2f
's', # 0x30
'sh', # 0x31
't', # 0x32
'd', # 0x33
'ch', # 0x34
'j', # 0x35
'y', # 0x36
'r', # 0x37
'w', # 0x38
'f', # 0x39
'k', # 0x3a
'kha', # 0x3b
'ts', # 0x3c
'z', # 0x3d
'h', # 0x3e
'zr', # 0x3f
'lh', # 0x40
'zh', # 0x41
'ch', # 0x42
'-', # 0x43
'e', # 0x44
'i', # 0x45
'o', # 0x46
'u', # 0x47
'O', # 0x48
'U', # 0x49
'ng', # 0x4a
'b', # 0x4b
'p', # 0x4c
'q', # 0x4d
'g', # 0x4e
'm', # 0x4f
't', # 0x50
'd', # 0x51
'ch', # 0x52
'j', # 0x53
'ts', # 0x54
'y', # 0x55
'w', # 0x56
'k', # 0x57
'g', # 0x58
'h', # 0x59
'jy', # 0x5a
'ny', # 0x5b
'dz', # 0x5c
'e', # 0x5d
'i', # 0x5e
'iy', # 0x5f
'U', # 0x60
'u', # 0x61
'ng', # 0x62
'k', # 0x63
'g', # 0x64
'h', # 0x65
'p', # 0x66
'sh', # 0x67
't', # 0x68
'd', # 0x69
'j', # 0x6a
'f', # 0x6b
'g', # 0x6c
'h', # 0x6d
'ts', # 0x6e
'z', # 0x6f
'r', # 0x70
'ch', # 0x71
'zh', # 0x72
'i', # 0x73
'k', # 0x74
'r', # 0x75
'f', # 0x76
'zh', # 0x77
'[?]', # 0x78
'[?]', # 0x79
'[?]', # 0x7a
'[?]', # 0x7b
'[?]', # 0x7c
'[?]', # 0x7d
'[?]', # 0x7e
'[?]', # 0x7f
'[?]', # 0x80
'H', # 0x81
'X', # 0x82
'W', # 0x83
'M', # 0x84
' 3 ', # 0x85
' 333 ', # 0x86
'a', # 0x87
'i', # 0x88
'k', # 0x89
'ng', # 0x8a
'c', # 0x8b
'tt', # 0x8c
'tth', # 0x8d
'dd', # 0x8e
'nn', # 0x8f
't', # 0x90
'd', # 0x91
'p', # 0x92
'ph', # 0x93
'ss', # 0x94
'zh', # 0x95
'z', # 0x96
'a', # 0x97
't', # 0x98
'zh', # 0x99
'gh', # 0x9a
'ng', # 0x9b
'c', # 0x9c
'jh', # 0x9d
'tta', # 0x9e
'ddh', # 0x9f
't', # 0xa0
'dh', # 0xa1
'ss', # 0xa2
'cy', # 0xa3
'zh', # 0xa4
'z', # 0xa5
'u', # 0xa6
'y', # 0xa7
'bh', # 0xa8
'\'', # 0xa9
'[?]', # 0xaa
'[?]', # 0xab
'[?]', # 0xac
'[?]', # 0xad
'[?]', # 0xae
'[?]', # 0xaf
'[?]', # 0xb0
'[?]', # 0xb1
'[?]', # 0xb2
'[?]', # 0xb3
'[?]', # 0xb4
'[?]', # 0xb5
'[?]', # 0xb6
'[?]', # 0xb7
'[?]', # 0xb8
'[?]', # 0xb9
'[?]', # 0xba
'[?]', # 0xbb
'[?]', # 0xbc
'[?]', # 0xbd
'[?]', # 0xbe
'[?]', # 0xbf
'[?]', # 0xc0
'[?]', # 0xc1
'[?]', # 0xc2
'[?]', # 0xc3
'[?]', # 0xc4
'[?]', # 0xc5
'[?]', # 0xc6
'[?]', # 0xc7
'[?]', # 0xc8
'[?]', # 0xc9
'[?]', # 0xca
'[?]', # 0xcb
'[?]', # 0xcc
'[?]', # 0xcd
'[?]', # 0xce
'[?]', # 0xcf
'[?]', # 0xd0
'[?]', # 0xd1
'[?]', # 0xd2
'[?]', # 0xd3
'[?]', # 0xd4
'[?]', # 0xd5
'[?]', # 0xd6
'[?]', # 0xd7
'[?]', # 0xd8
'[?]', # 0xd9
'[?]', # 0xda
'[?]', # 0xdb
'[?]', # 0xdc
'[?]', # 0xdd
'[?]', # 0xde
'[?]', # 0xdf
'[?]', # 0xe0
'[?]', # 0xe1
'[?]', # 0xe2
'[?]', # 0xe3
'[?]', # 0xe4
'[?]', # 0xe5
'[?]', # 0xe6
'[?]', # 0xe7
'[?]', # 0xe8
'[?]', # 0xe9
'[?]', # 0xea
'[?]', # 0xeb
'[?]', # 0xec
'[?]', # 0xed
'[?]', # 0xee
'[?]', # 0xef
'[?]', # 0xf0
'[?]', # 0xf1
'[?]', # 0xf2
'[?]', # 0xf3
'[?]', # 0xf4
'[?]', # 0xf5
'[?]', # 0xf6
'[?]', # 0xf7
'[?]', # 0xf8
'[?]', # 0xf9
'[?]', # 0xfa
'[?]', # 0xfb
'[?]', # 0xfc
'[?]', # 0xfd
'[?]', # 0xfe
)

257
libs/unidecode/x01d.py Normal file
View file

@ -0,0 +1,257 @@
data = (
'', # 0x00
'', # 0x01
'', # 0x02
'', # 0x03
'', # 0x04
'', # 0x05
'', # 0x06
'', # 0x07
'', # 0x08
'', # 0x09
'', # 0x0a
'', # 0x0b
'', # 0x0c
'', # 0x0d
'', # 0x0e
'', # 0x0f
'', # 0x10
'', # 0x11
'', # 0x12
'', # 0x13
'', # 0x14
'', # 0x15
'', # 0x16
'', # 0x17
'', # 0x18
'', # 0x19
'', # 0x1a
'', # 0x1b
'', # 0x1c
'', # 0x1d
'', # 0x1e
'', # 0x1f
'', # 0x20
'', # 0x21
'', # 0x22
'', # 0x23
'', # 0x24
'', # 0x25
'', # 0x26
'', # 0x27
'', # 0x28
'', # 0x29
'', # 0x2a
'', # 0x2b
'', # 0x2c
'', # 0x2d
'', # 0x2e
'', # 0x2f
'', # 0x30
'', # 0x31
'', # 0x32
'', # 0x33
'', # 0x34
'', # 0x35
'', # 0x36
'', # 0x37
'', # 0x38
'', # 0x39
'', # 0x3a
'', # 0x3b
'', # 0x3c
'', # 0x3d
'', # 0x3e
'', # 0x3f
'', # 0x40
'', # 0x41
'', # 0x42
'', # 0x43
'', # 0x44
'', # 0x45
'', # 0x46
'', # 0x47
'', # 0x48
'', # 0x49
'', # 0x4a
'', # 0x4b
'', # 0x4c
'', # 0x4d
'', # 0x4e
'', # 0x4f
'', # 0x50
'', # 0x51
'', # 0x52
'', # 0x53
'', # 0x54
'', # 0x55
'', # 0x56
'', # 0x57
'', # 0x58
'', # 0x59
'', # 0x5a
'', # 0x5b
'', # 0x5c
'', # 0x5d
'', # 0x5e
'', # 0x5f
'', # 0x60
'', # 0x61
'', # 0x62
'', # 0x63
'', # 0x64
'', # 0x65
'', # 0x66
'', # 0x67
'', # 0x68
'', # 0x69
'', # 0x6a
'', # 0x6b
'b', # 0x6c
'd', # 0x6d
'f', # 0x6e
'm', # 0x6f
'n', # 0x70
'p', # 0x71
'r', # 0x72
'r', # 0x73
's', # 0x74
't', # 0x75
'z', # 0x76
'g', # 0x77
'', # 0x78
'', # 0x79
'', # 0x7a
'', # 0x7b
'', # 0x7c
'p', # 0x7d
'', # 0x7e
'', # 0x7f
'b', # 0x80
'd', # 0x81
'f', # 0x82
'g', # 0x83
'k', # 0x84
'l', # 0x85
'm', # 0x86
'n', # 0x87
'p', # 0x88
'r', # 0x89
's', # 0x8a
'', # 0x8b
'v', # 0x8c
'x', # 0x8d
'z', # 0x8e
'', # 0x8f
'', # 0x90
'', # 0x91
'', # 0x92
'', # 0x93
'', # 0x94
'', # 0x95
'', # 0x96
'', # 0x97
'', # 0x98
'', # 0x99
'', # 0x9a
'', # 0x9b
'', # 0x9c
'', # 0x9d
'', # 0x9e
'', # 0x9f
'', # 0xa0
'', # 0xa1
'', # 0xa2
'', # 0xa3
'', # 0xa4
'', # 0xa5
'', # 0xa6
'', # 0xa7
'', # 0xa8
'', # 0xa9
'', # 0xaa
'', # 0xab
'', # 0xac
'', # 0xad
'', # 0xae
'', # 0xaf
'', # 0xb0
'', # 0xb1
'', # 0xb2
'', # 0xb3
'', # 0xb4
'', # 0xb5
'', # 0xb6
'', # 0xb7
'', # 0xb8
'', # 0xb9
'', # 0xba
'', # 0xbb
'', # 0xbc
'', # 0xbd
'', # 0xbe
'', # 0xbf
'', # 0xc0
'', # 0xc1
'', # 0xc2
'', # 0xc3
'', # 0xc4
'', # 0xc5
'', # 0xc6
'', # 0xc7
'', # 0xc8
'', # 0xc9
'', # 0xca
'', # 0xcb
'', # 0xcc
'', # 0xcd
'', # 0xce
'', # 0xcf
'', # 0xd0
'', # 0xd1
'', # 0xd2
'', # 0xd3
'', # 0xd4
'', # 0xd5
'', # 0xd6
'', # 0xd7
'', # 0xd8
'', # 0xd9
'', # 0xda
'', # 0xdb
'', # 0xdc
'', # 0xdd
'', # 0xde
'', # 0xdf
'', # 0xe0
'', # 0xe1
'', # 0xe2
'', # 0xe3
'', # 0xe4
'', # 0xe5
'', # 0xe6
'', # 0xe7
'', # 0xe8
'', # 0xe9
'', # 0xea
'', # 0xeb
'', # 0xec
'', # 0xed
'', # 0xee
'', # 0xef
'', # 0xf0
'', # 0xf1
'', # 0xf2
'', # 0xf3
'', # 0xf4
'', # 0xf5
'', # 0xf6
'', # 0xf7
'', # 0xf8
'', # 0xf9
'', # 0xfa
'', # 0xfb
'', # 0xfc
'', # 0xfd
'', # 0xfe
)

257
libs/unidecode/x01e.py Normal file
View file

@ -0,0 +1,257 @@
data = (
'A', # 0x00
'a', # 0x01
'B', # 0x02
'b', # 0x03
'B', # 0x04
'b', # 0x05
'B', # 0x06
'b', # 0x07
'C', # 0x08
'c', # 0x09
'D', # 0x0a
'd', # 0x0b
'D', # 0x0c
'd', # 0x0d
'D', # 0x0e
'd', # 0x0f
'D', # 0x10
'd', # 0x11
'D', # 0x12
'd', # 0x13
'E', # 0x14
'e', # 0x15
'E', # 0x16
'e', # 0x17
'E', # 0x18
'e', # 0x19
'E', # 0x1a
'e', # 0x1b
'E', # 0x1c
'e', # 0x1d
'F', # 0x1e
'f', # 0x1f
'G', # 0x20
'g', # 0x21
'H', # 0x22
'h', # 0x23
'H', # 0x24
'h', # 0x25
'H', # 0x26
'h', # 0x27
'H', # 0x28
'h', # 0x29
'H', # 0x2a
'h', # 0x2b
'I', # 0x2c
'i', # 0x2d
'I', # 0x2e
'i', # 0x2f
'K', # 0x30
'k', # 0x31
'K', # 0x32
'k', # 0x33
'K', # 0x34
'k', # 0x35
'L', # 0x36
'l', # 0x37
'L', # 0x38
'l', # 0x39
'L', # 0x3a
'l', # 0x3b
'L', # 0x3c
'l', # 0x3d
'M', # 0x3e
'm', # 0x3f
'M', # 0x40
'm', # 0x41
'M', # 0x42
'm', # 0x43
'N', # 0x44
'n', # 0x45
'N', # 0x46
'n', # 0x47
'N', # 0x48
'n', # 0x49
'N', # 0x4a
'n', # 0x4b
'O', # 0x4c
'o', # 0x4d
'O', # 0x4e
'o', # 0x4f
'O', # 0x50
'o', # 0x51
'O', # 0x52
'o', # 0x53
'P', # 0x54
'p', # 0x55
'P', # 0x56
'p', # 0x57
'R', # 0x58
'r', # 0x59
'R', # 0x5a
'r', # 0x5b
'R', # 0x5c
'r', # 0x5d
'R', # 0x5e
'r', # 0x5f
'S', # 0x60
's', # 0x61
'S', # 0x62
's', # 0x63
'S', # 0x64
's', # 0x65
'S', # 0x66
's', # 0x67
'S', # 0x68
's', # 0x69
'T', # 0x6a
't', # 0x6b
'T', # 0x6c
't', # 0x6d
'T', # 0x6e
't', # 0x6f
'T', # 0x70
't', # 0x71
'U', # 0x72
'u', # 0x73
'U', # 0x74
'u', # 0x75
'U', # 0x76
'u', # 0x77
'U', # 0x78
'u', # 0x79
'U', # 0x7a
'u', # 0x7b
'V', # 0x7c
'v', # 0x7d
'V', # 0x7e
'v', # 0x7f
'W', # 0x80
'w', # 0x81
'W', # 0x82
'w', # 0x83
'W', # 0x84
'w', # 0x85
'W', # 0x86
'w', # 0x87
'W', # 0x88
'w', # 0x89
'X', # 0x8a
'x', # 0x8b
'X', # 0x8c
'x', # 0x8d
'Y', # 0x8e
'y', # 0x8f
'Z', # 0x90
'z', # 0x91
'Z', # 0x92
'z', # 0x93
'Z', # 0x94
'z', # 0x95
'h', # 0x96
't', # 0x97
'w', # 0x98
'y', # 0x99
'a', # 0x9a
'S', # 0x9b
'[?]', # 0x9c
'[?]', # 0x9d
'Ss', # 0x9e
'[?]', # 0x9f
'A', # 0xa0
'a', # 0xa1
'A', # 0xa2
'a', # 0xa3
'A', # 0xa4
'a', # 0xa5
'A', # 0xa6
'a', # 0xa7
'A', # 0xa8
'a', # 0xa9
'A', # 0xaa
'a', # 0xab
'A', # 0xac
'a', # 0xad
'A', # 0xae
'a', # 0xaf
'A', # 0xb0
'a', # 0xb1
'A', # 0xb2
'a', # 0xb3
'A', # 0xb4
'a', # 0xb5
'A', # 0xb6
'a', # 0xb7
'E', # 0xb8
'e', # 0xb9
'E', # 0xba
'e', # 0xbb
'E', # 0xbc
'e', # 0xbd
'E', # 0xbe
'e', # 0xbf
'E', # 0xc0
'e', # 0xc1
'E', # 0xc2
'e', # 0xc3
'E', # 0xc4
'e', # 0xc5
'E', # 0xc6
'e', # 0xc7
'I', # 0xc8
'i', # 0xc9
'I', # 0xca
'i', # 0xcb
'O', # 0xcc
'o', # 0xcd
'O', # 0xce
'o', # 0xcf
'O', # 0xd0
'o', # 0xd1
'O', # 0xd2
'o', # 0xd3
'O', # 0xd4
'o', # 0xd5
'O', # 0xd6
'o', # 0xd7
'O', # 0xd8
'o', # 0xd9
'O', # 0xda
'o', # 0xdb
'O', # 0xdc
'o', # 0xdd
'O', # 0xde
'o', # 0xdf
'O', # 0xe0
'o', # 0xe1
'O', # 0xe2
'o', # 0xe3
'U', # 0xe4
'u', # 0xe5
'U', # 0xe6
'u', # 0xe7
'U', # 0xe8
'u', # 0xe9
'U', # 0xea
'u', # 0xeb
'U', # 0xec
'u', # 0xed
'U', # 0xee
'u', # 0xef
'U', # 0xf0
'u', # 0xf1
'Y', # 0xf2
'y', # 0xf3
'Y', # 0xf4
'y', # 0xf5
'Y', # 0xf6
'y', # 0xf7
'Y', # 0xf8
'y', # 0xf9
'[?]', # 0xfa
'[?]', # 0xfb
'[?]', # 0xfc
'[?]', # 0xfd
'[?]', # 0xfe
)

257
libs/unidecode/x01f.py Normal file
View file

@ -0,0 +1,257 @@
data = (
'a', # 0x00
'a', # 0x01
'a', # 0x02
'a', # 0x03
'a', # 0x04
'a', # 0x05
'a', # 0x06
'a', # 0x07
'A', # 0x08
'A', # 0x09
'A', # 0x0a
'A', # 0x0b
'A', # 0x0c
'A', # 0x0d
'A', # 0x0e
'A', # 0x0f
'e', # 0x10
'e', # 0x11
'e', # 0x12
'e', # 0x13
'e', # 0x14
'e', # 0x15
'[?]', # 0x16
'[?]', # 0x17
'E', # 0x18
'E', # 0x19
'E', # 0x1a
'E', # 0x1b
'E', # 0x1c
'E', # 0x1d
'[?]', # 0x1e
'[?]', # 0x1f
'e', # 0x20
'e', # 0x21
'e', # 0x22
'e', # 0x23
'e', # 0x24
'e', # 0x25
'e', # 0x26
'e', # 0x27
'E', # 0x28
'E', # 0x29
'E', # 0x2a
'E', # 0x2b
'E', # 0x2c
'E', # 0x2d
'E', # 0x2e
'E', # 0x2f
'i', # 0x30
'i', # 0x31
'i', # 0x32
'i', # 0x33
'i', # 0x34
'i', # 0x35
'i', # 0x36
'i', # 0x37
'I', # 0x38
'I', # 0x39
'I', # 0x3a
'I', # 0x3b
'I', # 0x3c
'I', # 0x3d
'I', # 0x3e
'I', # 0x3f
'o', # 0x40
'o', # 0x41
'o', # 0x42
'o', # 0x43
'o', # 0x44
'o', # 0x45
'[?]', # 0x46
'[?]', # 0x47
'O', # 0x48
'O', # 0x49
'O', # 0x4a
'O', # 0x4b
'O', # 0x4c
'O', # 0x4d
'[?]', # 0x4e
'[?]', # 0x4f
'u', # 0x50
'u', # 0x51
'u', # 0x52
'u', # 0x53
'u', # 0x54
'u', # 0x55
'u', # 0x56
'u', # 0x57
'[?]', # 0x58
'U', # 0x59
'[?]', # 0x5a
'U', # 0x5b
'[?]', # 0x5c
'U', # 0x5d
'[?]', # 0x5e
'U', # 0x5f
'o', # 0x60
'o', # 0x61
'o', # 0x62
'o', # 0x63
'o', # 0x64
'o', # 0x65
'o', # 0x66
'o', # 0x67
'O', # 0x68
'O', # 0x69
'O', # 0x6a
'O', # 0x6b
'O', # 0x6c
'O', # 0x6d
'O', # 0x6e
'O', # 0x6f
'a', # 0x70
'a', # 0x71
'e', # 0x72
'e', # 0x73
'e', # 0x74
'e', # 0x75
'i', # 0x76
'i', # 0x77
'o', # 0x78
'o', # 0x79
'u', # 0x7a
'u', # 0x7b
'o', # 0x7c
'o', # 0x7d
'[?]', # 0x7e
'[?]', # 0x7f
'a', # 0x80
'a', # 0x81
'a', # 0x82
'a', # 0x83
'a', # 0x84
'a', # 0x85
'a', # 0x86
'a', # 0x87
'A', # 0x88
'A', # 0x89
'A', # 0x8a
'A', # 0x8b
'A', # 0x8c
'A', # 0x8d
'A', # 0x8e
'A', # 0x8f
'e', # 0x90
'e', # 0x91
'e', # 0x92
'e', # 0x93
'e', # 0x94
'e', # 0x95
'e', # 0x96
'e', # 0x97
'E', # 0x98
'E', # 0x99
'E', # 0x9a
'E', # 0x9b
'E', # 0x9c
'E', # 0x9d
'E', # 0x9e
'E', # 0x9f
'o', # 0xa0
'o', # 0xa1
'o', # 0xa2
'o', # 0xa3
'o', # 0xa4
'o', # 0xa5
'o', # 0xa6
'o', # 0xa7
'O', # 0xa8
'O', # 0xa9
'O', # 0xaa
'O', # 0xab
'O', # 0xac
'O', # 0xad
'O', # 0xae
'O', # 0xaf
'a', # 0xb0
'a', # 0xb1
'a', # 0xb2
'a', # 0xb3
'a', # 0xb4
'[?]', # 0xb5
'a', # 0xb6
'a', # 0xb7
'A', # 0xb8
'A', # 0xb9
'A', # 0xba
'A', # 0xbb
'A', # 0xbc
'\'', # 0xbd
'i', # 0xbe
'\'', # 0xbf
'~', # 0xc0
'"~', # 0xc1
'e', # 0xc2
'e', # 0xc3
'e', # 0xc4
'[?]', # 0xc5
'e', # 0xc6
'e', # 0xc7
'E', # 0xc8
'E', # 0xc9
'E', # 0xca
'E', # 0xcb
'E', # 0xcc
'\'`', # 0xcd
'\'\'', # 0xce
'\'~', # 0xcf
'i', # 0xd0
'i', # 0xd1
'i', # 0xd2
'i', # 0xd3
'[?]', # 0xd4
'[?]', # 0xd5
'i', # 0xd6
'i', # 0xd7
'I', # 0xd8
'I', # 0xd9
'I', # 0xda
'I', # 0xdb
'[?]', # 0xdc
'`\'', # 0xdd
'`\'', # 0xde
'`~', # 0xdf
'u', # 0xe0
'u', # 0xe1
'u', # 0xe2
'u', # 0xe3
'R', # 0xe4
'R', # 0xe5
'u', # 0xe6
'u', # 0xe7
'U', # 0xe8
'U', # 0xe9
'U', # 0xea
'U', # 0xeb
'R', # 0xec
'"`', # 0xed
'"\'', # 0xee
'`', # 0xef
'[?]', # 0xf0
'[?]', # 0xf1
'o', # 0xf2
'o', # 0xf3
'o', # 0xf4
'[?]', # 0xf5
'o', # 0xf6
'o', # 0xf7
'O', # 0xf8
'O', # 0xf9
'O', # 0xfa
'O', # 0xfb
'O', # 0xfc
'\'', # 0xfd
'`', # 0xfe
)

257
libs/unidecode/x020.py Normal file
View file

@ -0,0 +1,257 @@
data = (
' ', # 0x00
' ', # 0x01
' ', # 0x02
' ', # 0x03
' ', # 0x04
' ', # 0x05
' ', # 0x06
' ', # 0x07
' ', # 0x08
' ', # 0x09
' ', # 0x0a
' ', # 0x0b
'', # 0x0c
'', # 0x0d
'', # 0x0e
'', # 0x0f
'-', # 0x10
'-', # 0x11
'-', # 0x12
'-', # 0x13
'--', # 0x14
'--', # 0x15
'||', # 0x16
'_', # 0x17
'\'', # 0x18
'\'', # 0x19
',', # 0x1a
'\'', # 0x1b
'"', # 0x1c
'"', # 0x1d
',,', # 0x1e
'"', # 0x1f
'+', # 0x20
'++', # 0x21
'*', # 0x22
'*>', # 0x23
'.', # 0x24
'..', # 0x25
'...', # 0x26
'.', # 0x27
'\x0a', # 0x28
'\x0a\x0a', # 0x29
'', # 0x2a
'', # 0x2b
'', # 0x2c
'', # 0x2d
'', # 0x2e
' ', # 0x2f
'%0', # 0x30
'%00', # 0x31
'\'', # 0x32
'\'\'', # 0x33
'\'\'\'', # 0x34
'`', # 0x35
'``', # 0x36
'```', # 0x37
'^', # 0x38
'<', # 0x39
'>', # 0x3a
'*', # 0x3b
'!!', # 0x3c
'!?', # 0x3d
'-', # 0x3e
'_', # 0x3f
'-', # 0x40
'^', # 0x41
'***', # 0x42
'--', # 0x43
'/', # 0x44
'-[', # 0x45
']-', # 0x46
'??', # 0x47
'?!', # 0x48
'!?', # 0x49
'7', # 0x4a
'PP', # 0x4b
'(]', # 0x4c
'[)', # 0x4d
'*', # 0x4e
'[?]', # 0x4f
'[?]', # 0x50
'[?]', # 0x51
'%', # 0x52
'~', # 0x53
'[?]', # 0x54
'[?]', # 0x55
'[?]', # 0x56
"''''", # 0x57
'[?]', # 0x58
'[?]', # 0x59
'[?]', # 0x5a
'[?]', # 0x5b
'[?]', # 0x5c
'[?]', # 0x5d
'[?]', # 0x5e
'[?]', # 0x5f
'', # 0x60
'[?]', # 0x61
'[?]', # 0x62
'[?]', # 0x63
'[?]', # 0x64
'[?]', # 0x65
'[?]', # 0x66
'[?]', # 0x67
'[?]', # 0x68
'[?]', # 0x69
'', # 0x6a
'', # 0x6b
'', # 0x6c
'', # 0x6d
'', # 0x6e
'', # 0x6f
'0', # 0x70
'', # 0x71
'', # 0x72
'', # 0x73
'4', # 0x74
'5', # 0x75
'6', # 0x76
'7', # 0x77
'8', # 0x78
'9', # 0x79
'+', # 0x7a
'-', # 0x7b
'=', # 0x7c
'(', # 0x7d
')', # 0x7e
'n', # 0x7f
'0', # 0x80
'1', # 0x81
'2', # 0x82
'3', # 0x83
'4', # 0x84
'5', # 0x85
'6', # 0x86
'7', # 0x87
'8', # 0x88
'9', # 0x89
'+', # 0x8a
'-', # 0x8b
'=', # 0x8c
'(', # 0x8d
')', # 0x8e
'[?]', # 0x8f
'[?]', # 0x90
'[?]', # 0x91
'[?]', # 0x92
'[?]', # 0x93
'[?]', # 0x94
'[?]', # 0x95
'[?]', # 0x96
'[?]', # 0x97
'[?]', # 0x98
'[?]', # 0x99
'[?]', # 0x9a
'[?]', # 0x9b
'[?]', # 0x9c
'[?]', # 0x9d
'[?]', # 0x9e
'[?]', # 0x9f
'ECU', # 0xa0
'CL', # 0xa1
'Cr', # 0xa2
'FF', # 0xa3
'L', # 0xa4
'mil', # 0xa5
'N', # 0xa6
'Pts', # 0xa7
'Rs', # 0xa8
'W', # 0xa9
'NS', # 0xaa
'D', # 0xab
'EU', # 0xac
'K', # 0xad
'T', # 0xae
'Dr', # 0xaf
'[?]', # 0xb0
'[?]', # 0xb1
'[?]', # 0xb2
'[?]', # 0xb3
'[?]', # 0xb4
'[?]', # 0xb5
'[?]', # 0xb6
'[?]', # 0xb7
'[?]', # 0xb8
'[?]', # 0xb9
'[?]', # 0xba
'[?]', # 0xbb
'[?]', # 0xbc
'[?]', # 0xbd
'[?]', # 0xbe
'[?]', # 0xbf
'[?]', # 0xc0
'[?]', # 0xc1
'[?]', # 0xc2
'[?]', # 0xc3
'[?]', # 0xc4
'[?]', # 0xc5
'[?]', # 0xc6
'[?]', # 0xc7
'[?]', # 0xc8
'[?]', # 0xc9
'[?]', # 0xca
'[?]', # 0xcb
'[?]', # 0xcc
'[?]', # 0xcd
'[?]', # 0xce
'[?]', # 0xcf
'', # 0xd0
'', # 0xd1
'', # 0xd2
'', # 0xd3
'', # 0xd4
'', # 0xd5
'', # 0xd6
'', # 0xd7
'', # 0xd8
'', # 0xd9
'', # 0xda
'', # 0xdb
'', # 0xdc
'', # 0xdd
'', # 0xde
'', # 0xdf
'', # 0xe0
'', # 0xe1
'', # 0xe2
'', # 0xe3
'[?]', # 0xe4
'', # 0xe5
'[?]', # 0xe6
'[?]', # 0xe7
'[?]', # 0xe8
'[?]', # 0xe9
'[?]', # 0xea
'[?]', # 0xeb
'[?]', # 0xec
'[?]', # 0xed
'[?]', # 0xee
'[?]', # 0xef
'[?]', # 0xf0
'[?]', # 0xf1
'[?]', # 0xf2
'[?]', # 0xf3
'[?]', # 0xf4
'[?]', # 0xf5
'[?]', # 0xf6
'[?]', # 0xf7
'[?]', # 0xf8
'[?]', # 0xf9
'[?]', # 0xfa
'[?]', # 0xfb
'[?]', # 0xfc
'[?]', # 0xfd
'[?]', # 0xfe
)

257
libs/unidecode/x021.py Normal file
View file

@ -0,0 +1,257 @@
data = (
'', # 0x00
'', # 0x01
'', # 0x02
'', # 0x03
'', # 0x04
'', # 0x05
'', # 0x06
'', # 0x07
'', # 0x08
'', # 0x09
'', # 0x0a
'', # 0x0b
'', # 0x0c
'', # 0x0d
'', # 0x0e
'', # 0x0f
'', # 0x10
'', # 0x11
'', # 0x12
'', # 0x13
'', # 0x14
'', # 0x15
'', # 0x16
'', # 0x17
'', # 0x18
'', # 0x19
'', # 0x1a
'', # 0x1b
'', # 0x1c
'', # 0x1d
'', # 0x1e
'', # 0x1f
'(sm)', # 0x20
'TEL', # 0x21
'(tm)', # 0x22
'', # 0x23
'', # 0x24
'', # 0x25
'', # 0x26
'', # 0x27
'', # 0x28
'', # 0x29
'K', # 0x2a
'A', # 0x2b
'', # 0x2c
'', # 0x2d
'', # 0x2e
'', # 0x2f
'', # 0x30
'', # 0x31
'F', # 0x32
'', # 0x33
'', # 0x34
'', # 0x35
'', # 0x36
'', # 0x37
'', # 0x38
'', # 0x39
'', # 0x3a
'FAX', # 0x3b
'[?]', # 0x3c
'[?]', # 0x3d
'[?]', # 0x3e
'[?]', # 0x3f
'[?]', # 0x40
'[?]', # 0x41
'[?]', # 0x42
'[?]', # 0x43
'[?]', # 0x44
'[?]', # 0x45
'[?]', # 0x46
'[?]', # 0x47
'[?]', # 0x48
'[?]', # 0x49
'[?]', # 0x4a
'[?]', # 0x4b
'[?]', # 0x4c
'[?]', # 0x4d
'F', # 0x4e
'[?]', # 0x4f
'[?]', # 0x50
'[?]', # 0x51
'[?]', # 0x52
' 1/3 ', # 0x53
' 2/3 ', # 0x54
' 1/5 ', # 0x55
' 2/5 ', # 0x56
' 3/5 ', # 0x57
' 4/5 ', # 0x58
' 1/6 ', # 0x59
' 5/6 ', # 0x5a
' 1/8 ', # 0x5b
' 3/8 ', # 0x5c
' 5/8 ', # 0x5d
' 7/8 ', # 0x5e
' 1/', # 0x5f
'I', # 0x60
'II', # 0x61
'III', # 0x62
'IV', # 0x63
'V', # 0x64
'VI', # 0x65
'VII', # 0x66
'VIII', # 0x67
'IX', # 0x68
'X', # 0x69
'XI', # 0x6a
'XII', # 0x6b
'L', # 0x6c
'C', # 0x6d
'D', # 0x6e
'M', # 0x6f
'i', # 0x70
'ii', # 0x71
'iii', # 0x72
'iv', # 0x73
'v', # 0x74
'vi', # 0x75
'vii', # 0x76
'viii', # 0x77
'ix', # 0x78
'x', # 0x79
'xi', # 0x7a
'xii', # 0x7b
'l', # 0x7c
'c', # 0x7d
'd', # 0x7e
'm', # 0x7f
'(D', # 0x80
'D)', # 0x81
'((|))', # 0x82
')', # 0x83
'[?]', # 0x84
'[?]', # 0x85
'[?]', # 0x86
'[?]', # 0x87
'[?]', # 0x88
'[?]', # 0x89
'[?]', # 0x8a
'[?]', # 0x8b
'[?]', # 0x8c
'[?]', # 0x8d
'[?]', # 0x8e
'[?]', # 0x8f
'-', # 0x90
'|', # 0x91
'-', # 0x92
'|', # 0x93
'-', # 0x94
'|', # 0x95
'\\', # 0x96
'/', # 0x97
'\\', # 0x98
'/', # 0x99
'-', # 0x9a
'-', # 0x9b
'~', # 0x9c
'~', # 0x9d
'-', # 0x9e
'|', # 0x9f
'-', # 0xa0
'|', # 0xa1
'-', # 0xa2
'-', # 0xa3
'-', # 0xa4
'|', # 0xa5
'-', # 0xa6
'|', # 0xa7
'|', # 0xa8
'-', # 0xa9
'-', # 0xaa
'-', # 0xab
'-', # 0xac
'-', # 0xad
'-', # 0xae
'|', # 0xaf
'|', # 0xb0
'|', # 0xb1
'|', # 0xb2
'|', # 0xb3
'|', # 0xb4
'|', # 0xb5
'^', # 0xb6
'V', # 0xb7
'\\', # 0xb8
'=', # 0xb9
'V', # 0xba
'^', # 0xbb
'-', # 0xbc
'-', # 0xbd
'|', # 0xbe
'|', # 0xbf
'-', # 0xc0
'-', # 0xc1
'|', # 0xc2
'|', # 0xc3
'=', # 0xc4
'|', # 0xc5
'=', # 0xc6
'=', # 0xc7
'|', # 0xc8
'=', # 0xc9
'|', # 0xca
'=', # 0xcb
'=', # 0xcc
'=', # 0xcd
'=', # 0xce
'=', # 0xcf
'=', # 0xd0
'|', # 0xd1
'=', # 0xd2
'|', # 0xd3
'=', # 0xd4
'|', # 0xd5
'\\', # 0xd6
'/', # 0xd7
'\\', # 0xd8
'/', # 0xd9
'=', # 0xda
'=', # 0xdb
'~', # 0xdc
'~', # 0xdd
'|', # 0xde
'|', # 0xdf
'-', # 0xe0
'|', # 0xe1
'-', # 0xe2
'|', # 0xe3
'-', # 0xe4
'-', # 0xe5
'-', # 0xe6
'|', # 0xe7
'-', # 0xe8
'|', # 0xe9
'|', # 0xea
'|', # 0xeb
'|', # 0xec
'|', # 0xed
'|', # 0xee
'|', # 0xef
'-', # 0xf0
'\\', # 0xf1
'\\', # 0xf2
'|', # 0xf3
'[?]', # 0xf4
'[?]', # 0xf5
'[?]', # 0xf6
'[?]', # 0xf7
'[?]', # 0xf8
'[?]', # 0xf9
'[?]', # 0xfa
'[?]', # 0xfb
'[?]', # 0xfc
'[?]', # 0xfd
'[?]', # 0xfe
)

257
libs/unidecode/x022.py Normal file
View file

@ -0,0 +1,257 @@
data = (
'[?]', # 0x00
'[?]', # 0x01
'[?]', # 0x02
'[?]', # 0x03
'[?]', # 0x04
'[?]', # 0x05
'[?]', # 0x06
'[?]', # 0x07
'[?]', # 0x08
'[?]', # 0x09
'[?]', # 0x0a
'[?]', # 0x0b
'[?]', # 0x0c
'[?]', # 0x0d
'[?]', # 0x0e
'[?]', # 0x0f
'[?]', # 0x10
'[?]', # 0x11
'-', # 0x12
'[?]', # 0x13
'[?]', # 0x14
'/', # 0x15
'\\', # 0x16
'*', # 0x17
'[?]', # 0x18
'[?]', # 0x19
'[?]', # 0x1a
'[?]', # 0x1b
'[?]', # 0x1c
'[?]', # 0x1d
'[?]', # 0x1e
'[?]', # 0x1f
'[?]', # 0x20
'[?]', # 0x21
'[?]', # 0x22
'|', # 0x23
'[?]', # 0x24
'[?]', # 0x25
'[?]', # 0x26
'[?]', # 0x27
'[?]', # 0x28
'[?]', # 0x29
'[?]', # 0x2a
'[?]', # 0x2b
'[?]', # 0x2c
'[?]', # 0x2d
'[?]', # 0x2e
'[?]', # 0x2f
'[?]', # 0x30
'[?]', # 0x31
'[?]', # 0x32
'[?]', # 0x33
'[?]', # 0x34
'[?]', # 0x35
':', # 0x36
'[?]', # 0x37
'[?]', # 0x38
'[?]', # 0x39
'[?]', # 0x3a
'[?]', # 0x3b
'~', # 0x3c
'[?]', # 0x3d
'[?]', # 0x3e
'[?]', # 0x3f
'[?]', # 0x40
'[?]', # 0x41
'[?]', # 0x42
'[?]', # 0x43
'[?]', # 0x44
'[?]', # 0x45
'[?]', # 0x46
'[?]', # 0x47
'[?]', # 0x48
'[?]', # 0x49
'[?]', # 0x4a
'[?]', # 0x4b
'[?]', # 0x4c
'[?]', # 0x4d
'[?]', # 0x4e
'[?]', # 0x4f
'[?]', # 0x50
'[?]', # 0x51
'[?]', # 0x52
'[?]', # 0x53
'[?]', # 0x54
'[?]', # 0x55
'[?]', # 0x56
'[?]', # 0x57
'[?]', # 0x58
'[?]', # 0x59
'[?]', # 0x5a
'[?]', # 0x5b
'[?]', # 0x5c
'[?]', # 0x5d
'[?]', # 0x5e
'[?]', # 0x5f
'[?]', # 0x60
'[?]', # 0x61
'[?]', # 0x62
'[?]', # 0x63
'<=', # 0x64
'>=', # 0x65
'<=', # 0x66
'>=', # 0x67
'[?]', # 0x68
'[?]', # 0x69
'[?]', # 0x6a
'[?]', # 0x6b
'[?]', # 0x6c
'[?]', # 0x6d
'[?]', # 0x6e
'[?]', # 0x6f
'[?]', # 0x70
'[?]', # 0x71
'[?]', # 0x72
'[?]', # 0x73
'[?]', # 0x74
'[?]', # 0x75
'[?]', # 0x76
'[?]', # 0x77
'[?]', # 0x78
'[?]', # 0x79
'[?]', # 0x7a
'[?]', # 0x7b
'[?]', # 0x7c
'[?]', # 0x7d
'[?]', # 0x7e
'[?]', # 0x7f
'[?]', # 0x80
'[?]', # 0x81
'[?]', # 0x82
'[?]', # 0x83
'[?]', # 0x84
'[?]', # 0x85
'[?]', # 0x86
'[?]', # 0x87
'[?]', # 0x88
'[?]', # 0x89
'[?]', # 0x8a
'[?]', # 0x8b
'[?]', # 0x8c
'[?]', # 0x8d
'[?]', # 0x8e
'[?]', # 0x8f
'[?]', # 0x90
'[?]', # 0x91
'[?]', # 0x92
'[?]', # 0x93
'[?]', # 0x94
'[?]', # 0x95
'[?]', # 0x96
'[?]', # 0x97
'[?]', # 0x98
'[?]', # 0x99
'[?]', # 0x9a
'[?]', # 0x9b
'[?]', # 0x9c
'[?]', # 0x9d
'[?]', # 0x9e
'[?]', # 0x9f
'[?]', # 0xa0
'[?]', # 0xa1
'[?]', # 0xa2
'[?]', # 0xa3
'[?]', # 0xa4
'[?]', # 0xa5
'[?]', # 0xa6
'[?]', # 0xa7
'[?]', # 0xa8
'[?]', # 0xa9
'[?]', # 0xaa
'[?]', # 0xab
'[?]', # 0xac
'[?]', # 0xad
'[?]', # 0xae
'[?]', # 0xaf
'[?]', # 0xb0
'[?]', # 0xb1
'[?]', # 0xb2
'[?]', # 0xb3
'[?]', # 0xb4
'[?]', # 0xb5
'[?]', # 0xb6
'[?]', # 0xb7
'[?]', # 0xb8
'[?]', # 0xb9
'[?]', # 0xba
'[?]', # 0xbb
'[?]', # 0xbc
'[?]', # 0xbd
'[?]', # 0xbe
'[?]', # 0xbf
'[?]', # 0xc0
'[?]', # 0xc1
'[?]', # 0xc2
'[?]', # 0xc3
'[?]', # 0xc4
'[?]', # 0xc5
'[?]', # 0xc6
'[?]', # 0xc7
'[?]', # 0xc8
'[?]', # 0xc9
'[?]', # 0xca
'[?]', # 0xcb
'[?]', # 0xcc
'[?]', # 0xcd
'[?]', # 0xce
'[?]', # 0xcf
'[?]', # 0xd0
'[?]', # 0xd1
'[?]', # 0xd2
'[?]', # 0xd3
'[?]', # 0xd4
'[?]', # 0xd5
'[?]', # 0xd6
'[?]', # 0xd7
'[?]', # 0xd8
'[?]', # 0xd9
'[?]', # 0xda
'[?]', # 0xdb
'[?]', # 0xdc
'[?]', # 0xdd
'[?]', # 0xde
'[?]', # 0xdf
'[?]', # 0xe0
'[?]', # 0xe1
'[?]', # 0xe2
'[?]', # 0xe3
'[?]', # 0xe4
'[?]', # 0xe5
'[?]', # 0xe6
'[?]', # 0xe7
'[?]', # 0xe8
'[?]', # 0xe9
'[?]', # 0xea
'[?]', # 0xeb
'[?]', # 0xec
'[?]', # 0xed
'[?]', # 0xee
'[?]', # 0xef
'[?]', # 0xf0
'[?]', # 0xf1
'[?]', # 0xf2
'[?]', # 0xf3
'[?]', # 0xf4
'[?]', # 0xf5
'[?]', # 0xf6
'[?]', # 0xf7
'[?]', # 0xf8
'[?]', # 0xf9
'[?]', # 0xfa
'[?]', # 0xfb
'[?]', # 0xfc
'[?]', # 0xfd
'[?]', # 0xfe
)

257
libs/unidecode/x023.py Normal file
View file

@ -0,0 +1,257 @@
data = (
'[?]', # 0x00
'[?]', # 0x01
'[?]', # 0x02
'^', # 0x03
'[?]', # 0x04
'[?]', # 0x05
'[?]', # 0x06
'[?]', # 0x07
'[?]', # 0x08
'[?]', # 0x09
'[?]', # 0x0a
'[?]', # 0x0b
'[?]', # 0x0c
'[?]', # 0x0d
'[?]', # 0x0e
'[?]', # 0x0f
'[?]', # 0x10
'[?]', # 0x11
'[?]', # 0x12
'[?]', # 0x13
'[?]', # 0x14
'[?]', # 0x15
'[?]', # 0x16
'[?]', # 0x17
'[?]', # 0x18
'[?]', # 0x19
'[?]', # 0x1a
'[?]', # 0x1b
'[?]', # 0x1c
'[?]', # 0x1d
'[?]', # 0x1e
'[?]', # 0x1f
'[?]', # 0x20
'[?]', # 0x21
'[?]', # 0x22
'[?]', # 0x23
'[?]', # 0x24
'[?]', # 0x25
'[?]', # 0x26
'[?]', # 0x27
'[?]', # 0x28
'<', # 0x29
'> ', # 0x2a
'[?]', # 0x2b
'[?]', # 0x2c
'[?]', # 0x2d
'[?]', # 0x2e
'[?]', # 0x2f
'[?]', # 0x30
'[?]', # 0x31
'[?]', # 0x32
'[?]', # 0x33
'[?]', # 0x34
'[?]', # 0x35
'[?]', # 0x36
'[?]', # 0x37
'[?]', # 0x38
'[?]', # 0x39
'[?]', # 0x3a
'[?]', # 0x3b
'[?]', # 0x3c
'[?]', # 0x3d
'[?]', # 0x3e
'[?]', # 0x3f
'[?]', # 0x40
'[?]', # 0x41
'[?]', # 0x42
'[?]', # 0x43
'[?]', # 0x44
'[?]', # 0x45
'[?]', # 0x46
'[?]', # 0x47
'[?]', # 0x48
'[?]', # 0x49
'[?]', # 0x4a
'[?]', # 0x4b
'[?]', # 0x4c
'[?]', # 0x4d
'[?]', # 0x4e
'[?]', # 0x4f
'[?]', # 0x50
'[?]', # 0x51
'[?]', # 0x52
'[?]', # 0x53
'[?]', # 0x54
'[?]', # 0x55
'[?]', # 0x56
'[?]', # 0x57
'[?]', # 0x58
'[?]', # 0x59
'[?]', # 0x5a
'[?]', # 0x5b
'[?]', # 0x5c
'[?]', # 0x5d
'[?]', # 0x5e
'[?]', # 0x5f
'[?]', # 0x60
'[?]', # 0x61
'[?]', # 0x62
'[?]', # 0x63
'[?]', # 0x64
'[?]', # 0x65
'[?]', # 0x66
'[?]', # 0x67
'[?]', # 0x68
'[?]', # 0x69
'[?]', # 0x6a
'[?]', # 0x6b
'[?]', # 0x6c
'[?]', # 0x6d
'[?]', # 0x6e
'[?]', # 0x6f
'[?]', # 0x70
'[?]', # 0x71
'[?]', # 0x72
'[?]', # 0x73
'[?]', # 0x74
'[?]', # 0x75
'[?]', # 0x76
'[?]', # 0x77
'[?]', # 0x78
'[?]', # 0x79
'[?]', # 0x7a
'[?]', # 0x7b
'[?]', # 0x7c
'[?]', # 0x7d
'[?]', # 0x7e
'[?]', # 0x7f
'[?]', # 0x80
'[?]', # 0x81
'[?]', # 0x82
'[?]', # 0x83
'[?]', # 0x84
'[?]', # 0x85
'[?]', # 0x86
'[?]', # 0x87
'[?]', # 0x88
'[?]', # 0x89
'[?]', # 0x8a
'[?]', # 0x8b
'[?]', # 0x8c
'[?]', # 0x8d
'[?]', # 0x8e
'[?]', # 0x8f
'[?]', # 0x90
'[?]', # 0x91
'[?]', # 0x92
'[?]', # 0x93
'[?]', # 0x94
'[?]', # 0x95
'[?]', # 0x96
'[?]', # 0x97
'[?]', # 0x98
'[?]', # 0x99
'[?]', # 0x9a
'[?]', # 0x9b
'[?]', # 0x9c
'[?]', # 0x9d
'[?]', # 0x9e
'[?]', # 0x9f
'[?]', # 0xa0
'[?]', # 0xa1
'[?]', # 0xa2
'[?]', # 0xa3
'[?]', # 0xa4
'[?]', # 0xa5
'[?]', # 0xa6
'[?]', # 0xa7
'[?]', # 0xa8
'[?]', # 0xa9
'[?]', # 0xaa
'[?]', # 0xab
'[?]', # 0xac
'[?]', # 0xad
'[?]', # 0xae
'[?]', # 0xaf
'[?]', # 0xb0
'[?]', # 0xb1
'[?]', # 0xb2
'[?]', # 0xb3
'[?]', # 0xb4
'[?]', # 0xb5
'[?]', # 0xb6
'[?]', # 0xb7
'[?]', # 0xb8
'[?]', # 0xb9
'[?]', # 0xba
'[?]', # 0xbb
'[?]', # 0xbc
'[?]', # 0xbd
'[?]', # 0xbe
'[?]', # 0xbf
'[?]', # 0xc0
'[?]', # 0xc1
'[?]', # 0xc2
'[?]', # 0xc3
'[?]', # 0xc4
'[?]', # 0xc5
'[?]', # 0xc6
'[?]', # 0xc7
'[?]', # 0xc8
'[?]', # 0xc9
'[?]', # 0xca
'[?]', # 0xcb
'[?]', # 0xcc
'[?]', # 0xcd
'[?]', # 0xce
'[?]', # 0xcf
'[?]', # 0xd0
'[?]', # 0xd1
'[?]', # 0xd2
'[?]', # 0xd3
'[?]', # 0xd4
'[?]', # 0xd5
'[?]', # 0xd6
'[?]', # 0xd7
'[?]', # 0xd8
'[?]', # 0xd9
'[?]', # 0xda
'[?]', # 0xdb
'[?]', # 0xdc
'[?]', # 0xdd
'[?]', # 0xde
'[?]', # 0xdf
'[?]', # 0xe0
'[?]', # 0xe1
'[?]', # 0xe2
'[?]', # 0xe3
'[?]', # 0xe4
'[?]', # 0xe5
'[?]', # 0xe6
'[?]', # 0xe7
'[?]', # 0xe8
'[?]', # 0xe9
'[?]', # 0xea
'[?]', # 0xeb
'[?]', # 0xec
'[?]', # 0xed
'[?]', # 0xee
'[?]', # 0xef
'[?]', # 0xf0
'[?]', # 0xf1
'[?]', # 0xf2
'[?]', # 0xf3
'[?]', # 0xf4
'[?]', # 0xf5
'[?]', # 0xf6
'[?]', # 0xf7
'[?]', # 0xf8
'[?]', # 0xf9
'[?]', # 0xfa
'[?]', # 0xfb
'[?]', # 0xfc
'[?]', # 0xfd
'[?]', # 0xfe
)

257
libs/unidecode/x024.py Normal file
View file

@ -0,0 +1,257 @@
data = (
'', # 0x00
'', # 0x01
'', # 0x02
'', # 0x03
'', # 0x04
'', # 0x05
'', # 0x06
'', # 0x07
'', # 0x08
'', # 0x09
'', # 0x0a
'', # 0x0b
'', # 0x0c
'', # 0x0d
'', # 0x0e
'', # 0x0f
'', # 0x10
'', # 0x11
'', # 0x12
'', # 0x13
'', # 0x14
'', # 0x15
'', # 0x16
'', # 0x17
'', # 0x18
'', # 0x19
'', # 0x1a
'', # 0x1b
'', # 0x1c
'', # 0x1d
'', # 0x1e
'', # 0x1f
'', # 0x20
'', # 0x21
'', # 0x22
'', # 0x23
'', # 0x24
'', # 0x25
'', # 0x26
'[?]', # 0x27
'[?]', # 0x28
'[?]', # 0x29
'[?]', # 0x2a
'[?]', # 0x2b
'[?]', # 0x2c
'[?]', # 0x2d
'[?]', # 0x2e
'[?]', # 0x2f
'[?]', # 0x30
'[?]', # 0x31
'[?]', # 0x32
'[?]', # 0x33
'[?]', # 0x34
'[?]', # 0x35
'[?]', # 0x36
'[?]', # 0x37
'[?]', # 0x38
'[?]', # 0x39
'[?]', # 0x3a
'[?]', # 0x3b
'[?]', # 0x3c
'[?]', # 0x3d
'[?]', # 0x3e
'[?]', # 0x3f
'', # 0x40
'', # 0x41
'', # 0x42
'', # 0x43
'', # 0x44
'', # 0x45
'', # 0x46
'', # 0x47
'', # 0x48
'', # 0x49
'', # 0x4a
'[?]', # 0x4b
'[?]', # 0x4c
'[?]', # 0x4d
'[?]', # 0x4e
'[?]', # 0x4f
'[?]', # 0x50
'[?]', # 0x51
'[?]', # 0x52
'[?]', # 0x53
'[?]', # 0x54
'[?]', # 0x55
'[?]', # 0x56
'[?]', # 0x57
'[?]', # 0x58
'[?]', # 0x59
'[?]', # 0x5a
'[?]', # 0x5b
'[?]', # 0x5c
'[?]', # 0x5d
'[?]', # 0x5e
'[?]', # 0x5f
'1', # 0x60
'2', # 0x61
'3', # 0x62
'4', # 0x63
'5', # 0x64
'6', # 0x65
'7', # 0x66
'8', # 0x67
'9', # 0x68
'10', # 0x69
'11', # 0x6a
'12', # 0x6b
'13', # 0x6c
'14', # 0x6d
'15', # 0x6e
'16', # 0x6f
'17', # 0x70
'18', # 0x71
'19', # 0x72
'20', # 0x73
'(1)', # 0x74
'(2)', # 0x75
'(3)', # 0x76
'(4)', # 0x77
'(5)', # 0x78
'(6)', # 0x79
'(7)', # 0x7a
'(8)', # 0x7b
'(9)', # 0x7c
'(10)', # 0x7d
'(11)', # 0x7e
'(12)', # 0x7f
'(13)', # 0x80
'(14)', # 0x81
'(15)', # 0x82
'(16)', # 0x83
'(17)', # 0x84
'(18)', # 0x85
'(19)', # 0x86
'(20)', # 0x87
'1.', # 0x88
'2.', # 0x89
'3.', # 0x8a
'4.', # 0x8b
'5.', # 0x8c
'6.', # 0x8d
'7.', # 0x8e
'8.', # 0x8f
'9.', # 0x90
'10.', # 0x91
'11.', # 0x92
'12.', # 0x93
'13.', # 0x94
'14.', # 0x95
'15.', # 0x96
'16.', # 0x97
'17.', # 0x98
'18.', # 0x99
'19.', # 0x9a
'20.', # 0x9b
'(a)', # 0x9c
'(b)', # 0x9d
'(c)', # 0x9e
'(d)', # 0x9f
'(e)', # 0xa0
'(f)', # 0xa1
'(g)', # 0xa2
'(h)', # 0xa3
'(i)', # 0xa4
'(j)', # 0xa5
'(k)', # 0xa6
'(l)', # 0xa7
'(m)', # 0xa8
'(n)', # 0xa9
'(o)', # 0xaa
'(p)', # 0xab
'(q)', # 0xac
'(r)', # 0xad
'(s)', # 0xae
'(t)', # 0xaf
'(u)', # 0xb0
'(v)', # 0xb1
'(w)', # 0xb2
'(x)', # 0xb3
'(y)', # 0xb4
'(z)', # 0xb5
'a', # 0xb6
'b', # 0xb7
'c', # 0xb8
'd', # 0xb9
'e', # 0xba
'f', # 0xbb
'g', # 0xbc
'h', # 0xbd
'i', # 0xbe
'j', # 0xbf
'k', # 0xc0
'l', # 0xc1
'm', # 0xc2
'n', # 0xc3
'o', # 0xc4
'p', # 0xc5
'q', # 0xc6
'r', # 0xc7
's', # 0xc8
't', # 0xc9
'u', # 0xca
'v', # 0xcb
'w', # 0xcc
'x', # 0xcd
'y', # 0xce
'z', # 0xcf
'a', # 0xd0
'b', # 0xd1
'c', # 0xd2
'd', # 0xd3
'e', # 0xd4
'f', # 0xd5
'g', # 0xd6
'h', # 0xd7
'i', # 0xd8
'j', # 0xd9
'k', # 0xda
'l', # 0xdb
'm', # 0xdc
'n', # 0xdd
'o', # 0xde
'p', # 0xdf
'q', # 0xe0
'r', # 0xe1
's', # 0xe2
't', # 0xe3
'u', # 0xe4
'v', # 0xe5
'w', # 0xe6
'x', # 0xe7
'y', # 0xe8
'z', # 0xe9
'0', # 0xea
'[?]', # 0xeb
'[?]', # 0xec
'[?]', # 0xed
'[?]', # 0xee
'[?]', # 0xef
'[?]', # 0xf0
'[?]', # 0xf1
'[?]', # 0xf2
'[?]', # 0xf3
'[?]', # 0xf4
'[?]', # 0xf5
'[?]', # 0xf6
'[?]', # 0xf7
'[?]', # 0xf8
'[?]', # 0xf9
'[?]', # 0xfa
'[?]', # 0xfb
'[?]', # 0xfc
'[?]', # 0xfd
'[?]', # 0xfe
)

257
libs/unidecode/x025.py Normal file
View file

@ -0,0 +1,257 @@
data = (
'-', # 0x00
'-', # 0x01
'|', # 0x02
'|', # 0x03
'-', # 0x04
'-', # 0x05
'|', # 0x06
'|', # 0x07
'-', # 0x08
'-', # 0x09
'|', # 0x0a
'|', # 0x0b
'+', # 0x0c
'+', # 0x0d
'+', # 0x0e
'+', # 0x0f
'+', # 0x10
'+', # 0x11
'+', # 0x12
'+', # 0x13
'+', # 0x14
'+', # 0x15
'+', # 0x16
'+', # 0x17
'+', # 0x18
'+', # 0x19
'+', # 0x1a
'+', # 0x1b
'+', # 0x1c
'+', # 0x1d
'+', # 0x1e
'+', # 0x1f
'+', # 0x20
'+', # 0x21
'+', # 0x22
'+', # 0x23
'+', # 0x24
'+', # 0x25
'+', # 0x26
'+', # 0x27
'+', # 0x28
'+', # 0x29
'+', # 0x2a
'+', # 0x2b
'+', # 0x2c
'+', # 0x2d
'+', # 0x2e
'+', # 0x2f
'+', # 0x30
'+', # 0x31
'+', # 0x32
'+', # 0x33
'+', # 0x34
'+', # 0x35
'+', # 0x36
'+', # 0x37
'+', # 0x38
'+', # 0x39
'+', # 0x3a
'+', # 0x3b
'+', # 0x3c
'+', # 0x3d
'+', # 0x3e
'+', # 0x3f
'+', # 0x40
'+', # 0x41
'+', # 0x42
'+', # 0x43
'+', # 0x44
'+', # 0x45
'+', # 0x46
'+', # 0x47
'+', # 0x48
'+', # 0x49
'+', # 0x4a
'+', # 0x4b
'-', # 0x4c
'-', # 0x4d
'|', # 0x4e
'|', # 0x4f
'-', # 0x50
'|', # 0x51
'+', # 0x52
'+', # 0x53
'+', # 0x54
'+', # 0x55
'+', # 0x56
'+', # 0x57
'+', # 0x58
'+', # 0x59
'+', # 0x5a
'+', # 0x5b
'+', # 0x5c
'+', # 0x5d
'+', # 0x5e
'+', # 0x5f
'+', # 0x60
'+', # 0x61
'+', # 0x62
'+', # 0x63
'+', # 0x64
'+', # 0x65
'+', # 0x66
'+', # 0x67
'+', # 0x68
'+', # 0x69
'+', # 0x6a
'+', # 0x6b
'+', # 0x6c
'+', # 0x6d
'+', # 0x6e
'+', # 0x6f
'+', # 0x70
'/', # 0x71
'\\', # 0x72
'X', # 0x73
'-', # 0x74
'|', # 0x75
'-', # 0x76
'|', # 0x77
'-', # 0x78
'|', # 0x79
'-', # 0x7a
'|', # 0x7b
'-', # 0x7c
'|', # 0x7d
'-', # 0x7e
'|', # 0x7f
'#', # 0x80
'#', # 0x81
'#', # 0x82
'#', # 0x83
'#', # 0x84
'#', # 0x85
'#', # 0x86
'#', # 0x87
'#', # 0x88
'#', # 0x89
'#', # 0x8a
'#', # 0x8b
'#', # 0x8c
'#', # 0x8d
'#', # 0x8e
'#', # 0x8f
'#', # 0x90
'#', # 0x91
'#', # 0x92
'#', # 0x93
'-', # 0x94
'|', # 0x95
'[?]', # 0x96
'[?]', # 0x97
'[?]', # 0x98
'[?]', # 0x99
'[?]', # 0x9a
'[?]', # 0x9b
'[?]', # 0x9c
'[?]', # 0x9d
'[?]', # 0x9e
'[?]', # 0x9f
'#', # 0xa0
'#', # 0xa1
'#', # 0xa2
'#', # 0xa3
'#', # 0xa4
'#', # 0xa5
'#', # 0xa6
'#', # 0xa7
'#', # 0xa8
'#', # 0xa9
'#', # 0xaa
'#', # 0xab
'#', # 0xac
'#', # 0xad
'#', # 0xae
'#', # 0xaf
'#', # 0xb0
'#', # 0xb1
'^', # 0xb2
'^', # 0xb3
'^', # 0xb4
'^', # 0xb5
'>', # 0xb6
'>', # 0xb7
'>', # 0xb8
'>', # 0xb9
'>', # 0xba
'>', # 0xbb
'V', # 0xbc
'V', # 0xbd
'V', # 0xbe
'V', # 0xbf
'<', # 0xc0
'<', # 0xc1
'<', # 0xc2
'<', # 0xc3
'<', # 0xc4
'<', # 0xc5
'*', # 0xc6
'*', # 0xc7
'*', # 0xc8
'*', # 0xc9
'*', # 0xca
'*', # 0xcb
'*', # 0xcc
'*', # 0xcd
'*', # 0xce
'*', # 0xcf
'*', # 0xd0
'*', # 0xd1
'*', # 0xd2
'*', # 0xd3
'*', # 0xd4
'*', # 0xd5
'*', # 0xd6
'*', # 0xd7
'*', # 0xd8
'*', # 0xd9
'*', # 0xda
'*', # 0xdb
'*', # 0xdc
'*', # 0xdd
'*', # 0xde
'*', # 0xdf
'*', # 0xe0
'*', # 0xe1
'*', # 0xe2
'*', # 0xe3
'*', # 0xe4
'*', # 0xe5
'*', # 0xe6
'#', # 0xe7
'#', # 0xe8
'#', # 0xe9
'#', # 0xea
'#', # 0xeb
'^', # 0xec
'^', # 0xed
'^', # 0xee
'O', # 0xef
'#', # 0xf0
'#', # 0xf1
'#', # 0xf2
'#', # 0xf3
'#', # 0xf4
'#', # 0xf5
'#', # 0xf6
'#', # 0xf7
'[?]', # 0xf8
'[?]', # 0xf9
'[?]', # 0xfa
'[?]', # 0xfb
'[?]', # 0xfc
'[?]', # 0xfd
'[?]', # 0xfe
)

257
libs/unidecode/x026.py Normal file
View file

@ -0,0 +1,257 @@
data = (
'', # 0x00
'', # 0x01
'', # 0x02
'', # 0x03
'', # 0x04
'', # 0x05
'', # 0x06
'', # 0x07
'', # 0x08
'', # 0x09
'', # 0x0a
'', # 0x0b
'', # 0x0c
'', # 0x0d
'', # 0x0e
'', # 0x0f
'', # 0x10
'', # 0x11
'', # 0x12
'', # 0x13
'[?]', # 0x14
'[?]', # 0x15
'[?]', # 0x16
'[?]', # 0x17
'[?]', # 0x18
'', # 0x19
'', # 0x1a
'', # 0x1b
'', # 0x1c
'', # 0x1d
'', # 0x1e
'', # 0x1f
'', # 0x20
'', # 0x21
'', # 0x22
'', # 0x23
'', # 0x24
'', # 0x25
'', # 0x26
'', # 0x27
'', # 0x28
'', # 0x29
'', # 0x2a
'', # 0x2b
'', # 0x2c
'', # 0x2d
'', # 0x2e
'', # 0x2f
'', # 0x30
'', # 0x31
'', # 0x32
'', # 0x33
'', # 0x34
'', # 0x35
'', # 0x36
'', # 0x37
'', # 0x38
'', # 0x39
'', # 0x3a
'', # 0x3b
'', # 0x3c
'', # 0x3d
'', # 0x3e
'', # 0x3f
'', # 0x40
'', # 0x41
'', # 0x42
'', # 0x43
'', # 0x44
'', # 0x45
'', # 0x46
'', # 0x47
'', # 0x48
'', # 0x49
'', # 0x4a
'', # 0x4b
'', # 0x4c
'', # 0x4d
'', # 0x4e
'', # 0x4f
'', # 0x50
'', # 0x51
'', # 0x52
'', # 0x53
'', # 0x54
'', # 0x55
'', # 0x56
'', # 0x57
'', # 0x58
'', # 0x59
'', # 0x5a
'', # 0x5b
'', # 0x5c
'', # 0x5d
'', # 0x5e
'', # 0x5f
'', # 0x60
'', # 0x61
'', # 0x62
'', # 0x63
'', # 0x64
'', # 0x65
'', # 0x66
'', # 0x67
'', # 0x68
'', # 0x69
'', # 0x6a
'', # 0x6b
'', # 0x6c
'', # 0x6d
'', # 0x6e
'#', # 0x6f
'', # 0x70
'', # 0x71
'[?]', # 0x72
'[?]', # 0x73
'[?]', # 0x74
'[?]', # 0x75
'[?]', # 0x76
'[?]', # 0x77
'[?]', # 0x78
'[?]', # 0x79
'[?]', # 0x7a
'[?]', # 0x7b
'[?]', # 0x7c
'[?]', # 0x7d
'[?]', # 0x7e
'[?]', # 0x7f
'[?]', # 0x80
'[?]', # 0x81
'[?]', # 0x82
'[?]', # 0x83
'[?]', # 0x84
'[?]', # 0x85
'[?]', # 0x86
'[?]', # 0x87
'[?]', # 0x88
'[?]', # 0x89
'[?]', # 0x8a
'[?]', # 0x8b
'[?]', # 0x8c
'[?]', # 0x8d
'[?]', # 0x8e
'[?]', # 0x8f
'[?]', # 0x90
'[?]', # 0x91
'[?]', # 0x92
'[?]', # 0x93
'[?]', # 0x94
'[?]', # 0x95
'[?]', # 0x96
'[?]', # 0x97
'[?]', # 0x98
'[?]', # 0x99
'[?]', # 0x9a
'[?]', # 0x9b
'[?]', # 0x9c
'[?]', # 0x9d
'[?]', # 0x9e
'[?]', # 0x9f
'[?]', # 0xa0
'[?]', # 0xa1
'[?]', # 0xa2
'[?]', # 0xa3
'[?]', # 0xa4
'[?]', # 0xa5
'[?]', # 0xa6
'[?]', # 0xa7
'[?]', # 0xa8
'[?]', # 0xa9
'[?]', # 0xaa
'[?]', # 0xab
'[?]', # 0xac
'[?]', # 0xad
'[?]', # 0xae
'[?]', # 0xaf
'[?]', # 0xb0
'[?]', # 0xb1
'[?]', # 0xb2
'[?]', # 0xb3
'[?]', # 0xb4
'[?]', # 0xb5
'[?]', # 0xb6
'[?]', # 0xb7
'[?]', # 0xb8
'[?]', # 0xb9
'[?]', # 0xba
'[?]', # 0xbb
'[?]', # 0xbc
'[?]', # 0xbd
'[?]', # 0xbe
'[?]', # 0xbf
'[?]', # 0xc0
'[?]', # 0xc1
'[?]', # 0xc2
'[?]', # 0xc3
'[?]', # 0xc4
'[?]', # 0xc5
'[?]', # 0xc6
'[?]', # 0xc7
'[?]', # 0xc8
'[?]', # 0xc9
'[?]', # 0xca
'[?]', # 0xcb
'[?]', # 0xcc
'[?]', # 0xcd
'[?]', # 0xce
'[?]', # 0xcf
'[?]', # 0xd0
'[?]', # 0xd1
'[?]', # 0xd2
'[?]', # 0xd3
'[?]', # 0xd4
'[?]', # 0xd5
'[?]', # 0xd6
'[?]', # 0xd7
'[?]', # 0xd8
'[?]', # 0xd9
'[?]', # 0xda
'[?]', # 0xdb
'[?]', # 0xdc
'[?]', # 0xdd
'[?]', # 0xde
'[?]', # 0xdf
'[?]', # 0xe0
'[?]', # 0xe1
'[?]', # 0xe2
'[?]', # 0xe3
'[?]', # 0xe4
'[?]', # 0xe5
'[?]', # 0xe6
'[?]', # 0xe7
'[?]', # 0xe8
'[?]', # 0xe9
'[?]', # 0xea
'[?]', # 0xeb
'[?]', # 0xec
'[?]', # 0xed
'[?]', # 0xee
'[?]', # 0xef
'[?]', # 0xf0
'[?]', # 0xf1
'[?]', # 0xf2
'[?]', # 0xf3
'[?]', # 0xf4
'[?]', # 0xf5
'[?]', # 0xf6
'[?]', # 0xf7
'[?]', # 0xf8
'[?]', # 0xf9
'[?]', # 0xfa
'[?]', # 0xfb
'[?]', # 0xfc
'[?]', # 0xfd
'[?]', # 0xfe
)

257
libs/unidecode/x027.py Normal file
View file

@ -0,0 +1,257 @@
data = (
'[?]', # 0x00
'', # 0x01
'', # 0x02
'', # 0x03
'', # 0x04
'', # 0x05
'', # 0x06
'', # 0x07
'', # 0x08
'', # 0x09
'', # 0x0a
'', # 0x0b
'', # 0x0c
'', # 0x0d
'', # 0x0e
'', # 0x0f
'', # 0x10
'', # 0x11
'', # 0x12
'', # 0x13
'', # 0x14
'', # 0x15
'', # 0x16
'', # 0x17
'', # 0x18
'', # 0x19
'', # 0x1a
'', # 0x1b
'', # 0x1c
'', # 0x1d
'', # 0x1e
'', # 0x1f
'', # 0x20
'', # 0x21
'', # 0x22
'', # 0x23
'', # 0x24
'', # 0x25
'', # 0x26
'', # 0x27
'', # 0x28
'', # 0x29
'', # 0x2a
'', # 0x2b
'', # 0x2c
'', # 0x2d
'', # 0x2e
'', # 0x2f
'', # 0x30
'*', # 0x31
'', # 0x32
'', # 0x33
'', # 0x34
'', # 0x35
'', # 0x36
'', # 0x37
'', # 0x38
'', # 0x39
'', # 0x3a
'', # 0x3b
'', # 0x3c
'', # 0x3d
'', # 0x3e
'', # 0x3f
'', # 0x40
'', # 0x41
'', # 0x42
'', # 0x43
'', # 0x44
'', # 0x45
'', # 0x46
'', # 0x47
'', # 0x48
'', # 0x49
'', # 0x4a
'', # 0x4b
'', # 0x4c
'', # 0x4d
'', # 0x4e
'', # 0x4f
'', # 0x50
'', # 0x51
'', # 0x52
'', # 0x53
'', # 0x54
'', # 0x55
'', # 0x56
'', # 0x57
'|', # 0x58
'', # 0x59
'', # 0x5a
'', # 0x5b
'', # 0x5c
'', # 0x5d
'', # 0x5e
'[?]', # 0x5f
'[?]', # 0x60
'', # 0x61
'!', # 0x62
'', # 0x63
'', # 0x64
'', # 0x65
'', # 0x66
'', # 0x67
'', # 0x68
'', # 0x69
'', # 0x6a
'', # 0x6b
'', # 0x6c
'', # 0x6d
'', # 0x6e
'', # 0x6f
'', # 0x70
'', # 0x71
'', # 0x72
'', # 0x73
'', # 0x74
'', # 0x75
'', # 0x76
'', # 0x77
'', # 0x78
'', # 0x79
'', # 0x7a
'', # 0x7b
'', # 0x7c
'', # 0x7d
'', # 0x7e
'', # 0x7f
'', # 0x80
'', # 0x81
'', # 0x82
'', # 0x83
'', # 0x84
'', # 0x85
'', # 0x86
'', # 0x87
'', # 0x88
'', # 0x89
'', # 0x8a
'', # 0x8b
'', # 0x8c
'', # 0x8d
'', # 0x8e
'', # 0x8f
'', # 0x90
'', # 0x91
'', # 0x92
'', # 0x93
'', # 0x94
'', # 0x95
'', # 0x96
'', # 0x97
'', # 0x98
'', # 0x99
'', # 0x9a
'', # 0x9b
'', # 0x9c
'', # 0x9d
'', # 0x9e
'', # 0x9f
'', # 0xa0
'', # 0xa1
'', # 0xa2
'', # 0xa3
'', # 0xa4
'', # 0xa5
'', # 0xa6
'', # 0xa7
'', # 0xa8
'', # 0xa9
'', # 0xaa
'', # 0xab
'', # 0xac
'', # 0xad
'', # 0xae
'', # 0xaf
'[?]', # 0xb0
'', # 0xb1
'', # 0xb2
'', # 0xb3
'', # 0xb4
'', # 0xb5
'', # 0xb6
'', # 0xb7
'', # 0xb8
'', # 0xb9
'', # 0xba
'', # 0xbb
'', # 0xbc
'', # 0xbd
'', # 0xbe
'[?]', # 0xbf
'[?]', # 0xc0
'[?]', # 0xc1
'[?]', # 0xc2
'[?]', # 0xc3
'[?]', # 0xc4
'[?]', # 0xc5
'[?]', # 0xc6
'[?]', # 0xc7
'[?]', # 0xc8
'[?]', # 0xc9
'[?]', # 0xca
'[?]', # 0xcb
'[?]', # 0xcc
'[?]', # 0xcd
'[?]', # 0xce
'[?]', # 0xcf
'[?]', # 0xd0
'[?]', # 0xd1
'[?]', # 0xd2
'[?]', # 0xd3
'[?]', # 0xd4
'[?]', # 0xd5
'[?]', # 0xd6
'[?]', # 0xd7
'[?]', # 0xd8
'[?]', # 0xd9
'[?]', # 0xda
'[?]', # 0xdb
'[?]', # 0xdc
'[?]', # 0xdd
'[?]', # 0xde
'[?]', # 0xdf
'[?]', # 0xe0
'[?]', # 0xe1
'[?]', # 0xe2
'[?]', # 0xe3
'[?]', # 0xe4
'[?]', # 0xe5
'[', # 0xe6
'[?]', # 0xe7
'<', # 0xe8
'> ', # 0xe9
'[?]', # 0xea
'[?]', # 0xeb
'[?]', # 0xec
'[?]', # 0xed
'[?]', # 0xee
'[?]', # 0xef
'[?]', # 0xf0
'[?]', # 0xf1
'[?]', # 0xf2
'[?]', # 0xf3
'[?]', # 0xf4
'[?]', # 0xf5
'[?]', # 0xf6
'[?]', # 0xf7
'[?]', # 0xf8
'[?]', # 0xf9
'[?]', # 0xfa
'[?]', # 0xfb
'[?]', # 0xfc
'[?]', # 0xfd
'[?]', # 0xfe
)

258
libs/unidecode/x028.py Normal file
View file

@ -0,0 +1,258 @@
data = (
' ', # 0x00
'a', # 0x01
'1', # 0x02
'b', # 0x03
'\'', # 0x04
'k', # 0x05
'2', # 0x06
'l', # 0x07
'@', # 0x08
'c', # 0x09
'i', # 0x0a
'f', # 0x0b
'/', # 0x0c
'm', # 0x0d
's', # 0x0e
'p', # 0x0f
'"', # 0x10
'e', # 0x11
'3', # 0x12
'h', # 0x13
'9', # 0x14
'o', # 0x15
'6', # 0x16
'r', # 0x17
'^', # 0x18
'd', # 0x19
'j', # 0x1a
'g', # 0x1b
'>', # 0x1c
'n', # 0x1d
't', # 0x1e
'q', # 0x1f
',', # 0x20
'*', # 0x21
'5', # 0x22
'<', # 0x23
'-', # 0x24
'u', # 0x25
'8', # 0x26
'v', # 0x27
'.', # 0x28
'%', # 0x29
'[', # 0x2a
'$', # 0x2b
'+', # 0x2c
'x', # 0x2d
'!', # 0x2e
'&', # 0x2f
';', # 0x30
':', # 0x31
'4', # 0x32
'\\', # 0x33
'0', # 0x34
'z', # 0x35
'7', # 0x36
'(', # 0x37
'_', # 0x38
'?', # 0x39
'w', # 0x3a
']', # 0x3b
'#', # 0x3c
'y', # 0x3d
')', # 0x3e
'=', # 0x3f
'[d7]', # 0x40
'[d17]', # 0x41
'[d27]', # 0x42
'[d127]', # 0x43
'[d37]', # 0x44
'[d137]', # 0x45
'[d237]', # 0x46
'[d1237]', # 0x47
'[d47]', # 0x48
'[d147]', # 0x49
'[d247]', # 0x4a
'[d1247]', # 0x4b
'[d347]', # 0x4c
'[d1347]', # 0x4d
'[d2347]', # 0x4e
'[d12347]', # 0x4f
'[d57]', # 0x50
'[d157]', # 0x51
'[d257]', # 0x52
'[d1257]', # 0x53
'[d357]', # 0x54
'[d1357]', # 0x55
'[d2357]', # 0x56
'[d12357]', # 0x57
'[d457]', # 0x58
'[d1457]', # 0x59
'[d2457]', # 0x5a
'[d12457]', # 0x5b
'[d3457]', # 0x5c
'[d13457]', # 0x5d
'[d23457]', # 0x5e
'[d123457]', # 0x5f
'[d67]', # 0x60
'[d167]', # 0x61
'[d267]', # 0x62
'[d1267]', # 0x63
'[d367]', # 0x64
'[d1367]', # 0x65
'[d2367]', # 0x66
'[d12367]', # 0x67
'[d467]', # 0x68
'[d1467]', # 0x69
'[d2467]', # 0x6a
'[d12467]', # 0x6b
'[d3467]', # 0x6c
'[d13467]', # 0x6d
'[d23467]', # 0x6e
'[d123467]', # 0x6f
'[d567]', # 0x70
'[d1567]', # 0x71
'[d2567]', # 0x72
'[d12567]', # 0x73
'[d3567]', # 0x74
'[d13567]', # 0x75
'[d23567]', # 0x76
'[d123567]', # 0x77
'[d4567]', # 0x78
'[d14567]', # 0x79
'[d24567]', # 0x7a
'[d124567]', # 0x7b
'[d34567]', # 0x7c
'[d134567]', # 0x7d
'[d234567]', # 0x7e
'[d1234567]', # 0x7f
'[d8]', # 0x80
'[d18]', # 0x81
'[d28]', # 0x82
'[d128]', # 0x83
'[d38]', # 0x84
'[d138]', # 0x85
'[d238]', # 0x86
'[d1238]', # 0x87
'[d48]', # 0x88
'[d148]', # 0x89
'[d248]', # 0x8a
'[d1248]', # 0x8b
'[d348]', # 0x8c
'[d1348]', # 0x8d
'[d2348]', # 0x8e
'[d12348]', # 0x8f
'[d58]', # 0x90
'[d158]', # 0x91
'[d258]', # 0x92
'[d1258]', # 0x93
'[d358]', # 0x94
'[d1358]', # 0x95
'[d2358]', # 0x96
'[d12358]', # 0x97
'[d458]', # 0x98
'[d1458]', # 0x99
'[d2458]', # 0x9a
'[d12458]', # 0x9b
'[d3458]', # 0x9c
'[d13458]', # 0x9d
'[d23458]', # 0x9e
'[d123458]', # 0x9f
'[d68]', # 0xa0
'[d168]', # 0xa1
'[d268]', # 0xa2
'[d1268]', # 0xa3
'[d368]', # 0xa4
'[d1368]', # 0xa5
'[d2368]', # 0xa6
'[d12368]', # 0xa7
'[d468]', # 0xa8
'[d1468]', # 0xa9
'[d2468]', # 0xaa
'[d12468]', # 0xab
'[d3468]', # 0xac
'[d13468]', # 0xad
'[d23468]', # 0xae
'[d123468]', # 0xaf
'[d568]', # 0xb0
'[d1568]', # 0xb1
'[d2568]', # 0xb2
'[d12568]', # 0xb3
'[d3568]', # 0xb4
'[d13568]', # 0xb5
'[d23568]', # 0xb6
'[d123568]', # 0xb7
'[d4568]', # 0xb8
'[d14568]', # 0xb9
'[d24568]', # 0xba
'[d124568]', # 0xbb
'[d34568]', # 0xbc
'[d134568]', # 0xbd
'[d234568]', # 0xbe
'[d1234568]', # 0xbf
'[d78]', # 0xc0
'[d178]', # 0xc1
'[d278]', # 0xc2
'[d1278]', # 0xc3
'[d378]', # 0xc4
'[d1378]', # 0xc5
'[d2378]', # 0xc6
'[d12378]', # 0xc7
'[d478]', # 0xc8
'[d1478]', # 0xc9
'[d2478]', # 0xca
'[d12478]', # 0xcb
'[d3478]', # 0xcc
'[d13478]', # 0xcd
'[d23478]', # 0xce
'[d123478]', # 0xcf
'[d578]', # 0xd0
'[d1578]', # 0xd1
'[d2578]', # 0xd2
'[d12578]', # 0xd3
'[d3578]', # 0xd4
'[d13578]', # 0xd5
'[d23578]', # 0xd6
'[d123578]', # 0xd7
'[d4578]', # 0xd8
'[d14578]', # 0xd9
'[d24578]', # 0xda
'[d124578]', # 0xdb
'[d34578]', # 0xdc
'[d134578]', # 0xdd
'[d234578]', # 0xde
'[d1234578]', # 0xdf
'[d678]', # 0xe0
'[d1678]', # 0xe1
'[d2678]', # 0xe2
'[d12678]', # 0xe3
'[d3678]', # 0xe4
'[d13678]', # 0xe5
'[d23678]', # 0xe6
'[d123678]', # 0xe7
'[d4678]', # 0xe8
'[d14678]', # 0xe9
'[d24678]', # 0xea
'[d124678]', # 0xeb
'[d34678]', # 0xec
'[d134678]', # 0xed
'[d234678]', # 0xee
'[d1234678]', # 0xef
'[d5678]', # 0xf0
'[d15678]', # 0xf1
'[d25678]', # 0xf2
'[d125678]', # 0xf3
'[d35678]', # 0xf4
'[d135678]', # 0xf5
'[d235678]', # 0xf6
'[d1235678]', # 0xf7
'[d45678]', # 0xf8
'[d145678]', # 0xf9
'[d245678]', # 0xfa
'[d1245678]', # 0xfb
'[d345678]', # 0xfc
'[d1345678]', # 0xfd
'[d2345678]', # 0xfe
'[d12345678]', # 0xff
)

257
libs/unidecode/x029.py Normal file
View file

@ -0,0 +1,257 @@
data = (
'', # 0x00
'', # 0x01
'', # 0x02
'', # 0x03
'', # 0x04
'', # 0x05
'', # 0x06
'', # 0x07
'', # 0x08
'', # 0x09
'', # 0x0a
'', # 0x0b
'', # 0x0c
'', # 0x0d
'', # 0x0e
'', # 0x0f
'', # 0x10
'', # 0x11
'', # 0x12
'', # 0x13
'', # 0x14
'', # 0x15
'', # 0x16
'', # 0x17
'', # 0x18
'', # 0x19
'', # 0x1a
'', # 0x1b
'', # 0x1c
'', # 0x1d
'', # 0x1e
'', # 0x1f
'', # 0x20
'', # 0x21
'', # 0x22
'', # 0x23
'', # 0x24
'', # 0x25
'', # 0x26
'', # 0x27
'', # 0x28
'', # 0x29
'', # 0x2a
'', # 0x2b
'', # 0x2c
'', # 0x2d
'', # 0x2e
'', # 0x2f
'', # 0x30
'', # 0x31
'', # 0x32
'', # 0x33
'', # 0x34
'', # 0x35
'', # 0x36
'', # 0x37
'', # 0x38
'', # 0x39
'', # 0x3a
'', # 0x3b
'', # 0x3c
'', # 0x3d
'', # 0x3e
'', # 0x3f
'', # 0x40
'', # 0x41
'', # 0x42
'', # 0x43
'', # 0x44
'', # 0x45
'', # 0x46
'', # 0x47
'', # 0x48
'', # 0x49
'', # 0x4a
'', # 0x4b
'', # 0x4c
'', # 0x4d
'', # 0x4e
'', # 0x4f
'', # 0x50
'', # 0x51
'', # 0x52
'', # 0x53
'', # 0x54
'', # 0x55
'', # 0x56
'', # 0x57
'', # 0x58
'', # 0x59
'', # 0x5a
'', # 0x5b
'', # 0x5c
'', # 0x5d
'', # 0x5e
'', # 0x5f
'', # 0x60
'', # 0x61
'', # 0x62
'', # 0x63
'', # 0x64
'', # 0x65
'', # 0x66
'', # 0x67
'', # 0x68
'', # 0x69
'', # 0x6a
'', # 0x6b
'', # 0x6c
'', # 0x6d
'', # 0x6e
'', # 0x6f
'', # 0x70
'', # 0x71
'', # 0x72
'', # 0x73
'', # 0x74
'', # 0x75
'', # 0x76
'', # 0x77
'', # 0x78
'', # 0x79
'', # 0x7a
'', # 0x7b
'', # 0x7c
'', # 0x7d
'', # 0x7e
'', # 0x7f
'', # 0x80
'', # 0x81
'', # 0x82
'{', # 0x83
'} ', # 0x84
'', # 0x85
'', # 0x86
'', # 0x87
'', # 0x88
'', # 0x89
'', # 0x8a
'', # 0x8b
'', # 0x8c
'', # 0x8d
'', # 0x8e
'', # 0x8f
'', # 0x90
'', # 0x91
'', # 0x92
'', # 0x93
'', # 0x94
'', # 0x95
'', # 0x96
'', # 0x97
'', # 0x98
'', # 0x99
'', # 0x9a
'', # 0x9b
'', # 0x9c
'', # 0x9d
'', # 0x9e
'', # 0x9f
'', # 0xa0
'', # 0xa1
'', # 0xa2
'', # 0xa3
'', # 0xa4
'', # 0xa5
'', # 0xa6
'', # 0xa7
'', # 0xa8
'', # 0xa9
'', # 0xaa
'', # 0xab
'', # 0xac
'', # 0xad
'', # 0xae
'', # 0xaf
'', # 0xb0
'', # 0xb1
'', # 0xb2
'', # 0xb3
'', # 0xb4
'', # 0xb5
'', # 0xb6
'', # 0xb7
'', # 0xb8
'', # 0xb9
'', # 0xba
'', # 0xbb
'', # 0xbc
'', # 0xbd
'', # 0xbe
'', # 0xbf
'', # 0xc0
'', # 0xc1
'', # 0xc2
'', # 0xc3
'', # 0xc4
'', # 0xc5
'', # 0xc6
'', # 0xc7
'', # 0xc8
'', # 0xc9
'', # 0xca
'', # 0xcb
'', # 0xcc
'', # 0xcd
'', # 0xce
'', # 0xcf
'', # 0xd0
'', # 0xd1
'', # 0xd2
'', # 0xd3
'', # 0xd4
'', # 0xd5
'', # 0xd6
'', # 0xd7
'', # 0xd8
'', # 0xd9
'', # 0xda
'', # 0xdb
'', # 0xdc
'', # 0xdd
'', # 0xde
'', # 0xdf
'', # 0xe0
'', # 0xe1
'', # 0xe2
'', # 0xe3
'', # 0xe4
'', # 0xe5
'', # 0xe6
'', # 0xe7
'', # 0xe8
'', # 0xe9
'', # 0xea
'', # 0xeb
'', # 0xec
'', # 0xed
'', # 0xee
'', # 0xef
'', # 0xf0
'', # 0xf1
'', # 0xf2
'', # 0xf3
'', # 0xf4
'', # 0xf5
'', # 0xf6
'', # 0xf7
'', # 0xf8
'', # 0xf9
'', # 0xfa
'', # 0xfb
'', # 0xfc
'', # 0xfd
'', # 0xfe
)

257
libs/unidecode/x02a.py Normal file
View file

@ -0,0 +1,257 @@
data = (
'', # 0x00
'', # 0x01
'', # 0x02
'', # 0x03
'', # 0x04
'', # 0x05
'', # 0x06
'', # 0x07
'', # 0x08
'', # 0x09
'', # 0x0a
'', # 0x0b
'', # 0x0c
'', # 0x0d
'', # 0x0e
'', # 0x0f
'', # 0x10
'', # 0x11
'', # 0x12
'', # 0x13
'', # 0x14
'', # 0x15
'', # 0x16
'', # 0x17
'', # 0x18
'', # 0x19
'', # 0x1a
'', # 0x1b
'', # 0x1c
'', # 0x1d
'', # 0x1e
'', # 0x1f
'', # 0x20
'', # 0x21
'', # 0x22
'', # 0x23
'', # 0x24
'', # 0x25
'', # 0x26
'', # 0x27
'', # 0x28
'', # 0x29
'', # 0x2a
'', # 0x2b
'', # 0x2c
'', # 0x2d
'', # 0x2e
'', # 0x2f
'', # 0x30
'', # 0x31
'', # 0x32
'', # 0x33
'', # 0x34
'', # 0x35
'', # 0x36
'', # 0x37
'', # 0x38
'', # 0x39
'', # 0x3a
'', # 0x3b
'', # 0x3c
'', # 0x3d
'', # 0x3e
'', # 0x3f
'', # 0x40
'', # 0x41
'', # 0x42
'', # 0x43
'', # 0x44
'', # 0x45
'', # 0x46
'', # 0x47
'', # 0x48
'', # 0x49
'', # 0x4a
'', # 0x4b
'', # 0x4c
'', # 0x4d
'', # 0x4e
'', # 0x4f
'', # 0x50
'', # 0x51
'', # 0x52
'', # 0x53
'', # 0x54
'', # 0x55
'', # 0x56
'', # 0x57
'', # 0x58
'', # 0x59
'', # 0x5a
'', # 0x5b
'', # 0x5c
'', # 0x5d
'', # 0x5e
'', # 0x5f
'', # 0x60
'', # 0x61
'', # 0x62
'', # 0x63
'', # 0x64
'', # 0x65
'', # 0x66
'', # 0x67
'', # 0x68
'', # 0x69
'', # 0x6a
'', # 0x6b
'', # 0x6c
'', # 0x6d
'', # 0x6e
'', # 0x6f
'', # 0x70
'', # 0x71
'', # 0x72
'', # 0x73
'::=', # 0x74
'==', # 0x75
'===', # 0x76
'', # 0x77
'', # 0x78
'', # 0x79
'', # 0x7a
'', # 0x7b
'', # 0x7c
'', # 0x7d
'', # 0x7e
'', # 0x7f
'', # 0x80
'', # 0x81
'', # 0x82
'', # 0x83
'', # 0x84
'', # 0x85
'', # 0x86
'', # 0x87
'', # 0x88
'', # 0x89
'', # 0x8a
'', # 0x8b
'', # 0x8c
'', # 0x8d
'', # 0x8e
'', # 0x8f
'', # 0x90
'', # 0x91
'', # 0x92
'', # 0x93
'', # 0x94
'', # 0x95
'', # 0x96
'', # 0x97
'', # 0x98
'', # 0x99
'', # 0x9a
'', # 0x9b
'', # 0x9c
'', # 0x9d
'', # 0x9e
'', # 0x9f
'', # 0xa0
'', # 0xa1
'', # 0xa2
'', # 0xa3
'', # 0xa4
'', # 0xa5
'', # 0xa6
'', # 0xa7
'', # 0xa8
'', # 0xa9
'', # 0xaa
'', # 0xab
'', # 0xac
'', # 0xad
'', # 0xae
'', # 0xaf
'', # 0xb0
'', # 0xb1
'', # 0xb2
'', # 0xb3
'', # 0xb4
'', # 0xb5
'', # 0xb6
'', # 0xb7
'', # 0xb8
'', # 0xb9
'', # 0xba
'', # 0xbb
'', # 0xbc
'', # 0xbd
'', # 0xbe
'', # 0xbf
'', # 0xc0
'', # 0xc1
'', # 0xc2
'', # 0xc3
'', # 0xc4
'', # 0xc5
'', # 0xc6
'', # 0xc7
'', # 0xc8
'', # 0xc9
'', # 0xca
'', # 0xcb
'', # 0xcc
'', # 0xcd
'', # 0xce
'', # 0xcf
'', # 0xd0
'', # 0xd1
'', # 0xd2
'', # 0xd3
'', # 0xd4
'', # 0xd5
'', # 0xd6
'', # 0xd7
'', # 0xd8
'', # 0xd9
'', # 0xda
'', # 0xdb
'', # 0xdc
'', # 0xdd
'', # 0xde
'', # 0xdf
'', # 0xe0
'', # 0xe1
'', # 0xe2
'', # 0xe3
'', # 0xe4
'', # 0xe5
'', # 0xe6
'', # 0xe7
'', # 0xe8
'', # 0xe9
'', # 0xea
'', # 0xeb
'', # 0xec
'', # 0xed
'', # 0xee
'', # 0xef
'', # 0xf0
'', # 0xf1
'', # 0xf2
'', # 0xf3
'', # 0xf4
'', # 0xf5
'', # 0xf6
'', # 0xf7
'', # 0xf8
'', # 0xf9
'', # 0xfa
'', # 0xfb
'', # 0xfc
'', # 0xfd
'', # 0xfe
)

257
libs/unidecode/x02c.py Normal file
View file

@ -0,0 +1,257 @@
data = (
'', # 0x00
'', # 0x01
'', # 0x02
'', # 0x03
'', # 0x04
'', # 0x05
'', # 0x06
'', # 0x07
'', # 0x08
'', # 0x09
'', # 0x0a
'', # 0x0b
'', # 0x0c
'', # 0x0d
'', # 0x0e
'', # 0x0f
'', # 0x10
'', # 0x11
'', # 0x12
'', # 0x13
'', # 0x14
'', # 0x15
'', # 0x16
'', # 0x17
'', # 0x18
'', # 0x19
'', # 0x1a
'', # 0x1b
'', # 0x1c
'', # 0x1d
'', # 0x1e
'', # 0x1f
'', # 0x20
'', # 0x21
'', # 0x22
'', # 0x23
'', # 0x24
'', # 0x25
'', # 0x26
'', # 0x27
'', # 0x28
'', # 0x29
'', # 0x2a
'', # 0x2b
'', # 0x2c
'', # 0x2d
'', # 0x2e
'', # 0x2f
'', # 0x30
'', # 0x31
'', # 0x32
'', # 0x33
'', # 0x34
'', # 0x35
'', # 0x36
'', # 0x37
'', # 0x38
'', # 0x39
'', # 0x3a
'', # 0x3b
'', # 0x3c
'', # 0x3d
'', # 0x3e
'', # 0x3f
'', # 0x40
'', # 0x41
'', # 0x42
'', # 0x43
'', # 0x44
'', # 0x45
'', # 0x46
'', # 0x47
'', # 0x48
'', # 0x49
'', # 0x4a
'', # 0x4b
'', # 0x4c
'', # 0x4d
'', # 0x4e
'', # 0x4f
'', # 0x50
'', # 0x51
'', # 0x52
'', # 0x53
'', # 0x54
'', # 0x55
'', # 0x56
'', # 0x57
'', # 0x58
'', # 0x59
'', # 0x5a
'', # 0x5b
'', # 0x5c
'', # 0x5d
'', # 0x5e
'', # 0x5f
'L', # 0x60
'l', # 0x61
'L', # 0x62
'P', # 0x63
'R', # 0x64
'a', # 0x65
't', # 0x66
'H', # 0x67
'h', # 0x68
'K', # 0x69
'k', # 0x6a
'Z', # 0x6b
'z', # 0x6c
'', # 0x6d
'M', # 0x6e
'A', # 0x6f
'', # 0x70
'', # 0x71
'', # 0x72
'', # 0x73
'', # 0x74
'', # 0x75
'', # 0x76
'', # 0x77
'', # 0x78
'', # 0x79
'', # 0x7a
'', # 0x7b
'', # 0x7c
'', # 0x7d
'', # 0x7e
'', # 0x7f
'', # 0x80
'', # 0x81
'', # 0x82
'', # 0x83
'', # 0x84
'', # 0x85
'', # 0x86
'', # 0x87
'', # 0x88
'', # 0x89
'', # 0x8a
'', # 0x8b
'', # 0x8c
'', # 0x8d
'', # 0x8e
'', # 0x8f
'', # 0x90
'', # 0x91
'', # 0x92
'', # 0x93
'', # 0x94
'', # 0x95
'', # 0x96
'', # 0x97
'', # 0x98
'', # 0x99
'', # 0x9a
'', # 0x9b
'', # 0x9c
'', # 0x9d
'', # 0x9e
'', # 0x9f
'', # 0xa0
'', # 0xa1
'', # 0xa2
'', # 0xa3
'', # 0xa4
'', # 0xa5
'', # 0xa6
'', # 0xa7
'', # 0xa8
'', # 0xa9
'', # 0xaa
'', # 0xab
'', # 0xac
'', # 0xad
'', # 0xae
'', # 0xaf
'', # 0xb0
'', # 0xb1
'', # 0xb2
'', # 0xb3
'', # 0xb4
'', # 0xb5
'', # 0xb6
'', # 0xb7
'', # 0xb8
'', # 0xb9
'', # 0xba
'', # 0xbb
'', # 0xbc
'', # 0xbd
'', # 0xbe
'', # 0xbf
'', # 0xc0
'', # 0xc1
'', # 0xc2
'', # 0xc3
'', # 0xc4
'', # 0xc5
'', # 0xc6
'', # 0xc7
'', # 0xc8
'', # 0xc9
'', # 0xca
'', # 0xcb
'', # 0xcc
'', # 0xcd
'', # 0xce
'', # 0xcf
'', # 0xd0
'', # 0xd1
'', # 0xd2
'', # 0xd3
'', # 0xd4
'', # 0xd5
'', # 0xd6
'', # 0xd7
'', # 0xd8
'', # 0xd9
'', # 0xda
'', # 0xdb
'', # 0xdc
'', # 0xdd
'', # 0xde
'', # 0xdf
'', # 0xe0
'', # 0xe1
'', # 0xe2
'', # 0xe3
'', # 0xe4
'', # 0xe5
'', # 0xe6
'', # 0xe7
'', # 0xe8
'', # 0xe9
'', # 0xea
'', # 0xeb
'', # 0xec
'', # 0xed
'', # 0xee
'', # 0xef
'', # 0xf0
'', # 0xf1
'', # 0xf2
'', # 0xf3
'', # 0xf4
'', # 0xf5
'', # 0xf6
'', # 0xf7
'', # 0xf8
'', # 0xf9
'', # 0xfa
'', # 0xfb
'', # 0xfc
'', # 0xfd
'', # 0xfe
)

257
libs/unidecode/x02e.py Normal file
View file

@ -0,0 +1,257 @@
data = (
'[?]', # 0x00
'[?]', # 0x01
'[?]', # 0x02
'[?]', # 0x03
'[?]', # 0x04
'[?]', # 0x05
'[?]', # 0x06
'[?]', # 0x07
'[?]', # 0x08
'[?]', # 0x09
'[?]', # 0x0a
'[?]', # 0x0b
'[?]', # 0x0c
'[?]', # 0x0d
'[?]', # 0x0e
'[?]', # 0x0f
'[?]', # 0x10
'[?]', # 0x11
'[?]', # 0x12
'[?]', # 0x13
'[?]', # 0x14
'[?]', # 0x15
'[?]', # 0x16
'[?]', # 0x17
'[?]', # 0x18
'[?]', # 0x19
'[?]', # 0x1a
'[?]', # 0x1b
'[?]', # 0x1c
'[?]', # 0x1d
'[?]', # 0x1e
'[?]', # 0x1f
'[?]', # 0x20
'[?]', # 0x21
'[?]', # 0x22
'[?]', # 0x23
'[?]', # 0x24
'[?]', # 0x25
'[?]', # 0x26
'[?]', # 0x27
'[?]', # 0x28
'[?]', # 0x29
'[?]', # 0x2a
'[?]', # 0x2b
'[?]', # 0x2c
'[?]', # 0x2d
'[?]', # 0x2e
'[?]', # 0x2f
'[?]', # 0x30
'[?]', # 0x31
'[?]', # 0x32
'[?]', # 0x33
'[?]', # 0x34
'[?]', # 0x35
'[?]', # 0x36
'[?]', # 0x37
'[?]', # 0x38
'[?]', # 0x39
'[?]', # 0x3a
'[?]', # 0x3b
'[?]', # 0x3c
'[?]', # 0x3d
'[?]', # 0x3e
'[?]', # 0x3f
'[?]', # 0x40
'[?]', # 0x41
'[?]', # 0x42
'[?]', # 0x43
'[?]', # 0x44
'[?]', # 0x45
'[?]', # 0x46
'[?]', # 0x47
'[?]', # 0x48
'[?]', # 0x49
'[?]', # 0x4a
'[?]', # 0x4b
'[?]', # 0x4c
'[?]', # 0x4d
'[?]', # 0x4e
'[?]', # 0x4f
'[?]', # 0x50
'[?]', # 0x51
'[?]', # 0x52
'[?]', # 0x53
'[?]', # 0x54
'[?]', # 0x55
'[?]', # 0x56
'[?]', # 0x57
'[?]', # 0x58
'[?]', # 0x59
'[?]', # 0x5a
'[?]', # 0x5b
'[?]', # 0x5c
'[?]', # 0x5d
'[?]', # 0x5e
'[?]', # 0x5f
'[?]', # 0x60
'[?]', # 0x61
'[?]', # 0x62
'[?]', # 0x63
'[?]', # 0x64
'[?]', # 0x65
'[?]', # 0x66
'[?]', # 0x67
'[?]', # 0x68
'[?]', # 0x69
'[?]', # 0x6a
'[?]', # 0x6b
'[?]', # 0x6c
'[?]', # 0x6d
'[?]', # 0x6e
'[?]', # 0x6f
'[?]', # 0x70
'[?]', # 0x71
'[?]', # 0x72
'[?]', # 0x73
'[?]', # 0x74
'[?]', # 0x75
'[?]', # 0x76
'[?]', # 0x77
'[?]', # 0x78
'[?]', # 0x79
'[?]', # 0x7a
'[?]', # 0x7b
'[?]', # 0x7c
'[?]', # 0x7d
'[?]', # 0x7e
'[?]', # 0x7f
'[?] ', # 0x80
'[?] ', # 0x81
'[?] ', # 0x82
'[?] ', # 0x83
'[?] ', # 0x84
'[?] ', # 0x85
'[?] ', # 0x86
'[?] ', # 0x87
'[?] ', # 0x88
'[?] ', # 0x89
'[?] ', # 0x8a
'[?] ', # 0x8b
'[?] ', # 0x8c
'[?] ', # 0x8d
'[?] ', # 0x8e
'[?] ', # 0x8f
'[?] ', # 0x90
'[?] ', # 0x91
'[?] ', # 0x92
'[?] ', # 0x93
'[?] ', # 0x94
'[?] ', # 0x95
'[?] ', # 0x96
'[?] ', # 0x97
'[?] ', # 0x98
'[?] ', # 0x99
'[?]', # 0x9a
'[?] ', # 0x9b
'[?] ', # 0x9c
'[?] ', # 0x9d
'[?] ', # 0x9e
'[?] ', # 0x9f
'[?] ', # 0xa0
'[?] ', # 0xa1
'[?] ', # 0xa2
'[?] ', # 0xa3
'[?] ', # 0xa4
'[?] ', # 0xa5
'[?] ', # 0xa6
'[?] ', # 0xa7
'[?] ', # 0xa8
'[?] ', # 0xa9
'[?] ', # 0xaa
'[?] ', # 0xab
'[?] ', # 0xac
'[?] ', # 0xad
'[?] ', # 0xae
'[?] ', # 0xaf
'[?] ', # 0xb0
'[?] ', # 0xb1
'[?] ', # 0xb2
'[?] ', # 0xb3
'[?] ', # 0xb4
'[?] ', # 0xb5
'[?] ', # 0xb6
'[?] ', # 0xb7
'[?] ', # 0xb8
'[?] ', # 0xb9
'[?] ', # 0xba
'[?] ', # 0xbb
'[?] ', # 0xbc
'[?] ', # 0xbd
'[?] ', # 0xbe
'[?] ', # 0xbf
'[?] ', # 0xc0
'[?] ', # 0xc1
'[?] ', # 0xc2
'[?] ', # 0xc3
'[?] ', # 0xc4
'[?] ', # 0xc5
'[?] ', # 0xc6
'[?] ', # 0xc7
'[?] ', # 0xc8
'[?] ', # 0xc9
'[?] ', # 0xca
'[?] ', # 0xcb
'[?] ', # 0xcc
'[?] ', # 0xcd
'[?] ', # 0xce
'[?] ', # 0xcf
'[?] ', # 0xd0
'[?] ', # 0xd1
'[?] ', # 0xd2
'[?] ', # 0xd3
'[?] ', # 0xd4
'[?] ', # 0xd5
'[?] ', # 0xd6
'[?] ', # 0xd7
'[?] ', # 0xd8
'[?] ', # 0xd9
'[?] ', # 0xda
'[?] ', # 0xdb
'[?] ', # 0xdc
'[?] ', # 0xdd
'[?] ', # 0xde
'[?] ', # 0xdf
'[?] ', # 0xe0
'[?] ', # 0xe1
'[?] ', # 0xe2
'[?] ', # 0xe3
'[?] ', # 0xe4
'[?] ', # 0xe5
'[?] ', # 0xe6
'[?] ', # 0xe7
'[?] ', # 0xe8
'[?] ', # 0xe9
'[?] ', # 0xea
'[?] ', # 0xeb
'[?] ', # 0xec
'[?] ', # 0xed
'[?] ', # 0xee
'[?] ', # 0xef
'[?] ', # 0xf0
'[?] ', # 0xf1
'[?] ', # 0xf2
'[?] ', # 0xf3
'[?]', # 0xf4
'[?]', # 0xf5
'[?]', # 0xf6
'[?]', # 0xf7
'[?]', # 0xf8
'[?]', # 0xf9
'[?]', # 0xfa
'[?]', # 0xfb
'[?]', # 0xfc
'[?]', # 0xfd
'[?]', # 0xfe
)

257
libs/unidecode/x02f.py Normal file
View file

@ -0,0 +1,257 @@
data = (
'[?] ', # 0x00
'[?] ', # 0x01
'[?] ', # 0x02
'[?] ', # 0x03
'[?] ', # 0x04
'[?] ', # 0x05
'[?] ', # 0x06
'[?] ', # 0x07
'[?] ', # 0x08
'[?] ', # 0x09
'[?] ', # 0x0a
'[?] ', # 0x0b
'[?] ', # 0x0c
'[?] ', # 0x0d
'[?] ', # 0x0e
'[?] ', # 0x0f
'[?] ', # 0x10
'[?] ', # 0x11
'[?] ', # 0x12
'[?] ', # 0x13
'[?] ', # 0x14
'[?] ', # 0x15
'[?] ', # 0x16
'[?] ', # 0x17
'[?] ', # 0x18
'[?] ', # 0x19
'[?] ', # 0x1a
'[?] ', # 0x1b
'[?] ', # 0x1c
'[?] ', # 0x1d
'[?] ', # 0x1e
'[?] ', # 0x1f
'[?] ', # 0x20
'[?] ', # 0x21
'[?] ', # 0x22
'[?] ', # 0x23
'[?] ', # 0x24
'[?] ', # 0x25
'[?] ', # 0x26
'[?] ', # 0x27
'[?] ', # 0x28
'[?] ', # 0x29
'[?] ', # 0x2a
'[?] ', # 0x2b
'[?] ', # 0x2c
'[?] ', # 0x2d
'[?] ', # 0x2e
'[?] ', # 0x2f
'[?] ', # 0x30
'[?] ', # 0x31
'[?] ', # 0x32
'[?] ', # 0x33
'[?] ', # 0x34
'[?] ', # 0x35
'[?] ', # 0x36
'[?] ', # 0x37
'[?] ', # 0x38
'[?] ', # 0x39
'[?] ', # 0x3a
'[?] ', # 0x3b
'[?] ', # 0x3c
'[?] ', # 0x3d
'[?] ', # 0x3e
'[?] ', # 0x3f
'[?] ', # 0x40
'[?] ', # 0x41
'[?] ', # 0x42
'[?] ', # 0x43
'[?] ', # 0x44
'[?] ', # 0x45
'[?] ', # 0x46
'[?] ', # 0x47
'[?] ', # 0x48
'[?] ', # 0x49
'[?] ', # 0x4a
'[?] ', # 0x4b
'[?] ', # 0x4c
'[?] ', # 0x4d
'[?] ', # 0x4e
'[?] ', # 0x4f
'[?] ', # 0x50
'[?] ', # 0x51
'[?] ', # 0x52
'[?] ', # 0x53
'[?] ', # 0x54
'[?] ', # 0x55
'[?] ', # 0x56
'[?] ', # 0x57
'[?] ', # 0x58
'[?] ', # 0x59
'[?] ', # 0x5a
'[?] ', # 0x5b
'[?] ', # 0x5c
'[?] ', # 0x5d
'[?] ', # 0x5e
'[?] ', # 0x5f
'[?] ', # 0x60
'[?] ', # 0x61
'[?] ', # 0x62
'[?] ', # 0x63
'[?] ', # 0x64
'[?] ', # 0x65
'[?] ', # 0x66
'[?] ', # 0x67
'[?] ', # 0x68
'[?] ', # 0x69
'[?] ', # 0x6a
'[?] ', # 0x6b
'[?] ', # 0x6c
'[?] ', # 0x6d
'[?] ', # 0x6e
'[?] ', # 0x6f
'[?] ', # 0x70
'[?] ', # 0x71
'[?] ', # 0x72
'[?] ', # 0x73
'[?] ', # 0x74
'[?] ', # 0x75
'[?] ', # 0x76
'[?] ', # 0x77
'[?] ', # 0x78
'[?] ', # 0x79
'[?] ', # 0x7a
'[?] ', # 0x7b
'[?] ', # 0x7c
'[?] ', # 0x7d
'[?] ', # 0x7e
'[?] ', # 0x7f
'[?] ', # 0x80
'[?] ', # 0x81
'[?] ', # 0x82
'[?] ', # 0x83
'[?] ', # 0x84
'[?] ', # 0x85
'[?] ', # 0x86
'[?] ', # 0x87
'[?] ', # 0x88
'[?] ', # 0x89
'[?] ', # 0x8a
'[?] ', # 0x8b
'[?] ', # 0x8c
'[?] ', # 0x8d
'[?] ', # 0x8e
'[?] ', # 0x8f
'[?] ', # 0x90
'[?] ', # 0x91
'[?] ', # 0x92
'[?] ', # 0x93
'[?] ', # 0x94
'[?] ', # 0x95
'[?] ', # 0x96
'[?] ', # 0x97
'[?] ', # 0x98
'[?] ', # 0x99
'[?] ', # 0x9a
'[?] ', # 0x9b
'[?] ', # 0x9c
'[?] ', # 0x9d
'[?] ', # 0x9e
'[?] ', # 0x9f
'[?] ', # 0xa0
'[?] ', # 0xa1
'[?] ', # 0xa2
'[?] ', # 0xa3
'[?] ', # 0xa4
'[?] ', # 0xa5
'[?] ', # 0xa6
'[?] ', # 0xa7
'[?] ', # 0xa8
'[?] ', # 0xa9
'[?] ', # 0xaa
'[?] ', # 0xab
'[?] ', # 0xac
'[?] ', # 0xad
'[?] ', # 0xae
'[?] ', # 0xaf
'[?] ', # 0xb0
'[?] ', # 0xb1
'[?] ', # 0xb2
'[?] ', # 0xb3
'[?] ', # 0xb4
'[?] ', # 0xb5
'[?] ', # 0xb6
'[?] ', # 0xb7
'[?] ', # 0xb8
'[?] ', # 0xb9
'[?] ', # 0xba
'[?] ', # 0xbb
'[?] ', # 0xbc
'[?] ', # 0xbd
'[?] ', # 0xbe
'[?] ', # 0xbf
'[?] ', # 0xc0
'[?] ', # 0xc1
'[?] ', # 0xc2
'[?] ', # 0xc3
'[?] ', # 0xc4
'[?] ', # 0xc5
'[?] ', # 0xc6
'[?] ', # 0xc7
'[?] ', # 0xc8
'[?] ', # 0xc9
'[?] ', # 0xca
'[?] ', # 0xcb
'[?] ', # 0xcc
'[?] ', # 0xcd
'[?] ', # 0xce
'[?] ', # 0xcf
'[?] ', # 0xd0
'[?] ', # 0xd1
'[?] ', # 0xd2
'[?] ', # 0xd3
'[?] ', # 0xd4
'[?] ', # 0xd5
'[?]', # 0xd6
'[?]', # 0xd7
'[?]', # 0xd8
'[?]', # 0xd9
'[?]', # 0xda
'[?]', # 0xdb
'[?]', # 0xdc
'[?]', # 0xdd
'[?]', # 0xde
'[?]', # 0xdf
'[?]', # 0xe0
'[?]', # 0xe1
'[?]', # 0xe2
'[?]', # 0xe3
'[?]', # 0xe4
'[?]', # 0xe5
'[?]', # 0xe6
'[?]', # 0xe7
'[?]', # 0xe8
'[?]', # 0xe9
'[?]', # 0xea
'[?]', # 0xeb
'[?]', # 0xec
'[?]', # 0xed
'[?]', # 0xee
'[?]', # 0xef
'[?] ', # 0xf0
'[?] ', # 0xf1
'[?] ', # 0xf2
'[?] ', # 0xf3
'[?] ', # 0xf4
'[?] ', # 0xf5
'[?] ', # 0xf6
'[?] ', # 0xf7
'[?] ', # 0xf8
'[?] ', # 0xf9
'[?] ', # 0xfa
'[?] ', # 0xfb
'[?]', # 0xfc
'[?]', # 0xfd
'[?]', # 0xfe
)

257
libs/unidecode/x030.py Normal file
View file

@ -0,0 +1,257 @@
data = (
' ', # 0x00
', ', # 0x01
'. ', # 0x02
'"', # 0x03
'[JIS]', # 0x04
'"', # 0x05
'/', # 0x06
'0', # 0x07
'<', # 0x08
'> ', # 0x09
'<<', # 0x0a
'>> ', # 0x0b
'[', # 0x0c
'] ', # 0x0d
'{', # 0x0e
'} ', # 0x0f
'[(', # 0x10
')] ', # 0x11
'@', # 0x12
'X ', # 0x13
'[', # 0x14
'] ', # 0x15
'[[', # 0x16
']] ', # 0x17
'((', # 0x18
')) ', # 0x19
'[[', # 0x1a
']] ', # 0x1b
'~ ', # 0x1c
'``', # 0x1d
'\'\'', # 0x1e
',,', # 0x1f
'@', # 0x20
'1', # 0x21
'2', # 0x22
'3', # 0x23
'4', # 0x24
'5', # 0x25
'6', # 0x26
'7', # 0x27
'8', # 0x28
'9', # 0x29
'', # 0x2a
'', # 0x2b
'', # 0x2c
'', # 0x2d
'', # 0x2e
'', # 0x2f
'~', # 0x30
'+', # 0x31
'+', # 0x32
'+', # 0x33
'+', # 0x34
'', # 0x35
'@', # 0x36
' // ', # 0x37
'+10+', # 0x38
'+20+', # 0x39
'+30+', # 0x3a
'[?]', # 0x3b
'[?]', # 0x3c
'[?]', # 0x3d
'', # 0x3e
'', # 0x3f
'[?]', # 0x40
'a', # 0x41
'a', # 0x42
'i', # 0x43
'i', # 0x44
'u', # 0x45
'u', # 0x46
'e', # 0x47
'e', # 0x48
'o', # 0x49
'o', # 0x4a
'ka', # 0x4b
'ga', # 0x4c
'ki', # 0x4d
'gi', # 0x4e
'ku', # 0x4f
'gu', # 0x50
'ke', # 0x51
'ge', # 0x52
'ko', # 0x53
'go', # 0x54
'sa', # 0x55
'za', # 0x56
'shi', # 0x57
'zi', # 0x58
'su', # 0x59
'zu', # 0x5a
'se', # 0x5b
'ze', # 0x5c
'so', # 0x5d
'zo', # 0x5e
'ta', # 0x5f
'da', # 0x60
'chi', # 0x61
'di', # 0x62
'tsu', # 0x63
'tsu', # 0x64
'du', # 0x65
'te', # 0x66
'de', # 0x67
'to', # 0x68
'do', # 0x69
'na', # 0x6a
'ni', # 0x6b
'nu', # 0x6c
'ne', # 0x6d
'no', # 0x6e
'ha', # 0x6f
'ba', # 0x70
'pa', # 0x71
'hi', # 0x72
'bi', # 0x73
'pi', # 0x74
'hu', # 0x75
'bu', # 0x76
'pu', # 0x77
'he', # 0x78
'be', # 0x79
'pe', # 0x7a
'ho', # 0x7b
'bo', # 0x7c
'po', # 0x7d
'ma', # 0x7e
'mi', # 0x7f
'mu', # 0x80
'me', # 0x81
'mo', # 0x82
'ya', # 0x83
'ya', # 0x84
'yu', # 0x85
'yu', # 0x86
'yo', # 0x87
'yo', # 0x88
'ra', # 0x89
'ri', # 0x8a
'ru', # 0x8b
're', # 0x8c
'ro', # 0x8d
'wa', # 0x8e
'wa', # 0x8f
'wi', # 0x90
'we', # 0x91
'wo', # 0x92
'n', # 0x93
'vu', # 0x94
'[?]', # 0x95
'[?]', # 0x96
'[?]', # 0x97
'[?]', # 0x98
'', # 0x99
'', # 0x9a
'', # 0x9b
'', # 0x9c
'"', # 0x9d
'"', # 0x9e
'[?]', # 0x9f
'[?]', # 0xa0
'a', # 0xa1
'a', # 0xa2
'i', # 0xa3
'i', # 0xa4
'u', # 0xa5
'u', # 0xa6
'e', # 0xa7
'e', # 0xa8
'o', # 0xa9
'o', # 0xaa
'ka', # 0xab
'ga', # 0xac
'ki', # 0xad
'gi', # 0xae
'ku', # 0xaf
'gu', # 0xb0
'ke', # 0xb1
'ge', # 0xb2
'ko', # 0xb3
'go', # 0xb4
'sa', # 0xb5
'za', # 0xb6
'shi', # 0xb7
'zi', # 0xb8
'su', # 0xb9
'zu', # 0xba
'se', # 0xbb
'ze', # 0xbc
'so', # 0xbd
'zo', # 0xbe
'ta', # 0xbf
'da', # 0xc0
'chi', # 0xc1
'di', # 0xc2
'tsu', # 0xc3
'tsu', # 0xc4
'du', # 0xc5
'te', # 0xc6
'de', # 0xc7
'to', # 0xc8
'do', # 0xc9
'na', # 0xca
'ni', # 0xcb
'nu', # 0xcc
'ne', # 0xcd
'no', # 0xce
'ha', # 0xcf
'ba', # 0xd0
'pa', # 0xd1
'hi', # 0xd2
'bi', # 0xd3
'pi', # 0xd4
'hu', # 0xd5
'bu', # 0xd6
'pu', # 0xd7
'he', # 0xd8
'be', # 0xd9
'pe', # 0xda
'ho', # 0xdb
'bo', # 0xdc
'po', # 0xdd
'ma', # 0xde
'mi', # 0xdf
'mu', # 0xe0
'me', # 0xe1
'mo', # 0xe2
'ya', # 0xe3
'ya', # 0xe4
'yu', # 0xe5
'yu', # 0xe6
'yo', # 0xe7
'yo', # 0xe8
'ra', # 0xe9
'ri', # 0xea
'ru', # 0xeb
're', # 0xec
'ro', # 0xed
'wa', # 0xee
'wa', # 0xef
'wi', # 0xf0
'we', # 0xf1
'wo', # 0xf2
'n', # 0xf3
'vu', # 0xf4
'ka', # 0xf5
'ke', # 0xf6
'va', # 0xf7
'vi', # 0xf8
've', # 0xf9
'vo', # 0xfa
'', # 0xfb
'', # 0xfc
'"', # 0xfd
'"', # 0xfe
)

257
libs/unidecode/x031.py Normal file
View file

@ -0,0 +1,257 @@
data = (
'[?]', # 0x00
'[?]', # 0x01
'[?]', # 0x02
'[?]', # 0x03
'[?]', # 0x04
'B', # 0x05
'P', # 0x06
'M', # 0x07
'F', # 0x08
'D', # 0x09
'T', # 0x0a
'N', # 0x0b
'L', # 0x0c
'G', # 0x0d
'K', # 0x0e
'H', # 0x0f
'J', # 0x10
'Q', # 0x11
'X', # 0x12
'ZH', # 0x13
'CH', # 0x14
'SH', # 0x15
'R', # 0x16
'Z', # 0x17
'C', # 0x18
'S', # 0x19
'A', # 0x1a
'O', # 0x1b
'E', # 0x1c
'EH', # 0x1d
'AI', # 0x1e
'EI', # 0x1f
'AU', # 0x20
'OU', # 0x21
'AN', # 0x22
'EN', # 0x23
'ANG', # 0x24
'ENG', # 0x25
'ER', # 0x26
'I', # 0x27
'U', # 0x28
'IU', # 0x29
'V', # 0x2a
'NG', # 0x2b
'GN', # 0x2c
'[?]', # 0x2d
'[?]', # 0x2e
'[?]', # 0x2f
'[?]', # 0x30
'g', # 0x31
'gg', # 0x32
'gs', # 0x33
'n', # 0x34
'nj', # 0x35
'nh', # 0x36
'd', # 0x37
'dd', # 0x38
'r', # 0x39
'lg', # 0x3a
'lm', # 0x3b
'lb', # 0x3c
'ls', # 0x3d
'lt', # 0x3e
'lp', # 0x3f
'rh', # 0x40
'm', # 0x41
'b', # 0x42
'bb', # 0x43
'bs', # 0x44
's', # 0x45
'ss', # 0x46
'', # 0x47
'j', # 0x48
'jj', # 0x49
'c', # 0x4a
'k', # 0x4b
't', # 0x4c
'p', # 0x4d
'h', # 0x4e
'a', # 0x4f
'ae', # 0x50
'ya', # 0x51
'yae', # 0x52
'eo', # 0x53
'e', # 0x54
'yeo', # 0x55
'ye', # 0x56
'o', # 0x57
'wa', # 0x58
'wae', # 0x59
'oe', # 0x5a
'yo', # 0x5b
'u', # 0x5c
'weo', # 0x5d
'we', # 0x5e
'wi', # 0x5f
'yu', # 0x60
'eu', # 0x61
'yi', # 0x62
'i', # 0x63
'', # 0x64
'nn', # 0x65
'nd', # 0x66
'ns', # 0x67
'nZ', # 0x68
'lgs', # 0x69
'ld', # 0x6a
'lbs', # 0x6b
'lZ', # 0x6c
'lQ', # 0x6d
'mb', # 0x6e
'ms', # 0x6f
'mZ', # 0x70
'mN', # 0x71
'bg', # 0x72
'', # 0x73
'bsg', # 0x74
'bst', # 0x75
'bj', # 0x76
'bt', # 0x77
'bN', # 0x78
'bbN', # 0x79
'sg', # 0x7a
'sn', # 0x7b
'sd', # 0x7c
'sb', # 0x7d
'sj', # 0x7e
'Z', # 0x7f
'', # 0x80
'N', # 0x81
'Ns', # 0x82
'NZ', # 0x83
'pN', # 0x84
'hh', # 0x85
'Q', # 0x86
'yo-ya', # 0x87
'yo-yae', # 0x88
'yo-i', # 0x89
'yu-yeo', # 0x8a
'yu-ye', # 0x8b
'yu-i', # 0x8c
'U', # 0x8d
'U-i', # 0x8e
'[?]', # 0x8f
'', # 0x90
'', # 0x91
'', # 0x92
'', # 0x93
'', # 0x94
'', # 0x95
'', # 0x96
'', # 0x97
'', # 0x98
'', # 0x99
'', # 0x9a
'', # 0x9b
'', # 0x9c
'', # 0x9d
'', # 0x9e
'', # 0x9f
'BU', # 0xa0
'ZI', # 0xa1
'JI', # 0xa2
'GU', # 0xa3
'EE', # 0xa4
'ENN', # 0xa5
'OO', # 0xa6
'ONN', # 0xa7
'IR', # 0xa8
'ANN', # 0xa9
'INN', # 0xaa
'UNN', # 0xab
'IM', # 0xac
'NGG', # 0xad
'AINN', # 0xae
'AUNN', # 0xaf
'AM', # 0xb0
'OM', # 0xb1
'ONG', # 0xb2
'INNN', # 0xb3
'P', # 0xb4
'T', # 0xb5
'K', # 0xb6
'H', # 0xb7
'[?]', # 0xb8
'[?]', # 0xb9
'[?]', # 0xba
'[?]', # 0xbb
'[?]', # 0xbc
'[?]', # 0xbd
'[?]', # 0xbe
'[?]', # 0xbf
'[?]', # 0xc0
'[?]', # 0xc1
'[?]', # 0xc2
'[?]', # 0xc3
'[?]', # 0xc4
'[?]', # 0xc5
'[?]', # 0xc6
'[?]', # 0xc7
'[?]', # 0xc8
'[?]', # 0xc9
'[?]', # 0xca
'[?]', # 0xcb
'[?]', # 0xcc
'[?]', # 0xcd
'[?]', # 0xce
'[?]', # 0xcf
'[?]', # 0xd0
'[?]', # 0xd1
'[?]', # 0xd2
'[?]', # 0xd3
'[?]', # 0xd4
'[?]', # 0xd5
'[?]', # 0xd6
'[?]', # 0xd7
'[?]', # 0xd8
'[?]', # 0xd9
'[?]', # 0xda
'[?]', # 0xdb
'[?]', # 0xdc
'[?]', # 0xdd
'[?]', # 0xde
'[?]', # 0xdf
'[?]', # 0xe0
'[?]', # 0xe1
'[?]', # 0xe2
'[?]', # 0xe3
'[?]', # 0xe4
'[?]', # 0xe5
'[?]', # 0xe6
'[?]', # 0xe7
'[?]', # 0xe8
'[?]', # 0xe9
'[?]', # 0xea
'[?]', # 0xeb
'[?]', # 0xec
'[?]', # 0xed
'[?]', # 0xee
'[?]', # 0xef
'[?]', # 0xf0
'[?]', # 0xf1
'[?]', # 0xf2
'[?]', # 0xf3
'[?]', # 0xf4
'[?]', # 0xf5
'[?]', # 0xf6
'[?]', # 0xf7
'[?]', # 0xf8
'[?]', # 0xf9
'[?]', # 0xfa
'[?]', # 0xfb
'[?]', # 0xfc
'[?]', # 0xfd
'[?]', # 0xfe
)

Some files were not shown because too many files have changed in this diff Show more