github/nzbToMedia
1
0
Fork 0
mirror of https://github.com/clinton-hall/nzbToMedia.git synced 2025-08-14 10:36:52 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Update vendored beets to 1.6.0

Browse source 
Updates colorama to 0.4.6
Adds confuse version 1.7.0
Updates jellyfish to 0.9.0
Adds mediafile 0.10.1
Updates munkres to 1.1.4
Updates musicbrainzngs to 0.7.1
Updates mutagen to 1.46.0
Updates pyyaml to 6.0
Updates unidecode to 1.3.6
This commit is contained in:
Labrys of Knossos 2022-11-28 18:02:40 -05:00
commit 56c6773c6b
385 changed files with 25143 additions and 18080 deletions
Download patch file Download diff file Expand all files Collapse all files

13
libs/common/confuse/__init__.py Normal file
View file

@ -0,0 +1,13 @@
"""Painless YAML configuration.
"""
from __future__ import division, absolute_import, print_function
__version__ = '1.7.0'
from .exceptions import * # NOQA
from .util import * # NOQA
from .yaml_util import * # NOQA
from .sources import * # NOQA
from .templates import * # NOQA
from .core import * # NOQA

724
libs/common/confuse/core.py Normal file
View file

@ -0,0 +1,724 @@
# -*- coding: utf-8 -*-
# This file is part of Confuse.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Worry-free YAML configuration files.
"""
from __future__ import division, absolute_import, print_function
import errno
import os
import yaml
from collections import OrderedDict
from . import util
from . import templates
from . import yaml_util
from .sources import ConfigSource, EnvSource, YamlSource
from .exceptions import ConfigTypeError, NotFoundError, ConfigError
CONFIG_FILENAME = 'config.yaml'
DEFAULT_FILENAME = 'config_default.yaml'
ROOT_NAME = 'root'
REDACTED_TOMBSTONE = 'REDACTED'
# Views and sources.
class ConfigView(object):
"""A configuration "view" is a query into a program's configuration
data. A view represents a hypothetical location in the configuration
tree; to extract the data from the location, a client typically
calls the ``view.get()`` method. The client can access children in
the tree (subviews) by subscripting the parent view (i.e.,
``view[key]``).
"""
name = None
"""The name of the view, depicting the path taken through the
configuration in Python-like syntax (e.g., ``foo['bar'][42]``).
"""
def resolve(self):
"""The core (internal) data retrieval method. Generates (value,
source) pairs for each source that contains a value for this
view. May raise `ConfigTypeError` if a type error occurs while
traversing a source.
"""
raise NotImplementedError
def first(self):
"""Return a (value, source) pair for the first object found for
this view. This amounts to the first element returned by
`resolve`. If no values are available, a `NotFoundError` is
raised.
"""
pairs = self.resolve()
try:
return util.iter_first(pairs)
except ValueError:
raise NotFoundError(u"{0} not found".format(self.name))
def exists(self):
"""Determine whether the view has a setting in any source.
"""
try:
self.first()
except NotFoundError:
return False
return True
def add(self, value):
"""Set the *default* value for this configuration view. The
specified value is added as the lowest-priority configuration
data source.
"""
raise NotImplementedError
def set(self, value):
"""*Override* the value for this configuration view. The
specified value is added as the highest-priority configuration
data source.
"""
raise NotImplementedError
def root(self):
"""The RootView object from which this view is descended.
"""
raise NotImplementedError
def __repr__(self):
return '<{}: {}>'.format(self.__class__.__name__, self.name)
def __iter__(self):
"""Iterate over the keys of a dictionary view or the *subviews*
of a list view.
"""
# Try iterating over the keys, if this is a dictionary view.
try:
for key in self.keys():
yield key
except ConfigTypeError:
# Otherwise, try iterating over a list view.
try:
for subview in self.sequence():
yield subview
except ConfigTypeError:
item, _ = self.first()
raise ConfigTypeError(
u'{0} must be a dictionary or a list, not {1}'.format(
self.name, type(item).__name__
)
)
def __getitem__(self, key):
"""Get a subview of this view."""
return Subview(self, key)
def __setitem__(self, key, value):
"""Create an overlay source to assign a given key under this
view.
"""
self.set({key: value})
def __contains__(self, key):
return self[key].exists()
def set_args(self, namespace, dots=False):
"""Overlay parsed command-line arguments, generated by a library
like argparse or optparse, onto this view's value.
:param namespace: Dictionary or Namespace to overlay this config with.
Supports nested Dictionaries and Namespaces.
:type namespace: dict or Namespace
:param dots: If True, any properties on namespace that contain dots (.)
will be broken down into child dictionaries.
:Example:
{'foo.bar': 'car'}
# Will be turned into
{'foo': {'bar': 'car'}}
:type dots: bool
"""
self.set(util.build_dict(namespace, sep='.' if dots else ''))
# Magical conversions. These special methods make it possible to use
# View objects somewhat transparently in certain circumstances. For
# example, rather than using ``view.get(bool)``, it's possible to
# just say ``bool(view)`` or use ``view`` in a conditional.
def __str__(self):
"""Get the value for this view as a bytestring.
"""
if util.PY3:
return self.__unicode__()
else:
return bytes(self.get())
def __unicode__(self):
"""Get the value for this view as a Unicode string.
"""
return util.STRING(self.get())
def __nonzero__(self):
"""Gets the value for this view as a boolean. (Python 2 only.)
"""
return self.__bool__()
def __bool__(self):
"""Gets the value for this view as a boolean. (Python 3 only.)
"""
return bool(self.get())
# Dictionary emulation methods.
def keys(self):
"""Returns a list containing all the keys available as subviews
of the current views. This enumerates all the keys in *all*
dictionaries matching the current view, in contrast to
``view.get(dict).keys()``, which gets all the keys for the
*first* dict matching the view. If the object for this view in
any source is not a dict, then a `ConfigTypeError` is raised. The
keys are ordered according to how they appear in each source.
"""
keys = []
for dic, _ in self.resolve():
try:
cur_keys = dic.keys()
except AttributeError:
raise ConfigTypeError(
u'{0} must be a dict, not {1}'.format(
self.name, type(dic).__name__
)
)
for key in cur_keys:
if key not in keys:
keys.append(key)
return keys
def items(self):
"""Iterates over (key, subview) pairs contained in dictionaries
from *all* sources at this view. If the object for this view in
any source is not a dict, then a `ConfigTypeError` is raised.
"""
for key in self.keys():
yield key, self[key]
def values(self):
"""Iterates over all the subviews contained in dictionaries from
*all* sources at this view. If the object for this view in any
source is not a dict, then a `ConfigTypeError` is raised.
"""
for key in self.keys():
yield self[key]
# List/sequence emulation.
def sequence(self):
"""Iterates over the subviews contained in lists from the *first*
source at this view. If the object for this view in the first source
is not a list or tuple, then a `ConfigTypeError` is raised.
"""
try:
collection, _ = self.first()
except NotFoundError:
return
if not isinstance(collection, (list, tuple)):
raise ConfigTypeError(
u'{0} must be a list, not {1}'.format(
self.name, type(collection).__name__
)
)
# Yield all the indices in the sequence.
for index in range(len(collection)):
yield self[index]
def all_contents(self):
"""Iterates over all subviews from collections at this view from
*all* sources. If the object for this view in any source is not
iterable, then a `ConfigTypeError` is raised. This method is
intended to be used when the view indicates a list; this method
will concatenate the contents of the list from all sources.
"""
for collection, _ in self.resolve():
try:
it = iter(collection)
except TypeError:
raise ConfigTypeError(
u'{0} must be an iterable, not {1}'.format(
self.name, type(collection).__name__
)
)
for value in it:
yield value
# Validation and conversion.
def flatten(self, redact=False):
"""Create a hierarchy of OrderedDicts containing the data from
this view, recursively reifying all views to get their
represented values.
If `redact` is set, then sensitive values are replaced with
the string "REDACTED".
"""
od = OrderedDict()
for key, view in self.items():
if redact and view.redact:
od[key] = REDACTED_TOMBSTONE
else:
try:
od[key] = view.flatten(redact=redact)
except ConfigTypeError:
od[key] = view.get()
return od
def get(self, template=templates.REQUIRED):
"""Retrieve the value for this view according to the template.
The `template` against which the values are checked can be
anything convertible to a `Template` using `as_template`. This
means you can pass in a default integer or string value, for
example, or a type to just check that something matches the type
you expect.
May raise a `ConfigValueError` (or its subclass,
`ConfigTypeError`) or a `NotFoundError` when the configuration
doesn't satisfy the template.
"""
return templates.as_template(template).value(self, template)
# Shortcuts for common templates.
def as_filename(self):
"""Get the value as a path. Equivalent to `get(Filename())`.
"""
return self.get(templates.Filename())
def as_path(self):
"""Get the value as a `pathlib.Path` object. Equivalent to `get(Path())`.
"""
return self.get(templates.Path())
def as_choice(self, choices):
"""Get the value from a list of choices. Equivalent to
`get(Choice(choices))`.
"""
return self.get(templates.Choice(choices))
def as_number(self):
"""Get the value as any number type: int or float. Equivalent to
`get(Number())`.
"""
return self.get(templates.Number())
def as_str_seq(self, split=True):
"""Get the value as a sequence of strings. Equivalent to
`get(StrSeq(split=split))`.
"""
return self.get(templates.StrSeq(split=split))
def as_pairs(self, default_value=None):
"""Get the value as a sequence of pairs of two strings. Equivalent to
`get(Pairs(default_value=default_value))`.
"""
return self.get(templates.Pairs(default_value=default_value))
def as_str(self):
"""Get the value as a (Unicode) string. Equivalent to
`get(unicode)` on Python 2 and `get(str)` on Python 3.
"""
return self.get(templates.String())
def as_str_expanded(self):
"""Get the value as a (Unicode) string, with env vars
expanded by `os.path.expandvars()`.
"""
return self.get(templates.String(expand_vars=True))
# Redaction.
@property
def redact(self):
"""Whether the view contains sensitive information and should be
redacted from output.
"""
return () in self.get_redactions()
@redact.setter
def redact(self, flag):
self.set_redaction((), flag)
def set_redaction(self, path, flag):
"""Add or remove a redaction for a key path, which should be an
iterable of keys.
"""
raise NotImplementedError()
def get_redactions(self):
"""Get the set of currently-redacted sub-key-paths at this view.
"""
raise NotImplementedError()
class RootView(ConfigView):
"""The base of a view hierarchy. This view keeps track of the
sources that may be accessed by subviews.
"""
def __init__(self, sources):
"""Create a configuration hierarchy for a list of sources. At
least one source must be provided. The first source in the list
has the highest priority.
"""
self.sources = list(sources)
self.name = ROOT_NAME
self.redactions = set()
def add(self, obj):
self.sources.append(ConfigSource.of(obj))
def set(self, value):
self.sources.insert(0, ConfigSource.of(value))
def resolve(self):
return ((dict(s), s) for s in self.sources)
def clear(self):
"""Remove all sources (and redactions) from this
configuration.
"""
del self.sources[:]
self.redactions.clear()
def root(self):
return self
def set_redaction(self, path, flag):
if flag:
self.redactions.add(path)
elif path in self.redactions:
self.redactions.remove(path)
def get_redactions(self):
return self.redactions
class Subview(ConfigView):
"""A subview accessed via a subscript of a parent view."""
def __init__(self, parent, key):
"""Make a subview of a parent view for a given subscript key.
"""
self.parent = parent
self.key = key
# Choose a human-readable name for this view.
if isinstance(self.parent, RootView):
self.name = ''
else:
self.name = self.parent.name
if not isinstance(self.key, int):
self.name += '.'
if isinstance(self.key, int):
self.name += u'#{0}'.format(self.key)
elif isinstance(self.key, bytes):
self.name += self.key.decode('utf-8')
elif isinstance(self.key, util.STRING):
self.name += self.key
else:
self.name += repr(self.key)
def resolve(self):
for collection, source in self.parent.resolve():
try:
value = collection[self.key]
except IndexError:
# List index out of bounds.
continue
except KeyError:
# Dict key does not exist.
continue
except TypeError:
# Not subscriptable.
raise ConfigTypeError(
u"{0} must be a collection, not {1}".format(
self.parent.name, type(collection).__name__
)
)
yield value, source
def set(self, value):
self.parent.set({self.key: value})
def add(self, value):
self.parent.add({self.key: value})
def root(self):
return self.parent.root()
def set_redaction(self, path, flag):
self.parent.set_redaction((self.key,) + path, flag)
def get_redactions(self):
return (kp[1:] for kp in self.parent.get_redactions()
if kp and kp[0] == self.key)
# Main interface.
class Configuration(RootView):
def __init__(self, appname, modname=None, read=True,
loader=yaml_util.Loader):
"""Create a configuration object by reading the
automatically-discovered config files for the application for a
given name. If `modname` is specified, it should be the import
name of a module whose package will be searched for a default
config file. (Otherwise, no defaults are used.) Pass `False` for
`read` to disable automatic reading of all discovered
configuration files. Use this when creating a configuration
object at module load time and then call the `read` method
later. Specify the Loader class as `loader`.
"""
super(Configuration, self).__init__([])
self.appname = appname
self.modname = modname
self.loader = loader
# Resolve default source location. We do this ahead of time to
# avoid unexpected problems if the working directory changes.
if self.modname:
self._package_path = util.find_package_path(self.modname)
else:
self._package_path = None
self._env_var = '{0}DIR'.format(self.appname.upper())
if read:
self.read()
def user_config_path(self):
"""Points to the location of the user configuration.
The file may not exist.
"""
return os.path.join(self.config_dir(), CONFIG_FILENAME)
def _add_user_source(self):
"""Add the configuration options from the YAML file in the
user's configuration directory (given by `config_dir`) if it
exists.
"""
filename = self.user_config_path()
self.add(YamlSource(filename, loader=self.loader, optional=True))
def _add_default_source(self):
"""Add the package's default configuration settings. This looks
for a YAML file located inside the package for the module
`modname` if it was given.
"""
if self.modname:
if self._package_path:
filename = os.path.join(self._package_path, DEFAULT_FILENAME)
self.add(YamlSource(filename, loader=self.loader,
optional=True, default=True))
def read(self, user=True, defaults=True):
"""Find and read the files for this configuration and set them
as the sources for this configuration. To disable either
discovered user configuration files or the in-package defaults,
set `user` or `defaults` to `False`.
"""
if user:
self._add_user_source()
if defaults:
self._add_default_source()
def config_dir(self):
"""Get the path to the user configuration directory. The
directory is guaranteed to exist as a postcondition (one may be
created if none exist).
If the application's ``...DIR`` environment variable is set, it
is used as the configuration directory. Otherwise,
platform-specific standard configuration locations are searched
for a ``config.yaml`` file. If no configuration file is found, a
fallback path is used.
"""
# If environment variable is set, use it.
if self._env_var in os.environ:
appdir = os.environ[self._env_var]
appdir = os.path.abspath(os.path.expanduser(appdir))
if os.path.isfile(appdir):
raise ConfigError(u'{0} must be a directory'.format(
self._env_var
))
else:
# Search platform-specific locations. If no config file is
# found, fall back to the first directory in the list.
configdirs = util.config_dirs()
for confdir in configdirs:
appdir = os.path.join(confdir, self.appname)
if os.path.isfile(os.path.join(appdir, CONFIG_FILENAME)):
break
else:
appdir = os.path.join(configdirs[0], self.appname)
# Ensure that the directory exists.
try:
os.makedirs(appdir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
return appdir
def set_file(self, filename, base_for_paths=False):
"""Parses the file as YAML and inserts it into the configuration
sources with highest priority.
:param filename: Filename of the YAML file to load.
:param base_for_paths: Indicates whether the directory containing the
YAML file will be used as the base directory for resolving relative
path values stored in the YAML file. Otherwise, by default, the
directory returned by `config_dir()` will be used as the base.
"""
self.set(YamlSource(filename, base_for_paths=base_for_paths,
loader=self.loader))
def set_env(self, prefix=None, sep='__'):
"""Create a configuration overlay at the highest priority from
environment variables.
After prefix matching and removal, environment variable names will be
converted to lowercase for use as keys within the configuration. If
there are nested keys, list-like dicts (ie, `{0: 'a', 1: 'b'}`) will
be converted into corresponding lists (ie, `['a', 'b']`). The values
of all environment variables will be parsed as YAML scalars using the
`self.loader` Loader class to ensure type conversion is consistent
with YAML file sources. Use the `EnvSource` class directly to load
environment variables using non-default behavior and to enable full
YAML parsing of values.
:param prefix: The prefix to identify the environment variables to use.
Defaults to uppercased `self.appname` followed by an underscore.
:param sep: Separator within variable names to define nested keys.
"""
if prefix is None:
prefix = '{0}_'.format(self.appname.upper())
self.set(EnvSource(prefix, sep=sep, loader=self.loader))
def dump(self, full=True, redact=False):
"""Dump the Configuration object to a YAML file.
The order of the keys is determined from the default
configuration file. All keys not in the default configuration
will be appended to the end of the file.
:param full: Dump settings that don't differ from the defaults
as well
:param redact: Remove sensitive information (views with the `redact`
flag set) from the output
"""
if full:
out_dict = self.flatten(redact=redact)
else:
# Exclude defaults when flattening.
sources = [s for s in self.sources if not s.default]
temp_root = RootView(sources)
temp_root.redactions = self.redactions
out_dict = temp_root.flatten(redact=redact)
yaml_out = yaml.dump(out_dict, Dumper=yaml_util.Dumper,
default_flow_style=None, indent=4,
width=1000)
# Restore comments to the YAML text.
default_source = None
for source in self.sources:
if source.default:
default_source = source
break
if default_source and default_source.filename:
with open(default_source.filename, 'rb') as fp:
default_data = fp.read()
yaml_out = yaml_util.restore_yaml_comments(
yaml_out, default_data.decode('utf-8'))
return yaml_out
def reload(self):
"""Reload all sources from the file system.
This only affects sources that come from files (i.e.,
`YamlSource` objects); other sources, such as dictionaries
inserted with `add` or `set`, will remain unchanged.
"""
for source in self.sources:
if isinstance(source, YamlSource):
source.load()
class LazyConfig(Configuration):
"""A Configuration at reads files on demand when it is first
accessed. This is appropriate for using as a global config object at
the module level.
"""
def __init__(self, appname, modname=None):
super(LazyConfig, self).__init__(appname, modname, False)
self._materialized = False # Have we read the files yet?
self._lazy_prefix = [] # Pre-materialization calls to set().
self._lazy_suffix = [] # Calls to add().
def read(self, user=True, defaults=True):
self._materialized = True
super(LazyConfig, self).read(user, defaults)
def resolve(self):
if not self._materialized:
# Read files and unspool buffers.
self.read()
self.sources += self._lazy_suffix
self.sources[:0] = self._lazy_prefix
return super(LazyConfig, self).resolve()
def add(self, value):
super(LazyConfig, self).add(value)
if not self._materialized:
# Buffer additions to end.
self._lazy_suffix += self.sources
del self.sources[:]
def set(self, value):
super(LazyConfig, self).set(value)
if not self._materialized:
# Buffer additions to beginning.
self._lazy_prefix[:0] = self.sources
del self.sources[:]
def clear(self):
"""Remove all sources from this configuration."""
super(LazyConfig, self).clear()
self._lazy_suffix = []
self._lazy_prefix = []
# "Validated" configuration views: experimental!

56
libs/common/confuse/exceptions.py Normal file
View file

@ -0,0 +1,56 @@
from __future__ import division, absolute_import, print_function
import yaml
__all__ = [
'ConfigError', 'NotFoundError', 'ConfigValueError', 'ConfigTypeError',
'ConfigTemplateError', 'ConfigReadError']
YAML_TAB_PROBLEM = "found character '\\t' that cannot start any token"
# Exceptions.
class ConfigError(Exception):
"""Base class for exceptions raised when querying a configuration.
"""
class NotFoundError(ConfigError):
"""A requested value could not be found in the configuration trees.
"""
class ConfigValueError(ConfigError):
"""The value in the configuration is illegal."""
class ConfigTypeError(ConfigValueError):
"""The value in the configuration did not match the expected type.
"""
class ConfigTemplateError(ConfigError):
"""Base class for exceptions raised because of an invalid template.
"""
class ConfigReadError(ConfigError):
"""A configuration source could not be read."""
def __init__(self, name, reason=None):
self.name = name
self.reason = reason
message = u'{0} could not be read'.format(name)
if (isinstance(reason, yaml.scanner.ScannerError)
and reason.problem == YAML_TAB_PROBLEM):
# Special-case error message for tab indentation in YAML markup.
message += u': found tab character at line {0}, column {1}'.format(
reason.problem_mark.line + 1,
reason.problem_mark.column + 1,
)
elif reason:
# Generic error message uses exception's message.
message += u': {0}'.format(reason)
super(ConfigReadError, self).__init__(message)

184
libs/common/confuse/sources.py Normal file
View file

@ -0,0 +1,184 @@
from __future__ import division, absolute_import, print_function
from .util import BASESTRING, build_dict
from . import yaml_util
import os
class ConfigSource(dict):
"""A dictionary augmented with metadata about the source of the
configuration.
"""
def __init__(self, value, filename=None, default=False,
base_for_paths=False):
"""Create a configuration source from a dictionary.
:param filename: The file with the data for this configuration source.
:param default: Indicates whether this source provides the
application's default configuration settings.
:param base_for_paths: Indicates whether the source file's directory
(i.e., the directory component of `self.filename`) should be used as
the base directory for resolving relative path values provided by this
source, instead of using the application's configuration directory. If
no `filename` is provided, `base_for_paths` will be treated as False.
See `templates.Filename` for details of the relative path resolution
behavior.
"""
super(ConfigSource, self).__init__(value)
if (filename is not None
and not isinstance(filename, BASESTRING)):
raise TypeError(u'filename must be a string or None')
self.filename = filename
self.default = default
self.base_for_paths = base_for_paths if filename is not None else False
def __repr__(self):
return 'ConfigSource({0!r}, {1!r}, {2!r}, {3!r})'.format(
super(ConfigSource, self),
self.filename,
self.default,
self.base_for_paths,
)
@classmethod
def of(cls, value):
"""Given either a dictionary or a `ConfigSource` object, return
a `ConfigSource` object. This lets a function accept either type
of object as an argument.
"""
if isinstance(value, ConfigSource):
return value
elif isinstance(value, dict):
return ConfigSource(value)
else:
raise TypeError(u'source value must be a dict')
class YamlSource(ConfigSource):
"""A configuration data source that reads from a YAML file.
"""
def __init__(self, filename=None, default=False, base_for_paths=False,
optional=False, loader=yaml_util.Loader):
"""Create a YAML data source by reading data from a file.
May raise a `ConfigReadError`. However, if `optional` is
enabled, this exception will not be raised in the case when the
file does not exist---instead, the source will be silently
empty.
"""
filename = os.path.abspath(filename)
super(YamlSource, self).__init__({}, filename, default, base_for_paths)
self.loader = loader
self.optional = optional
self.load()
def load(self):
"""Load YAML data from the source's filename.
"""
if self.optional and not os.path.isfile(self.filename):
value = {}
else:
value = yaml_util.load_yaml(self.filename,
loader=self.loader) or {}
self.update(value)
class EnvSource(ConfigSource):
"""A configuration data source loaded from environment variables.
"""
def __init__(self, prefix, sep='__', lower=True, handle_lists=True,
parse_yaml_docs=False, loader=yaml_util.Loader):
"""Create a configuration source from the environment.
:param prefix: The prefix used to identify the environment variables
to be loaded into this configuration source.
:param sep: Separator within variable names to define nested keys.
:param lower: Indicates whether to convert variable names to lowercase
after prefix matching.
:param handle_lists: If variables are split into nested keys, indicates
whether to search for sub-dicts with keys that are sequential
integers starting from 0 and convert those dicts to lists.
:param parse_yaml_docs: Enable parsing the values of environment
variables as full YAML documents. By default, when False, values
are parsed only as YAML scalars.
:param loader: PyYAML Loader class to use to parse YAML values.
"""
super(EnvSource, self).__init__({}, filename=None, default=False,
base_for_paths=False)
self.prefix = prefix
self.sep = sep
self.lower = lower
self.handle_lists = handle_lists
self.parse_yaml_docs = parse_yaml_docs
self.loader = loader
self.load()
def load(self):
"""Load configuration data from the environment.
"""
# Read config variables with prefix from the environment.
config_vars = {}
for var, value in os.environ.items():
if var.startswith(self.prefix):
key = var[len(self.prefix):]
if self.lower:
key = key.lower()
if self.parse_yaml_docs:
# Parse the value as a YAML document, which will convert
# string representations of dicts and lists into the
# appropriate object (ie, '{foo: bar}' to {'foo': 'bar'}).
# Will raise a ConfigReadError if YAML parsing fails.
value = yaml_util.load_yaml_string(value,
'env variable ' + var,
loader=self.loader)
else:
# Parse the value as a YAML scalar so that values are type
# converted using the same rules as the YAML Loader (ie,
# numeric string to int/float, 'true' to True, etc.). Will
# not raise a ConfigReadError.
value = yaml_util.parse_as_scalar(value,
loader=self.loader)
config_vars[key] = value
if self.sep:
# Build a nested dict, keeping keys with `None` values to allow
# environment variables to unset values from lower priority sources
config_vars = build_dict(config_vars, self.sep, keep_none=True)
if self.handle_lists:
for k, v in config_vars.items():
config_vars[k] = self._convert_dict_lists(v)
self.update(config_vars)
@classmethod
def _convert_dict_lists(cls, obj):
"""Recursively search for dicts where all of the keys are integers
from 0 to the length of the dict, and convert them to lists.
"""
# We only deal with dictionaries
if not isinstance(obj, dict):
return obj
# Recursively search values for additional dicts to convert to lists
for k, v in obj.items():
obj[k] = cls._convert_dict_lists(v)
try:
# Convert the keys to integers, mapping the ints back to the keys
int_to_key = {int(k): k for k in obj.keys()}
except (ValueError):
# Not all of the keys represent integers
return obj
try:
# For the integers from 0 to the length of the dict, try to create
# a list from the dict values using the integer to key mapping
return [obj[int_to_key[i]] for i in range(len(obj))]
except (KeyError):
# At least one integer within the range is not a key of the dict
return obj

741
libs/common/confuse/templates.py Normal file
View file

@ -0,0 +1,741 @@
from __future__ import division, absolute_import, print_function
import os
import re
import sys
from . import util
from . import exceptions
try:
import enum
SUPPORTS_ENUM = True
except ImportError:
SUPPORTS_ENUM = False
try:
import pathlib
SUPPORTS_PATHLIB = True
except ImportError:
SUPPORTS_PATHLIB = False
if sys.version_info >= (3, 3):
from collections import abc
else:
import collections as abc
REQUIRED = object()
"""A sentinel indicating that there is no default value and an exception
should be raised when the value is missing.
"""
class Template(object):
"""A value template for configuration fields.
The template works like a type and instructs Confuse about how to
interpret a deserialized YAML value. This includes type conversions,
providing a default value, and validating for errors. For example, a
filepath type might expand tildes and check that the file exists.
"""
def __init__(self, default=REQUIRED):
"""Create a template with a given default value.
If `default` is the sentinel `REQUIRED` (as it is by default),
then an error will be raised when a value is missing. Otherwise,
missing values will instead return `default`.
"""
self.default = default
def __call__(self, view):
"""Invoking a template on a view gets the view's value according
to the template.
"""
return self.value(view, self)
def value(self, view, template=None):
"""Get the value for a `ConfigView`.
May raise a `NotFoundError` if the value is missing (and the
template requires it) or a `ConfigValueError` for invalid values.
"""
try:
value, _ = view.first()
return self.convert(value, view)
except exceptions.NotFoundError:
pass
# Get default value, or raise if required.
return self.get_default_value(view.name)
def get_default_value(self, key_name='default'):
"""Get the default value to return when the value is missing.
May raise a `NotFoundError` if the value is required.
"""
if not hasattr(self, 'default') or self.default is REQUIRED:
# The value is required. A missing value is an error.
raise exceptions.NotFoundError(u"{} not found".format(key_name))
# The value is not required.
return self.default
def convert(self, value, view):
"""Convert the YAML-deserialized value to a value of the desired
type.
Subclasses should override this to provide useful conversions.
May raise a `ConfigValueError` when the configuration is wrong.
"""
# Default implementation does no conversion.
return value
def fail(self, message, view, type_error=False):
"""Raise an exception indicating that a value cannot be
accepted.
`type_error` indicates whether the error is due to a type
mismatch rather than a malformed value. In this case, a more
specific exception is raised.
"""
exc_class = (
exceptions.ConfigTypeError if type_error
else exceptions.ConfigValueError)
raise exc_class(u'{0}: {1}'.format(view.name, message))
def __repr__(self):
return '{0}({1})'.format(
type(self).__name__,
'' if self.default is REQUIRED else repr(self.default),
)
class Integer(Template):
"""An integer configuration value template.
"""
def convert(self, value, view):
"""Check that the value is an integer. Floats are rounded.
"""
if isinstance(value, int):
return value
elif isinstance(value, float):
return int(value)
else:
self.fail(u'must be a number', view, True)
class Number(Template):
"""A numeric type: either an integer or a floating-point number.
"""
def convert(self, value, view):
"""Check that the value is an int or a float.
"""
if isinstance(value, util.NUMERIC_TYPES):
return value
else:
self.fail(
u'must be numeric, not {0}'.format(type(value).__name__),
view,
True
)
class MappingTemplate(Template):
"""A template that uses a dictionary to specify other types for the
values for a set of keys and produce a validated `AttrDict`.
"""
def __init__(self, mapping):
"""Create a template according to a dict (mapping). The
mapping's values should themselves either be Types or
convertible to Types.
"""
subtemplates = {}
for key, typ in mapping.items():
subtemplates[key] = as_template(typ)
self.subtemplates = subtemplates
def value(self, view, template=None):
"""Get a dict with the same keys as the template and values
validated according to the value types.
"""
out = AttrDict()
for key, typ in self.subtemplates.items():
out[key] = typ.value(view[key], self)
return out
def __repr__(self):
return 'MappingTemplate({0})'.format(repr(self.subtemplates))
class Sequence(Template):
"""A template used to validate lists of similar items,
based on a given subtemplate.
"""
def __init__(self, subtemplate):
"""Create a template for a list with items validated
on a given subtemplate.
"""
self.subtemplate = as_template(subtemplate)
def value(self, view, template=None):
"""Get a list of items validated against the template.
"""
out = []
for item in view.sequence():
out.append(self.subtemplate.value(item, self))
return out
def __repr__(self):
return 'Sequence({0})'.format(repr(self.subtemplate))
class MappingValues(Template):
"""A template used to validate mappings of similar items,
based on a given subtemplate applied to the values.
All keys in the mapping are considered valid, but values
must pass validation by the subtemplate. Similar to the
Sequence template but for mappings.
"""
def __init__(self, subtemplate):
"""Create a template for a mapping with variable keys
and item values validated on a given subtemplate.
"""
self.subtemplate = as_template(subtemplate)
def value(self, view, template=None):
"""Get a dict with the same keys as the view and the
value of each item validated against the subtemplate.
"""
out = {}
for key, item in view.items():
out[key] = self.subtemplate.value(item, self)
return out
def __repr__(self):
return 'MappingValues({0})'.format(repr(self.subtemplate))
class String(Template):
"""A string configuration value template.
"""
def __init__(self, default=REQUIRED, pattern=None, expand_vars=False):
"""Create a template with the added optional `pattern` argument,
a regular expression string that the value should match.
"""
super(String, self).__init__(default)
self.pattern = pattern
self.expand_vars = expand_vars
if pattern:
self.regex = re.compile(pattern)
def __repr__(self):
args = []
if self.default is not REQUIRED:
args.append(repr(self.default))
if self.pattern is not None:
args.append('pattern=' + repr(self.pattern))
return 'String({0})'.format(', '.join(args))
def convert(self, value, view):
"""Check that the value is a string and matches the pattern.
"""
if not isinstance(value, util.BASESTRING):
self.fail(u'must be a string', view, True)
if self.pattern and not self.regex.match(value):
self.fail(
u"must match the pattern {0}".format(self.pattern),
view
)
if self.expand_vars:
return os.path.expandvars(value)
else:
return value
class Choice(Template):
"""A template that permits values from a sequence of choices.
"""
def __init__(self, choices, default=REQUIRED):
"""Create a template that validates any of the values from the
iterable `choices`.
If `choices` is a map, then the corresponding value is emitted.
Otherwise, the value itself is emitted.
If `choices` is a `Enum`, then the enum entry with the value is
emitted.
"""
super(Choice, self).__init__(default)
self.choices = choices
def convert(self, value, view):
"""Ensure that the value is among the choices (and remap if the
choices are a mapping).
"""
if (SUPPORTS_ENUM and isinstance(self.choices, type)
and issubclass(self.choices, enum.Enum)):
try:
return self.choices(value)
except ValueError:
self.fail(
u'must be one of {0!r}, not {1!r}'.format(
[c.value for c in self.choices], value
),
view
)
if value not in self.choices:
self.fail(
u'must be one of {0!r}, not {1!r}'.format(
list(self.choices), value
),
view
)
if isinstance(self.choices, abc.Mapping):
return self.choices[value]
else:
return value
def __repr__(self):
return 'Choice({0!r})'.format(self.choices)
class OneOf(Template):
"""A template that permits values complying to one of the given templates.
"""
def __init__(self, allowed, default=REQUIRED):
super(OneOf, self).__init__(default)
self.allowed = list(allowed)
def __repr__(self):
args = []
if self.allowed is not None:
args.append('allowed=' + repr(self.allowed))
if self.default is not REQUIRED:
args.append(repr(self.default))
return 'OneOf({0})'.format(', '.join(args))
def value(self, view, template):
self.template = template
return super(OneOf, self).value(view, template)
def convert(self, value, view):
"""Ensure that the value follows at least one template.
"""
is_mapping = isinstance(self.template, MappingTemplate)
for candidate in self.allowed:
try:
if is_mapping:
if isinstance(candidate, Filename) and \
candidate.relative_to:
next_template = candidate.template_with_relatives(
view,
self.template
)
next_template.subtemplates[view.key] = as_template(
candidate
)
else:
next_template = MappingTemplate({view.key: candidate})
return view.parent.get(next_template)[view.key]
else:
return view.get(candidate)
except exceptions.ConfigTemplateError:
raise
except exceptions.ConfigError:
pass
except ValueError as exc:
raise exceptions.ConfigTemplateError(exc)
self.fail(
u'must be one of {0}, not {1}'.format(
repr(self.allowed), repr(value)
),
view
)
class StrSeq(Template):
"""A template for values that are lists of strings.
Validates both actual YAML string lists and single strings. Strings
can optionally be split on whitespace.
"""
def __init__(self, split=True, default=REQUIRED):
"""Create a new template.
`split` indicates whether, when the underlying value is a single
string, it should be split on whitespace. Otherwise, the
resulting value is a list containing a single string.
"""
super(StrSeq, self).__init__(default)
self.split = split
def _convert_value(self, x, view):
if isinstance(x, util.STRING):
return x
elif isinstance(x, bytes):
return x.decode('utf-8', 'ignore')
else:
self.fail(u'must be a list of strings', view, True)
def convert(self, value, view):
if isinstance(value, bytes):
value = value.decode('utf-8', 'ignore')
if isinstance(value, util.STRING):
if self.split:
value = value.split()
else:
value = [value]
else:
try:
value = list(value)
except TypeError:
self.fail(u'must be a whitespace-separated string or a list',
view, True)
return [self._convert_value(v, view) for v in value]
class Pairs(StrSeq):
"""A template for ordered key-value pairs.
This can either be given with the same syntax as for `StrSeq` (i.e. without
values), or as a list of strings and/or single-element mappings such as::
- key: value
- [key, value]
- key
The result is a list of two-element tuples. If no value is provided, the
`default_value` will be returned as the second element.
"""
def __init__(self, default_value=None):
"""Create a new template.
`default` is the dictionary value returned for items that are not
a mapping, but a single string.
"""
super(Pairs, self).__init__(split=True)
self.default_value = default_value
def _convert_value(self, x, view):
try:
return (super(Pairs, self)._convert_value(x, view),
self.default_value)
except exceptions.ConfigTypeError:
if isinstance(x, abc.Mapping):
if len(x) != 1:
self.fail(u'must be a single-element mapping', view, True)
k, v = util.iter_first(x.items())
elif isinstance(x, abc.Sequence):
if len(x) != 2:
self.fail(u'must be a two-element list', view, True)
k, v = x
else:
# Is this even possible? -> Likely, if some !directive cause
# YAML to parse this to some custom type.
self.fail(u'must be a single string, mapping, or a list'
u'' + str(x),
view, True)
return (super(Pairs, self)._convert_value(k, view),
super(Pairs, self)._convert_value(v, view))
class Filename(Template):
"""A template that validates strings as filenames.
Filenames are returned as absolute, tilde-free paths.
Relative paths are relative to the template's `cwd` argument
when it is specified. Otherwise, if the paths come from a file,
they will be relative to the configuration directory (see the
`config_dir` method) by default or to the base directory of the
config file if either the source has `base_for_paths` set to True
or the template has `in_source_dir` set to True. Paths from sources
without a file are relative to the current working directory. This
helps attain the expected behavior when using command-line options.
"""
def __init__(self, default=REQUIRED, cwd=None, relative_to=None,
in_app_dir=False, in_source_dir=False):
"""`relative_to` is the name of a sibling value that is
being validated at the same time.
`in_app_dir` indicates whether the path should be resolved
inside the application's config directory (even when the setting
does not come from a file).
`in_source_dir` indicates whether the path should be resolved
relative to the directory containing the source file, if there is
one, taking precedence over the application's config directory.
"""
super(Filename, self).__init__(default)
self.cwd = cwd
self.relative_to = relative_to
self.in_app_dir = in_app_dir
self.in_source_dir = in_source_dir
def __repr__(self):
args = []
if self.default is not REQUIRED:
args.append(repr(self.default))
if self.cwd is not None:
args.append('cwd=' + repr(self.cwd))
if self.relative_to is not None:
args.append('relative_to=' + repr(self.relative_to))
if self.in_app_dir:
args.append('in_app_dir=True')
if self.in_source_dir:
args.append('in_source_dir=True')
return 'Filename({0})'.format(', '.join(args))
def resolve_relative_to(self, view, template):
if not isinstance(template, (abc.Mapping, MappingTemplate)):
# disallow config.get(Filename(relative_to='foo'))
raise exceptions.ConfigTemplateError(
u'relative_to may only be used when getting multiple values.'
)
elif self.relative_to == view.key:
raise exceptions.ConfigTemplateError(
u'{0} is relative to itself'.format(view.name)
)
elif self.relative_to not in view.parent.keys():
# self.relative_to is not in the config
self.fail(
(
u'needs sibling value "{0}" to expand relative path'
).format(self.relative_to),
view
)
old_template = {}
old_template.update(template.subtemplates)
# save time by skipping MappingTemplate's init loop
next_template = MappingTemplate({})
next_relative = self.relative_to
# gather all the needed templates and nothing else
while next_relative is not None:
try:
# pop to avoid infinite loop because of recursive
# relative paths
rel_to_template = old_template.pop(next_relative)
except KeyError:
if next_relative in template.subtemplates:
# we encountered this config key previously
raise exceptions.ConfigTemplateError((
u'{0} and {1} are recursively relative'
).format(view.name, self.relative_to))
else:
raise exceptions.ConfigTemplateError((
u'missing template for {0}, needed to expand {1}\'s'
u'relative path'
).format(self.relative_to, view.name))
next_template.subtemplates[next_relative] = rel_to_template
next_relative = rel_to_template.relative_to
return view.parent.get(next_template)[self.relative_to]
def value(self, view, template=None):
try:
path, source = view.first()
except exceptions.NotFoundError:
return self.get_default_value(view.name)
if not isinstance(path, util.BASESTRING):
self.fail(
u'must be a filename, not {0}'.format(type(path).__name__),
view,
True
)
path = os.path.expanduser(util.STRING(path))
if not os.path.isabs(path):
if self.cwd is not None:
# relative to the template's argument
path = os.path.join(self.cwd, path)
elif self.relative_to is not None:
path = os.path.join(
self.resolve_relative_to(view, template),
path,
)
elif ((source.filename and self.in_source_dir)
or (source.base_for_paths and not self.in_app_dir)):
# relative to the directory the source file is in.
path = os.path.join(os.path.dirname(source.filename), path)
elif source.filename or self.in_app_dir:
# From defaults: relative to the app's directory.
path = os.path.join(view.root().config_dir(), path)
return os.path.abspath(path)
class Path(Filename):
"""A template that validates strings as `pathlib.Path` objects.
Filenames are parsed equivalent to the `Filename` template and then
converted to `pathlib.Path` objects.
For Python 2 it returns the original path as returned by the `Filename`
template.
"""
def value(self, view, template=None):
value = super(Path, self).value(view, template)
if value is None:
return
import pathlib
return pathlib.Path(value)
class Optional(Template):
"""A template that makes a subtemplate optional.
If the value is present and not null, it must validate against the
subtemplate. However, if the value is null or missing, the template will
still validate, returning a default value. If `allow_missing` is False,
the template will not allow missing values while still permitting null.
"""
def __init__(self, subtemplate, default=None, allow_missing=True):
self.subtemplate = as_template(subtemplate)
if default is None:
# When no default is passed, try to use the subtemplate's
# default value as the default for this template
try:
default = self.subtemplate.get_default_value()
except exceptions.NotFoundError:
pass
self.default = default
self.allow_missing = allow_missing
def value(self, view, template=None):
try:
value, _ = view.first()
except exceptions.NotFoundError:
if self.allow_missing:
# Value is missing but not required
return self.default
# Value must be present even though it can be null. Raise an error.
raise exceptions.NotFoundError(u'{} not found'.format(view.name))
if value is None:
# None (ie, null) is always a valid value
return self.default
return self.subtemplate.value(view, self)
def __repr__(self):
return 'Optional({0}, {1}, allow_missing={2})'.format(
repr(self.subtemplate),
repr(self.default),
self.allow_missing,
)
class TypeTemplate(Template):
"""A simple template that checks that a value is an instance of a
desired Python type.
"""
def __init__(self, typ, default=REQUIRED):
"""Create a template that checks that the value is an instance
of `typ`.
"""
super(TypeTemplate, self).__init__(default)
self.typ = typ
def convert(self, value, view):
if not isinstance(value, self.typ):
self.fail(
u'must be a {0}, not {1}'.format(
self.typ.__name__,
type(value).__name__,
),
view,
True
)
return value
class AttrDict(dict):
"""A `dict` subclass that can be accessed via attributes (dot
notation) for convenience.
"""
def __getattr__(self, key):
if key in self:
return self[key]
else:
raise AttributeError(key)
def __setattr__(self, key, value):
self[key] = value
def as_template(value):
"""Convert a simple "shorthand" Python value to a `Template`.
"""
if isinstance(value, Template):
# If it's already a Template, pass it through.
return value
elif isinstance(value, abc.Mapping):
# Dictionaries work as templates.
return MappingTemplate(value)
elif value is int:
return Integer()
elif isinstance(value, int):
return Integer(value)
elif isinstance(value, type) and issubclass(value, util.BASESTRING):
return String()
elif isinstance(value, util.BASESTRING):
return String(value)
elif isinstance(value, set):
# convert to list to avoid hash related problems
return Choice(list(value))
elif (SUPPORTS_ENUM and isinstance(value, type)
and issubclass(value, enum.Enum)):
return Choice(value)
elif isinstance(value, list):
return OneOf(value)
elif value is float:
return Number()
elif isinstance(value, float):
return Number(value)
elif SUPPORTS_PATHLIB and isinstance(value, pathlib.PurePath):
return Path(value)
elif value is None:
return Template(None)
elif value is REQUIRED:
return Template()
elif value is dict:
return TypeTemplate(abc.Mapping)
elif value is list:
return TypeTemplate(abc.Sequence)
elif isinstance(value, type):
return TypeTemplate(value)
else:
raise ValueError(u'cannot convert to template: {0!r}'.format(value))

186
libs/common/confuse/util.py Normal file
View file

@ -0,0 +1,186 @@
from __future__ import division, absolute_import, print_function
import os
import sys
import argparse
import optparse
import platform
import pkgutil
PY3 = sys.version_info[0] == 3
STRING = str if PY3 else unicode # noqa: F821
BASESTRING = str if PY3 else basestring # noqa: F821
NUMERIC_TYPES = (int, float) if PY3 else (int, float, long) # noqa: F821
UNIX_DIR_FALLBACK = '~/.config'
WINDOWS_DIR_VAR = 'APPDATA'
WINDOWS_DIR_FALLBACK = '~\\AppData\\Roaming'
MAC_DIR = '~/Library/Application Support'
def iter_first(sequence):
"""Get the first element from an iterable or raise a ValueError if
the iterator generates no values.
"""
it = iter(sequence)
try:
return next(it)
except StopIteration:
raise ValueError()
def namespace_to_dict(obj):
"""If obj is argparse.Namespace or optparse.Values we'll return
a dict representation of it, else return the original object.
Redefine this method if using other parsers.
:param obj: *
:return:
:rtype: dict or *
"""
if isinstance(obj, (argparse.Namespace, optparse.Values)):
return vars(obj)
return obj
def build_dict(obj, sep='', keep_none=False):
"""Recursively builds a dictionary from an argparse.Namespace,
optparse.Values, or dict object.
Additionally, if `sep` is a non-empty string, the keys will be split
by `sep` and expanded into a nested dict. Keys with a `None` value
are dropped by default to avoid unsetting options but can be kept
by setting `keep_none` to `True`.
:param obj: Namespace, Values, or dict to iterate over. Other
values will simply be returned.
:type obj: argparse.Namespace or optparse.Values or dict or *
:param sep: Separator to use for splitting properties/keys of `obj`
for expansion into nested dictionaries.
:type sep: str
:param keep_none: Whether to keep keys whose value is `None`.
:type keep_none: bool
:return: A new dictionary or the value passed if obj was not a
dict, Namespace, or Values.
:rtype: dict or *
"""
# We expect our root object to be a dict, but it may come in as
# a namespace
obj = namespace_to_dict(obj)
# We only deal with dictionaries
if not isinstance(obj, dict):
return obj
# Get keys iterator
keys = obj.keys() if PY3 else obj.iterkeys()
if sep:
# Splitting keys by `sep` needs sorted keys to prevent parents
# from clobbering children
keys = sorted(list(keys))
output = {}
for key in keys:
value = obj[key]
if value is None and not keep_none: # Avoid unset options.
continue
save_to = output
result = build_dict(value, sep, keep_none)
if sep:
# Split keys by `sep` as this signifies nesting
split = key.split(sep)
if len(split) > 1:
# The last index will be the key we assign result to
key = split.pop()
# Build the dict tree if needed and change where
# we're saving to
for child_key in split:
if child_key in save_to and \
isinstance(save_to[child_key], dict):
save_to = save_to[child_key]
else:
# Clobber or create
save_to[child_key] = {}
save_to = save_to[child_key]
# Save
if key in save_to:
save_to[key].update(result)
else:
save_to[key] = result
return output
# Config file paths, including platform-specific paths and in-package
# defaults.
def find_package_path(name):
"""Returns the path to the package containing the named module or
None if the path could not be identified (e.g., if
``name == "__main__"``).
"""
# Based on get_root_path from Flask by Armin Ronacher.
loader = pkgutil.get_loader(name)
if loader is None or name == '__main__':
return None
if hasattr(loader, 'get_filename'):
filepath = loader.get_filename(name)
else:
# Fall back to importing the specified module.
__import__(name)
filepath = sys.modules[name].__file__
return os.path.dirname(os.path.abspath(filepath))
def xdg_config_dirs():
"""Returns a list of paths taken from the XDG_CONFIG_DIRS
and XDG_CONFIG_HOME environment varibables if they exist
"""
paths = []
if 'XDG_CONFIG_HOME' in os.environ:
paths.append(os.environ['XDG_CONFIG_HOME'])
if 'XDG_CONFIG_DIRS' in os.environ:
paths.extend(os.environ['XDG_CONFIG_DIRS'].split(':'))
else:
paths.append('/etc/xdg')
paths.append('/etc')
return paths
def config_dirs():
"""Return a platform-specific list of candidates for user
configuration directories on the system.
The candidates are in order of priority, from highest to lowest. The
last element is the "fallback" location to be used when no
higher-priority config file exists.
"""
paths = []
if platform.system() == 'Darwin':
paths.append(UNIX_DIR_FALLBACK)
paths.append(MAC_DIR)
paths.extend(xdg_config_dirs())
elif platform.system() == 'Windows':
paths.append(WINDOWS_DIR_FALLBACK)
if WINDOWS_DIR_VAR in os.environ:
paths.append(os.environ[WINDOWS_DIR_VAR])
else:
# Assume Unix.
paths.append(UNIX_DIR_FALLBACK)
paths.extend(xdg_config_dirs())
# Expand and deduplicate paths.
out = []
for path in paths:
path = os.path.abspath(os.path.expanduser(path))
if path not in out:
out.append(path)
return out

228
libs/common/confuse/yaml_util.py Normal file
View file

@ -0,0 +1,228 @@
from __future__ import division, absolute_import, print_function
from collections import OrderedDict
import yaml
from .exceptions import ConfigReadError
from .util import BASESTRING
# YAML loading.
class Loader(yaml.SafeLoader):
"""A customized YAML loader. This loader deviates from the official
YAML spec in a few convenient ways:
- All strings as are Unicode objects.
- All maps are OrderedDicts.
- Strings can begin with % without quotation.
"""
# All strings should be Unicode objects, regardless of contents.
def _construct_unicode(self, node):
return self.construct_scalar(node)
# Use ordered dictionaries for every YAML map.
# From https://gist.github.com/844388
def construct_yaml_map(self, node):
data = OrderedDict()
yield data
value = self.construct_mapping(node)
data.update(value)
def construct_mapping(self, node, deep=False):
if isinstance(node, yaml.MappingNode):
self.flatten_mapping(node)
else:
raise yaml.constructor.ConstructorError(
None, None,
u'expected a mapping node, but found %s' % node.id,
node.start_mark
)
mapping = OrderedDict()
for key_node, value_node in node.value:
key = self.construct_object(key_node, deep=deep)
try:
hash(key)
except TypeError as exc:
raise yaml.constructor.ConstructorError(
u'while constructing a mapping',
node.start_mark, 'found unacceptable key (%s)' % exc,
key_node.start_mark
)
value = self.construct_object(value_node, deep=deep)
mapping[key] = value
return mapping
# Allow bare strings to begin with %. Directives are still detected.
def check_plain(self):
plain = super(Loader, self).check_plain()
return plain or self.peek() == '%'
@staticmethod
def add_constructors(loader):
"""Modify a PyYAML Loader class to add extra constructors for strings
and maps. Call this method on a custom Loader class to make it behave
like Confuse's own Loader
"""
loader.add_constructor('tag:yaml.org,2002:str',
Loader._construct_unicode)
loader.add_constructor('tag:yaml.org,2002:map',
Loader.construct_yaml_map)
loader.add_constructor('tag:yaml.org,2002:omap',
Loader.construct_yaml_map)
Loader.add_constructors(Loader)
def load_yaml(filename, loader=Loader):
"""Read a YAML document from a file. If the file cannot be read or
parsed, a ConfigReadError is raised.
loader is the PyYAML Loader class to use to parse the YAML. By default,
this is Confuse's own Loader class, which is like SafeLoader with
extra constructors.
"""
try:
with open(filename, 'rb') as f:
return yaml.load(f, Loader=loader)
except (IOError, yaml.error.YAMLError) as exc:
raise ConfigReadError(filename, exc)
def load_yaml_string(yaml_string, name, loader=Loader):
"""Read a YAML document from a string. If the string cannot be parsed,
a ConfigReadError is raised.
`yaml_string` is a string to be parsed as a YAML document.
`name` is the name to use in error messages.
`loader` is the PyYAML Loader class to use to parse the YAML. By default,
this is Confuse's own Loader class, which is like SafeLoader with
extra constructors.
"""
try:
return yaml.load(yaml_string, Loader=loader)
except yaml.error.YAMLError as exc:
raise ConfigReadError(name, exc)
def parse_as_scalar(value, loader=Loader):
"""Parse a value as if it were a YAML scalar to perform type conversion
that is consistent with YAML documents.
`value` should be a string. Non-string inputs or strings that raise YAML
errors will be returned unchanged.
`Loader` is the PyYAML Loader class to use for parsing, defaulting to
Confuse's own Loader class.
Examples with the default Loader:
- '1' will return 1 as an integer
- '1.0' will return 1 as a float
- 'true' will return True
- The empty string '' will return None
"""
# We only deal with strings
if not isinstance(value, BASESTRING):
return value
try:
loader = loader('')
tag = loader.resolve(yaml.ScalarNode, value, (True, False))
node = yaml.ScalarNode(tag, value)
return loader.construct_object(node)
except yaml.error.YAMLError:
# Fallback to returning the value unchanged
return value
# YAML dumping.
class Dumper(yaml.SafeDumper):
"""A PyYAML Dumper that represents OrderedDicts as ordinary mappings
(in order, of course).
"""
# From http://pyyaml.org/attachment/ticket/161/use_ordered_dict.py
def represent_mapping(self, tag, mapping, flow_style=None):
value = []
node = yaml.MappingNode(tag, value, flow_style=flow_style)
if self.alias_key is not None:
self.represented_objects[self.alias_key] = node
best_style = False
if hasattr(mapping, 'items'):
mapping = list(mapping.items())
for item_key, item_value in mapping:
node_key = self.represent_data(item_key)
node_value = self.represent_data(item_value)
if not (isinstance(node_key, yaml.ScalarNode)
and not node_key.style):
best_style = False
if not (isinstance(node_value, yaml.ScalarNode)
and not node_value.style):
best_style = False
value.append((node_key, node_value))
if flow_style is None:
if self.default_flow_style is not None:
node.flow_style = self.default_flow_style
else:
node.flow_style = best_style
return node
def represent_list(self, data):
"""If a list has less than 4 items, represent it in inline style
(i.e. comma separated, within square brackets).
"""
node = super(Dumper, self).represent_list(data)
length = len(data)
if self.default_flow_style is None and length < 4:
node.flow_style = True
elif self.default_flow_style is None:
node.flow_style = False
return node
def represent_bool(self, data):
"""Represent bool as 'yes' or 'no' instead of 'true' or 'false'.
"""
if data:
value = u'yes'
else:
value = u'no'
return self.represent_scalar('tag:yaml.org,2002:bool', value)
def represent_none(self, data):
"""Represent a None value with nothing instead of 'none'.
"""
return self.represent_scalar('tag:yaml.org,2002:null', '')
Dumper.add_representer(OrderedDict, Dumper.represent_dict)
Dumper.add_representer(bool, Dumper.represent_bool)
Dumper.add_representer(type(None), Dumper.represent_none)
Dumper.add_representer(list, Dumper.represent_list)
def restore_yaml_comments(data, default_data):
"""Scan default_data for comments (we include empty lines in our
definition of comments) and place them before the same keys in data.
Only works with comments that are on one or more own lines, i.e.
not next to a yaml mapping.
"""
comment_map = dict()
default_lines = iter(default_data.splitlines())
for line in default_lines:
if not line:
comment = "\n"
elif line.startswith("#"):
comment = "{0}\n".format(line)
else:
continue
while True:
line = next(default_lines)
if line and not line.startswith("#"):
break
comment += "{0}\n".format(line)
key = line.split(':')[0].strip()
comment_map[key] = comment
out_lines = iter(data.splitlines())
out_data = ""
for line in out_lines:
key = line.split(':')[0].strip()
if key in comment_map:
out_data += comment_map[key]
out_data += "{0}\n".format(line)
return out_data