Updates vendored subliminal to 2.1.0

Updates rarfile to 3.1
Updates stevedore to 3.5.0
Updates appdirs to 1.4.4
Updates click to 8.1.3
Updates decorator to 5.1.1
Updates dogpile.cache to 1.1.8
Updates pbr to 5.11.0
Updates pysrt to 1.1.2
Updates pytz to 2022.6
Adds importlib-metadata version 3.1.1
Adds typing-extensions version 4.1.1
Adds zipp version 3.11.0

libs/common/pbr/build.py

@@ -0,0 +1,61 @@
+# Copyright 2021 Monty Taylor
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""pep-517 support
+
+Add::
+
+      [build-system]
+      requires = ["pbr>=5.7.0", "setuptools>=36.6.0", "wheel"]
+      build-backend = "pbr.build"
+
+to pyproject.toml to use this
+"""
+
+from setuptools import build_meta
+
+__all__ = [
+    'get_requires_for_build_sdist',
+    'get_requires_for_build_wheel',
+    'prepare_metadata_for_build_wheel',
+    'build_wheel',
+    'build_sdist',
+]
+
+
+def get_requires_for_build_wheel(config_settings=None):
+    return build_meta.get_requires_for_build_wheel(config_settings)
+
+
+def get_requires_for_build_sdist(config_settings=None):
+    return build_meta.get_requires_for_build_sdist(config_settings)
+
+
+def prepare_metadata_for_build_wheel(metadata_directory, config_settings=None):
+    return build_meta.prepare_metadata_for_build_wheel(
+        metadata_directory, config_settings)
+
+
+def build_wheel(
+    wheel_directory,
+    config_settings=None,
+    metadata_directory=None,
+):
+    return build_meta.build_wheel(
+        wheel_directory, config_settings, metadata_directory,
+    )
+
+
+def build_sdist(sdist_directory, config_settings=None):
+    return build_meta.build_sdist(sdist_directory, config_settings)

libs/common/pbr/builddoc.py

@@ -132,11 +132,11 @@ class LocalBuildDoc(setup_command.BuildDoc):
                 autoindex.write("   %s.rst\n" % module)
 
     def _sphinx_tree(self):
-            source_dir = self._get_source_dir()
-            cmd = ['-H', 'Modules', '-o', source_dir, '.']
-            if apidoc_use_padding:
-                cmd.insert(0, 'apidoc')
-            apidoc.main(cmd + self.autodoc_tree_excludes)
+        source_dir = self._get_source_dir()
+        cmd = ['-H', 'Modules', '-o', source_dir, '.']
+        if apidoc_use_padding:
+            cmd.insert(0, 'apidoc')
+        apidoc.main(cmd + self.autodoc_tree_excludes)
 
     def _sphinx_run(self):
         if not self.verbose:

libs/common/pbr/cmd/main.py

@@ -40,8 +40,11 @@ def get_sha(args):
 
 
 def get_info(args):
-    print("{name}\t{version}\t{released}\t{sha}".format(
-        **_get_info(args.name)))
+    if args.short:
+        print("{version}".format(**_get_info(args.name)))
+    else:
+        print("{name}\t{version}\t{released}\t{sha}".format(
+            **_get_info(args.name)))
 
 
 def _get_info(name):
@@ -86,7 +89,9 @@ def main():
         version=str(pbr.version.VersionInfo('pbr')))
 
     subparsers = parser.add_subparsers(
-        title='commands', description='valid commands', help='additional help')
+        title='commands', description='valid commands', help='additional help',
+        dest='cmd')
+    subparsers.required = True
 
     cmd_sha = subparsers.add_parser('sha', help='print sha of package')
     cmd_sha.set_defaults(func=get_sha)
@@ -96,6 +101,8 @@ def main():
         'info', help='print version info for package')
     cmd_info.set_defaults(func=get_info)
     cmd_info.add_argument('name', help='package to print info of')
+    cmd_info.add_argument('-s', '--short', action="store_true",
+                          help='only display package version')
 
     cmd_freeze = subparsers.add_parser(
         'freeze', help='print version info for all installed packages')

libs/common/pbr/core.py

@@ -61,6 +61,11 @@ else:
     integer_types = (int, long)  # noqa
 
 
+# We use this canary to detect whether the module has already been called,
+# in order to avoid recursion
+in_use = False
+
+
 def pbr(dist, attr, value):
     """Implements the actual pbr setup() keyword.
 
@@ -81,6 +86,16 @@ def pbr(dist, attr, value):
     not work well with distributions that do use a `Distribution` subclass.
     """
 
+    # Distribution.finalize_options() is what calls this method. That means
+    # there is potential for recursion here. Recursion seems to be an issue
+    # particularly when using PEP517 build-system configs without
+    # setup_requires in setup.py. We can avoid the recursion by setting
+    # this canary so we don't repeat ourselves.
+    global in_use
+    if in_use:
+        return
+    in_use = True
+
     if not value:
         return
     if isinstance(value, string_type):

libs/common/pbr/git.py

@@ -156,9 +156,9 @@ def _clean_changelog_message(msg):
     * Escapes '`' which is interpreted as a literal
     """
 
-    msg = msg.replace('*', '\*')
-    msg = msg.replace('_', '\_')
-    msg = msg.replace('`', '\`')
+    msg = msg.replace('*', r'\*')
+    msg = msg.replace('_', r'\_')
+    msg = msg.replace('`', r'\`')
 
     return msg
 
@@ -223,6 +223,11 @@ def _iter_log_inner(git_dir):
     presentation logic to the output - making it suitable for different
     uses.
 
+    .. caution:: this function risk to return a tag that doesn't exist really
+                 inside the git objects list due to replacement made
+                 to tag name to also list pre-release suffix.
+                 Compliant with the SemVer specification (e.g 1.2.3-rc1)
+
     :return: An iterator of (hash, tags_set, 1st_line) tuples.
     """
     log.info('[pbr] Generating ChangeLog')
@@ -248,7 +253,7 @@ def _iter_log_inner(git_dir):
             for tag_string in refname.split("refs/tags/")[1:]:
                 # git tag does not allow : or " " in tag names, so we split
                 # on ", " which is the separator between elements
-                candidate = tag_string.split(", ")[0]
+                candidate = tag_string.split(", ")[0].replace("-", ".")
                 if _is_valid_version(candidate):
                     tags.add(candidate)
 
@@ -271,13 +276,14 @@ def write_git_changelog(git_dir=None, dest_dir=os.path.curdir,
             changelog = _iter_changelog(changelog)
     if not changelog:
         return
+
     new_changelog = os.path.join(dest_dir, 'ChangeLog')
-    # If there's already a ChangeLog and it's not writable, just use it
-    if (os.path.exists(new_changelog)
-            and not os.access(new_changelog, os.W_OK)):
+    if os.path.exists(new_changelog) and not os.access(new_changelog, os.W_OK):
+        # If there's already a ChangeLog and it's not writable, just use it
         log.info('[pbr] ChangeLog not written (file already'
                  ' exists and it is not writeable)')
         return
+
     log.info('[pbr] Writing ChangeLog')
     with io.open(new_changelog, "w", encoding="utf-8") as changelog_file:
         for release, content in changelog:
@@ -292,13 +298,14 @@ def generate_authors(git_dir=None, dest_dir='.', option_dict=dict()):
                                              'SKIP_GENERATE_AUTHORS')
     if should_skip:
         return
+
     start = time.time()
     old_authors = os.path.join(dest_dir, 'AUTHORS.in')
     new_authors = os.path.join(dest_dir, 'AUTHORS')
-    # If there's already an AUTHORS file and it's not writable, just use it
-    if (os.path.exists(new_authors)
-            and not os.access(new_authors, os.W_OK)):
+    if os.path.exists(new_authors) and not os.access(new_authors, os.W_OK):
+        # If there's already an AUTHORS file and it's not writable, just use it
         return
+
     log.info('[pbr] Generating AUTHORS')
     ignore_emails = '((jenkins|zuul)@review|infra@lists|jenkins@openstack)'
     if git_dir is None:

libs/common/pbr/hooks/files.py

@@ -14,6 +14,7 @@
 # under the License.
 
 import os
+import shlex
 import sys
 
 from pbr import find_package
@@ -35,6 +36,21 @@ def get_man_section(section):
     return os.path.join(get_manpath(), 'man%s' % section)
 
 
+def unquote_path(path):
+    # unquote the full path, e.g: "'a/full/path'" becomes "a/full/path", also
+    # strip the quotes off individual path components because os.walk cannot
+    # handle paths like: "'i like spaces'/'another dir'", so we will pass it
+    # "i like spaces/another dir" instead.
+
+    if os.name == 'nt':
+        # shlex cannot handle paths that contain backslashes, treating those
+        # as escape characters.
+        path = path.replace("\\", "/")
+        return "".join(shlex.split(path)).replace("/", "\\")
+
+    return "".join(shlex.split(path))
+
+
 class FilesConfig(base.BaseConfig):
 
     section = 'files'
@@ -57,21 +73,28 @@ class FilesConfig(base.BaseConfig):
                 target = target.strip()
                 if not target.endswith(os.path.sep):
                     target += os.path.sep
-                for (dirpath, dirnames, fnames) in os.walk(source_prefix):
-                    finished.append(
-                        "%s = " % dirpath.replace(source_prefix, target))
+                unquoted_prefix = unquote_path(source_prefix)
+                unquoted_target = unquote_path(target)
+                for (dirpath, dirnames, fnames) in os.walk(unquoted_prefix):
+                    # As source_prefix is always matched, using replace with a
+                    # a limit of one is always going to replace the path prefix
+                    # and not accidentally replace some text in the middle of
+                    # the path
+                    new_prefix = dirpath.replace(unquoted_prefix,
+                                                 unquoted_target, 1)
+                    finished.append("'%s' = " % new_prefix)
                     finished.extend(
-                        [" %s" % os.path.join(dirpath, f) for f in fnames])
+                        [" '%s'" % os.path.join(dirpath, f) for f in fnames])
             else:
                 finished.append(line)
 
         self.data_files = "\n".join(finished)
 
     def add_man_path(self, man_path):
-        self.data_files = "%s\n%s =" % (self.data_files, man_path)
+        self.data_files = "%s\n'%s' =" % (self.data_files, man_path)
 
     def add_man_page(self, man_page):
-        self.data_files = "%s\n  %s" % (self.data_files, man_page)
+        self.data_files = "%s\n  '%s'" % (self.data_files, man_page)
 
     def get_man_sections(self):
         man_sections = dict()

libs/common/pbr/options.py

@@ -48,6 +48,6 @@ TRUE_VALUES = ('true', '1', 'yes')
 
 
 def get_boolean_option(option_dict, option_name, env_name):
-    return ((option_name in option_dict
-             and option_dict[option_name][1].lower() in TRUE_VALUES) or
+    return ((option_name in option_dict and
+             option_dict[option_name][1].lower() in TRUE_VALUES) or
             str(os.getenv(env_name)).lower() in TRUE_VALUES)

libs/common/pbr/packaging.py

@@ -22,6 +22,16 @@ from __future__ import unicode_literals
 
 from distutils.command import install as du_install
 from distutils import log
+
+# (hberaud) do not use six here to import urlparse
+# to keep this module free from external dependencies
+# to avoid cross dependencies errors on minimal system
+# free from dependencies.
+try:
+    from urllib.parse import urlparse
+except ImportError:
+    from urlparse import urlparse
+
 import email
 import email.errors
 import os
@@ -98,19 +108,31 @@ def get_reqs_from_files(requirements_files):
     return []
 
 
+def egg_fragment(match):
+    return re.sub(r'(?P<PackageName>[\w.-]+)-'
+                  r'(?P<GlobalVersion>'
+                  r'(?P<VersionTripple>'
+                  r'(?P<Major>0|[1-9][0-9]*)\.'
+                  r'(?P<Minor>0|[1-9][0-9]*)\.'
+                  r'(?P<Patch>0|[1-9][0-9]*)){1}'
+                  r'(?P<Tags>(?:\-'
+                  r'(?P<Prerelease>(?:(?=[0]{1}[0-9A-Za-z-]{0})(?:[0]{1})|'
+                  r'(?=[1-9]{1}[0-9]*[A-Za-z]{0})(?:[0-9]+)|'
+                  r'(?=[0-9]*[A-Za-z-]+[0-9A-Za-z-]*)(?:[0-9A-Za-z-]+)){1}'
+                  r'(?:\.(?=[0]{1}[0-9A-Za-z-]{0})(?:[0]{1})|'
+                  r'\.(?=[1-9]{1}[0-9]*[A-Za-z]{0})(?:[0-9]+)|'
+                  r'\.(?=[0-9]*[A-Za-z-]+[0-9A-Za-z-]*)'
+                  r'(?:[0-9A-Za-z-]+))*){1}){0,1}(?:\+'
+                  r'(?P<Meta>(?:[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))){0,1}))',
+                  r'\g<PackageName>>=\g<GlobalVersion>',
+                  match.groups()[-1])
+
+
 def parse_requirements(requirements_files=None, strip_markers=False):
 
     if requirements_files is None:
         requirements_files = get_requirements_files()
 
-    def egg_fragment(match):
-        # take a versioned egg fragment and return a
-        # versioned package requirement e.g.
-        # nova-1.2.3 becomes nova>=1.2.3
-        return re.sub(r'([\w.]+)-([\w.-]+)',
-                      r'\1>=\2',
-                      match.groups()[-1])
-
     requirements = []
     for line in get_reqs_from_files(requirements_files):
         # Ignore comments
@@ -118,7 +140,8 @@ def parse_requirements(requirements_files=None, strip_markers=False):
             continue
 
         # Ignore index URL lines
-        if re.match(r'^\s*(-i|--index-url|--extra-index-url).*', line):
+        if re.match(r'^\s*(-i|--index-url|--extra-index-url|--find-links).*',
+                    line):
             continue
 
         # Handle nested requirements files such as:
@@ -140,16 +163,19 @@ def parse_requirements(requirements_files=None, strip_markers=False):
         # -e git://github.com/openstack/nova/master#egg=nova
         # -e git://github.com/openstack/nova/master#egg=nova-1.2.3
         # -e git+https://foo.com/zipball#egg=bar&subdirectory=baz
-        if re.match(r'\s*-e\s+', line):
-            line = re.sub(r'\s*-e\s+.*#egg=([^&]+).*$', egg_fragment, line)
-        # such as:
         # http://github.com/openstack/nova/zipball/master#egg=nova
         # http://github.com/openstack/nova/zipball/master#egg=nova-1.2.3
         # git+https://foo.com/zipball#egg=bar&subdirectory=baz
-        elif re.match(r'\s*(https?|git(\+(https|ssh))?):', line):
-            line = re.sub(r'\s*(https?|git(\+(https|ssh))?):.*#egg=([^&]+).*$',
-                          egg_fragment, line)
+        # git+[ssh]://github.com/openstack/nova/zipball/master#egg=nova-1.2.3
+        # hg+[ssh]://github.com/openstack/nova/zipball/master#egg=nova-1.2.3
+        # svn+[proto]://github.com/openstack/nova/zipball/master#egg=nova-1.2.3
         # -f lines are for index locations, and don't get used here
+        if re.match(r'\s*-e\s+', line):
+            extract = re.match(r'\s*-e\s+(.*)$', line)
+            line = extract.group(1)
+        egg = urlparse(line)
+        if egg.scheme:
+            line = re.sub(r'egg=([^&]+).*$', egg_fragment, egg.fragment)
         elif re.match(r'\s*-f\s+', line):
             line = None
             reason = 'Index Location'
@@ -183,7 +209,7 @@ def parse_dependency_links(requirements_files=None):
         if re.match(r'\s*-[ef]\s+', line):
             dependency_links.append(re.sub(r'\s*-[ef]\s+', '', line))
         # lines that are only urls can go in unmolested
-        elif re.match(r'\s*(https?|git(\+(https|ssh))?):', line):
+        elif re.match(r'^\s*(https?|git(\+(https|ssh))?|svn|hg)\S*:', line):
             dependency_links.append(line)
     return dependency_links
 
@@ -302,6 +328,7 @@ except ImportError:
 def have_nose():
     return _have_nose
 
+
 _wsgi_text = """#PBR Generated from %(group)r
 
 import threading
@@ -404,9 +431,13 @@ def generate_script(group, entry_point, header, template):
 
 
 def override_get_script_args(
-        dist, executable=os.path.normpath(sys.executable), is_wininst=False):
+        dist, executable=os.path.normpath(sys.executable)):
     """Override entrypoints console_script."""
-    header = easy_install.get_script_header("", executable, is_wininst)
+    # get_script_header() is deprecated since Setuptools 12.0
+    try:
+        header = easy_install.ScriptWriter.get_header("", executable)
+    except AttributeError:
+        header = easy_install.get_script_header("", executable)
     for group, template in ENTRY_POINTS_MAP.items():
         for name, ep in dist.get_entry_map(group).items():
             yield (name, generate_script(group, ep, header, template))
@@ -428,8 +459,12 @@ class LocalInstallScripts(install_scripts.install_scripts):
     """Intercepts console scripts entry_points."""
     command_name = 'install_scripts'
 
-    def _make_wsgi_scripts_only(self, dist, executable, is_wininst):
-        header = easy_install.get_script_header("", executable, is_wininst)
+    def _make_wsgi_scripts_only(self, dist, executable):
+        # get_script_header() is deprecated since Setuptools 12.0
+        try:
+            header = easy_install.ScriptWriter.get_header("", executable)
+        except AttributeError:
+            header = easy_install.get_script_header("", executable)
         wsgi_script_template = ENTRY_POINTS_MAP['wsgi_scripts']
         for name, ep in dist.get_entry_map('wsgi_scripts').items():
             content = generate_script(
@@ -455,16 +490,12 @@ class LocalInstallScripts(install_scripts.install_scripts):
         bs_cmd = self.get_finalized_command('build_scripts')
         executable = getattr(
             bs_cmd, 'executable', easy_install.sys_executable)
-        is_wininst = getattr(
-            self.get_finalized_command("bdist_wininst"), '_is_running', False
-        )
-
         if 'bdist_wheel' in self.distribution.have_run:
             # We're building a wheel which has no way of generating mod_wsgi
             # scripts for us. Let's build them.
             # NOTE(sigmavirus24): This needs to happen here because, as the
             # comment below indicates, no_ep is True when building a wheel.
-            self._make_wsgi_scripts_only(dist, executable, is_wininst)
+            self._make_wsgi_scripts_only(dist, executable)
 
         if self.no_ep:
             # no_ep is True if we're installing into an .egg file or building
@@ -478,7 +509,7 @@ class LocalInstallScripts(install_scripts.install_scripts):
             get_script_args = easy_install.get_script_args
             executable = '"%s"' % executable
 
-        for args in get_script_args(dist, executable, is_wininst):
+        for args in get_script_args(dist, executable):
             self.write_script(*args)
 
 
@@ -550,8 +581,9 @@ class LocalEggInfo(egg_info.egg_info):
         else:
             log.info("[pbr] Reusing existing SOURCES.txt")
             self.filelist = egg_info.FileList()
-            for entry in open(manifest_filename, 'r').read().split('\n'):
-                self.filelist.append(entry)
+            with open(manifest_filename, 'r') as fil:
+                for entry in fil.read().split('\n'):
+                    self.filelist.append(entry)
 
 
 def _from_git(distribution):
@@ -626,6 +658,7 @@ class LocalSDist(sdist.sdist):
             self.filelist.sort()
         sdist.sdist.make_distribution(self)
 
+
 try:
     from pbr import builddoc
     _have_sphinx = True
@@ -659,12 +692,14 @@ def _get_increment_kwargs(git_dir, tag):
     # git log output affecting out ability to have working sem ver headers.
     changelog = git._run_git_command(['log', '--pretty=%B', version_spec],
                                      git_dir)
-    header_len = len('sem-ver:')
-    commands = [line[header_len:].strip() for line in changelog.split('\n')
-                if line.lower().startswith('sem-ver:')]
     symbols = set()
-    for command in commands:
-        symbols.update([symbol.strip() for symbol in command.split(',')])
+    header = 'sem-ver:'
+    for line in changelog.split("\n"):
+        line = line.lower().strip()
+        if not line.lower().strip().startswith(header):
+            continue
+        new_symbols = line[len(header):].strip().split(",")
+        symbols.update([symbol.strip() for symbol in new_symbols])
 
     def _handle_symbol(symbol, symbols, impact):
         if symbol in symbols:
@@ -791,12 +826,9 @@ def _get_version_from_pkg_metadata(package_name):
     pkg_metadata = {}
     for filename in pkg_metadata_filenames:
         try:
-            pkg_metadata_file = open(filename, 'r')
-        except (IOError, OSError):
-            continue
-        try:
-            pkg_metadata = email.message_from_file(pkg_metadata_file)
-        except email.errors.MessageError:
+            with open(filename, 'r') as pkg_metadata_file:
+                pkg_metadata = email.message_from_file(pkg_metadata_file)
+        except (IOError, OSError, email.errors.MessageError):
             continue
 
     # Check to make sure we're in our own dir

libs/common/pbr/tests/base.py

@@ -187,7 +187,9 @@ class CapturedSubprocess(fixtures.Fixture):
         self.addDetail(self.label + '-stderr', content.text_content(self.err))
         self.returncode = proc.returncode
         if proc.returncode:
-            raise AssertionError('Failed process %s' % proc.returncode)
+            raise AssertionError(
+                'Failed process args=%r, kwargs=%r, returncode=%s' % (
+                    self.args, self.kwargs, proc.returncode))
         self.addCleanup(delattr, self, 'out')
         self.addCleanup(delattr, self, 'err')
         self.addCleanup(delattr, self, 'returncode')
@@ -200,12 +202,15 @@ def _run_cmd(args, cwd):
     :param cwd: The directory to run the comamnd in.
     :return: ((stdout, stderr), returncode)
     """
+    print('Running %s' % ' '.join(args))
     p = subprocess.Popen(
         args, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
         stderr=subprocess.PIPE, cwd=cwd)
     streams = tuple(s.decode('latin1').strip() for s in p.communicate())
-    for stream_content in streams:
-        print(stream_content)
+    print('STDOUT:')
+    print(streams[0])
+    print('STDERR:')
+    print(streams[1])
     return (streams) + (p.returncode,)
 
 

libs/common/pbr/tests/test_commands.py

@@ -78,7 +78,7 @@ class TestCommands(base.BaseTestCase):
         stdout, stderr, return_code = self.run_pbr('freeze')
         self.assertEqual(0, return_code)
         pkgs = []
-        for l in stdout.split('\n'):
-            pkgs.append(l.split('==')[0].lower())
+        for line in stdout.split('\n'):
+            pkgs.append(line.split('==')[0].lower())
         pkgs_sort = sorted(pkgs[:])
         self.assertEqual(pkgs_sort, pkgs)

libs/common/pbr/tests/test_core.py

@@ -40,6 +40,7 @@
 
 import glob
 import os
+import sys
 import tarfile
 
 import fixtures
@@ -74,7 +75,7 @@ class TestCore(base.BaseTestCase):
 
         self.run_setup('egg_info')
         stdout, _, _ = self.run_setup('--keywords')
-        assert stdout == 'packaging,distutils,setuptools'
+        assert stdout == 'packaging, distutils, setuptools'
 
     def test_setup_py_build_sphinx(self):
         stdout, _, return_code = self.run_setup('build_sphinx')
@@ -113,6 +114,12 @@ class TestCore(base.BaseTestCase):
     def test_console_script_develop(self):
         """Test that we develop a non-pkg-resources console script."""
 
+        if sys.version_info < (3, 0):
+            self.skipTest(
+                'Fails with recent virtualenv due to '
+                'https://github.com/pypa/virtualenv/issues/1638'
+            )
+
         if os.name == 'nt':
             self.skipTest('Windows support is passthrough')
 

libs/common/pbr/tests/test_files.py

@@ -35,17 +35,31 @@ class FilesConfigTest(base.BaseTestCase):
             ])
         self.useFixture(pkg_fixture)
         pkg_etc = os.path.join(pkg_fixture.base, 'etc')
+        pkg_ansible = os.path.join(pkg_fixture.base, 'ansible',
+                                   'kolla-ansible', 'test')
+        dir_spcs = os.path.join(pkg_fixture.base, 'dir with space')
+        dir_subdir_spc = os.path.join(pkg_fixture.base, 'multi space',
+                                      'more spaces')
         pkg_sub = os.path.join(pkg_etc, 'sub')
         subpackage = os.path.join(
             pkg_fixture.base, 'fake_package', 'subpackage')
         os.makedirs(pkg_sub)
         os.makedirs(subpackage)
+        os.makedirs(pkg_ansible)
+        os.makedirs(dir_spcs)
+        os.makedirs(dir_subdir_spc)
         with open(os.path.join(pkg_etc, "foo"), 'w') as foo_file:
             foo_file.write("Foo Data")
         with open(os.path.join(pkg_sub, "bar"), 'w') as foo_file:
             foo_file.write("Bar Data")
+        with open(os.path.join(pkg_ansible, "baz"), 'w') as baz_file:
+            baz_file.write("Baz Data")
         with open(os.path.join(subpackage, "__init__.py"), 'w') as foo_file:
             foo_file.write("# empty")
+        with open(os.path.join(dir_spcs, "file with spc"), 'w') as spc_file:
+            spc_file.write("# empty")
+        with open(os.path.join(dir_subdir_spc, "file with spc"), 'w') as file_:
+            file_.write("# empty")
 
         self.useFixture(base.DiveDir(pkg_fixture.base))
 
@@ -74,5 +88,61 @@ class FilesConfigTest(base.BaseTestCase):
         )
         files.FilesConfig(config, 'fake_package').run()
         self.assertIn(
-            '\netc/pbr/ = \n etc/foo\netc/pbr/sub = \n etc/sub/bar',
+            "\n'etc/pbr/' = \n 'etc/foo'\n'etc/pbr/sub' = \n 'etc/sub/bar'",
+            config['files']['data_files'])
+
+    def test_data_files_with_spaces(self):
+        config = dict(
+            files=dict(
+                data_files="\n  'i like spaces' = 'dir with space'/*"
+            )
+        )
+        files.FilesConfig(config, 'fake_package').run()
+        self.assertIn(
+            "\n'i like spaces/' = \n 'dir with space/file with spc'",
+            config['files']['data_files'])
+
+    def test_data_files_with_spaces_subdirectories(self):
+        # test that we can handle whitespace in subdirectories
+        data_files = "\n 'one space/two space' = 'multi space/more spaces'/*"
+        expected = (
+            "\n'one space/two space/' = "
+            "\n 'multi space/more spaces/file with spc'")
+        config = dict(
+            files=dict(
+                data_files=data_files
+            )
+        )
+        files.FilesConfig(config, 'fake_package').run()
+        self.assertIn(expected, config['files']['data_files'])
+
+    def test_data_files_with_spaces_quoted_components(self):
+        # test that we can quote individual path components
+        data_files = (
+            "\n'one space'/'two space' = 'multi space'/'more spaces'/*"
+        )
+        expected = ("\n'one space/two space/' = "
+                    "\n 'multi space/more spaces/file with spc'")
+        config = dict(
+            files=dict(
+                data_files=data_files
+            )
+        )
+        files.FilesConfig(config, 'fake_package').run()
+        self.assertIn(expected, config['files']['data_files'])
+
+    def test_data_files_globbing_source_prefix_in_directory_name(self):
+        # We want to test that the string, "docs", is not replaced in a
+        # subdirectory name, "sub-docs"
+        config = dict(
+            files=dict(
+                data_files="\n  share/ansible = ansible/*"
+            )
+        )
+        files.FilesConfig(config, 'fake_package').run()
+        self.assertIn(
+            "\n'share/ansible/' = "
+            "\n'share/ansible/kolla-ansible' = "
+            "\n'share/ansible/kolla-ansible/test' = "
+            "\n 'ansible/kolla-ansible/test/baz'",
             config['files']['data_files'])

libs/common/pbr/tests/test_integration.py

@@ -11,7 +11,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+try:
+    import configparser
+except ImportError:
+    import ConfigParser as configparser
 import os.path
+import pkg_resources
 import shlex
 import sys
 
@@ -77,19 +82,35 @@ class TestIntegration(base.BaseTestCase):
         # We don't break these into separate tests because we'd need separate
         # source dirs to isolate from side effects of running pip, and the
         # overheads of setup would start to beat the benefits of parallelism.
-        self.useFixture(base.CapturedSubprocess(
-            'sync-req',
-            ['python', 'update.py', os.path.join(REPODIR, self.short_name)],
-            cwd=os.path.join(REPODIR, 'requirements')))
-        self.useFixture(base.CapturedSubprocess(
-            'commit-requirements',
-            'git diff --quiet || git commit -amrequirements',
-            cwd=os.path.join(REPODIR, self.short_name), shell=True))
-        path = os.path.join(
-            self.useFixture(fixtures.TempDir()).path, 'project')
-        self.useFixture(base.CapturedSubprocess(
-            'clone',
-            ['git', 'clone', os.path.join(REPODIR, self.short_name), path]))
+        path = os.path.join(REPODIR, self.short_name)
+        setup_cfg = os.path.join(path, 'setup.cfg')
+        project_name = pkg_resources.safe_name(self.short_name).lower()
+        # These projects should all have setup.cfg files but we'll be careful
+        if os.path.exists(setup_cfg):
+            config = configparser.ConfigParser()
+            config.read(setup_cfg)
+            if config.has_section('metadata'):
+                raw_name = config.get('metadata', 'name',
+                                      fallback='notapackagename')
+                # Technically we should really only need to use the raw
+                # name because all our projects should be good and use
+                # normalized names but they don't...
+                project_name = pkg_resources.safe_name(raw_name).lower()
+        constraints = os.path.join(REPODIR, 'requirements',
+                                   'upper-constraints.txt')
+        tmp_constraints = os.path.join(
+            self.useFixture(fixtures.TempDir()).path,
+            'upper-constraints.txt')
+        # We need to filter out the package we are installing to avoid
+        # conflicts with the constraints.
+        with open(constraints, 'r') as src:
+            with open(tmp_constraints, 'w') as dest:
+                for line in src:
+                    constraint = line.split('===')[0]
+                    if project_name != constraint:
+                        dest.write(line)
+        pip_cmd = PIP_CMD + ['-c', tmp_constraints]
+
         venv = self.useFixture(
             test_packaging.Venv('sdist',
                                 modules=['pip', 'wheel', PBRVERSION],
@@ -105,7 +126,7 @@ class TestIntegration(base.BaseTestCase):
         filename = os.path.join(
             path, 'dist', os.listdir(os.path.join(path, 'dist'))[0])
         self.useFixture(base.CapturedSubprocess(
-            'tarball', [python] + PIP_CMD + [filename]))
+            'tarball', [python] + pip_cmd + [filename]))
         venv = self.useFixture(
             test_packaging.Venv('install-git',
                                 modules=['pip', 'wheel', PBRVERSION],
@@ -113,7 +134,7 @@ class TestIntegration(base.BaseTestCase):
         root = venv.path
         python = venv.python
         self.useFixture(base.CapturedSubprocess(
-            'install-git', [python] + PIP_CMD + ['git+file://' + path]))
+            'install-git', [python] + pip_cmd + ['git+file://' + path]))
         if self.short_name == 'nova':
             found = False
             for _, _, filenames in os.walk(root):
@@ -127,7 +148,7 @@ class TestIntegration(base.BaseTestCase):
         root = venv.path
         python = venv.python
         self.useFixture(base.CapturedSubprocess(
-            'install-e', [python] + PIP_CMD + ['-e', path]))
+            'install-e', [python] + pip_cmd + ['-e', path]))
 
 
 class TestInstallWithoutPbr(base.BaseTestCase):
@@ -188,12 +209,16 @@ class TestInstallWithoutPbr(base.BaseTestCase):
 class TestMarkersPip(base.BaseTestCase):
 
     scenarios = [
-        ('pip-1.5', {'modules': ['pip>=1.5,<1.6']}),
-        ('pip-6.0', {'modules': ['pip>=6.0,<6.1']}),
         ('pip-latest', {'modules': ['pip']}),
-        ('setuptools-EL7', {'modules': ['pip==1.4.1', 'setuptools==0.9.8']}),
-        ('setuptools-Trusty', {'modules': ['pip==1.5', 'setuptools==2.2']}),
-        ('setuptools-minimum', {'modules': ['pip==1.5', 'setuptools==0.7.2']}),
+        ('setuptools-Bionic', {
+            'modules': ['pip==9.0.1', 'setuptools==39.0.1']}),
+        ('setuptools-Stretch', {
+            'modules': ['pip==9.0.1', 'setuptools==33.1.1']}),
+        ('setuptools-EL8', {'modules': ['pip==9.0.3', 'setuptools==39.2.0']}),
+        ('setuptools-Buster', {
+            'modules': ['pip==18.1', 'setuptools==40.8.0']}),
+        ('setuptools-Focal', {
+            'modules': ['pip==20.0.2', 'setuptools==45.2.0']}),
     ]
 
     @testtools.skipUnless(
@@ -240,25 +265,17 @@ class TestLTSSupport(base.BaseTestCase):
     # These versions come from the versions installed from the 'virtualenv'
     # command from the 'python-virtualenv' package.
     scenarios = [
-        ('EL7', {'modules': ['pip==1.4.1', 'setuptools==0.9.8'],
-                 'py3support': True}),  # And EPEL6
-        ('Trusty', {'modules': ['pip==1.5', 'setuptools==2.2'],
-                    'py3support': True}),
-        ('Jessie', {'modules': ['pip==1.5.6', 'setuptools==5.5.1'],
-                    'py3support': True}),
-        # Wheezy has pip1.1, which cannot be called with '-m pip'
-        # So we'll use a different version of pip here.
-        ('WheezyPrecise', {'modules': ['pip==1.4.1', 'setuptools==0.6c11'],
-                           'py3support': False})
+        ('Bionic', {'modules': ['pip==9.0.1', 'setuptools==39.0.1']}),
+        ('Stretch', {'modules': ['pip==9.0.1', 'setuptools==33.1.1']}),
+        ('EL8', {'modules': ['pip==9.0.3', 'setuptools==39.2.0']}),
+        ('Buster', {'modules': ['pip==18.1', 'setuptools==40.8.0']}),
+        ('Focal', {'modules': ['pip==20.0.2', 'setuptools==45.2.0']}),
     ]
 
     @testtools.skipUnless(
         os.environ.get('PBR_INTEGRATION', None) == '1',
         'integration tests not enabled')
     def test_lts_venv_default_versions(self):
-        if (sys.version_info[0] == 3 and not self.py3support):
-            self.skipTest('This combination will not install with py3, '
-                          'skipping test')
         venv = self.useFixture(
             test_packaging.Venv('setuptools', modules=self.modules))
         bin_python = venv.python

libs/common/pbr/tests/test_packaging.py

@@ -48,7 +48,10 @@ import tempfile
 import textwrap
 
 import fixtures
-import mock
+try:
+    from unittest import mock
+except ImportError:
+    import mock
 import pkg_resources
 import six
 import testscenarios
@@ -108,7 +111,7 @@ class GPGKeyFixture(fixtures.Fixture):
     def setUp(self):
         super(GPGKeyFixture, self).setUp()
         tempdir = self.useFixture(fixtures.TempDir())
-        gnupg_version_re = re.compile('^gpg\s.*\s([\d+])\.([\d+])\.([\d+])')
+        gnupg_version_re = re.compile(r'^gpg\s.*\s([\d+])\.([\d+])\.([\d+])')
         gnupg_version = base._run_cmd(['gpg', '--version'], tempdir.path)
         for line in gnupg_version[0].split('\n'):
             gnupg_version = gnupg_version_re.match(line)
@@ -120,9 +123,9 @@ class GPGKeyFixture(fixtures.Fixture):
         else:
             if gnupg_version is None:
                 gnupg_version = (0, 0, 0)
-        config_file = tempdir.path + '/key-config'
-        f = open(config_file, 'wt')
-        try:
+
+        config_file = os.path.join(tempdir.path, 'key-config')
+        with open(config_file, 'wt') as f:
             if gnupg_version[0] == 2 and gnupg_version[1] >= 1:
                 f.write("""
                 %no-protection
@@ -135,11 +138,9 @@ class GPGKeyFixture(fixtures.Fixture):
             Name-Comment: N/A
             Name-Email: example@example.com
             Expire-Date: 2d
-            Preferences: (setpref)
             %commit
             """)
-        finally:
-            f.close()
+
         # Note that --quick-random (--debug-quick-random in GnuPG 2.x)
         # does not have a corresponding preferences file setting and
         # must be passed explicitly on the command line instead
@@ -149,6 +150,7 @@ class GPGKeyFixture(fixtures.Fixture):
             gnupg_random = '--debug-quick-random'
         else:
             gnupg_random = ''
+
         base._run_cmd(
             ['gpg', '--gen-key', '--batch', gnupg_random, config_file],
             tempdir.path)
@@ -173,17 +175,17 @@ class Venv(fixtures.Fixture):
         """
         self._reason = reason
         if modules == ():
-            pbr = 'file://%s#egg=pbr' % PBR_ROOT
-            modules = ['pip', 'wheel', pbr]
+            modules = ['pip', 'wheel', 'build', PBR_ROOT]
         self.modules = modules
         if pip_cmd is None:
-            self.pip_cmd = ['-m', 'pip', 'install']
+            self.pip_cmd = ['-m', 'pip', '-v', 'install']
         else:
             self.pip_cmd = pip_cmd
 
     def _setUp(self):
         path = self.useFixture(fixtures.TempDir()).path
-        virtualenv.create_environment(path, clear=True)
+        virtualenv.cli_run([path])
+
         python = os.path.join(path, 'bin', 'python')
         command = [python] + self.pip_cmd + ['-U']
         if self.modules and len(self.modules) > 0:
@@ -293,23 +295,23 @@ class TestPackagingInGitRepoWithCommit(base.BaseTestCase):
         self.run_setup('sdist', allow_fail=False)
         with open(os.path.join(self.package_dir, 'ChangeLog'), 'r') as f:
             body = f.read()
-        self.assertIn('\*', body)
+        self.assertIn(r'\*', body)
 
     def test_changelog_handles_dead_links_in_commit(self):
         self.repo.commit(message_content="See os_ for to_do about qemu_.")
         self.run_setup('sdist', allow_fail=False)
         with open(os.path.join(self.package_dir, 'ChangeLog'), 'r') as f:
             body = f.read()
-        self.assertIn('os\_', body)
-        self.assertIn('to\_do', body)
-        self.assertIn('qemu\_', body)
+        self.assertIn(r'os\_', body)
+        self.assertIn(r'to\_do', body)
+        self.assertIn(r'qemu\_', body)
 
     def test_changelog_handles_backticks(self):
         self.repo.commit(message_content="Allow `openstack.org` to `work")
         self.run_setup('sdist', allow_fail=False)
         with open(os.path.join(self.package_dir, 'ChangeLog'), 'r') as f:
             body = f.read()
-        self.assertIn('\`', body)
+        self.assertIn(r'\`', body)
 
     def test_manifest_exclude_honoured(self):
         self.run_setup('sdist', allow_fail=False)
@@ -379,6 +381,12 @@ class TestPackagingWheels(base.BaseTestCase):
         wheel_file.extractall(self.extracted_wheel_dir)
         wheel_file.close()
 
+    def test_metadata_directory_has_pbr_json(self):
+        # Build the path to the scripts directory
+        pbr_json = os.path.join(
+            self.extracted_wheel_dir, 'pbr_testpackage-0.0.dist-info/pbr.json')
+        self.assertTrue(os.path.exists(pbr_json))
+
     def test_data_directory_has_wsgi_scripts(self):
         # Build the path to the scripts directory
         scripts_dir = os.path.join(
@@ -531,11 +539,13 @@ class ParseRequirementsTest(base.BaseTestCase):
         tempdir = tempfile.mkdtemp()
         requirements = os.path.join(tempdir, 'requirements.txt')
         with open(requirements, 'w') as f:
-            f.write('-i https://myindex.local')
-            f.write('  --index-url https://myindex.local')
-            f.write(' --extra-index-url https://myindex.local')
+            f.write('-i https://myindex.local\n')
+            f.write('  --index-url https://myindex.local\n')
+            f.write(' --extra-index-url https://myindex.local\n')
+            f.write('--find-links https://myindex.local\n')
+            f.write('arequirement>=1.0\n')
         result = packaging.parse_requirements([requirements])
-        self.assertEqual([], result)
+        self.assertEqual(['arequirement>=1.0'], result)
 
     def test_nested_requirements(self):
         tempdir = tempfile.mkdtemp()
@@ -662,12 +672,65 @@ class TestVersions(base.BaseTestCase):
         version = packaging._get_version_from_git()
         self.assertThat(version, matchers.StartsWith('2.0.0.dev1'))
 
+    def test_multi_inline_symbols_no_space(self):
+        self.repo.commit()
+        self.repo.tag('1.2.3')
+        self.repo.commit('Sem-ver: feature,api-break')
+        version = packaging._get_version_from_git()
+        self.assertThat(version, matchers.StartsWith('2.0.0.dev1'))
+
+    def test_multi_inline_symbols_spaced(self):
+        self.repo.commit()
+        self.repo.tag('1.2.3')
+        self.repo.commit('Sem-ver: feature, api-break')
+        version = packaging._get_version_from_git()
+        self.assertThat(version, matchers.StartsWith('2.0.0.dev1'))
+
+    def test_multi_inline_symbols_reversed(self):
+        self.repo.commit()
+        self.repo.tag('1.2.3')
+        self.repo.commit('Sem-ver: api-break,feature')
+        version = packaging._get_version_from_git()
+        self.assertThat(version, matchers.StartsWith('2.0.0.dev1'))
+
+    def test_leading_space(self):
+        self.repo.commit()
+        self.repo.tag('1.2.3')
+        self.repo.commit('   sem-ver: api-break')
+        version = packaging._get_version_from_git()
+        self.assertThat(version, matchers.StartsWith('2.0.0.dev1'))
+
+    def test_leading_space_multiline(self):
+        self.repo.commit()
+        self.repo.tag('1.2.3')
+        self.repo.commit(
+            (
+                '   Some cool text\n'
+                '   sem-ver: api-break'
+            )
+        )
+        version = packaging._get_version_from_git()
+        self.assertThat(version, matchers.StartsWith('2.0.0.dev1'))
+
+    def test_leading_characters_symbol_not_found(self):
+        self.repo.commit()
+        self.repo.tag('1.2.3')
+        self.repo.commit('  ssem-ver: api-break')
+        version = packaging._get_version_from_git()
+        self.assertThat(version, matchers.StartsWith('1.2.4.dev1'))
+
     def test_tagged_version_has_tag_version(self):
         self.repo.commit()
         self.repo.tag('1.2.3')
         version = packaging._get_version_from_git('1.2.3')
         self.assertEqual('1.2.3', version)
 
+    def test_tagged_version_with_semver_compliant_prerelease(self):
+        self.repo.commit()
+        self.repo.tag('1.2.3-rc2')
+        version = packaging._get_version_from_git()
+        self.assertEqual('1.2.3.0rc2', version)
+
     def test_non_canonical_tagged_version_bump(self):
         self.repo.commit()
         self.repo.tag('1.4')
@@ -724,6 +787,13 @@ class TestVersions(base.BaseTestCase):
         version = packaging._get_version_from_git('1.2.3')
         self.assertThat(version, matchers.StartsWith('1.2.3.0a2.dev1'))
 
+    def test_untagged_version_after_semver_compliant_prerelease_tag(self):
+        self.repo.commit()
+        self.repo.tag('1.2.3-rc2')
+        self.repo.commit()
+        version = packaging._get_version_from_git()
+        self.assertEqual('1.2.3.0rc3.dev1', version)
+
     def test_preversion_too_low_simple(self):
         # That is, the target version is either already released or not high
         # enough for the semver requirements given api breaks etc.
@@ -750,8 +820,10 @@ class TestVersions(base.BaseTestCase):
 
     def test_get_kwargs_corner_cases(self):
         # No tags:
-        git_dir = self.repo._basedir + '/.git'
-        get_kwargs = lambda tag: packaging._get_increment_kwargs(git_dir, tag)
+
+        def get_kwargs(tag):
+            git_dir = self.repo._basedir + '/.git'
+            return packaging._get_increment_kwargs(git_dir, tag)
 
         def _check_combinations(tag):
             self.repo.commit()
@@ -903,6 +975,235 @@ class TestRequirementParsing(base.BaseTestCase):
             self.assertEqual(exp_parsed, gen_parsed)
 
 
+class TestPEP517Support(base.BaseTestCase):
+    def test_pep_517_support(self):
+        # Note that the current PBR PEP517 entrypoints rely on a valid
+        # PBR setup.py existing.
+        pkgs = {
+            'test_pep517':
+                {
+                    'requirements.txt': textwrap.dedent("""\
+                        sphinx
+                        iso8601
+                    """),
+                    # Override default setup.py to remove setup_requires.
+                    'setup.py': textwrap.dedent("""\
+                        #!/usr/bin/env python
+                        import setuptools
+                        setuptools.setup(pbr=True)
+                    """),
+                    'setup.cfg': textwrap.dedent("""\
+                        [metadata]
+                        name = test_pep517
+                        summary = A tiny test project
+                        author = PBR Team
+                        author-email = foo@example.com
+                        home-page = https://example.com/
+                        classifier =
+                            Intended Audience :: Information Technology
+                            Intended Audience :: System Administrators
+                            License :: OSI Approved :: Apache Software License
+                            Operating System :: POSIX :: Linux
+                            Programming Language :: Python
+                            Programming Language :: Python :: 2
+                            Programming Language :: Python :: 2.7
+                            Programming Language :: Python :: 3
+                            Programming Language :: Python :: 3.6
+                            Programming Language :: Python :: 3.7
+                            Programming Language :: Python :: 3.8
+                    """),
+                    'pyproject.toml': textwrap.dedent("""\
+                        [build-system]
+                        requires = ["pbr", "setuptools>=36.6.0", "wheel"]
+                        build-backend = "pbr.build"
+                    """)},
+        }
+        pkg_dirs = self.useFixture(CreatePackages(pkgs)).package_dirs
+        pkg_dir = pkg_dirs['test_pep517']
+        venv = self.useFixture(Venv('PEP517'))
+
+        # Test building sdists and wheels works. Note we do not use pip here
+        # because pip will forcefully install the latest version of PBR on
+        # pypi to satisfy the build-system requires. This means we can't self
+        # test changes using pip. Build with --no-isolation appears to avoid
+        # this problem.
+        self._run_cmd(venv.python, ('-m', 'build', '--no-isolation', '.'),
+                      allow_fail=False, cwd=pkg_dir)
+
+
+class TestRepositoryURLDependencies(base.BaseTestCase):
+
+    def setUp(self):
+        super(TestRepositoryURLDependencies, self).setUp()
+        self.requirements = os.path.join(tempfile.mkdtemp(),
+                                         'requirements.txt')
+        with open(self.requirements, 'w') as f:
+            f.write('\n'.join([
+                '-e git+git://git.pro-ject.org/oslo.messaging#egg=oslo.messaging-1.0.0-rc',  # noqa
+                '-e git+git://git.pro-ject.org/django-thumborize#egg=django-thumborize',  # noqa
+                '-e git+git://git.pro-ject.org/django-thumborize#egg=django-thumborize-beta',  # noqa
+                '-e git+git://git.pro-ject.org/django-thumborize#egg=django-thumborize2-beta',  # noqa
+                '-e git+git://git.pro-ject.org/django-thumborize#egg=django-thumborize2-beta-4.0.1',  # noqa
+                '-e git+git://git.pro-ject.org/django-thumborize#egg=django-thumborize2-beta-1.0.0-alpha.beta.1',  # noqa
+                '-e git+git://git.pro-ject.org/django-thumborize#egg=django-thumborize2-beta-1.0.0-alpha-a.b-c-somethinglong+build.1-aef.1-its-okay',  # noqa
+                '-e git+git://git.pro-ject.org/django-thumborize#egg=django-thumborize2-beta-2.0.0-rc.1+build.123',  # noqa
+                '-e git+git://git.project.org/Proj#egg=Proj1',
+                'git+https://git.project.org/Proj#egg=Proj2-0.0.1',
+                '-e git+ssh://git.project.org/Proj#egg=Proj3',
+                'svn+svn://svn.project.org/svn/Proj#egg=Proj4-0.0.2',
+                '-e svn+http://svn.project.org/svn/Proj/trunk@2019#egg=Proj5',
+                'hg+http://hg.project.org/Proj@da39a3ee5e6b#egg=Proj-0.0.3',
+                '-e hg+http://hg.project.org/Proj@2019#egg=Proj',
+                'hg+http://hg.project.org/Proj@v1.0#egg=Proj-0.0.4',
+                '-e hg+http://hg.project.org/Proj@special_feature#egg=Proj',
+                'git://foo.com/zipball#egg=foo-bar-1.2.4',
+                'pypi-proj1', 'pypi-proj2']))
+
+    def test_egg_fragment(self):
+        expected = [
+            'django-thumborize',
+            'django-thumborize-beta',
+            'django-thumborize2-beta',
+            'django-thumborize2-beta>=4.0.1',
+            'django-thumborize2-beta>=1.0.0-alpha.beta.1',
+            'django-thumborize2-beta>=1.0.0-alpha-a.b-c-long+build.1-aef.1-its-okay',  # noqa
+            'django-thumborize2-beta>=2.0.0-rc.1+build.123',
+            'django-thumborize-beta>=0.0.4',
+            'django-thumborize-beta>=1.2.3',
+            'django-thumborize-beta>=10.20.30',
+            'django-thumborize-beta>=1.1.2-prerelease+meta',
+            'django-thumborize-beta>=1.1.2+meta',
+            'django-thumborize-beta>=1.1.2+meta-valid',
+            'django-thumborize-beta>=1.0.0-alpha',
+            'django-thumborize-beta>=1.0.0-beta',
+            'django-thumborize-beta>=1.0.0-alpha.beta',
+            'django-thumborize-beta>=1.0.0-alpha.beta.1',
+            'django-thumborize-beta>=1.0.0-alpha.1',
+            'django-thumborize-beta>=1.0.0-alpha0.valid',
+            'django-thumborize-beta>=1.0.0-alpha.0valid',
+            'django-thumborize-beta>=1.0.0-alpha-a.b-c-somethinglong+build.1-aef.1-its-okay',  # noqa
+            'django-thumborize-beta>=1.0.0-rc.1+build.1',
+            'django-thumborize-beta>=2.0.0-rc.1+build.123',
+            'django-thumborize-beta>=1.2.3-beta',
+            'django-thumborize-beta>=10.2.3-DEV-SNAPSHOT',
+            'django-thumborize-beta>=1.2.3-SNAPSHOT-123',
+            'django-thumborize-beta>=1.0.0',
+            'django-thumborize-beta>=2.0.0',
+            'django-thumborize-beta>=1.1.7',
+            'django-thumborize-beta>=2.0.0+build.1848',
+            'django-thumborize-beta>=2.0.1-alpha.1227',
+            'django-thumborize-beta>=1.0.0-alpha+beta',
+            'django-thumborize-beta>=1.2.3----RC-SNAPSHOT.12.9.1--.12+788',
+            'django-thumborize-beta>=1.2.3----R-S.12.9.1--.12+meta',
+            'django-thumborize-beta>=1.2.3----RC-SNAPSHOT.12.9.1--.12',
+            'django-thumborize-beta>=1.0.0+0.build.1-rc.10000aaa-kk-0.1',
+            'django-thumborize-beta>=999999999999999999.99999999999999.9999999999999',  # noqa
+            'Proj1',
+            'Proj2>=0.0.1',
+            'Proj3',
+            'Proj4>=0.0.2',
+            'Proj5',
+            'Proj>=0.0.3',
+            'Proj',
+            'Proj>=0.0.4',
+            'Proj',
+            'foo-bar>=1.2.4',
+        ]
+        tests = [
+            'egg=django-thumborize',
+            'egg=django-thumborize-beta',
+            'egg=django-thumborize2-beta',
+            'egg=django-thumborize2-beta-4.0.1',
+            'egg=django-thumborize2-beta-1.0.0-alpha.beta.1',
+            'egg=django-thumborize2-beta-1.0.0-alpha-a.b-c-long+build.1-aef.1-its-okay',  # noqa
+            'egg=django-thumborize2-beta-2.0.0-rc.1+build.123',
+            'egg=django-thumborize-beta-0.0.4',
+            'egg=django-thumborize-beta-1.2.3',
+            'egg=django-thumborize-beta-10.20.30',
+            'egg=django-thumborize-beta-1.1.2-prerelease+meta',
+            'egg=django-thumborize-beta-1.1.2+meta',
+            'egg=django-thumborize-beta-1.1.2+meta-valid',
+            'egg=django-thumborize-beta-1.0.0-alpha',
+            'egg=django-thumborize-beta-1.0.0-beta',
+            'egg=django-thumborize-beta-1.0.0-alpha.beta',
+            'egg=django-thumborize-beta-1.0.0-alpha.beta.1',
+            'egg=django-thumborize-beta-1.0.0-alpha.1',
+            'egg=django-thumborize-beta-1.0.0-alpha0.valid',
+            'egg=django-thumborize-beta-1.0.0-alpha.0valid',
+            'egg=django-thumborize-beta-1.0.0-alpha-a.b-c-somethinglong+build.1-aef.1-its-okay',  # noqa
+            'egg=django-thumborize-beta-1.0.0-rc.1+build.1',
+            'egg=django-thumborize-beta-2.0.0-rc.1+build.123',
+            'egg=django-thumborize-beta-1.2.3-beta',
+            'egg=django-thumborize-beta-10.2.3-DEV-SNAPSHOT',
+            'egg=django-thumborize-beta-1.2.3-SNAPSHOT-123',
+            'egg=django-thumborize-beta-1.0.0',
+            'egg=django-thumborize-beta-2.0.0',
+            'egg=django-thumborize-beta-1.1.7',
+            'egg=django-thumborize-beta-2.0.0+build.1848',
+            'egg=django-thumborize-beta-2.0.1-alpha.1227',
+            'egg=django-thumborize-beta-1.0.0-alpha+beta',
+            'egg=django-thumborize-beta-1.2.3----RC-SNAPSHOT.12.9.1--.12+788',  # noqa
+            'egg=django-thumborize-beta-1.2.3----R-S.12.9.1--.12+meta',
+            'egg=django-thumborize-beta-1.2.3----RC-SNAPSHOT.12.9.1--.12',
+            'egg=django-thumborize-beta-1.0.0+0.build.1-rc.10000aaa-kk-0.1',  # noqa
+            'egg=django-thumborize-beta-999999999999999999.99999999999999.9999999999999',  # noqa
+            'egg=Proj1',
+            'egg=Proj2-0.0.1',
+            'egg=Proj3',
+            'egg=Proj4-0.0.2',
+            'egg=Proj5',
+            'egg=Proj-0.0.3',
+            'egg=Proj',
+            'egg=Proj-0.0.4',
+            'egg=Proj',
+            'egg=foo-bar-1.2.4',
+        ]
+        for index, test in enumerate(tests):
+            self.assertEqual(expected[index],
+                             re.sub(r'egg=([^&]+).*$',
+                                    packaging.egg_fragment,
+                                    test))
+
+    def test_parse_repo_url_requirements(self):
+        result = packaging.parse_requirements([self.requirements])
+        self.assertEqual(['oslo.messaging>=1.0.0-rc',
+                          'django-thumborize',
+                          'django-thumborize-beta',
+                          'django-thumborize2-beta',
+                          'django-thumborize2-beta>=4.0.1',
+                          'django-thumborize2-beta>=1.0.0-alpha.beta.1',
+                          'django-thumborize2-beta>=1.0.0-alpha-a.b-c-somethinglong+build.1-aef.1-its-okay',  # noqa
+                          'django-thumborize2-beta>=2.0.0-rc.1+build.123',
+                          'Proj1', 'Proj2>=0.0.1', 'Proj3',
+                          'Proj4>=0.0.2', 'Proj5', 'Proj>=0.0.3',
+                          'Proj', 'Proj>=0.0.4', 'Proj',
+                          'foo-bar>=1.2.4', 'pypi-proj1',
+                          'pypi-proj2'], result)
+
+    def test_parse_repo_url_dependency_links(self):
+        result = packaging.parse_dependency_links([self.requirements])
+        self.assertEqual(
+            [
+             'git+git://git.pro-ject.org/oslo.messaging#egg=oslo.messaging-1.0.0-rc',  # noqa
+             'git+git://git.pro-ject.org/django-thumborize#egg=django-thumborize',  # noqa
+             'git+git://git.pro-ject.org/django-thumborize#egg=django-thumborize-beta',  # noqa
+             'git+git://git.pro-ject.org/django-thumborize#egg=django-thumborize2-beta',  # noqa
+             'git+git://git.pro-ject.org/django-thumborize#egg=django-thumborize2-beta-4.0.1',  # noqa
+             'git+git://git.pro-ject.org/django-thumborize#egg=django-thumborize2-beta-1.0.0-alpha.beta.1',  # noqa
+             'git+git://git.pro-ject.org/django-thumborize#egg=django-thumborize2-beta-1.0.0-alpha-a.b-c-somethinglong+build.1-aef.1-its-okay',  # noqa
+             'git+git://git.pro-ject.org/django-thumborize#egg=django-thumborize2-beta-2.0.0-rc.1+build.123',  # noqa
+             'git+git://git.project.org/Proj#egg=Proj1',
+             'git+https://git.project.org/Proj#egg=Proj2-0.0.1',
+             'git+ssh://git.project.org/Proj#egg=Proj3',
+             'svn+svn://svn.project.org/svn/Proj#egg=Proj4-0.0.2',
+             'svn+http://svn.project.org/svn/Proj/trunk@2019#egg=Proj5',
+             'hg+http://hg.project.org/Proj@da39a3ee5e6b#egg=Proj-0.0.3',
+             'hg+http://hg.project.org/Proj@2019#egg=Proj',
+             'hg+http://hg.project.org/Proj@v1.0#egg=Proj-0.0.4',
+             'hg+http://hg.project.org/Proj@special_feature#egg=Proj',
+             'git://foo.com/zipball#egg=foo-bar-1.2.4'], result)
+
+
 def get_soabi():
     soabi = None
     try:

libs/common/pbr/tests/test_pbr_json.py

@@ -10,7 +10,10 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import mock
+try:
+    from unittest import mock
+except ImportError:
+    import mock
 
 from pbr import pbr_json
 from pbr.tests import base

libs/common/pbr/tests/test_setup.py

@@ -93,8 +93,9 @@ class SkipFileWrites(base.BaseTestCase):
                       option_dict=self.option_dict)
         self.assertEqual(
             not os.path.exists(self.filename),
-            (self.option_value.lower() in options.TRUE_VALUES
-             or self.env_value is not None))
+            (self.option_value.lower() in options.TRUE_VALUES or
+             self.env_value is not None))
+
 
 _changelog_content = """7780758\x00Break parser\x00 (tag: refs/tags/1_foo.1)
 04316fe\x00Make python\x00 (refs/heads/review/monty_taylor/27519)
@@ -125,6 +126,7 @@ def _make_old_git_changelog_format(line):
     refname = refname.replace('tag: ', '')
     return '\x00'.join((sha, msg, refname))
 
+
 _old_git_changelog_content = '\n'.join(
     _make_old_git_changelog_format(line)
     for line in _changelog_content.split('\n'))
@@ -162,7 +164,7 @@ class GitLogsTest(base.BaseTestCase):
             self.assertIn("------", changelog_contents)
             self.assertIn("Refactor hooks file", changelog_contents)
             self.assertIn(
-                "Bug fix: create\_stack() fails when waiting",
+                r"Bug fix: create\_stack() fails when waiting",
                 changelog_contents)
             self.assertNotIn("Refactor hooks file.", changelog_contents)
             self.assertNotIn("182feb3", changelog_contents)
@@ -176,7 +178,7 @@ class GitLogsTest(base.BaseTestCase):
             self.assertNotIn("ev)il", changelog_contents)
             self.assertNotIn("e(vi)l", changelog_contents)
             self.assertNotIn('Merge "', changelog_contents)
-            self.assertNotIn('1\_foo.1', changelog_contents)
+            self.assertNotIn(r'1\_foo.1', changelog_contents)
 
     def test_generate_authors(self):
         author_old = u"Foo Foo <email@foo.com>"
@@ -216,9 +218,9 @@ class GitLogsTest(base.BaseTestCase):
 
         with open(os.path.join(self.temp_path, "AUTHORS"), "r") as auth_fh:
             authors = auth_fh.read()
-            self.assertTrue(author_old in authors)
-            self.assertTrue(author_new in authors)
-            self.assertTrue(co_author in authors)
+            self.assertIn(author_old, authors)
+            self.assertIn(author_new, authors)
+            self.assertIn(co_author, authors)
 
 
 class _SphinxConfig(object):

libs/common/pbr/tests/test_util.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Copyright (c) 2015 Hewlett-Packard Development Company, L.P. (HP)
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -13,6 +14,7 @@
 # under the License.
 
 import io
+import tempfile
 import textwrap
 
 import six
@@ -23,6 +25,122 @@ from pbr.tests import base
 from pbr import util
 
 
+def config_from_ini(ini):
+    config = {}
+    ini = textwrap.dedent(six.u(ini))
+    if sys.version_info >= (3, 2):
+        parser = configparser.ConfigParser()
+        parser.read_file(io.StringIO(ini))
+    else:
+        parser = configparser.SafeConfigParser()
+        parser.readfp(io.StringIO(ini))
+    for section in parser.sections():
+        config[section] = dict(parser.items(section))
+    return config
+
+
+class TestBasics(base.BaseTestCase):
+
+    def test_basics(self):
+        self.maxDiff = None
+        config_text = """
+            [metadata]
+            name = foo
+            version = 1.0
+            author = John Doe
+            author_email = jd@example.com
+            maintainer = Jim Burke
+            maintainer_email = jb@example.com
+            home_page = http://example.com
+            summary = A foobar project.
+            description = Hello, world. This is a long description.
+            download_url = http://opendev.org/x/pbr
+            classifier =
+                Development Status :: 5 - Production/Stable
+                Programming Language :: Python
+            platform =
+                any
+            license = Apache 2.0
+            requires_dist =
+                Sphinx
+                requests
+            setup_requires_dist =
+                docutils
+            python_requires = >=3.6
+            provides_dist =
+                bax
+            provides_extras =
+                bar
+            obsoletes_dist =
+                baz
+
+            [files]
+            packages_root = src
+            packages =
+                foo
+            package_data =
+                "" = *.txt, *.rst
+                foo = *.msg
+            namespace_packages =
+                hello
+            data_files =
+                bitmaps =
+                    bm/b1.gif
+                    bm/b2.gif
+                config =
+                    cfg/data.cfg
+            scripts =
+                scripts/hello-world.py
+            modules =
+                mod1
+            """
+        expected = {
+            'name': u'foo',
+            'version': u'1.0',
+            'author': u'John Doe',
+            'author_email': u'jd@example.com',
+            'maintainer': u'Jim Burke',
+            'maintainer_email': u'jb@example.com',
+            'url': u'http://example.com',
+            'description': u'A foobar project.',
+            'long_description': u'Hello, world. This is a long description.',
+            'download_url': u'http://opendev.org/x/pbr',
+            'classifiers': [
+                u'Development Status :: 5 - Production/Stable',
+                u'Programming Language :: Python',
+            ],
+            'platforms': [u'any'],
+            'license': u'Apache 2.0',
+            'install_requires': [
+                u'Sphinx',
+                u'requests',
+            ],
+            'setup_requires': [u'docutils'],
+            'python_requires': u'>=3.6',
+            'provides': [u'bax'],
+            'provides_extras': [u'bar'],
+            'obsoletes': [u'baz'],
+            'extras_require': {},
+
+            'package_dir': {'': u'src'},
+            'packages': [u'foo'],
+            'package_data': {
+                '': ['*.txt,', '*.rst'],
+                'foo': ['*.msg'],
+            },
+            'namespace_packages': [u'hello'],
+            'data_files': [
+                ('bitmaps', ['bm/b1.gif', 'bm/b2.gif']),
+                ('config', ['cfg/data.cfg']),
+            ],
+            'scripts': [u'scripts/hello-world.py'],
+            'py_modules': [u'mod1'],
+        }
+        config = config_from_ini(config_text)
+        actual = util.setup_cfg_to_setup_kwargs(config)
+        self.assertDictEqual(expected, actual)
+
+
 class TestExtrasRequireParsingScenarios(base.BaseTestCase):
 
     scenarios = [
@@ -64,20 +182,8 @@ class TestExtrasRequireParsingScenarios(base.BaseTestCase):
             {}
         })]
 
-    def config_from_ini(self, ini):
-        config = {}
-        if sys.version_info >= (3, 2):
-            parser = configparser.ConfigParser()
-        else:
-            parser = configparser.SafeConfigParser()
-        ini = textwrap.dedent(six.u(ini))
-        parser.readfp(io.StringIO(ini))
-        for section in parser.sections():
-            config[section] = dict(parser.items(section))
-        return config
-
     def test_extras_parsing(self):
-        config = self.config_from_ini(self.config_text)
+        config = config_from_ini(self.config_text)
         kwargs = util.setup_cfg_to_setup_kwargs(config)
 
         self.assertEqual(self.expected_extra_requires,
@@ -89,3 +195,127 @@ class TestInvalidMarkers(base.BaseTestCase):
     def test_invalid_marker_raises_error(self):
         config = {'extras': {'test': "foo :bad_marker>'1.0'"}}
         self.assertRaises(SyntaxError, util.setup_cfg_to_setup_kwargs, config)
+
+
+class TestMapFieldsParsingScenarios(base.BaseTestCase):
+
+    scenarios = [
+        ('simple_project_urls', {
+            'config_text': """
+                [metadata]
+                project_urls =
+                    Bug Tracker = https://bugs.launchpad.net/pbr/
+                    Documentation = https://docs.openstack.org/pbr/
+                    Source Code = https://opendev.org/openstack/pbr
+                """,  # noqa: E501
+            'expected_project_urls': {
+                'Bug Tracker': 'https://bugs.launchpad.net/pbr/',
+                'Documentation': 'https://docs.openstack.org/pbr/',
+                'Source Code': 'https://opendev.org/openstack/pbr',
+            },
+        }),
+        ('query_parameters', {
+            'config_text': """
+                [metadata]
+                project_urls =
+                    Bug Tracker = https://bugs.launchpad.net/pbr/?query=true
+                    Documentation = https://docs.openstack.org/pbr/?foo=bar
+                    Source Code = https://git.openstack.org/cgit/openstack-dev/pbr/commit/?id=hash
+                """,  # noqa: E501
+            'expected_project_urls': {
+                'Bug Tracker': 'https://bugs.launchpad.net/pbr/?query=true',
+                'Documentation': 'https://docs.openstack.org/pbr/?foo=bar',
+                'Source Code': 'https://git.openstack.org/cgit/openstack-dev/pbr/commit/?id=hash',  # noqa: E501
+            },
+        }),
+    ]
+
+    def test_project_url_parsing(self):
+        config = config_from_ini(self.config_text)
+        kwargs = util.setup_cfg_to_setup_kwargs(config)
+
+        self.assertEqual(self.expected_project_urls, kwargs['project_urls'])
+
+
+class TestKeywordsParsingScenarios(base.BaseTestCase):
+
+    scenarios = [
+        ('keywords_list', {
+            'config_text': """
+                [metadata]
+                keywords =
+                    one
+                    two
+                    three
+                """,  # noqa: E501
+            'expected_keywords': ['one', 'two', 'three'],
+        },
+        ),
+        ('inline_keywords', {
+            'config_text': """
+                [metadata]
+                keywords = one, two, three
+                """,  # noqa: E501
+            'expected_keywords': ['one, two, three'],
+        }),
+    ]
+
+    def test_keywords_parsing(self):
+        config = config_from_ini(self.config_text)
+        kwargs = util.setup_cfg_to_setup_kwargs(config)
+
+        self.assertEqual(self.expected_keywords, kwargs['keywords'])
+
+
+class TestProvidesExtras(base.BaseTestCase):
+    def test_provides_extras(self):
+        ini = """
+        [metadata]
+        provides_extras = foo
+                          bar
+        """
+        config = config_from_ini(ini)
+        kwargs = util.setup_cfg_to_setup_kwargs(config)
+        self.assertEqual(['foo', 'bar'], kwargs['provides_extras'])
+
+
+class TestDataFilesParsing(base.BaseTestCase):
+
+    scenarios = [
+        ('data_files', {
+            'config_text': """
+            [files]
+            data_files =
+                'i like spaces/' =
+                    'dir with space/file with spc 2'
+                    'dir with space/file with spc 1'
+            """,
+            'data_files': [
+                ('i like spaces/', ['dir with space/file with spc 2',
+                                    'dir with space/file with spc 1'])
+            ]
+        })]
+
+    def test_handling_of_whitespace_in_data_files(self):
+        config = config_from_ini(self.config_text)
+        kwargs = util.setup_cfg_to_setup_kwargs(config)
+
+        self.assertEqual(self.data_files, kwargs['data_files'])
+
+
+class TestUTF8DescriptionFile(base.BaseTestCase):
+    def test_utf8_description_file(self):
+        _, path = tempfile.mkstemp()
+        ini_template = """
+        [metadata]
+        description_file = %s
+        """
+        # Two \n's because pbr strips the file content and adds \n\n
+        # This way we can use it directly as the assert comparison
+        unicode_description = u'UTF8 description: é"…-ʃŋ\'\n\n'
+        ini = ini_template % path
+        with io.open(path, 'w', encoding='utf8') as f:
+            f.write(unicode_description)
+        config = config_from_ini(ini)
+        kwargs = util.setup_cfg_to_setup_kwargs(config)
+        self.assertEqual(unicode_description, kwargs['long_description'])

libs/common/pbr/tests/test_wsgi.py

@@ -77,8 +77,8 @@ class TestWsgiScripts(base.BaseTestCase):
 
     def _test_wsgi(self, cmd_name, output, extra_args=None):
         cmd = os.path.join(self.temp_dir, 'bin', cmd_name)
-        print("Running %s -p 0" % cmd)
-        popen_cmd = [cmd, '-p', '0']
+        print("Running %s -p 0 -b 127.0.0.1" % cmd)
+        popen_cmd = [cmd, '-p', '0', '-b', '127.0.0.1']
         if extra_args:
             popen_cmd.extend(extra_args)
 
@@ -98,7 +98,7 @@ class TestWsgiScripts(base.BaseTestCase):
 
         stdoutdata = p.stdout.readline()  # Available at ...
         print(stdoutdata)
-        m = re.search(b'(http://[^:]+:\d+)/', stdoutdata)
+        m = re.search(br'(http://[^:]+:\d+)/', stdoutdata)
         self.assertIsNotNone(m, "Regex failed to match on %s" % stdoutdata)
 
         stdoutdata = p.stdout.readline()  # DANGER! ...

libs/common/pbr/tests/testpackage/doc/source/conf.py

@@ -12,17 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
-import sys
 
-sys.path.insert(0, os.path.abspath('../..'))
 # -- General configuration ----------------------------------------------------
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = [
     'sphinx.ext.autodoc',
-    #'sphinx.ext.intersphinx',
 ]
 
 # autodoc generation is a bit aggressive and a nuisance when doing heavy
@@ -49,17 +45,9 @@ add_module_names = True
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = 'sphinx'
 
+
 # -- Options for HTML output --------------------------------------------------
 
-# The theme to use for HTML and HTML Help pages.  Major themes that come with
-# Sphinx are currently 'default' and 'sphinxdoc'.
-# html_theme_path = ["."]
-# html_theme = '_theme'
-# html_static_path = ['static']
-
-# Output file base name for HTML help builder.
-htmlhelp_basename = '%sdoc' % project
-
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, documentclass
 # [howto/manual]).
@@ -69,6 +57,3 @@ latex_documents = [
      u'%s Documentation' % project,
      u'OpenStack Foundation', 'manual'),
 ]
-
-# Example configuration for intersphinx: refer to the Python standard library.
-#intersphinx_mapping = {'http://docs.python.org/': None}

libs/common/pbr/tests/util.py

@@ -53,9 +53,9 @@ except ImportError:
 @contextlib.contextmanager
 def open_config(filename):
     if sys.version_info >= (3, 2):
-            cfg = configparser.ConfigParser()
+        cfg = configparser.ConfigParser()
     else:
-            cfg = configparser.SafeConfigParser()
+        cfg = configparser.SafeConfigParser()
     cfg.read(filename)
     yield cfg
     with open(filename, 'w') as fp:

libs/common/pbr/util.py

@@ -62,8 +62,10 @@ except ImportError:
 import logging  # noqa
 
 from collections import defaultdict
+import io
 import os
 import re
+import shlex
 import sys
 import traceback
 
@@ -86,50 +88,52 @@ import pbr.hooks
 # predicates in ()
 _VERSION_SPEC_RE = re.compile(r'\s*(.*?)\s*\((.*)\)\s*$')
 
-
-# Mappings from setup() keyword arguments to setup.cfg options;
-# The values are (section, option) tuples, or simply (section,) tuples if
-# the option has the same name as the setup() argument
-D1_D2_SETUP_ARGS = {
-    "name": ("metadata",),
-    "version": ("metadata",),
-    "author": ("metadata",),
-    "author_email": ("metadata",),
-    "maintainer": ("metadata",),
-    "maintainer_email": ("metadata",),
-    "url": ("metadata", "home_page"),
-    "project_urls": ("metadata",),
-    "description": ("metadata", "summary"),
-    "keywords": ("metadata",),
-    "long_description": ("metadata", "description"),
-    "long_description_content_type": ("metadata", "description_content_type"),
-    "download_url": ("metadata",),
-    "classifiers": ("metadata", "classifier"),
-    "platforms": ("metadata", "platform"),  # **
-    "license": ("metadata",),
+# Mappings from setup.cfg options, in (section, option) form, to setup()
+# keyword arguments
+CFG_TO_PY_SETUP_ARGS = (
+    (('metadata', 'name'), 'name'),
+    (('metadata', 'version'), 'version'),
+    (('metadata', 'author'), 'author'),
+    (('metadata', 'author_email'), 'author_email'),
+    (('metadata', 'maintainer'), 'maintainer'),
+    (('metadata', 'maintainer_email'), 'maintainer_email'),
+    (('metadata', 'home_page'), 'url'),
+    (('metadata', 'project_urls'), 'project_urls'),
+    (('metadata', 'summary'), 'description'),
+    (('metadata', 'keywords'), 'keywords'),
+    (('metadata', 'description'), 'long_description'),
+    (
+        ('metadata', 'description_content_type'),
+        'long_description_content_type',
+    ),
+    (('metadata', 'download_url'), 'download_url'),
+    (('metadata', 'classifier'), 'classifiers'),
+    (('metadata', 'platform'), 'platforms'),  # **
+    (('metadata', 'license'), 'license'),
     # Use setuptools install_requires, not
     # broken distutils requires
-    "install_requires": ("metadata", "requires_dist"),
-    "setup_requires": ("metadata", "setup_requires_dist"),
-    "python_requires": ("metadata",),
-    "provides": ("metadata", "provides_dist"),  # **
-    "obsoletes": ("metadata", "obsoletes_dist"),  # **
-    "package_dir": ("files", 'packages_root'),
-    "packages": ("files",),
-    "package_data": ("files",),
-    "namespace_packages": ("files",),
-    "data_files": ("files",),
-    "scripts": ("files",),
-    "py_modules": ("files", "modules"),   # **
-    "cmdclass": ("global", "commands"),
+    (('metadata', 'requires_dist'), 'install_requires'),
+    (('metadata', 'setup_requires_dist'), 'setup_requires'),
+    (('metadata', 'python_requires'), 'python_requires'),
+    (('metadata', 'requires_python'), 'python_requires'),
+    (('metadata', 'provides_dist'), 'provides'),  # **
+    (('metadata', 'provides_extras'), 'provides_extras'),
+    (('metadata', 'obsoletes_dist'), 'obsoletes'),  # **
+    (('files', 'packages_root'), 'package_dir'),
+    (('files', 'packages'), 'packages'),
+    (('files', 'package_data'), 'package_data'),
+    (('files', 'namespace_packages'), 'namespace_packages'),
+    (('files', 'data_files'), 'data_files'),
+    (('files', 'scripts'), 'scripts'),
+    (('files', 'modules'), 'py_modules'),   # **
+    (('global', 'commands'), 'cmdclass'),
     # Not supported in distutils2, but provided for
     # backwards compatibility with setuptools
-    "use_2to3": ("backwards_compat", "use_2to3"),
-    "zip_safe": ("backwards_compat", "zip_safe"),
-    "tests_require": ("backwards_compat", "tests_require"),
-    "dependency_links": ("backwards_compat",),
-    "include_package_data": ("backwards_compat",),
-}
+    (('backwards_compat', 'zip_safe'), 'zip_safe'),
+    (('backwards_compat', 'tests_require'), 'tests_require'),
+    (('backwards_compat', 'dependency_links'), 'dependency_links'),
+    (('backwards_compat', 'include_package_data'), 'include_package_data'),
+)
 
 # setup() arguments that can have multiple values in setup.cfg
 MULTI_FIELDS = ("classifiers",
@@ -146,16 +150,27 @@ MULTI_FIELDS = ("classifiers",
                 "dependency_links",
                 "setup_requires",
                 "tests_require",
-                "cmdclass")
+                "keywords",
+                "cmdclass",
+                "provides_extras")
 
 # setup() arguments that can have mapping values in setup.cfg
 MAP_FIELDS = ("project_urls",)
 
 # setup() arguments that contain boolean values
-BOOL_FIELDS = ("use_2to3", "zip_safe", "include_package_data")
+BOOL_FIELDS = ("zip_safe", "include_package_data")
+
+CSV_FIELDS = ()
 
 
-CSV_FIELDS = ("keywords",)
+def shlex_split(path):
+    if os.name == 'nt':
+        # shlex cannot handle paths that contain backslashes, treating those
+        # as escape characters.
+        path = path.replace("\\", "/")
+        return [x.replace("/", "\\") for x in shlex.split(path)]
+
+    return shlex.split(path)
 
 
 def resolve_name(name):
@@ -205,10 +220,11 @@ def cfg_to_args(path='setup.cfg', script_args=()):
     """
 
     # The method source code really starts here.
-    if sys.version_info >= (3, 2):
-            parser = configparser.ConfigParser()
+    if sys.version_info >= (3, 0):
+        parser = configparser.ConfigParser()
     else:
-            parser = configparser.SafeConfigParser()
+        parser = configparser.SafeConfigParser()
+
     if not os.path.exists(path):
         raise errors.DistutilsFileError("file '%s' does not exist" %
                                         os.path.abspath(path))
@@ -297,34 +313,25 @@ def setup_cfg_to_setup_kwargs(config, script_args=()):
     # parse env_markers.
     all_requirements = {}
 
-    for arg in D1_D2_SETUP_ARGS:
-        if len(D1_D2_SETUP_ARGS[arg]) == 2:
-            # The distutils field name is different than distutils2's.
-            section, option = D1_D2_SETUP_ARGS[arg]
-
-        elif len(D1_D2_SETUP_ARGS[arg]) == 1:
-            # The distutils field name is the same thant distutils2's.
-            section = D1_D2_SETUP_ARGS[arg][0]
-            option = arg
+    for alias, arg in CFG_TO_PY_SETUP_ARGS:
+        section, option = alias
 
         in_cfg_value = has_get_option(config, section, option)
+        if not in_cfg_value and arg == "long_description":
+            in_cfg_value = has_get_option(config, section, "description_file")
+            if in_cfg_value:
+                in_cfg_value = split_multiline(in_cfg_value)
+                value = ''
+                for filename in in_cfg_value:
+                    description_file = io.open(filename, encoding='utf-8')
+                    try:
+                        value += description_file.read().strip() + '\n\n'
+                    finally:
+                        description_file.close()
+                in_cfg_value = value
+
         if not in_cfg_value:
-            # There is no such option in the setup.cfg
-            if arg == "long_description":
-                in_cfg_value = has_get_option(config, section,
-                                              "description_file")
-                if in_cfg_value:
-                    in_cfg_value = split_multiline(in_cfg_value)
-                    value = ''
-                    for filename in in_cfg_value:
-                        description_file = open(filename)
-                        try:
-                            value += description_file.read().strip() + '\n\n'
-                        finally:
-                            description_file.close()
-                    in_cfg_value = value
-            else:
-                continue
+            continue
 
         if arg in CSV_FIELDS:
             in_cfg_value = split_csv(in_cfg_value)
@@ -333,7 +340,7 @@ def setup_cfg_to_setup_kwargs(config, script_args=()):
         elif arg in MAP_FIELDS:
             in_cfg_map = {}
             for i in split_multiline(in_cfg_value):
-                k, v = i.split('=')
+                k, v = i.split('=', 1)
                 in_cfg_map[k.strip()] = v.strip()
             in_cfg_value = in_cfg_map
         elif arg in BOOL_FIELDS:
@@ -370,26 +377,27 @@ def setup_cfg_to_setup_kwargs(config, script_args=()):
                 for line in in_cfg_value:
                     if '=' in line:
                         key, value = line.split('=', 1)
-                        key, value = (key.strip(), value.strip())
+                        key_unquoted = shlex_split(key.strip())[0]
+                        key, value = (key_unquoted, value.strip())
                         if key in data_files:
                             # Multiple duplicates of the same package name;
                             # this is for backwards compatibility of the old
                             # format prior to d2to1 0.2.6.
                             prev = data_files[key]
-                            prev.extend(value.split())
+                            prev.extend(shlex_split(value))
                         else:
-                            prev = data_files[key.strip()] = value.split()
+                            prev = data_files[key.strip()] = shlex_split(value)
                     elif firstline:
                         raise errors.DistutilsOptionError(
                             'malformed package_data first line %r (misses '
                             '"=")' % line)
                     else:
-                        prev.extend(line.strip().split())
+                        prev.extend(shlex_split(line.strip()))
                     firstline = False
                 if arg == 'data_files':
                     # the data_files value is a pointlessly different structure
                     # from the package_data value
-                    data_files = data_files.items()
+                    data_files = sorted(data_files.items())
                 in_cfg_value = data_files
             elif arg == 'cmdclass':
                 cmdclass = {}
@@ -532,7 +540,7 @@ def get_extension_modules(config):
         else:
             # Backwards compatibility for old syntax; don't use this though
             labels = section.split('=', 1)
-        labels = [l.strip() for l in labels]
+        labels = [label.strip() for label in labels]
         if (len(labels) == 2) and (labels[0] == 'extension'):
             ext_args = {}
             for field in EXTENSION_FIELDS:

libs/common/pbr/version.py

@@ -15,13 +15,24 @@
 #    under the License.
 
 """
-Utilities for consuming the version from pkg_resources.
+Utilities for consuming the version from importlib-metadata.
 """
 
 import itertools
 import operator
 import sys
 
+# TODO(stephenfin): Remove this once we drop support for Python < 3.8
+if sys.version_info >= (3, 8):
+    from importlib import metadata as importlib_metadata
+    use_importlib = True
+else:
+    try:
+        import importlib_metadata
+        use_importlib = True
+    except ImportError:
+        use_importlib = False
+
 
 def _is_int(string):
     try:
@@ -323,8 +334,8 @@ class SemanticVersion(object):
             version number of the component to preserve sorting. (Used for
             rpm support)
         """
-        if ((self._prerelease_type or self._dev_count)
-                and pre_separator is None):
+        if ((self._prerelease_type or self._dev_count) and
+                pre_separator is None):
             segments = [self.decrement().brief_string()]
             pre_separator = "."
         else:
@@ -431,12 +442,15 @@ class VersionInfo(object):
         """Obtain a version from pkg_resources or setup-time logic if missing.
 
         This will try to get the version of the package from the pkg_resources
+        This will try to get the version of the package from the
         record associated with the package, and if there is no such record
+        importlib_metadata record associated with the package, and if there
         falls back to the logic sdist would use.
+
+        is no such record falls back to the logic sdist would use.
         """
-        # Lazy import because pkg_resources is costly to import so defer until
-        # we absolutely need it.
         import pkg_resources
+
         try:
             requirement = pkg_resources.Requirement.parse(self.package)
             provider = pkg_resources.get_provider(requirement)
@@ -447,6 +461,25 @@ class VersionInfo(object):
             # installed into anything. Revert to setup-time logic.
             from pbr import packaging
             result_string = packaging.get_version(self.package)
+
+        return SemanticVersion.from_pip_string(result_string)
+
+    def _get_version_from_importlib_metadata(self):
+        """Obtain a version from importlib or setup-time logic if missing.
+
+        This will try to get the version of the package from the
+        importlib_metadata record associated with the package, and if there
+        is no such record falls back to the logic sdist would use.
+        """
+        try:
+            distribution = importlib_metadata.distribution(self.package)
+            result_string = distribution.version
+        except importlib_metadata.PackageNotFoundError:
+            # The most likely cause for this is running tests in a tree
+            # produced from a tarball where the package itself has not been
+            # installed into anything. Revert to setup-time logic.
+            from pbr import packaging
+            result_string = packaging.get_version(self.package)
         return SemanticVersion.from_pip_string(result_string)
 
     def release_string(self):
@@ -459,7 +492,12 @@ class VersionInfo(object):
     def semantic_version(self):
         """Return the SemanticVersion object for this version."""
         if self._semantic is None:
-            self._semantic = self._get_version_from_pkg_resources()
+            # TODO(damami): simplify this once Python 3.8 is the oldest
+            # we support
+            if use_importlib:
+                self._semantic = self._get_version_from_importlib_metadata()
+            else:
+                self._semantic = self._get_version_from_pkg_resources()
         return self._semantic
 
     def version_string(self):