diff options
| -rw-r--r-- | .zuul.yaml | 2 | ||||
| -rw-r--r-- | doc/requirements.txt | 4 | ||||
| -rw-r--r-- | doc/source/conf.py | 10 | ||||
| -rw-r--r-- | doc/source/reference/index.rst | 3 | ||||
| -rw-r--r-- | doc/source/user/history.rst | 5 | ||||
| -rw-r--r-- | doc/source/user/index.rst | 1 | ||||
| -rw-r--r-- | lower-constraints.txt | 1 | ||||
| -rw-r--r-- | pbr/cmd/main.py | 4 | ||||
| -rw-r--r-- | pbr/git.py | 7 | ||||
| -rw-r--r-- | pbr/hooks/files.py | 24 | ||||
| -rw-r--r-- | pbr/tests/test_files.py | 59 | ||||
| -rw-r--r-- | pbr/tests/test_packaging.py | 13 | ||||
| -rw-r--r-- | pbr/tests/test_util.py | 45 | ||||
| -rw-r--r-- | pbr/tests/test_wsgi.py | 4 | ||||
| -rw-r--r-- | pbr/util.py | 13 | ||||
| -rw-r--r-- | playbooks/legacy/pbr-installation-devstack/run.yaml | 1 | ||||
| -rw-r--r-- | playbooks/legacy/pbr-installation-upstream-devstack/run.yaml | 1 | ||||
| -rw-r--r-- | releasenotes/notes/fix-handling-of-spaces-in-data-files-glob-0fe0c398d70dfea8.yaml | 5 | ||||
| -rw-r--r-- | setup.cfg | 13 | ||||
| -rw-r--r-- | test-requirements.txt | 3 | ||||
| -rw-r--r-- | tox.ini | 22 |
21 files changed, 187 insertions, 53 deletions
@@ -35,7 +35,6 @@ - openstack/manila-ui - openstack/neutron - openstack/neutron-fwaas - - openstack/neutron-lbaas - openstack/neutron-vpnaas - openstack/nova - openstack/octavia @@ -112,6 +111,7 @@ - openstack-python-jobs - openstack-python35-jobs - openstack-python36-jobs + - openstack-python37-jobs - periodic-stable-jobs - publish-openstack-docs-pti check: diff --git a/doc/requirements.txt b/doc/requirements.txt index b9c5e1f..d0c1f36 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,3 +1,5 @@ -sphinx!=1.6.6,!=1.6.7,>=1.6.2 # BSD +sphinx!=1.6.6,!=1.6.7,>=1.6.2,<2.0.0;python_version=='2.7' # BSD +sphinx!=1.6.6,!=1.6.7,>=1.6.2;python_version>='3.4' # BSD +sphinxcontrib-apidoc>=0.2.0 # BSD openstackdocstheme>=1.18.1 # Apache-2.0 reno>=2.5.0 # Apache-2.0 diff --git a/doc/source/conf.py b/doc/source/conf.py index cc7f4b3..13f63a0 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -8,7 +8,7 @@ sys.path.insert(0, os.path.abspath('../..')) # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo'] +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinxcontrib.apidoc'] # make openstackdocstheme optional to not increase the needed dependencies try: import openstackdocstheme @@ -72,3 +72,11 @@ latex_documents = [ '%s Documentation' % project, 'OpenStack Foundation', 'manual'), ] + +# -- sphinxcontrib.apidoc configuration -------------------------------------- + +apidoc_module_dir = '../../pbr' +apidoc_output_dir = 'reference/api' +apidoc_excluded_paths = [ + 'tests', +] diff --git a/doc/source/reference/index.rst b/doc/source/reference/index.rst index 68a9c32..3162d67 100644 --- a/doc/source/reference/index.rst +++ b/doc/source/reference/index.rst @@ -3,6 +3,5 @@ =================== .. toctree:: - :glob: - api/* + api/modules diff --git a/doc/source/user/history.rst b/doc/source/user/history.rst deleted file mode 100644 index 55439e7..0000000 --- a/doc/source/user/history.rst +++ /dev/null @@ -1,5 +0,0 @@ -================= - Release History -================= - -.. include:: ../../../ChangeLog diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst index 7854dc5..0629e2e 100644 --- a/doc/source/user/index.rst +++ b/doc/source/user/index.rst @@ -9,4 +9,3 @@ packagers semver compatibility - history diff --git a/lower-constraints.txt b/lower-constraints.txt index 173a299..fc576a5 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -25,6 +25,7 @@ requests==2.14.2 six==1.10.0 snowballstemmer==1.2.1 Sphinx==1.6.5 +sphinxcontrib-apidoc==0.2.0 sphinxcontrib-websupport==1.0.1 stestr==2.1.0 testrepository==0.0.18 diff --git a/pbr/cmd/main.py b/pbr/cmd/main.py index 29cd61d..91ea384 100644 --- a/pbr/cmd/main.py +++ b/pbr/cmd/main.py @@ -86,7 +86,9 @@ def main(): version=str(pbr.version.VersionInfo('pbr'))) subparsers = parser.add_subparsers( - title='commands', description='valid commands', help='additional help') + title='commands', description='valid commands', help='additional help', + dest='cmd') + subparsers.required = True cmd_sha = subparsers.add_parser('sha', help='print sha of package') cmd_sha.set_defaults(func=get_sha) @@ -223,6 +223,11 @@ def _iter_log_inner(git_dir): presentation logic to the output - making it suitable for different uses. + .. caution:: this function risk to return a tag that doesn't exist really + inside the git objects list due to replacement made + to tag name to also list pre-release suffix. + Compliant with the SemVer specification (e.g 1.2.3-rc1) + :return: An iterator of (hash, tags_set, 1st_line) tuples. """ log.info('[pbr] Generating ChangeLog') @@ -248,7 +253,7 @@ def _iter_log_inner(git_dir): for tag_string in refname.split("refs/tags/")[1:]: # git tag does not allow : or " " in tag names, so we split # on ", " which is the separator between elements - candidate = tag_string.split(", ")[0] + candidate = tag_string.split(", ")[0].replace("-", ".") if _is_valid_version(candidate): tags.add(candidate) diff --git a/pbr/hooks/files.py b/pbr/hooks/files.py index 750ac32..0fe0df5 100644 --- a/pbr/hooks/files.py +++ b/pbr/hooks/files.py @@ -14,6 +14,7 @@ # under the License. import os +import shlex import sys from pbr import find_package @@ -35,6 +36,14 @@ def get_man_section(section): return os.path.join(get_manpath(), 'man%s' % section) +def unquote_path(path): + # unquote the full path, e.g: "'a/full/path'" becomes "a/full/path", also + # strip the quotes off individual path components because os.walk cannot + # handle paths like: "'i like spaces'/'another dir'", so we will pass it + # "i like spaces/another dir" instead. + return "".join(shlex.split(path)) + + class FilesConfig(base.BaseConfig): section = 'files' @@ -57,25 +66,28 @@ class FilesConfig(base.BaseConfig): target = target.strip() if not target.endswith(os.path.sep): target += os.path.sep - for (dirpath, dirnames, fnames) in os.walk(source_prefix): + unquoted_prefix = unquote_path(source_prefix) + unquoted_target = unquote_path(target) + for (dirpath, dirnames, fnames) in os.walk(unquoted_prefix): # As source_prefix is always matched, using replace with a # a limit of one is always going to replace the path prefix # and not accidentally replace some text in the middle of # the path - new_prefix = dirpath.replace(source_prefix, target, 1) - finished.append("%s = " % new_prefix) + new_prefix = dirpath.replace(unquoted_prefix, + unquoted_target, 1) + finished.append("'%s' = " % new_prefix) finished.extend( - [" %s" % os.path.join(dirpath, f) for f in fnames]) + [" '%s'" % os.path.join(dirpath, f) for f in fnames]) else: finished.append(line) self.data_files = "\n".join(finished) def add_man_path(self, man_path): - self.data_files = "%s\n%s =" % (self.data_files, man_path) + self.data_files = "%s\n'%s' =" % (self.data_files, man_path) def add_man_page(self, man_page): - self.data_files = "%s\n %s" % (self.data_files, man_page) + self.data_files = "%s\n '%s'" % (self.data_files, man_page) def get_man_sections(self): man_sections = dict() diff --git a/pbr/tests/test_files.py b/pbr/tests/test_files.py index ed67f7b..94a2d9a 100644 --- a/pbr/tests/test_files.py +++ b/pbr/tests/test_files.py @@ -37,12 +37,17 @@ class FilesConfigTest(base.BaseTestCase): pkg_etc = os.path.join(pkg_fixture.base, 'etc') pkg_ansible = os.path.join(pkg_fixture.base, 'ansible', 'kolla-ansible', 'test') + dir_spcs = os.path.join(pkg_fixture.base, 'dir with space') + dir_subdir_spc = os.path.join(pkg_fixture.base, 'multi space', + 'more spaces') pkg_sub = os.path.join(pkg_etc, 'sub') subpackage = os.path.join( pkg_fixture.base, 'fake_package', 'subpackage') os.makedirs(pkg_sub) os.makedirs(subpackage) os.makedirs(pkg_ansible) + os.makedirs(dir_spcs) + os.makedirs(dir_subdir_spc) with open(os.path.join(pkg_etc, "foo"), 'w') as foo_file: foo_file.write("Foo Data") with open(os.path.join(pkg_sub, "bar"), 'w') as foo_file: @@ -51,6 +56,10 @@ class FilesConfigTest(base.BaseTestCase): baz_file.write("Baz Data") with open(os.path.join(subpackage, "__init__.py"), 'w') as foo_file: foo_file.write("# empty") + with open(os.path.join(dir_spcs, "file with spc"), 'w') as spc_file: + spc_file.write("# empty") + with open(os.path.join(dir_subdir_spc, "file with spc"), 'w') as file_: + file_.write("# empty") self.useFixture(base.DiveDir(pkg_fixture.base)) @@ -79,9 +88,49 @@ class FilesConfigTest(base.BaseTestCase): ) files.FilesConfig(config, 'fake_package').run() self.assertIn( - '\netc/pbr/ = \n etc/foo\netc/pbr/sub = \n etc/sub/bar', + "\n'etc/pbr/' = \n 'etc/foo'\n'etc/pbr/sub' = \n 'etc/sub/bar'", config['files']['data_files']) + def test_data_files_with_spaces(self): + config = dict( + files=dict( + data_files="\n 'i like spaces' = 'dir with space'/*" + ) + ) + files.FilesConfig(config, 'fake_package').run() + self.assertIn( + "\n'i like spaces/' = \n 'dir with space/file with spc'", + config['files']['data_files']) + + def test_data_files_with_spaces_subdirectories(self): + # test that we can handle whitespace in subdirectories + data_files = "\n 'one space/two space' = 'multi space/more spaces'/*" + expected = ( + "\n'one space/two space/' = " + "\n 'multi space/more spaces/file with spc'") + config = dict( + files=dict( + data_files=data_files + ) + ) + files.FilesConfig(config, 'fake_package').run() + self.assertIn(expected, config['files']['data_files']) + + def test_data_files_with_spaces_quoted_components(self): + # test that we can quote individual path components + data_files = ( + "\n'one space'/'two space' = 'multi space'/'more spaces'/*" + ) + expected = ("\n'one space/two space/' = " + "\n 'multi space/more spaces/file with spc'") + config = dict( + files=dict( + data_files=data_files + ) + ) + files.FilesConfig(config, 'fake_package').run() + self.assertIn(expected, config['files']['data_files']) + def test_data_files_globbing_source_prefix_in_directory_name(self): # We want to test that the string, "docs", is not replaced in a # subdirectory name, "sub-docs" @@ -92,8 +141,8 @@ class FilesConfigTest(base.BaseTestCase): ) files.FilesConfig(config, 'fake_package').run() self.assertIn( - '\nshare/ansible/ = ' - '\nshare/ansible/kolla-ansible = ' - '\nshare/ansible/kolla-ansible/test = ' - '\n ansible/kolla-ansible/test/baz', + "\n'share/ansible/' = " + "\n'share/ansible/kolla-ansible' = " + "\n'share/ansible/kolla-ansible/test' = " + "\n 'ansible/kolla-ansible/test/baz'", config['files']['data_files']) diff --git a/pbr/tests/test_packaging.py b/pbr/tests/test_packaging.py index 853844f..308ae89 100644 --- a/pbr/tests/test_packaging.py +++ b/pbr/tests/test_packaging.py @@ -670,6 +670,12 @@ class TestVersions(base.BaseTestCase): version = packaging._get_version_from_git('1.2.3') self.assertEqual('1.2.3', version) + def test_tagged_version_with_semver_compliant_prerelease(self): + self.repo.commit() + self.repo.tag('1.2.3-rc2') + version = packaging._get_version_from_git() + self.assertEqual('1.2.3.0rc2', version) + def test_non_canonical_tagged_version_bump(self): self.repo.commit() self.repo.tag('1.4') @@ -726,6 +732,13 @@ class TestVersions(base.BaseTestCase): version = packaging._get_version_from_git('1.2.3') self.assertThat(version, matchers.StartsWith('1.2.3.0a2.dev1')) + def test_untagged_version_after_semver_compliant_prerelease_tag(self): + self.repo.commit() + self.repo.tag('1.2.3-rc2') + self.repo.commit() + version = packaging._get_version_from_git() + self.assertEqual('1.2.3.0rc3.dev1', version) + def test_preversion_too_low_simple(self): # That is, the target version is either already released or not high # enough for the semver requirements given api breaks etc. diff --git a/pbr/tests/test_util.py b/pbr/tests/test_util.py index 6814ac7..1cbb2d2 100644 --- a/pbr/tests/test_util.py +++ b/pbr/tests/test_util.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Copyright (c) 2015 Hewlett-Packard Development Company, L.P. (HP) # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -13,6 +14,7 @@ # under the License. import io +import tempfile import textwrap import six @@ -172,3 +174,46 @@ class TestProvidesExtras(base.BaseTestCase): config = config_from_ini(ini) kwargs = util.setup_cfg_to_setup_kwargs(config) self.assertEqual(['foo', 'bar'], kwargs['provides_extras']) + + +class TestDataFilesParsing(base.BaseTestCase): + + scenarios = [ + ('data_files', { + 'config_text': """ + [files] + data_files = + 'i like spaces/' = + 'dir with space/file with spc 2' + 'dir with space/file with spc 1' + """, + 'data_files': [ + ('i like spaces/', ['dir with space/file with spc 2', + 'dir with space/file with spc 1']) + ] + })] + + def test_handling_of_whitespace_in_data_files(self): + config = config_from_ini(self.config_text) + kwargs = util.setup_cfg_to_setup_kwargs(config) + + self.assertEqual(self.data_files, + list(kwargs['data_files'])) + + +class TestUTF8DescriptionFile(base.BaseTestCase): + def test_utf8_description_file(self): + _, path = tempfile.mkstemp() + ini_template = """ + [metadata] + description_file = %s + """ + # Two \n's because pbr strips the file content and adds \n\n + # This way we can use it directly as the assert comparison + unicode_description = u'UTF8 description: é"…-ʃŋ\'\n\n' + ini = ini_template % path + with io.open(path, 'w', encoding='utf8') as f: + f.write(unicode_description) + config = config_from_ini(ini) + kwargs = util.setup_cfg_to_setup_kwargs(config) + self.assertEqual(unicode_description, kwargs['long_description']) diff --git a/pbr/tests/test_wsgi.py b/pbr/tests/test_wsgi.py index f840610..18732f7 100644 --- a/pbr/tests/test_wsgi.py +++ b/pbr/tests/test_wsgi.py @@ -77,8 +77,8 @@ class TestWsgiScripts(base.BaseTestCase): def _test_wsgi(self, cmd_name, output, extra_args=None): cmd = os.path.join(self.temp_dir, 'bin', cmd_name) - print("Running %s -p 0" % cmd) - popen_cmd = [cmd, '-p', '0'] + print("Running %s -p 0 -b 127.0.0.1" % cmd) + popen_cmd = [cmd, '-p', '0', '-b', '127.0.0.1'] if extra_args: popen_cmd.extend(extra_args) diff --git a/pbr/util.py b/pbr/util.py index 55d73f8..323747e 100644 --- a/pbr/util.py +++ b/pbr/util.py @@ -62,8 +62,10 @@ except ImportError: import logging # noqa from collections import defaultdict +import io import os import re +import shlex import sys import traceback @@ -319,7 +321,7 @@ def setup_cfg_to_setup_kwargs(config, script_args=()): in_cfg_value = split_multiline(in_cfg_value) value = '' for filename in in_cfg_value: - description_file = open(filename) + description_file = io.open(filename, encoding='utf-8') try: value += description_file.read().strip() + '\n\n' finally: @@ -372,21 +374,22 @@ def setup_cfg_to_setup_kwargs(config, script_args=()): for line in in_cfg_value: if '=' in line: key, value = line.split('=', 1) - key, value = (key.strip(), value.strip()) + key_unquoted = shlex.split(key.strip())[0] + key, value = (key_unquoted, value.strip()) if key in data_files: # Multiple duplicates of the same package name; # this is for backwards compatibility of the old # format prior to d2to1 0.2.6. prev = data_files[key] - prev.extend(value.split()) + prev.extend(shlex.split(value)) else: - prev = data_files[key.strip()] = value.split() + prev = data_files[key.strip()] = shlex.split(value) elif firstline: raise errors.DistutilsOptionError( 'malformed package_data first line %r (misses ' '"=")' % line) else: - prev.extend(line.strip().split()) + prev.extend(shlex.split(line.strip())) firstline = False if arg == 'data_files': # the data_files value is a pointlessly different structure diff --git a/playbooks/legacy/pbr-installation-devstack/run.yaml b/playbooks/legacy/pbr-installation-devstack/run.yaml index c3591ce..96f863a 100644 --- a/playbooks/legacy/pbr-installation-devstack/run.yaml +++ b/playbooks/legacy/pbr-installation-devstack/run.yaml @@ -63,7 +63,6 @@ export PROJECTS="openstack/zaqar $PROJECTS" export PROJECTS="openstack/neutron $PROJECTS" export PROJECTS="openstack/neutron-fwaas $PROJECTS" - export PROJECTS="openstack/neutron-lbaas $PROJECTS" export PROJECTS="openstack/octavia $PROJECTS" export PROJECTS="openstack/neutron-vpnaas $PROJECTS" export PROJECTS="openstack/nova $PROJECTS" diff --git a/playbooks/legacy/pbr-installation-upstream-devstack/run.yaml b/playbooks/legacy/pbr-installation-upstream-devstack/run.yaml index 554d44b..544dd43 100644 --- a/playbooks/legacy/pbr-installation-upstream-devstack/run.yaml +++ b/playbooks/legacy/pbr-installation-upstream-devstack/run.yaml @@ -63,7 +63,6 @@ export PROJECTS="openstack/zaqar $PROJECTS" export PROJECTS="openstack/neutron $PROJECTS" export PROJECTS="openstack/neutron-fwaas $PROJECTS" - export PROJECTS="openstack/neutron-lbaas $PROJECTS" export PROJECTS="openstack/octavia $PROJECTS" export PROJECTS="openstack/neutron-vpnaas $PROJECTS" export PROJECTS="openstack/nova $PROJECTS" diff --git a/releasenotes/notes/fix-handling-of-spaces-in-data-files-glob-0fe0c398d70dfea8.yaml b/releasenotes/notes/fix-handling-of-spaces-in-data-files-glob-0fe0c398d70dfea8.yaml new file mode 100644 index 0000000..793ba73 --- /dev/null +++ b/releasenotes/notes/fix-handling-of-spaces-in-data-files-glob-0fe0c398d70dfea8.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixes the handling of spaces in data_files globs. Please see `bug 1810934 + <https://bugs.launchpad.net/pbr/+bug/1810934>`_ for more details. @@ -32,13 +32,6 @@ classifier = packages = pbr -[pbr] -autodoc_tree_index_modules = True -autodoc_tree_excludes = - setup.py - pbr/tests/ -api_doc_dir = reference/api - [entry_points] distutils.setup_keywords = pbr = pbr.core:pbr @@ -47,11 +40,5 @@ egg_info.writers = console_scripts = pbr = pbr.cmd.main:main -[build_sphinx] -all-files = 1 -build-dir = doc/build -source-dir = doc/source -warning-is-error = 1 - [bdist_wheel] universal = 1 diff --git a/test-requirements.txt b/test-requirements.txt index 70e4ca0..c30a2b1 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -14,5 +14,6 @@ virtualenv>=14.0.6 # MIT coverage!=4.4,>=4.0 # Apache-2.0 # optionally exposed by distutils commands -sphinx!=1.6.6,!=1.6.7,>=1.6.2 # BSD +sphinx!=1.6.6,!=1.6.7,>=1.6.2,<2.0.0;python_version=='2.7' # BSD +sphinx!=1.6.6,!=1.6.7,>=1.6.2;python_version>='3.4' # BSD testrepository>=0.0.18 # Apache-2.0/BSD @@ -1,6 +1,8 @@ [tox] -minversion = 2.0 -envlist = py{27,35,36},pep8,docs +minversion = 3.1 +envlist = pep8,py{37,36,35,27},docs +ignore_basepython_conflict = True +skip_missing_interpreters = True [testenv] usedevelop = True @@ -11,7 +13,7 @@ setenv = OS_STDERR_CAPTURE={env:OS_STDERR_CAPTURE:1} OS_TEST_TIMEOUT={env:OS_TEST_TIMEOUT:60} deps = - -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} + -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} -r{toxinidir}/test-requirements.txt commands = stestr run --suppress-attachments {posargs} @@ -21,12 +23,20 @@ commands = flake8 {posargs} [testenv:docs] basepython = python3 -deps = -r{toxinidir}/doc/requirements.txt -commands = python setup.py build_sphinx +whitelist_externals = rm +deps = + -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} + -r{toxinidir}/doc/requirements.txt +commands = + rm -rf doc/build doc/source/reference/api + sphinx-build -W -b html doc/source doc/build/html {posargs} [testenv:releasenotes] basepython = python3 -deps = -r{toxinidir}/doc/requirements.txt +whitelist_externals = rm +deps = + -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} + -r{toxinidir}/doc/requirements.txt commands = rm -rf releasenotes/build sphinx-build -W -b html -d releasenotes/build/doctrees releasenotes/source releasenotes/build/html |