diff options
| -rw-r--r-- | doc/source/index.rst | 49 | ||||
| -rw-r--r-- | pbr/git.py | 36 | ||||
| -rw-r--r-- | pbr/hooks/commands.py | 2 | ||||
| -rw-r--r-- | pbr/packaging.py | 74 | ||||
| -rw-r--r-- | pbr/tests/test_packaging.py | 36 | ||||
| -rw-r--r-- | pbr/tests/test_util.py | 2 | ||||
| -rw-r--r-- | pbr/tests/test_wsgi.py | 171 | ||||
| -rw-r--r-- | pbr/tests/testpackage/pbr_testpackage/wsgi.py | 31 | ||||
| -rw-r--r-- | pbr/tests/testpackage/setup.cfg | 4 | ||||
| -rw-r--r-- | pbr/util.py | 2 | ||||
| -rw-r--r-- | test-requirements.txt | 4 |
11 files changed, 378 insertions, 33 deletions
diff --git a/doc/source/index.rst b/doc/source/index.rst index 331a2d3..eea043f 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -95,6 +95,8 @@ Sphinx documentation setups are altered to generate man pages by default. They also have several pieces of information that are known to setup.py injected into the sphinx config. +See the pbr_ section for details on configuring your project for autodoc. + Requirements ------------ @@ -157,10 +159,10 @@ constraints on the environment, you can use:: [extras] security = aleph - bet :python_environment=='3.2' - gimel :python_environment=='2.7' + bet:python_version=='3.2' + gimel:python_version=='2.7' testing = - quux :python_environment=='2.7' + quux:python_version=='2.7' long_description ---------------- @@ -172,7 +174,7 @@ for you. Usage ===== -pbr requires a distribution to use distribute. Your distribution +pbr requires a distribution to use setuptools. Your distribution must include a distutils2-like setup.cfg file, and a minimal setup.py script. A simple sample can be found in pbr's own setup.cfg @@ -296,8 +298,12 @@ pbr --- The pbr section controls pbr specific options and behaviours. -The `autodoc_tree_index_modules` is a boolean value controlling whether pbr -should generate an index of modules using `sphinx-apidoc`. +The `autodoc_tree_index_modules` is a boolean option controlling whether pbr +should generate an index of modules using `sphinx-apidoc`. By default, setup.py +is excluded. The list of excluded modules can be specified with the +`autodoc_tree_excludes` option. See the +`sphinx-apidoc man page <http://sphinx-doc.org/man/sphinx-apidoc.html>`_ +for more information. The `autodoc_index_modules` is a boolean option controlling whether pbr should itself generates documentation for Python modules of the project. By default, @@ -305,6 +311,37 @@ all found Python modules are included; some of them can be excluded by listing them in `autodoc_exclude_modules`. This list of modules can contains `fnmatch` style pattern (e.g. `myapp.tests.*`) to exclude some modules. +The `warnerrors` boolean option is used to tell Sphinx builders to treat +warnings as errors which will cause sphinx-build to fail if it encounters +warnings. This is generally useful to ensure your documentation stays clean +once you have a good docs build. + +.. note:: + + When using `autodoc_tree_excludes` or `autodoc_index_modules` you may also + need to set `exclude_patterns` in your Sphinx configuration file (generally + found at doc/source/conf.py in most OpenStack projects) otherwise + Sphinx may complain about documents that are not in a toctree. This is + especially true if the `warnerrors=True` option is set. See + http://sphinx-doc.org/config.html for more information on configuring + Sphinx. + +Comments +-------- + +Comments may be used in setup.cfg, however all comments should start with a +`#` and may be on a single line, or in line, with at least one white space +character immediately preceding the `#`. Semicolons are not a supported +comment delimiter. For instance:: + + [section] + # A comment at the start of a dedicated line + key = + value1 # An in line comment + value2 + # A comment on a dedicated line + value3 + Additional Docs =============== @@ -18,10 +18,12 @@ from __future__ import unicode_literals import distutils.errors from distutils import log +import errno import io import os import re import subprocess +import time import pkg_resources @@ -63,7 +65,13 @@ def _run_git_command(cmd, git_dir, **kwargs): def _get_git_directory(): - return _run_shell_command(['git', 'rev-parse', '--git-dir']) + try: + return _run_shell_command(['git', 'rev-parse', '--git-dir']) + except OSError as e: + if e.errno == errno.ENOENT: + # git not installed. + return '' + raise def _git_is_installed(): @@ -161,7 +169,7 @@ def _iter_changelog(changelog): first_line = False -def _iter_log_oneline(git_dir=None, option_dict=None): +def _iter_log_oneline(git_dir=None): """Iterate over --oneline log entries if possible. This parses the output into a structured form but does not apply @@ -171,16 +179,10 @@ def _iter_log_oneline(git_dir=None, option_dict=None): :return: An iterator of (hash, tags_set, 1st_line) tuples, or None if changelog generation is disabled / not available. """ - if not option_dict: - option_dict = {} - should_skip = options.get_boolean_option(option_dict, 'skip_changelog', - 'SKIP_WRITE_GIT_CHANGELOG') - if should_skip: - return if git_dir is None: git_dir = _get_git_directory() if not git_dir: - return + return [] return _iter_log_inner(git_dir) @@ -219,10 +221,17 @@ def _iter_log_inner(git_dir): def write_git_changelog(git_dir=None, dest_dir=os.path.curdir, - option_dict=dict(), changelog=None): + option_dict=None, changelog=None): """Write a changelog based on the git changelog.""" + start = time.time() + if not option_dict: + option_dict = {} + should_skip = options.get_boolean_option(option_dict, 'skip_changelog', + 'SKIP_WRITE_GIT_CHANGELOG') + if should_skip: + return if not changelog: - changelog = _iter_log_oneline(git_dir=git_dir, option_dict=option_dict) + changelog = _iter_log_oneline(git_dir=git_dir) if changelog: changelog = _iter_changelog(changelog) if not changelog: @@ -236,6 +245,8 @@ def write_git_changelog(git_dir=None, dest_dir=os.path.curdir, with io.open(new_changelog, "w", encoding="utf-8") as changelog_file: for release, content in changelog: changelog_file.write(content) + stop = time.time() + log.info('[pbr] ChangeLog complete (%0.1fs)' % (stop - start)) def generate_authors(git_dir=None, dest_dir='.', option_dict=dict()): @@ -244,6 +255,7 @@ def generate_authors(git_dir=None, dest_dir='.', option_dict=dict()): 'SKIP_GENERATE_AUTHORS') if should_skip: return + start = time.time() old_authors = os.path.join(dest_dir, 'AUTHORS.in') new_authors = os.path.join(dest_dir, 'AUTHORS') # If there's already an AUTHORS file and it's not writable, just use it @@ -278,3 +290,5 @@ def generate_authors(git_dir=None, dest_dir='.', option_dict=dict()): new_authors_fh.write(old_authors_fh.read()) new_authors_fh.write(('\n'.join(authors) + '\n') .encode('utf-8')) + stop = time.time() + log.info('[pbr] AUTHORS complete (%0.1fs)' % (stop - start)) diff --git a/pbr/hooks/commands.py b/pbr/hooks/commands.py index 6de1257..fd757e4 100644 --- a/pbr/hooks/commands.py +++ b/pbr/hooks/commands.py @@ -62,3 +62,5 @@ class CommandsConfig(base.BaseConfig): # We always want non-egg install unless explicitly requested if 'manpages' in self.pbr_config or not use_egg: self.add_command('pbr.packaging.LocalInstall') + else: + self.add_command('pbr.packaging.InstallWithGit') diff --git a/pbr/packaging.py b/pbr/packaging.py index 3ab930f..ec4388b 100644 --- a/pbr/packaging.py +++ b/pbr/packaging.py @@ -164,6 +164,20 @@ def parse_dependency_links(requirements_files=None): return dependency_links +class InstallWithGit(install.install): + """Extracts ChangeLog and AUTHORS from git then installs. + + This is useful for e.g. readthedocs where the package is + installed and then docs built. + """ + + command_name = 'install' + + def run(self): + _from_git(self.distribution) + return install.install.run(self) + + class LocalInstall(install.install): """Runs python setup.py install in a sensible manner. @@ -175,6 +189,7 @@ class LocalInstall(install.install): command_name = 'install' def run(self): + _from_git(self.distribution) return du_install.install.run(self) @@ -232,6 +247,36 @@ except ImportError: def have_nose(): return _have_nose +_wsgi_text = """#PBR Generated from %(group)r + +from %(module_name)s import %(import_target)s + +if __name__ == "__main__": + import argparse + import socket + import wsgiref.simple_server as wss + + my_ip = socket.gethostbyname(socket.gethostname()) + parser = argparse.ArgumentParser( + description=%(import_target)s.__doc__, + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('--port', '-p', type=int, default=8000, + help='TCP port to listen on') + args = parser.parse_args() + server = wss.make_server('', args.port, %(invoke_target)s()) + + print("*" * 80) + print("STARTING test server %(module_name)s.%(invoke_target)s") + url = "http://%%s:%%d/" %% (my_ip, server.server_port) + print("Available at %%s" %% url) + print("DANGER! For testing only, do not use in production") + print("*" * 80) + + server.serve_forever() +else: + application = %(invoke_target)s() + +""" _script_text = """# PBR Generated from %(group)r @@ -245,16 +290,25 @@ if __name__ == "__main__": """ +# the following allows us to specify different templates per entry +# point group when generating pbr scripts. +ENTRY_POINTS_MAP = { + 'console_scripts': _script_text, + 'gui_scripts': _script_text, + 'wsgi_scripts': _wsgi_text +} + + def override_get_script_args( dist, executable=os.path.normpath(sys.executable), is_wininst=False): """Override entrypoints console_script.""" header = easy_install.get_script_header("", executable, is_wininst) - for group in 'console_scripts', 'gui_scripts': + for group, template in ENTRY_POINTS_MAP.items(): for name, ep in dist.get_entry_map(group).items(): if not ep.attrs or len(ep.attrs) > 2: raise ValueError("Script targets must be of the form " "'func' or 'Class.class_method'.") - script_text = _script_text % dict( + script_text = template % dict( group=group, module_name=ep.module_name, import_target=ep.attrs[0], @@ -378,18 +432,22 @@ class LocalEggInfo(egg_info.egg_info): self.filelist.append(entry) +def _from_git(distribution): + option_dict = distribution.get_option_dict('pbr') + changelog = git._iter_log_oneline() + if changelog: + changelog = git._iter_changelog(changelog) + git.write_git_changelog(option_dict=option_dict, changelog=changelog) + git.generate_authors(option_dict=option_dict) + + class LocalSDist(sdist.sdist): """Builds the ChangeLog and Authors files from VC first.""" command_name = 'sdist' def run(self): - option_dict = self.distribution.get_option_dict('pbr') - changelog = git._iter_log_oneline(option_dict=option_dict) - if changelog: - changelog = git._iter_changelog(changelog) - git.write_git_changelog(option_dict=option_dict, changelog=changelog) - git.generate_authors(option_dict=option_dict) + _from_git(self.distribution) # sdist.sdist is an old style class, can't use super() sdist.sdist.run(self) diff --git a/pbr/tests/test_packaging.py b/pbr/tests/test_packaging.py index 81701d1..4a188d0 100644 --- a/pbr/tests/test_packaging.py +++ b/pbr/tests/test_packaging.py @@ -156,21 +156,23 @@ class TestPackagingInGitRepoWithCommit(base.BaseTestCase): super(TestPackagingInGitRepoWithCommit, self).setUp() repo = self.useFixture(TestRepo(self.package_dir)) repo.commit() - self.run_setup('sdist', allow_fail=False) def test_authors(self): + self.run_setup('sdist', allow_fail=False) # One commit, something should be in the authors list with open(os.path.join(self.package_dir, 'AUTHORS'), 'r') as f: body = f.read() self.assertNotEqual(body, '') def test_changelog(self): + self.run_setup('sdist', allow_fail=False) with open(os.path.join(self.package_dir, 'ChangeLog'), 'r') as f: body = f.read() # One commit, something should be in the ChangeLog list self.assertNotEqual(body, '') def test_manifest_exclude_honoured(self): + self.run_setup('sdist', allow_fail=False) with open(os.path.join( self.package_dir, 'pbr_testpackage.egg-info/SOURCES.txt'), 'r') as f: @@ -179,6 +181,12 @@ class TestPackagingInGitRepoWithCommit(base.BaseTestCase): body, matchers.Not(matchers.Contains('pbr_testpackage/extra.py'))) self.assertThat(body, matchers.Contains('pbr_testpackage/__init__.py')) + def test_install_writes_changelog(self): + stdout, _, _ = self.run_setup( + 'install', '--root', self.temp_dir + 'installed', + allow_fail=False) + self.expectThat(stdout, matchers.Contains('Generating ChangeLog')) + class TestPackagingInGitRepoWithoutCommit(base.BaseTestCase): @@ -204,18 +212,26 @@ class TestPackagingInPlainDirectory(base.BaseTestCase): def setUp(self): super(TestPackagingInPlainDirectory, self).setUp() - self.run_setup('sdist', allow_fail=False) def test_authors(self): + self.run_setup('sdist', allow_fail=False) # Not a git repo, no AUTHORS file created filename = os.path.join(self.package_dir, 'AUTHORS') self.assertFalse(os.path.exists(filename)) def test_changelog(self): + self.run_setup('sdist', allow_fail=False) # Not a git repo, no ChangeLog created filename = os.path.join(self.package_dir, 'ChangeLog') self.assertFalse(os.path.exists(filename)) + def test_install_no_ChangeLog(self): + stdout, _, _ = self.run_setup( + 'install', '--root', self.temp_dir + 'installed', + allow_fail=False) + self.expectThat( + stdout, matchers.Not(matchers.Contains('Generating ChangeLog'))) + class TestPresenceOfGit(base.BaseTestCase): @@ -428,6 +444,18 @@ class TestVersions(base.BaseTestCase): version = packaging._get_version_from_git() self.assertEqual('1.3.0.0a1', version) + def test_skip_write_git_changelog(self): + # Fix for bug 1467440 + self.repo.commit() + self.repo.tag('1.2.3') + os.environ['SKIP_WRITE_GIT_CHANGELOG'] = '1' + version = packaging._get_version_from_git('1.2.3') + self.assertEqual('1.2.3', version) + + def tearDown(self): + super(TestVersions, self).tearDown() + os.environ.pop('SKIP_WRITE_GIT_CHANGELOG', None) + class TestRequirementParsing(base.BaseTestCase): @@ -454,8 +482,8 @@ class TestRequirementParsing(base.BaseTestCase): # anonymous section instead of the empty string. Weird. expected_requirements = { None: ['bar'], - ":python_version=='2.6'": ['quux<1.0'], - "test:python_version=='2.7'": ['baz>3.2'], + ":(python_version=='2.6')": ['quux<1.0'], + "test:(python_version=='2.7')": ['baz>3.2'], "test": ['foo'] } setup_py = os.path.join(tempdir, 'setup.py') diff --git a/pbr/tests/test_util.py b/pbr/tests/test_util.py index 7a4c6bd..5999b17 100644 --- a/pbr/tests/test_util.py +++ b/pbr/tests/test_util.py @@ -48,7 +48,7 @@ class TestExtrasRequireParsingScenarios(base.BaseTestCase): baz<1.6 :python_version=='2.6' """, 'expected_extra_requires': { - "test:python_version=='2.6'": ['foo', 'baz<1.6'], + "test:(python_version=='2.6')": ['foo', 'baz<1.6'], "test": ['bar']}}), ('no_extras', { 'config_text': """ diff --git a/pbr/tests/test_wsgi.py b/pbr/tests/test_wsgi.py new file mode 100644 index 0000000..9eded63 --- /dev/null +++ b/pbr/tests/test_wsgi.py @@ -0,0 +1,171 @@ +# Copyright (c) 2015 Hewlett-Packard Development Company, L.P. (HP) +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import re +import subprocess +import sys +import tempfile +import time +try: + # python 2 + from urllib2 import urlopen +except ImportError: + # python 3 + from urllib.request import urlopen + +import fixtures + +from pbr.tests import base + + +class TestWsgiScripts(base.BaseTestCase): + + cmd_names = ('pbr_test_wsgi', 'pbr_test_wsgi_with_class') + + def test_wsgi_script_install(self): + """Test that we install a non-pkg-resources wsgi script.""" + if os.name == 'nt': + self.skipTest('Windows support is passthrough') + + stdout, _, return_code = self.run_setup( + 'install', '--prefix=%s' % self.temp_dir) + + self.useFixture( + fixtures.EnvironmentVariable( + 'PYTHONPATH', ".:%s/lib/python%s.%s/site-packages" % ( + self.temp_dir, + sys.version_info[0], + sys.version_info[1]))) + + self._check_wsgi_install_content(stdout) + + def test_wsgi_script_run(self): + """Test that we install a runnable wsgi script. + + This test actually attempts to start and interact with the + wsgi script in question to demonstrate that it's a working + wsgi script using simple server. It's a bit hokey because of + process management that has to be done. + + """ + self.skipTest("Test skipped until we can determine a reliable " + "way to capture subprocess stdout without blocking") + + if os.name == 'nt': + self.skipTest('Windows support is passthrough') + + stdout, _, return_code = self.run_setup( + 'install', '--prefix=%s' % self.temp_dir) + + self.useFixture( + fixtures.EnvironmentVariable( + 'PYTHONPATH', ".:%s/lib/python%s.%s/site-packages" % ( + self.temp_dir, + sys.version_info[0], + sys.version_info[1]))) + # NOTE(sdague): making python unbuffered is critical to + # getting output out of the subprocess. + self.useFixture( + fixtures.EnvironmentVariable( + 'PYTHONUNBUFFERED', '1')) + + self._check_wsgi_install_content(stdout) + + # Live test run the scripts and see that they respond to wsgi + # requests. + self._test_wsgi() + + def _test_wsgi(self): + for cmd_name in self.cmd_names: + cmd = os.path.join(self.temp_dir, 'bin', cmd_name) + stdout = tempfile.NamedTemporaryFile() + print("Running %s > %s" % (cmd, stdout.name)) + # NOTE(sdague): ok, this looks a little janky, and it + # is. However getting python to not hang with + # popen.communicate is beyond me. + # + # We're opening with a random port (so no conflicts), and + # redirecting all stdout and stderr to files. We can then + # safely read these files and not deadlock later in the + # test. This requires shell expansion. + p = subprocess.Popen( + "%s -p 0 > %s 2>&1" % (cmd, stdout.name), + shell=True, + close_fds=True, + cwd=self.temp_dir) + + self.addCleanup(p.kill) + + # the sleep is important to force a context switch to the + # subprocess + time.sleep(0.1) + + stdoutdata = stdout.read() + self.assertIn( + "STARTING test server pbr_testpackage.wsgi", + stdoutdata) + self.assertIn( + "DANGER! For testing only, do not use in production", + stdoutdata) + + m = re.search('(http://[^:]+:\d+)/', stdoutdata) + self.assertIsNotNone(m, "Regex failed to match on %s" % stdoutdata) + + f = urlopen(m.group(1)) + self.assertEqual("Hello World", f.read()) + + # the sleep is important to force a context switch to the + # subprocess + time.sleep(0.1) + + # Kill off the child, it should force a flush of the stdout. + p.kill() + time.sleep(0.1) + + stdoutdata = stdout.read() + # we should have logged an HTTP request, return code 200, that + # returned 11 bytes + self.assertIn('"GET / HTTP/1.1" 200 11', stdoutdata) + + def _check_wsgi_install_content(self, install_stdout): + for cmd_name in self.cmd_names: + install_txt = 'Installing %s script to %s' % (cmd_name, + self.temp_dir) + self.assertIn(install_txt, install_stdout) + + cmd_filename = os.path.join(self.temp_dir, 'bin', cmd_name) + + script_txt = open(cmd_filename, 'r').read() + self.assertNotIn('pkg_resources', script_txt) + + main_block = """if __name__ == "__main__": + import argparse + import socket + import wsgiref.simple_server as wss""" + + if cmd_name == 'pbr_test_wsgi': + app_name = "main" + else: + app_name = "WSGI.app" + + starting_block = ("STARTING test server pbr_testpackage.wsgi." + "%s" % app_name) + + else_block = """else: + application = %s()""" % app_name + + self.assertIn(main_block, script_txt) + self.assertIn(starting_block, script_txt) + self.assertIn(else_block, script_txt) diff --git a/pbr/tests/testpackage/pbr_testpackage/wsgi.py b/pbr/tests/testpackage/pbr_testpackage/wsgi.py new file mode 100644 index 0000000..7b96e66 --- /dev/null +++ b/pbr/tests/testpackage/pbr_testpackage/wsgi.py @@ -0,0 +1,31 @@ +# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import print_function + + +def application(env, start_response): + start_response('200 OK', [('Content-Type', 'text/html')]) + return ["Hello World"] + + +def main(): + return application + + +class WSGI(object): + + @classmethod + def app(self): + return application diff --git a/pbr/tests/testpackage/setup.cfg b/pbr/tests/testpackage/setup.cfg index 0188bd2..7ba209f 100644 --- a/pbr/tests/testpackage/setup.cfg +++ b/pbr/tests/testpackage/setup.cfg @@ -38,6 +38,10 @@ console_scripts = pbr_test_cmd = pbr_testpackage.cmd:main pbr_test_cmd_with_class = pbr_testpackage.cmd:Foo.bar +wsgi_scripts = + pbr_test_wsgi = pbr_testpackage.wsgi:main + pbr_test_wsgi_with_class = pbr_testpackage.wsgi:WSGI.app + [extension=pbr_testpackage.testext] sources = src/testext.c optional = True diff --git a/pbr/util.py b/pbr/util.py index 929a234..644bcd8 100644 --- a/pbr/util.py +++ b/pbr/util.py @@ -418,7 +418,7 @@ def setup_cfg_to_setup_kwargs(config): for req_group in all_requirements: for requirement, env_marker in all_requirements[req_group]: if env_marker: - extras_key = '%s:%s' % (req_group, env_marker) + extras_key = '%s:(%s)' % (req_group, env_marker) else: extras_key = req_group extras_require.setdefault(extras_key, []).append(requirement) diff --git a/test-requirements.txt b/test-requirements.txt index 39867c1..5802d7c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,9 +3,9 @@ # process, which may cause wedges in the gate later. coverage>=3.6 discover -fixtures>=0.3.14 +fixtures>=1.3.1 hacking<0.11,>=0.10.0 -mock>=1.0 +mock>=1.2 python-subunit>=0.0.18 sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2 six>=1.9.0 |