From 65f4fafd907a16ea1952ab7072676db2e9e0c51d Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 17 Dec 2014 14:26:03 -0500 Subject: Only import sphinx during hook processing When pbr is imported to handle writing the egg_info file because of the entry point, it's causing sphinx to get imported. This has a cascading effect once docutils is trying to be installed on a system with pbr installed. If some of the imports fail along the way, allow pbr to continue usefully but without the Sphinx extensions available. Eventually, when everything is installed, those extensions will work again when the commands for build_sphinx, etc. are run separately. Also slip in a change to reorder the default list of environments run by tox so the testr database is created using a dbm format available to all python versions. Change-Id: I79d67bf41a09d7e5aad8ed32eaf107f139167eb8 Closes-bug: #1403510 --- pbr/builddoc.py | 189 ++++++++++++++++++++++++++++++++++++++++++++++++++ pbr/hooks/commands.py | 7 +- pbr/options.py | 48 +++++++++++++ pbr/packaging.py | 184 ++++-------------------------------------------- pbr/tests/base.py | 6 +- pbr/util.py | 4 +- tox.ini | 8 ++- 7 files changed, 267 insertions(+), 179 deletions(-) create mode 100644 pbr/builddoc.py create mode 100644 pbr/options.py diff --git a/pbr/builddoc.py b/pbr/builddoc.py new file mode 100644 index 0000000..7242deb --- /dev/null +++ b/pbr/builddoc.py @@ -0,0 +1,189 @@ +# Copyright 2011 OpenStack LLC. +# Copyright 2012-2013 Hewlett-Packard Development Company, L.P. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from distutils import log +import os +import sys + +try: + import cStringIO +except ImportError: + import io as cStringIO + +try: + from sphinx import apidoc + from sphinx import application + from sphinx import config + from sphinx import setup_command +except Exception as e: + # NOTE(dhellmann): During the installation of docutils, setuptools + # tries to import pbr code to find the egg_info.writer hooks. That + # imports this module, which imports sphinx, which imports + # docutils, which is being installed. Because docutils uses 2to3 + # to convert its code during installation under python 3, the + # import fails, but it fails with an error other than ImportError + # (today it's a NameError on StandardError, an exception base + # class). Convert the exception type here so it can be caught in + # packaging.py where we try to determine if we can import and use + # sphinx by importing this module. See bug #1403510 for details. + raise ImportError(str(e)) +from pbr import options + + +_rst_template = """%(heading)s +%(underline)s + +.. automodule:: %(module)s + :members: + :undoc-members: + :show-inheritance: +""" + + +def _find_modules(arg, dirname, files): + for filename in files: + if filename.endswith('.py') and filename != '__init__.py': + arg["%s.%s" % (dirname.replace('/', '.'), + filename[:-3])] = True + + +class LocalBuildDoc(setup_command.BuildDoc): + + command_name = 'build_sphinx' + builders = ['html', 'man'] + + def _get_source_dir(self): + option_dict = self.distribution.get_option_dict('build_sphinx') + if 'source_dir' in option_dict: + source_dir = os.path.join(option_dict['source_dir'][1], 'api') + else: + source_dir = 'doc/source/api' + if not os.path.exists(source_dir): + os.makedirs(source_dir) + return source_dir + + def generate_autoindex(self): + log.info("[pbr] Autodocumenting from %s" + % os.path.abspath(os.curdir)) + modules = {} + source_dir = self._get_source_dir() + for pkg in self.distribution.packages: + if '.' not in pkg: + for dirpath, dirnames, files in os.walk(pkg): + _find_modules(modules, dirpath, files) + module_list = list(modules.keys()) + module_list.sort() + autoindex_filename = os.path.join(source_dir, 'autoindex.rst') + with open(autoindex_filename, 'w') as autoindex: + autoindex.write(""".. toctree:: + :maxdepth: 1 + +""") + for module in module_list: + output_filename = os.path.join(source_dir, + "%s.rst" % module) + heading = "The :mod:`%s` Module" % module + underline = "=" * len(heading) + values = dict(module=module, heading=heading, + underline=underline) + + log.info("[pbr] Generating %s" + % output_filename) + with open(output_filename, 'w') as output_file: + output_file.write(_rst_template % values) + autoindex.write(" %s.rst\n" % module) + + def _sphinx_tree(self): + source_dir = self._get_source_dir() + apidoc.main(['apidoc', '.', '-H', 'Modules', '-o', source_dir]) + + def _sphinx_run(self): + if not self.verbose: + status_stream = cStringIO.StringIO() + else: + status_stream = sys.stdout + confoverrides = {} + if self.version: + confoverrides['version'] = self.version + if self.release: + confoverrides['release'] = self.release + if self.today: + confoverrides['today'] = self.today + sphinx_config = config.Config(self.config_dir, 'conf.py', {}, []) + sphinx_config.init_values() + if self.builder == 'man' and len(sphinx_config.man_pages) == 0: + return + app = application.Sphinx( + self.source_dir, self.config_dir, + self.builder_target_dir, self.doctree_dir, + self.builder, confoverrides, status_stream, + freshenv=self.fresh_env, warningiserror=True) + + try: + app.build(force_all=self.all_files) + except Exception as err: + from docutils import utils + if isinstance(err, utils.SystemMessage): + sys.stder.write('reST markup error:\n') + sys.stderr.write(err.args[0].encode('ascii', + 'backslashreplace')) + sys.stderr.write('\n') + else: + raise + + if self.link_index: + src = app.config.master_doc + app.builder.out_suffix + dst = app.builder.get_outfilename('index') + os.symlink(src, dst) + + def run(self): + option_dict = self.distribution.get_option_dict('pbr') + tree_index = options.get_boolean_option(option_dict, + 'autodoc_tree_index_modules', + 'AUTODOC_TREE_INDEX_MODULES') + auto_index = options.get_boolean_option(option_dict, + 'autodoc_index_modules', + 'AUTODOC_INDEX_MODULES') + if not os.getenv('SPHINX_DEBUG'): + #NOTE(afazekas): These options can be used together, + # but they do a very similar thing in a difffernet way + if tree_index: + self._sphinx_tree() + if auto_index: + self.generate_autoindex() + + for builder in self.builders: + self.builder = builder + self.finalize_options() + self.project = self.distribution.get_name() + self.version = self.distribution.get_version() + self.release = self.distribution.get_version() + if 'warnerrors' in option_dict: + self._sphinx_run() + else: + setup_command.BuildDoc.run(self) + + def finalize_options(self): + # Not a new style class, super keyword does not work. + setup_command.BuildDoc.finalize_options(self) + # Allow builders to be configurable - as a comma separated list. + if not isinstance(self.builders, list) and self.builders: + self.builders = self.builders.split(',') + + +class LocalBuildLatex(LocalBuildDoc): + builders = ['latex'] + command_name = 'build_sphinx_latex' diff --git a/pbr/hooks/commands.py b/pbr/hooks/commands.py index b4206ed..3033119 100644 --- a/pbr/hooks/commands.py +++ b/pbr/hooks/commands.py @@ -20,6 +20,7 @@ import os from setuptools.command import easy_install from pbr.hooks import base +from pbr import options from pbr import packaging @@ -46,8 +47,8 @@ class CommandsConfig(base.BaseConfig): easy_install.get_script_args = packaging.override_get_script_args if packaging.have_sphinx(): - self.add_command('pbr.packaging.LocalBuildDoc') - self.add_command('pbr.packaging.LocalBuildLatex') + self.add_command('pbr.builddoc.LocalBuildDoc') + self.add_command('pbr.builddoc.LocalBuildLatex') if os.path.exists('.testr.conf') and packaging.have_testr(): # There is a .testr.conf file. We want to use it. @@ -56,7 +57,7 @@ class CommandsConfig(base.BaseConfig): # We seem to still have nose configured self.add_command('pbr.packaging.NoseTest') - use_egg = packaging.get_boolean_option( + use_egg = options.get_boolean_option( self.pbr_config, 'use-egg', 'PBR_USE_EGG') # We always want non-egg install unless explicitly requested if 'manpages' in self.pbr_config or not use_egg: diff --git a/pbr/options.py b/pbr/options.py new file mode 100644 index 0000000..5a7023c --- /dev/null +++ b/pbr/options.py @@ -0,0 +1,48 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright (C) 2013 Association of Universities for Research in Astronomy +# (AURA) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# 3. The name of AURA and its representatives may not be used to +# endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY AURA ``AS IS'' AND ANY EXPRESS OR IMPLIED +# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL AURA BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + +import os + + +TRUE_VALUES = ('true', '1', 'yes') + + +def get_boolean_option(option_dict, option_name, env_name): + return ((option_name in option_dict + and option_dict[option_name][1].lower() in TRUE_VALUES) or + str(os.getenv(env_name)).lower() in TRUE_VALUES) diff --git a/pbr/packaging.py b/pbr/packaging.py index 5bd3b06..820cb9c 100644 --- a/pbr/packaging.py +++ b/pbr/packaging.py @@ -40,14 +40,9 @@ from setuptools.command import install from setuptools.command import install_scripts from setuptools.command import sdist -try: - import cStringIO -except ImportError: - import io as cStringIO - from pbr import extra_files +from pbr import options -TRUE_VALUES = ('true', '1', 'yes') REQUIREMENTS_FILES = ('requirements.txt', 'tools/pip-requires') TEST_REQUIREMENTS_FILES = ('test-requirements.txt', 'tools/test-requires') @@ -76,7 +71,7 @@ def append_text_list(config, key, text_list): def _pip_install(links, requires, root=None, option_dict=dict()): - if get_boolean_option( + if options.get_boolean_option( option_dict, 'skip_pip_install', 'SKIP_PIP_INSTALL'): return cmd = [sys.executable, '-m', 'pip.__init__', 'install'] @@ -236,17 +231,11 @@ def _get_highest_tag(tags): return max(tags, key=pkg_resources.parse_version) -def get_boolean_option(option_dict, option_name, env_name): - return ((option_name in option_dict - and option_dict[option_name][1].lower() in TRUE_VALUES) or - str(os.getenv(env_name)).lower() in TRUE_VALUES) - - def write_git_changelog(git_dir=None, dest_dir=os.path.curdir, option_dict=dict()): """Write a changelog based on the git changelog.""" - should_skip = get_boolean_option(option_dict, 'skip_changelog', - 'SKIP_WRITE_GIT_CHANGELOG') + should_skip = options.get_boolean_option(option_dict, 'skip_changelog', + 'SKIP_WRITE_GIT_CHANGELOG') if should_skip: return @@ -303,8 +292,8 @@ def write_git_changelog(git_dir=None, dest_dir=os.path.curdir, def generate_authors(git_dir=None, dest_dir='.', option_dict=dict()): """Create AUTHORS file using git commits.""" - should_skip = get_boolean_option(option_dict, 'skip_authors', - 'SKIP_GENERATE_AUTHORS') + should_skip = options.get_boolean_option(option_dict, 'skip_authors', + 'SKIP_GENERATE_AUTHORS') if should_skip: return @@ -360,23 +349,6 @@ def _find_git_files(dirname='', git_dir=None): return [f for f in file_list if f] -_rst_template = """%(heading)s -%(underline)s - -.. automodule:: %(module)s - :members: - :undoc-members: - :show-inheritance: -""" - - -def _find_modules(arg, dirname, files): - for filename in files: - if filename.endswith('.py') and filename != '__init__.py': - arg["%s.%s" % (dirname.replace('/', '.'), - filename[:-3])] = True - - class LocalInstall(install.install): """Runs python setup.py install in a sensible manner. @@ -583,8 +555,8 @@ class LocalManifestMaker(egg_info.manifest_maker): self.filelist.append(self.template) self.filelist.append(self.manifest) self.filelist.extend(extra_files.get_extra_files()) - should_skip = get_boolean_option(option_dict, 'skip_git_sdist', - 'SKIP_GIT_SDIST') + should_skip = options.get_boolean_option(option_dict, 'skip_git_sdist', + 'SKIP_GIT_SDIST') if not should_skip: rcfiles = _find_git_files() if rcfiles: @@ -635,142 +607,16 @@ class LocalSDist(sdist.sdist): sdist.sdist.run(self) try: - from sphinx import apidoc - from sphinx import application - from sphinx import config - from sphinx import setup_command - - class LocalBuildDoc(setup_command.BuildDoc): - - command_name = 'build_sphinx' - builders = ['html', 'man'] - - def _get_source_dir(self): - option_dict = self.distribution.get_option_dict('build_sphinx') - if 'source_dir' in option_dict: - source_dir = os.path.join(option_dict['source_dir'][1], 'api') - else: - source_dir = 'doc/source/api' - if not os.path.exists(source_dir): - os.makedirs(source_dir) - return source_dir - - def generate_autoindex(self): - log.info("[pbr] Autodocumenting from %s" - % os.path.abspath(os.curdir)) - modules = {} - source_dir = self._get_source_dir() - for pkg in self.distribution.packages: - if '.' not in pkg: - for dirpath, dirnames, files in os.walk(pkg): - _find_modules(modules, dirpath, files) - module_list = list(modules.keys()) - module_list.sort() - autoindex_filename = os.path.join(source_dir, 'autoindex.rst') - with open(autoindex_filename, 'w') as autoindex: - autoindex.write(""".. toctree:: - :maxdepth: 1 - -""") - for module in module_list: - output_filename = os.path.join(source_dir, - "%s.rst" % module) - heading = "The :mod:`%s` Module" % module - underline = "=" * len(heading) - values = dict(module=module, heading=heading, - underline=underline) - - log.info("[pbr] Generating %s" - % output_filename) - with open(output_filename, 'w') as output_file: - output_file.write(_rst_template % values) - autoindex.write(" %s.rst\n" % module) - - def _sphinx_tree(self): - source_dir = self._get_source_dir() - apidoc.main(['apidoc', '.', '-H', 'Modules', '-o', source_dir]) - - def _sphinx_run(self): - if not self.verbose: - status_stream = cStringIO.StringIO() - else: - status_stream = sys.stdout - confoverrides = {} - if self.version: - confoverrides['version'] = self.version - if self.release: - confoverrides['release'] = self.release - if self.today: - confoverrides['today'] = self.today - sphinx_config = config.Config(self.config_dir, 'conf.py', {}, []) - sphinx_config.init_values() - if self.builder == 'man' and len(sphinx_config.man_pages) == 0: - return - app = application.Sphinx( - self.source_dir, self.config_dir, - self.builder_target_dir, self.doctree_dir, - self.builder, confoverrides, status_stream, - freshenv=self.fresh_env, warningiserror=True) - - try: - app.build(force_all=self.all_files) - except Exception as err: - from docutils import utils - if isinstance(err, utils.SystemMessage): - sys.stder.write('reST markup error:\n') - sys.stderr.write(err.args[0].encode('ascii', - 'backslashreplace')) - sys.stderr.write('\n') - else: - raise - - if self.link_index: - src = app.config.master_doc + app.builder.out_suffix - dst = app.builder.get_outfilename('index') - os.symlink(src, dst) - - def run(self): - option_dict = self.distribution.get_option_dict('pbr') - tree_index = get_boolean_option(option_dict, - 'autodoc_tree_index_modules', - 'AUTODOC_TREE_INDEX_MODULES') - auto_index = get_boolean_option(option_dict, - 'autodoc_index_modules', - 'AUTODOC_INDEX_MODULES') - if not os.getenv('SPHINX_DEBUG'): - #NOTE(afazekas): These options can be used together, - # but they do a very similar thing in a difffernet way - if tree_index: - self._sphinx_tree() - if auto_index: - self.generate_autoindex() - - for builder in self.builders: - self.builder = builder - self.finalize_options() - self.project = self.distribution.get_name() - self.version = self.distribution.get_version() - self.release = self.distribution.get_version() - if 'warnerrors' in option_dict: - self._sphinx_run() - else: - setup_command.BuildDoc.run(self) - - def finalize_options(self): - # Not a new style class, super keyword does not work. - setup_command.BuildDoc.finalize_options(self) - # Allow builders to be configurable - as a comma separated list. - if not isinstance(self.builders, list) and self.builders: - self.builders = self.builders.split(',') - - class LocalBuildLatex(LocalBuildDoc): - builders = ['latex'] - command_name = 'build_sphinx_latex' - + from pbr import builddoc _have_sphinx = True - + # Import the symbols from their new home so the package API stays + # compatible. + LocalBuildDoc = builddoc.LocalBuildDoc + LocalBuildLatex = builddoc.LocalBuildLatex except ImportError: _have_sphinx = False + LocalBuildDoc = None + LocalBuildLatex = None def have_sphinx(): diff --git a/pbr/tests/base.py b/pbr/tests/base.py index ce20de1..9134086 100644 --- a/pbr/tests/base.py +++ b/pbr/tests/base.py @@ -50,7 +50,7 @@ import fixtures import testresources import testtools -from pbr import packaging +from pbr import options class DiveDir(fixtures.Fixture): @@ -83,10 +83,10 @@ class BaseTestCase(testtools.TestCase, testresources.ResourcedTestCase): if test_timeout > 0: self.useFixture(fixtures.Timeout(test_timeout, gentle=True)) - if os.environ.get('OS_STDOUT_CAPTURE') in packaging.TRUE_VALUES: + if os.environ.get('OS_STDOUT_CAPTURE') in options.TRUE_VALUES: stdout = self.useFixture(fixtures.StringStream('stdout')).stream self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout)) - if os.environ.get('OS_STDERR_CAPTURE') in packaging.TRUE_VALUES: + if os.environ.get('OS_STDERR_CAPTURE') in options.TRUE_VALUES: stderr = self.useFixture(fixtures.StringStream('stderr')).stream self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr)) self.log_fixture = self.useFixture( diff --git a/pbr/util.py b/pbr/util.py index 8c7c2c9..ad76366 100644 --- a/pbr/util.py +++ b/pbr/util.py @@ -354,8 +354,8 @@ def setup_cfg_to_setup_kwargs(config): elif arg == 'cmdclass': cmdclass = {} dist = Distribution() - for cls in in_cfg_value: - cls = resolve_name(cls) + for cls_name in in_cfg_value: + cls = resolve_name(cls_name) cmd = cls(dist) cmdclass[cmd.get_command_name()] = cls in_cfg_value = cmdclass diff --git a/tox.ini b/tox.ini index 51d55e2..68057b2 100644 --- a/tox.ini +++ b/tox.ini @@ -1,13 +1,17 @@ [tox] minversion = 1.6 skipsdist = True -envlist = py26,py27,py33,pypy,pep8 +envlist = py33,py34,py26,py27,pypy,pep8 [testenv] usedevelop = True install_command = pip install {opts} {packages} setenv = VIRTUAL_ENV={envdir} -deps = -r{toxinidir}/requirements.txt +# NOTE(dhellmann): List ourself as a dependency first to ensure that +# the source being tested is used to install all of the other +# dependencies that want to use pbr for installation. +deps = . + -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt commands = python setup.py testr --testr-args='{posargs}' -- cgit v1.2.1