Only import sphinx during hook processing

When pbr is imported to handle writing the egg_info file because of
the entry point, it's causing sphinx to get imported. This has a
cascading effect once docutils is trying to be installed on a
system with pbr installed. If some of the imports fail along the way,
allow pbr to continue usefully but without the Sphinx extensions
available. Eventually, when everything is installed, those extensions
will work again when the commands for build_sphinx, etc. are run
separately.

Also slip in a change to reorder the default list of environments run by
tox so the testr database is created using a dbm format available to all
python versions.

Change-Id: I79d67bf41a09d7e5aad8ed32eaf107f139167eb8
Closes-bug: #1403510

7 files changed, 267 insertions, 179 deletions

diff --git a/pbr/builddoc.py b/pbr/builddoc.py
new file mode 100644
index 0000000..7242deb
--- /dev/null
+++ b/pbr/builddoc.py
@@ -0,0 +1,189 @@
+# Copyright 2011 OpenStack LLC.
+# Copyright 2012-2013 Hewlett-Packard Development Company, L.P.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from distutils import log
+import os
+import sys
+
+try:
+    import cStringIO
+except ImportError:
+    import io as cStringIO
+
+try:
+    from sphinx import apidoc
+    from sphinx import application
+    from sphinx import config
+    from sphinx import setup_command
+except Exception as e:
+    # NOTE(dhellmann): During the installation of docutils, setuptools
+    # tries to import pbr code to find the egg_info.writer hooks. That
+    # imports this module, which imports sphinx, which imports
+    # docutils, which is being installed. Because docutils uses 2to3
+    # to convert its code during installation under python 3, the
+    # import fails, but it fails with an error other than ImportError
+    # (today it's a NameError on StandardError, an exception base
+    # class). Convert the exception type here so it can be caught in
+    # packaging.py where we try to determine if we can import and use
+    # sphinx by importing this module.  See bug #1403510 for details.
+    raise ImportError(str(e))
+from pbr import options
+
+
+_rst_template = """%(heading)s
+%(underline)s
+
+.. automodule:: %(module)s
+  :members:
+  :undoc-members:
+  :show-inheritance:
+"""
+
+
+def _find_modules(arg, dirname, files):
+    for filename in files:
+        if filename.endswith('.py') and filename != '__init__.py':
+            arg["%s.%s" % (dirname.replace('/', '.'),
+                           filename[:-3])] = True
+
+
+class LocalBuildDoc(setup_command.BuildDoc):
+
+    command_name = 'build_sphinx'
+    builders = ['html', 'man']
+
+    def _get_source_dir(self):
+        option_dict = self.distribution.get_option_dict('build_sphinx')
+        if 'source_dir' in option_dict:
+            source_dir = os.path.join(option_dict['source_dir'][1], 'api')
+        else:
+            source_dir = 'doc/source/api'
+        if not os.path.exists(source_dir):
+            os.makedirs(source_dir)
+        return source_dir
+
+    def generate_autoindex(self):
+        log.info("[pbr] Autodocumenting from %s"
+                 % os.path.abspath(os.curdir))
+        modules = {}
+        source_dir = self._get_source_dir()
+        for pkg in self.distribution.packages:
+            if '.' not in pkg:
+                for dirpath, dirnames, files in os.walk(pkg):
+                    _find_modules(modules, dirpath, files)
+        module_list = list(modules.keys())
+        module_list.sort()
+        autoindex_filename = os.path.join(source_dir, 'autoindex.rst')
+        with open(autoindex_filename, 'w') as autoindex:
+            autoindex.write(""".. toctree::
+   :maxdepth: 1
+
+""")
+            for module in module_list:
+                output_filename = os.path.join(source_dir,
+                                               "%s.rst" % module)
+                heading = "The :mod:`%s` Module" % module
+                underline = "=" * len(heading)
+                values = dict(module=module, heading=heading,
+                              underline=underline)
+
+                log.info("[pbr] Generating %s"
+                         % output_filename)
+                with open(output_filename, 'w') as output_file:
+                    output_file.write(_rst_template % values)
+                autoindex.write("   %s.rst\n" % module)
+
+    def _sphinx_tree(self):
+            source_dir = self._get_source_dir()
+            apidoc.main(['apidoc', '.', '-H', 'Modules', '-o', source_dir])
+
+    def _sphinx_run(self):
+        if not self.verbose:
+            status_stream = cStringIO.StringIO()
+        else:
+            status_stream = sys.stdout
+        confoverrides = {}
+        if self.version:
+            confoverrides['version'] = self.version
+        if self.release:
+            confoverrides['release'] = self.release
+        if self.today:
+            confoverrides['today'] = self.today
+        sphinx_config = config.Config(self.config_dir, 'conf.py', {}, [])
+        sphinx_config.init_values()
+        if self.builder == 'man' and len(sphinx_config.man_pages) == 0:
+            return
+        app = application.Sphinx(
+            self.source_dir, self.config_dir,
+            self.builder_target_dir, self.doctree_dir,
+            self.builder, confoverrides, status_stream,
+            freshenv=self.fresh_env, warningiserror=True)
+
+        try:
+            app.build(force_all=self.all_files)
+        except Exception as err:
+            from docutils import utils
+            if isinstance(err, utils.SystemMessage):
+                sys.stder.write('reST markup error:\n')
+                sys.stderr.write(err.args[0].encode('ascii',
+                                                    'backslashreplace'))
+                sys.stderr.write('\n')
+            else:
+                raise
+
+        if self.link_index:
+            src = app.config.master_doc + app.builder.out_suffix
+            dst = app.builder.get_outfilename('index')
+            os.symlink(src, dst)
+
+    def run(self):
+        option_dict = self.distribution.get_option_dict('pbr')
+        tree_index = options.get_boolean_option(option_dict,
+                                                'autodoc_tree_index_modules',
+                                                'AUTODOC_TREE_INDEX_MODULES')
+        auto_index = options.get_boolean_option(option_dict,
+                                                'autodoc_index_modules',
+                                                'AUTODOC_INDEX_MODULES')
+        if not os.getenv('SPHINX_DEBUG'):
+            #NOTE(afazekas): These options can be used together,
+            # but they do a very similar thing in a difffernet way
+            if tree_index:
+                self._sphinx_tree()
+            if auto_index:
+                self.generate_autoindex()
+
+        for builder in self.builders:
+            self.builder = builder
+            self.finalize_options()
+            self.project = self.distribution.get_name()
+            self.version = self.distribution.get_version()
+            self.release = self.distribution.get_version()
+            if 'warnerrors' in option_dict:
+                self._sphinx_run()
+            else:
+                setup_command.BuildDoc.run(self)
+
+    def finalize_options(self):
+        # Not a new style class, super keyword does not work.
+        setup_command.BuildDoc.finalize_options(self)
+        # Allow builders to be configurable - as a comma separated list.
+        if not isinstance(self.builders, list) and self.builders:
+            self.builders = self.builders.split(',')
+
+
+class LocalBuildLatex(LocalBuildDoc):
+    builders = ['latex']
+    command_name = 'build_sphinx_latex'
diff --git a/pbr/hooks/commands.py b/pbr/hooks/commands.py
index b4206ed..3033119 100644
--- a/pbr/hooks/commands.py
+++ b/pbr/hooks/commands.py
@@ -20,6 +20,7 @@ import os
 from setuptools.command import easy_install
 
 from pbr.hooks import base
+from pbr import options
 from pbr import packaging
 
 
@@ -46,8 +47,8 @@ class CommandsConfig(base.BaseConfig):
             easy_install.get_script_args = packaging.override_get_script_args
 
         if packaging.have_sphinx():
-            self.add_command('pbr.packaging.LocalBuildDoc')
-            self.add_command('pbr.packaging.LocalBuildLatex')
+            self.add_command('pbr.builddoc.LocalBuildDoc')
+            self.add_command('pbr.builddoc.LocalBuildLatex')
 
         if os.path.exists('.testr.conf') and packaging.have_testr():
             # There is a .testr.conf file. We want to use it.
@@ -56,7 +57,7 @@ class CommandsConfig(base.BaseConfig):
             # We seem to still have nose configured
             self.add_command('pbr.packaging.NoseTest')
 
-        use_egg = packaging.get_boolean_option(
+        use_egg = options.get_boolean_option(
             self.pbr_config, 'use-egg', 'PBR_USE_EGG')
         # We always want non-egg install unless explicitly requested
         if 'manpages' in self.pbr_config or not use_egg:
diff --git a/pbr/options.py b/pbr/options.py
new file mode 100644
index 0000000..5a7023c
--- /dev/null
+++ b/pbr/options.py
@@ -0,0 +1,48 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Copyright (C) 2013 Association of Universities for Research in Astronomy
+#                    (AURA)
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+#     1. Redistributions of source code must retain the above copyright
+#        notice, this list of conditions and the following disclaimer.
+#
+#     2. Redistributions in binary form must reproduce the above
+#        copyright notice, this list of conditions and the following
+#        disclaimer in the documentation and/or other materials provided
+#        with the distribution.
+#
+#     3. The name of AURA and its representatives may not be used to
+#        endorse or promote products derived from this software without
+#        specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY AURA ``AS IS'' AND ANY EXPRESS OR IMPLIED
+# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL AURA BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+
+import os
+
+
+TRUE_VALUES = ('true', '1', 'yes')
+
+
+def get_boolean_option(option_dict, option_name, env_name):
+    return ((option_name in option_dict
+             and option_dict[option_name][1].lower() in TRUE_VALUES) or
+            str(os.getenv(env_name)).lower() in TRUE_VALUES)
diff --git a/pbr/packaging.py b/pbr/packaging.py
index 5bd3b06..820cb9c 100644
--- a/pbr/packaging.py
+++ b/pbr/packaging.py
@@ -40,14 +40,9 @@ from setuptools.command import install
 from setuptools.command import install_scripts
 from setuptools.command import sdist
 
-try:
-    import cStringIO
-except ImportError:
-    import io as cStringIO
-
 from pbr import extra_files
+from pbr import options
 
-TRUE_VALUES = ('true', '1', 'yes')
 REQUIREMENTS_FILES = ('requirements.txt', 'tools/pip-requires')
 TEST_REQUIREMENTS_FILES = ('test-requirements.txt', 'tools/test-requires')
 
@@ -76,7 +71,7 @@ def append_text_list(config, key, text_list):
 
 
 def _pip_install(links, requires, root=None, option_dict=dict()):
-    if get_boolean_option(
+    if options.get_boolean_option(
             option_dict, 'skip_pip_install', 'SKIP_PIP_INSTALL'):
         return
     cmd = [sys.executable, '-m', 'pip.__init__', 'install']
@@ -236,17 +231,11 @@ def _get_highest_tag(tags):
     return max(tags, key=pkg_resources.parse_version)
 
 
-def get_boolean_option(option_dict, option_name, env_name):
-    return ((option_name in option_dict
-             and option_dict[option_name][1].lower() in TRUE_VALUES) or
-            str(os.getenv(env_name)).lower() in TRUE_VALUES)
-
-
 def write_git_changelog(git_dir=None, dest_dir=os.path.curdir,
                         option_dict=dict()):
     """Write a changelog based on the git changelog."""
-    should_skip = get_boolean_option(option_dict, 'skip_changelog',
-                                     'SKIP_WRITE_GIT_CHANGELOG')
+    should_skip = options.get_boolean_option(option_dict, 'skip_changelog',
+                                             'SKIP_WRITE_GIT_CHANGELOG')
     if should_skip:
         return
 
@@ -303,8 +292,8 @@ def write_git_changelog(git_dir=None, dest_dir=os.path.curdir,
 
 def generate_authors(git_dir=None, dest_dir='.', option_dict=dict()):
     """Create AUTHORS file using git commits."""
-    should_skip = get_boolean_option(option_dict, 'skip_authors',
-                                     'SKIP_GENERATE_AUTHORS')
+    should_skip = options.get_boolean_option(option_dict, 'skip_authors',
+                                             'SKIP_GENERATE_AUTHORS')
     if should_skip:
         return
 
@@ -360,23 +349,6 @@ def _find_git_files(dirname='', git_dir=None):
     return [f for f in file_list if f]
 
 
-_rst_template = """%(heading)s
-%(underline)s
-
-.. automodule:: %(module)s
-  :members:
-  :undoc-members:
-  :show-inheritance:
-"""
-
-
-def _find_modules(arg, dirname, files):
-    for filename in files:
-        if filename.endswith('.py') and filename != '__init__.py':
-            arg["%s.%s" % (dirname.replace('/', '.'),
-                           filename[:-3])] = True
-
-
 class LocalInstall(install.install):
     """Runs python setup.py install in a sensible manner.
 
@@ -583,8 +555,8 @@ class LocalManifestMaker(egg_info.manifest_maker):
         self.filelist.append(self.template)
         self.filelist.append(self.manifest)
         self.filelist.extend(extra_files.get_extra_files())
-        should_skip = get_boolean_option(option_dict, 'skip_git_sdist',
-                                         'SKIP_GIT_SDIST')
+        should_skip = options.get_boolean_option(option_dict, 'skip_git_sdist',
+                                                 'SKIP_GIT_SDIST')
         if not should_skip:
             rcfiles = _find_git_files()
             if rcfiles:
@@ -635,142 +607,16 @@ class LocalSDist(sdist.sdist):
         sdist.sdist.run(self)
 
 try:
-    from sphinx import apidoc
-    from sphinx import application
-    from sphinx import config
-    from sphinx import setup_command
-
-    class LocalBuildDoc(setup_command.BuildDoc):
-
-        command_name = 'build_sphinx'
-        builders = ['html', 'man']
-
-        def _get_source_dir(self):
-            option_dict = self.distribution.get_option_dict('build_sphinx')
-            if 'source_dir' in option_dict:
-                source_dir = os.path.join(option_dict['source_dir'][1], 'api')
-            else:
-                source_dir = 'doc/source/api'
-            if not os.path.exists(source_dir):
-                os.makedirs(source_dir)
-            return source_dir
-
-        def generate_autoindex(self):
-            log.info("[pbr] Autodocumenting from %s"
-                     % os.path.abspath(os.curdir))
-            modules = {}
-            source_dir = self._get_source_dir()
-            for pkg in self.distribution.packages:
-                if '.' not in pkg:
-                    for dirpath, dirnames, files in os.walk(pkg):
-                        _find_modules(modules, dirpath, files)
-            module_list = list(modules.keys())
-            module_list.sort()
-            autoindex_filename = os.path.join(source_dir, 'autoindex.rst')
-            with open(autoindex_filename, 'w') as autoindex:
-                autoindex.write(""".. toctree::
-   :maxdepth: 1
-
-""")
-                for module in module_list:
-                    output_filename = os.path.join(source_dir,
-                                                   "%s.rst" % module)
-                    heading = "The :mod:`%s` Module" % module
-                    underline = "=" * len(heading)
-                    values = dict(module=module, heading=heading,
-                                  underline=underline)
-
-                    log.info("[pbr] Generating %s"
-                             % output_filename)
-                    with open(output_filename, 'w') as output_file:
-                        output_file.write(_rst_template % values)
-                    autoindex.write("   %s.rst\n" % module)
-
-        def _sphinx_tree(self):
-                source_dir = self._get_source_dir()
-                apidoc.main(['apidoc', '.', '-H', 'Modules', '-o', source_dir])
-
-        def _sphinx_run(self):
-            if not self.verbose:
-                status_stream = cStringIO.StringIO()
-            else:
-                status_stream = sys.stdout
-            confoverrides = {}
-            if self.version:
-                confoverrides['version'] = self.version
-            if self.release:
-                confoverrides['release'] = self.release
-            if self.today:
-                confoverrides['today'] = self.today
-            sphinx_config = config.Config(self.config_dir, 'conf.py', {}, [])
-            sphinx_config.init_values()
-            if self.builder == 'man' and len(sphinx_config.man_pages) == 0:
-                return
-            app = application.Sphinx(
-                self.source_dir, self.config_dir,
-                self.builder_target_dir, self.doctree_dir,
-                self.builder, confoverrides, status_stream,
-                freshenv=self.fresh_env, warningiserror=True)
-
-            try:
-                app.build(force_all=self.all_files)
-            except Exception as err:
-                from docutils import utils
-                if isinstance(err, utils.SystemMessage):
-                    sys.stder.write('reST markup error:\n')
-                    sys.stderr.write(err.args[0].encode('ascii',
-                                                        'backslashreplace'))
-                    sys.stderr.write('\n')
-                else:
-                    raise
-
-            if self.link_index:
-                src = app.config.master_doc + app.builder.out_suffix
-                dst = app.builder.get_outfilename('index')
-                os.symlink(src, dst)
-
-        def run(self):
-            option_dict = self.distribution.get_option_dict('pbr')
-            tree_index = get_boolean_option(option_dict,
-                                            'autodoc_tree_index_modules',
-                                            'AUTODOC_TREE_INDEX_MODULES')
-            auto_index = get_boolean_option(option_dict,
-                                            'autodoc_index_modules',
-                                            'AUTODOC_INDEX_MODULES')
-            if not os.getenv('SPHINX_DEBUG'):
-                #NOTE(afazekas): These options can be used together,
-                # but they do a very similar thing in a difffernet way
-                if tree_index:
-                    self._sphinx_tree()
-                if auto_index:
-                    self.generate_autoindex()
-
-            for builder in self.builders:
-                self.builder = builder
-                self.finalize_options()
-                self.project = self.distribution.get_name()
-                self.version = self.distribution.get_version()
-                self.release = self.distribution.get_version()
-                if 'warnerrors' in option_dict:
-                    self._sphinx_run()
-                else:
-                    setup_command.BuildDoc.run(self)
-
-        def finalize_options(self):
-            # Not a new style class, super keyword does not work.
-            setup_command.BuildDoc.finalize_options(self)
-            # Allow builders to be configurable - as a comma separated list.
-            if not isinstance(self.builders, list) and self.builders:
-                self.builders = self.builders.split(',')
-
-    class LocalBuildLatex(LocalBuildDoc):
-        builders = ['latex']
-        command_name = 'build_sphinx_latex'
-
+    from pbr import builddoc
     _have_sphinx = True
-
+    # Import the symbols from their new home so the package API stays
+    # compatible.
+    LocalBuildDoc = builddoc.LocalBuildDoc
+    LocalBuildLatex = builddoc.LocalBuildLatex
 except ImportError:
     _have_sphinx = False
+    LocalBuildDoc = None
+    LocalBuildLatex = None
 
 
 def have_sphinx():
diff --git a/pbr/tests/base.py b/pbr/tests/base.py
index ce20de1..9134086 100644
--- a/pbr/tests/base.py
+++ b/pbr/tests/base.py
@@ -50,7 +50,7 @@ import fixtures
 import testresources
 import testtools
 
-from pbr import packaging
+from pbr import options
 
 
 class DiveDir(fixtures.Fixture):
@@ -83,10 +83,10 @@ class BaseTestCase(testtools.TestCase, testresources.ResourcedTestCase):
         if test_timeout > 0:
             self.useFixture(fixtures.Timeout(test_timeout, gentle=True))
 
-        if os.environ.get('OS_STDOUT_CAPTURE') in packaging.TRUE_VALUES:
+        if os.environ.get('OS_STDOUT_CAPTURE') in options.TRUE_VALUES:
             stdout = self.useFixture(fixtures.StringStream('stdout')).stream
             self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
-        if os.environ.get('OS_STDERR_CAPTURE') in packaging.TRUE_VALUES:
+        if os.environ.get('OS_STDERR_CAPTURE') in options.TRUE_VALUES:
             stderr = self.useFixture(fixtures.StringStream('stderr')).stream
             self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
         self.log_fixture = self.useFixture(
diff --git a/pbr/util.py b/pbr/util.py
index 8c7c2c9..ad76366 100644
--- a/pbr/util.py
+++ b/pbr/util.py
@@ -354,8 +354,8 @@ def setup_cfg_to_setup_kwargs(config):
             elif arg == 'cmdclass':
                 cmdclass = {}
                 dist = Distribution()
-                for cls in in_cfg_value:
-                    cls = resolve_name(cls)
+                for cls_name in in_cfg_value:
+                    cls = resolve_name(cls_name)
                     cmd = cls(dist)
                     cmdclass[cmd.get_command_name()] = cls
                 in_cfg_value = cmdclass
diff --git a/tox.ini b/tox.ini
index 51d55e2..68057b2 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,13 +1,17 @@
 [tox]
 minversion = 1.6
 skipsdist = True
-envlist = py26,py27,py33,pypy,pep8
+envlist = py33,py34,py26,py27,pypy,pep8
 
 [testenv]
 usedevelop = True
 install_command = pip install {opts} {packages}
 setenv = VIRTUAL_ENV={envdir}
-deps = -r{toxinidir}/requirements.txt
+# NOTE(dhellmann): List ourself as a dependency first to ensure that
+# the source being tested is used to install all of the other
+# dependencies that want to use pbr for installation.
+deps = .
+       -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
 commands =
   python setup.py testr --testr-args='{posargs}'