Parallelise integration tests.

This involves moving the inner loop to python because
existing-tooling.

Change-Id: Iaad811a0248a3f700e655bd8be656d183deead93

6 files changed, 196 insertions, 75 deletions

diff --git a/.testr.conf b/.testr.conf
index 1641f86..b3cfc54 100644
--- a/.testr.conf
+++ b/.testr.conf
@@ -1,4 +1,4 @@
 [DEFAULT]
-test_command=OS_STDOUT_CAPTURE=1 OS_STDERR_CAPTURE=1 OS_TEST_TIMEOUT=60 ${PYTHON:-python} -m subunit.run discover -t ./ . $LISTOPT $IDOPTION
+test_command=OS_STDOUT_CAPTURE=1 OS_STDERR_CAPTURE=1 OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} ${PYTHON:-python} -m subunit.run discover -t ./ . $LISTOPT $IDOPTION
 test_id_option=--load-list $IDFILE
 test_list_option=--list
diff --git a/pbr/tests/base.py b/pbr/tests/base.py
index e1068f1..09bd642 100644
--- a/pbr/tests/base.py
+++ b/pbr/tests/base.py
@@ -161,3 +161,15 @@ def _run_cmd(args, cwd):
     for content in streams:
         print(content)
     return (streams) + (p.returncode,)
+
+
+def _config_git():
+    _run_cmd(
+        ['git', 'config', '--global', 'user.email', 'example@example.com'],
+        None)
+    _run_cmd(
+        ['git', 'config', '--global', 'user.name', 'OpenStack Developer'],
+        None)
+    _run_cmd(
+        ['git', 'config', '--global', 'user.signingkey',
+         'example@example.com'], None)
diff --git a/pbr/tests/test_integration.py b/pbr/tests/test_integration.py
new file mode 100644
index 0000000..dcc71df
--- /dev/null
+++ b/pbr/tests/test_integration.py
@@ -0,0 +1,165 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os.path
+import shlex
+import subprocess
+
+import fixtures
+import testscenarios
+import testtools
+from testtools import content
+import virtualenv
+
+from pbr.tests import base
+
+PIPFLAGS = shlex.split(os.environ.get('PIPFLAGS', ''))
+PIPVERSION = os.environ.get('PIPVERSION', 'pip')
+PBRVERSION = os.environ.get('PBRVERSION', 'pbr')
+REPODIR = os.environ.get('REPODIR', '')
+WHEELHOUSE = os.environ.get('WHEELHOUSE', '')
+PIP_CMD = ['-m', 'pip'] + PIPFLAGS + ['install', '-f', WHEELHOUSE]
+PROJECTS = shlex.split(os.environ.get('PROJECTS', ''))
+
+
+def all_projects():
+    if not REPODIR:
+        return
+    # Future: make this path parameterisable.
+    excludes = set(['pypi-mirror', 'jeepyb', 'tempest', 'requirements'])
+    for name in PROJECTS:
+        name = name.strip()
+        short_name = name.split('/')[-1]
+        try:
+            with open(os.path.join(
+                    REPODIR, short_name, 'setup.py'), 'rt') as f:
+                if 'pbr' not in f.read():
+                    continue
+        except IOError:
+            continue
+        if short_name in excludes:
+            continue
+        yield (short_name, dict(name=name, short_name=short_name))
+
+
+class CapturedSubprocess(fixtures.Fixture):
+    """Run a process and capture its output.
+
+    :attr stdout: The output (a string).
+    :attr stderr: The standard error (a string).
+    :attr returncode: The return code of the process.
+
+    Note that stdout and stderr are decoded from the bytestrings subprocess
+    returns using error=replace.
+    """
+
+    def __init__(self, label, *args, **kwargs):
+        """Create a CapturedSubprocess.
+
+        :param label: A label for the subprocess in the test log. E.g. 'foo'.
+        :param *args: The *args to pass to Popen.
+        :param **kwargs: The **kwargs to pass to Popen.
+        """
+        super(CapturedSubprocess, self).__init__()
+        self.label = label
+        self.args = args
+        self.kwargs = kwargs
+        self.kwargs['stderr'] = subprocess.PIPE
+        self.kwargs['stdin'] = subprocess.PIPE
+        self.kwargs['stdout'] = subprocess.PIPE
+
+    def setUp(self):
+        super(CapturedSubprocess, self).setUp()
+        proc = subprocess.Popen(*self.args, **self.kwargs)
+        out, err = proc.communicate()
+        self.out = out.decode('utf-8', 'replace')
+        self.err = err.decode('utf-8', 'replace')
+        self.addDetail(self.label + '-stdout', content.text_content(self.out))
+        self.addDetail(self.label + '-stderr', content.text_content(self.err))
+        self.returncode = proc.returncode
+        if proc.returncode:
+            raise AssertionError('Failed process %s' % proc.returncode)
+        self.addCleanup(delattr, self, 'out')
+        self.addCleanup(delattr, self, 'err')
+        self.addCleanup(delattr, self, 'returncode')
+
+
+class TestIntegration(base.BaseTestCase):
+
+    scenarios = list(all_projects())
+
+    def setUp(self):
+        # Integration tests need a higher default - big repos can be slow to
+        # clone, particularly under guest load.
+        os.environ['OS_TEST_TIMEOUT'] = os.environ.get('OS_TEST_TIMEOUT', 240)
+        super(TestIntegration, self).setUp()
+        base._config_git()
+
+    def venv(self, reason):
+        path = self.useFixture(fixtures.TempDir()).path
+        virtualenv.create_environment(path, clear=True)
+        python = os.path.join(path, 'bin', 'python')
+        self.useFixture(CapturedSubprocess(
+            'mkvenv-' + reason, [python] + PIP_CMD + [
+                '-U', PIPVERSION, 'wheel', PBRVERSION]))
+        return path, python
+
+    @testtools.skipUnless(
+        os.environ.get('PBR_INTEGRATION', None) == '1',
+        'integration tests not enabled')
+    def test_integration(self):
+        # Test that we can:
+        # - run sdist from the repo in a venv
+        # - install the resulting tarball in a new venv
+        # - pip install the repo
+        # - pip install -e the repo
+        # We don't break these into separate tests because we'd need separate
+        # source dirs to isolate from side effects of running pip, and the
+        # overheads of setup would start to beat the benefits of parallelism.
+        self.useFixture(CapturedSubprocess(
+            'sync-req',
+            ['python', 'update.py', os.path.join(REPODIR, self.short_name)],
+            cwd=os.path.join(REPODIR, 'requirements')))
+        self.useFixture(CapturedSubprocess(
+            'commit-requirements',
+            'git diff --quiet || git commit -amrequirements',
+            cwd=os.path.join(REPODIR, self.short_name), shell=True))
+        path = os.path.join(
+            self.useFixture(fixtures.TempDir()).path, 'project')
+        self.useFixture(CapturedSubprocess(
+            'clone',
+            ['git', 'clone', os.path.join(REPODIR, self.short_name), path]))
+        _, python = self.venv('sdist')
+        self.useFixture(CapturedSubprocess(
+            'sdist', [python, 'setup.py', 'sdist'], cwd=path))
+        _, python = self.venv('tarball')
+        filename = os.path.join(
+            path, 'dist', os.listdir(os.path.join(path, 'dist'))[0])
+        self.useFixture(CapturedSubprocess(
+            'tarball', [python] + PIP_CMD + [filename]))
+        root, python = self.venv('install-git')
+        self.useFixture(CapturedSubprocess(
+            'install-git', [python] + PIP_CMD + ['git+file://' + path]))
+        if self.short_name == 'nova':
+            found = False
+            for _, _, filenames in os.walk(root):
+                if 'migrate.cfg' in filenames:
+                    found = True
+            self.assertTrue(found)
+        _, python = self.venv('install-e')
+        self.useFixture(CapturedSubprocess(
+            'install-e', [python] + PIP_CMD + ['-e', path]))
+
+
+def load_tests(loader, in_tests, pattern):
+    return testscenarios.load_tests_apply_scenarios(loader, in_tests, pattern)
diff --git a/pbr/tests/test_packaging.py b/pbr/tests/test_packaging.py
index 3b19516..741934e 100644
--- a/pbr/tests/test_packaging.py
+++ b/pbr/tests/test_packaging.py
@@ -67,15 +67,7 @@ class TestRepo(fixtures.Fixture):
     def setUp(self):
         super(TestRepo, self).setUp()
         base._run_cmd(['git', 'init', '.'], self._basedir)
-        base._run_cmd(
-            ['git', 'config', '--global', 'user.email', 'example@example.com'],
-            self._basedir)
-        base._run_cmd(
-            ['git', 'config', '--global', 'user.name', 'OpenStack Developer'],
-            self._basedir)
-        base._run_cmd(
-            ['git', 'config', '--global', 'user.signingkey',
-             'example@example.com'], self._basedir)
+        base._config_git()
         base._run_cmd(['git', 'add', '.'], self._basedir)
 
     def commit(self, message_content='test commit'):
diff --git a/test-requirements.txt b/test-requirements.txt
index 75ad928..5f8cfb8 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -9,3 +9,4 @@ testrepository>=0.0.18
 testresources>=0.2.4
 testscenarios>=0.4
 testtools>=0.9.34
+virtualenv
diff --git a/tools/integration.sh b/tools/integration.sh
index c03c6f3..e58267e 100644
--- a/tools/integration.sh
+++ b/tools/integration.sh
@@ -148,68 +148,19 @@ $epvenv/bin/test_cmd | grep 'Test cmd'
 
 projectdir=$tmpdir/projects
 mkdir -p $projectdir
-
-for PROJECT in $PROJECTS ; do
-    SHORT_PROJECT=$(basename $PROJECT)
-    if ! grep 'pbr' $REPODIR/$SHORT_PROJECT/setup.py >/dev/null 2>&1
-    then
-        # project doesn't use pbr
-        continue
-    fi
-    if [ $SHORT_PROJECT = 'pypi-mirror' ]; then
-        # pypi-mirror doesn't consume the mirror
-        continue
-    fi
-    if [ $SHORT_PROJECT = 'jeepyb' ]; then
-        # pypi-mirror doesn't consume the mirror
-        continue
-    fi
-    if [ $SHORT_PROJECT = 'tempest' ]; then
-        # Tempest doesn't really install
-        continue
-    fi
-    if [ $SHORT_PROJECT = 'requirements' ]; then
-        # requirements doesn't really install
-        continue
-    fi
-
-    # set up the project synced with the global requirements
-    sudo chown -R $USER $REPODIR/$SHORT_PROJECT
-    (cd $REPODIR/requirements && python update.py $REPODIR/$SHORT_PROJECT)
-    pushd $REPODIR/$SHORT_PROJECT
-    if ! git diff --quiet ; then
-        git commit -a -m'Update requirements'
-    fi
-    popd
-
-    # Clone from synced repo
-    shortprojectdir=$projectdir/$SHORT_PROJECT
-    git clone $REPODIR/$SHORT_PROJECT $shortprojectdir
-
-    # Test that we can make a tarball from scratch
-    sdistvenv=$tmpdir/sdist
-    mkvenv $sdistvenv
-    cd $shortprojectdir
-    $sdistvenv/bin/python setup.py sdist
-
-    cd $tmpdir
-
-    # Test that the tarball installs
-    tarballvenv=$tmpdir/tarball
-    mkvenv $tarballvenv
-    $tarballvenv/bin/pip $PIPFLAGS install -f $WHEELHOUSE $shortprojectdir/dist/*tar.gz
-
-    # Test pip installing
-    pipvenv=$tmpdir/pip
-    mkvenv $pipvenv
-    $pipvenv/bin/pip $PIPFLAGS install -f $WHEELHOUSE git+file://$shortprojectdir
-    # Ensure the install_package_data is doing the thing it should do
-    if [ $SHORT_PROJECT = 'nova' ]; then
-        find $pipvenv | grep migrate.cfg
-    fi
-
-    # Test pip install -e
-    pipvenv=$tmpdir/pip
-    mkvenv $pipvenv
-    $pipvenv/bin/pip $PIPFLAGS install -f $WHEELHOUSE -e $shortprojectdir
-done
+sudo chown -R $USER $REPODIR
+
+export PBR_INTEGRATION=1
+export PIPFLAGS
+export PIPVERSION
+PBRVERSION=pbr
+if [ -n "$PBR_CHANGE" ] ; then
+    PBRVERSION=$(ls $pbrsdistdir/dist/pbr-*.whl)
+fi
+export PBRVERSION
+export PROJECTS
+export REPODIR
+export WHEELHOUSE
+export OS_TEST_TIMEOUT=240
+cd $REPODIR/pbr
+tox -epy27 -- test_integration