Add support for virtualenv 20.x

virtualenv is undergoing a rewrite and has changed how the programmatic
API works [1]. Switch to the "new way".

While we're here, we also need to get Python 2.7 tests passing again.
That requires dropping support for upper-constraints and using our own,
limited local constraints based on supported Python versions. We also
need to migrate integration tests since those run with Python 3 now.
Update the scenarios for pip/setuptools in integration testing
similarly. Finally clean up the installation of all devstack repos as
requirements are managed different now. Instead of worrying about
syncing them we use constraints.

[1] https://github.com/pypa/virtualenv/issues/1585#issuecomment-585228492

Change-Id: I493e88985d2c4d09612fea4d20d8ffa20043a0cb
Signed-off-by: Stephen Finucane <sfinucan@redhat.com>
Depends-On: https://review.opendev.org/739014

7 files changed, 78 insertions, 53 deletions

diff --git a/lower-constraints.txt b/lower-constraints.txt
index fc576a5..46ad80b 100644
--- a/lower-constraints.txt
+++ b/lower-constraints.txt
@@ -22,7 +22,7 @@ pytz==2013.6
 PyYAML==3.12
 reno==2.5.0
 requests==2.14.2
-six==1.10.0
+six==1.12.0
 snowballstemmer==1.2.1
 Sphinx==1.6.5
 sphinxcontrib-apidoc==0.2.0
@@ -34,4 +34,4 @@ testscenarios==0.4
 testtools==2.2.0
 traceback2==1.4.0
 unittest2==1.1.0
-virtualenv==14.0.6
+virtualenv==20.0.3
diff --git a/pbr/tests/test_core.py b/pbr/tests/test_core.py
index ccd14ab..edb7c7b 100644
--- a/pbr/tests/test_core.py
+++ b/pbr/tests/test_core.py
@@ -40,6 +40,7 @@
 
 import glob
 import os
+import sys
 import tarfile
 
 import fixtures
@@ -113,6 +114,12 @@ class TestCore(base.BaseTestCase):
     def test_console_script_develop(self):
         """Test that we develop a non-pkg-resources console script."""
 
+        if sys.version_info < (3, 0):
+            self.skipTest(
+                'Fails with recent virtualenv due to '
+                'https://github.com/pypa/virtualenv/issues/1638'
+            )
+
         if os.name == 'nt':
             self.skipTest('Windows support is passthrough')
 
diff --git a/pbr/tests/test_integration.py b/pbr/tests/test_integration.py
index 8e96f21..25473b0 100644
--- a/pbr/tests/test_integration.py
+++ b/pbr/tests/test_integration.py
@@ -11,7 +11,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+try:
+    import configparser
+except ImportError:
+    import ConfigParser as configparser
 import os.path
+import pkg_resources
 import shlex
 import sys
 
@@ -77,19 +82,35 @@ class TestIntegration(base.BaseTestCase):
         # We don't break these into separate tests because we'd need separate
         # source dirs to isolate from side effects of running pip, and the
         # overheads of setup would start to beat the benefits of parallelism.
-        self.useFixture(base.CapturedSubprocess(
-            'sync-req',
-            ['python', 'update.py', os.path.join(REPODIR, self.short_name)],
-            cwd=os.path.join(REPODIR, 'requirements')))
-        self.useFixture(base.CapturedSubprocess(
-            'commit-requirements',
-            'git diff --quiet || git commit -amrequirements',
-            cwd=os.path.join(REPODIR, self.short_name), shell=True))
-        path = os.path.join(
-            self.useFixture(fixtures.TempDir()).path, 'project')
-        self.useFixture(base.CapturedSubprocess(
-            'clone',
-            ['git', 'clone', os.path.join(REPODIR, self.short_name), path]))
+        path = os.path.join(REPODIR, self.short_name)
+        setup_cfg = os.path.join(path, 'setup.cfg')
+        project_name = pkg_resources.safe_name(self.short_name).lower()
+        # These projects should all have setup.cfg files but we'll be careful
+        if os.path.exists(setup_cfg):
+            config = configparser.ConfigParser()
+            config.read(setup_cfg)
+            if config.has_section('metadata'):
+                raw_name = config.get('metadata', 'name',
+                                      fallback='notapackagename')
+                # Technically we should really only need to use the raw
+                # name because all our projects should be good and use
+                # normalized names but they don't...
+                project_name = pkg_resources.safe_name(raw_name).lower()
+        constraints = os.path.join(REPODIR, 'requirements',
+                                   'upper-constraints.txt')
+        tmp_constraints = os.path.join(
+            self.useFixture(fixtures.TempDir()).path,
+            'upper-constraints.txt')
+        # We need to filter out the package we are installing to avoid
+        # conflicts with the constraints.
+        with open(constraints, 'r') as src:
+            with open(tmp_constraints, 'w') as dest:
+                for line in src:
+                    constraint = line.split('===')[0]
+                    if project_name != constraint:
+                        dest.write(line)
+        pip_cmd = PIP_CMD + ['-c', tmp_constraints]
+
         venv = self.useFixture(
             test_packaging.Venv('sdist',
                                 modules=['pip', 'wheel', PBRVERSION],
@@ -105,7 +126,7 @@ class TestIntegration(base.BaseTestCase):
         filename = os.path.join(
             path, 'dist', os.listdir(os.path.join(path, 'dist'))[0])
         self.useFixture(base.CapturedSubprocess(
-            'tarball', [python] + PIP_CMD + [filename]))
+            'tarball', [python] + pip_cmd + [filename]))
         venv = self.useFixture(
             test_packaging.Venv('install-git',
                                 modules=['pip', 'wheel', PBRVERSION],
@@ -113,7 +134,7 @@ class TestIntegration(base.BaseTestCase):
         root = venv.path
         python = venv.python
         self.useFixture(base.CapturedSubprocess(
-            'install-git', [python] + PIP_CMD + ['git+file://' + path]))
+            'install-git', [python] + pip_cmd + ['git+file://' + path]))
         if self.short_name == 'nova':
             found = False
             for _, _, filenames in os.walk(root):
@@ -127,7 +148,7 @@ class TestIntegration(base.BaseTestCase):
         root = venv.path
         python = venv.python
         self.useFixture(base.CapturedSubprocess(
-            'install-e', [python] + PIP_CMD + ['-e', path]))
+            'install-e', [python] + pip_cmd + ['-e', path]))
 
 
 class TestInstallWithoutPbr(base.BaseTestCase):
@@ -188,12 +209,16 @@ class TestInstallWithoutPbr(base.BaseTestCase):
 class TestMarkersPip(base.BaseTestCase):
 
     scenarios = [
-        ('pip-1.5', {'modules': ['pip>=1.5,<1.6']}),
-        ('pip-6.0', {'modules': ['pip>=6.0,<6.1']}),
         ('pip-latest', {'modules': ['pip']}),
-        ('setuptools-EL7', {'modules': ['pip==1.4.1', 'setuptools==0.9.8']}),
-        ('setuptools-Trusty', {'modules': ['pip==1.5', 'setuptools==2.2']}),
-        ('setuptools-minimum', {'modules': ['pip==1.5', 'setuptools==0.7.2']}),
+        ('setuptools-Bionic', {
+            'modules': ['pip==9.0.1', 'setuptools==39.0.1']}),
+        ('setuptools-Stretch', {
+            'modules': ['pip==9.0.1', 'setuptools==33.1.1']}),
+        ('setuptools-EL8', {'modules': ['pip==9.0.3', 'setuptools==39.2.0']}),
+        ('setuptools-Buster', {
+            'modules': ['pip==18.1', 'setuptools==40.8.0']}),
+        ('setuptools-Focal', {
+            'modules': ['pip==20.0.2', 'setuptools==45.2.0']}),
     ]
 
     @testtools.skipUnless(
@@ -240,25 +265,17 @@ class TestLTSSupport(base.BaseTestCase):
     # These versions come from the versions installed from the 'virtualenv'
     # command from the 'python-virtualenv' package.
     scenarios = [
-        ('EL7', {'modules': ['pip==1.4.1', 'setuptools==0.9.8'],
-                 'py3support': True}),  # And EPEL6
-        ('Trusty', {'modules': ['pip==1.5', 'setuptools==2.2'],
-                    'py3support': True}),
-        ('Jessie', {'modules': ['pip==1.5.6', 'setuptools==5.5.1'],
-                    'py3support': True}),
-        # Wheezy has pip1.1, which cannot be called with '-m pip'
-        # So we'll use a different version of pip here.
-        ('WheezyPrecise', {'modules': ['pip==1.4.1', 'setuptools==0.6c11'],
-                           'py3support': False})
+        ('Bionic', {'modules': ['pip==9.0.1', 'setuptools==39.0.1']}),
+        ('Stretch', {'modules': ['pip==9.0.1', 'setuptools==33.1.1']}),
+        ('EL8', {'modules': ['pip==9.0.3', 'setuptools==39.2.0']}),
+        ('Buster', {'modules': ['pip==18.1', 'setuptools==40.8.0']}),
+        ('Focal', {'modules': ['pip==20.0.2', 'setuptools==45.2.0']}),
     ]
 
     @testtools.skipUnless(
         os.environ.get('PBR_INTEGRATION', None) == '1',
         'integration tests not enabled')
     def test_lts_venv_default_versions(self):
-        if (sys.version_info[0] == 3 and not self.py3support):
-            self.skipTest('This combination will not install with py3, '
-                          'skipping test')
         venv = self.useFixture(
             test_packaging.Venv('setuptools', modules=self.modules))
         bin_python = venv.python
diff --git a/pbr/tests/test_packaging.py b/pbr/tests/test_packaging.py
index 07be547..3a70319 100644
--- a/pbr/tests/test_packaging.py
+++ b/pbr/tests/test_packaging.py
@@ -183,7 +183,8 @@ class Venv(fixtures.Fixture):
 
     def _setUp(self):
         path = self.useFixture(fixtures.TempDir()).path
-        virtualenv.create_environment(path, clear=True)
+        virtualenv.cli_run([path])
+
         python = os.path.join(path, 'bin', 'python')
         command = [python] + self.pip_cmd + ['-U']
         if self.modules and len(self.modules) > 0:
diff --git a/test-requirements.txt b/test-requirements.txt
index 09efec0..0358a79 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -6,12 +6,13 @@ wheel>=0.32.0 # MIT
 fixtures>=3.0.0 # Apache-2.0/BSD
 hacking>=1.1.0,<1.2.0 # Apache-2.0
 mock>=2.0.0 # BSD
-six>=1.10.0 # MIT
-stestr>=2.1.0 # Apache-2.0
+six>=1.12.0 # MIT
+stestr>=2.1.0,<3.0;python_version=='2.7' # Apache-2.0
+stestr>=2.1.0;python_version>='3.0' # Apache-2.0
 testresources>=2.0.0 # Apache-2.0/BSD
 testscenarios>=0.4 # Apache-2.0/BSD
 testtools>=2.2.0 # MIT
-virtualenv>=14.0.6 # MIT
+virtualenv>=20.0.3 # MIT
 coverage!=4.4,>=4.0 # Apache-2.0
 
 # optionally exposed by distutils commands
diff --git a/tools/integration.sh b/tools/integration.sh
index 6c4dc16..ff337ef 100644
--- a/tools/integration.sh
+++ b/tools/integration.sh
@@ -97,10 +97,6 @@ name = test_project
 [entry_points]
 console_scripts =
     test_cmd = test_project:main
-
-[global]
-setup-hooks =
-    pbr.hooks.setup_hook
 EOF
 
 cat <<EOF > setup.py
@@ -115,18 +111,19 @@ from socket import error as SocketError
 try:
     setuptools.setup(
         setup_requires=['pbr'],
-        pbr=True)
+        pbr=True,
+    )
 except (SocketError, Timeout):
     setuptools.setup(
         setup_requires=['pbr'],
-        pbr=True)
-
+        pbr=True,
+    )
 EOF
 
 mkdir test_project
 cat <<EOF > test_project/__init__.py
 def main():
-    print "Test cmd"
+    print("Test cmd")
 EOF
 
 epvenv=$eptest/venv
@@ -165,6 +162,8 @@ export REPODIR
 export WHEELHOUSE
 export OS_TEST_TIMEOUT=600
 cd $REPODIR/pbr
-tox -epy27 --notest
-.tox/py27/bin/python -m pip install ${REPODIR}/requirements
-tox -epy27 -- test_integration
+mkvenv .venv
+source .venv/bin/activate
+pip install -r test-requirements.txt
+pip install ${REPODIR}/requirements
+stestr run --suppress-attachments test_integration
diff --git a/tox.ini b/tox.ini
index d48c770..70f2127 100644
--- a/tox.ini
+++ b/tox.ini
@@ -11,8 +11,9 @@ setenv =
   OS_STDOUT_CAPTURE={env:OS_STDOUT_CAPTURE:1}
   OS_STDERR_CAPTURE={env:OS_STDERR_CAPTURE:1}
   OS_TEST_TIMEOUT={env:OS_TEST_TIMEOUT:60}
+# NOTE(stephenfin): pbr intentionally does not use constraints since we support
+# a broader range of Python versions than OpenStack as a whole
 deps =
-  -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
   -r{toxinidir}/test-requirements.txt
 commands = stestr run --suppress-attachments {posargs}
 
@@ -22,7 +23,6 @@ commands = flake8 {posargs}
 [testenv:docs]
 whitelist_externals = rm
 deps =
-  -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
   -r{toxinidir}/doc/requirements.txt
 commands =
   rm -rf doc/build doc/source/reference/api