Merge "Teach pbr to read extras and env markers"1.0.0

5 files changed, 235 insertions, 0 deletions

diff --git a/doc/source/index.rst b/doc/source/index.rst
index bc5441d..59fe16f 100644
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -123,6 +123,45 @@ version number used to install the package):
 Only the first file found is used to install the list of packages it
 contains.
 
+Extra requirements
+------------------
+
+Groups of optional dependencies (`"extra" requirements
+<https://www.python.org/dev/peps/pep-0426/#extras-optional-dependencies>`_)
+can be described in your setup.cfg, rather than needing to be added to
+setup.py. An example (which also demonstrates the use of environment
+markers) is shown below.
+
+Environment markers
+-------------------
+
+Environment markers are `conditional dependencies
+<https://www.python.org/dev/peps/pep-0426/#environment-markers>`_
+which can be added to the requirements (or to a group of extra
+requirements) automatically, depending on the environment the
+installer is running in. They can be added to requirements in the
+requirements file, or to extras definied in setup.cfg - but the format
+is slightly different for each.
+
+For ``requirements.txt``::
+
+    argparse; python=='2.6'
+
+will result in the package depending on ``argparse`` only if it's being
+installed into python2.6
+
+For extras specifed in setup.cfg, add an ``extras`` section. For
+instance, to create two groups of extra requirements with additional
+constraints on the environment, you can use::
+
+    [extras]
+    security =
+        aleph
+        bet :python_environment=='3.2'
+        gimel :python_environment=='2.7'
+    testing =
+        quux :python_environment=='2.7'
+
 long_description
 ----------------
 
diff --git a/pbr/tests/test_packaging.py b/pbr/tests/test_packaging.py
index 741934e..9e846d8 100644
--- a/pbr/tests/test_packaging.py
+++ b/pbr/tests/test_packaging.py
@@ -40,10 +40,14 @@
 
 import os
 import re
+import sys
 import tempfile
+import textwrap
 
 import fixtures
 import mock
+import pkg_resources
+import six
 import testscenarios
 from testtools import matchers
 
@@ -417,5 +421,57 @@ class TestVersions(base.BaseTestCase):
         self.assertEqual('1.3.0.0a1', version)
 
 
+class TestRequirementParsing(base.BaseTestCase):
+
+    def test_requirement_parsing(self):
+        tempdir = self.useFixture(fixtures.TempDir()).path
+        requirements = os.path.join(tempdir, 'requirements.txt')
+        with open(requirements, 'wt') as f:
+            f.write(textwrap.dedent(six.u("""\
+                bar
+                quux<1.0; python_version=='2.6'
+            """)))
+        setup_cfg = os.path.join(tempdir, 'setup.cfg')
+        with open(setup_cfg, 'wt') as f:
+            f.write(textwrap.dedent(six.u("""\
+                [metadata]
+                name = test_reqparse
+
+                [extras]
+                test =
+                    foo
+                    baz>3.2 :python_version=='2.7'
+            """)))
+        # pkg_resources.split_sections uses None as the title of an
+        # anonymous section instead of the empty string. Weird.
+        expected_requirements = {
+            None: ['bar'],
+            ":python_version=='2.6'": ['quux<1.0'],
+            "test:python_version=='2.7'": ['baz>3.2'],
+            "test": ['foo']
+        }
+        setup_py = os.path.join(tempdir, 'setup.py')
+        with open(setup_py, 'wt') as f:
+            f.write(textwrap.dedent(six.u("""\
+                #!/usr/bin/env python
+                import setuptools
+                setuptools.setup(
+                    setup_requires=['pbr'],
+                    pbr=True,
+                )
+            """)))
+
+        self._run_cmd(sys.executable, (setup_py, 'egg_info'),
+                      allow_fail=False, cwd=tempdir)
+        egg_info = os.path.join(tempdir, 'test_reqparse.egg-info')
+
+        requires_txt = os.path.join(egg_info, 'requires.txt')
+        with open(requires_txt, 'rt') as requires:
+            generated_requirements = dict(
+                pkg_resources.split_sections(requires))
+
+        self.assertEqual(expected_requirements, generated_requirements)
+
+
 def load_tests(loader, in_tests, pattern):
     return testscenarios.load_tests_apply_scenarios(loader, in_tests, pattern)
diff --git a/pbr/tests/test_util.py b/pbr/tests/test_util.py
new file mode 100644
index 0000000..7a4c6bd
--- /dev/null
+++ b/pbr/tests/test_util.py
@@ -0,0 +1,80 @@
+# Copyright (c) 2015 Hewlett-Packard Development Company, L.P. (HP)
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import io
+import textwrap
+
+import six
+from six.moves import configparser
+import testscenarios
+
+from pbr.tests import base
+from pbr import util
+
+
+class TestExtrasRequireParsingScenarios(base.BaseTestCase):
+
+    scenarios = [
+        ('simple_extras', {
+            'config_text': """
+                [extras]
+                first =
+                    foo
+                    bar==1.0
+                second =
+                    baz>=3.2
+                    foo
+                """,
+            'expected_extra_requires': {'first': ['foo', 'bar==1.0'],
+                                        'second': ['baz>=3.2', 'foo']}
+        }),
+        ('with_markers', {
+            'config_text': """
+                [extras]
+                test =
+                    foo:python_version=='2.6'
+                    bar
+                    baz<1.6 :python_version=='2.6'
+                """,
+            'expected_extra_requires': {
+                "test:python_version=='2.6'": ['foo', 'baz<1.6'],
+                "test": ['bar']}}),
+        ('no_extras', {
+            'config_text': """
+            [metadata]
+            long_description = foo
+            """,
+            'expected_extra_requires':
+            {}
+        })]
+
+    def config_from_ini(self, ini):
+        config = {}
+        parser = configparser.SafeConfigParser()
+        ini = textwrap.dedent(six.u(ini))
+        parser.readfp(io.StringIO(ini))
+        for section in parser.sections():
+            config[section] = dict(parser.items(section))
+        return config
+
+    def test_extras_parsing(self):
+        config = self.config_from_ini(self.config_text)
+        kwargs = util.setup_cfg_to_setup_kwargs(config)
+
+        self.assertEqual(self.expected_extra_requires,
+                         kwargs['extras_require'])
+
+
+def load_tests(loader, in_tests, pattern):
+    return testscenarios.load_tests_apply_scenarios(loader, in_tests, pattern)
diff --git a/pbr/util.py b/pbr/util.py
index 63566eb..929a234 100644
--- a/pbr/util.py
+++ b/pbr/util.py
@@ -280,6 +280,10 @@ def setup_cfg_to_setup_kwargs(config):
 
     kwargs = {}
 
+    # Temporarily holds install_reqires and extra_requires while we
+    # parse env_markers.
+    all_requirements = {}
+
     for arg in D1_D2_SETUP_ARGS:
         if len(D1_D2_SETUP_ARGS[arg]) == 2:
             # The distutils field name is different than distutils2's.
@@ -326,6 +330,17 @@ def setup_cfg_to_setup_kwargs(config):
                 # setuptools
                 in_cfg_value = [_VERSION_SPEC_RE.sub(r'\1\2', pred)
                                 for pred in in_cfg_value]
+            if arg == 'install_requires':
+                # Split install_requires into package,env_marker tuples
+                # These will be re-assembled later
+                install_requires = []
+                requirement_pattern = '(?P<package>[^;]*);?(?P<env_marker>.*)$'
+                for requirement in in_cfg_value:
+                    m = re.match(requirement_pattern, requirement)
+                    requirement_package = m.group('package').strip()
+                    env_marker = m.group('env_marker').strip()
+                    install_requires.append((requirement_package,env_marker))
+                all_requirements[''] = install_requires
             elif arg == 'package_dir':
                 in_cfg_value = {'': in_cfg_value}
             elif arg in ('package_data', 'data_files'):
@@ -367,6 +382,50 @@ def setup_cfg_to_setup_kwargs(config):
 
         kwargs[arg] = in_cfg_value
 
+    # Transform requirements with embedded environment markers to
+    # setuptools' supported marker-per-requirement format.
+    #
+    # install_requires are treated as a special case of extras, before
+    # being put back in the expected place
+    #
+    # fred =
+    #     foo:marker
+    #     bar
+    # -> {'fred': ['bar'], 'fred:marker':['foo']}
+
+    if 'extras' in config:
+        requirement_pattern = '(?P<package>[^:]*):?(?P<env_marker>.*)$'
+        extras = config['extras']
+        for extra in extras:
+            extra_requirements = []
+            requirements = split_multiline(extras[extra])
+            for requirement in requirements:
+                m = re.match(requirement_pattern, requirement)
+                extras_value = m.group('package').strip()
+                env_marker = m.group('env_marker')
+                extra_requirements.append((extras_value,env_marker))
+            all_requirements[extra] = extra_requirements
+
+    # Transform the full list of requirements into:
+    # - install_requires, for those that have no extra and no
+    #   env_marker
+    # - named extras, for those with an extra name (which may include
+    #   an env_marker)
+    # - and as a special case, install_requires with an env_marker are
+    #   treated as named extras where the name is the empty string
+
+    extras_require = {}
+    for req_group in all_requirements:
+        for requirement, env_marker in all_requirements[req_group]:
+            if env_marker:
+                extras_key = '%s:%s' % (req_group, env_marker)
+            else:
+                extras_key = req_group
+            extras_require.setdefault(extras_key, []).append(requirement)
+
+    kwargs['install_requires'] = extras_require.pop('', [])
+    kwargs['extras_require'] = extras_require
+
     return kwargs
 
 
diff --git a/test-requirements.txt b/test-requirements.txt
index 5f8cfb8..2b33504 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -5,6 +5,7 @@ hacking>=0.9.2,<0.10
 mock>=1.0
 python-subunit>=0.0.18
 sphinx>=1.1.2,<1.2
+six>=1.9.0
 testrepository>=0.0.18
 testresources>=0.2.4
 testscenarios>=0.4