Add collection config support to ansible-test.

7 files changed, 351 insertions, 20 deletions

diff --git a/test/lib/ansible_test/_internal/compat/__init__.py b/test/lib/ansible_test/_internal/compat/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/test/lib/ansible_test/_internal/compat/__init__.py
diff --git a/test/lib/ansible_test/_internal/compat/packaging.py b/test/lib/ansible_test/_internal/compat/packaging.py
new file mode 100644
index 0000000000..e91d14a583
--- /dev/null
+++ b/test/lib/ansible_test/_internal/compat/packaging.py
@@ -0,0 +1,12 @@
+"""Packaging compatibility."""
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+try:
+    from packaging.specifiers import SpecifierSet
+    from packaging.version import Version
+    PACKAGING_IMPORT_ERROR = None
+except ImportError as ex:
+    SpecifierSet = None
+    Version = None
+    PACKAGING_IMPORT_ERROR = ex
diff --git a/test/lib/ansible_test/_internal/compat/yaml.py b/test/lib/ansible_test/_internal/compat/yaml.py
new file mode 100644
index 0000000000..11740ce06b
--- /dev/null
+++ b/test/lib/ansible_test/_internal/compat/yaml.py
@@ -0,0 +1,21 @@
+"""PyYAML compatibility."""
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+from functools import (
+    partial,
+)
+
+try:
+    import yaml as _yaml
+    YAML_IMPORT_ERROR = None
+except ImportError as ex:
+    yaml_load = None  # pylint: disable=invalid-name
+    YAML_IMPORT_ERROR = ex
+else:
+    try:
+        _SafeLoader = _yaml.CSafeLoader
+    except AttributeError:
+        _SafeLoader = _yaml.SafeLoader
+
+    yaml_load = partial(_yaml.load, Loader=_SafeLoader)
diff --git a/test/lib/ansible_test/_internal/content_config.py b/test/lib/ansible_test/_internal/content_config.py
new file mode 100644
index 0000000000..7802dc355e
--- /dev/null
+++ b/test/lib/ansible_test/_internal/content_config.py
@@ -0,0 +1,155 @@
+"""Content configuration."""
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import os
+
+from . import types as t
+
+from .compat.packaging import (
+    PACKAGING_IMPORT_ERROR,
+    SpecifierSet,
+    Version,
+)
+
+from .compat.yaml import (
+    YAML_IMPORT_ERROR,
+    yaml_load,
+)
+
+from .io import (
+    read_text_file,
+)
+
+from .util import (
+    ApplicationError,
+    CONTROLLER_PYTHON_VERSIONS,
+    SUPPORTED_PYTHON_VERSIONS,
+    display,
+    str_to_version,
+)
+
+from .data import (
+    data_context,
+)
+
+
+MISSING = object()
+
+
+class BaseConfig:
+    """Base class for content configuration."""
+    def __init__(self, data):  # type: (t.Any) -> None
+        if not isinstance(data, dict):
+            raise Exception('config must be type `dict` not `%s`' % type(data))
+
+
+class ModulesConfig(BaseConfig):
+    """Configuration for modules."""
+    def __init__(self, data):  # type: (t.Any) -> None
+        super(ModulesConfig, self).__init__(data)
+
+        python_requires = data.get('python_requires', MISSING)
+
+        if python_requires == MISSING:
+            raise KeyError('python_requires is required')
+
+        self.python_requires = python_requires
+        self.python_versions = parse_python_requires(python_requires)
+        self.controller_only = python_requires == 'controller'
+
+
+class ContentConfig(BaseConfig):
+    """Configuration for all content."""
+    def __init__(self, data):  # type: (t.Any) -> None
+        super(ContentConfig, self).__init__(data)
+
+        # Configuration specific to modules/module_utils.
+        self.modules = ModulesConfig(data.get('modules', {}))
+
+        # Python versions supported by the controller, combined with Python versions supported by modules/module_utils.
+        # Mainly used for display purposes and to limit the Python versions used for sanity tests.
+        self.python_versions = [version for version in SUPPORTED_PYTHON_VERSIONS
+                                if version in CONTROLLER_PYTHON_VERSIONS or version in self.modules.python_versions]
+
+        # True if Python 2.x is supported.
+        self.py2_support = any(version for version in self.python_versions if str_to_version(version)[0] == 2)
+
+
+def load_config(path):  # type: (str) -> t.Optional[ContentConfig]
+    """Load and parse the specified config file and return the result or None if loading/parsing failed."""
+    if YAML_IMPORT_ERROR:
+        raise ApplicationError('The "PyYAML" module is required to parse config: %s' % YAML_IMPORT_ERROR)
+
+    if PACKAGING_IMPORT_ERROR:
+        raise ApplicationError('The "packaging" module is required to parse config: %s' % PACKAGING_IMPORT_ERROR)
+
+    value = read_text_file(path)
+
+    try:
+        yaml_value = yaml_load(value)
+    except Exception as ex:  # pylint: disable=broad-except
+        display.warning('Ignoring config "%s" due to a YAML parsing error: %s' % (path, ex))
+        return None
+
+    try:
+        config = ContentConfig(yaml_value)
+    except Exception as ex:  # pylint: disable=broad-except
+        display.warning('Ignoring config "%s" due a config parsing error: %s' % (path, ex))
+        return None
+
+    display.info('Loaded configuration: %s' % path, verbosity=1)
+
+    return config
+
+
+def get_content_config():  # type: () -> ContentConfig
+    """
+    Parse and return the content configuration (if any) for the current collection.
+    For ansible-core, a default configuration is used.
+    Results are cached.
+    """
+    try:
+        return get_content_config.config
+    except AttributeError:
+        pass
+
+    collection_config_path = 'tests/config.yml'
+
+    config = None
+
+    if data_context().content.collection and os.path.exists(collection_config_path):
+        config = load_config(collection_config_path)
+
+    if not config:
+        config = ContentConfig(dict(
+            modules=dict(
+                python_requires='default',
+            ),
+        ))
+
+    get_content_config.config = config
+
+    if not config.modules.python_versions:
+        raise ApplicationError('This collection does not declare support for modules/module_utils on any known Python version.\n'
+                               'Ansible supports modules/module_utils on Python versions: %s\n'
+                               'This collection provides the Python requirement: %s' % (
+                                   ', '.join(SUPPORTED_PYTHON_VERSIONS), config.modules.python_requires))
+
+    return config
+
+
+def parse_python_requires(value):  # type: (t.Any) -> t.List[str]
+    """Parse the given 'python_requires' version specifier and return the matching Python versions."""
+    if not isinstance(value, str):
+        raise ValueError('python_requires must must be of type `str` not type `%s`' % type(value))
+
+    if value == 'default':
+        versions = list(SUPPORTED_PYTHON_VERSIONS)
+    elif value == 'controller':
+        versions = list(CONTROLLER_PYTHON_VERSIONS)
+    else:
+        specifier_set = SpecifierSet(value)
+        versions = [version for version in SUPPORTED_PYTHON_VERSIONS if specifier_set.contains(Version(version))]
+
+    return versions
diff --git a/test/lib/ansible_test/_internal/sanity/__init__.py b/test/lib/ansible_test/_internal/sanity/__init__.py
index 6613abb99b..30cb6dd47a 100644
--- a/test/lib/ansible_test/_internal/sanity/__init__.py
+++ b/test/lib/ansible_test/_internal/sanity/__init__.py
@@ -33,6 +33,7 @@ from ..util import (
     str_to_version,
     SUPPORTED_PYTHON_VERSIONS,
     CONTROLLER_PYTHON_VERSIONS,
+    REMOTE_ONLY_PYTHON_VERSIONS,
 )
 
 from ..util_common import (
@@ -74,6 +75,10 @@ from ..data import (
     data_context,
 )
 
+from ..content_config import (
+    get_content_config,
+)
+
 COMMAND = 'sanity'
 SANITY_ROOT = os.path.join(ANSIBLE_TEST_DATA_ROOT, 'sanity')
 
@@ -142,18 +147,21 @@ def command_sanity(args):
             options = ''
 
             if test.supported_python_versions and version not in test.supported_python_versions:
-                display.warning("Skipping sanity test '%s' on Python %s. Supported Python versions: %s" % (
-                    test.name, version, ', '.join(test.supported_python_versions)))
-                result = SanitySkipped(test.name, skip_version)
-            elif not args.python and version not in available_versions:
-                display.warning("Skipping sanity test '%s' on Python %s due to missing interpreter." % (test.name, version))
+                # There are two ways this situation can occur:
+                #
+                # - A specific Python version was requested with the `--python` option and that version is not supported by the test.
+                #   This means that the test supports only a subset of the controller supported Python versions, and not the one given by the `--python` option.
+                #   Or that a remote-only Python version was specified for a Python based sanity test that is not multi-version.
+                #
+                # - No specific Python version was requested and no supported version was found on the system.
+                #   This means that the test supports only a subset of the controller supported Python versions, and not the one used to run ansible-test.
+                #   Or that the Python version used to run ansible-test is not supported by the controller, a condition which will soon not be possible.
+                #
+                # Neither of these are affected by the Python versions supported by a collection.
                 result = SanitySkipped(test.name, skip_version)
+                result.reason = "Skipping sanity test '%s' on Python %s. Supported Python versions: %s" % (
+                    test.name, version, ', '.join(test.supported_python_versions))
             else:
-                if test.supported_python_versions:
-                    display.info("Running sanity test '%s' with Python %s" % (test.name, version))
-                else:
-                    display.info("Running sanity test '%s'" % test.name)
-
                 if isinstance(test, SanityCodeSmellTest):
                     settings = test.load_processor(args)
                 elif isinstance(test, SanityMultipleVersion):
@@ -177,11 +185,27 @@ def command_sanity(args):
                 all_targets = SanityTargets.filter_and_inject_targets(test, all_targets)
                 usable_targets = SanityTargets.filter_and_inject_targets(test, usable_targets)
 
-                usable_targets = sorted(test.filter_targets(list(usable_targets)))
+                usable_targets = sorted(test.filter_targets_by_version(list(usable_targets), version))
                 usable_targets = settings.filter_skipped_targets(usable_targets)
                 sanity_targets = SanityTargets(tuple(all_targets), tuple(usable_targets))
 
-                if usable_targets or test.no_targets:
+                test_needed = bool(usable_targets or test.no_targets)
+                result = None
+
+                if test_needed and not args.python and version not in available_versions:
+                    # Deferred checking of Python availability. Done here since it is now known to be required for running the test.
+                    # Earlier checking could cause a spurious warning to be generated for a collection which does not support the Python version.
+                    # If the `--python` option was used, this warning will be skipped and an error will be reported when running the test instead.
+                    result = SanitySkipped(test.name, skip_version)
+                    result.reason = "Skipping sanity test '%s' on Python %s due to missing interpreter." % (test.name, version)
+
+                if not result:
+                    if test.supported_python_versions:
+                        display.info("Running sanity test '%s' with Python %s" % (test.name, version))
+                    else:
+                        display.info("Running sanity test '%s'" % test.name)
+
+                if test_needed and not result:
                     install_command_requirements(args, version, context=test.name, enable_pyyaml_check=True)
 
                     if isinstance(test, SanityCodeSmellTest):
@@ -195,6 +219,8 @@ def command_sanity(args):
                         result = test.test(args, sanity_targets)
                     else:
                         raise Exception('Unsupported test type: %s' % type(test))
+                elif result:
+                    pass
                 else:
                     result = SanitySkipped(test.name, skip_version)
 
@@ -274,13 +300,18 @@ class SanityIgnoreParser:
         for test in sanity_get_tests():
             test_targets = SanityTargets.filter_and_inject_targets(test, targets)
 
-            paths_by_test[test.name] = set(target.path for target in test.filter_targets(test_targets))
-
             if isinstance(test, SanityMultipleVersion):
                 versioned_test_names.add(test.name)
-                tests_by_name.update(dict(('%s-%s' % (test.name, python_version), test) for python_version in test.supported_python_versions))
+
+                for python_version in test.supported_python_versions:
+                    test_name = '%s-%s' % (test.name, python_version)
+
+                    paths_by_test[test_name] = set(target.path for target in test.filter_targets_by_version(test_targets, python_version))
+                    tests_by_name[test_name] = test
             else:
                 unversioned_test_names.update(dict(('%s-%s' % (test.name, python_version), test.name) for python_version in SUPPORTED_PYTHON_VERSIONS))
+
+                paths_by_test[test.name] = set(target.path for target in test.filter_targets_by_version(test_targets, ''))
                 tests_by_name[test.name] = test
 
         for line_no, line in enumerate(lines, start=1):
@@ -347,7 +378,7 @@ class SanityIgnoreParser:
                 self.parse_errors.append((line_no, 1, "Sanity test '%s' does not support directory paths" % test_name))
                 continue
 
-            if path not in paths_by_test[test.name] and not test.no_targets:
+            if path not in paths_by_test[test_name] and not test.no_targets:
                 self.parse_errors.append((line_no, 1, "Sanity test '%s' does not test path '%s'" % (test_name, path)))
                 continue
 
@@ -658,6 +689,11 @@ class SanityTest(ABC):
         return False
 
     @property
+    def py2_compat(self):  # type: () -> bool
+        """True if the test only applies to code that runs on Python 2.x."""
+        return False
+
+    @property
     def supported_python_versions(self):  # type: () -> t.Optional[t.Tuple[str, ...]]
         """A tuple of supported Python versions or None if the test does not depend on specific Python versions."""
         return CONTROLLER_PYTHON_VERSIONS
@@ -669,6 +705,47 @@ class SanityTest(ABC):
 
         raise NotImplementedError('Sanity test "%s" must implement "filter_targets" or set "no_targets" to True.' % self.name)
 
+    def filter_targets_by_version(self, targets, python_version):  # type: (t.List[TestTarget], str) -> t.List[TestTarget]
+        """Return the given list of test targets, filtered to include only those relevant for the test, taking into account the Python version."""
+        del python_version  # python_version is not used here, but derived classes may make use of it
+
+        targets = self.filter_targets(targets)
+
+        if self.py2_compat:
+            # This sanity test is a Python 2.x compatibility test.
+            content_config = get_content_config()
+
+            if content_config.py2_support:
+                # This collection supports Python 2.x.
+                # Filter targets to include only those that require support for remote-only Python versions.
+                targets = self.filter_remote_targets(targets)
+            else:
+                # This collection does not support Python 2.x.
+                # There are no targets to test.
+                targets = []
+
+        return targets
+
+    def filter_remote_targets(self, targets):  # type: (t.List[TestTarget]) -> t.List[TestTarget]
+        """Return a filtered list of the given targets, including only those that require support for remote-only Python versions."""
+        targets = [target for target in targets if (
+            is_subdir(target.path, data_context().content.module_path) or
+            is_subdir(target.path, data_context().content.module_utils_path) or
+            is_subdir(target.path, data_context().content.unit_module_path) or
+            is_subdir(target.path, data_context().content.unit_module_utils_path) or
+            # include modules/module_utils within integration test library directories
+            re.search('^%s/.*/library/' % re.escape(data_context().content.integration_targets_path), target.path) or
+            # special handling for content in ansible-core
+            (data_context().content.is_ansible and (
+                # temporary solution until ansible-test code is reorganized when the split controller/remote implementation is complete
+                is_subdir(target.path, 'test/lib/ansible_test/') or
+                # integration test support modules/module_utils continue to require support for remote-only Python versions
+                re.search('^test/support/integration/.*/(modules|module_utils)/', target.path)
+            ))
+        )]
+
+        return targets
+
 
 class SanityCodeSmellTest(SanityTest):
     """Sanity test script."""
@@ -701,6 +778,7 @@ class SanityCodeSmellTest(SanityTest):
             self.__no_targets = self.config.get('no_targets')  # type: bool
             self.__include_directories = self.config.get('include_directories')  # type: bool
             self.__include_symlinks = self.config.get('include_symlinks')  # type: bool
+            self.__py2_compat = self.config.get('py2_compat', False)  # type: bool
         else:
             self.output = None
             self.extensions = []
@@ -715,6 +793,7 @@ class SanityCodeSmellTest(SanityTest):
             self.__no_targets = True
             self.__include_directories = False
             self.__include_symlinks = False
+            self.__py2_compat = False
 
         if self.no_targets:
             mutually_exclusive = (
@@ -754,6 +833,11 @@ class SanityCodeSmellTest(SanityTest):
         return self.__include_symlinks
 
     @property
+    def py2_compat(self):  # type: () -> bool
+        """True if the test only applies to code that runs on Python 2.x."""
+        return self.__py2_compat
+
+    @property
     def supported_python_versions(self):  # type: () -> t.Optional[t.Tuple[str, ...]]
         """A tuple of supported Python versions or None if the test does not depend on specific Python versions."""
         versions = super(SanityCodeSmellTest, self).supported_python_versions
@@ -937,6 +1021,25 @@ class SanityMultipleVersion(SanityFunc):
         """A tuple of supported Python versions or None if the test does not depend on specific Python versions."""
         return SUPPORTED_PYTHON_VERSIONS
 
+    def filter_targets_by_version(self, targets, python_version):  # type: (t.List[TestTarget], str) -> t.List[TestTarget]
+        """Return the given list of test targets, filtered to include only those relevant for the test, taking into account the Python version."""
+        if not python_version:
+            raise Exception('python_version is required to filter multi-version tests')
+
+        targets = super(SanityMultipleVersion, self).filter_targets_by_version(targets, python_version)
+
+        if python_version in REMOTE_ONLY_PYTHON_VERSIONS:
+            content_config = get_content_config()
+
+            if python_version not in content_config.modules.python_versions:
+                # when a remote-only python version is not supported there are no paths to test
+                return []
+
+            # when a remote-only python version is supported, tests must be applied only to targets that support remote-only Python versions
+            targets = self.filter_remote_targets(targets)
+
+        return targets
+
 
 SANITY_TESTS = (
 )
diff --git a/test/lib/ansible_test/_internal/test.py b/test/lib/ansible_test/_internal/test.py
index f0a0c83c25..952bc28df0 100644
--- a/test/lib/ansible_test/_internal/test.py
+++ b/test/lib/ansible_test/_internal/test.py
@@ -228,16 +228,29 @@ class TestSuccess(TestResult):
 
 class TestSkipped(TestResult):
     """Test skipped."""
+    def __init__(self, command, test, python_version=None):
+        """
+        :type command: str
+        :type test: str
+        :type python_version: str
+        """
+        super(TestSkipped, self).__init__(command, test, python_version)
+
+        self.reason = None  # type: t.Optional[str]
+
     def write_console(self):
         """Write results to console."""
-        display.info('No tests applicable.', verbosity=1)
+        if self.reason:
+            display.warning(self.reason)
+        else:
+            display.info('No tests applicable.', verbosity=1)
 
     def write_junit(self, args):
         """
         :type args: TestConfig
         """
         test_case = self.junit.TestCase(classname=self.command, name=self.name)
-        test_case.add_skipped_info('No tests applicable.')
+        test_case.add_skipped_info(self.reason or 'No tests applicable.')
 
         self.save_junit(args, test_case)
 
diff --git a/test/lib/ansible_test/_internal/units/__init__.py b/test/lib/ansible_test/_internal/units/__init__.py
index d27ed7d53f..a2f49e29bd 100644
--- a/test/lib/ansible_test/_internal/units/__init__.py
+++ b/test/lib/ansible_test/_internal/units/__init__.py
@@ -60,6 +60,10 @@ from ..executor import (
     install_command_requirements,
 )
 
+from ..content_config import (
+    get_content_config,
+)
+
 
 class TestContext:
     """Contexts that unit tests run in based on the type of content."""
@@ -80,8 +84,18 @@ def command_units(args):
 
     paths = [target.path for target in include]
 
-    module_paths = [path for path in paths if is_subdir(path, data_context().content.unit_module_path)]
-    module_utils_paths = [path for path in paths if is_subdir(path, data_context().content.unit_module_utils_path)]
+    content_config = get_content_config()
+    supported_remote_python_versions = content_config.modules.python_versions
+
+    if content_config.modules.controller_only:
+        # controller-only collections run modules/module_utils unit tests as controller-only tests
+        module_paths = []
+        module_utils_paths = []
+    else:
+        # normal collections run modules/module_utils unit tests isolated from controller code due to differences in python version requirements
+        module_paths = [path for path in paths if is_subdir(path, data_context().content.unit_module_path)]
+        module_utils_paths = [path for path in paths if is_subdir(path, data_context().content.unit_module_utils_path)]
+
     controller_paths = sorted(path for path in set(paths) - set(module_paths) - set(module_utils_paths))
 
     remote_paths = module_paths or module_utils_paths
@@ -96,10 +110,20 @@ def command_units(args):
         raise AllTargetsSkipped()
 
     if args.python and args.python in REMOTE_ONLY_PYTHON_VERSIONS:
+        if args.python not in supported_remote_python_versions:
+            display.warning('Python %s is not supported by this collection. Supported Python versions are: %s' % (
+                args.python, ', '.join(content_config.python_versions)))
+            raise AllTargetsSkipped()
+
         if not remote_paths:
             display.warning('Python %s is only supported by module and module_utils unit tests, but none were selected.' % args.python)
             raise AllTargetsSkipped()
 
+    if args.python and args.python not in supported_remote_python_versions and not controller_paths:
+        display.warning('Python %s is not supported by this collection for modules/module_utils. Supported Python versions are: %s' % (
+            args.python, ', '.join(supported_remote_python_versions)))
+        raise AllTargetsSkipped()
+
     if args.delegate:
         raise Delegate(require=changes, exclude=args.exclude)
 
@@ -118,6 +142,9 @@ def command_units(args):
             if test_context == TestContext.controller:
                 if version not in CONTROLLER_PYTHON_VERSIONS:
                     continue
+            else:
+                if version not in supported_remote_python_versions:
+                    continue
 
             if not paths:
                 continue