diff options
76 files changed, 2901 insertions, 1093 deletions
diff --git a/changelogs/fragments/collection_routing.yml b/changelogs/fragments/collection_routing.yml new file mode 100644 index 0000000000..fd1a5579bf --- /dev/null +++ b/changelogs/fragments/collection_routing.yml @@ -0,0 +1,10 @@ +major_changes: + - Plugin routing allows collections to declare deprecation, redirection targets, and removals for all plugin types. + - Routing data built into Ansible 2.10 ensures that 2.9 content should work unmodified on 2.10. Formerly included + modules and plugins that were moved to collections are still accessible by their original unqualified names, so long + as their destination collections are installed. + - Collections may declare a list of supported/tested Ansible versions for the collection. A warning is issued if a + collection does not support the Ansible version that loads it (can also be configured as silent or a fatal error). + Collections that do not declare supported Ansible versions do not issue a warning/error. + - Plugins that import module_utils and other ansible namespaces that have moved to collections should + continue to work unmodified. diff --git a/changelogs/fragments/validate-modules-deprecated-collections.yml b/changelogs/fragments/validate-modules-deprecated-collections.yml index cbd1366824..092eb5930d 100644 --- a/changelogs/fragments/validate-modules-deprecated-collections.yml +++ b/changelogs/fragments/validate-modules-deprecated-collections.yml @@ -1,2 +1,2 @@ minor_changes: - - validate-modules checks for deprecated in collections against meta/routing.yml + - validate-modules checks for deprecated in collections against meta/runtime.yml diff --git a/lib/ansible/cli/__init__.py b/lib/ansible/cli/__init__.py index 3adf5eb5a4..59b19c636f 100644 --- a/lib/ansible/cli/__init__.py +++ b/lib/ansible/cli/__init__.py @@ -26,7 +26,8 @@ from ansible.parsing.dataloader import DataLoader from ansible.parsing.vault import PromptVaultSecret, get_file_vault_secret from ansible.plugins.loader import add_all_plugin_dirs from ansible.release import __version__ -from ansible.utils.collection_loader import AnsibleCollectionLoader, get_collection_name_from_path, set_collection_playbook_paths +from ansible.utils.collection_loader import AnsibleCollectionConfig +from ansible.utils.collection_loader._collection_finder import _get_collection_name_from_path from ansible.utils.display import Display from ansible.utils.path import unfrackpath from ansible.utils.unsafe_proxy import to_unsafe_text @@ -455,11 +456,11 @@ class CLI(with_metaclass(ABCMeta, object)): if basedir: loader.set_basedir(basedir) add_all_plugin_dirs(basedir) - set_collection_playbook_paths(basedir) - default_collection = get_collection_name_from_path(basedir) + AnsibleCollectionConfig.playbook_paths = basedir + default_collection = _get_collection_name_from_path(basedir) if default_collection: display.warning(u'running with default collection {0}'.format(default_collection)) - AnsibleCollectionLoader().set_default_collection(default_collection) + AnsibleCollectionConfig.default_collection = default_collection vault_ids = list(options['vault_ids']) default_vault_ids = C.DEFAULT_VAULT_IDENTITY_LIST diff --git a/lib/ansible/cli/doc.py b/lib/ansible/cli/doc.py index 8e96376ab6..bfb67e66ff 100644 --- a/lib/ansible/cli/doc.py +++ b/lib/ansible/cli/doc.py @@ -28,7 +28,8 @@ from ansible.parsing.metadata import extract_metadata from ansible.parsing.plugin_docs import read_docstub from ansible.parsing.yaml.dumper import AnsibleDumper from ansible.plugins.loader import action_loader, fragment_loader -from ansible.utils.collection_loader import set_collection_playbook_paths, get_collection_name_from_path +from ansible.utils.collection_loader import AnsibleCollectionConfig +from ansible.utils.collection_loader._collection_finder import _get_collection_name_from_path from ansible.utils.display import Display from ansible.utils.plugin_docs import BLACKLIST, get_docstring, get_versioned_doclink @@ -44,11 +45,11 @@ def jdump(text): def add_collection_plugins(plugin_list, plugin_type, coll_filter=None): - # TODO: take into account routing.yml once implemented + # TODO: take into account runtime.yml once implemented b_colldirs = list_collection_dirs(coll_filter=coll_filter) for b_path in b_colldirs: path = to_text(b_path, errors='surrogate_or_strict') - collname = get_collection_name_from_path(b_path) + collname = _get_collection_name_from_path(b_path) ptype = C.COLLECTION_PTYPE_COMPAT.get(plugin_type, plugin_type) plugin_list.update(DocCLI.find_plugins(os.path.join(path, 'plugins', ptype), plugin_type, collection=collname)) @@ -127,7 +128,7 @@ class DocCLI(CLI): # add to plugin paths from command line basedir = context.CLIARGS['basedir'] if basedir: - set_collection_playbook_paths(basedir) + AnsibleCollectionConfig.playbook_paths = basedir loader.add_directory(basedir, with_subdir=True) if context.CLIARGS['module_path']: for path in context.CLIARGS['module_path']: diff --git a/lib/ansible/cli/playbook.py b/lib/ansible/cli/playbook.py index 300d913349..3429a28396 100644 --- a/lib/ansible/cli/playbook.py +++ b/lib/ansible/cli/playbook.py @@ -16,7 +16,8 @@ from ansible.executor.playbook_executor import PlaybookExecutor from ansible.module_utils._text import to_bytes from ansible.playbook.block import Block from ansible.utils.display import Display -from ansible.utils.collection_loader import AnsibleCollectionLoader, get_collection_name_from_path, set_collection_playbook_paths +from ansible.utils.collection_loader import AnsibleCollectionConfig +from ansible.utils.collection_loader._collection_finder import _get_collection_name_from_path from ansible.plugins.loader import add_all_plugin_dirs @@ -90,13 +91,13 @@ class PlaybookCLI(CLI): b_playbook_dirs.append(b_playbook_dir) - set_collection_playbook_paths(b_playbook_dirs) + AnsibleCollectionConfig.playbook_paths = b_playbook_dirs - playbook_collection = get_collection_name_from_path(b_playbook_dirs[0]) + playbook_collection = _get_collection_name_from_path(b_playbook_dirs[0]) if playbook_collection: display.warning("running playbook inside collection {0}".format(playbook_collection)) - AnsibleCollectionLoader().set_default_collection(playbook_collection) + AnsibleCollectionConfig.default_collection = playbook_collection # don't deal with privilege escalation or passwords when we don't need to if not (context.CLIARGS['listhosts'] or context.CLIARGS['listtasks'] or diff --git a/lib/ansible/collections/list.py b/lib/ansible/collections/list.py index cd207dafdf..8e8bc66490 100644 --- a/lib/ansible/collections/list.py +++ b/lib/ansible/collections/list.py @@ -10,7 +10,7 @@ from collections import defaultdict from ansible.collections import is_collection_path from ansible.module_utils._text import to_bytes -from ansible.utils.collection_loader import AnsibleCollectionLoader +from ansible.utils.collection_loader import AnsibleCollectionConfig from ansible.utils.display import Display display = Display() @@ -27,7 +27,7 @@ def list_valid_collection_paths(search_paths=None, warn=False): if search_paths is None: search_paths = [] - search_paths.extend(AnsibleCollectionLoader().n_collection_paths) + search_paths.extend(AnsibleCollectionConfig.collection_paths) for path in search_paths: diff --git a/lib/ansible/config/routing.yml b/lib/ansible/config/ansible_builtin_runtime.yml index cf8f2e49d1..8064babf80 100644 --- a/lib/ansible/config/routing.yml +++ b/lib/ansible/config/ansible_builtin_runtime.yml @@ -1,8 +1,10 @@ # Copyright (c) 2020 Ansible Project # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) -# This file is only for 2.9 backwards compatiblity, expect it to go away in future ansible versions. plugin_routing: connection: + # test entry + redirected_local: + redirect: ansible.builtin.local buildah: redirect: containers.podman.buildah podman: @@ -48,6 +50,12 @@ plugin_routing: persistent: redirect: ansible.netcommon.persistent modules: + # test entry + formerly_core_ping: + redirect: testns.testcoll.ping + # test entry + uses_redirected_action: + redirect: ansible.builtin.ping podman_container_info: redirect: containers.podman.podman_container_info podman_image_info: @@ -4100,8 +4108,8 @@ plugin_routing: redirect: ansible.posix.sysctl async_status: redirect: ansible.windows.async_status - setup: - redirect: ansible.windows.setup + setup.ps1: + redirect: ansible.windows.setup.ps1 slurp: redirect: ansible.windows.slurp win_acl: @@ -7553,6 +7561,10 @@ plugin_routing: cpm_user: redirect: wti.remote.cpm_user module_utils: + formerly_core: + redirect: ansible_collections.testns.testcoll.plugins.module_utils.base + sub1.sub2.formerly_core: + redirect: ansible_collections.testns.testcoll.plugins.module_utils.base common: redirect: f5networks.f5_modules.common frr: @@ -8052,6 +8064,9 @@ plugin_routing: vyos: redirect: vyos.vyos.vyos action: + # test entry, overloaded with module of same name to use a different base action (ie not "normal.py") + uses_redirected_action: + redirect: testns.testcoll.subclassed_norm aireos: redirect: community.general.aireos aruba: @@ -8493,6 +8508,11 @@ plugin_routing: vyos: redirect: vyos.vyos.vyos filter: + # test entries + formerly_core_filter: + redirect: ansible.builtin.bool + formerly_core_masked_filter: + redirect: ansible.builtin.bool gcp_kms_encrypt: redirect: google.cloud.gcp_kms_encrypt gcp_kms_decrypt: @@ -8575,6 +8595,9 @@ plugin_routing: qradar: redirect: ibm.qradar.qradar inventory: + # test entry + formerly_core_inventory: + redirect: testns.content_adj.statichost cloudscale: redirect: community.general.cloudscale docker_machine: @@ -8622,6 +8645,9 @@ plugin_routing: azure_rm: redirect: azure.azcollection.azure_rm lookup: + # test entry + formerly_core_lookup: + redirect: testns.testcoll.mylookup avi: redirect: community.general.avi cartesian: @@ -8714,7 +8740,29 @@ plugin_routing: junos: redirect: junipernetworks.junos.junos shell: + # test entry + formerly_core_powershell: + redirect: ansible.builtin.powershell csh: redirect: ansible.posix.csh fish: redirect: ansible.posix.fish + test: + # test entries + formerly_core_test: + redirect: ansible.builtin.search + formerly_core_masked_test: + redirect: ansible.builtin.search +import_redirection: + # test entry + ansible.module_utils.formerly_core: + redirect: ansible_collections.testns.testcoll.plugins.module_utils.base + ansible.module_utils.known_hosts: + redirect: ansible_collections.community.general.plugins.module_utils.known_hosts + # ansible.builtin synthetic collection redirection hackery + ansible_collections.ansible.builtin.plugins.modules: + redirect: ansible.modules + ansible_collections.ansible.builtin.plugins.module_utils: + redirect: ansible.module_utils + ansible_collections.ansible.builtin.plugins: + redirect: ansible.plugins diff --git a/lib/ansible/config/base.yml b/lib/ansible/config/base.yml index 9c500b7c7d..7029c65189 100644 --- a/lib/ansible/config/base.yml +++ b/lib/ansible/config/base.yml @@ -215,6 +215,14 @@ CACHE_PLUGIN_TIMEOUT: - {key: fact_caching_timeout, section: defaults} type: integer yaml: {key: facts.cache.timeout} +COLLECTIONS_SCAN_SYS_PATH: + name: enable/disable scanning sys.path for installed collections + default: true + type: boolean + env: + - {name: ANSIBLE_COLLECTIONS_SCAN_SYS_PATH} + ini: + - {key: collections_scan_sys_path, section: defaults} COLLECTIONS_PATHS: name: ordered list of root paths for loading installed Ansible collections content description: Colon separated paths in which Ansible will search for collections content. @@ -224,6 +232,16 @@ COLLECTIONS_PATHS: - {name: ANSIBLE_COLLECTIONS_PATHS} ini: - {key: collections_paths, section: defaults} +COLLECTIONS_ON_ANSIBLE_VERSION_MISMATCH: + name: Defines behavior when loading a collection that does not support the current Ansible version + description: + - When a collection is loaded that does not support the running Ansible version (via the collection metadata key + `requires_ansible`), the default behavior is to issue a warning and continue anyway. Setting this value to `ignore` + skips the warning entirely, while setting it to `fatal` will immediately halt Ansible execution. + env: [{name: ANSIBLE_COLLECTIONS_ON_ANSIBLE_VERSION_MISMATCH}] + ini: [{key: collections_on_ansible_version_mismatch, section: defaults}] + choices: [error, warning, ignore] + default: warning COLOR_CHANGED: name: Color for 'changed' task status default: yellow diff --git a/lib/ansible/errors/__init__.py b/lib/ansible/errors/__init__.py index 243ce9955a..93871b9e9a 100644 --- a/lib/ansible/errors/__init__.py +++ b/lib/ansible/errors/__init__.py @@ -276,6 +276,21 @@ class AnsibleFileNotFound(AnsibleRuntimeError): suppress_extended_error=suppress_extended_error, orig_exc=orig_exc) +class AnsiblePluginRemoved(AnsibleRuntimeError): + ''' a requested plugin has been removed ''' + pass + + +class AnsiblePluginCircularRedirect(AnsibleRuntimeError): + '''a cycle was detected in plugin redirection''' + pass + + +class AnsibleCollectionUnsupportedVersionError(AnsibleRuntimeError): + '''a collection is not supported by this version of Ansible''' + pass + + # These Exceptions are temporary, using them as flow control until we can get a better solution. # DO NOT USE as they will probably be removed soon. # We will port the action modules in our tree to use a context manager instead. diff --git a/lib/ansible/executor/module_common.py b/lib/ansible/executor/module_common.py index a29502e533..fea097cf90 100644 --- a/lib/ansible/executor/module_common.py +++ b/lib/ansible/executor/module_common.py @@ -36,9 +36,10 @@ from ansible import constants as C from ansible.errors import AnsibleError from ansible.executor.interpreter_discovery import InterpreterDiscoveryRequiredError from ansible.executor.powershell import module_manifest as ps_manifest -from ansible.module_utils._text import to_bytes, to_text, to_native -from ansible.module_utils.compat.importlib import import_module +from ansible.module_utils.common.text.converters import to_bytes, to_text, to_native from ansible.plugins.loader import module_utils_loader +from ansible.utils.collection_loader._collection_finder import _get_collection_metadata + # Must import strategy and use write_locks from there # If we import write_locks directly then we end up binding a # variable to the object and then it never gets updated. @@ -601,7 +602,8 @@ class ModuleInfo: path = None if imp is None: - self._info = info = importlib.machinery.PathFinder.find_spec(name, paths) + # don't pretend this is a top-level module, prefix the rest of the namespace + self._info = info = importlib.machinery.PathFinder.find_spec('ansible.module_utils.' + name, paths) if info is not None: self.py_src = os.path.splitext(info.origin)[1] in importlib.machinery.SOURCE_SUFFIXES self.pkg_dir = info.origin.endswith('/__init__.py') @@ -632,33 +634,61 @@ class ModuleInfo: class CollectionModuleInfo(ModuleInfo): - def __init__(self, name, paths): + def __init__(self, name, pkg): self._mod_name = name self.py_src = True - # FIXME: Implement pkg_dir so that we can place __init__.py files self.pkg_dir = False - for path in paths: - self._package_name = '.'.join(path.split('/')) - try: - self.get_source() - except FileNotFoundError: - pass - else: - self.path = os.path.join(path, self._mod_name) + '.py' - break - else: - # FIXME (nitz): implement package fallback code + split_name = pkg.split('.') + split_name.append(name) + if len(split_name) < 5 or split_name[0] != 'ansible_collections' or split_name[3] != 'plugins' or split_name[4] != 'module_utils': + raise ValueError('must search for something beneath a collection module_utils, not {0}.{1}'.format(to_native(pkg), to_native(name))) + + # NB: we can't use pkgutil.get_data safely here, since we don't want to import/execute package/module code on + # the controller while analyzing/assembling the module, so we'll have to manually import the collection's + # Python package to locate it (import root collection, reassemble resource path beneath, fetch source) + + # FIXME: handle MU redirection logic here + + collection_pkg_name = '.'.join(split_name[0:3]) + resource_base_path = os.path.join(*split_name[3:]) + # look for package_dir first, then module + + self._src = pkgutil.get_data(collection_pkg_name, to_native(os.path.join(resource_base_path, '__init__.py'))) + + if self._src is not None: # empty string is OK + return + + self._src = pkgutil.get_data(collection_pkg_name, to_native(resource_base_path + '.py')) + + if not self._src: raise ImportError('unable to load collection-hosted module_util' - ' {0}.{1}'.format(to_native(self._package_name), - to_native(name))) + ' {0}.{1}'.format(to_native(pkg), to_native(name))) def get_source(self): - # FIXME (nitz): need this in py2 for some reason TBD, but we shouldn't (get_data delegates - # to wrong loader without it) - pkg = import_module(self._package_name) - data = pkgutil.get_data(to_native(self._package_name), to_native(self._mod_name + '.py')) - return data + return self._src + + +class InternalRedirectModuleInfo(ModuleInfo): + def __init__(self, name, full_name): + self.pkg_dir = None + self._original_name = full_name + self.path = full_name.replace('.', '/') + '.py' + collection_meta = _get_collection_metadata('ansible.builtin') + redirect = collection_meta.get('plugin_routing', {}).get('module_utils', {}).get(name, {}).get('redirect', None) + if not redirect: + raise ImportError('no redirect found for {0}'.format(name)) + self._redirect = redirect + self.py_src = True + self._shim_src = """ +import sys +import {1} as mod + +sys.modules['{0}'] = mod +""".format(self._original_name, self._redirect) + + def get_source(self): + return self._shim_src def recursive_finder(name, module_fqn, data, py_module_names, py_module_cache, zf): @@ -721,8 +751,7 @@ def recursive_finder(name, module_fqn, data, py_module_names, py_module_cache, z break try: # this is a collection-hosted MU; look it up with pkgutil.get_data() - module_info = CollectionModuleInfo(py_module_name[-idx], - [os.path.join(*py_module_name[:-idx])]) + module_info = CollectionModuleInfo(py_module_name[-idx], '.'.join(py_module_name[:-idx])) break except ImportError: continue @@ -740,7 +769,13 @@ def recursive_finder(name, module_fqn, data, py_module_names, py_module_cache, z [os.path.join(p, *relative_module_utils_dir[:-idx]) for p in module_utils_paths]) break except ImportError: - continue + # check metadata for redirect, generate stub if present + try: + module_info = InternalRedirectModuleInfo(py_module_name[-idx], + '.'.join(py_module_name[:(None if idx == 1 else -1)])) + break + except ImportError: + continue else: # If we get here, it's because of a bug in ModuleDepFinder. If we get a reproducer we # should then fix ModuleDepFinder diff --git a/lib/ansible/executor/powershell/module_manifest.py b/lib/ansible/executor/powershell/module_manifest.py index 321ff83455..83a1c3a739 100644 --- a/lib/ansible/executor/powershell/module_manifest.py +++ b/lib/ansible/executor/powershell/module_manifest.py @@ -142,7 +142,7 @@ class PSModuleDepFinder(object): if name in self.exec_scripts.keys(): return - data = pkgutil.get_data("ansible.executor.powershell", name + ".ps1") + data = pkgutil.get_data("ansible.executor.powershell", to_native(name + ".ps1")) if data is None: raise AnsibleError("Could not find executor powershell script " "for '%s'" % name) diff --git a/lib/ansible/executor/task_executor.py b/lib/ansible/executor/task_executor.py index 3a5c98dd58..f4a5fd834f 100644 --- a/lib/ansible/executor/task_executor.py +++ b/lib/ansible/executor/task_executor.py @@ -27,7 +27,7 @@ from ansible.playbook.conditional import Conditional from ansible.playbook.task import Task from ansible.plugins.loader import become_loader, cliconf_loader, connection_loader, httpapi_loader, netconf_loader, terminal_loader from ansible.template import Templar -from ansible.utils.collection_loader import AnsibleCollectionLoader +from ansible.utils.collection_loader import AnsibleCollectionConfig from ansible.utils.listify import listify_lookup_plugin_terms from ansible.utils.unsafe_proxy import to_unsafe_text, wrap_var from ansible.vars.clean import namespace_facts, clean_facts @@ -1101,7 +1101,7 @@ def start_connection(play_context, variables, task_uuid): # can. 'ANSIBLE_BECOME_PLUGINS': become_loader.print_paths(), 'ANSIBLE_CLICONF_PLUGINS': cliconf_loader.print_paths(), - 'ANSIBLE_COLLECTIONS_PATHS': os.pathsep.join(AnsibleCollectionLoader().n_collection_paths), + 'ANSIBLE_COLLECTIONS_PATHS': to_native(os.pathsep.join(AnsibleCollectionConfig.collection_paths)), 'ANSIBLE_CONNECTION_PLUGINS': connection_loader.print_paths(), 'ANSIBLE_HTTPAPI_PLUGINS': httpapi_loader.print_paths(), 'ANSIBLE_NETCONF_PLUGINS': netconf_loader.print_paths(), diff --git a/lib/ansible/playbook/collectionsearch.py b/lib/ansible/playbook/collectionsearch.py index d80b6a1c6a..fb69519b40 100644 --- a/lib/ansible/playbook/collectionsearch.py +++ b/lib/ansible/playbook/collectionsearch.py @@ -6,7 +6,7 @@ __metaclass__ = type from ansible.module_utils.six import string_types from ansible.playbook.attribute import FieldAttribute -from ansible.utils.collection_loader import AnsibleCollectionLoader +from ansible.utils.collection_loader import AnsibleCollectionConfig from ansible.template import is_template, Environment from ansible.utils.display import Display @@ -14,7 +14,7 @@ display = Display() def _ensure_default_collection(collection_list=None): - default_collection = AnsibleCollectionLoader().default_collection + default_collection = AnsibleCollectionConfig.default_collection # Will be None when used as the default if collection_list is None: diff --git a/lib/ansible/playbook/helpers.py b/lib/ansible/playbook/helpers.py index 4eff569943..2a8c5aecbe 100644 --- a/lib/ansible/playbook/helpers.py +++ b/lib/ansible/playbook/helpers.py @@ -25,7 +25,6 @@ from ansible.errors import AnsibleParserError, AnsibleUndefinedVariable, Ansible from ansible.module_utils._text import to_native from ansible.module_utils.six import string_types from ansible.parsing.mod_args import ModuleArgsParser -from ansible.utils.collection_loader import AnsibleCollectionLoader from ansible.utils.display import Display display = Display() diff --git a/lib/ansible/playbook/role/__init__.py b/lib/ansible/playbook/role/__init__.py index 4b43b3eb64..b7456afcf7 100644 --- a/lib/ansible/playbook/role/__init__.py +++ b/lib/ansible/playbook/role/__init__.py @@ -32,7 +32,7 @@ from ansible.playbook.helpers import load_list_of_blocks from ansible.playbook.role.metadata import RoleMetadata from ansible.playbook.taggable import Taggable from ansible.plugins.loader import add_all_plugin_dirs -from ansible.utils.collection_loader import AnsibleCollectionLoader +from ansible.utils.collection_loader import AnsibleCollectionConfig from ansible.utils.vars import combine_vars @@ -234,7 +234,7 @@ class Role(Base, Conditional, Taggable, CollectionSearch): if self._role_collection: # this is a collection-hosted role self.collections.insert(0, self._role_collection) else: # this is a legacy role, but set the default collection if there is one - default_collection = AnsibleCollectionLoader().default_collection + default_collection = AnsibleCollectionConfig.default_collection if default_collection: self.collections.insert(0, default_collection) # legacy role, ensure all plugin dirs under the role are added to plugin search path diff --git a/lib/ansible/playbook/role/definition.py b/lib/ansible/playbook/role/definition.py index b859f63a95..20d69ebd6b 100644 --- a/lib/ansible/playbook/role/definition.py +++ b/lib/ansible/playbook/role/definition.py @@ -31,7 +31,8 @@ from ansible.playbook.collectionsearch import CollectionSearch from ansible.playbook.conditional import Conditional from ansible.playbook.taggable import Taggable from ansible.template import Templar -from ansible.utils.collection_loader import get_collection_role_path, AnsibleCollectionRef +from ansible.utils.collection_loader import AnsibleCollectionRef +from ansible.utils.collection_loader._collection_finder import _get_collection_role_path from ansible.utils.path import unfrackpath from ansible.utils.display import Display @@ -155,7 +156,7 @@ class RoleDefinition(Base, Conditional, Taggable, CollectionSearch): # try to load as a collection-based role first if self._collection_list or AnsibleCollectionRef.is_valid_fqcr(role_name): - role_tuple = get_collection_role_path(role_name, self._collection_list) + role_tuple = _get_collection_role_path(role_name, self._collection_list) if role_tuple: # we found it, stash collection data and return the name/path tuple diff --git a/lib/ansible/playbook/task.py b/lib/ansible/playbook/task.py index 972707f5cf..e9e5d876ef 100644 --- a/lib/ansible/playbook/task.py +++ b/lib/ansible/playbook/task.py @@ -36,7 +36,7 @@ from ansible.playbook.conditional import Conditional from ansible.playbook.loop_control import LoopControl from ansible.playbook.role import Role from ansible.playbook.taggable import Taggable -from ansible.utils.collection_loader import AnsibleCollectionLoader +from ansible.utils.collection_loader import AnsibleCollectionConfig from ansible.utils.display import Display from ansible.utils.sentinel import Sentinel @@ -182,7 +182,7 @@ class Task(Base, Conditional, Taggable, CollectionSearch): new_ds.ansible_pos = ds.ansible_pos # since this affects the task action parsing, we have to resolve in preprocess instead of in typical validator - default_collection = AnsibleCollectionLoader().default_collection + default_collection = AnsibleCollectionConfig.default_collection collections_list = ds.get('collections') if collections_list is None: @@ -293,7 +293,7 @@ class Task(Base, Conditional, Taggable, CollectionSearch): if self._parent: self._parent.post_validate(templar) - if AnsibleCollectionLoader().default_collection: + if AnsibleCollectionConfig.default_collection: pass super(Task, self).post_validate(templar) diff --git a/lib/ansible/plugins/inventory/__init__.py b/lib/ansible/plugins/inventory/__init__.py index fc6046c628..c0ecb4c1cd 100644 --- a/lib/ansible/plugins/inventory/__init__.py +++ b/lib/ansible/plugins/inventory/__init__.py @@ -216,10 +216,13 @@ class BaseInventoryPlugin(AnsiblePlugin): except Exception as e: raise AnsibleParserError(to_native(e)) + # a plugin can be loaded via many different names with redirection- if so, we want to accept any of those names + valid_names = getattr(self, '_redirected_names') or [self.NAME] + if not config: # no data raise AnsibleParserError("%s is empty" % (to_native(path))) - elif config.get('plugin') != self.NAME: + elif config.get('plugin') not in valid_names: # this is not my config file raise AnsibleParserError("Incorrect plugin name in file: %s" % config.get('plugin', 'none found')) elif not isinstance(config, Mapping): diff --git a/lib/ansible/plugins/loader.py b/lib/ansible/plugins/loader.py index fb61fd2084..e52eebc49a 100644 --- a/lib/ansible/plugins/loader.py +++ b/lib/ansible/plugins/loader.py @@ -16,25 +16,45 @@ import warnings from collections import defaultdict from ansible import constants as C -from ansible.errors import AnsibleError +from ansible.errors import AnsibleError, AnsiblePluginCircularRedirect, AnsiblePluginRemoved, AnsibleCollectionUnsupportedVersionError from ansible.module_utils._text import to_bytes, to_text, to_native from ansible.module_utils.compat.importlib import import_module from ansible.module_utils.six import string_types from ansible.parsing.utils.yaml import from_yaml from ansible.parsing.yaml.loader import AnsibleLoader from ansible.plugins import get_plugin_class, MODULE_CACHE, PATH_CACHE, PLUGIN_PATH_CACHE -from ansible.utils.collection_loader import AnsibleCollectionLoader, AnsibleFlatMapLoader, AnsibleCollectionRef +from ansible.utils.collection_loader import AnsibleCollectionConfig, AnsibleCollectionRef +from ansible.utils.collection_loader._collection_finder import _AnsibleCollectionFinder, _get_collection_metadata from ansible.utils.display import Display from ansible.utils.plugin_docs import add_fragments +from ansible import __version__ as ansible_version + +# TODO: take the packaging dep, or vendor SpecifierSet? + +try: + from packaging.specifiers import SpecifierSet + from packaging.version import Version +except ImportError: + SpecifierSet = None + Version = None + +try: + # use C version if possible for speedup + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader try: import importlib.util imp = None except ImportError: import imp + ModuleNotFoundError = None display = Display() +_tombstones = None + def get_all_plugin_loaders(): return [(name, obj) for (name, obj) in globals().items() if isinstance(obj, PluginLoader)] @@ -93,6 +113,61 @@ def add_dirs_to_loader(which_loader, paths): loader.add_directory(path, with_subdir=True) +class PluginLoadContext(object): + def __init__(self): + self.original_name = None + self.redirect_list = [] + self.error_list = [] + self.import_error_list = [] + self.load_attempts = [] + self.pending_redirect = None + self.exit_reason = None + self.plugin_resolved_path = None + self.plugin_resolved_name = None + self.deprecated = False + self.removal_date = None + self.deprecation_warnings = [] + self.resolved = False + + def record_deprecation(self, name, deprecation): + if not deprecation: + return self + + warning_text = deprecation.get('warning_text', None) + removal_date = deprecation.get('removal_date', None) + if not warning_text: + if removal_date: + warning_text = '{0} has been deprecated and will be removed in a release after {1}'.format(name, removal_date) + else: + warning_text = '{0} has been deprecated and will be removed in a future release'.format(name) + + self.deprecated = True + if removal_date: + self.removal_date = removal_date + self.deprecation_warnings.append(warning_text) + return self + + def resolve(self, resolved_name, resolved_path, exit_reason): + self.pending_redirect = None + self.plugin_resolved_name = resolved_name + self.plugin_resolved_path = resolved_path + self.exit_reason = exit_reason + self.resolved = True + return self + + def redirect(self, redirect_name): + self.pending_redirect = redirect_name + self.exit_reason = 'pending redirect resolution from {0} to {1}'.format(self.original_name, redirect_name) + self.resolved = False + return self + + def nope(self, exit_reason): + self.pending_redirect = None + self.exit_reason = exit_reason + self.resolved = False + return self + + class PluginLoader: ''' PluginLoader loads plugins from the configured plugin directories. @@ -136,6 +211,9 @@ class PluginLoader: self._searched_paths = set() + def __repr__(self): + return 'PluginLoader(type={0})'.format(AnsibleCollectionRef.legacy_plugin_dir_to_plugin_type(self.subdir)) + def _clear_caches(self): if C.OLD_PLUGIN_CACHE_CLEARING: @@ -307,15 +385,80 @@ class PluginLoader: self._clear_caches() display.debug('Added %s to loader search path' % (directory)) - def _find_fq_plugin(self, fq_name, extension): + def _query_collection_routing_meta(self, acr, plugin_type, extension=None): + collection_pkg = import_module(acr.n_python_collection_package_name) + if not collection_pkg: + return None + + # FIXME: shouldn't need this... + try: + # force any type-specific metadata postprocessing to occur + import_module(acr.n_python_collection_package_name + '.plugins.{0}'.format(plugin_type)) + except ImportError: + pass + + # this will be created by the collection PEP302 loader + collection_meta = getattr(collection_pkg, '_collection_meta', None) + + if not collection_meta: + return None + + # TODO: add subdirs support + # check for extension-specific entry first (eg 'setup.ps1') + # TODO: str/bytes on extension/name munging + if acr.subdirs: + subdir_qualified_resource = '.'.join([acr.subdirs, acr.resource]) + else: + subdir_qualified_resource = acr.resource + entry = collection_meta.get('plugin_routing', {}).get(plugin_type, {}).get(subdir_qualified_resource + extension, None) + if not entry: + # try for extension-agnostic entry + entry = collection_meta.get('plugin_routing', {}).get(plugin_type, {}).get(subdir_qualified_resource, None) + return entry + + def _find_fq_plugin(self, fq_name, extension, plugin_load_context): """Search builtin paths to find a plugin. No external paths are searched, meaning plugins inside roles inside collections will be ignored. """ + plugin_load_context.resolved = False + plugin_type = AnsibleCollectionRef.legacy_plugin_dir_to_plugin_type(self.subdir) acr = AnsibleCollectionRef.from_fqcr(fq_name, plugin_type) + # check collection metadata to see if any special handling is required for this plugin + routing_metadata = self._query_collection_routing_meta(acr, plugin_type, extension=extension) + + # TODO: factor this into a wrapper method + if routing_metadata: + deprecation = routing_metadata.get('deprecation', None) + + # this will no-op if there's no deprecation metadata for this plugin + plugin_load_context.record_deprecation(fq_name, deprecation) + + tombstone = routing_metadata.get('tombstone', None) + + if tombstone: + redirect = tombstone.get('redirect', None) + removal_date = tombstone.get('removal_date') + if removal_date: + removed_msg = '{0} was removed on {1}'.format(fq_name, removal_date) + else: + removed_msg = '{0} was removed in a previous release'.format(fq_name) + plugin_load_context.removal_date = removal_date + plugin_load_context.resolved = True + plugin_load_context.exit_reason = removed_msg + return plugin_load_context + + redirect = routing_metadata.get('redirect', None) + + if redirect: + # FIXME: remove once this is covered in debug or whatever + display.vv("redirecting (type: {0}) {1} to {2}".format(plugin_type, fq_name, redirect)) + return plugin_load_context.redirect(redirect) + # TODO: non-FQCN case, do we support `.` prefix for current collection, assume it with no dots, require it for subdirs in current, or ? + n_resource = to_native(acr.resource, errors='strict') # we want this before the extension is added full_name = '{0}.{1}'.format(acr.n_python_package_name, n_resource) @@ -326,17 +469,10 @@ class PluginLoader: pkg = sys.modules.get(acr.n_python_package_name) if not pkg: # FIXME: there must be cheaper/safer way to do this - pkg = import_module(acr.n_python_package_name) - - # if the package is one of our flatmaps, we need to consult its loader to find the path, since the file could be - # anywhere in the tree - if hasattr(pkg, '__loader__') and isinstance(pkg.__loader__, AnsibleFlatMapLoader): try: - file_path = pkg.__loader__.find_file(n_resource) - return full_name, to_text(file_path) - except IOError: - # this loader already takes care of extensionless files, so if we didn't find it, just bail - return None, None + pkg = import_module(acr.n_python_package_name) + except (ImportError, ModuleNotFoundError): + return plugin_load_context.nope('Python package {0} not found'.format(acr.n_python_package_name)) pkg_path = os.path.dirname(pkg.__file__) @@ -344,7 +480,11 @@ class PluginLoader: # FIXME: and is file or file link or ... if os.path.exists(n_resource_path): - return full_name, to_text(n_resource_path) + return plugin_load_context.resolve(full_name, to_text(n_resource_path), 'found exact match for {0} in {1}'.format(full_name, acr.collection)) + + if extension: + # the request was extension-specific, don't try for an extensionless match + return plugin_load_context.nope('no match for {0} in {1}'.format(to_text(n_resource), acr.collection)) # look for any matching extension in the package location (sans filter) found_files = [f @@ -352,24 +492,63 @@ class PluginLoader: if os.path.isfile(f) and not f.endswith(C.MODULE_IGNORE_EXTS)] if not found_files: - return None, None + return plugin_load_context.nope('failed fuzzy extension match for {0} in {1}'.format(full_name, acr.collection)) if len(found_files) > 1: # TODO: warn? pass - return full_name, to_text(found_files[0]) + return plugin_load_context.resolve(full_name, to_text(found_files[0]), 'found fuzzy extension match for {0} in {1}'.format(full_name, acr.collection)) def find_plugin(self, name, mod_type='', ignore_deprecated=False, check_aliases=False, collection_list=None): ''' Find a plugin named name ''' - return self.find_plugin_with_name(name, mod_type, ignore_deprecated, check_aliases, collection_list)[1] + result = self.find_plugin_with_context(name, mod_type, ignore_deprecated, check_aliases, collection_list) + if result.resolved and result.plugin_resolved_path: + return result.plugin_resolved_path - def find_plugin_with_name(self, name, mod_type='', ignore_deprecated=False, check_aliases=False, collection_list=None): - ''' Find a plugin named name ''' + return None + + def find_plugin_with_context(self, name, mod_type='', ignore_deprecated=False, check_aliases=False, collection_list=None): + ''' Find a plugin named name, returning contextual info about the load, recursively resolving redirection ''' + plugin_load_context = PluginLoadContext() + plugin_load_context.original_name = name + while True: + result = self._resolve_plugin_step(name, mod_type, ignore_deprecated, check_aliases, collection_list, plugin_load_context=plugin_load_context) + if result.pending_redirect: + if result.pending_redirect in result.redirect_list: + raise AnsiblePluginCircularRedirect('plugin redirect loop resolving {0} (path: {1})'.format(result.original_name, result.redirect_list)) + name = result.pending_redirect + result.pending_redirect = None + plugin_load_context = result + else: + break + + # TODO: smuggle these to the controller when we're in a worker, reduce noise from normal things like missing plugin packages during collection search + if plugin_load_context.error_list: + display.warning("errors were encountered during the plugin load for {0}:\n{1}".format(name, plugin_load_context.error_list)) + + # TODO: display/return import_error_list? Only useful for forensics... + + if plugin_load_context.deprecated and C.config.get_config_value('DEPRECATION_WARNINGS'): + for dw in plugin_load_context.deprecation_warnings: + # TODO: need to smuggle these to the controller if we're in a worker context + display.warning('[DEPRECATION WARNING] ' + dw) + + return plugin_load_context + + # FIXME: name bikeshed + def _resolve_plugin_step(self, name, mod_type='', ignore_deprecated=False, + check_aliases=False, collection_list=None, plugin_load_context=PluginLoadContext()): + if not plugin_load_context: + raise ValueError('A PluginLoadContext is required') + + plugin_load_context.redirect_list.append(name) + plugin_load_context.resolved = False global _PLUGIN_FILTERS if name in _PLUGIN_FILTERS[self.package]: - return None, None + plugin_load_context.exit_reason = '{0} matched a defined plugin filter'.format(name) + return plugin_load_context if mod_type: suffix = mod_type @@ -387,37 +566,46 @@ class PluginLoader: candidates = [name] else: candidates = ['{0}.{1}'.format(c, name) for c in collection_list] - # TODO: keep actual errors, not just assembled messages - errors = [] + for candidate_name in candidates: try: + plugin_load_context.load_attempts.append(candidate_name) # HACK: refactor this properly if candidate_name.startswith('ansible.legacy'): # 'ansible.legacy' refers to the plugin finding behavior used before collections existed. # They need to search 'library' and the various '*_plugins' directories in order to find the file. - full_name = name - p = self._find_plugin_legacy(name.replace('ansible.legacy.', '', 1), ignore_deprecated, check_aliases, suffix) + plugin_load_context = self._find_plugin_legacy(name.replace('ansible.legacy.', '', 1), + plugin_load_context, ignore_deprecated, check_aliases, suffix) else: # 'ansible.builtin' should be handled here. This means only internal, or builtin, paths are searched. - full_name, p = self._find_fq_plugin(candidate_name, suffix) - if p: - return full_name, p + plugin_load_context = self._find_fq_plugin(candidate_name, suffix, plugin_load_context=plugin_load_context) + if plugin_load_context.resolved or plugin_load_context.pending_redirect: # if we got an answer or need to chase down a redirect, return + return plugin_load_context + except (AnsiblePluginRemoved, AnsiblePluginCircularRedirect, AnsibleCollectionUnsupportedVersionError): + # these are generally fatal, let them fly + raise + except ImportError as ie: + plugin_load_context.import_error_list.append(ie) except Exception as ex: - errors.append(to_native(ex)) + # FIXME: keep actual errors, not just assembled messages + plugin_load_context.error_list.append(to_native(ex)) + + if plugin_load_context.error_list: + display.debug(msg='plugin lookup for {0} failed; errors: {1}'.format(name, '; '.join(plugin_load_context.error_list))) - if errors: - display.debug(msg='plugin lookup for {0} failed; errors: {1}'.format(name, '; '.join(errors))) + plugin_load_context.exit_reason = 'no matches found for {0}'.format(name) - return None, None + return plugin_load_context # if we got here, there's no collection list and it's not an FQ name, so do legacy lookup - return name, self._find_plugin_legacy(name, ignore_deprecated, check_aliases, suffix) + return self._find_plugin_legacy(name, plugin_load_context, ignore_deprecated, check_aliases, suffix) - def _find_plugin_legacy(self, name, ignore_deprecated=False, check_aliases=False, suffix=None): + def _find_plugin_legacy(self, name, plugin_load_context, ignore_deprecated=False, check_aliases=False, suffix=None): """Search library and various *_plugins paths in order to find the file. This was behavior prior to the existence of collections. """ + plugin_load_context.resolved = False if check_aliases: name = self.aliases.get(name, name) @@ -426,7 +614,10 @@ class PluginLoader: # requested mod_type pull_cache = self._plugin_path_cache[suffix] try: - return pull_cache[name] + plugin_load_context.plugin_resolved_path = pull_cache[name] + plugin_load_context.plugin_resolved_name = name + plugin_load_context.resolved = True + return plugin_load_context except KeyError: # Cache miss. Now let's find the plugin pass @@ -438,6 +629,7 @@ class PluginLoader: # We can use _get_paths() since add_directory() forces a cache refresh. for path in (p for p in self._get_paths() if p not in self._searched_paths and os.path.isdir(p)): display.debug('trying %s' % path) + plugin_load_context.load_attempts.append(path) try: full_paths = (os.path.join(path, f) for f in os.listdir(path)) except OSError as e: @@ -474,7 +666,10 @@ class PluginLoader: self._searched_paths.add(path) try: - return pull_cache[name] + plugin_load_context.plugin_resolved_path = pull_cache[name] + plugin_load_context.plugin_resolved_name = name + plugin_load_context.resolved = True + return plugin_load_context except KeyError: # Didn't find the plugin in this directory. Load modules from the next one pass @@ -488,9 +683,17 @@ class PluginLoader: # FIXME: this is not always the case, some are just aliases display.deprecated('%s is kept for backwards compatibility but usage is discouraged. ' # pylint: disable=ansible-deprecated-no-version 'The module documentation details page may explain more about this rationale.' % name.lstrip('_')) - return pull_cache[alias_name] + plugin_load_context.plugin_resolved_path = pull_cache[alias_name] + plugin_load_context.plugin_resolved_name = alias_name + plugin_load_context.resolved = True + return plugin_load_context - return None + # last ditch, if it's something that can be redirected, look for a builtin redirect before giving up + candidate_fqcr = 'ansible.builtin.{0}'.format(name) + if '.' not in name and AnsibleCollectionRef.is_valid_fqcr(candidate_fqcr): + return self._find_fq_plugin(fq_name=candidate_fqcr, extension=suffix, plugin_load_context=plugin_load_context) + + return plugin_load_context.nope('{0} is not eligible for last-chance resolution'.format(name)) def has_plugin(self, name, collection_list=None): ''' Checks if a plugin named name exists ''' @@ -530,11 +733,12 @@ class PluginLoader: module = imp.load_source(to_native(full_name), to_native(path), module_file) return module - def _update_object(self, obj, name, path): + def _update_object(self, obj, name, path, redirected_names=None): # set extra info on the module, in case we want it later setattr(obj, '_original_path', path) setattr(obj, '_load_name', name) + setattr(obj, '_redirected_names', redirected_names or []) def get(self, name, *args, **kwargs): ''' instantiates a plugin of the given name using arguments ''' @@ -544,10 +748,15 @@ class PluginLoader: collection_list = kwargs.pop('collection_list', None) if name in self.aliases: name = self.aliases[name] - name, path = self.find_plugin_with_name(name, collection_list=collection_list) - if path is None: + plugin_load_context = self.find_plugin_with_context(name, collection_list=collection_list) + if not plugin_load_context.resolved or not plugin_load_context.plugin_resolved_path: + # FIXME: this is probably an error (eg removed plugin) return None + name = plugin_load_context.plugin_resolved_name + path = plugin_load_context.plugin_resolved_path + redirected_names = plugin_load_context.redirect_list or [] + if path not in self._module_cache: self._module_cache[path] = self._load_module_source(name, path) self._load_config_defs(name, self._module_cache[path], path) @@ -566,6 +775,7 @@ class PluginLoader: if not issubclass(obj, plugin_class): return None + # FIXME: update this to use the load context self._display_plugin_load(self.class_name, name, self._searched_paths, path, found_in_cache=found_in_cache, class_only=class_only) if not class_only: @@ -573,7 +783,7 @@ class PluginLoader: # A plugin may need to use its _load_name in __init__ (for example, to set # or get options from config), so update the object before using the constructor instance = object.__new__(obj) - self._update_object(instance, name, path) + self._update_object(instance, name, path, redirected_names) obj.__init__(instance, *args, **kwargs) obj = instance except TypeError as e: @@ -583,7 +793,7 @@ class PluginLoader: return None raise - self._update_object(obj, name, path) + self._update_object(obj, name, path, redirected_names) return obj def _display_plugin_load(self, class_name, name, searched_paths, path, found_in_cache=None, class_only=None): @@ -818,9 +1028,52 @@ def _load_plugin_filter(): return filters +# since we don't want the actual collection loader understanding metadata, we'll do it in an event handler +def _on_collection_load_handler(collection_name, collection_path): + display.vvvv(to_text('Loading collection {0} from {1}'.format(collection_name, collection_path))) + + collection_meta = _get_collection_metadata(collection_name) + + try: + if not _does_collection_support_ansible_version(collection_meta.get('requires_ansible', ''), ansible_version): + mismatch_behavior = C.config.get_config_value('COLLECTIONS_ON_ANSIBLE_VERSION_MISMATCH') + message = 'Collection {0} does not support Ansible version {1}'.format(collection_name, ansible_version) + if mismatch_behavior == 'warning': + display.warning(message) + elif mismatch_behavior == 'error': + raise AnsibleCollectionUnsupportedVersionError(message) + except AnsibleError: + raise + except Exception as ex: + display.warning('Error parsing collection metadata requires_ansible value from collection {0}: {1}'.format(collection_name, ex)) + + +def _does_collection_support_ansible_version(requirement_string, ansible_version): + if not requirement_string: + return True + + if not SpecifierSet: + display.warning('packaging Python module unavailable; unable to validate collection Ansible version requirements') + return True + + ss = SpecifierSet(requirement_string) + + # ignore prerelease/postrelease/beta/dev flags for simplicity + base_ansible_version = Version(ansible_version).base_version + + return ss.contains(base_ansible_version) + + def _configure_collection_loader(): - if not any((isinstance(l, AnsibleCollectionLoader) for l in sys.meta_path)): - sys.meta_path.insert(0, AnsibleCollectionLoader(C.config)) + if AnsibleCollectionConfig.collection_finder: + display.warning('AnsibleCollectionFinder has already been configured') + return + + finder = _AnsibleCollectionFinder(C.config.get_config_value('COLLECTIONS_PATHS'), C.config.get_config_value('COLLECTIONS_SCAN_SYS_PATH')) + finder._install() + + # this should succeed now + AnsibleCollectionConfig.on_collection_load += _on_collection_load_handler # TODO: All of the following is initialization code It should be moved inside of an initialization diff --git a/lib/ansible/plugins/strategy/linear.py b/lib/ansible/plugins/strategy/linear.py index 6883c9c1cb..5a7b979714 100644 --- a/lib/ansible/plugins/strategy/linear.py +++ b/lib/ansible/plugins/strategy/linear.py @@ -240,10 +240,28 @@ class StrategyModule(StrategyBase): run_once = False work_to_do = True + # check to see if this task should be skipped, due to it being a member of a + # role which has already run (and whether that role allows duplicate execution) + if task._role and task._role.has_run(host): + # If there is no metadata, the default behavior is to not allow duplicates, + # if there is metadata, check to see if the allow_duplicates flag was set to true + if task._role._metadata is None or task._role._metadata and not task._role._metadata.allow_duplicates: + display.debug("'%s' skipped because role has already run" % task) + continue + + display.debug("getting variables") + task_vars = self._variable_manager.get_vars(play=iterator._play, host=host, task=task, + _hosts=self._hosts_cache, _hosts_all=self._hosts_cache_all) + self.add_tqm_variables(task_vars, play=iterator._play) + templar = Templar(loader=self._loader, variables=task_vars) + display.debug("done getting variables") + # test to see if the task across all hosts points to an action plugin which # sets BYPASS_HOST_LOOP to true, or if it has run_once enabled. If so, we # will only send this task to the first host in the list. + task.action = templar.template(task.action) + try: action = action_loader.get(task.action, class_only=True) except KeyError: @@ -251,15 +269,6 @@ class StrategyModule(StrategyBase): # corresponding action plugin action = None - # check to see if this task should be skipped, due to it being a member of a - # role which has already run (and whether that role allows duplicate execution) - if task._role and task._role.has_run(host): - # If there is no metadata, the default behavior is to not allow duplicates, - # if there is metadata, check to see if the allow_duplicates flag was set to true - if task._role._metadata is None or task._role._metadata and not task._role._metadata.allow_duplicates: - display.debug("'%s' skipped because role has already run" % task) - continue - if task.action == 'meta': # for the linear strategy, we run meta tasks just once and for # all hosts currently being iterated over rather than one host @@ -277,13 +286,6 @@ class StrategyModule(StrategyBase): skip_rest = True break - display.debug("getting variables") - task_vars = self._variable_manager.get_vars(play=iterator._play, host=host, task=task, - _hosts=self._hosts_cache, _hosts_all=self._hosts_cache_all) - self.add_tqm_variables(task_vars, play=iterator._play) - templar = Templar(loader=self._loader, variables=task_vars) - display.debug("done getting variables") - run_once = templar.template(task.run_once) or action and getattr(action, 'BYPASS_HOST_LOOP', False) if (task.any_errors_fatal or run_once) and not task.ignore_errors: diff --git a/lib/ansible/template/__init__.py b/lib/ansible/template/__init__.py index 10fa6e128b..aa779e82f3 100644 --- a/lib/ansible/template/__init__.py +++ b/lib/ansible/template/__init__.py @@ -30,6 +30,7 @@ import time from contextlib import contextmanager from distutils.version import LooseVersion from numbers import Number +from traceback import format_exc try: from hashlib import sha1 @@ -53,6 +54,7 @@ from ansible.template.template import AnsibleJ2Template from ansible.template.vars import AnsibleJ2Vars from ansible.utils.collection_loader import AnsibleCollectionRef from ansible.utils.display import Display +from ansible.utils.collection_loader._collection_finder import _get_collection_metadata from ansible.utils.unsafe_proxy import wrap_var display = Display() @@ -350,52 +352,75 @@ class JinjaPluginIntercept(MutableMapping): # FUTURE: we can cache FQ filter/test calls for the entire duration of a run, since a given collection's impl's # aren't supposed to change during a run def __getitem__(self, key): - if not isinstance(key, string_types): - raise ValueError('key must be a string') - - key = to_native(key) + try: + if not isinstance(key, string_types): + raise ValueError('key must be a string') - if '.' not in key: # might be a built-in value, delegate to base dict - return self._delegatee.__getitem__(key) + key = to_native(key) - func = self._collection_jinja_func_cache.get(key) + if '.' not in key: # might be a built-in or legacy, check the delegatee dict first, then try for a last-chance base redirect + func = self._delegatee.get(key) - if func: - return func + if func: + return func - acr = AnsibleCollectionRef.try_parse_fqcr(key, self._dirname) + ts = _get_collection_metadata('ansible.builtin') - if not acr: - raise KeyError('invalid plugin name: {0}'.format(key)) + # TODO: implement support for collection-backed redirect (currently only builtin) + # TODO: implement cycle detection (unified across collection redir as well) + redirect_fqcr = ts.get('plugin_routing', {}).get(self._dirname, {}).get(key, {}).get('redirect', None) + if redirect_fqcr: + acr = AnsibleCollectionRef.from_fqcr(ref=redirect_fqcr, ref_type=self._dirname) + display.vvv('redirecting {0} {1} to {2}.{3}'.format(self._dirname, key, acr.collection, acr.resource)) + key = redirect_fqcr + # TODO: handle recursive forwarding (not necessary for builtin, but definitely for further collection redirs) - try: - pkg = import_module(acr.n_python_package_name) - except ImportError: - raise KeyError() + func = self._collection_jinja_func_cache.get(key) - parent_prefix = acr.collection + if func: + return func - if acr.subdirs: - parent_prefix = '{0}.{1}'.format(parent_prefix, acr.subdirs) + acr = AnsibleCollectionRef.try_parse_fqcr(key, self._dirname) - for dummy, module_name, ispkg in pkgutil.iter_modules(pkg.__path__, prefix=parent_prefix + '.'): - if ispkg: - continue + if not acr: + raise KeyError('invalid plugin name: {0}'.format(key)) try: - plugin_impl = self._pluginloader.get(module_name) - except Exception as e: - raise TemplateSyntaxError(to_native(e), 0) + pkg = import_module(acr.n_python_package_name) + except ImportError: + raise KeyError() - method_map = getattr(plugin_impl, self._method_map_name) + parent_prefix = acr.collection - for f in iteritems(method_map()): - fq_name = '.'.join((parent_prefix, f[0])) - # FIXME: detect/warn on intra-collection function name collisions - self._collection_jinja_func_cache[fq_name] = f[1] + if acr.subdirs: + parent_prefix = '{0}.{1}'.format(parent_prefix, acr.subdirs) + + # TODO: implement collection-level redirect + + for dummy, module_name, ispkg in pkgutil.iter_modules(pkg.__path__, prefix=parent_prefix + '.'): + if ispkg: + continue - function_impl = self._collection_jinja_func_cache[key] - return function_impl + try: + plugin_impl = self._pluginloader.get(module_name) + except Exception as e: + raise TemplateSyntaxError(to_native(e), 0) + + method_map = getattr(plugin_impl, self._method_map_name) + + for f in iteritems(method_map()): + fq_name = '.'.join((parent_prefix, f[0])) + # FIXME: detect/warn on intra-collection function name collisions + self._collection_jinja_func_cache[fq_name] = f[1] + + function_impl = self._collection_jinja_func_cache[key] + return function_impl + except KeyError: + raise + except Exception as ex: + display.warning('an unexpected error occurred during Jinja2 environment setup: {0}'.format(to_native(ex))) + display.vvv('exception during Jinja2 environment setup: {0}'.format(format_exc())) + raise def __setitem__(self, key, value): return self._delegatee.__setitem__(key, value) diff --git a/lib/ansible/utils/collection_loader.py b/lib/ansible/utils/collection_loader.py deleted file mode 100644 index e38d4137f0..0000000000 --- a/lib/ansible/utils/collection_loader.py +++ /dev/null @@ -1,603 +0,0 @@ -# (c) 2019 Ansible Project -# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) - -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type - -import os -import os.path -import re -import sys - -from types import ModuleType - -from ansible.module_utils._text import to_bytes, to_native, to_text -from ansible.module_utils.compat.importlib import import_module -from ansible.module_utils.six import iteritems, string_types, with_metaclass -from ansible.utils.singleton import Singleton - -_SYNTHETIC_PACKAGES = { - # these provide fallback package definitions when there are no on-disk paths - 'ansible_collections': dict(type='pkg_only', allow_external_subpackages=True), - 'ansible_collections.ansible': dict(type='pkg_only', allow_external_subpackages=True), - # these implement the ansible.builtin synthetic collection mapped to the packages inside the ansible distribution - 'ansible_collections.ansible.builtin': dict(type='pkg_only'), - 'ansible_collections.ansible.builtin.plugins': dict(type='map', map='ansible.plugins'), - 'ansible_collections.ansible.builtin.plugins.module_utils': dict(type='map', map='ansible.module_utils', graft=True), - 'ansible_collections.ansible.builtin.plugins.modules': dict(type='flatmap', flatmap='ansible.modules', graft=True), -} - - -# FIXME: exception handling/error logging -class AnsibleCollectionLoader(with_metaclass(Singleton, object)): - def __init__(self, config=None): - if config: - paths = config.get_config_value('COLLECTIONS_PATHS') - else: - paths = os.environ.get('ANSIBLE_COLLECTIONS_PATHS', '').split(os.pathsep) - - if isinstance(paths, string_types): - paths = [paths] - elif paths is None: - paths = [] - - # expand any placeholders in configured paths - paths = [ - to_native(os.path.expanduser(p), errors='surrogate_or_strict') - for p in paths - ] - - # Append all ``ansible_collections`` dirs from sys.path to the end - for path in sys.path: - if ( - path not in paths and - os.path.isdir(to_bytes( - os.path.join(path, 'ansible_collections'), - errors='surrogate_or_strict', - )) - ): - paths.append(path) - - self._n_configured_paths = paths - - self._n_playbook_paths = [] - self._default_collection = None - # pre-inject grafted package maps so we can force them to use the right loader instead of potentially delegating to a "normal" loader - for syn_pkg_def in (p for p in iteritems(_SYNTHETIC_PACKAGES) if p[1].get('graft')): - pkg_name = syn_pkg_def[0] - pkg_def = syn_pkg_def[1] - - newmod = ModuleType(pkg_name) - newmod.__package__ = pkg_name - newmod.__file__ = '<ansible_synthetic_collection_package>' - pkg_type = pkg_def.get('type') - - # TODO: need to rethink map style so we can just delegate all the loading - - if pkg_type == 'flatmap': - newmod.__loader__ = AnsibleFlatMapLoader(import_module(pkg_def['flatmap'])) - newmod.__path__ = [] - - sys.modules[pkg_name] = newmod - - @property - def n_collection_paths(self): - return self._n_playbook_paths + self._n_configured_paths - - def get_collection_path(self, collection_name): - if not AnsibleCollectionRef.is_valid_collection_name(collection_name): - raise ValueError('{0} is not a valid collection name'.format(to_native(collection_name))) - - m = import_module('ansible_collections.{0}'.format(collection_name)) - - return m.__file__ - - def set_playbook_paths(self, b_playbook_paths): - if isinstance(b_playbook_paths, string_types): - b_playbook_paths = [b_playbook_paths] - - # track visited paths; we have to preserve the dir order as-passed in case there are duplicate collections (first one wins) - added_paths = set() - - # de-dupe and ensure the paths are native strings (Python seems to do this for package paths etc, so assume it's safe) - self._n_playbook_paths = [os.path.join(to_native(p), 'collections') for p in b_playbook_paths if not (p in added_paths or added_paths.add(p))] - # FIXME: only allow setting this once, or handle any necessary cache/package path invalidations internally? - - # FIXME: is there a better place to store this? - # FIXME: only allow setting this once - def set_default_collection(self, collection_name): - self._default_collection = collection_name - - @property - def default_collection(self): - return self._default_collection - - def find_module(self, fullname, path=None): - if self._find_module(fullname, path, load=False)[0]: - return self - - return None - - def load_module(self, fullname): - mod = self._find_module(fullname, None, load=True)[1] - - if not mod: - raise ImportError('module {0} not found'.format(fullname)) - - return mod - - def _find_module(self, fullname, path, load): - # this loader is only concerned with items under the Ansible Collections namespace hierarchy, ignore others - if not fullname.startswith('ansible_collections.') and fullname != 'ansible_collections': - return False, None - - if sys.modules.get(fullname): - if not load: - return True, None - - return True, sys.modules[fullname] - - newmod = None - - # this loader implements key functionality for Ansible collections - # * implicit distributed namespace packages for the root Ansible namespace (no pkgutil.extend_path hackery reqd) - # * implicit package support for Python 2.7 (no need for __init__.py in collections, except to use standard Py2.7 tooling) - # * preventing controller-side code injection during collection loading - # * (default loader would execute arbitrary package code from all __init__.py's) - - parent_pkg_name = '.'.join(fullname.split('.')[:-1]) - - parent_pkg = sys.modules.get(parent_pkg_name) - - if parent_pkg_name and not parent_pkg: - raise ImportError('parent package {0} not found'.format(parent_pkg_name)) - - # are we at or below the collection level? eg a.mynamespace.mycollection.something.else - # if so, we don't want distributed namespace behavior; first mynamespace.mycollection on the path is where - # we'll load everything from (ie, don't fall back to another mynamespace.mycollection lower on the path) - sub_collection = fullname.count('.') > 1 - - synpkg_def = _SYNTHETIC_PACKAGES.get(fullname) - synpkg_remainder = '' - - if not synpkg_def: - # if the parent is a grafted package, we have some special work to do, otherwise just look for stuff on disk - parent_synpkg_def = _SYNTHETIC_PACKAGES.get(parent_pkg_name) - if parent_synpkg_def and parent_synpkg_def.get('graft'): - synpkg_def = parent_synpkg_def - synpkg_remainder = '.' + fullname.rpartition('.')[2] - - # FUTURE: collapse as much of this back to on-demand as possible (maybe stub packages that get replaced when actually loaded?) - if synpkg_def: - pkg_type = synpkg_def.get('type') - if not pkg_type: - raise KeyError('invalid synthetic package type (no package "type" specified)') - if pkg_type == 'map': - map_package = synpkg_def.get('map') - - if not map_package: - raise KeyError('invalid synthetic map package definition (no target "map" defined)') - - if not load: - return True, None - - mod = import_module(map_package + synpkg_remainder) - - sys.modules[fullname] = mod - - return True, mod - elif pkg_type == 'flatmap': - raise NotImplementedError() - elif pkg_type == 'pkg_only': - if not load: - return True, None - - newmod = ModuleType(fullname) - newmod.__package__ = fullname - newmod.__file__ = '<ansible_synthetic_collection_package>' - newmod.__loader__ = self - newmod.__path__ = [] - - if not synpkg_def.get('allow_external_subpackages'): - # if external subpackages are NOT allowed, we're done - sys.modules[fullname] = newmod - return True, newmod - - # if external subpackages ARE allowed, check for on-disk implementations and return a normal - # package if we find one, otherwise return the one we created here - - if not parent_pkg: # top-level package, look for NS subpackages on all collection paths - package_paths = [self._extend_path_with_ns(p, fullname) for p in self.n_collection_paths] - else: # subpackage; search in all subpaths (we'll limit later inside a collection) - package_paths = [self._extend_path_with_ns(p, fullname) for p in parent_pkg.__path__] - - for candidate_child_path in package_paths: - code_object = None - is_package = True - location = None - # check for implicit sub-package first - if os.path.isdir(to_bytes(candidate_child_path)): - # Py3.x implicit namespace packages don't have a file location, so they don't support get_data - # (which assumes the parent dir or that the loader has an internal mapping); so we have to provide - # a bogus leaf file on the __file__ attribute for pkgutil.get_data to strip off - location = os.path.join(candidate_child_path, '__synthetic__') - else: - for source_path in [os.path.join(candidate_child_path, '__init__.py'), - candidate_child_path + '.py']: - if not os.path.isfile(to_bytes(source_path)): - continue - - if not load: - return True, None - - with open(to_bytes(source_path), 'rb') as fd: - source = fd.read() - - code_object = compile(source=source, filename=source_path, mode='exec', flags=0, dont_inherit=True) - location = source_path - is_package = source_path.endswith('__init__.py') - break - - if not location: - continue - - newmod = ModuleType(fullname) - newmod.__file__ = location - newmod.__loader__ = self - - if is_package: - if sub_collection: # we never want to search multiple instances of the same collection; use first found - newmod.__path__ = [candidate_child_path] - else: - newmod.__path__ = package_paths - - newmod.__package__ = fullname - else: - newmod.__package__ = parent_pkg_name - - sys.modules[fullname] = newmod - - if code_object: - # FIXME: decide cases where we don't actually want to exec the code? - exec(code_object, newmod.__dict__) - - return True, newmod - - # even if we didn't find one on disk, fall back to a synthetic package if we have one... - if newmod: - sys.modules[fullname] = newmod - return True, newmod - - # FIXME: need to handle the "no dirs present" case for at least the root and synthetic internal collections like ansible.builtin - - return False, None - - @staticmethod - def _extend_path_with_ns(path, ns): - ns_path_add = ns.rsplit('.', 1)[-1] - - return os.path.join(path, ns_path_add) - - def get_data(self, filename): - with open(filename, 'rb') as fd: - return fd.read() - - -class AnsibleFlatMapLoader(object): - _extension_blacklist = ['.pyc', '.pyo'] - - def __init__(self, root_package): - self._root_package = root_package - self._dirtree = None - - def _init_dirtree(self): - # FIXME: thread safety - root_path = os.path.dirname(self._root_package.__file__) - flat_files = [] - # FIXME: make this a dict of filename->dir for faster direct lookup? - # FIXME: deal with _ prefixed deprecated files (or require another method for collections?) - # FIXME: fix overloaded filenames (eg, rename Windows setup to win_setup) - for root, dirs, files in os.walk(root_path): - # add all files in this dir that don't have a blacklisted extension - flat_files.extend(((root, f) for f in files if not any((f.endswith(ext) for ext in self._extension_blacklist)))) - - # HACK: Put Windows modules at the end of the list. This makes collection_loader behave - # the same way as plugin loader, preventing '.ps1' from modules being selected before '.py' - # modules simply because '.ps1' files may be above '.py' files in the flat_files list. - # - # The expected sort order is paths in the order they were in 'flat_files' - # with paths ending in '/windows' at the end, also in the original order they were - # in 'flat_files'. The .sort() method is guaranteed to be stable, so original order is preserved. - flat_files.sort(key=lambda p: p[0].endswith('/windows')) - self._dirtree = flat_files - - def find_file(self, filename): - # FIXME: thread safety - if not self._dirtree: - self._init_dirtree() - - if '.' not in filename: # no extension specified, use extension regex to filter - extensionless_re = re.compile(r'^{0}(\..+)?$'.format(re.escape(filename))) - # why doesn't Python have first()? - try: - # FIXME: store extensionless in a separate direct lookup? - filepath = next(os.path.join(r, f) for r, f in self._dirtree if extensionless_re.match(f)) - except StopIteration: - raise IOError("couldn't find {0}".format(filename)) - else: # actual filename, just look it up - # FIXME: this case sucks; make it a lookup - try: - filepath = next(os.path.join(r, f) for r, f in self._dirtree if f == filename) - except StopIteration: - raise IOError("couldn't find {0}".format(filename)) - - return filepath - - def get_data(self, filename): - found_file = self.find_file(filename) - - with open(found_file, 'rb') as fd: - return fd.read() - - -# TODO: implement these for easier inline debugging? -# def get_source(self, fullname): -# def get_code(self, fullname): -# def is_package(self, fullname): - - -class AnsibleCollectionRef: - # FUTURE: introspect plugin loaders to get these dynamically? - VALID_REF_TYPES = frozenset(to_text(r) for r in ['action', 'become', 'cache', 'callback', 'cliconf', 'connection', - 'doc_fragments', 'filter', 'httpapi', 'inventory', 'lookup', - 'module_utils', 'modules', 'netconf', 'role', 'shell', 'strategy', - 'terminal', 'test', 'vars']) - - # FIXME: tighten this up to match Python identifier reqs, etc - VALID_COLLECTION_NAME_RE = re.compile(to_text(r'^(\w+)\.(\w+)$')) - VALID_SUBDIRS_RE = re.compile(to_text(r'^\w+(\.\w+)*$')) - VALID_FQCR_RE = re.compile(to_text(r'^\w+\.\w+\.\w+(\.\w+)*$')) # can have 0-N included subdirs as well - - def __init__(self, collection_name, subdirs, resource, ref_type): - """ - Create an AnsibleCollectionRef from components - :param collection_name: a collection name of the form 'namespace.collectionname' - :param subdirs: optional subdir segments to be appended below the plugin type (eg, 'subdir1.subdir2') - :param resource: the name of the resource being references (eg, 'mymodule', 'someaction', 'a_role') - :param ref_type: the type of the reference, eg 'module', 'role', 'doc_fragment' - """ - collection_name = to_text(collection_name, errors='strict') - if subdirs is not None: - subdirs = to_text(subdirs, errors='strict') - resource = to_text(resource, errors='strict') - ref_type = to_text(ref_type, errors='strict') - - if not self.is_valid_collection_name(collection_name): - raise ValueError('invalid collection name (must be of the form namespace.collection): {0}'.format(to_native(collection_name))) - - if ref_type not in self.VALID_REF_TYPES: - raise ValueError('invalid collection ref_type: {0}'.format(ref_type)) - - self.collection = collection_name - if subdirs: - if not re.match(self.VALID_SUBDIRS_RE, subdirs): - raise ValueError('invalid subdirs entry: {0} (must be empty/None or of the form subdir1.subdir2)'.format(to_native(subdirs))) - self.subdirs = subdirs - else: - self.subdirs = u'' - - self.resource = resource - self.ref_type = ref_type - - package_components = [u'ansible_collections', self.collection] - - if self.ref_type == u'role': - package_components.append(u'roles') - else: - # we assume it's a plugin - package_components += [u'plugins', self.ref_type] - - if self.subdirs: - package_components.append(self.subdirs) - - if self.ref_type == u'role': - # roles are their own resource - package_components.append(self.resource) - - self.n_python_package_name = to_native('.'.join(package_components)) - - @staticmethod - def from_fqcr(ref, ref_type): - """ - Parse a string as a fully-qualified collection reference, raises ValueError if invalid - :param ref: collection reference to parse (a valid ref is of the form 'ns.coll.resource' or 'ns.coll.subdir1.subdir2.resource') - :param ref_type: the type of the reference, eg 'module', 'role', 'doc_fragment' - :return: a populated AnsibleCollectionRef object - """ - # assuming the fq_name is of the form (ns).(coll).(optional_subdir_N).(resource_name), - # we split the resource name off the right, split ns and coll off the left, and we're left with any optional - # subdirs that need to be added back below the plugin-specific subdir we'll add. So: - # ns.coll.resource -> ansible_collections.ns.coll.plugins.(plugintype).resource - # ns.coll.subdir1.resource -> ansible_collections.ns.coll.plugins.subdir1.(plugintype).resource - # ns.coll.rolename -> ansible_collections.ns.coll.roles.rolename - if not AnsibleCollectionRef.is_valid_fqcr(ref): - raise ValueError('{0} is not a valid collection reference'.format(to_native(ref))) - - ref = to_text(ref, errors='strict') - ref_type = to_text(ref_type, errors='strict') - - resource_splitname = ref.rsplit(u'.', 1) - package_remnant = resource_splitname[0] - resource = resource_splitname[1] - - # split the left two components of the collection package name off, anything remaining is plugin-type - # specific subdirs to be added back on below the plugin type - package_splitname = package_remnant.split(u'.', 2) - if len(package_splitname) == 3: - subdirs = package_splitname[2] - else: - subdirs = u'' - - collection_name = u'.'.join(package_splitname[0:2]) - - return AnsibleCollectionRef(collection_name, subdirs, resource, ref_type) - - @staticmethod - def try_parse_fqcr(ref, ref_type): - """ - Attempt to parse a string as a fully-qualified collection reference, returning None on failure (instead of raising an error) - :param ref: collection reference to parse (a valid ref is of the form 'ns.coll.resource' or 'ns.coll.subdir1.subdir2.resource') - :param ref_type: the type of the reference, eg 'module', 'role', 'doc_fragment' - :return: a populated AnsibleCollectionRef object on successful parsing, else None - """ - try: - return AnsibleCollectionRef.from_fqcr(ref, ref_type) - except ValueError: - pass - - @staticmethod - def legacy_plugin_dir_to_plugin_type(legacy_plugin_dir_name): - """ - Utility method to convert from a PluginLoader dir name to a plugin ref_type - :param legacy_plugin_dir_name: PluginLoader dir name (eg, 'action_plugins', 'library') - :return: the corresponding plugin ref_type (eg, 'action', 'role') - """ - legacy_plugin_dir_name = to_text(legacy_plugin_dir_name) - - plugin_type = legacy_plugin_dir_name.replace(u'_plugins', u'') - - if plugin_type == u'library': - plugin_type = u'modules' - - if plugin_type not in AnsibleCollectionRef.VALID_REF_TYPES: - raise ValueError('{0} cannot be mapped to a valid collection ref type'.format(to_native(legacy_plugin_dir_name))) - - return plugin_type - - @staticmethod - def is_valid_fqcr(ref, ref_type=None): - """ - Validates if is string is a well-formed fully-qualified collection reference (does not look up the collection itself) - :param ref: candidate collection reference to validate (a valid ref is of the form 'ns.coll.resource' or 'ns.coll.subdir1.subdir2.resource') - :param ref_type: optional reference type to enable deeper validation, eg 'module', 'role', 'doc_fragment' - :return: True if the collection ref passed is well-formed, False otherwise - """ - - ref = to_text(ref) - - if not ref_type: - return bool(re.match(AnsibleCollectionRef.VALID_FQCR_RE, ref)) - - return bool(AnsibleCollectionRef.try_parse_fqcr(ref, ref_type)) - - @staticmethod - def is_valid_collection_name(collection_name): - """ - Validates if the given string is a well-formed collection name (does not look up the collection itself) - :param collection_name: candidate collection name to validate (a valid name is of the form 'ns.collname') - :return: True if the collection name passed is well-formed, False otherwise - """ - - collection_name = to_text(collection_name) - - return bool(re.match(AnsibleCollectionRef.VALID_COLLECTION_NAME_RE, collection_name)) - - -def get_collection_role_path(role_name, collection_list=None): - acr = AnsibleCollectionRef.try_parse_fqcr(role_name, 'role') - - if acr: - # looks like a valid qualified collection ref; skip the collection_list - collection_list = [acr.collection] - subdirs = acr.subdirs - resource = acr.resource - elif not collection_list: - return None # not a FQ role and no collection search list spec'd, nothing to do - else: - resource = role_name # treat as unqualified, loop through the collection search list to try and resolve - subdirs = '' - - for collection_name in collection_list: - try: - acr = AnsibleCollectionRef(collection_name=collection_name, subdirs=subdirs, resource=resource, ref_type='role') - # FIXME: error handling/logging; need to catch any import failures and move along - - # FIXME: this line shouldn't be necessary, but py2 pkgutil.get_data is delegating back to built-in loader when it shouldn't - pkg = import_module(acr.n_python_package_name) - - if pkg is not None: - # the package is now loaded, get the collection's package and ask where it lives - path = os.path.dirname(to_bytes(sys.modules[acr.n_python_package_name].__file__, errors='surrogate_or_strict')) - return resource, to_text(path, errors='surrogate_or_strict'), collection_name - - except IOError: - continue - except Exception as ex: - # FIXME: pick out typical import errors first, then error logging - continue - - return None - - -_N_COLLECTION_PATH_RE = re.compile(r'/ansible_collections/([^/]+)/([^/]+)') - - -def get_collection_name_from_path(path): - """ - Return the containing collection name for a given path, or None if the path is not below a configured collection, or - the collection cannot be loaded (eg, the collection is masked by another of the same name higher in the configured - collection roots). - :param n_path: native-string path to evaluate for collection containment - :return: collection name or None - """ - n_collection_paths = [to_native(os.path.abspath(to_bytes(p))) for p in AnsibleCollectionLoader().n_collection_paths] - - b_path = os.path.abspath(to_bytes(path)) - n_path = to_native(b_path) - - for coll_path in n_collection_paths: - common_prefix = to_native(os.path.commonprefix([b_path, to_bytes(coll_path)])) - if common_prefix == coll_path: - # strip off the common prefix (handle weird testing cases of nested collection roots, eg) - collection_remnant = n_path[len(coll_path):] - # commonprefix may include the trailing /, prepend to the remnant if necessary (eg trailing / on root) - if collection_remnant and collection_remnant[0] != '/': - collection_remnant = '/' + collection_remnant - # the path lives under this collection root, see if it maps to a collection - found_collection = _N_COLLECTION_PATH_RE.search(collection_remnant) - if not found_collection: - continue - n_collection_name = '{0}.{1}'.format(*found_collection.groups()) - - loaded_collection_path = AnsibleCollectionLoader().get_collection_path(n_collection_name) - - if not loaded_collection_path: - return None - - # ensure we're using the canonical real path, with the bogus __synthetic__ stripped off - b_loaded_collection_path = os.path.dirname(os.path.abspath(to_bytes(loaded_collection_path))) - - # if the collection path prefix matches the path prefix we were passed, it's the same collection that's loaded - if os.path.commonprefix([b_path, b_loaded_collection_path]) == b_loaded_collection_path: - return n_collection_name - - return None # if not, it's a collection, but not the same collection the loader sees, so ignore it - - -def set_collection_playbook_paths(b_playbook_paths): - AnsibleCollectionLoader().set_playbook_paths(b_playbook_paths) - - -def resource_from_fqcr(ref): - """ - Return resource from a fully-qualified collection reference, - or from a simple resource name. - - For fully-qualified collection references, this is equivalent to - ``AnsibleCollectionRef.from_fqcr(ref).resource``. - - :param ref: collection reference to parse - :return: the resource as a unicode string - """ - ref = to_text(ref, errors='strict') - return ref.split(u'.')[-1] diff --git a/lib/ansible/utils/collection_loader/__init__.py b/lib/ansible/utils/collection_loader/__init__.py new file mode 100644 index 0000000000..a81f503966 --- /dev/null +++ b/lib/ansible/utils/collection_loader/__init__.py @@ -0,0 +1,23 @@ +# (c) 2019 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +# FIXME: decide what of this we want to actually be public/toplevel, put other stuff on a utility class? +from ._collection_config import AnsibleCollectionConfig +from ._collection_finder import AnsibleCollectionRef +from ansible.module_utils.common.text.converters import to_text + + +def resource_from_fqcr(ref): + """ + Return resource from a fully-qualified collection reference, + or from a simple resource name. + For fully-qualified collection references, this is equivalent to + ``AnsibleCollectionRef.from_fqcr(ref).resource``. + :param ref: collection reference to parse + :return: the resource as a unicode string + """ + ref = to_text(ref, errors='strict') + return ref.split(u'.')[-1] diff --git a/lib/ansible/utils/collection_loader/_collection_config.py b/lib/ansible/utils/collection_loader/_collection_config.py new file mode 100644 index 0000000000..e717cde957 --- /dev/null +++ b/lib/ansible/utils/collection_loader/_collection_config.py @@ -0,0 +1,101 @@ +# (c) 2019 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible.module_utils.common.text.converters import to_text +from ansible.module_utils.six import with_metaclass + + +class _EventSource: + def __init__(self): + self._handlers = set() + + def __iadd__(self, handler): + if not callable(handler): + raise ValueError('handler must be callable') + self._handlers.add(handler) + return self + + def __isub__(self, handler): + try: + self._handlers.remove(handler) + except KeyError: + pass + + return self + + def _on_exception(self, handler, exc, *args, **kwargs): + # if we return True, we want the caller to re-raise + return True + + def fire(self, *args, **kwargs): + for h in self._handlers: + try: + h(*args, **kwargs) + except Exception as ex: + if self._on_exception(h, ex, *args, **kwargs): + raise + + +class _AnsibleCollectionConfig(type): + def __init__(cls, meta, name, bases): + cls._collection_finder = None + cls._default_collection = None + cls._on_collection_load = _EventSource() + + @property + def collection_finder(cls): + return cls._collection_finder + + @collection_finder.setter + def collection_finder(cls, value): + if cls._collection_finder: + raise ValueError('an AnsibleCollectionFinder has already been configured') + + cls._collection_finder = value + + @property + def collection_paths(cls): + cls._require_finder() + return [to_text(p) for p in cls._collection_finder._n_collection_paths] + + @property + def default_collection(cls): + return cls._default_collection + + @default_collection.setter + def default_collection(cls, value): + if cls._default_collection: + raise ValueError('default collection {0} has already been configured'.format(value)) + + cls._default_collection = value + + @property + def on_collection_load(cls): + return cls._on_collection_load + + @on_collection_load.setter + def on_collection_load(cls, value): + if value is not cls._on_collection_load: + raise ValueError('on_collection_load is not directly settable (use +=)') + + @property + def playbook_paths(cls): + cls._require_finder() + return [to_text(p) for p in cls._collection_finder._n_playbook_paths] + + @playbook_paths.setter + def playbook_paths(cls, value): + cls._require_finder() + cls._collection_finder.set_playbook_paths(value) + + def _require_finder(cls): + if not cls._collection_finder: + raise NotImplementedError('an AnsibleCollectionFinder has not been installed in this process') + + +# concrete class of our metaclass type that defines the class properties we want +class AnsibleCollectionConfig(with_metaclass(_AnsibleCollectionConfig)): + pass diff --git a/lib/ansible/utils/collection_loader/_collection_finder.py b/lib/ansible/utils/collection_loader/_collection_finder.py new file mode 100644 index 0000000000..c2366680ac --- /dev/null +++ b/lib/ansible/utils/collection_loader/_collection_finder.py @@ -0,0 +1,953 @@ +# (c) 2019 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import os +import os.path +import pkgutil +import re +import sys + + +# DO NOT add new non-stdlib import deps here, this loader is used by external tools (eg ansible-test import sanity) +# that only allow stdlib and module_utils +from ansible.module_utils.common.text.converters import to_native, to_text, to_bytes +from ansible.module_utils.six import string_types, PY3 +from ._collection_config import AnsibleCollectionConfig + +from contextlib import contextmanager +from types import ModuleType + +try: + from importlib import import_module +except ImportError: + def import_module(name): + __import__(name) + return sys.modules[name] + +try: + from importlib import reload as reload_module +except ImportError: + # 2.7 has a global reload function instead... + reload_module = reload # pylint:disable=undefined-variable + +# NB: this supports import sanity test providing a different impl +try: + from ._collection_meta import _meta_yml_to_dict +except ImportError: + _meta_yml_to_dict = None + + +class _AnsibleCollectionFinder: + def __init__(self, paths=None, scan_sys_paths=True): + # TODO: accept metadata loader override + self._ansible_pkg_path = to_native(os.path.dirname(to_bytes(sys.modules['ansible'].__file__))) + + if isinstance(paths, string_types): + paths = [paths] + elif paths is None: + paths = [] + + # expand any placeholders in configured paths + paths = [os.path.expanduser(to_native(p, errors='surrogate_or_strict')) for p in paths] + + if scan_sys_paths: + # append all sys.path entries with an ansible_collections package + for path in sys.path: + if ( + path not in paths and + os.path.isdir(to_bytes( + os.path.join(path, 'ansible_collections'), + errors='surrogate_or_strict', + )) + ): + paths.append(path) + + self._n_configured_paths = paths + self._n_cached_collection_paths = None + self._n_cached_collection_qualified_paths = None + + self._n_playbook_paths = [] + + @classmethod + def _remove(cls): + for mps in sys.meta_path: + if isinstance(mps, _AnsibleCollectionFinder): + sys.meta_path.remove(mps) + + # remove any path hooks that look like ours + for ph in sys.path_hooks: + if hasattr(ph, '__self__') and isinstance(ph.__self__, _AnsibleCollectionFinder): + sys.path_hooks.remove(ph) + + # zap any cached path importer cache entries that might refer to us + sys.path_importer_cache.clear() + + AnsibleCollectionConfig._collection_finder = None + + # validate via the public property that we really killed it + if AnsibleCollectionConfig.collection_finder is not None: + raise AssertionError('_AnsibleCollectionFinder remove did not reset AnsibleCollectionConfig.collection_finder') + + def _install(self): + self._remove() + sys.meta_path.insert(0, self) + + sys.path_hooks.insert(0, self._ansible_collection_path_hook) + + AnsibleCollectionConfig.collection_finder = self + + def _ansible_collection_path_hook(self, path): + path = to_native(path) + interesting_paths = self._n_cached_collection_qualified_paths + if not interesting_paths: + interesting_paths = [os.path.join(p, 'ansible_collections') for p in + self._n_collection_paths] + interesting_paths.insert(0, self._ansible_pkg_path) + self._n_cached_collection_qualified_paths = interesting_paths + + if any(path.startswith(p) for p in interesting_paths): + return _AnsiblePathHookFinder(self, path) + + raise ImportError('not interested') + + @property + def _n_collection_paths(self): + paths = self._n_cached_collection_paths + if not paths: + self._n_cached_collection_paths = paths = self._n_playbook_paths + self._n_configured_paths + return paths + + def set_playbook_paths(self, playbook_paths): + if isinstance(playbook_paths, string_types): + playbook_paths = [playbook_paths] + + # track visited paths; we have to preserve the dir order as-passed in case there are duplicate collections (first one wins) + added_paths = set() + + # de-dupe + self._n_playbook_paths = [os.path.join(to_native(p), 'collections') for p in playbook_paths if not (p in added_paths or added_paths.add(p))] + self._n_cached_collection_paths = None + # HACK: playbook CLI sets this relatively late, so we've already loaded some packages whose paths might depend on this. Fix those up. + # NB: this should NOT be used for late additions; ideally we'd fix the playbook dir setup earlier in Ansible init + # to prevent this from occurring + for pkg in ['ansible_collections', 'ansible_collections.ansible']: + self._reload_hack(pkg) + + def _reload_hack(self, fullname): + m = sys.modules.get(fullname) + if not m: + return + reload_module(m) + + def find_module(self, fullname, path=None): + # Figure out what's being asked for, and delegate to a special-purpose loader + + split_name = fullname.split('.') + toplevel_pkg = split_name[0] + module_to_find = split_name[-1] + part_count = len(split_name) + + if toplevel_pkg not in ['ansible', 'ansible_collections']: + # not interested in anything other than ansible_collections (and limited cases under ansible) + return None + + # sanity check what we're getting from import, canonicalize path values + if part_count == 1: + if path: + raise ValueError('path should not be specified for top-level packages (trying to find {0})'.format(fullname)) + else: + # seed the path to the configured collection roots + path = self._n_collection_paths + + if part_count > 1 and path is None: + raise ValueError('path must be specified for subpackages (trying to find {0})'.format(fullname)) + + # NB: actual "find"ing is delegated to the constructors on the various loaders; they'll ImportError if not found + try: + if toplevel_pkg == 'ansible': + # something under the ansible package, delegate to our internal loader in case of redirections + return _AnsibleInternalRedirectLoader(fullname=fullname, path_list=path) + if part_count == 1: + return _AnsibleCollectionRootPkgLoader(fullname=fullname, path_list=path) + if part_count == 2: # ns pkg eg, ansible_collections, ansible_collections.somens + return _AnsibleCollectionNSPkgLoader(fullname=fullname, path_list=path) + elif part_count == 3: # collection pkg eg, ansible_collections.somens.somecoll + return _AnsibleCollectionPkgLoader(fullname=fullname, path_list=path) + # anything below the collection + return _AnsibleCollectionLoader(fullname=fullname, path_list=path) + except ImportError: + # TODO: log attempt to load context + return None + + +# Implements a path_hook finder for iter_modules (since it's only path based). This finder does not need to actually +# function as a finder in most cases, since our meta_path finder is consulted first for *almost* everything, except +# pkgutil.iter_modules, and under py2, pkgutil.get_data if the parent package passed has not been loaded yet. +class _AnsiblePathHookFinder: + def __init__(self, collection_finder, pathctx): + # when called from a path_hook, find_module doesn't usually get the path arg, so this provides our context + self._pathctx = to_native(pathctx) + self._collection_finder = collection_finder + if PY3: + # cache the native FileFinder (take advantage of its filesystem cache for future find/load requests) + self._file_finder = None + + # class init is fun- this method has a self arg that won't get used + def _get_filefinder_path_hook(self=None): + _file_finder_hook = None + if PY3: + # try to find the FileFinder hook to call for fallback path-based imports in Py3 + _file_finder_hook = [ph for ph in sys.path_hooks if 'FileFinder' in repr(ph)] + if len(_file_finder_hook) != 1: + raise Exception('need exactly one FileFinder import hook (found {0})'.format(len(_file_finder_hook))) + _file_finder_hook = _file_finder_hook[0] + + return _file_finder_hook + + _filefinder_path_hook = _get_filefinder_path_hook() + + def find_module(self, fullname, path=None): + # we ignore the passed in path here- use what we got from the path hook init + split_name = fullname.split('.') + toplevel_pkg = split_name[0] + + if toplevel_pkg == 'ansible_collections': + # collections content? delegate to the collection finder + return self._collection_finder.find_module(fullname, path=[self._pathctx]) + else: + # Something else; we'd normally restrict this to `ansible` descendent modules so that any weird loader + # behavior that arbitrary Python modules have can be serviced by those loaders. In some dev/test + # scenarios (eg a venv under a collection) our path_hook signs us up to load non-Ansible things, and + # it's too late by the time we've reached this point, but also too expensive for the path_hook to figure + # out what we *shouldn't* be loading with the limited info it has. So we'll just delegate to the + # normal path-based loader as best we can to service it. This also allows us to take advantage of Python's + # built-in FS caching and byte-compilation for most things. + if PY3: + # create or consult our cached file finder for this path + if not self._file_finder: + self._file_finder = _AnsiblePathHookFinder._filefinder_path_hook(self._pathctx) + + spec = self._file_finder.find_spec(fullname) + if not spec: + return None + return spec.loader + else: + # call py2's internal loader + return pkgutil.ImpImporter(self._pathctx).find_module(fullname) + + def iter_modules(self, prefix): + # NB: this currently represents only what's on disk, and does not handle package redirection + return _iter_modules_impl([self._pathctx], prefix) + + def __repr__(self): + return "{0}(path='{1}')".format(self.__class__.__name__, self._pathctx) + + +class _AnsibleCollectionPkgLoaderBase: + _allows_package_code = False + + def __init__(self, fullname, path_list=None): + self._fullname = fullname + self._redirect_module = None + self._split_name = fullname.split('.') + self._rpart_name = fullname.rpartition('.') + self._parent_package_name = self._rpart_name[0] # eg ansible_collections for ansible_collections.somens, '' for toplevel + self._package_to_load = self._rpart_name[2] # eg somens for ansible_collections.somens + + self._source_code_path = None + self._decoded_source = None + self._compiled_code = None + + self._validate_args() + + self._candidate_paths = self._get_candidate_paths([to_native(p) for p in path_list]) + self._subpackage_search_paths = self._get_subpackage_search_paths(self._candidate_paths) + + self._validate_final() + + # allow subclasses to validate args and sniff split values before we start digging around + def _validate_args(self): + if self._split_name[0] != 'ansible_collections': + raise ImportError('this loader can only load packages from the ansible_collections package, not {0}'.format(self._fullname)) + + # allow subclasses to customize candidate path filtering + def _get_candidate_paths(self, path_list): + return [os.path.join(p, self._package_to_load) for p in path_list] + + # allow subclasses to customize finding paths + def _get_subpackage_search_paths(self, candidate_paths): + # filter candidate paths for existence (NB: silently ignoring package init code and same-named modules) + return [p for p in candidate_paths if os.path.isdir(to_bytes(p))] + + # allow subclasses to customize state validation/manipulation before we return the loader instance + def _validate_final(self): + return + + @staticmethod + @contextmanager + def _new_or_existing_module(name, **kwargs): + # handle all-or-nothing sys.modules creation/use-existing/delete-on-exception-if-created behavior + created_module = False + module = sys.modules.get(name) + try: + if not module: + module = ModuleType(name) + created_module = True + sys.modules[name] = module + # always override the values passed, except name (allow reference aliasing) + for attr, value in kwargs.items(): + setattr(module, attr, value) + yield module + except Exception: + if created_module: + if sys.modules.get(name): + sys.modules.pop(name) + raise + + # basic module/package location support + # NB: this does not support distributed packages! + @staticmethod + def _module_file_from_path(leaf_name, path): + has_code = True + package_path = os.path.join(to_native(path), to_native(leaf_name)) + module_path = None + + # if the submodule is a package, assemble valid submodule paths, but stop looking for a module + if os.path.isdir(to_bytes(package_path)): + # is there a package init? + module_path = os.path.join(package_path, '__init__.py') + if not os.path.isfile(to_bytes(module_path)): + module_path = os.path.join(package_path, '__synthetic__') + has_code = False + else: + module_path = package_path + '.py' + package_path = None + if not os.path.isfile(to_bytes(module_path)): + raise ImportError('{0} not found at {1}'.format(leaf_name, path)) + + return module_path, has_code, package_path + + def load_module(self, fullname): + # short-circuit redirect; we've already imported the redirected module, so just alias it and return it + if self._redirect_module: + sys.modules[self._fullname] = self._redirect_module + return self._redirect_module + + # we're actually loading a module/package + module_attrs = dict( + __loader__=self, + __file__=self.get_filename(fullname), + __package__=self._parent_package_name # sane default for non-packages + ) + + # eg, I am a package + if self._subpackage_search_paths is not None: # empty is legal + module_attrs['__path__'] = self._subpackage_search_paths + module_attrs['__package__'] = fullname # per PEP366 + + with self._new_or_existing_module(fullname, **module_attrs) as module: + # execute the module's code in its namespace + exec(self.get_code(fullname), module.__dict__) + + return module + + def is_package(self, fullname): + if fullname != self._fullname: + raise ValueError('this loader cannot answer is_package for {0}, only {1}'.format(fullname, self._fullname)) + return self._subpackage_search_paths is not None + + def get_source(self, fullname): + if self._decoded_source: + return self._decoded_source + if fullname != self._fullname: + raise ValueError('this loader cannot load source for {0}, only {1}'.format(fullname, self._fullname)) + if not self._source_code_path: + return None + # FIXME: what do we want encoding/newline requirements to be? + self._decoded_source = self.get_data(self._source_code_path) + return self._decoded_source + + def get_data(self, path): + if not path: + raise ValueError('a path must be specified') + + # TODO: ensure we're being asked for a path below something we own + # TODO: try to handle redirects internally? + + if not path[0] == '/': + # relative to current package, search package paths if possible (this may not be necessary) + # candidate_paths = [os.path.join(ssp, path) for ssp in self._subpackage_search_paths] + raise ValueError('relative resource paths not supported') + else: + candidate_paths = [path] + + for p in candidate_paths: + b_path = to_bytes(p) + if os.path.isfile(b_path): + with open(b_path, 'rb') as fd: + return fd.read() + + return None + + def _synthetic_filename(self, fullname): + return '<ansible_synthetic_collection_package>' + + def get_filename(self, fullname): + if fullname != self._fullname: + raise ValueError('this loader cannot find files for {0}, only {1}'.format(fullname, self._fullname)) + + filename = self._source_code_path + + if not filename and self.is_package(fullname): + if len(self._subpackage_search_paths) == 1: + filename = os.path.join(self._subpackage_search_paths[0], '__synthetic__') + else: + filename = self._synthetic_filename(fullname) + + return filename + + def get_code(self, fullname): + if self._compiled_code: + return self._compiled_code + + # this may or may not be an actual filename, but it's the value we'll use for __file__ + filename = self.get_filename(fullname) + if not filename: + filename = '<string>' + + source_code = self.get_source(fullname) + if not source_code: + source_code = '' + + self._compiled_code = compile(source=source_code, filename=filename, mode='exec', flags=0, dont_inherit=True) + + return self._compiled_code + + def iter_modules(self, prefix): + return _iter_modules_impl(self._subpackage_search_paths, prefix) + + def __repr__(self): + return '{0}(path={1})'.format(self.__class__.__name__, self._subpackage_search_paths or self._source_code_path) + + +class _AnsibleCollectionRootPkgLoader(_AnsibleCollectionPkgLoaderBase): + def _validate_args(self): + super(_AnsibleCollectionRootPkgLoader, self)._validate_args() + if len(self._split_name) != 1: + raise ImportError('this loader can only load the ansible_collections toplevel package, not {0}'.format(self._fullname)) + + +# Implements Ansible's custom namespace package support. +# The ansible_collections package and one level down (collections namespaces) are Python namespace packages +# that search across all configured collection roots. The collection package (two levels down) is the first one found +# on the configured collection root path, and Python namespace package aggregation is not allowed at or below +# the collection. Implements implicit package (package dir) support for both Py2/3. Package init code is ignored +# by this loader. +class _AnsibleCollectionNSPkgLoader(_AnsibleCollectionPkgLoaderBase): + def _validate_args(self): + super(_AnsibleCollectionNSPkgLoader, self)._validate_args() + if len(self._split_name) != 2: + raise ImportError('this loader can only load collections namespace packages, not {0}'.format(self._fullname)) + + def _validate_final(self): + # special-case the `ansible` namespace, since `ansible.builtin` is magical + if not self._subpackage_search_paths and self._package_to_load != 'ansible': + raise ImportError('no {0} found in {1}'.format(self._package_to_load, self._candidate_paths)) + + +# handles locating the actual collection package and associated metadata +class _AnsibleCollectionPkgLoader(_AnsibleCollectionPkgLoaderBase): + def _validate_args(self): + super(_AnsibleCollectionPkgLoader, self)._validate_args() + if len(self._split_name) != 3: + raise ImportError('this loader can only load collection packages, not {0}'.format(self._fullname)) + + def _validate_final(self): + if self._split_name[1:3] == ['ansible', 'builtin']: + # we don't want to allow this one to have on-disk search capability + self._subpackage_search_paths = [] + elif not self._subpackage_search_paths: + raise ImportError('no {0} found in {1}'.format(self._package_to_load, self._candidate_paths)) + else: + # only search within the first collection we found + self._subpackage_search_paths = [self._subpackage_search_paths[0]] + + def load_module(self, fullname): + if not _meta_yml_to_dict: + raise ValueError('ansible.utils.collection_loader._meta_yml_to_dict is not set') + + module = super(_AnsibleCollectionPkgLoader, self).load_module(fullname) + + module._collection_meta = {} + # TODO: load collection metadata, cache in __loader__ state + + collection_name = '.'.join(self._split_name[1:3]) + + if collection_name == 'ansible.builtin': + # ansible.builtin is a synthetic collection, get its routing config from the Ansible distro + raw_routing = pkgutil.get_data('ansible.config', 'ansible_builtin_runtime.yml') + else: + b_routing_meta_path = to_bytes(os.path.join(module.__path__[0], 'meta/runtime.yml')) + if os.path.isfile(b_routing_meta_path): + with open(b_routing_meta_path, 'rb') as fd: + raw_routing = fd.read() + else: + raw_routing = '' + try: + if raw_routing: + routing_dict = _meta_yml_to_dict(raw_routing, (collection_name, 'runtime.yml')) + module._collection_meta = self._canonicalize_meta(routing_dict) + except Exception as ex: + raise ValueError('error parsing collection metadata: {0}'.format(to_native(ex))) + + AnsibleCollectionConfig.on_collection_load.fire(collection_name=collection_name, collection_path=os.path.dirname(module.__file__)) + + return module + + def _canonicalize_meta(self, meta_dict): + # TODO: rewrite import keys and all redirect targets that start with .. (current namespace) and . (current collection) + # OR we could do it all on the fly? + # if not meta_dict: + # return {} + # + # ns_name = '.'.join(self._split_name[0:2]) + # collection_name = '.'.join(self._split_name[0:3]) + # + # # + # for routing_type, routing_type_dict in iteritems(meta_dict.get('plugin_routing', {})): + # for plugin_key, plugin_dict in iteritems(routing_type_dict): + # redirect = plugin_dict.get('redirect', '') + # if redirect.startswith('..'): + # redirect = redirect[2:] + + return meta_dict + + +# loads everything under a collection, including handling redirections defined by the collection +class _AnsibleCollectionLoader(_AnsibleCollectionPkgLoaderBase): + # HACK: stash this in a better place + _redirected_package_map = {} + _allows_package_code = True + + def _validate_args(self): + super(_AnsibleCollectionLoader, self)._validate_args() + if len(self._split_name) < 4: + raise ValueError('this loader is only for sub-collection modules/packages, not {0}'.format(self._fullname)) + + def _get_candidate_paths(self, path_list): + if len(path_list) != 1 and self._split_name[1:3] != ['ansible', 'builtin']: + raise ValueError('this loader requires exactly one path to search') + + return path_list + + def _get_subpackage_search_paths(self, candidate_paths): + collection_meta = _get_collection_metadata('.'.join(self._split_name[1:3])) + + # check for explicit redirection, as well as ancestor package-level redirection (only load the actual code once!) + redirect = None + explicit_redirect = False + + routing_entry = _nested_dict_get(collection_meta, ['import_redirection', self._fullname]) + if routing_entry: + redirect = routing_entry.get('redirect') + + if redirect: + explicit_redirect = True + else: + redirect = _get_ancestor_redirect(self._redirected_package_map, self._fullname) + + # NB: package level redirection requires hooking all future imports beneath the redirected source package + # in order to ensure sanity on future relative imports. We always import everything under its "real" name, + # then add a sys.modules entry with the redirected name using the same module instance. If we naively imported + # the source for each redirection, most submodules would import OK, but we'd have N runtime copies of the module + # (one for each name), and relative imports that ascend above the redirected package would break (since they'd + # see the redirected ancestor package contents instead of the package where they actually live). + if redirect: + # FIXME: wrap this so we can be explicit about a failed redirection + self._redirect_module = import_module(redirect) + if explicit_redirect and hasattr(self._redirect_module, '__path__') and self._redirect_module.__path__: + # if the import target looks like a package, store its name so we can rewrite future descendent loads + # FIXME: shouldn't this be in a shared location? This is currently per loader instance, so + self._redirected_package_map[self._fullname] = redirect + + # if we redirected, don't do any further custom package logic + return None + + # we're not doing a redirect- try to find what we need to actually load a module/package + + # this will raise ImportError if we can't find the requested module/package at all + if not candidate_paths: + # noplace to look, just ImportError + raise ImportError('package has no paths') + + found_path, has_code, package_path = self._module_file_from_path(self._package_to_load, candidate_paths[0]) + + # still here? we found something to load... + if has_code: + self._source_code_path = found_path + + if package_path: + return [package_path] # always needs to be a list + + return None + + +# This loader only answers for intercepted Ansible Python modules. Normal imports will fail here and be picked up later +# by our path_hook importer (which proxies the built-in import mechanisms, allowing normal caching etc to occur) +class _AnsibleInternalRedirectLoader: + def __init__(self, fullname, path_list): + self._redirect = None + + split_name = fullname.split('.') + toplevel_pkg = split_name[0] + module_to_load = split_name[-1] + + if toplevel_pkg != 'ansible': + raise ImportError('not interested') + + builtin_meta = _get_collection_metadata('ansible.builtin') + + routing_entry = _nested_dict_get(builtin_meta, ['import_redirection', fullname]) + if routing_entry: + self._redirect = routing_entry.get('redirect') + + if not self._redirect: + raise ImportError('not redirected, go ask path_hook') + + def load_module(self, fullname): + # since we're delegating to other loaders, this should only be called for internal redirects where we answered + # find_module with this loader, in which case we'll just directly import the redirection target, insert it into + # sys.modules under the name it was requested by, and return the original module. + + # should never see this + if not self._redirect: + raise ValueError('no redirect found for {0}'.format(fullname)) + + # FIXME: smuggle redirection context, provide warning/error that we tried and failed to redirect + mod = import_module(self._redirect) + sys.modules[fullname] = mod + return mod + + +class AnsibleCollectionRef: + # FUTURE: introspect plugin loaders to get these dynamically? + VALID_REF_TYPES = frozenset(to_text(r) for r in ['action', 'become', 'cache', 'callback', 'cliconf', 'connection', + 'doc_fragments', 'filter', 'httpapi', 'inventory', 'lookup', + 'module_utils', 'modules', 'netconf', 'role', 'shell', 'strategy', + 'terminal', 'test', 'vars']) + + # FIXME: tighten this up to match Python identifier reqs, etc + VALID_COLLECTION_NAME_RE = re.compile(to_text(r'^(\w+)\.(\w+)$')) + VALID_SUBDIRS_RE = re.compile(to_text(r'^\w+(\.\w+)*$')) + VALID_FQCR_RE = re.compile(to_text(r'^\w+\.\w+\.\w+(\.\w+)*$')) # can have 0-N included subdirs as well + + def __init__(self, collection_name, subdirs, resource, ref_type): + """ + Create an AnsibleCollectionRef from components + :param collection_name: a collection name of the form 'namespace.collectionname' + :param subdirs: optional subdir segments to be appended below the plugin type (eg, 'subdir1.subdir2') + :param resource: the name of the resource being references (eg, 'mymodule', 'someaction', 'a_role') + :param ref_type: the type of the reference, eg 'module', 'role', 'doc_fragment' + """ + collection_name = to_text(collection_name, errors='strict') + if subdirs is not None: + subdirs = to_text(subdirs, errors='strict') + resource = to_text(resource, errors='strict') + ref_type = to_text(ref_type, errors='strict') + + if not self.is_valid_collection_name(collection_name): + raise ValueError('invalid collection name (must be of the form namespace.collection): {0}'.format(to_native(collection_name))) + + if ref_type not in self.VALID_REF_TYPES: + raise ValueError('invalid collection ref_type: {0}'.format(ref_type)) + + self.collection = collection_name + if subdirs: + if not re.match(self.VALID_SUBDIRS_RE, subdirs): + raise ValueError('invalid subdirs entry: {0} (must be empty/None or of the form subdir1.subdir2)'.format(to_native(subdirs))) + self.subdirs = subdirs + else: + self.subdirs = u'' + + self.resource = resource + self.ref_type = ref_type + + package_components = [u'ansible_collections', self.collection] + fqcr_components = [self.collection] + + self.n_python_collection_package_name = to_native('.'.join(package_components)) + + if self.ref_type == u'role': + package_components.append(u'roles') + else: + # we assume it's a plugin + package_components += [u'plugins', self.ref_type] + + if self.subdirs: + package_components.append(self.subdirs) + fqcr_components.append(self.subdirs) + + if self.ref_type == u'role': + # roles are their own resource + package_components.append(self.resource) + + fqcr_components.append(self.resource) + + self.n_python_package_name = to_native('.'.join(package_components)) + self._fqcr = u'.'.join(fqcr_components) + + def __repr__(self): + return 'AnsibleCollectionRef(collection={0!r}, subdirs={1!r}, resource={2!r})'.format(self.collection, self.subdirs, self.resource) + + @property + def fqcr(self): + return self._fqcr + + @staticmethod + def from_fqcr(ref, ref_type): + """ + Parse a string as a fully-qualified collection reference, raises ValueError if invalid + :param ref: collection reference to parse (a valid ref is of the form 'ns.coll.resource' or 'ns.coll.subdir1.subdir2.resource') + :param ref_type: the type of the reference, eg 'module', 'role', 'doc_fragment' + :return: a populated AnsibleCollectionRef object + """ + # assuming the fq_name is of the form (ns).(coll).(optional_subdir_N).(resource_name), + # we split the resource name off the right, split ns and coll off the left, and we're left with any optional + # subdirs that need to be added back below the plugin-specific subdir we'll add. So: + # ns.coll.resource -> ansible_collections.ns.coll.plugins.(plugintype).resource + # ns.coll.subdir1.resource -> ansible_collections.ns.coll.plugins.subdir1.(plugintype).resource + # ns.coll.rolename -> ansible_collections.ns.coll.roles.rolename + if not AnsibleCollectionRef.is_valid_fqcr(ref): + raise ValueError('{0} is not a valid collection reference'.format(to_native(ref))) + + ref = to_text(ref, errors='strict') + ref_type = to_text(ref_type, errors='strict') + + resource_splitname = ref.rsplit(u'.', 1) + package_remnant = resource_splitname[0] + resource = resource_splitname[1] + + # split the left two components of the collection package name off, anything remaining is plugin-type + # specific subdirs to be added back on below the plugin type + package_splitname = package_remnant.split(u'.', 2) + if len(package_splitname) == 3: + subdirs = package_splitname[2] + else: + subdirs = u'' + + collection_name = u'.'.join(package_splitname[0:2]) + + return AnsibleCollectionRef(collection_name, subdirs, resource, ref_type) + + @staticmethod + def try_parse_fqcr(ref, ref_type): + """ + Attempt to parse a string as a fully-qualified collection reference, returning None on failure (instead of raising an error) + :param ref: collection reference to parse (a valid ref is of the form 'ns.coll.resource' or 'ns.coll.subdir1.subdir2.resource') + :param ref_type: the type of the reference, eg 'module', 'role', 'doc_fragment' + :return: a populated AnsibleCollectionRef object on successful parsing, else None + """ + try: + return AnsibleCollectionRef.from_fqcr(ref, ref_type) + except ValueError: + pass + + @staticmethod + def legacy_plugin_dir_to_plugin_type(legacy_plugin_dir_name): + """ + Utility method to convert from a PluginLoader dir name to a plugin ref_type + :param legacy_plugin_dir_name: PluginLoader dir name (eg, 'action_plugins', 'library') + :return: the corresponding plugin ref_type (eg, 'action', 'role') + """ + legacy_plugin_dir_name = to_text(legacy_plugin_dir_name) + + plugin_type = legacy_plugin_dir_name.replace(u'_plugins', u'') + + if plugin_type == u'library': + plugin_type = u'modules' + + if plugin_type not in AnsibleCollectionRef.VALID_REF_TYPES: + raise ValueError('{0} cannot be mapped to a valid collection ref type'.format(to_native(legacy_plugin_dir_name))) + + return plugin_type + + @staticmethod + def is_valid_fqcr(ref, ref_type=None): + """ + Validates if is string is a well-formed fully-qualified collection reference (does not look up the collection itself) + :param ref: candidate collection reference to validate (a valid ref is of the form 'ns.coll.resource' or 'ns.coll.subdir1.subdir2.resource') + :param ref_type: optional reference type to enable deeper validation, eg 'module', 'role', 'doc_fragment' + :return: True if the collection ref passed is well-formed, False otherwise + """ + + ref = to_text(ref) + + if not ref_type: + return bool(re.match(AnsibleCollectionRef.VALID_FQCR_RE, ref)) + + return bool(AnsibleCollectionRef.try_parse_fqcr(ref, ref_type)) + + @staticmethod + def is_valid_collection_name(collection_name): + """ + Validates if the given string is a well-formed collection name (does not look up the collection itself) + :param collection_name: candidate collection name to validate (a valid name is of the form 'ns.collname') + :return: True if the collection name passed is well-formed, False otherwise + """ + + collection_name = to_text(collection_name) + + return bool(re.match(AnsibleCollectionRef.VALID_COLLECTION_NAME_RE, collection_name)) + + +def _get_collection_role_path(role_name, collection_list=None): + acr = AnsibleCollectionRef.try_parse_fqcr(role_name, 'role') + + if acr: + # looks like a valid qualified collection ref; skip the collection_list + collection_list = [acr.collection] + subdirs = acr.subdirs + resource = acr.resource + elif not collection_list: + return None # not a FQ role and no collection search list spec'd, nothing to do + else: + resource = role_name # treat as unqualified, loop through the collection search list to try and resolve + subdirs = '' + + for collection_name in collection_list: + try: + acr = AnsibleCollectionRef(collection_name=collection_name, subdirs=subdirs, resource=resource, ref_type='role') + # FIXME: error handling/logging; need to catch any import failures and move along + pkg = import_module(acr.n_python_package_name) + + if pkg is not None: + # the package is now loaded, get the collection's package and ask where it lives + path = os.path.dirname(to_bytes(sys.modules[acr.n_python_package_name].__file__, errors='surrogate_or_strict')) + return resource, to_text(path, errors='surrogate_or_strict'), collection_name + + except IOError: + continue + except Exception as ex: + # FIXME: pick out typical import errors first, then error logging + continue + + return None + + +def _get_collection_name_from_path(path): + """ + Return the containing collection name for a given path, or None if the path is not below a configured collection, or + the collection cannot be loaded (eg, the collection is masked by another of the same name higher in the configured + collection roots). + :param path: path to evaluate for collection containment + :return: collection name or None + """ + + # FIXME: mess with realpath canonicalization or not? + path = to_native(path) + + path_parts = path.split('/') + if path_parts.count('ansible_collections') != 1: + return None + + ac_pos = path_parts.index('ansible_collections') + + # make sure it's followed by at least a namespace and collection name + if len(path_parts) < ac_pos + 3: + return None + + candidate_collection_name = '.'.join(path_parts[ac_pos + 1:ac_pos + 3]) + + try: + # we've got a name for it, now see if the path prefix matches what the loader sees + imported_pkg_path = to_native(os.path.dirname(to_bytes(import_module('ansible_collections.' + candidate_collection_name).__file__))) + except ImportError: + return None + + # reassemble the original path prefix up the collection name, and it should match what we just imported. If not + # this is probably a collection root that's not configured. + + original_path_prefix = os.path.join('/', *path_parts[0:ac_pos + 3]) + + if original_path_prefix != imported_pkg_path: + return None + + return candidate_collection_name + + +def _get_import_redirect(collection_meta_dict, fullname): + if not collection_meta_dict: + return None + + return _nested_dict_get(collection_meta_dict, ['import_redirection', fullname, 'redirect']) + + +def _get_ancestor_redirect(redirected_package_map, fullname): + # walk the requested module's ancestor packages to see if any have been previously redirected + cur_pkg = fullname + while cur_pkg: + cur_pkg = cur_pkg.rpartition('.')[0] + ancestor_redirect = redirected_package_map.get(cur_pkg) + if ancestor_redirect: + # rewrite the prefix on fullname so we import the target first, then alias it + redirect = ancestor_redirect + fullname[len(cur_pkg):] + return redirect + return None + + +def _nested_dict_get(root_dict, key_list): + cur_value = root_dict + for key in key_list: + cur_value = cur_value.get(key) + if not cur_value: + return None + + return cur_value + + +def _iter_modules_impl(paths, prefix=''): + # NB: this currently only iterates what's on disk- redirected modules are not considered + if not prefix: + prefix = '' + else: + prefix = to_native(prefix) + # yield (module_loader, name, ispkg) for each module/pkg under path + # TODO: implement ignore/silent catch for unreadable? + for b_path in map(to_bytes, paths): + if not os.path.isdir(b_path): + continue + for b_basename in sorted(os.listdir(b_path)): + b_candidate_module_path = os.path.join(b_path, b_basename) + if os.path.isdir(b_candidate_module_path): + # exclude things that obviously aren't Python package dirs + # FIXME: this dir is adjustable in py3.8+, check for it + if b'.' in b_basename or b_basename == b'__pycache__': + continue + + # TODO: proper string handling? + yield prefix + to_native(b_basename), True + else: + # FIXME: match builtin ordering for package/dir/file, support compiled? + if b_basename.endswith(b'.py') and b_basename != b'__init__.py': + yield prefix + to_native(os.path.splitext(b_basename)[0]), False + + +def _get_collection_metadata(collection_name): + collection_name = to_native(collection_name) + if not collection_name or not isinstance(collection_name, string_types) or len(collection_name.split('.')) != 2: + raise ValueError('collection_name must be a non-empty string of the form namespace.collection') + + try: + collection_pkg = import_module('ansible_collections.' + collection_name) + except ImportError: + raise ValueError('unable to locate collection {0}'.format(collection_name)) + + _collection_meta = getattr(collection_pkg, '_collection_meta', None) + + if _collection_meta is None: + raise ValueError('collection metadata was not loaded for collection {0}'.format(collection_name)) + + return _collection_meta diff --git a/lib/ansible/utils/collection_loader/_collection_meta.py b/lib/ansible/utils/collection_loader/_collection_meta.py new file mode 100644 index 0000000000..979a921cab --- /dev/null +++ b/lib/ansible/utils/collection_loader/_collection_meta.py @@ -0,0 +1,17 @@ +# (c) 2019 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from yaml import safe_load + + +def _meta_yml_to_dict(yaml_string_data, content_id): + routing_dict = safe_load(yaml_string_data) + if not routing_dict: + routing_dict = {} + # TODO: change this to Mapping abc? + if not isinstance(routing_dict, dict): + raise ValueError('collection metadata must be a dictionary') + return routing_dict diff --git a/requirements.txt b/requirements.txt index 2b77e2f880..44689bb54e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ jinja2 PyYAML cryptography +packaging diff --git a/test/integration/targets/ansible-doc/aliases b/test/integration/targets/ansible-doc/aliases index a6dafcf8cd..268da2c702 100644 --- a/test/integration/targets/ansible-doc/aliases +++ b/test/integration/targets/ansible-doc/aliases @@ -1 +1,2 @@ shippable/posix/group1 +disabled diff --git a/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/meta/runtime.yml b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/meta/runtime.yml new file mode 100644 index 0000000000..cb21fee66f --- /dev/null +++ b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/meta/runtime.yml @@ -0,0 +1,39 @@ +plugin_routing: + action: + uses_redirected_action: + redirect: testns.testcoll.subclassed_normal + connection: + redirected_local: + redirect: ansible.builtin.local + modules: + multilevel1: + redirect: testns.testcoll.multilevel2 + multilevel2: + redirect: testns.testcoll.multilevel3 + multilevel3: + redirect: testns.testcoll.ping + uses_redirected_action: + redirect: ansible.builtin.ping + setup.ps1: ansible.windows.setup + looped_ping: + redirect: testns.testcoll.looped_ping2 + looped_ping2: + redirect: testns.testcoll.looped_ping + bogus_redirect: + redirect: bogus.collection.shouldbomb + deprecated_ping: + deprecation: + removal_date: 2020-12-31 + warning_text: old_ping will be removed in a future release of this collection. Use new_ping instead. + foobar_facts: + redirect: foobar_info + aliased_ping: + redirect: ansible.builtin.ping + dead_ping: + tombstone: + removal_date: 2019-12-31 + warning_text: dead_ping has been removed + module_utils: + moved_out_root: + redirect: testns.content_adj.sub1.foomodule +requires_ansible: '>=2.11' diff --git a/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/action/subclassed_normal.py b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/action/subclassed_normal.py new file mode 100644 index 0000000000..f0eff30bf3 --- /dev/null +++ b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/action/subclassed_normal.py @@ -0,0 +1,11 @@ +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible.plugins.action.normal import ActionModule as NormalAction + + +class ActionModule(NormalAction): + def run(self, *args, **kwargs): + result = super(ActionModule, self).run(*args, **kwargs) + result['hacked'] = 'I got run under a subclassed normal, yay' + return result diff --git a/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/action/uses_redirected_import.py b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/action/uses_redirected_import.py new file mode 100644 index 0000000000..701d7b468e --- /dev/null +++ b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/action/uses_redirected_import.py @@ -0,0 +1,20 @@ +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible.plugins.action import ActionBase +from ansible.module_utils.formerly_core import thingtocall + + +class ActionModule(ActionBase): + TRANSFERS_FILES = False + _VALID_ARGS = frozenset() + + def run(self, tmp=None, task_vars=None): + if task_vars is None: + task_vars = dict() + + result = super(ActionModule, self).run(None, task_vars) + + result = dict(changed=False, ttc_res=thingtocall()) + + return result diff --git a/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/module_utils/nested_same/nested_same/nested_same.py b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/module_utils/nested_same/nested_same/nested_same.py new file mode 100644 index 0000000000..774075646a --- /dev/null +++ b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/module_utils/nested_same/nested_same/nested_same.py @@ -0,0 +1,6 @@ +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +def nested_same(): + return 'hello from nested_same' diff --git a/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/module_utils/subpkg_with_init.py b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/module_utils/subpkg_with_init.py new file mode 100644 index 0000000000..b48a717cae --- /dev/null +++ b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/module_utils/subpkg_with_init.py @@ -0,0 +1,11 @@ +# NB: this module should never be loaded, since we'll see the subpkg_with_init package dir first +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +def thingtocall(): + raise Exception('this should never be called (loaded discrete module instead of package module)') + + +def anotherthingtocall(): + raise Exception('this should never be called (loaded discrete module instead of package module)') diff --git a/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/modules/deprecated_ping.py b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/modules/deprecated_ping.py new file mode 100644 index 0000000000..9698ba6f07 --- /dev/null +++ b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/modules/deprecated_ping.py @@ -0,0 +1,13 @@ +#!/usr/bin/python +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import json + + +def main(): + print(json.dumps(dict(changed=False, source='user', is_deprecated=True))) + + +if __name__ == '__main__': + main() diff --git a/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/modules/uses_collection_redirected_mu.py b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/modules/uses_collection_redirected_mu.py new file mode 100644 index 0000000000..736eb400f0 --- /dev/null +++ b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/modules/uses_collection_redirected_mu.py @@ -0,0 +1,19 @@ +#!/usr/bin/python +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import json +import sys + +from ansible_collections.testns.testcoll.plugins.module_utils.moved_out_root import importme + + +def main(): + mu_result = importme() + print(json.dumps(dict(changed=False, source='user', mu_result=mu_result))) + + sys.exit() + + +if __name__ == '__main__': + main() diff --git a/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/modules/uses_core_redirected_mu.py b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/modules/uses_core_redirected_mu.py new file mode 100644 index 0000000000..28a077293a --- /dev/null +++ b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/modules/uses_core_redirected_mu.py @@ -0,0 +1,19 @@ +#!/usr/bin/python +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import json +import sys + +from ansible.module_utils.formerly_core import thingtocall + + +def main(): + mu_result = thingtocall() + print(json.dumps(dict(changed=False, source='user', mu_result=mu_result))) + + sys.exit() + + +if __name__ == '__main__': + main() diff --git a/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/modules/uses_nested_same_as_func.py b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/modules/uses_nested_same_as_func.py new file mode 100644 index 0000000000..26fa53c077 --- /dev/null +++ b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/modules/uses_nested_same_as_func.py @@ -0,0 +1,19 @@ +#!/usr/bin/python +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import json +import sys + +from ansible_collections.testns.testcoll.plugins.module_utils.nested_same.nested_same.nested_same import nested_same + + +def main(): + mu_result = nested_same() + print(json.dumps(dict(changed=False, source='user', mu_result=mu_result))) + + sys.exit() + + +if __name__ == '__main__': + main() diff --git a/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/modules/uses_nested_same_as_module.py b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/modules/uses_nested_same_as_module.py new file mode 100644 index 0000000000..e017c14f8e --- /dev/null +++ b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/modules/uses_nested_same_as_module.py @@ -0,0 +1,19 @@ +#!/usr/bin/python +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import json +import sys + +from ansible_collections.testns.testcoll.plugins.module_utils.nested_same.nested_same import nested_same + + +def main(): + mu_result = nested_same.nested_same() + print(json.dumps(dict(changed=False, source='user', mu_result=mu_result))) + + sys.exit() + + +if __name__ == '__main__': + main() diff --git a/test/integration/targets/collections/collections/ansible_collections/testns/content_adj/plugins/module_utils/sub1/foomodule.py b/test/integration/targets/collections/collections/ansible_collections/testns/content_adj/plugins/module_utils/sub1/foomodule.py new file mode 100644 index 0000000000..eeffe01e29 --- /dev/null +++ b/test/integration/targets/collections/collections/ansible_collections/testns/content_adj/plugins/module_utils/sub1/foomodule.py @@ -0,0 +1,6 @@ +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +def importme(): + return "hello from {0}".format(__name__) diff --git a/test/integration/targets/collections/filter_plugins/override_formerly_core_masked_filter.py b/test/integration/targets/collections/filter_plugins/override_formerly_core_masked_filter.py new file mode 100644 index 0000000000..600b1fd8c2 --- /dev/null +++ b/test/integration/targets/collections/filter_plugins/override_formerly_core_masked_filter.py @@ -0,0 +1,13 @@ +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +def override_formerly_core_masked_filter(*args, **kwargs): + return 'hello from overridden formerly_core_masked_filter' + + +class FilterModule(object): + def filters(self): + return { + 'formerly_core_masked_filter': override_formerly_core_masked_filter + } diff --git a/test/integration/targets/collections/inventory_test.yml b/test/integration/targets/collections/inventory_test.yml new file mode 100644 index 0000000000..b50892788d --- /dev/null +++ b/test/integration/targets/collections/inventory_test.yml @@ -0,0 +1,26 @@ +- name: test a collection-hosted connection plugin against hosts from collection-hosted inventory plugins + hosts: dynamic_host_a, dynamic_host_redirected + gather_facts: no + vars: + ansible_connection: testns.testcoll.localconn + ansible_localconn_connectionvar: from_play + tasks: + - raw: echo 'hello world' + register: connection_out + + - assert: + that: + - connection_out.stdout == "localconn ran echo 'hello world'" + # ensure that the connection var we overrode above made it into the running config + - connection_out.stderr == "connectionvar is from_play" + + +- hosts: localhost + gather_facts: no + tasks: + - assert: + that: + - hostvars['dynamic_host_a'] is defined + - hostvars['dynamic_host_a'].connection_out.stdout == "localconn ran echo 'hello world'" + - hostvars['dynamic_host_redirected'] is defined + - hostvars['dynamic_host_redirected'].connection_out.stdout == "localconn ran echo 'hello world'" diff --git a/test/integration/targets/collections/posix.yml b/test/integration/targets/collections/posix.yml index 61f950f50b..ab74aedb23 100644 --- a/test/integration/targets/collections/posix.yml +++ b/test/integration/targets/collections/posix.yml @@ -63,6 +63,16 @@ testns.testcoll.uses_leaf_mu_module_import_from: register: from_out + # module with multiple levels of the same nested package name and imported as a function + - name: exec module with multiple levels of the same nested package name imported as a function + testns.testcoll.uses_nested_same_as_func: + register: from_nested_func + + # module with multiple levels of the same nested package name and imported as a module + - name: exec module with multiple levels of the same nested package name imported as a module + testns.testcoll.uses_nested_same_as_module: + register: from_nested_module + - assert: that: - testmodule_out.source == 'user' @@ -79,6 +89,8 @@ - flat_out.mu_result == 'thingtocall in leaf' - from_out.mu_result == 'thingtocall in leaf' - from_out.mu2_result == 'thingtocall in secondary' + - from_nested_func.mu_result == 'hello from nested_same' + - from_nested_module.mu_result == 'hello from nested_same' - hosts: testhost tasks: @@ -373,28 +385,7 @@ - include_tasks: includeme.yml -- name: test a collection-hosted connection plugin against a host from a collection-hosted inventory plugin - hosts: dynamic_host_a - vars: - ansible_connection: testns.testcoll.localconn - ansible_localconn_connectionvar: from_play - tasks: - - raw: echo 'hello world' - register: connection_out - - - assert: - that: - - connection_out.stdout == "localconn ran echo 'hello world'" - # ensure that the connection var we overrode above made it into the running config - - connection_out.stderr == "connectionvar is from_play" - -- hosts: testhost - tasks: - - assert: - that: - - hostvars['dynamic_host_a'] is defined - - hostvars['dynamic_host_a'].connection_out.stdout == "localconn ran echo 'hello world'" - +- import_playbook: test_collection_meta.yml - name: Test FQCN handlers hosts: testhost vars: diff --git a/test/integration/targets/collections/redirected.statichost.yml b/test/integration/targets/collections/redirected.statichost.yml new file mode 100644 index 0000000000..9fd2c2d87b --- /dev/null +++ b/test/integration/targets/collections/redirected.statichost.yml @@ -0,0 +1,3 @@ +# use a plugin redirected by core to a collection to ensure inventory redirection and redirected config names are working +plugin: formerly_core_inventory # this is defined in the ansible-base runtime.yml routing to point at testns.content_adj.statichost +hostname: dynamic_host_redirected diff --git a/test/integration/targets/collections/runme.sh b/test/integration/targets/collections/runme.sh index 98471b2761..e0c7ef0319 100755 --- a/test/integration/targets/collections/runme.sh +++ b/test/integration/targets/collections/runme.sh @@ -38,19 +38,22 @@ else export TEST_PLAYBOOK=posix.yml echo "testing default collection support" - ansible-playbook -i "${INVENTORY_PATH}" collection_root_user/ansible_collections/testns/testcoll/playbooks/default_collection_playbook.yml + ansible-playbook -i "${INVENTORY_PATH}" collection_root_user/ansible_collections/testns/testcoll/playbooks/default_collection_playbook.yml "$@" fi # run test playbooks -ansible-playbook -i "${INVENTORY_PATH}" -i ./a.statichost.yml -v "${TEST_PLAYBOOK}" "$@" +ansible-playbook -i "${INVENTORY_PATH}" -v "${TEST_PLAYBOOK}" "$@" if [[ ${INVENTORY_PATH} != *.winrm ]]; then - ansible-playbook -i "${INVENTORY_PATH}" -i ./a.statichost.yml -v invocation_tests.yml "$@" + ansible-playbook -i "${INVENTORY_PATH}" -v invocation_tests.yml "$@" fi +# test collection inventories +ansible-playbook inventory_test.yml -i a.statichost.yml -i redirected.statichost.yml "$@" + # test adjacent with --playbook-dir export ANSIBLE_COLLECTIONS_PATHS='' -ANSIBLE_INVENTORY_ANY_UNPARSED_IS_FAILED=1 ansible-inventory -i a.statichost.yml --list --export --playbook-dir=. -v "$@" +ANSIBLE_INVENTORY_ANY_UNPARSED_IS_FAILED=1 ansible-inventory --list --export --playbook-dir=. -v "$@" # use an inventory source with caching enabled ansible-playbook -i a.statichost.yml -i ./cache.statichost.yml -v check_populated_inventory.yml diff --git a/test/integration/targets/collections/test_collection_meta.yml b/test/integration/targets/collections/test_collection_meta.yml new file mode 100644 index 0000000000..22a00b2197 --- /dev/null +++ b/test/integration/targets/collections/test_collection_meta.yml @@ -0,0 +1,46 @@ +- hosts: localhost + gather_facts: no + collections: + - testns.testcoll + vars: + # redirect connection + ansible_connection: testns.testcoll.redirected_local + tasks: + - assert: + that: ('data' | testns.testcoll.testfilter) == 'data_via_testfilter_from_userdir' + + # redirect module (multiple levels) + - multilevel1: + # redirect action + - uses_redirected_action: + # redirect import (consumed via action) + - uses_redirected_import: + # redirect lookup + - assert: + that: lookup('formerly_core_lookup') == 'mylookup_from_user_dir' + # redirect filter + - assert: + that: ('yes' | formerly_core_filter) == True + # legacy filter should mask redirected + - assert: + that: ('' | formerly_core_masked_filter) == 'hello from overridden formerly_core_masked_filter' + # redirect test + - assert: + that: + - "'stuff' is formerly_core_test('tuf')" + - "'hello override' is formerly_core_masked_test" + # redirect module (formerly internal) + - formerly_core_ping: + # redirect module from collection (with subdir) + - testns.testcoll.module_subdir.subdir_ping_module: + # redirect module_utils plugin (consumed via module) + - uses_core_redirected_mu: + # deprecated module (issues warning) + - deprecated_ping: + # redirect module (internal alias) + - aliased_ping: + # redirect module (cycle detection, fatal) +# - looped_ping: + + # removed module (fatal) +# - dead_ping: diff --git a/test/integration/targets/collections/test_plugins/override_formerly_core_masked_test.py b/test/integration/targets/collections/test_plugins/override_formerly_core_masked_test.py new file mode 100644 index 0000000000..11c7f7a7e3 --- /dev/null +++ b/test/integration/targets/collections/test_plugins/override_formerly_core_masked_test.py @@ -0,0 +1,16 @@ +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +def override_formerly_core_masked_test(value, *args, **kwargs): + if value != 'hello override': + raise Exception('expected "hello override" only...') + + return True + + +class TestModule(object): + def tests(self): + return { + 'formerly_core_masked_test': override_formerly_core_masked_test + } diff --git a/test/lib/ansible_test/_data/pytest/plugins/ansible_pytest_collections.py b/test/lib/ansible_test/_data/pytest/plugins/ansible_pytest_collections.py index 91e2047878..906fdc7c70 100644 --- a/test/lib/ansible_test/_data/pytest/plugins/ansible_pytest_collections.py +++ b/test/lib/ansible_test/_data/pytest/plugins/ansible_pytest_collections.py @@ -26,10 +26,11 @@ def pytest_configure(): except AttributeError: pytest_configure.executed = True - from ansible.utils.collection_loader import AnsibleCollectionLoader + from ansible.utils.collection_loader._collection_finder import _AnsibleCollectionFinder # allow unit tests to import code from collections - sys.meta_path.insert(0, AnsibleCollectionLoader()) + + _AnsibleCollectionFinder(paths=[os.path.dirname(ANSIBLE_COLLECTIONS_PATH)])._install() # pylint: disable=protected-access # noinspection PyProtectedMember import py._path.local diff --git a/test/lib/ansible_test/_data/requirements/integration.txt b/test/lib/ansible_test/_data/requirements/integration.txt index 469d21f998..2c562615a8 100644 --- a/test/lib/ansible_test/_data/requirements/integration.txt +++ b/test/lib/ansible_test/_data/requirements/integration.txt @@ -2,4 +2,5 @@ cryptography jinja2 junit-xml ordereddict ; python_version < '2.7' +packaging pyyaml diff --git a/test/lib/ansible_test/_data/requirements/network-integration.txt b/test/lib/ansible_test/_data/requirements/network-integration.txt index f5a45ebe88..726d29438f 100644 --- a/test/lib/ansible_test/_data/requirements/network-integration.txt +++ b/test/lib/ansible_test/_data/requirements/network-integration.txt @@ -2,5 +2,6 @@ cryptography jinja2 junit-xml ordereddict ; python_version < '2.7' # ansible-test junit callback plugin requirement +packaging paramiko pyyaml diff --git a/test/lib/ansible_test/_data/requirements/windows-integration.txt b/test/lib/ansible_test/_data/requirements/windows-integration.txt index 1c56a8d178..86de35eee8 100644 --- a/test/lib/ansible_test/_data/requirements/windows-integration.txt +++ b/test/lib/ansible_test/_data/requirements/windows-integration.txt @@ -5,6 +5,7 @@ ntlm-auth ordereddict ; python_version < '2.7' # ansible-test junit callback plugin requirement requests-ntlm requests-credssp +packaging pypsrp pywinrm[credssp] pyyaml diff --git a/test/lib/ansible_test/_data/sanity/import/importer.py b/test/lib/ansible_test/_data/sanity/import/importer.py index 2b81190c5e..80b5fc9026 100755 --- a/test/lib/ansible_test/_data/sanity/import/importer.py +++ b/test/lib/ansible_test/_data/sanity/import/importer.py @@ -9,17 +9,20 @@ def main(): Main program function used to isolate globals from imported code. Changes to globals in imported modules on Python 2.x will overwrite our own globals. """ + import ansible import contextlib + import json import os import re import runpy + import subprocess import sys import traceback - import types import warnings - ansible_path = os.environ['PYTHONPATH'] + ansible_path = os.path.dirname(os.path.dirname(ansible.__file__)) temp_path = os.environ['SANITY_TEMP_PATH'] + os.path.sep + external_python = os.environ.get('SANITY_EXTERNAL_PYTHON') collection_full_name = os.environ.get('SANITY_COLLECTION_FULL_NAME') collection_root = os.environ.get('ANSIBLE_COLLECTIONS_PATHS') @@ -37,63 +40,45 @@ def main(): except ImportError: from io import StringIO - # pre-load an empty ansible package to prevent unwanted code in __init__.py from loading - # without this the ansible.release import there would pull in many Python modules which Ansible modules should not have access to - ansible_module = types.ModuleType('ansible') - ansible_module.__file__ = os.path.join(os.environ['PYTHONPATH'], 'ansible', '__init__.py') - ansible_module.__path__ = [os.path.dirname(ansible_module.__file__)] - ansible_module.__package__ = 'ansible' - - sys.modules['ansible'] = ansible_module - if collection_full_name: # allow importing code from collections when testing a collection - from ansible.utils.collection_loader import AnsibleCollectionLoader - from ansible.module_utils._text import to_bytes - - def get_source(self, fullname): - with open(to_bytes(self.get_filename(fullname)), 'rb') as mod_file: - return mod_file.read() - - def get_code(self, fullname): - return compile(source=self.get_source(fullname), filename=self.get_filename(fullname), mode='exec', flags=0, dont_inherit=True) + from ansible.module_utils.common.text.converters import to_bytes, to_text, to_native + from ansible.utils.collection_loader import AnsibleCollectionConfig + from ansible.utils.collection_loader._collection_finder import _AnsibleCollectionFinder + from ansible.utils.collection_loader import _collection_finder - def is_package(self, fullname): - return os.path.basename(self.get_filename(fullname)) in ('__init__.py', '__synthetic__') + yaml_to_json_path = os.path.join(os.path.dirname(__file__), 'yaml_to_json.py') + yaml_to_dict_cache = {} - def get_filename(self, fullname): - if fullname in sys.modules: - return sys.modules[fullname].__file__ - - # find the module without importing it - # otherwise an ImportError during module load will prevent us from getting the filename of the module - loader = self.find_module(fullname) + def yaml_to_dict(yaml, content_id): + """ + Return a Python dict version of the provided YAML. + Conversion is done in a subprocess since the current Python interpreter does not have access to PyYAML. + """ + if content_id in yaml_to_dict_cache: + return yaml_to_dict_cache[content_id] - if not loader: - raise ImportError('module {0} not found'.format(fullname)) + try: + cmd = [external_python, yaml_to_json_path] + proc = subprocess.Popen([to_bytes(c) for c in cmd], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout_bytes, stderr_bytes = proc.communicate(to_bytes(yaml)) - # determine the filename of the module that was found - filename = os.path.join(collection_root, fullname.replace('.', os.path.sep)) + if proc.returncode != 0: + raise Exception('command %s failed with return code %d: %s' % ([to_native(c) for c in cmd], proc.returncode, to_native(stderr_bytes))) - if os.path.isdir(filename): - init_filename = os.path.join(filename, '__init__.py') - filename = init_filename if os.path.exists(init_filename) else os.path.join(filename, '__synthetic__') - else: - filename += '.py' + data = yaml_to_dict_cache[content_id] = json.loads(to_text(stdout_bytes)) - return filename + return data + except Exception as ex: + raise Exception('internal importer error - failed to parse yaml: %s' % to_native(ex)) - # monkeypatch collection loader to work with runpy - # remove this (and the associated code above) once implemented natively in the collection loader - AnsibleCollectionLoader.get_source = get_source - AnsibleCollectionLoader.get_code = get_code - AnsibleCollectionLoader.is_package = is_package - AnsibleCollectionLoader.get_filename = get_filename + _collection_finder._meta_yml_to_dict = yaml_to_dict # pylint: disable=protected-access - collection_loader = AnsibleCollectionLoader() + collection_loader = _AnsibleCollectionFinder(paths=[collection_root]) + collection_loader._install() # pylint: disable=protected-access + nuke_modules = list(m for m in sys.modules if m.partition('.')[0] == 'ansible') + map(sys.modules.pop, nuke_modules) - # noinspection PyCallingNonCallable - sys.meta_path.insert(0, collection_loader) else: # do not support collection loading when not testing a collection collection_loader = None diff --git a/test/lib/ansible_test/_data/sanity/import/yaml_to_json.py b/test/lib/ansible_test/_data/sanity/import/yaml_to_json.py new file mode 100644 index 0000000000..00269bd80b --- /dev/null +++ b/test/lib/ansible_test/_data/sanity/import/yaml_to_json.py @@ -0,0 +1,16 @@ +"""Read YAML from stdin and write JSON to stdout.""" +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import json +import sys + +from yaml import load + +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader + + +json.dump(load(sys.stdin, Loader=SafeLoader), sys.stdout) diff --git a/test/lib/ansible_test/_data/sanity/validate-modules/validate_modules/main.py b/test/lib/ansible_test/_data/sanity/validate-modules/validate_modules/main.py index 2e24095c6d..52a6017edd 100644 --- a/test/lib/ansible_test/_data/sanity/validate-modules/validate_modules/main.py +++ b/test/lib/ansible_test/_data/sanity/validate-modules/validate_modules/main.py @@ -42,7 +42,7 @@ from ansible.executor.module_common import REPLACER_WINDOWS from ansible.module_utils.common._collections_compat import Mapping from ansible.module_utils._text import to_bytes from ansible.plugins.loader import fragment_loader -from ansible.utils.collection_loader import AnsibleCollectionLoader +from ansible.utils.collection_loader._collection_finder import _AnsibleCollectionFinder from ansible.utils.plugin_docs import BLACKLIST, add_fragments, get_docstring from ansible.utils.version import SemanticVersion @@ -942,7 +942,7 @@ class ModuleValidator(Validator): else: # We are testing a collection if self.routing and self.routing.get('plugin_routing', {}).get('modules', {}).get(self.name, {}).get('deprecation', {}): - # meta/routing.yml says this is deprecated + # meta/runtime.yml says this is deprecated routing_says_deprecated = True deprecated = True @@ -1118,15 +1118,15 @@ class ModuleValidator(Validator): self.reporter.error( path=self.object_path, code='collections-no-underscore-on-deprecation', - msg='Deprecated content in collections MUST NOT start with "_", update meta/routing.yml instead', + msg='Deprecated content in collections MUST NOT start with "_", update meta/runtime.yml instead', ) if not (doc_deprecated == routing_says_deprecated): - # DOCUMENTATION.deprecated and meta/routing.yml disagree + # DOCUMENTATION.deprecated and meta/runtime.yml disagree self.reporter.error( path=self.object_path, code='deprecation-mismatch', - msg='"meta/routing.yml" and DOCUMENTATION.deprecation do not agree.' + msg='"meta/runtime.yml" and DOCUMENTATION.deprecation do not agree.' ) # In the future we should error if ANSIBLE_METADATA exists in a collection @@ -2140,39 +2140,8 @@ class PythonPackageValidator(Validator): def setup_collection_loader(): - def get_source(self, fullname): - mod = sys.modules.get(fullname) - if not mod: - mod = self.load_module(fullname) - - with open(to_bytes(mod.__file__), 'rb') as mod_file: - source = mod_file.read() - - return source - - def get_code(self, fullname): - return compile(source=self.get_source(fullname), filename=self.get_filename(fullname), mode='exec', flags=0, dont_inherit=True) - - def is_package(self, fullname): - return self.get_filename(fullname).endswith('__init__.py') - - def get_filename(self, fullname): - mod = sys.modules.get(fullname) or self.load_module(fullname) - - return mod.__file__ - - # monkeypatch collection loader to work with runpy - # remove this (and the associated code above) once implemented natively in the collection loader - AnsibleCollectionLoader.get_source = get_source - AnsibleCollectionLoader.get_code = get_code - AnsibleCollectionLoader.is_package = is_package - AnsibleCollectionLoader.get_filename = get_filename - - collection_loader = AnsibleCollectionLoader() - - # allow importing code from collections when testing a collection - # noinspection PyCallingNonCallable - sys.meta_path.insert(0, collection_loader) + collections_paths = os.environ.get('ANSIBLE_COLLECTIONS_PATHS', '').split(os.pathsep) + _AnsibleCollectionFinder(collections_paths) def re_compile(value): @@ -2228,8 +2197,8 @@ def run(): routing = None if args.collection: setup_collection_loader() - routing_file = 'meta/routing.yml' - # Load meta/routing.yml if it exists, as it may contain deprecation information + routing_file = 'meta/runtime.yml' + # Load meta/runtime.yml if it exists, as it may contain deprecation information if os.path.isfile(routing_file): try: with open(routing_file) as f: diff --git a/test/lib/ansible_test/_internal/sanity/import.py b/test/lib/ansible_test/_internal/sanity/import.py index 7d566fd9ee..b6e4a0a08a 100644 --- a/test/lib/ansible_test/_internal/sanity/import.py +++ b/test/lib/ansible_test/_internal/sanity/import.py @@ -76,10 +76,12 @@ class ImportTest(SanityMultipleVersion): """ capture_pip = args.verbosity < 2 + python = find_python(python_version) + if python_version.startswith('2.') and args.requirements: # hack to make sure that virtualenv is available under Python 2.x # on Python 3.x we can use the built-in venv - pip = generate_pip_command(find_python(python_version)) + pip = generate_pip_command(python) run_command(args, generate_pip_install(pip, 'sanity.import', packages=['virtualenv']), capture=capture_pip) settings = self.load_processor(args, python_version) @@ -102,8 +104,10 @@ class ImportTest(SanityMultipleVersion): # add the importer to our virtual environment so it can be accessed through the coverage injector importer_path = os.path.join(virtual_environment_bin, 'importer.py') + yaml_to_json_path = os.path.join(virtual_environment_bin, 'yaml_to_json.py') if not args.explain: os.symlink(os.path.abspath(os.path.join(SANITY_ROOT, 'import', 'importer.py')), importer_path) + os.symlink(os.path.abspath(os.path.join(SANITY_ROOT, 'import', 'yaml_to_json.py')), yaml_to_json_path) # activate the virtual environment env['PATH'] = '%s:%s' % (virtual_environment_bin, env['PATH']) @@ -115,6 +119,7 @@ class ImportTest(SanityMultipleVersion): if data_context().content.collection: env.update( SANITY_COLLECTION_FULL_NAME=data_context().content.collection.full_name, + SANITY_EXTERNAL_PYTHON=python, ) virtualenv_python = os.path.join(virtual_environment_bin, 'python') diff --git a/test/sanity/code-smell/no-unwanted-files.py b/test/sanity/code-smell/no-unwanted-files.py index 93260623b3..bff0915277 100755 --- a/test/sanity/code-smell/no-unwanted-files.py +++ b/test/sanity/code-smell/no-unwanted-files.py @@ -19,7 +19,7 @@ def main(): ) skip_paths = set([ - 'lib/ansible/config/routing.yml', # not included in the sanity ignore file since it won't exist until after migration + 'lib/ansible/config/ansible_builtin_runtime.yml', # not included in the sanity ignore file since it won't exist until after migration ]) skip_directories = ( diff --git a/test/sanity/ignore.txt b/test/sanity/ignore.txt index 8e7cbb9b41..bb5981ae60 100644 --- a/test/sanity/ignore.txt +++ b/test/sanity/ignore.txt @@ -550,12 +550,19 @@ test/units/plugins/test_plugins.py pylint:blacklisted-name test/units/template/test_templar.py pylint:blacklisted-name test/units/test_constants.py future-import-boilerplate test/units/test_context.py future-import-boilerplate -test/units/utils/fixtures/collections/ansible_collections/my_namespace/my_collection/plugins/action/my_action.py future-import-boilerplate -test/units/utils/fixtures/collections/ansible_collections/my_namespace/my_collection/plugins/action/my_action.py metaclass-boilerplate -test/units/utils/fixtures/collections/ansible_collections/my_namespace/my_collection/plugins/module_utils/my_other_util.py future-import-boilerplate -test/units/utils/fixtures/collections/ansible_collections/my_namespace/my_collection/plugins/module_utils/my_other_util.py metaclass-boilerplate -test/units/utils/fixtures/collections/ansible_collections/my_namespace/my_collection/plugins/module_utils/my_util.py future-import-boilerplate -test/units/utils/fixtures/collections/ansible_collections/my_namespace/my_collection/plugins/module_utils/my_util.py metaclass-boilerplate +test/units/utils/collection_loader/fixtures/collections/ansible_collections/testns/testcoll/plugins/action/my_action.py future-import-boilerplate +test/units/utils/collection_loader/fixtures/collections/ansible_collections/testns/testcoll/plugins/action/my_action.py metaclass-boilerplate +test/units/utils/collection_loader/fixtures/collections/ansible_collections/testns/testcoll/plugins/action/my_action.py pylint:relative-beyond-top-level +test/units/utils/collection_loader/fixtures/collections/ansible_collections/testns/testcoll/plugins/module_utils/my_other_util.py future-import-boilerplate +test/units/utils/collection_loader/fixtures/collections/ansible_collections/testns/testcoll/plugins/module_utils/my_other_util.py metaclass-boilerplate +test/units/utils/collection_loader/fixtures/collections/ansible_collections/testns/testcoll/plugins/module_utils/my_util.py future-import-boilerplate +test/units/utils/collection_loader/fixtures/collections/ansible_collections/testns/testcoll/plugins/module_utils/my_util.py metaclass-boilerplate +test/units/utils/collection_loader/fixtures/collections/ansible_collections/testns/testcoll/plugins/modules/__init__.py empty-init # testing that collections don't need inits +test/units/utils/collection_loader/fixtures/collections_masked/ansible_collections/__init__.py empty-init # testing that collections don't need inits +test/units/utils/collection_loader/fixtures/collections_masked/ansible_collections/ansible/__init__.py empty-init # testing that collections don't need inits +test/units/utils/collection_loader/fixtures/collections_masked/ansible_collections/testns/__init__.py empty-init # testing that collections don't need inits +test/units/utils/collection_loader/fixtures/collections_masked/ansible_collections/testns/testcoll/__init__.py empty-init # testing that collections don't need inits +test/units/utils/collection_loader/test_collection_loader.py pylint:undefined-variable # magic runtime local var splatting test/units/utils/test_cleanup_tmp_file.py future-import-boilerplate test/units/utils/test_encrypt.py future-import-boilerplate test/units/utils/test_encrypt.py metaclass-boilerplate diff --git a/test/units/utils/fixtures/collections/ansible_collections/my_namespace/my_collection/plugins/__init__.py b/test/units/utils/collection_loader/__init__.py index e69de29bb2..e69de29bb2 100644 --- a/test/units/utils/fixtures/collections/ansible_collections/my_namespace/my_collection/plugins/__init__.py +++ b/test/units/utils/collection_loader/__init__.py diff --git a/test/units/utils/collection_loader/fixtures/collections/ansible_collections/ansible/builtin/plugins/modules/shouldnotload.py b/test/units/utils/collection_loader/fixtures/collections/ansible_collections/ansible/builtin/plugins/modules/shouldnotload.py new file mode 100644 index 0000000000..4041a338ba --- /dev/null +++ b/test/units/utils/collection_loader/fixtures/collections/ansible_collections/ansible/builtin/plugins/modules/shouldnotload.py @@ -0,0 +1,4 @@ +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +raise Exception('this module should never be loaded') diff --git a/test/units/utils/collection_loader/fixtures/collections/ansible_collections/testns/testcoll/meta/runtime.yml b/test/units/utils/collection_loader/fixtures/collections/ansible_collections/testns/testcoll/meta/runtime.yml new file mode 100644 index 0000000000..f2e2fdec2f --- /dev/null +++ b/test/units/utils/collection_loader/fixtures/collections/ansible_collections/testns/testcoll/meta/runtime.yml @@ -0,0 +1,4 @@ +plugin_routing: + modules: + rerouted_module: + redirect: ansible.builtin.ping diff --git a/test/units/utils/collection_loader/fixtures/collections/ansible_collections/testns/testcoll/plugins/action/my_action.py b/test/units/utils/collection_loader/fixtures/collections/ansible_collections/testns/testcoll/plugins/action/my_action.py new file mode 100644 index 0000000000..5b19fa1a50 --- /dev/null +++ b/test/units/utils/collection_loader/fixtures/collections/ansible_collections/testns/testcoll/plugins/action/my_action.py @@ -0,0 +1,5 @@ +from ..module_utils.my_util import question + + +def action_code(): + return "hello from my_action.py" diff --git a/test/units/utils/fixtures/collections/ansible_collections/my_namespace/my_collection/plugins/action/__init__.py b/test/units/utils/collection_loader/fixtures/collections/ansible_collections/testns/testcoll/plugins/module_utils/__init__.py index e69de29bb2..e69de29bb2 100644 --- a/test/units/utils/fixtures/collections/ansible_collections/my_namespace/my_collection/plugins/action/__init__.py +++ b/test/units/utils/collection_loader/fixtures/collections/ansible_collections/testns/testcoll/plugins/module_utils/__init__.py diff --git a/test/units/utils/fixtures/collections/ansible_collections/my_namespace/my_collection/plugins/module_utils/my_other_util.py b/test/units/utils/collection_loader/fixtures/collections/ansible_collections/testns/testcoll/plugins/module_utils/my_other_util.py index 6b194179aa..6b194179aa 100644 --- a/test/units/utils/fixtures/collections/ansible_collections/my_namespace/my_collection/plugins/module_utils/my_other_util.py +++ b/test/units/utils/collection_loader/fixtures/collections/ansible_collections/testns/testcoll/plugins/module_utils/my_other_util.py diff --git a/test/units/utils/fixtures/collections/ansible_collections/my_namespace/my_collection/plugins/module_utils/my_util.py b/test/units/utils/collection_loader/fixtures/collections/ansible_collections/testns/testcoll/plugins/module_utils/my_util.py index c431c34ca4..c431c34ca4 100644 --- a/test/units/utils/fixtures/collections/ansible_collections/my_namespace/my_collection/plugins/module_utils/my_util.py +++ b/test/units/utils/collection_loader/fixtures/collections/ansible_collections/testns/testcoll/plugins/module_utils/my_util.py diff --git a/test/units/utils/collection_loader/fixtures/collections/ansible_collections/testns/testcoll/plugins/modules/__init__.py b/test/units/utils/collection_loader/fixtures/collections/ansible_collections/testns/testcoll/plugins/modules/__init__.py new file mode 100644 index 0000000000..6d69703453 --- /dev/null +++ b/test/units/utils/collection_loader/fixtures/collections/ansible_collections/testns/testcoll/plugins/modules/__init__.py @@ -0,0 +1,5 @@ +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +raise Exception('this should never run') diff --git a/test/units/utils/collection_loader/fixtures/collections/ansible_collections/testns/testcoll/plugins/modules/amodule.py b/test/units/utils/collection_loader/fixtures/collections/ansible_collections/testns/testcoll/plugins/modules/amodule.py new file mode 100644 index 0000000000..99320a0c37 --- /dev/null +++ b/test/units/utils/collection_loader/fixtures/collections/ansible_collections/testns/testcoll/plugins/modules/amodule.py @@ -0,0 +1,6 @@ +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +def module_code(): + return "hello from amodule.py" diff --git a/test/units/utils/fixtures/collections/ansible_collections/my_namespace/my_collection/plugins/module_utils/__init__.py b/test/units/utils/collection_loader/fixtures/collections/ansible_collections/testns/testcoll/roles/some_role/.gitkeep index e69de29bb2..e69de29bb2 100644 --- a/test/units/utils/fixtures/collections/ansible_collections/my_namespace/my_collection/plugins/module_utils/__init__.py +++ b/test/units/utils/collection_loader/fixtures/collections/ansible_collections/testns/testcoll/roles/some_role/.gitkeep diff --git a/test/units/utils/collection_loader/fixtures/collections_masked/ansible_collections/__init__.py b/test/units/utils/collection_loader/fixtures/collections_masked/ansible_collections/__init__.py new file mode 100644 index 0000000000..6068ac1a2b --- /dev/null +++ b/test/units/utils/collection_loader/fixtures/collections_masked/ansible_collections/__init__.py @@ -0,0 +1,5 @@ +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +raise Exception('this code should never execute') diff --git a/test/units/utils/collection_loader/fixtures/collections_masked/ansible_collections/ansible/__init__.py b/test/units/utils/collection_loader/fixtures/collections_masked/ansible_collections/ansible/__init__.py new file mode 100644 index 0000000000..6068ac1a2b --- /dev/null +++ b/test/units/utils/collection_loader/fixtures/collections_masked/ansible_collections/ansible/__init__.py @@ -0,0 +1,5 @@ +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +raise Exception('this code should never execute') diff --git a/test/units/utils/collection_loader/fixtures/collections_masked/ansible_collections/testns/__init__.py b/test/units/utils/collection_loader/fixtures/collections_masked/ansible_collections/testns/__init__.py new file mode 100644 index 0000000000..6068ac1a2b --- /dev/null +++ b/test/units/utils/collection_loader/fixtures/collections_masked/ansible_collections/testns/__init__.py @@ -0,0 +1,5 @@ +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +raise Exception('this code should never execute') diff --git a/test/units/utils/collection_loader/fixtures/collections_masked/ansible_collections/testns/testcoll/__init__.py b/test/units/utils/collection_loader/fixtures/collections_masked/ansible_collections/testns/testcoll/__init__.py new file mode 100644 index 0000000000..6068ac1a2b --- /dev/null +++ b/test/units/utils/collection_loader/fixtures/collections_masked/ansible_collections/testns/testcoll/__init__.py @@ -0,0 +1,5 @@ +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +raise Exception('this code should never execute') diff --git a/test/units/utils/collection_loader/fixtures/playbook_path/collections/ansible_collections/ansible/playbook_adj_other/.gitkeep b/test/units/utils/collection_loader/fixtures/playbook_path/collections/ansible_collections/ansible/playbook_adj_other/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/test/units/utils/collection_loader/fixtures/playbook_path/collections/ansible_collections/ansible/playbook_adj_other/.gitkeep diff --git a/test/units/utils/collection_loader/fixtures/playbook_path/collections/ansible_collections/freshns/playbook_adj_other/.gitkeep b/test/units/utils/collection_loader/fixtures/playbook_path/collections/ansible_collections/freshns/playbook_adj_other/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/test/units/utils/collection_loader/fixtures/playbook_path/collections/ansible_collections/freshns/playbook_adj_other/.gitkeep diff --git a/test/units/utils/collection_loader/fixtures/playbook_path/collections/ansible_collections/testns/playbook_adj_other/.gitkeep b/test/units/utils/collection_loader/fixtures/playbook_path/collections/ansible_collections/testns/playbook_adj_other/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/test/units/utils/collection_loader/fixtures/playbook_path/collections/ansible_collections/testns/playbook_adj_other/.gitkeep diff --git a/test/units/utils/collection_loader/test_collection_loader.py b/test/units/utils/collection_loader/test_collection_loader.py new file mode 100644 index 0000000000..fccc3a7798 --- /dev/null +++ b/test/units/utils/collection_loader/test_collection_loader.py @@ -0,0 +1,809 @@ +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import os +import pkgutil +import pytest +import re +import sys + +from ansible.module_utils.six import PY3, string_types +from ansible.module_utils.compat.importlib import import_module +from ansible.utils.collection_loader import AnsibleCollectionConfig, AnsibleCollectionRef +from ansible.utils.collection_loader._collection_finder import ( + _AnsibleCollectionFinder, _AnsibleCollectionLoader, _AnsibleCollectionNSPkgLoader, _AnsibleCollectionPkgLoader, + _AnsibleCollectionPkgLoaderBase, _AnsibleCollectionRootPkgLoader, _AnsiblePathHookFinder, + _get_collection_name_from_path, _get_collection_role_path, _get_collection_metadata, _iter_modules_impl +) +from ansible.utils.collection_loader._collection_config import _EventSource +from units.compat.mock import MagicMock, NonCallableMagicMock, patch + + +# fixture to ensure we always clean up the import stuff when we're done +@pytest.fixture(autouse=True, scope='function') +def teardown(*args, **kwargs): + yield + reset_collections_loader_state() + +# BEGIN STANDALONE TESTS - these exercise behaviors of the individual components without the import machinery + + +def test_finder_setup(): + # ensure scalar path is listified + f = _AnsibleCollectionFinder(paths='/bogus/bogus') + assert isinstance(f._n_collection_paths, list) + + # ensure sys.path paths that have an ansible_collections dir are added to the end of the collections paths + with patch.object(sys, 'path', ['/bogus', default_test_collection_paths[1], '/morebogus', default_test_collection_paths[0]]): + f = _AnsibleCollectionFinder(paths=['/explicit', '/other']) + assert f._n_collection_paths == ['/explicit', '/other', default_test_collection_paths[1], default_test_collection_paths[0]] + + configured_paths = ['/bogus'] + playbook_paths = ['/playbookdir'] + f = _AnsibleCollectionFinder(paths=configured_paths) + assert f._n_collection_paths == configured_paths + f.set_playbook_paths(playbook_paths) + assert f._n_collection_paths == extend_paths(playbook_paths, 'collections') + configured_paths + + # ensure scalar playbook_paths gets listified + f.set_playbook_paths(playbook_paths[0]) + assert f._n_collection_paths == extend_paths(playbook_paths, 'collections') + configured_paths + + +def test_finder_not_interested(): + f = get_default_finder() + assert f.find_module('nothanks') is None + assert f.find_module('nothanks.sub', path=['/bogus/dir']) is None + + +def test_finder_ns(): + # ensure we can still load ansible_collections and ansible_collections.ansible when they don't exist on disk + f = _AnsibleCollectionFinder(paths=['/bogus/bogus']) + loader = f.find_module('ansible_collections') + assert isinstance(loader, _AnsibleCollectionRootPkgLoader) + + loader = f.find_module('ansible_collections.ansible', path=['/bogus/bogus']) + assert isinstance(loader, _AnsibleCollectionNSPkgLoader) + + f = get_default_finder() + loader = f.find_module('ansible_collections') + assert isinstance(loader, _AnsibleCollectionRootPkgLoader) + + # path is not allowed for top-level + with pytest.raises(ValueError): + f.find_module('ansible_collections', path=['whatever']) + + # path is required for subpackages + with pytest.raises(ValueError): + f.find_module('ansible_collections.whatever', path=None) + + paths = [os.path.join(p, 'ansible_collections/nonexistns') for p in default_test_collection_paths] + + # test missing + loader = f.find_module('ansible_collections.nonexistns', paths) + assert loader is None + + +# keep these up top to make sure the loader install/remove are working, since we rely on them heavily in the tests +def test_loader_remove(): + fake_mp = [MagicMock(), _AnsibleCollectionFinder(), MagicMock(), _AnsibleCollectionFinder()] + fake_ph = [MagicMock().m1, MagicMock().m2, _AnsibleCollectionFinder()._ansible_collection_path_hook, NonCallableMagicMock] + # must nest until 2.6 compilation is totally donezo + with patch.object(sys, 'meta_path', fake_mp): + with patch.object(sys, 'path_hooks', fake_ph): + _AnsibleCollectionFinder()._remove() + assert len(sys.meta_path) == 2 + # no AnsibleCollectionFinders on the meta path after remove is called + assert all((not isinstance(mpf, _AnsibleCollectionFinder) for mpf in sys.meta_path)) + assert len(sys.path_hooks) == 3 + # none of the remaining path hooks should point at an AnsibleCollectionFinder + assert all((not isinstance(ph.__self__, _AnsibleCollectionFinder) for ph in sys.path_hooks if hasattr(ph, '__self__'))) + assert AnsibleCollectionConfig.collection_finder is None + + +def test_loader_install(): + fake_mp = [MagicMock(), _AnsibleCollectionFinder(), MagicMock(), _AnsibleCollectionFinder()] + fake_ph = [MagicMock().m1, MagicMock().m2, _AnsibleCollectionFinder()._ansible_collection_path_hook, NonCallableMagicMock] + # must nest until 2.6 compilation is totally donezo + with patch.object(sys, 'meta_path', fake_mp): + with patch.object(sys, 'path_hooks', fake_ph): + f = _AnsibleCollectionFinder() + f._install() + assert len(sys.meta_path) == 3 # should have removed the existing ACFs and installed a new one + assert sys.meta_path[0] is f # at the front + # the rest of the meta_path should not be AnsibleCollectionFinders + assert all((not isinstance(mpf, _AnsibleCollectionFinder) for mpf in sys.meta_path[1:])) + assert len(sys.path_hooks) == 4 # should have removed the existing ACF path hooks and installed a new one + # the first path hook should be ours, make sure it's pointing at the right instance + assert hasattr(sys.path_hooks[0], '__self__') and sys.path_hooks[0].__self__ is f + # the rest of the path_hooks should not point at an AnsibleCollectionFinder + assert all((not isinstance(ph.__self__, _AnsibleCollectionFinder) for ph in sys.path_hooks[1:] if hasattr(ph, '__self__'))) + assert AnsibleCollectionConfig.collection_finder is f + with pytest.raises(ValueError): + AnsibleCollectionConfig.collection_finder = f + + +def test_finder_coll(): + f = get_default_finder() + + tests = [ + {'name': 'ansible_collections.testns.testcoll', 'test_paths': [default_test_collection_paths]}, + {'name': 'ansible_collections.ansible.builtin', 'test_paths': [['/bogus'], default_test_collection_paths]}, + ] + # ensure finder works for legit paths and bogus paths + for test_dict in tests: + # splat the dict values to our locals + globals().update(test_dict) + parent_pkg = name.rpartition('.')[0] + for paths in test_paths: + paths = [os.path.join(p, parent_pkg.replace('.', '/')) for p in paths] + loader = f.find_module(name, path=paths) + assert isinstance(loader, _AnsibleCollectionPkgLoader) + + +def test_root_loader_not_interested(): + with pytest.raises(ImportError): + _AnsibleCollectionRootPkgLoader('not_ansible_collections_toplevel', path_list=[]) + + with pytest.raises(ImportError): + _AnsibleCollectionRootPkgLoader('ansible_collections.somens', path_list=['/bogus']) + + +def test_root_loader(): + name = 'ansible_collections' + # ensure this works even when ansible_collections doesn't exist on disk + for paths in [], default_test_collection_paths: + if name in sys.modules: + del sys.modules[name] + loader = _AnsibleCollectionRootPkgLoader(name, paths) + assert repr(loader).startswith('_AnsibleCollectionRootPkgLoader(path=') + module = loader.load_module(name) + assert module.__name__ == name + assert module.__path__ == [p for p in extend_paths(paths, name) if os.path.isdir(p)] + # even if the dir exists somewhere, this loader doesn't support get_data, so make __file__ a non-file + assert module.__file__ == '<ansible_synthetic_collection_package>' + assert module.__package__ == name + assert sys.modules.get(name) == module + + +def test_nspkg_loader_not_interested(): + with pytest.raises(ImportError): + _AnsibleCollectionNSPkgLoader('not_ansible_collections_toplevel.something', path_list=[]) + + with pytest.raises(ImportError): + _AnsibleCollectionNSPkgLoader('ansible_collections.somens.somecoll', path_list=[]) + + +def test_nspkg_loader_load_module(): + # ensure the loader behaves on the toplevel and ansible packages for both legit and missing/bogus paths + for name in ['ansible_collections.ansible', 'ansible_collections.testns']: + parent_pkg = name.partition('.')[0] + module_to_load = name.rpartition('.')[2] + paths = extend_paths(default_test_collection_paths, parent_pkg) + existing_child_paths = [p for p in extend_paths(paths, module_to_load) if os.path.exists(p)] + if name in sys.modules: + del sys.modules[name] + loader = _AnsibleCollectionNSPkgLoader(name, path_list=paths) + assert repr(loader).startswith('_AnsibleCollectionNSPkgLoader(path=') + module = loader.load_module(name) + assert module.__name__ == name + assert isinstance(module.__loader__, _AnsibleCollectionNSPkgLoader) + assert module.__path__ == existing_child_paths + assert module.__package__ == name + assert module.__file__ == '<ansible_synthetic_collection_package>' + assert sys.modules.get(name) == module + + +def test_collpkg_loader_not_interested(): + with pytest.raises(ImportError): + _AnsibleCollectionPkgLoader('not_ansible_collections', path_list=[]) + + with pytest.raises(ImportError): + _AnsibleCollectionPkgLoader('ansible_collections.ns', path_list=['/bogus/bogus']) + + +def test_collpkg_loader_load_module(): + reset_collections_loader_state() + with patch('ansible.utils.collection_loader.AnsibleCollectionConfig') as p: + for name in ['ansible_collections.ansible.builtin', 'ansible_collections.testns.testcoll']: + parent_pkg = name.rpartition('.')[0] + module_to_load = name.rpartition('.')[2] + paths = extend_paths(default_test_collection_paths, parent_pkg) + existing_child_paths = [p for p in extend_paths(paths, module_to_load) if os.path.exists(p)] + is_builtin = 'ansible.builtin' in name + if name in sys.modules: + del sys.modules[name] + loader = _AnsibleCollectionPkgLoader(name, path_list=paths) + assert repr(loader).startswith('_AnsibleCollectionPkgLoader(path=') + module = loader.load_module(name) + assert module.__name__ == name + assert isinstance(module.__loader__, _AnsibleCollectionPkgLoader) + if is_builtin: + assert module.__path__ == [] + else: + assert module.__path__ == [existing_child_paths[0]] + + assert module.__package__ == name + if is_builtin: + assert module.__file__ == '<ansible_synthetic_collection_package>' + else: + assert module.__file__.endswith('__synthetic__') and os.path.isdir(os.path.dirname(module.__file__)) + assert sys.modules.get(name) == module + + assert hasattr(module, '_collection_meta') and isinstance(module._collection_meta, dict) + + # FIXME: validate _collection_meta contents match what's on disk (or not) + + # if the module has metadata, try loading it with busted metadata + if module._collection_meta: + _collection_finder = import_module('ansible.utils.collection_loader._collection_finder') + with patch.object(_collection_finder, '_meta_yml_to_dict', side_effect=Exception('bang')): + with pytest.raises(Exception) as ex: + _AnsibleCollectionPkgLoader(name, path_list=paths).load_module(name) + assert 'error parsing collection metadata' in str(ex.value) + + +def test_coll_loader(): + with patch('ansible.utils.collection_loader.AnsibleCollectionConfig'): + with pytest.raises(ValueError): + # not a collection + _AnsibleCollectionLoader('ansible_collections') + + with pytest.raises(ValueError): + # bogus paths + _AnsibleCollectionLoader('ansible_collections.testns.testcoll', path_list=[]) + + # FIXME: more + + +def test_path_hook_setup(): + with patch.object(sys, 'path_hooks', []): + found_hook = None + pathhook_exc = None + try: + found_hook = _AnsiblePathHookFinder._get_filefinder_path_hook() + except Exception as phe: + pathhook_exc = phe + + if PY3: + assert str(pathhook_exc) == 'need exactly one FileFinder import hook (found 0)' + else: + assert found_hook is None + + assert repr(_AnsiblePathHookFinder(object(), '/bogus/path')) == "_AnsiblePathHookFinder(path='/bogus/path')" + + +def test_new_or_existing_module(): + module_name = 'blar.test.module' + pkg_name = module_name.rpartition('.')[0] + + # create new module case + nuke_module_prefix(module_name) + with _AnsibleCollectionPkgLoaderBase._new_or_existing_module(module_name, __package__=pkg_name) as new_module: + # the module we just created should now exist in sys.modules + assert sys.modules.get(module_name) is new_module + assert new_module.__name__ == module_name + + # the module should stick since we didn't raise an exception in the contextmgr + assert sys.modules.get(module_name) is new_module + + # reuse existing module case + with _AnsibleCollectionPkgLoaderBase._new_or_existing_module(module_name, __attr1__=42, blar='yo') as existing_module: + assert sys.modules.get(module_name) is new_module # should be the same module we created earlier + assert hasattr(existing_module, '__package__') and existing_module.__package__ == pkg_name + assert hasattr(existing_module, '__attr1__') and existing_module.__attr1__ == 42 + assert hasattr(existing_module, 'blar') and existing_module.blar == 'yo' + + # exception during update existing shouldn't zap existing module from sys.modules + with pytest.raises(ValueError) as ve: + with _AnsibleCollectionPkgLoaderBase._new_or_existing_module(module_name) as existing_module: + err_to_raise = ValueError('bang') + raise err_to_raise + # make sure we got our error + assert ve.value is err_to_raise + # and that the module still exists + assert sys.modules.get(module_name) is existing_module + + # test module removal after exception during creation + nuke_module_prefix(module_name) + with pytest.raises(ValueError) as ve: + with _AnsibleCollectionPkgLoaderBase._new_or_existing_module(module_name) as new_module: + err_to_raise = ValueError('bang') + raise err_to_raise + # make sure we got our error + assert ve.value is err_to_raise + # and that the module was removed + assert sys.modules.get(module_name) is None + + +def test_iter_modules_impl(): + modules_trailer = 'ansible_collections/testns/testcoll/plugins' + modules_pkg_prefix = modules_trailer.replace('/', '.') + '.' + modules_path = os.path.join(default_test_collection_paths[0], modules_trailer) + modules = list(_iter_modules_impl([modules_path], modules_pkg_prefix)) + + assert modules + assert set([('ansible_collections.testns.testcoll.plugins.action', True), + ('ansible_collections.testns.testcoll.plugins.module_utils', True), + ('ansible_collections.testns.testcoll.plugins.modules', True)]) == set(modules) + + modules_trailer = 'ansible_collections/testns/testcoll/plugins/modules' + modules_pkg_prefix = modules_trailer.replace('/', '.') + '.' + modules_path = os.path.join(default_test_collection_paths[0], modules_trailer) + modules = list(_iter_modules_impl([modules_path], modules_pkg_prefix)) + + assert modules + assert len(modules) == 1 + assert modules[0][0] == 'ansible_collections.testns.testcoll.plugins.modules.amodule' # name + assert modules[0][1] is False # is_pkg + + # FIXME: more + + +# BEGIN IN-CIRCUIT TESTS - these exercise behaviors of the loader when wired up to the import machinery + + +def test_import_from_collection(monkeypatch): + collection_root = os.path.join(os.path.dirname(__file__), 'fixtures', 'collections') + collection_path = os.path.join(collection_root, 'ansible_collections/testns/testcoll/plugins/module_utils/my_util.py') + + # THIS IS UNSTABLE UNDER A DEBUGGER + # the trace we're expecting to be generated when running the code below: + # answer = question() + expected_trace_log = [ + (collection_path, 5, 'call'), + (collection_path, 6, 'line'), + (collection_path, 6, 'return'), + ] + + # define the collection root before any ansible code has been loaded + # otherwise config will have already been loaded and changing the environment will have no effect + monkeypatch.setenv('ANSIBLE_COLLECTIONS_PATHS', collection_root) + + finder = _AnsibleCollectionFinder(paths=[collection_root]) + reset_collections_loader_state(finder) + + from ansible_collections.testns.testcoll.plugins.module_utils.my_util import question + + original_trace_function = sys.gettrace() + trace_log = [] + + if original_trace_function: + # enable tracing while preserving the existing trace function (coverage) + def my_trace_function(frame, event, arg): + trace_log.append((frame.f_code.co_filename, frame.f_lineno, event)) + + # the original trace function expects to have itself set as the trace function + sys.settrace(original_trace_function) + # call the original trace function + original_trace_function(frame, event, arg) + # restore our trace function + sys.settrace(my_trace_function) + + return my_trace_function + else: + # no existing trace function, so our trace function is much simpler + def my_trace_function(frame, event, arg): + trace_log.append((frame.f_code.co_filename, frame.f_lineno, event)) + + return my_trace_function + + sys.settrace(my_trace_function) + + try: + # run a minimal amount of code while the trace is running + # adding more code here, including use of a context manager, will add more to our trace + answer = question() + finally: + sys.settrace(original_trace_function) + + # make sure 'import ... as ...' works on builtin synthetic collections + # the following import is not supported (it tries to find module_utils in ansible.plugins) + # import ansible_collections.ansible.builtin.plugins.module_utils as c1 + import ansible_collections.ansible.builtin.plugins.action as c2 + import ansible_collections.ansible.builtin.plugins as c3 + import ansible_collections.ansible.builtin as c4 + import ansible_collections.ansible as c5 + import ansible_collections as c6 + + # make sure 'import ...' works on builtin synthetic collections + import ansible_collections.ansible.builtin.plugins.module_utils + + import ansible_collections.ansible.builtin.plugins.action + assert ansible_collections.ansible.builtin.plugins.action == c3.action == c2 + + import ansible_collections.ansible.builtin.plugins + assert ansible_collections.ansible.builtin.plugins == c4.plugins == c3 + + import ansible_collections.ansible.builtin + assert ansible_collections.ansible.builtin == c5.builtin == c4 + + import ansible_collections.ansible + assert ansible_collections.ansible == c6.ansible == c5 + + import ansible_collections + assert ansible_collections == c6 + + # make sure 'from ... import ...' works on builtin synthetic collections + from ansible_collections.ansible import builtin + from ansible_collections.ansible.builtin import plugins + assert builtin.plugins == plugins + + from ansible_collections.ansible.builtin.plugins import action + from ansible_collections.ansible.builtin.plugins.action import command + assert action.command == command + + from ansible_collections.ansible.builtin.plugins.module_utils import basic + from ansible_collections.ansible.builtin.plugins.module_utils.basic import AnsibleModule + assert basic.AnsibleModule == AnsibleModule + + # make sure relative imports work from collections code + # these require __package__ to be set correctly + import ansible_collections.testns.testcoll.plugins.module_utils.my_other_util + import ansible_collections.testns.testcoll.plugins.action.my_action + + # verify that code loaded from a collection does not inherit __future__ statements from the collection loader + if sys.version_info[0] == 2: + # if the collection code inherits the division future feature from the collection loader this will fail + assert answer == 1 + else: + assert answer == 1.5 + + # verify that the filename and line number reported by the trace is correct + # this makes sure that collection loading preserves file paths and line numbers + assert trace_log == expected_trace_log + + +def test_eventsource(): + es = _EventSource() + # fire when empty should succeed + es.fire(42) + handler1 = MagicMock() + handler2 = MagicMock() + es += handler1 + es.fire(99, my_kwarg='blah') + handler1.assert_called_with(99, my_kwarg='blah') + es += handler2 + es.fire(123, foo='bar') + handler1.assert_called_with(123, foo='bar') + handler2.assert_called_with(123, foo='bar') + es -= handler2 + handler1.reset_mock() + handler2.reset_mock() + es.fire(123, foo='bar') + handler1.assert_called_with(123, foo='bar') + handler2.assert_not_called() + es -= handler1 + handler1.reset_mock() + es.fire('blah', kwarg=None) + handler1.assert_not_called() + handler2.assert_not_called() + es -= handler1 # should succeed silently + handler_bang = MagicMock(side_effect=Exception('bang')) + es += handler_bang + with pytest.raises(Exception) as ex: + es.fire(123) + assert 'bang' in str(ex.value) + handler_bang.assert_called_with(123) + with pytest.raises(ValueError): + es += 42 + + +def test_on_collection_load(): + finder = get_default_finder() + reset_collections_loader_state(finder) + + load_handler = MagicMock() + AnsibleCollectionConfig.on_collection_load += load_handler + + m = import_module('ansible_collections.testns.testcoll') + load_handler.assert_called_once_with(collection_name='testns.testcoll', collection_path=os.path.dirname(m.__file__)) + + _meta = _get_collection_metadata('testns.testcoll') + assert _meta + # FIXME: compare to disk + + finder = get_default_finder() + reset_collections_loader_state(finder) + + AnsibleCollectionConfig.on_collection_load += MagicMock(side_effect=Exception('bang')) + with pytest.raises(Exception) as ex: + import_module('ansible_collections.testns.testcoll') + assert 'bang' in str(ex.value) + + +def test_default_collection_config(): + finder = get_default_finder() + reset_collections_loader_state(finder) + assert AnsibleCollectionConfig.default_collection is None + AnsibleCollectionConfig.default_collection = 'foo.bar' + assert AnsibleCollectionConfig.default_collection == 'foo.bar' + with pytest.raises(ValueError): + AnsibleCollectionConfig.default_collection = 'bar.baz' + + +def test_default_collection_detection(): + finder = get_default_finder() + reset_collections_loader_state(finder) + + # we're clearly not under a collection path + assert _get_collection_name_from_path('/') is None + + # something that looks like a collection path but isn't importable by our finder + assert _get_collection_name_from_path('/foo/ansible_collections/bogusns/boguscoll/bar') is None + + # legit, at the top of the collection + live_collection_path = os.path.join(os.path.dirname(__file__), 'fixtures/collections/ansible_collections/testns/testcoll') + assert _get_collection_name_from_path(live_collection_path) == 'testns.testcoll' + + # legit, deeper inside the collection + live_collection_deep_path = os.path.join(live_collection_path, 'plugins/modules') + assert _get_collection_name_from_path(live_collection_deep_path) == 'testns.testcoll' + + # this one should be hidden by the real testns.testcoll, so should not resolve + masked_collection_path = os.path.join(os.path.dirname(__file__), 'fixtures/collections_masked/ansible_collections/testns/testcoll') + assert _get_collection_name_from_path(masked_collection_path) is None + + +@pytest.mark.parametrize( + 'role_name,collection_list,expected_collection_name,expected_path_suffix', + [ + ('some_role', ['testns.testcoll', 'ansible.bogus'], 'testns.testcoll', 'testns/testcoll/roles/some_role'), + ('testns.testcoll.some_role', ['ansible.bogus', 'testns.testcoll'], 'testns.testcoll', 'testns/testcoll/roles/some_role'), + ('testns.testcoll.some_role', [], 'testns.testcoll', 'testns/testcoll/roles/some_role'), + ('testns.testcoll.some_role', None, 'testns.testcoll', 'testns/testcoll/roles/some_role'), + ('some_role', [], None, None), + ('some_role', None, None, None), + ]) +def test_collection_role_name_location(role_name, collection_list, expected_collection_name, expected_path_suffix): + finder = get_default_finder() + reset_collections_loader_state(finder) + + expected_path = None + if expected_path_suffix: + expected_path = os.path.join(os.path.dirname(__file__), 'fixtures/collections/ansible_collections', expected_path_suffix) + + found = _get_collection_role_path(role_name, collection_list) + + if found: + assert found[0] == role_name.rpartition('.')[2] + assert found[1] == expected_path + assert found[2] == expected_collection_name + else: + assert expected_collection_name is None and expected_path_suffix is None + + +def test_bogus_imports(): + finder = get_default_finder() + reset_collections_loader_state(finder) + + # ensure ImportError on known-bogus imports + bogus_imports = ['bogus_toplevel', 'ansible_collections.bogusns', 'ansible_collections.testns.boguscoll', + 'ansible_collections.testns.testcoll.bogussub', 'ansible_collections.ansible.builtin.bogussub'] + for bogus_import in bogus_imports: + with pytest.raises(ImportError): + import_module(bogus_import) + + +def test_finder_playbook_paths(): + finder = get_default_finder() + reset_collections_loader_state(finder) + + import ansible_collections + import ansible_collections.ansible + import ansible_collections.testns + + # ensure the package modules look like we expect + assert hasattr(ansible_collections, '__path__') and len(ansible_collections.__path__) > 0 + assert hasattr(ansible_collections.ansible, '__path__') and len(ansible_collections.ansible.__path__) > 0 + assert hasattr(ansible_collections.testns, '__path__') and len(ansible_collections.testns.__path__) > 0 + + # these shouldn't be visible yet, since we haven't added the playbook dir + with pytest.raises(ImportError): + import ansible_collections.ansible.playbook_adj_other + + with pytest.raises(ImportError): + import ansible_collections.testns.playbook_adj_other + + assert AnsibleCollectionConfig.playbook_paths == [] + playbook_path_fixture_dir = os.path.join(os.path.dirname(__file__), 'fixtures/playbook_path') + + # configure the playbook paths + AnsibleCollectionConfig.playbook_paths = [playbook_path_fixture_dir] + + # playbook paths go to the front of the line + assert AnsibleCollectionConfig.collection_paths[0] == os.path.join(playbook_path_fixture_dir, 'collections') + + # playbook paths should be updated on the existing root ansible_collections path, as well as on the 'ansible' namespace (but no others!) + assert ansible_collections.__path__[0] == os.path.join(playbook_path_fixture_dir, 'collections/ansible_collections') + assert ansible_collections.ansible.__path__[0] == os.path.join(playbook_path_fixture_dir, 'collections/ansible_collections/ansible') + assert all('playbook_path' not in p for p in ansible_collections.testns.__path__) + + # should succeed since we fixed up the package path + import ansible_collections.ansible.playbook_adj_other + # should succeed since we didn't import freshns before hacking in the path + import ansible_collections.freshns.playbook_adj_other + # should fail since we've already imported something from this path and didn't fix up its package path + with pytest.raises(ImportError): + import ansible_collections.testns.playbook_adj_other + + +def test_toplevel_iter_modules(): + finder = get_default_finder() + reset_collections_loader_state(finder) + + modules = list(pkgutil.iter_modules(default_test_collection_paths, '')) + assert len(modules) == 1 + assert modules[0][1] == 'ansible_collections' + + +def test_iter_modules_namespaces(): + finder = get_default_finder() + reset_collections_loader_state(finder) + + paths = extend_paths(default_test_collection_paths, 'ansible_collections') + modules = list(pkgutil.iter_modules(paths, 'ansible_collections.')) + assert len(modules) == 2 + assert all(m[2] is True for m in modules) + assert all(isinstance(m[0], _AnsiblePathHookFinder) for m in modules) + assert set(['ansible_collections.testns', 'ansible_collections.ansible']) == set(m[1] for m in modules) + + +def test_collection_get_data(): + finder = get_default_finder() + reset_collections_loader_state(finder) + + # something that's there + d = pkgutil.get_data('ansible_collections.testns.testcoll', 'plugins/action/my_action.py') + assert b'hello from my_action.py' in d + + # something that's not there + d = pkgutil.get_data('ansible_collections.testns.testcoll', 'bogus/bogus') + assert d is None + + with pytest.raises(ValueError): + plugins_pkg = import_module('ansible_collections.ansible.builtin') + assert not os.path.exists(os.path.dirname(plugins_pkg.__file__)) + d = pkgutil.get_data('ansible_collections.ansible.builtin', 'plugins/connection/local.py') + + +@pytest.mark.parametrize( + 'ref,ref_type,expected_collection,expected_subdirs,expected_resource,expected_python_pkg_name', + [ + ('ns.coll.myaction', 'action', 'ns.coll', '', 'myaction', 'ansible_collections.ns.coll.plugins.action'), + ('ns.coll.subdir1.subdir2.myaction', 'action', 'ns.coll', 'subdir1.subdir2', 'myaction', 'ansible_collections.ns.coll.plugins.action.subdir1.subdir2'), + ('ns.coll.myrole', 'role', 'ns.coll', '', 'myrole', 'ansible_collections.ns.coll.roles.myrole'), + ('ns.coll.subdir1.subdir2.myrole', 'role', 'ns.coll', 'subdir1.subdir2', 'myrole', 'ansible_collections.ns.coll.roles.subdir1.subdir2.myrole'), + ]) +def test_fqcr_parsing_valid(ref, ref_type, expected_collection, + expected_subdirs, expected_resource, expected_python_pkg_name): + assert AnsibleCollectionRef.is_valid_fqcr(ref, ref_type) + + r = AnsibleCollectionRef.from_fqcr(ref, ref_type) + assert r.collection == expected_collection + assert r.subdirs == expected_subdirs + assert r.resource == expected_resource + assert r.n_python_package_name == expected_python_pkg_name + + r = AnsibleCollectionRef.try_parse_fqcr(ref, ref_type) + assert r.collection == expected_collection + assert r.subdirs == expected_subdirs + assert r.resource == expected_resource + assert r.n_python_package_name == expected_python_pkg_name + + +@pytest.mark.parametrize( + 'ref,ref_type,expected_error_type,expected_error_expression', + [ + ('no_dots_at_all_action', 'action', ValueError, 'is not a valid collection reference'), + ('no_nscoll.myaction', 'action', ValueError, 'is not a valid collection reference'), + ('ns.coll.myaction', 'bogus', ValueError, 'invalid collection ref_type'), + ]) +def test_fqcr_parsing_invalid(ref, ref_type, expected_error_type, expected_error_expression): + assert not AnsibleCollectionRef.is_valid_fqcr(ref, ref_type) + + with pytest.raises(expected_error_type) as curerr: + AnsibleCollectionRef.from_fqcr(ref, ref_type) + + assert re.search(expected_error_expression, str(curerr.value)) + + r = AnsibleCollectionRef.try_parse_fqcr(ref, ref_type) + assert r is None + + +@pytest.mark.parametrize( + 'name,subdirs,resource,ref_type,python_pkg_name', + [ + ('ns.coll', None, 'res', 'doc_fragments', 'ansible_collections.ns.coll.plugins.doc_fragments'), + ('ns.coll', 'subdir1', 'res', 'doc_fragments', 'ansible_collections.ns.coll.plugins.doc_fragments.subdir1'), + ('ns.coll', 'subdir1.subdir2', 'res', 'action', 'ansible_collections.ns.coll.plugins.action.subdir1.subdir2'), + ]) +def test_collectionref_components_valid(name, subdirs, resource, ref_type, python_pkg_name): + x = AnsibleCollectionRef(name, subdirs, resource, ref_type) + + assert x.collection == name + if subdirs: + assert x.subdirs == subdirs + else: + assert x.subdirs == '' + + assert x.resource == resource + assert x.ref_type == ref_type + assert x.n_python_package_name == python_pkg_name + + +@pytest.mark.parametrize( + 'dirname,expected_result', + [ + ('become_plugins', 'become'), + ('cache_plugins', 'cache'), + ('connection_plugins', 'connection'), + ('library', 'modules'), + ('filter_plugins', 'filter'), + ('bogus_plugins', ValueError), + (None, ValueError) + ] +) +def test_legacy_plugin_dir_to_plugin_type(dirname, expected_result): + if isinstance(expected_result, string_types): + assert AnsibleCollectionRef.legacy_plugin_dir_to_plugin_type(dirname) == expected_result + else: + with pytest.raises(expected_result): + AnsibleCollectionRef.legacy_plugin_dir_to_plugin_type(dirname) + + +@pytest.mark.parametrize( + 'name,subdirs,resource,ref_type,expected_error_type,expected_error_expression', + [ + ('bad_ns', '', 'resource', 'action', ValueError, 'invalid collection name'), + ('ns.coll.', '', 'resource', 'action', ValueError, 'invalid collection name'), + ('ns.coll', 'badsubdir#', 'resource', 'action', ValueError, 'invalid subdirs entry'), + ('ns.coll', 'badsubdir.', 'resource', 'action', ValueError, 'invalid subdirs entry'), + ('ns.coll', '.badsubdir', 'resource', 'action', ValueError, 'invalid subdirs entry'), + ('ns.coll', '', 'resource', 'bogus', ValueError, 'invalid collection ref_type'), + ]) +def test_collectionref_components_invalid(name, subdirs, resource, ref_type, expected_error_type, expected_error_expression): + with pytest.raises(expected_error_type) as curerr: + AnsibleCollectionRef(name, subdirs, resource, ref_type) + + assert re.search(expected_error_expression, str(curerr.value)) + + +# BEGIN TEST SUPPORT + +default_test_collection_paths = [ + os.path.join(os.path.dirname(__file__), 'fixtures', 'collections'), + os.path.join(os.path.dirname(__file__), 'fixtures', 'collections_masked'), + '/bogus/bogussub' +] + + +def get_default_finder(): + return _AnsibleCollectionFinder(paths=default_test_collection_paths) + + +def extend_paths(path_list, suffix): + suffix = suffix.replace('.', '/') + return [os.path.join(p, suffix) for p in path_list] + + +def nuke_module_prefix(prefix): + for module_to_nuke in [m for m in sys.modules if m.startswith(prefix)]: + sys.modules.pop(module_to_nuke) + + +def reset_collections_loader_state(metapath_finder=None): + _AnsibleCollectionFinder._remove() + + nuke_module_prefix('ansible_collections') + nuke_module_prefix('ansible.modules') + nuke_module_prefix('ansible.plugins') + + # FIXME: better to move this someplace else that gets cleaned up automatically? + _AnsibleCollectionLoader._redirected_package_map = {} + + AnsibleCollectionConfig._default_collection = None + AnsibleCollectionConfig._on_collection_load = _EventSource() + + if metapath_finder: + metapath_finder._install() diff --git a/test/units/utils/fixtures/collections/ansible_collections/my_namespace/my_collection/plugins/action/my_action.py b/test/units/utils/fixtures/collections/ansible_collections/my_namespace/my_collection/plugins/action/my_action.py deleted file mode 100644 index 085f54f111..0000000000 --- a/test/units/utils/fixtures/collections/ansible_collections/my_namespace/my_collection/plugins/action/my_action.py +++ /dev/null @@ -1 +0,0 @@ -from ..module_utils.my_util import question diff --git a/test/units/utils/test_collection_loader.py b/test/units/utils/test_collection_loader.py deleted file mode 100644 index 7800f96007..0000000000 --- a/test/units/utils/test_collection_loader.py +++ /dev/null @@ -1,215 +0,0 @@ -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type - -import os -import pytest -import re -import sys - -from ansible.utils.collection_loader import AnsibleCollectionRef - - -def test_import_from_collection(monkeypatch): - collection_root = os.path.join(os.path.dirname(__file__), 'fixtures', 'collections') - collection_path = os.path.join(collection_root, 'ansible_collections/my_namespace/my_collection/plugins/module_utils/my_util.py') - - # the trace we're expecting to be generated when running the code below: - # answer = question() - expected_trace_log = [ - (collection_path, 5, 'call'), - (collection_path, 6, 'line'), - (collection_path, 6, 'return'), - ] - - # define the collection root before any ansible code has been loaded - # otherwise config will have already been loaded and changing the environment will have no effect - monkeypatch.setenv('ANSIBLE_COLLECTIONS_PATHS', collection_root) - - from ansible.utils.collection_loader import AnsibleCollectionLoader - - # zap the singleton collection loader instance if it exists - AnsibleCollectionLoader._Singleton__instance = None - - for index in [idx for idx, obj in enumerate(sys.meta_path) if isinstance(obj, AnsibleCollectionLoader)]: - # replace any existing collection loaders that may exist - # since these were loaded during unit test collection - # they will not have the correct configuration - sys.meta_path[index] = AnsibleCollectionLoader() - - # make sure the collection loader is installed - # this will be a no-op if the collection loader is already installed - # which will depend on whether or not any tests being run imported ansible.plugins.loader during unit test collection - from ansible.plugins.loader import _configure_collection_loader - _configure_collection_loader() # currently redundant, the import above already calls this - - from ansible_collections.my_namespace.my_collection.plugins.module_utils.my_util import question - - original_trace_function = sys.gettrace() - trace_log = [] - - if original_trace_function: - # enable tracing while preserving the existing trace function (coverage) - def my_trace_function(frame, event, arg): - trace_log.append((frame.f_code.co_filename, frame.f_lineno, event)) - - # the original trace function expects to have itself set as the trace function - sys.settrace(original_trace_function) - # call the original trace function - original_trace_function(frame, event, arg) - # restore our trace function - sys.settrace(my_trace_function) - - return my_trace_function - else: - # no existing trace function, so our trace function is much simpler - def my_trace_function(frame, event, arg): - trace_log.append((frame.f_code.co_filename, frame.f_lineno, event)) - - return my_trace_function - - sys.settrace(my_trace_function) - - try: - # run a minimal amount of code while the trace is running - # adding more code here, including use of a context manager, will add more to our trace - answer = question() - finally: - sys.settrace(original_trace_function) - - # make sure 'import ... as ...' works on builtin synthetic collections - # the following import is not supported (it tries to find module_utils in ansible.plugins) - # import ansible_collections.ansible.builtin.plugins.module_utils as c1 - import ansible_collections.ansible.builtin.plugins.action as c2 - import ansible_collections.ansible.builtin.plugins as c3 - import ansible_collections.ansible.builtin as c4 - import ansible_collections.ansible as c5 - import ansible_collections as c6 - - # make sure 'import ...' works on builtin synthetic collections - import ansible_collections.ansible.builtin.plugins.module_utils - - import ansible_collections.ansible.builtin.plugins.action - assert ansible_collections.ansible.builtin.plugins.action == c3.action == c2 - - import ansible_collections.ansible.builtin.plugins - assert ansible_collections.ansible.builtin.plugins == c4.plugins == c3 - - import ansible_collections.ansible.builtin - assert ansible_collections.ansible.builtin == c5.builtin == c4 - - import ansible_collections.ansible - assert ansible_collections.ansible == c6.ansible == c5 - - import ansible_collections - assert ansible_collections == c6 - - # make sure 'from ... import ...' works on builtin synthetic collections - from ansible_collections.ansible import builtin - from ansible_collections.ansible.builtin import plugins - assert builtin.plugins == plugins - - from ansible_collections.ansible.builtin.plugins import action - from ansible_collections.ansible.builtin.plugins.action import command - assert action.command == command - - from ansible_collections.ansible.builtin.plugins.module_utils import basic - from ansible_collections.ansible.builtin.plugins.module_utils.basic import AnsibleModule - assert basic.AnsibleModule == AnsibleModule - - # make sure relative imports work from collections code - # these require __package__ to be set correctly - import ansible_collections.my_namespace.my_collection.plugins.module_utils.my_other_util - import ansible_collections.my_namespace.my_collection.plugins.action.my_action - - # verify that code loaded from a collection does not inherit __future__ statements from the collection loader - if sys.version_info[0] == 2: - # if the collection code inherits the division future feature from the collection loader this will fail - assert answer == 1 - else: - assert answer == 1.5 - - # verify that the filename and line number reported by the trace is correct - # this makes sure that collection loading preserves file paths and line numbers - assert trace_log == expected_trace_log - - -@pytest.mark.parametrize( - 'ref,ref_type,expected_collection,expected_subdirs,expected_resource,expected_python_pkg_name', - [ - ('ns.coll.myaction', 'action', 'ns.coll', '', 'myaction', 'ansible_collections.ns.coll.plugins.action'), - ('ns.coll.subdir1.subdir2.myaction', 'action', 'ns.coll', 'subdir1.subdir2', 'myaction', 'ansible_collections.ns.coll.plugins.action.subdir1.subdir2'), - ('ns.coll.myrole', 'role', 'ns.coll', '', 'myrole', 'ansible_collections.ns.coll.roles.myrole'), - ('ns.coll.subdir1.subdir2.myrole', 'role', 'ns.coll', 'subdir1.subdir2', 'myrole', 'ansible_collections.ns.coll.roles.subdir1.subdir2.myrole'), - ]) -def test_fqcr_parsing_valid(ref, ref_type, expected_collection, - expected_subdirs, expected_resource, expected_python_pkg_name): - assert AnsibleCollectionRef.is_valid_fqcr(ref, ref_type) - - r = AnsibleCollectionRef.from_fqcr(ref, ref_type) - assert r.collection == expected_collection - assert r.subdirs == expected_subdirs - assert r.resource == expected_resource - assert r.n_python_package_name == expected_python_pkg_name - - r = AnsibleCollectionRef.try_parse_fqcr(ref, ref_type) - assert r.collection == expected_collection - assert r.subdirs == expected_subdirs - assert r.resource == expected_resource - assert r.n_python_package_name == expected_python_pkg_name - - -@pytest.mark.parametrize( - 'ref,ref_type,expected_error_type,expected_error_expression', - [ - ('no_dots_at_all_action', 'action', ValueError, 'is not a valid collection reference'), - ('no_nscoll.myaction', 'action', ValueError, 'is not a valid collection reference'), - ('ns.coll.myaction', 'bogus', ValueError, 'invalid collection ref_type'), - ]) -def test_fqcr_parsing_invalid(ref, ref_type, expected_error_type, expected_error_expression): - assert not AnsibleCollectionRef.is_valid_fqcr(ref, ref_type) - - with pytest.raises(expected_error_type) as curerr: - AnsibleCollectionRef.from_fqcr(ref, ref_type) - - assert re.search(expected_error_expression, str(curerr.value)) - - r = AnsibleCollectionRef.try_parse_fqcr(ref, ref_type) - assert r is None - - -@pytest.mark.parametrize( - 'name,subdirs,resource,ref_type,python_pkg_name', - [ - ('ns.coll', None, 'res', 'doc_fragments', 'ansible_collections.ns.coll.plugins.doc_fragments'), - ('ns.coll', 'subdir1', 'res', 'doc_fragments', 'ansible_collections.ns.coll.plugins.doc_fragments.subdir1'), - ('ns.coll', 'subdir1.subdir2', 'res', 'action', 'ansible_collections.ns.coll.plugins.action.subdir1.subdir2'), - ]) -def test_collectionref_components_valid(name, subdirs, resource, ref_type, python_pkg_name): - x = AnsibleCollectionRef(name, subdirs, resource, ref_type) - - assert x.collection == name - if subdirs: - assert x.subdirs == subdirs - else: - assert x.subdirs == '' - - assert x.resource == resource - assert x.ref_type == ref_type - assert x.n_python_package_name == python_pkg_name - - -@pytest.mark.parametrize( - 'name,subdirs,resource,ref_type,expected_error_type,expected_error_expression', - [ - ('bad_ns', '', 'resource', 'action', ValueError, 'invalid collection name'), - ('ns.coll.', '', 'resource', 'action', ValueError, 'invalid collection name'), - ('ns.coll', 'badsubdir#', 'resource', 'action', ValueError, 'invalid subdirs entry'), - ('ns.coll', 'badsubdir.', 'resource', 'action', ValueError, 'invalid subdirs entry'), - ('ns.coll', '.badsubdir', 'resource', 'action', ValueError, 'invalid subdirs entry'), - ('ns.coll', '', 'resource', 'bogus', ValueError, 'invalid collection ref_type'), - ]) -def test_collectionref_components_invalid(name, subdirs, resource, ref_type, expected_error_type, expected_error_expression): - with pytest.raises(expected_error_type) as curerr: - AnsibleCollectionRef(name, subdirs, resource, ref_type) - - assert re.search(expected_error_expression, str(curerr.value)) |