Update collection loader for Python 3.10 (#76225)

* Implement find_spec and exec_module to remove reliance on deprecated methods in the collection loader

ci_complete

* Move module execution to exec_module

Remove extra sys.modules handling

Use default module initialization by returning None from loader.create_module

Refactor

ci_complete

* Remove ansible-test's copy of the collection loader

ci_complete

* Fix metaclass for Python 2.x

ci_complete

* Fix Py2/Py3 syntax compatibility

* Refactor

ci_complete

* update collection_loader comments

ci_complete

* simplify find_module

ci_complete

* Fix Py2 compatibility - don't get loader from nonexistent spec

Remove unnecessary PY3 checking

* Refactor common code in load_module and exec_module

ci_complete

* tidy diff

ci_complete

* Include collection_loader in target paths for 'compile' sanity test

* add changelog

* Add "return None" instead of doing it implicitly

Remove get_filename

short-circuit exec_module if it's a redirect

ci_complete

4 files changed, 111 insertions, 55 deletions

diff --git a/lib/ansible/utils/collection_loader/__init__.py b/lib/ansible/utils/collection_loader/__init__.py
index 21c49c4730..83cc2462eb 100644
--- a/lib/ansible/utils/collection_loader/__init__.py
+++ b/lib/ansible/utils/collection_loader/__init__.py
@@ -1,13 +1,8 @@
 # (c) 2019 Ansible Project
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
 
-# CAUTION: There are two implementations of the collection loader.
-#          They must be kept functionally identical, although their implementations may differ.
-#
-# 1) The controller implementation resides in the "lib/ansible/utils/collection_loader/" directory.
-#    It must function on all Python versions supported on the controller.
-# 2) The ansible-test implementation resides in the "test/lib/ansible_test/_util/target/legacy_collection_loader/" directory.
-#    It must function on all Python versions supported on managed hosts which are not supported by the controller.
+# CAUTION: This implementation of the collection loader is used by ansible-test.
+#          Because of this, it must be compatible with all Python versions supported on the controller or remote.
 
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
diff --git a/lib/ansible/utils/collection_loader/_collection_config.py b/lib/ansible/utils/collection_loader/_collection_config.py
index 49de68b135..4f73a1a731 100644
--- a/lib/ansible/utils/collection_loader/_collection_config.py
+++ b/lib/ansible/utils/collection_loader/_collection_config.py
@@ -1,18 +1,14 @@
 # (c) 2019 Ansible Project
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
 
-# CAUTION: There are two implementations of the collection loader.
-#          They must be kept functionally identical, although their implementations may differ.
-#
-# 1) The controller implementation resides in the "lib/ansible/utils/collection_loader/" directory.
-#    It must function on all Python versions supported on the controller.
-# 2) The ansible-test implementation resides in the "test/lib/ansible_test/_util/target/legacy_collection_loader/" directory.
-#    It must function on all Python versions supported on managed hosts which are not supported by the controller.
+# CAUTION: This implementation of the collection loader is used by ansible-test.
+#          Because of this, it must be compatible with all Python versions supported on the controller or remote.
 
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 from ansible.module_utils.common.text.converters import to_text
+from ansible.module_utils.six import add_metaclass
 
 
 class _EventSource:
@@ -102,5 +98,6 @@ class _AnsibleCollectionConfig(type):
 
 
 # concrete class of our metaclass type that defines the class properties we want
-class AnsibleCollectionConfig(metaclass=_AnsibleCollectionConfig):
+@add_metaclass(_AnsibleCollectionConfig)
+class AnsibleCollectionConfig(object):
     pass
diff --git a/lib/ansible/utils/collection_loader/_collection_finder.py b/lib/ansible/utils/collection_loader/_collection_finder.py
index 13fab8922d..70e2d1038e 100644
--- a/lib/ansible/utils/collection_loader/_collection_finder.py
+++ b/lib/ansible/utils/collection_loader/_collection_finder.py
@@ -1,13 +1,8 @@
 # (c) 2019 Ansible Project
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
 
-# CAUTION: There are two implementations of the collection loader.
-#          They must be kept functionally identical, although their implementations may differ.
-#
-# 1) The controller implementation resides in the "lib/ansible/utils/collection_loader/" directory.
-#    It must function on all Python versions supported on the controller.
-# 2) The ansible-test implementation resides in the "test/lib/ansible_test/_util/target/legacy_collection_loader/" directory.
-#    It must function on all Python versions supported on managed hosts which are not supported by the controller.
+# CAUTION: This implementation of the collection loader is used by ansible-test.
+#          Because of this, it must be compatible with all Python versions supported on the controller or remote.
 
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
@@ -43,6 +38,11 @@ except ImportError:
     # 2.7 has a global reload function instead...
     reload_module = reload  # pylint:disable=undefined-variable
 
+try:
+    from importlib.util import spec_from_loader
+except ImportError:
+    pass
+
 # NB: this supports import sanity test providing a different impl
 try:
     from ._collection_meta import _meta_yml_to_dict
@@ -184,9 +184,7 @@ class _AnsibleCollectionFinder:
             return
         reload_module(m)
 
-    def find_module(self, fullname, path=None):
-        # Figure out what's being asked for, and delegate to a special-purpose loader
-
+    def _get_loader(self, fullname, path=None):
         split_name = fullname.split('.')
         toplevel_pkg = split_name[0]
         module_to_find = split_name[-1]
@@ -207,23 +205,40 @@ class _AnsibleCollectionFinder:
         if part_count > 1 and path is None:
             raise ValueError('path must be specified for subpackages (trying to find {0})'.format(fullname))
 
+        if toplevel_pkg == 'ansible':
+            # something under the ansible package, delegate to our internal loader in case of redirections
+            initialize_loader = _AnsibleInternalRedirectLoader
+        elif part_count == 1:
+            initialize_loader = _AnsibleCollectionRootPkgLoader
+        elif part_count == 2:  # ns pkg eg, ansible_collections, ansible_collections.somens
+            initialize_loader = _AnsibleCollectionNSPkgLoader
+        elif part_count == 3:  # collection pkg eg, ansible_collections.somens.somecoll
+            initialize_loader = _AnsibleCollectionPkgLoader
+        else:
+            # anything below the collection
+            initialize_loader = _AnsibleCollectionLoader
+
         # NB: actual "find"ing is delegated to the constructors on the various loaders; they'll ImportError if not found
         try:
-            if toplevel_pkg == 'ansible':
-                # something under the ansible package, delegate to our internal loader in case of redirections
-                return _AnsibleInternalRedirectLoader(fullname=fullname, path_list=path)
-            if part_count == 1:
-                return _AnsibleCollectionRootPkgLoader(fullname=fullname, path_list=path)
-            if part_count == 2:  # ns pkg eg, ansible_collections, ansible_collections.somens
-                return _AnsibleCollectionNSPkgLoader(fullname=fullname, path_list=path)
-            elif part_count == 3:  # collection pkg eg, ansible_collections.somens.somecoll
-                return _AnsibleCollectionPkgLoader(fullname=fullname, path_list=path)
-            # anything below the collection
-            return _AnsibleCollectionLoader(fullname=fullname, path_list=path)
+            return initialize_loader(fullname=fullname, path_list=path)
         except ImportError:
             # TODO: log attempt to load context
             return None
 
+    def find_module(self, fullname, path=None):
+        # Figure out what's being asked for, and delegate to a special-purpose loader
+        return self._get_loader(fullname, path)
+
+    def find_spec(self, fullname, path, target=None):
+        loader = self._get_loader(fullname, path)
+        if loader:
+            spec = spec_from_loader(fullname, loader)
+            if spec is not None and hasattr(loader, '_subpackage_search_paths'):
+                spec.submodule_search_locations = loader._subpackage_search_paths
+            return spec
+        else:
+            return None
+
 
 # Implements a path_hook finder for iter_modules (since it's only path based). This finder does not need to actually
 # function as a finder in most cases, since our meta_path finder is consulted first for *almost* everything, except
@@ -251,14 +266,13 @@ class _AnsiblePathHookFinder:
 
     _filefinder_path_hook = _get_filefinder_path_hook()
 
-    def find_module(self, fullname, path=None):
-        # we ignore the passed in path here- use what we got from the path hook init
+    def _get_finder(self, fullname):
         split_name = fullname.split('.')
         toplevel_pkg = split_name[0]
 
         if toplevel_pkg == 'ansible_collections':
             # collections content? delegate to the collection finder
-            return self._collection_finder.find_module(fullname, path=[self._pathctx])
+            return self._collection_finder
         else:
             # Something else; we'd normally restrict this to `ansible` descendent modules so that any weird loader
             # behavior that arbitrary Python modules have can be serviced by those loaders. In some dev/test
@@ -277,13 +291,31 @@ class _AnsiblePathHookFinder:
                         # might not be in some other situation...
                         return None
 
-                spec = self._file_finder.find_spec(fullname)
-                if not spec:
-                    return None
-                return spec.loader
+                return self._file_finder
+
+            # call py2's internal loader
+            return pkgutil.ImpImporter(self._pathctx)
+
+    def find_module(self, fullname, path=None):
+        # we ignore the passed in path here- use what we got from the path hook init
+        finder = self._get_finder(fullname)
+        if finder is not None:
+            return finder.find_module(fullname, path=[self._pathctx])
+        else:
+            return None
+
+    def find_spec(self, fullname, target=None):
+        split_name = fullname.split('.')
+        toplevel_pkg = split_name[0]
+
+        finder = self._get_finder(fullname)
+        if finder is not None:
+            if toplevel_pkg == 'ansible_collections':
+                return finder.find_spec(fullname, path=[self._pathctx])
             else:
-                # call py2's internal loader
-                return pkgutil.ImpImporter(self._pathctx).find_module(fullname)
+                return finder.find_spec(fullname)
+        else:
+            return None
 
     def iter_modules(self, prefix):
         # NB: this currently represents only what's on disk, and does not handle package redirection
@@ -377,6 +409,23 @@ class _AnsibleCollectionPkgLoaderBase:
 
         return module_path, has_code, package_path
 
+    def exec_module(self, module):
+        # short-circuit redirect; avoid reinitializing existing modules
+        if self._redirect_module:
+            return
+
+        # execute the module's code in its namespace
+        code_obj = self.get_code(self._fullname)
+        if code_obj is not None:  # things like NS packages that can't have code on disk will return None
+            exec(code_obj, module.__dict__)
+
+    def create_module(self, spec):
+        # short-circuit redirect; we've already imported the redirected module, so just alias it and return it
+        if self._redirect_module:
+            return self._redirect_module
+        else:
+            return None
+
     def load_module(self, fullname):
         # short-circuit redirect; we've already imported the redirected module, so just alias it and return it
         if self._redirect_module:
@@ -531,12 +580,10 @@ class _AnsibleCollectionPkgLoader(_AnsibleCollectionPkgLoaderBase):
             # only search within the first collection we found
             self._subpackage_search_paths = [self._subpackage_search_paths[0]]
 
-    def load_module(self, fullname):
+    def _load_module(self, module):
         if not _meta_yml_to_dict:
             raise ValueError('ansible.utils.collection_loader._meta_yml_to_dict is not set')
 
-        module = super(_AnsibleCollectionPkgLoader, self).load_module(fullname)
-
         module._collection_meta = {}
         # TODO: load collection metadata, cache in __loader__ state
 
@@ -566,6 +613,17 @@ class _AnsibleCollectionPkgLoader(_AnsibleCollectionPkgLoaderBase):
 
         return module
 
+    def exec_module(self, module):
+        super(_AnsibleCollectionPkgLoader, self).exec_module(module)
+        self._load_module(module)
+
+    def create_module(self, spec):
+        return None
+
+    def load_module(self, fullname):
+        module = super(_AnsibleCollectionPkgLoader, self).load_module(fullname)
+        return self._load_module(module)
+
     def _canonicalize_meta(self, meta_dict):
         # TODO: rewrite import keys and all redirect targets that start with .. (current namespace) and . (current collection)
         # OR we could do it all on the fly?
@@ -676,6 +734,17 @@ class _AnsibleInternalRedirectLoader:
         if not self._redirect:
             raise ImportError('not redirected, go ask path_hook')
 
+    def exec_module(self, module):
+        # should never see this
+        if not self._redirect:
+            raise ValueError('no redirect found for {0}'.format(module.__spec__.name))
+
+        # Replace the module with the redirect
+        sys.modules[module.__spec__.name] = import_module(self._redirect)
+
+    def create_module(self, spec):
+        return None
+
     def load_module(self, fullname):
         # since we're delegating to other loaders, this should only be called for internal redirects where we answered
         # find_module with this loader, in which case we'll just directly import the redirection target, insert it into
diff --git a/lib/ansible/utils/collection_loader/_collection_meta.py b/lib/ansible/utils/collection_loader/_collection_meta.py
index 6baa9dca05..9dcb702cf7 100644
--- a/lib/ansible/utils/collection_loader/_collection_meta.py
+++ b/lib/ansible/utils/collection_loader/_collection_meta.py
@@ -1,13 +1,8 @@
 # (c) 2019 Ansible Project
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
 
-# CAUTION: There are two implementations of the collection loader.
-#          They must be kept functionally identical, although their implementations may differ.
-#
-# 1) The controller implementation resides in the "lib/ansible/utils/collection_loader/" directory.
-#    It must function on all Python versions supported on the controller.
-# 2) The ansible-test implementation resides in the "test/lib/ansible_test/_util/target/legacy_collection_loader/" directory.
-#    It must function on all Python versions supported on managed hosts which are not supported by the controller.
+# CAUTION: This implementation of the collection loader is used by ansible-test.
+#          Because of this, it must be compatible with all Python versions supported on the controller or remote.
 
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type