collection routing (#67684)

* `meta/` directory in collections
* runtime metadata for redirection/deprecation/removal of plugin loads
* a compatibility layer to keep existing content working on ansible-base + collections
* a Python import redirection layer to keep collections-hosted (and otherwise moved) content importable by things that don't know better
* supported Ansible version validation on collection loads

18 files changed, 306 insertions, 26 deletions

diff --git a/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/meta/runtime.yml b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/meta/runtime.yml
new file mode 100644
index 0000000000..cb21fee66f
--- /dev/null
+++ b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/meta/runtime.yml
@@ -0,0 +1,39 @@
+plugin_routing:
+  action:
+    uses_redirected_action:
+      redirect: testns.testcoll.subclassed_normal
+  connection:
+    redirected_local:
+      redirect: ansible.builtin.local
+  modules:
+    multilevel1:
+      redirect: testns.testcoll.multilevel2
+    multilevel2:
+      redirect: testns.testcoll.multilevel3
+    multilevel3:
+      redirect: testns.testcoll.ping
+    uses_redirected_action:
+      redirect: ansible.builtin.ping
+    setup.ps1: ansible.windows.setup
+    looped_ping:
+      redirect: testns.testcoll.looped_ping2
+    looped_ping2:
+      redirect: testns.testcoll.looped_ping
+    bogus_redirect:
+      redirect: bogus.collection.shouldbomb
+    deprecated_ping:
+      deprecation:
+        removal_date: 2020-12-31
+        warning_text: old_ping will be removed in a future release of this collection. Use new_ping instead.
+    foobar_facts:
+      redirect: foobar_info
+    aliased_ping:
+      redirect: ansible.builtin.ping
+    dead_ping:
+      tombstone:
+        removal_date: 2019-12-31
+        warning_text: dead_ping has been removed
+  module_utils:
+    moved_out_root:
+      redirect: testns.content_adj.sub1.foomodule
+requires_ansible: '>=2.11'
diff --git a/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/action/subclassed_normal.py b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/action/subclassed_normal.py
new file mode 100644
index 0000000000..f0eff30bf3
--- /dev/null
+++ b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/action/subclassed_normal.py
@@ -0,0 +1,11 @@
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+from ansible.plugins.action.normal import ActionModule as NormalAction
+
+
+class ActionModule(NormalAction):
+    def run(self, *args, **kwargs):
+        result = super(ActionModule, self).run(*args, **kwargs)
+        result['hacked'] = 'I got run under a subclassed normal, yay'
+        return result
diff --git a/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/action/uses_redirected_import.py b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/action/uses_redirected_import.py
new file mode 100644
index 0000000000..701d7b468e
--- /dev/null
+++ b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/action/uses_redirected_import.py
@@ -0,0 +1,20 @@
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+from ansible.plugins.action import ActionBase
+from ansible.module_utils.formerly_core import thingtocall
+
+
+class ActionModule(ActionBase):
+    TRANSFERS_FILES = False
+    _VALID_ARGS = frozenset()
+
+    def run(self, tmp=None, task_vars=None):
+        if task_vars is None:
+            task_vars = dict()
+
+        result = super(ActionModule, self).run(None, task_vars)
+
+        result = dict(changed=False, ttc_res=thingtocall())
+
+        return result
diff --git a/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/module_utils/nested_same/nested_same/nested_same.py b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/module_utils/nested_same/nested_same/nested_same.py
new file mode 100644
index 0000000000..774075646a
--- /dev/null
+++ b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/module_utils/nested_same/nested_same/nested_same.py
@@ -0,0 +1,6 @@
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+def nested_same():
+    return 'hello from nested_same'
diff --git a/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/module_utils/subpkg_with_init.py b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/module_utils/subpkg_with_init.py
new file mode 100644
index 0000000000..b48a717cae
--- /dev/null
+++ b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/module_utils/subpkg_with_init.py
@@ -0,0 +1,11 @@
+# NB: this module should never be loaded, since we'll see the subpkg_with_init package dir first
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+def thingtocall():
+    raise Exception('this should never be called (loaded discrete module instead of package module)')
+
+
+def anotherthingtocall():
+    raise Exception('this should never be called (loaded discrete module instead of package module)')
diff --git a/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/modules/deprecated_ping.py b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/modules/deprecated_ping.py
new file mode 100644
index 0000000000..9698ba6f07
--- /dev/null
+++ b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/modules/deprecated_ping.py
@@ -0,0 +1,13 @@
+#!/usr/bin/python
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import json
+
+
+def main():
+    print(json.dumps(dict(changed=False, source='user', is_deprecated=True)))
+
+
+if __name__ == '__main__':
+    main()
diff --git a/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/modules/uses_collection_redirected_mu.py b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/modules/uses_collection_redirected_mu.py
new file mode 100644
index 0000000000..736eb400f0
--- /dev/null
+++ b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/modules/uses_collection_redirected_mu.py
@@ -0,0 +1,19 @@
+#!/usr/bin/python
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import json
+import sys
+
+from ansible_collections.testns.testcoll.plugins.module_utils.moved_out_root import importme
+
+
+def main():
+    mu_result = importme()
+    print(json.dumps(dict(changed=False, source='user', mu_result=mu_result)))
+
+    sys.exit()
+
+
+if __name__ == '__main__':
+    main()
diff --git a/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/modules/uses_core_redirected_mu.py b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/modules/uses_core_redirected_mu.py
new file mode 100644
index 0000000000..28a077293a
--- /dev/null
+++ b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/modules/uses_core_redirected_mu.py
@@ -0,0 +1,19 @@
+#!/usr/bin/python
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import json
+import sys
+
+from ansible.module_utils.formerly_core import thingtocall
+
+
+def main():
+    mu_result = thingtocall()
+    print(json.dumps(dict(changed=False, source='user', mu_result=mu_result)))
+
+    sys.exit()
+
+
+if __name__ == '__main__':
+    main()
diff --git a/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/modules/uses_nested_same_as_func.py b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/modules/uses_nested_same_as_func.py
new file mode 100644
index 0000000000..26fa53c077
--- /dev/null
+++ b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/modules/uses_nested_same_as_func.py
@@ -0,0 +1,19 @@
+#!/usr/bin/python
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import json
+import sys
+
+from ansible_collections.testns.testcoll.plugins.module_utils.nested_same.nested_same.nested_same import nested_same
+
+
+def main():
+    mu_result = nested_same()
+    print(json.dumps(dict(changed=False, source='user', mu_result=mu_result)))
+
+    sys.exit()
+
+
+if __name__ == '__main__':
+    main()
diff --git a/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/modules/uses_nested_same_as_module.py b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/modules/uses_nested_same_as_module.py
new file mode 100644
index 0000000000..e017c14f8e
--- /dev/null
+++ b/test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/modules/uses_nested_same_as_module.py
@@ -0,0 +1,19 @@
+#!/usr/bin/python
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import json
+import sys
+
+from ansible_collections.testns.testcoll.plugins.module_utils.nested_same.nested_same import nested_same
+
+
+def main():
+    mu_result = nested_same.nested_same()
+    print(json.dumps(dict(changed=False, source='user', mu_result=mu_result)))
+
+    sys.exit()
+
+
+if __name__ == '__main__':
+    main()
diff --git a/test/integration/targets/collections/collections/ansible_collections/testns/content_adj/plugins/module_utils/sub1/foomodule.py b/test/integration/targets/collections/collections/ansible_collections/testns/content_adj/plugins/module_utils/sub1/foomodule.py
new file mode 100644
index 0000000000..eeffe01e29
--- /dev/null
+++ b/test/integration/targets/collections/collections/ansible_collections/testns/content_adj/plugins/module_utils/sub1/foomodule.py
@@ -0,0 +1,6 @@
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+def importme():
+    return "hello from {0}".format(__name__)
diff --git a/test/integration/targets/collections/filter_plugins/override_formerly_core_masked_filter.py b/test/integration/targets/collections/filter_plugins/override_formerly_core_masked_filter.py
new file mode 100644
index 0000000000..600b1fd8c2
--- /dev/null
+++ b/test/integration/targets/collections/filter_plugins/override_formerly_core_masked_filter.py
@@ -0,0 +1,13 @@
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+def override_formerly_core_masked_filter(*args, **kwargs):
+    return 'hello from overridden formerly_core_masked_filter'
+
+
+class FilterModule(object):
+    def filters(self):
+        return {
+            'formerly_core_masked_filter': override_formerly_core_masked_filter
+        }
diff --git a/test/integration/targets/collections/inventory_test.yml b/test/integration/targets/collections/inventory_test.yml
new file mode 100644
index 0000000000..b50892788d
--- /dev/null
+++ b/test/integration/targets/collections/inventory_test.yml
@@ -0,0 +1,26 @@
+- name: test a collection-hosted connection plugin against hosts from collection-hosted inventory plugins
+  hosts: dynamic_host_a, dynamic_host_redirected
+  gather_facts: no
+  vars:
+    ansible_connection: testns.testcoll.localconn
+    ansible_localconn_connectionvar: from_play
+  tasks:
+  - raw: echo 'hello world'
+    register: connection_out
+
+  - assert:
+      that:
+      - connection_out.stdout == "localconn ran echo 'hello world'"
+      # ensure that the connection var we overrode above made it into the running config
+      - connection_out.stderr == "connectionvar is from_play"
+
+
+- hosts: localhost
+  gather_facts: no
+  tasks:
+  - assert:
+      that:
+      - hostvars['dynamic_host_a'] is defined
+      - hostvars['dynamic_host_a'].connection_out.stdout == "localconn ran echo 'hello world'"
+      - hostvars['dynamic_host_redirected'] is defined
+      - hostvars['dynamic_host_redirected'].connection_out.stdout == "localconn ran echo 'hello world'"
diff --git a/test/integration/targets/collections/posix.yml b/test/integration/targets/collections/posix.yml
index 61f950f50b..ab74aedb23 100644
--- a/test/integration/targets/collections/posix.yml
+++ b/test/integration/targets/collections/posix.yml
@@ -63,6 +63,16 @@
     testns.testcoll.uses_leaf_mu_module_import_from:
     register: from_out
 
+  # module with multiple levels of the same nested package name and imported as a function
+  - name: exec module with multiple levels of the same nested package name imported as a function
+    testns.testcoll.uses_nested_same_as_func:
+    register: from_nested_func
+
+  # module with multiple levels of the same nested package name and imported as a module
+  - name: exec module with multiple levels of the same nested package name imported as a module
+    testns.testcoll.uses_nested_same_as_module:
+    register: from_nested_module
+
   - assert:
       that:
       - testmodule_out.source == 'user'
@@ -79,6 +89,8 @@
       - flat_out.mu_result == 'thingtocall in leaf'
       - from_out.mu_result == 'thingtocall in leaf'
       - from_out.mu2_result == 'thingtocall in secondary'
+      - from_nested_func.mu_result == 'hello from nested_same'
+      - from_nested_module.mu_result == 'hello from nested_same'
 
 - hosts: testhost
   tasks:
@@ -373,28 +385,7 @@
   - include_tasks: includeme.yml
 
 
-- name: test a collection-hosted connection plugin against a host from a collection-hosted inventory plugin
-  hosts: dynamic_host_a
-  vars:
-    ansible_connection: testns.testcoll.localconn
-    ansible_localconn_connectionvar: from_play
-  tasks:
-  - raw: echo 'hello world'
-    register: connection_out
-
-  - assert:
-      that:
-      - connection_out.stdout == "localconn ran echo 'hello world'"
-      # ensure that the connection var we overrode above made it into the running config
-      - connection_out.stderr == "connectionvar is from_play"
-
-- hosts: testhost
-  tasks:
-  - assert:
-      that:
-      - hostvars['dynamic_host_a'] is defined
-      - hostvars['dynamic_host_a'].connection_out.stdout == "localconn ran echo 'hello world'"
-
+- import_playbook: test_collection_meta.yml
 - name: Test FQCN handlers
   hosts: testhost
   vars:
diff --git a/test/integration/targets/collections/redirected.statichost.yml b/test/integration/targets/collections/redirected.statichost.yml
new file mode 100644
index 0000000000..9fd2c2d87b
--- /dev/null
+++ b/test/integration/targets/collections/redirected.statichost.yml
@@ -0,0 +1,3 @@
+# use a plugin redirected by core to a collection to ensure inventory redirection and redirected config names are working
+plugin: formerly_core_inventory  # this is defined in the ansible-base runtime.yml routing to point at testns.content_adj.statichost
+hostname: dynamic_host_redirected
diff --git a/test/integration/targets/collections/runme.sh b/test/integration/targets/collections/runme.sh
index 98471b2761..e0c7ef0319 100755
--- a/test/integration/targets/collections/runme.sh
+++ b/test/integration/targets/collections/runme.sh
@@ -38,19 +38,22 @@ else
   export TEST_PLAYBOOK=posix.yml
 
   echo "testing default collection support"
-  ansible-playbook -i "${INVENTORY_PATH}" collection_root_user/ansible_collections/testns/testcoll/playbooks/default_collection_playbook.yml
+  ansible-playbook -i "${INVENTORY_PATH}" collection_root_user/ansible_collections/testns/testcoll/playbooks/default_collection_playbook.yml "$@"
 fi
 
 # run test playbooks
-ansible-playbook -i "${INVENTORY_PATH}"  -i ./a.statichost.yml -v "${TEST_PLAYBOOK}" "$@"
+ansible-playbook -i "${INVENTORY_PATH}" -v "${TEST_PLAYBOOK}" "$@"
 
 if [[ ${INVENTORY_PATH} != *.winrm ]]; then
-	ansible-playbook -i "${INVENTORY_PATH}"  -i ./a.statichost.yml -v invocation_tests.yml "$@"
+	ansible-playbook -i "${INVENTORY_PATH}" -v invocation_tests.yml "$@"
 fi
 
+# test collection inventories
+ansible-playbook inventory_test.yml -i a.statichost.yml -i redirected.statichost.yml "$@"
+
 # test adjacent with --playbook-dir
 export ANSIBLE_COLLECTIONS_PATHS=''
-ANSIBLE_INVENTORY_ANY_UNPARSED_IS_FAILED=1 ansible-inventory -i a.statichost.yml --list --export --playbook-dir=. -v "$@"
+ANSIBLE_INVENTORY_ANY_UNPARSED_IS_FAILED=1 ansible-inventory --list --export --playbook-dir=. -v "$@"
 
 # use an inventory source with caching enabled
 ansible-playbook -i a.statichost.yml -i ./cache.statichost.yml -v check_populated_inventory.yml
diff --git a/test/integration/targets/collections/test_collection_meta.yml b/test/integration/targets/collections/test_collection_meta.yml
new file mode 100644
index 0000000000..22a00b2197
--- /dev/null
+++ b/test/integration/targets/collections/test_collection_meta.yml
@@ -0,0 +1,46 @@
+- hosts: localhost
+  gather_facts: no
+  collections:
+  - testns.testcoll
+  vars:
+    # redirect connection
+    ansible_connection: testns.testcoll.redirected_local
+  tasks:
+  - assert:
+      that: ('data' | testns.testcoll.testfilter) == 'data_via_testfilter_from_userdir'
+
+  # redirect module (multiple levels)
+  - multilevel1:
+  # redirect action
+  - uses_redirected_action:
+  # redirect import (consumed via action)
+  - uses_redirected_import:
+  # redirect lookup
+  - assert:
+      that: lookup('formerly_core_lookup') == 'mylookup_from_user_dir'
+  # redirect filter
+  - assert:
+      that: ('yes' | formerly_core_filter) == True
+  # legacy filter should mask redirected
+  - assert:
+      that: ('' | formerly_core_masked_filter) == 'hello from overridden formerly_core_masked_filter'
+  # redirect test
+  - assert:
+      that:
+      - "'stuff' is formerly_core_test('tuf')"
+      - "'hello override' is formerly_core_masked_test"
+  # redirect module (formerly internal)
+  - formerly_core_ping:
+  # redirect module from collection (with subdir)
+  - testns.testcoll.module_subdir.subdir_ping_module:
+  # redirect module_utils plugin (consumed via module)
+  - uses_core_redirected_mu:
+  # deprecated module (issues warning)
+  - deprecated_ping:
+  # redirect module (internal alias)
+  - aliased_ping:
+  # redirect module (cycle detection, fatal)
+#  - looped_ping:
+
+  # removed module (fatal)
+#  - dead_ping:
diff --git a/test/integration/targets/collections/test_plugins/override_formerly_core_masked_test.py b/test/integration/targets/collections/test_plugins/override_formerly_core_masked_test.py
new file mode 100644
index 0000000000..11c7f7a7e3
--- /dev/null
+++ b/test/integration/targets/collections/test_plugins/override_formerly_core_masked_test.py
@@ -0,0 +1,16 @@
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+def override_formerly_core_masked_test(value, *args, **kwargs):
+    if value != 'hello override':
+        raise Exception('expected "hello override" only...')
+
+    return True
+
+
+class TestModule(object):
+    def tests(self):
+        return {
+            'formerly_core_masked_test': override_formerly_core_masked_test
+        }