Prepare ansible-test for inclusion in setup.py (#60294)

* Cache ansible version lookup.
* Fix method of determining ANSIBLE_ROOT.
* Clean up based on PyCharm inspections.
* Generate minimal PKG-INFO without setup.py.
* Use ANSIBLE_LIB_ROOT where possible.
* Use import instead of subprocess to get version.
* Fix install layout type.
* Correct required paths message for installs.
* Update list of files copied during delegation.
* Fix ansible-test entry point.
* Fix pylint issue.
* Fix version lookup on Python 2.x.
* Fix pylint issue.
* Remove unwanted print statement.

13 files changed, 87 insertions, 53 deletions

diff --git a/test/lib/ansible_test/_data/cli/ansible_test_cli_stub.py b/test/lib/ansible_test/_data/cli/ansible_test_cli_stub.py
index 4c075075f5..d12b6334ef 100755
--- a/test/lib/ansible_test/_data/cli/ansible_test_cli_stub.py
+++ b/test/lib/ansible_test/_data/cli/ansible_test_cli_stub.py
@@ -11,13 +11,14 @@ import sys
 
 def main():
     """Main program entry point."""
-    ansible_root = os.path.abspath(os.path.join(os.path.dirname(os.path.dirname(__file__))))
+    ansible_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
     source_root = os.path.join(ansible_root, 'test', 'lib')
 
-    if os.path.exists(os.path.join(ansible_root, 'setup.py')) and os.path.exists(os.path.join(source_root, 'ansible_test', '_internal', 'cli.py')):
+    if os.path.exists(os.path.join(source_root, 'ansible_test', '_internal', 'cli.py')):
         # running from source, use that version of ansible-test instead of any version that may already be installed
         sys.path.insert(0, source_root)
 
+    # noinspection PyProtectedMember
     from ansible_test._internal.cli import main as cli_main
 
     cli_main()
diff --git a/test/lib/ansible_test/_data/sanity/import/importer.py b/test/lib/ansible_test/_data/sanity/import/importer.py
index e0606950b6..561ebf9b59 100755
--- a/test/lib/ansible_test/_data/sanity/import/importer.py
+++ b/test/lib/ansible_test/_data/sanity/import/importer.py
@@ -24,6 +24,7 @@ def main():
         import imp
 
     try:
+        # noinspection PyCompatibility
         from StringIO import StringIO
     except ImportError:
         from io import StringIO
@@ -34,6 +35,7 @@ def main():
     try:
         from ansible.utils.collection_loader import AnsibleCollectionLoader
     except ImportError:
+        # noinspection PyPep8Naming
         AnsibleCollectionLoader = None
 
     class ImporterAnsibleModuleException(Exception):
@@ -58,6 +60,7 @@ def main():
 
         if AnsibleCollectionLoader:
             # allow importing code from collections
+            # noinspection PyCallingNonCallable
             sys.meta_path.insert(0, AnsibleCollectionLoader())
 
         for path in sys.argv[1:] or sys.stdin.read().splitlines():
diff --git a/test/lib/ansible_test/_internal/ansible_util.py b/test/lib/ansible_test/_internal/ansible_util.py
index e36829e1dc..75be193152 100644
--- a/test/lib/ansible_test/_internal/ansible_util.py
+++ b/test/lib/ansible_test/_internal/ansible_util.py
@@ -15,6 +15,7 @@ from .util import (
     find_python,
     ApplicationError,
     ANSIBLE_ROOT,
+    ANSIBLE_LIB_ROOT,
     ANSIBLE_TEST_DATA_ROOT,
 )
 
@@ -67,7 +68,7 @@ def ansible_environment(args, color=True, ansible_config=None):
         ANSIBLE_RETRY_FILES_ENABLED='false',
         ANSIBLE_CONFIG=os.path.abspath(ansible_config),
         ANSIBLE_LIBRARY='/dev/null',
-        PYTHONPATH=os.path.join(ANSIBLE_ROOT, 'lib'),
+        PYTHONPATH=os.path.dirname(ANSIBLE_LIB_ROOT),
         PAGER='/bin/cat',
         PATH=path,
     )
diff --git a/test/lib/ansible_test/_internal/data.py b/test/lib/ansible_test/_internal/data.py
index a99b573abc..0b777e4f85 100644
--- a/test/lib/ansible_test/_internal/data.py
+++ b/test/lib/ansible_test/_internal/data.py
@@ -11,6 +11,7 @@ from .util import (
     import_plugins,
     ANSIBLE_ROOT,
     is_subdir,
+    ANSIBLE_IS_INSTALLED,
 )
 
 from .provider import (
@@ -57,7 +58,7 @@ class DataContext:
             content = self.create_content_layout(self.__layout_providers, self.__source_providers, content_path, False)
 
             if content.is_ansible:
-                install = content
+                install = InstallLayout(ANSIBLE_ROOT, content.all_files())
             else:
                 install = None
         elif is_subdir(current_path, ANSIBLE_ROOT):
@@ -129,12 +130,18 @@ def data_init():  # type: () -> DataContext
     try:
         context = DataContext()
     except ProviderNotFoundForPath:
-        raise ApplicationError('''The current working directory must be at or below one of:
+        options = [
+            ' - an Ansible collection: {...}/ansible_collections/{namespace}/{collection}/',
+        ]
 
- - Ansible source: %s/
- - Ansible collection: {...}/ansible_collections/{namespace}/{collection}/
+        if not ANSIBLE_IS_INSTALLED:
+            options.insert(0, ' - the Ansible source: %s/' % ANSIBLE_ROOT)
 
-Current working directory: %s''' % (ANSIBLE_ROOT, os.getcwd()))
+        raise ApplicationError('''The current working directory must be at or below:
+
+%s
+
+Current working directory: %s''' % ('\n'.join(options), os.getcwd()))
 
     return context
 
diff --git a/test/lib/ansible_test/_internal/env.py b/test/lib/ansible_test/_internal/env.py
index 2a2e0aa880..ce7a65876a 100644
--- a/test/lib/ansible_test/_internal/env.py
+++ b/test/lib/ansible_test/_internal/env.py
@@ -20,13 +20,10 @@ from .config import (
 from .util import (
     display,
     find_executable,
-    raw_command,
     SubprocessError,
     ApplicationError,
-)
-
-from .ansible_util import (
-    ansible_environment,
+    load_module,
+    ANSIBLE_LIB_ROOT,
 )
 
 from .git import (
@@ -81,7 +78,7 @@ def show_dump_env(args):
 
     data = dict(
         ansible=dict(
-            version=get_ansible_version(args),
+            version=get_ansible_version(),
         ),
         docker=get_docker_details(args),
         environ=os.environ.copy(),
@@ -238,21 +235,21 @@ def show_dict(data, verbose, root_verbosity=0, path=None):
             display.info(indent + '%s: %s' % (key, value), verbosity=verbosity)
 
 
-def get_ansible_version(args):
-    """
-    :type args: CommonConfig
-    :rtype: str | None
-    """
-    code = 'from __future__ import (print_function); from ansible.release import __version__; print(__version__)'
-    cmd = [sys.executable, '-c', code]
-    env = ansible_environment(args)
-
+def get_ansible_version():  # type: () -> str
+    """Return the Ansible version."""
     try:
-        ansible_version, _dummy = raw_command(cmd, env=env, capture=True)
-        ansible_version = ansible_version.strip()
-    except SubprocessError as ex:
-        display.warning('Unable to get Ansible version:\n%s' % ex)
-        ansible_version = None
+        return get_ansible_version.version
+    except AttributeError:
+        pass
+
+    # ansible may not be in our sys.path
+    # avoids a symlink to release.py since ansible placement relative to ansible-test may change during delegation
+    load_module(os.path.join(ANSIBLE_LIB_ROOT, 'release.py'), 'ansible_release')
+
+    # noinspection PyUnresolvedReferences
+    from ansible_release import __version__ as ansible_version  # pylint: disable=import-error
+
+    get_ansible_version.version = ansible_version
 
     return ansible_version
 
diff --git a/test/lib/ansible_test/_internal/executor.py b/test/lib/ansible_test/_internal/executor.py
index 0323d756b1..ce278543be 100644
--- a/test/lib/ansible_test/_internal/executor.py
+++ b/test/lib/ansible_test/_internal/executor.py
@@ -60,7 +60,7 @@ from .util import (
     get_remote_completion,
     COVERAGE_OUTPUT_NAME,
     cmd_quote,
-    ANSIBLE_ROOT,
+    ANSIBLE_LIB_ROOT,
     ANSIBLE_TEST_DATA_ROOT,
     get_available_python_versions,
     is_subdir,
@@ -87,6 +87,10 @@ from .ansible_util import (
     check_pyyaml,
 )
 
+from .env import (
+    get_ansible_version,
+)
+
 from .target import (
     IntegrationTarget,
     walk_internal_targets,
@@ -299,13 +303,34 @@ def generate_egg_info(args):
     """
     :type args: EnvironmentConfig
     """
-    if not os.path.exists(os.path.join(ANSIBLE_ROOT, 'setup.py')):
+    if args.explain:
         return
 
-    if os.path.isdir(os.path.join(ANSIBLE_ROOT, 'lib/ansible.egg-info')):
+    egg_info_path = ANSIBLE_LIB_ROOT + '.egg-info'
+
+    if os.path.exists(egg_info_path):
         return
 
-    run_command(args, [args.python_executable, 'setup.py', 'egg_info'], cwd=ANSIBLE_ROOT, capture=args.verbosity < 3)
+    # minimal PKG-INFO stub following the format defined in PEP 241
+    # required for older setuptools versions to avoid a traceback when importing pkg_resources from packages like cryptography
+    # newer setuptools versions are happy with an empty directory
+    # including a stub here means we don't need to locate the existing file or have setup.py generate it when running from source
+    pkg_info = '''
+Metadata-Version: 1.0
+Name: ansible
+Version: %s
+Platform: UNKNOWN
+Summary: Radically simple IT automation
+Author-email: info@ansible.com
+License: GPLv3+
+''' % get_ansible_version()
+
+    os.mkdir(egg_info_path)
+
+    pkg_info_path = os.path.join(egg_info_path, 'PKG-INFO')
+
+    with open(pkg_info_path, 'w') as pkg_info_fd:
+        pkg_info_fd.write(pkg_info.lstrip())
 
 
 def generate_pip_install(pip, command, packages=None):
diff --git a/test/lib/ansible_test/_internal/payload.py b/test/lib/ansible_test/_internal/payload.py
index 12127e75d0..f3c59ec30a 100644
--- a/test/lib/ansible_test/_internal/payload.py
+++ b/test/lib/ansible_test/_internal/payload.py
@@ -41,17 +41,13 @@ def create_payload(args, dst_path):  # type: (CommonConfig, str) -> None
                  f[1].startswith('bin/') or
                  f[1].startswith('lib/') or
                  f[1].startswith('test/lib/') or
-                 f[1].startswith('packaging/requirements/') or
                  f[1] in (
-                     'setup.py',
-                     'README.rst',
-                     'requirements.txt',
-                     # units only
-                     'test/units/ansible.cfg',
-                     # integration only
                      'test/integration/integration.cfg',
                      'test/integration/integration_config.yml',
                      'test/integration/inventory',
+                     'test/integration/network-integration.cfg',
+                     'test/integration/target-prefixes.network',
+                     'test/integration/windows-integration.cfg',
                  )]
 
         if not isinstance(args, (ShellConfig, IntegrationConfig)):
diff --git a/test/lib/ansible_test/_internal/provider/layout/ansible.py b/test/lib/ansible_test/_internal/provider/layout/ansible.py
index 4ed6e835a4..49bbe601f6 100644
--- a/test/lib/ansible_test/_internal/provider/layout/ansible.py
+++ b/test/lib/ansible_test/_internal/provider/layout/ansible.py
@@ -3,14 +3,9 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 import os
-import re
 
 from ... import types as t
 
-from ...util import (
-    ANSIBLE_TEST_ROOT,
-)
-
 from . import (
     ContentLayout,
     LayoutProvider,
diff --git a/test/lib/ansible_test/_internal/provider/layout/collection.py b/test/lib/ansible_test/_internal/provider/layout/collection.py
index a09f0bfd03..a0633f4d51 100644
--- a/test/lib/ansible_test/_internal/provider/layout/collection.py
+++ b/test/lib/ansible_test/_internal/provider/layout/collection.py
@@ -3,7 +3,6 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 import os
-import re
 
 from ... import types as t
 
diff --git a/test/lib/ansible_test/_internal/sanity/__init__.py b/test/lib/ansible_test/_internal/sanity/__init__.py
index 4dcde04f07..d4e4814b79 100644
--- a/test/lib/ansible_test/_internal/sanity/__init__.py
+++ b/test/lib/ansible_test/_internal/sanity/__init__.py
@@ -238,7 +238,7 @@ class SanityIgnoreParser:
 
     def __init__(self, args):  # type: (SanityConfig) -> None
         if data_context().content.collection:
-            ansible_version = '%s.%s' % tuple(get_ansible_version(args).split('.')[:2])
+            ansible_version = '%s.%s' % tuple(get_ansible_version().split('.')[:2])
 
             ansible_label = 'Ansible %s' % ansible_version
             file_name = 'ignore-%s.txt' % ansible_version
diff --git a/test/lib/ansible_test/_internal/sanity/import.py b/test/lib/ansible_test/_internal/sanity/import.py
index 921565e549..2e9215bf54 100644
--- a/test/lib/ansible_test/_internal/sanity/import.py
+++ b/test/lib/ansible_test/_internal/sanity/import.py
@@ -26,6 +26,7 @@ from ..util import (
     parse_to_list_of_dict,
     make_dirs,
     is_subdir,
+    ANSIBLE_LIB_ROOT,
 )
 
 from ..util_common import (
@@ -51,7 +52,6 @@ from ..coverage_util import (
 
 from ..data import (
     data_context,
-    ANSIBLE_ROOT,
 )
 
 
@@ -109,7 +109,7 @@ class ImportTest(SanityMultipleVersion):
             with open(ansible_init, 'w'):
                 pass
 
-            os.symlink(os.path.join(ANSIBLE_ROOT, 'lib/ansible/module_utils'), ansible_link)
+            os.symlink(os.path.join(ANSIBLE_LIB_ROOT, 'module_utils'), ansible_link)
 
             if data_context().content.collection:
                 # inject just enough Ansible code for the collections loader to work on all supported Python versions
@@ -120,8 +120,8 @@ class ImportTest(SanityMultipleVersion):
                 with open(os.path.join(ansible_path, 'utils/__init__.py'), 'w'):
                     pass
 
-                os.symlink(os.path.join(ANSIBLE_ROOT, 'lib/ansible/utils/collection_loader.py'), os.path.join(ansible_path, 'utils/collection_loader.py'))
-                os.symlink(os.path.join(ANSIBLE_ROOT, 'lib/ansible/utils/singleton.py'), os.path.join(ansible_path, 'utils/singleton.py'))
+                os.symlink(os.path.join(ANSIBLE_LIB_ROOT, 'utils', 'collection_loader.py'), os.path.join(ansible_path, 'utils', 'collection_loader.py'))
+                os.symlink(os.path.join(ANSIBLE_LIB_ROOT, 'utils', 'singleton.py'), os.path.join(ansible_path, 'utils', 'singleton.py'))
 
                 make_dirs(os.path.join(ansible_path, 'modules'))
                 with open(os.path.join(ansible_path, 'modules/__init__.py'), 'w'):
diff --git a/test/lib/ansible_test/_internal/util.py b/test/lib/ansible_test/_internal/util.py
index 3b830b5629..ccad8e20a0 100644
--- a/test/lib/ansible_test/_internal/util.py
+++ b/test/lib/ansible_test/_internal/util.py
@@ -62,8 +62,19 @@ except AttributeError:
 COVERAGE_CONFIG_NAME = 'coveragerc'
 COVERAGE_OUTPUT_NAME = 'coverage'
 
-ANSIBLE_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
-ANSIBLE_TEST_ROOT = os.path.join(ANSIBLE_ROOT, 'test', 'lib', 'ansible_test')
+ANSIBLE_TEST_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+# assume running from install
+ANSIBLE_ROOT = os.path.dirname(ANSIBLE_TEST_ROOT)
+ANSIBLE_LIB_ROOT = os.path.join(ANSIBLE_ROOT, 'ansible')
+ANSIBLE_IS_INSTALLED = True
+
+if not os.path.exists(ANSIBLE_LIB_ROOT):
+    # running from source
+    ANSIBLE_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(ANSIBLE_TEST_ROOT)))
+    ANSIBLE_LIB_ROOT = os.path.join(ANSIBLE_ROOT, 'lib', 'ansible')
+    ANSIBLE_IS_INSTALLED = False
+
 ANSIBLE_TEST_DATA_ROOT = os.path.join(ANSIBLE_TEST_ROOT, '_data')
 ANSIBLE_TEST_CONFIG_ROOT = os.path.join(ANSIBLE_TEST_ROOT, 'config')
 
diff --git a/test/lib/ansible_test/_internal/util_common.py b/test/lib/ansible_test/_internal/util_common.py
index ed323552d6..5649234d6b 100644
--- a/test/lib/ansible_test/_internal/util_common.py
+++ b/test/lib/ansible_test/_internal/util_common.py
@@ -15,7 +15,6 @@ from .util import (
     COVERAGE_OUTPUT_NAME,
     display,
     find_python,
-    ANSIBLE_ROOT,
     is_shippable,
     MODE_DIRECTORY,
     MODE_FILE_EXECUTE,