Add support for ansible-test --coverage-check. (#53573)

8 files changed, 49 insertions, 13 deletions

diff --git a/lib/ansible/executor/module_common.py b/lib/ansible/executor/module_common.py
index d4266464fa..021ea5349b 100644
--- a/lib/ansible/executor/module_common.py
+++ b/lib/ansible/executor/module_common.py
@@ -339,7 +339,12 @@ ANSIBALLZ_COVERAGE_TEMPLATE = '''
         os.environ['COVERAGE_FILE'] = '%(coverage_output)s'
 
         import atexit
-        import coverage
+
+        try:
+            import coverage
+        except ImportError:
+            print('{"msg": "Could not import `coverage` module.", "failed": true}')
+            sys.exit(1)
 
         cov = coverage.Coverage(config_file='%(coverage_config)s')
 
@@ -352,6 +357,14 @@ ANSIBALLZ_COVERAGE_TEMPLATE = '''
         cov.start()
 '''
 
+ANSIBALLZ_COVERAGE_CHECK_TEMPLATE = '''
+        try:
+            imp.find_module('coverage')
+        except ImportError:
+            print('{"msg": "Could not find `coverage` module.", "failed": true}')
+            sys.exit(1)
+'''
+
 ANSIBALLZ_RLIMIT_TEMPLATE = '''
     import resource
 
@@ -829,12 +842,19 @@ def _find_module_utils(module_name, b_module_data, module_path, module_args, tas
         coverage_config = os.environ.get('_ANSIBLE_COVERAGE_CONFIG')
 
         if coverage_config:
-            # Enable code coverage analysis of the module.
-            # This feature is for internal testing and may change without notice.
-            coverage = ANSIBALLZ_COVERAGE_TEMPLATE % dict(
-                coverage_config=coverage_config,
-                coverage_output=os.environ['_ANSIBLE_COVERAGE_OUTPUT']
-            )
+            coverage_output = os.environ['_ANSIBLE_COVERAGE_OUTPUT']
+
+            if coverage_output:
+                # Enable code coverage analysis of the module.
+                # This feature is for internal testing and may change without notice.
+                coverage = ANSIBALLZ_COVERAGE_TEMPLATE % dict(
+                    coverage_config=coverage_config,
+                    coverage_output=coverage_output,
+                )
+            else:
+                # Verify coverage is available without importing it.
+                # This will detect when a module would fail with coverage enabled with minimal overhead.
+                coverage = ANSIBALLZ_COVERAGE_CHECK_TEMPLATE
         else:
             coverage = ''
 
diff --git a/test/runner/lib/cli.py b/test/runner/lib/cli.py
index 6d6e5653b8..577495c408 100644
--- a/test/runner/lib/cli.py
+++ b/test/runner/lib/cli.py
@@ -209,6 +209,10 @@ def parse_args():
                       default='',
                       help='label to include in coverage output file names')
 
+    test.add_argument('--coverage-check',
+                      action='store_true',
+                      help='only verify code coverage can be enabled')
+
     test.add_argument('--metadata',
                       help=argparse.SUPPRESS)
 
diff --git a/test/runner/lib/config.py b/test/runner/lib/config.py
index a4c3ff09c8..324bcdccb9 100644
--- a/test/runner/lib/config.py
+++ b/test/runner/lib/config.py
@@ -104,6 +104,7 @@ class TestConfig(EnvironmentConfig):
 
         self.coverage = args.coverage  # type: bool
         self.coverage_label = args.coverage_label  # type: str
+        self.coverage_check = args.coverage_check  # type: bool
         self.include = args.include or []  # type: list [str]
         self.exclude = args.exclude or []  # type: list [str]
         self.require = args.require or []  # type: list [str]
@@ -124,6 +125,9 @@ class TestConfig(EnvironmentConfig):
         self.metadata = Metadata.from_file(args.metadata) if args.metadata else Metadata()
         self.metadata_path = None
 
+        if self.coverage_check:
+            self.coverage = True
+
 
 class ShellConfig(EnvironmentConfig):
     """Configuration for the shell command."""
diff --git a/test/runner/lib/util.py b/test/runner/lib/util.py
index 75166fe352..5893a6fcde 100644
--- a/test/runner/lib/util.py
+++ b/test/runner/lib/util.py
@@ -211,6 +211,9 @@ def intercept_command(args, cmd, target_name, capture=False, env=None, data=None
     coverage_file = os.path.abspath(os.path.join(inject_path, '..', 'output', '%s=%s=%s=%s=coverage' % (
         args.command, target_name, args.coverage_label or 'local-%s' % version, 'python-%s' % version)))
 
+    if args.coverage_check:
+        coverage_file = ''
+
     env['PATH'] = inject_path + os.path.pathsep + env['PATH']
     env['ANSIBLE_TEST_PYTHON_VERSION'] = version
     env['ANSIBLE_TEST_PYTHON_INTERPRETER'] = interpreter
diff --git a/test/units/conftest.py b/test/units/conftest.py
index c7163201d0..0fb7955081 100644
--- a/test/units/conftest.py
+++ b/test/units/conftest.py
@@ -29,6 +29,11 @@ def pytest_configure():
         if not coverage_config:
             return
 
+        coverage_output = os.environ.get('_ANSIBLE_COVERAGE_OUTPUT')
+
+        if not coverage_output:
+            return
+
         cov = coverage.Coverage(config_file=coverage_config)
         coverage_instances.append(cov)
     else:
diff --git a/test/utils/shippable/network.sh b/test/utils/shippable/network.sh
index b63e7b3917..51093feba4 100755
--- a/test/utils/shippable/network.sh
+++ b/test/utils/shippable/network.sh
@@ -6,7 +6,7 @@ set -o pipefail -eux
 ansible-test network-integration --explain ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} 2>&1 \
     | { grep ' network-integration: .* (targeted)$' || true; } > /tmp/network.txt
 
-if [ "${COVERAGE}" ]; then
+if [ "${COVERAGE}" == "--coverage" ]; then
     # when on-demand coverage is enabled, force tests to run for all network platforms
     echo "coverage" > /tmp/network.txt
 fi
diff --git a/test/utils/shippable/shippable.sh b/test/utils/shippable/shippable.sh
index 22ff3c0cce..8e90ab9095 100755
--- a/test/utils/shippable/shippable.sh
+++ b/test/utils/shippable/shippable.sh
@@ -46,7 +46,7 @@ elif [[ "${COMMIT_MESSAGE}" =~ ci_coverage ]]; then
     export COVERAGE="--coverage"
 else
     # on-demand coverage reporting disabled (default behavior, always-on coverage reporting remains enabled)
-    export COVERAGE=""
+    export COVERAGE="--coverage-check"
 fi
 
 if [ -n "${COMPLETE:-}" ]; then
@@ -75,7 +75,7 @@ function cleanup
 {
     if find test/results/coverage/ -mindepth 1 -name '.*' -prune -o -print -quit | grep -q .; then
         # for complete on-demand coverage generate a report for all files with no coverage on the "other" job so we only have one copy
-        if [ "${COVERAGE}" ] && [ "${CHANGED}" == "" ] && [ "${test}" == "sanity/1" ]; then
+        if [ "${COVERAGE}" == "--coverage" ] && [ "${CHANGED}" == "" ] && [ "${test}" == "sanity/1" ]; then
             stub="--stub"
         else
             stub=""
@@ -86,7 +86,7 @@ function cleanup
         cp -a test/results/reports/coverage=*.xml shippable/codecoverage/
 
         # upload coverage report to codecov.io only when using complete on-demand coverage
-        if [ "${COVERAGE}" ] && [ "${CHANGED}" == "" ]; then
+        if [ "${COVERAGE}" == "--coverage" ] && [ "${CHANGED}" == "" ]; then
             for file in test/results/reports/coverage=*.xml; do
                 flags="${file##*/coverage=}"
                 flags="${flags%.xml}"
@@ -116,7 +116,7 @@ function cleanup
 
 trap cleanup EXIT
 
-if [[ "${COVERAGE:-}" ]]; then
+if [[ "${COVERAGE:-}" == "--coverage" ]]; then
     timeout=60
 else
     timeout=45
diff --git a/test/utils/shippable/units.sh b/test/utils/shippable/units.sh
index 11d1a792ca..89712811c6 100755
--- a/test/utils/shippable/units.sh
+++ b/test/utils/shippable/units.sh
@@ -7,7 +7,7 @@ IFS='/:' read -ra args <<< "$1"
 
 version="${args[1]}"
 
-if [[ "${COVERAGE:-}" ]]; then
+if [[ "${COVERAGE:-}" == "--coverage" ]]; then
     timeout=90
 else
     timeout=10