Implement on-demand coverage on Shippable. (#24535)

8 files changed, 25 insertions, 10 deletions

diff --git a/test/utils/shippable/cloud.sh b/test/utils/shippable/cloud.sh
index 37535cf5e7..918cbbdb4f 100755
--- a/test/utils/shippable/cloud.sh
+++ b/test/utils/shippable/cloud.sh
@@ -8,4 +8,5 @@ IFS='/:' read -ra args <<< "${TEST}"
 image="ansible/ansible:${args[1]}"
 target="posix/ci/cloud/"
 
-ansible-test integration --color -v --retry-on-error "${target}" --docker "${image}"
+# shellcheck disable=SC2086
+ansible-test integration --color -v --retry-on-error "${target}" --docker "${image}" ${COVERAGE:+"$COVERAGE"}
diff --git a/test/utils/shippable/freebsd.sh b/test/utils/shippable/freebsd.sh
index 5af7eab052..7b255194e3 100755
--- a/test/utils/shippable/freebsd.sh
+++ b/test/utils/shippable/freebsd.sh
@@ -9,4 +9,5 @@ platform="${args[0]}"
 version="${args[1]}"
 target="posix/ci/"
 
-ansible-test integration --color -v --retry-on-error "${target}" --remote "${platform}/${version}" --exclude "posix/ci/cloud/"
+# shellcheck disable=SC2086
+ansible-test integration --color -v --retry-on-error "${target}" --remote "${platform}/${version}" --exclude "posix/ci/cloud/" ${COVERAGE:+"$COVERAGE"}
diff --git a/test/utils/shippable/linux.sh b/test/utils/shippable/linux.sh
index 1d98735c7e..d4f28ac67b 100755
--- a/test/utils/shippable/linux.sh
+++ b/test/utils/shippable/linux.sh
@@ -8,4 +8,5 @@ IFS='/:' read -ra args <<< "${TEST}"
 image="ansible/ansible:${args[1]}"
 target="posix/ci/group${args[2]}/"
 
-ansible-test integration --color -v --retry-on-error "${target}" --docker "${image}"
+# shellcheck disable=SC2086
+ansible-test integration --color -v --retry-on-error "${target}" --docker "${image}" ${COVERAGE:+"$COVERAGE"}
diff --git a/test/utils/shippable/network.sh b/test/utils/shippable/network.sh
index 57892bc4b3..9cf076f7d7 100755
--- a/test/utils/shippable/network.sh
+++ b/test/utils/shippable/network.sh
@@ -12,7 +12,8 @@ if [ -s /tmp/network.txt ]; then
 
     echo "Running network integration tests for multiple platforms concurrently."
 
-    ansible-test network-integration --color -v --retry-on-error "${target}" --requirements \
+    # shellcheck disable=SC2086
+    ansible-test network-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} \
         --platform vyos/1.1.0 \
         --platform ios/csr1000v \
 
@@ -20,6 +21,7 @@ else
     echo "No changes requiring integration tests specific to networking were detected."
     echo "Running network integration tests for a single platform only."
 
-    ansible-test network-integration --color -v --retry-on-error "${target}" --requirements \
+    # shellcheck disable=SC2086
+    ansible-test network-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} \
         --platform vyos/1.1.0
 fi
diff --git a/test/utils/shippable/osx.sh b/test/utils/shippable/osx.sh
index 5e6c18c91f..872f1dd560 100755
--- a/test/utils/shippable/osx.sh
+++ b/test/utils/shippable/osx.sh
@@ -9,4 +9,6 @@ platform="${args[0]}"
 version="${args[1]}"
 target="posix/ci/"
 
-ansible-test integration --color -v --retry-on-error "${target}" --remote "${platform}/${version}" --remote-terminate success --exclude "posix/ci/cloud/"
+# shellcheck disable=SC2086
+ansible-test integration --color -v --retry-on-error "${target}" --remote "${platform}/${version}" --remote-terminate success --exclude "posix/ci/cloud/" \
+    ${COVERAGE:+"$COVERAGE"}
diff --git a/test/utils/shippable/other.sh b/test/utils/shippable/other.sh
index 3af0811913..3ca5b6a53c 100755
--- a/test/utils/shippable/other.sh
+++ b/test/utils/shippable/other.sh
@@ -10,8 +10,8 @@ retry.py pip install tox --disable-pip-version-check
 
 echo '{"verified": false, "results": []}' > test/results/bot/ansible-test-failure.json
 
-ansible-test compile --failure-ok --color -v --junit --requirements
-ansible-test sanity  --failure-ok --color -v --junit --tox --skip-test ansible-doc --python 3.5
+ansible-test compile --failure-ok --color -v --junit --requirements --coverage
+ansible-test sanity  --failure-ok --color -v --junit --tox --skip-test ansible-doc --python 3.5 --coverage
 ansible-test sanity  --failure-ok --color -v --junit --tox --test ansible-doc --coverage
 
 rm test/results/bot/ansible-test-failure.json
diff --git a/test/utils/shippable/shippable.sh b/test/utils/shippable/shippable.sh
index 4dde3766dc..516c2dc881 100755
--- a/test/utils/shippable/shippable.sh
+++ b/test/utils/shippable/shippable.sh
@@ -23,6 +23,12 @@ pip list --disable-pip-version-check
 
 export PATH="test/runner:${PATH}"
 export PYTHONIOENCODING='utf-8'
+export COVERAGE="${COVERAGE:-}"
+
+# run integration coverage if 'ci_coverage' is in the commit message or the COVERAGE var is non-empty
+if [[ "${COMMIT_MESSAGE}" =~ ci_coverage ]] || [ -n "${COVERAGE}" ]; then
+    export COVERAGE="--coverage"
+fi
 
 # remove empty core/extras module directories from PRs created prior to the repo-merge
 find lib/ansible/modules -type d -empty -print -delete
diff --git a/test/utils/shippable/windows.sh b/test/utils/shippable/windows.sh
index 7ac724c8ce..0276733ddc 100755
--- a/test/utils/shippable/windows.sh
+++ b/test/utils/shippable/windows.sh
@@ -22,7 +22,8 @@ if [ -s /tmp/windows.txt ]; then
 
     target="windows/ci/"
 
-    ansible-test windows-integration --color -v --retry-on-error "${target}" --requirements \
+    # shellcheck disable=SC2086
+    ansible-test windows-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} \
         --windows 2008-SP2 \
         --windows 2008-R2_SP1 \
         --windows 2012-RTM \
@@ -34,6 +35,7 @@ else
 
     target="windows/ci/group${job}/"
 
-    ansible-test windows-integration --color -v --retry-on-error "${target}" --requirements \
+    # shellcheck disable=SC2086
+    ansible-test windows-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} \
         --windows 2012-R2_RTM
 fi