Add support for on-demand coverage on Shippable.

Triggered by setting the COVERAGE environment var to a non-empty value.
Alternatively the string `ci_coverage` can be added to a commit message.

8 files changed, 17 insertions, 10 deletions

diff --git a/test/utils/shippable/cloud.sh b/test/utils/shippable/cloud.sh
index 37535cf5e7..9d101db56a 100755
--- a/test/utils/shippable/cloud.sh
+++ b/test/utils/shippable/cloud.sh
@@ -8,4 +8,4 @@ IFS='/:' read -ra args <<< "${TEST}"
 image="ansible/ansible:${args[1]}"
 target="posix/ci/cloud/"
 
-ansible-test integration --color -v --retry-on-error "${target}" --docker "${image}"
+ansible-test integration --color -v --retry-on-error "${target}" --docker "${image}" "${COVERAGE}"
diff --git a/test/utils/shippable/freebsd.sh b/test/utils/shippable/freebsd.sh
index 5af7eab052..ded7ca1f58 100755
--- a/test/utils/shippable/freebsd.sh
+++ b/test/utils/shippable/freebsd.sh
@@ -9,4 +9,4 @@ platform="${args[0]}"
 version="${args[1]}"
 target="posix/ci/"
 
-ansible-test integration --color -v --retry-on-error "${target}" --remote "${platform}/${version}" --exclude "posix/ci/cloud/"
+ansible-test integration --color -v --retry-on-error "${target}" --remote "${platform}/${version}" --exclude "posix/ci/cloud/" "${COVERAGE}"
diff --git a/test/utils/shippable/linux.sh b/test/utils/shippable/linux.sh
index 1d98735c7e..9375e0699e 100755
--- a/test/utils/shippable/linux.sh
+++ b/test/utils/shippable/linux.sh
@@ -8,4 +8,4 @@ IFS='/:' read -ra args <<< "${TEST}"
 image="ansible/ansible:${args[1]}"
 target="posix/ci/group${args[2]}/"
 
-ansible-test integration --color -v --retry-on-error "${target}" --docker "${image}"
+ansible-test integration --color -v --retry-on-error "${target}" --docker "${image}" "${COVERAGE}"
diff --git a/test/utils/shippable/network.sh b/test/utils/shippable/network.sh
index 57892bc4b3..41fb6cbe6a 100755
--- a/test/utils/shippable/network.sh
+++ b/test/utils/shippable/network.sh
@@ -12,7 +12,7 @@ if [ -s /tmp/network.txt ]; then
 
     echo "Running network integration tests for multiple platforms concurrently."
 
-    ansible-test network-integration --color -v --retry-on-error "${target}" --requirements \
+    ansible-test network-integration --color -v --retry-on-error "${target}" --requirements "${COVERAGE}" \
         --platform vyos/1.1.0 \
         --platform ios/csr1000v \
 
@@ -20,6 +20,6 @@ else
     echo "No changes requiring integration tests specific to networking were detected."
     echo "Running network integration tests for a single platform only."
 
-    ansible-test network-integration --color -v --retry-on-error "${target}" --requirements \
+    ansible-test network-integration --color -v --retry-on-error "${target}" --requirements "${COVERAGE}" \
         --platform vyos/1.1.0
 fi
diff --git a/test/utils/shippable/osx.sh b/test/utils/shippable/osx.sh
index 5e6c18c91f..abdc3db6e7 100755
--- a/test/utils/shippable/osx.sh
+++ b/test/utils/shippable/osx.sh
@@ -9,4 +9,5 @@ platform="${args[0]}"
 version="${args[1]}"
 target="posix/ci/"
 
-ansible-test integration --color -v --retry-on-error "${target}" --remote "${platform}/${version}" --remote-terminate success --exclude "posix/ci/cloud/"
+ansible-test integration --color -v --retry-on-error "${target}" --remote "${platform}/${version}" --remote-terminate success --exclude "posix/ci/cloud/" \
+    "${COVERAGE}"
diff --git a/test/utils/shippable/other.sh b/test/utils/shippable/other.sh
index 3af0811913..3ca5b6a53c 100755
--- a/test/utils/shippable/other.sh
+++ b/test/utils/shippable/other.sh
@@ -10,8 +10,8 @@ retry.py pip install tox --disable-pip-version-check
 
 echo '{"verified": false, "results": []}' > test/results/bot/ansible-test-failure.json
 
-ansible-test compile --failure-ok --color -v --junit --requirements
-ansible-test sanity  --failure-ok --color -v --junit --tox --skip-test ansible-doc --python 3.5
+ansible-test compile --failure-ok --color -v --junit --requirements --coverage
+ansible-test sanity  --failure-ok --color -v --junit --tox --skip-test ansible-doc --python 3.5 --coverage
 ansible-test sanity  --failure-ok --color -v --junit --tox --test ansible-doc --coverage
 
 rm test/results/bot/ansible-test-failure.json
diff --git a/test/utils/shippable/shippable.sh b/test/utils/shippable/shippable.sh
index 4dde3766dc..516c2dc881 100755
--- a/test/utils/shippable/shippable.sh
+++ b/test/utils/shippable/shippable.sh
@@ -23,6 +23,12 @@ pip list --disable-pip-version-check
 
 export PATH="test/runner:${PATH}"
 export PYTHONIOENCODING='utf-8'
+export COVERAGE="${COVERAGE:-}"
+
+# run integration coverage if 'ci_coverage' is in the commit message or the COVERAGE var is non-empty
+if [[ "${COMMIT_MESSAGE}" =~ ci_coverage ]] || [ -n "${COVERAGE}" ]; then
+    export COVERAGE="--coverage"
+fi
 
 # remove empty core/extras module directories from PRs created prior to the repo-merge
 find lib/ansible/modules -type d -empty -print -delete
diff --git a/test/utils/shippable/windows.sh b/test/utils/shippable/windows.sh
index 7ac724c8ce..0c7a0dd8cc 100755
--- a/test/utils/shippable/windows.sh
+++ b/test/utils/shippable/windows.sh
@@ -22,7 +22,7 @@ if [ -s /tmp/windows.txt ]; then
 
     target="windows/ci/"
 
-    ansible-test windows-integration --color -v --retry-on-error "${target}" --requirements \
+    ansible-test windows-integration --color -v --retry-on-error "${target}" --requirements "${COVERAGE}" \
         --windows 2008-SP2 \
         --windows 2008-R2_SP1 \
         --windows 2012-RTM \
@@ -34,6 +34,6 @@ else
 
     target="windows/ci/group${job}/"
 
-    ansible-test windows-integration --color -v --retry-on-error "${target}" --requirements \
+    ansible-test windows-integration --color -v --retry-on-error "${target}" --requirements "${COVERAGE}" \
         --windows 2012-R2_RTM
 fi