summaryrefslogtreecommitdiff
path: root/test/utils
diff options
context:
space:
mode:
authorMatt Clay <matt@mystile.com>2017-05-14 15:04:52 +0800
committerMatt Clay <matt@mystile.com>2017-05-14 18:27:58 +0800
commitfdcad0f44d0a10470be7270196765bf1f46864d4 (patch)
tree0388fba08cf74087bf0b351c8146fef1410520ef /test/utils
parent5cb5228cde65d33744b91f6099d92368d15cf0b7 (diff)
downloadansible-fdcad0f44d0a10470be7270196765bf1f46864d4.tar.gz
Improve on-demand coverage for Shippable.
Diffstat (limited to 'test/utils')
-rwxr-xr-xtest/utils/shippable/cloud.sh2
-rwxr-xr-xtest/utils/shippable/freebsd.sh3
-rwxr-xr-xtest/utils/shippable/linux.sh2
-rwxr-xr-xtest/utils/shippable/network.sh12
-rwxr-xr-xtest/utils/shippable/osx.sh4
-rwxr-xr-xtest/utils/shippable/other.sh11
-rwxr-xr-xtest/utils/shippable/shippable.sh22
-rwxr-xr-xtest/utils/shippable/units.sh3
-rwxr-xr-xtest/utils/shippable/windows.sh7
9 files changed, 47 insertions, 19 deletions
diff --git a/test/utils/shippable/cloud.sh b/test/utils/shippable/cloud.sh
index 918cbbdb4f..647f6f834a 100755
--- a/test/utils/shippable/cloud.sh
+++ b/test/utils/shippable/cloud.sh
@@ -9,4 +9,4 @@ image="ansible/ansible:${args[1]}"
target="posix/ci/cloud/"
# shellcheck disable=SC2086
-ansible-test integration --color -v --retry-on-error "${target}" --docker "${image}" ${COVERAGE:+"$COVERAGE"}
+ansible-test integration --color -v --retry-on-error "${target}" --docker "${image}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
diff --git a/test/utils/shippable/freebsd.sh b/test/utils/shippable/freebsd.sh
index 7b255194e3..1410abeddf 100755
--- a/test/utils/shippable/freebsd.sh
+++ b/test/utils/shippable/freebsd.sh
@@ -10,4 +10,5 @@ version="${args[1]}"
target="posix/ci/"
# shellcheck disable=SC2086
-ansible-test integration --color -v --retry-on-error "${target}" --remote "${platform}/${version}" --exclude "posix/ci/cloud/" ${COVERAGE:+"$COVERAGE"}
+ansible-test integration --color -v --retry-on-error "${target}" --remote "${platform}/${version}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
+ --exclude "posix/ci/cloud/"
diff --git a/test/utils/shippable/linux.sh b/test/utils/shippable/linux.sh
index d4f28ac67b..c42af17941 100755
--- a/test/utils/shippable/linux.sh
+++ b/test/utils/shippable/linux.sh
@@ -9,4 +9,4 @@ image="ansible/ansible:${args[1]}"
target="posix/ci/group${args[2]}/"
# shellcheck disable=SC2086
-ansible-test integration --color -v --retry-on-error "${target}" --docker "${image}" ${COVERAGE:+"$COVERAGE"}
+ansible-test integration --color -v --retry-on-error "${target}" --docker "${image}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
diff --git a/test/utils/shippable/network.sh b/test/utils/shippable/network.sh
index 9cf076f7d7..e54a08d43d 100755
--- a/test/utils/shippable/network.sh
+++ b/test/utils/shippable/network.sh
@@ -2,7 +2,13 @@
set -o pipefail
-ansible-test network-integration --explain 2>&1 | { grep ' network-integration: .* (targeted)$' || true; } > /tmp/network.txt
+# shellcheck disable=SC2086
+ansible-test network-integration --explain ${CHANGED:+"$CHANGED"} 2>&1 | { grep ' network-integration: .* (targeted)$' || true; } > /tmp/network.txt
+
+if [ "${COVERAGE}" ]; then
+ # when on-demand coverage is enabled, force tests to run for all network platforms
+ echo "coverage" > /tmp/network.txt
+fi
target="network/ci/"
@@ -13,7 +19,7 @@ if [ -s /tmp/network.txt ]; then
echo "Running network integration tests for multiple platforms concurrently."
# shellcheck disable=SC2086
- ansible-test network-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} \
+ ansible-test network-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
--platform vyos/1.1.0 \
--platform ios/csr1000v \
@@ -22,6 +28,6 @@ else
echo "Running network integration tests for a single platform only."
# shellcheck disable=SC2086
- ansible-test network-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} \
+ ansible-test network-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
--platform vyos/1.1.0
fi
diff --git a/test/utils/shippable/osx.sh b/test/utils/shippable/osx.sh
index 872f1dd560..d66112c7f2 100755
--- a/test/utils/shippable/osx.sh
+++ b/test/utils/shippable/osx.sh
@@ -10,5 +10,5 @@ version="${args[1]}"
target="posix/ci/"
# shellcheck disable=SC2086
-ansible-test integration --color -v --retry-on-error "${target}" --remote "${platform}/${version}" --remote-terminate success --exclude "posix/ci/cloud/" \
- ${COVERAGE:+"$COVERAGE"}
+ansible-test integration --color -v --retry-on-error "${target}" --remote "${platform}/${version}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
+ --exclude "posix/ci/cloud/" --remote-terminate always
diff --git a/test/utils/shippable/other.sh b/test/utils/shippable/other.sh
index 3ca5b6a53c..d42d671851 100755
--- a/test/utils/shippable/other.sh
+++ b/test/utils/shippable/other.sh
@@ -10,13 +10,16 @@ retry.py pip install tox --disable-pip-version-check
echo '{"verified": false, "results": []}' > test/results/bot/ansible-test-failure.json
-ansible-test compile --failure-ok --color -v --junit --requirements --coverage
-ansible-test sanity --failure-ok --color -v --junit --tox --skip-test ansible-doc --python 3.5 --coverage
-ansible-test sanity --failure-ok --color -v --junit --tox --test ansible-doc --coverage
+# shellcheck disable=SC2086
+ansible-test compile --failure-ok --color -v --junit --requirements --coverage ${CHANGED:+"$CHANGED"}
+# shellcheck disable=SC2086
+ansible-test sanity --failure-ok --color -v --junit --tox --skip-test ansible-doc --python 3.5 --coverage ${CHANGED:+"$CHANGED"}
+# shellcheck disable=SC2086
+ansible-test sanity --failure-ok --color -v --junit --tox --test ansible-doc --coverage ${CHANGED:+"$CHANGED"}
rm test/results/bot/ansible-test-failure.json
if find test/results/bot/ -mindepth 1 -name '.*' -prune -o -print -quit | grep -q .; then
- echo "One or more of the above ansible-test commands recorded at least one test failure."
+ echo "One or more of the above tests reported at least one failure."
exit 1
fi
diff --git a/test/utils/shippable/shippable.sh b/test/utils/shippable/shippable.sh
index 516c2dc881..e330154aa2 100755
--- a/test/utils/shippable/shippable.sh
+++ b/test/utils/shippable/shippable.sh
@@ -23,11 +23,27 @@ pip list --disable-pip-version-check
export PATH="test/runner:${PATH}"
export PYTHONIOENCODING='utf-8'
-export COVERAGE="${COVERAGE:-}"
-# run integration coverage if 'ci_coverage' is in the commit message or the COVERAGE var is non-empty
-if [[ "${COMMIT_MESSAGE}" =~ ci_coverage ]] || [ -n "${COVERAGE}" ]; then
+if [ -n "${COVERAGE:-}" ]; then
+ # on-demand coverage reporting triggered by setting the COVERAGE environment variable to a non-empty value
export COVERAGE="--coverage"
+elif [[ "${COMMIT_MESSAGE}" =~ ci_coverage ]]; then
+ # on-demand coverage reporting triggered by having 'ci_coverage' in the latest commit message
+ export COVERAGE="--coverage"
+else
+ # on-demand coverage reporting disabled (default behavior, always-on coverage reporting remains enabled)
+ export COVERAGE=""
+fi
+
+if [ -n "${COMPLETE:-}" ]; then
+ # disable change detection triggered by setting the COMPLETE environment variable to a non-empty value
+ export CHANGED=""
+elif [[ "${COMMIT_MESSAGE}" =~ ci_complete ]]; then
+ # disable change detection triggered by having 'ci_complete' in the latest commit message
+ export CHANGED=""
+else
+ # enable change detection (default behavior)
+ export CHANGED="--changed"
fi
# remove empty core/extras module directories from PRs created prior to the repo-merge
diff --git a/test/utils/shippable/units.sh b/test/utils/shippable/units.sh
index 1e57ba1a97..74332762f9 100755
--- a/test/utils/shippable/units.sh
+++ b/test/utils/shippable/units.sh
@@ -9,4 +9,5 @@ version="${args[1]}"
retry.py pip install tox --disable-pip-version-check
-ansible-test units --color -v --tox --coverage --python "${version}"
+# shellcheck disable=SC2086
+ansible-test units --color -v --tox --python "${version}" --coverage ${CHANGED:+"$CHANGED"} \
diff --git a/test/utils/shippable/windows.sh b/test/utils/shippable/windows.sh
index 0276733ddc..34fd22d3d9 100755
--- a/test/utils/shippable/windows.sh
+++ b/test/utils/shippable/windows.sh
@@ -7,7 +7,8 @@ IFS='/:' read -ra args <<< "${TEST}"
job="${args[1]}"
-ansible-test windows-integration --explain 2>&1 | { grep ' windows-integration: .* (targeted)$' || true; } > /tmp/windows.txt
+# shellcheck disable=SC2086
+ansible-test windows-integration --explain ${CHANGED:+"$CHANGED"} 2>&1 | { grep ' windows-integration: .* (targeted)$' || true; } > /tmp/windows.txt
if [ -s /tmp/windows.txt ]; then
echo "Detected changes requiring integration tests specific to Windows:"
@@ -23,7 +24,7 @@ if [ -s /tmp/windows.txt ]; then
target="windows/ci/"
# shellcheck disable=SC2086
- ansible-test windows-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} \
+ ansible-test windows-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
--windows 2008-SP2 \
--windows 2008-R2_SP1 \
--windows 2012-RTM \
@@ -36,6 +37,6 @@ else
target="windows/ci/group${job}/"
# shellcheck disable=SC2086
- ansible-test windows-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} \
+ ansible-test windows-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
--windows 2012-R2_RTM
fi