diff options
author | Matt Clay <matt@mystile.com> | 2017-05-14 15:04:52 +0800 |
---|---|---|
committer | Matt Clay <matt@mystile.com> | 2017-05-14 18:27:58 +0800 |
commit | fdcad0f44d0a10470be7270196765bf1f46864d4 (patch) | |
tree | 0388fba08cf74087bf0b351c8146fef1410520ef /test/utils | |
parent | 5cb5228cde65d33744b91f6099d92368d15cf0b7 (diff) | |
download | ansible-fdcad0f44d0a10470be7270196765bf1f46864d4.tar.gz |
Improve on-demand coverage for Shippable.
Diffstat (limited to 'test/utils')
-rwxr-xr-x | test/utils/shippable/cloud.sh | 2 | ||||
-rwxr-xr-x | test/utils/shippable/freebsd.sh | 3 | ||||
-rwxr-xr-x | test/utils/shippable/linux.sh | 2 | ||||
-rwxr-xr-x | test/utils/shippable/network.sh | 12 | ||||
-rwxr-xr-x | test/utils/shippable/osx.sh | 4 | ||||
-rwxr-xr-x | test/utils/shippable/other.sh | 11 | ||||
-rwxr-xr-x | test/utils/shippable/shippable.sh | 22 | ||||
-rwxr-xr-x | test/utils/shippable/units.sh | 3 | ||||
-rwxr-xr-x | test/utils/shippable/windows.sh | 7 |
9 files changed, 47 insertions, 19 deletions
diff --git a/test/utils/shippable/cloud.sh b/test/utils/shippable/cloud.sh index 918cbbdb4f..647f6f834a 100755 --- a/test/utils/shippable/cloud.sh +++ b/test/utils/shippable/cloud.sh @@ -9,4 +9,4 @@ image="ansible/ansible:${args[1]}" target="posix/ci/cloud/" # shellcheck disable=SC2086 -ansible-test integration --color -v --retry-on-error "${target}" --docker "${image}" ${COVERAGE:+"$COVERAGE"} +ansible-test integration --color -v --retry-on-error "${target}" --docker "${image}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \ diff --git a/test/utils/shippable/freebsd.sh b/test/utils/shippable/freebsd.sh index 7b255194e3..1410abeddf 100755 --- a/test/utils/shippable/freebsd.sh +++ b/test/utils/shippable/freebsd.sh @@ -10,4 +10,5 @@ version="${args[1]}" target="posix/ci/" # shellcheck disable=SC2086 -ansible-test integration --color -v --retry-on-error "${target}" --remote "${platform}/${version}" --exclude "posix/ci/cloud/" ${COVERAGE:+"$COVERAGE"} +ansible-test integration --color -v --retry-on-error "${target}" --remote "${platform}/${version}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \ + --exclude "posix/ci/cloud/" diff --git a/test/utils/shippable/linux.sh b/test/utils/shippable/linux.sh index d4f28ac67b..c42af17941 100755 --- a/test/utils/shippable/linux.sh +++ b/test/utils/shippable/linux.sh @@ -9,4 +9,4 @@ image="ansible/ansible:${args[1]}" target="posix/ci/group${args[2]}/" # shellcheck disable=SC2086 -ansible-test integration --color -v --retry-on-error "${target}" --docker "${image}" ${COVERAGE:+"$COVERAGE"} +ansible-test integration --color -v --retry-on-error "${target}" --docker "${image}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \ diff --git a/test/utils/shippable/network.sh b/test/utils/shippable/network.sh index 9cf076f7d7..e54a08d43d 100755 --- a/test/utils/shippable/network.sh +++ b/test/utils/shippable/network.sh @@ -2,7 +2,13 @@ set -o pipefail -ansible-test network-integration --explain 2>&1 | { grep ' network-integration: .* (targeted)$' || true; } > /tmp/network.txt +# shellcheck disable=SC2086 +ansible-test network-integration --explain ${CHANGED:+"$CHANGED"} 2>&1 | { grep ' network-integration: .* (targeted)$' || true; } > /tmp/network.txt + +if [ "${COVERAGE}" ]; then + # when on-demand coverage is enabled, force tests to run for all network platforms + echo "coverage" > /tmp/network.txt +fi target="network/ci/" @@ -13,7 +19,7 @@ if [ -s /tmp/network.txt ]; then echo "Running network integration tests for multiple platforms concurrently." # shellcheck disable=SC2086 - ansible-test network-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} \ + ansible-test network-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \ --platform vyos/1.1.0 \ --platform ios/csr1000v \ @@ -22,6 +28,6 @@ else echo "Running network integration tests for a single platform only." # shellcheck disable=SC2086 - ansible-test network-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} \ + ansible-test network-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \ --platform vyos/1.1.0 fi diff --git a/test/utils/shippable/osx.sh b/test/utils/shippable/osx.sh index 872f1dd560..d66112c7f2 100755 --- a/test/utils/shippable/osx.sh +++ b/test/utils/shippable/osx.sh @@ -10,5 +10,5 @@ version="${args[1]}" target="posix/ci/" # shellcheck disable=SC2086 -ansible-test integration --color -v --retry-on-error "${target}" --remote "${platform}/${version}" --remote-terminate success --exclude "posix/ci/cloud/" \ - ${COVERAGE:+"$COVERAGE"} +ansible-test integration --color -v --retry-on-error "${target}" --remote "${platform}/${version}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \ + --exclude "posix/ci/cloud/" --remote-terminate always diff --git a/test/utils/shippable/other.sh b/test/utils/shippable/other.sh index 3ca5b6a53c..d42d671851 100755 --- a/test/utils/shippable/other.sh +++ b/test/utils/shippable/other.sh @@ -10,13 +10,16 @@ retry.py pip install tox --disable-pip-version-check echo '{"verified": false, "results": []}' > test/results/bot/ansible-test-failure.json -ansible-test compile --failure-ok --color -v --junit --requirements --coverage -ansible-test sanity --failure-ok --color -v --junit --tox --skip-test ansible-doc --python 3.5 --coverage -ansible-test sanity --failure-ok --color -v --junit --tox --test ansible-doc --coverage +# shellcheck disable=SC2086 +ansible-test compile --failure-ok --color -v --junit --requirements --coverage ${CHANGED:+"$CHANGED"} +# shellcheck disable=SC2086 +ansible-test sanity --failure-ok --color -v --junit --tox --skip-test ansible-doc --python 3.5 --coverage ${CHANGED:+"$CHANGED"} +# shellcheck disable=SC2086 +ansible-test sanity --failure-ok --color -v --junit --tox --test ansible-doc --coverage ${CHANGED:+"$CHANGED"} rm test/results/bot/ansible-test-failure.json if find test/results/bot/ -mindepth 1 -name '.*' -prune -o -print -quit | grep -q .; then - echo "One or more of the above ansible-test commands recorded at least one test failure." + echo "One or more of the above tests reported at least one failure." exit 1 fi diff --git a/test/utils/shippable/shippable.sh b/test/utils/shippable/shippable.sh index 516c2dc881..e330154aa2 100755 --- a/test/utils/shippable/shippable.sh +++ b/test/utils/shippable/shippable.sh @@ -23,11 +23,27 @@ pip list --disable-pip-version-check export PATH="test/runner:${PATH}" export PYTHONIOENCODING='utf-8' -export COVERAGE="${COVERAGE:-}" -# run integration coverage if 'ci_coverage' is in the commit message or the COVERAGE var is non-empty -if [[ "${COMMIT_MESSAGE}" =~ ci_coverage ]] || [ -n "${COVERAGE}" ]; then +if [ -n "${COVERAGE:-}" ]; then + # on-demand coverage reporting triggered by setting the COVERAGE environment variable to a non-empty value export COVERAGE="--coverage" +elif [[ "${COMMIT_MESSAGE}" =~ ci_coverage ]]; then + # on-demand coverage reporting triggered by having 'ci_coverage' in the latest commit message + export COVERAGE="--coverage" +else + # on-demand coverage reporting disabled (default behavior, always-on coverage reporting remains enabled) + export COVERAGE="" +fi + +if [ -n "${COMPLETE:-}" ]; then + # disable change detection triggered by setting the COMPLETE environment variable to a non-empty value + export CHANGED="" +elif [[ "${COMMIT_MESSAGE}" =~ ci_complete ]]; then + # disable change detection triggered by having 'ci_complete' in the latest commit message + export CHANGED="" +else + # enable change detection (default behavior) + export CHANGED="--changed" fi # remove empty core/extras module directories from PRs created prior to the repo-merge diff --git a/test/utils/shippable/units.sh b/test/utils/shippable/units.sh index 1e57ba1a97..74332762f9 100755 --- a/test/utils/shippable/units.sh +++ b/test/utils/shippable/units.sh @@ -9,4 +9,5 @@ version="${args[1]}" retry.py pip install tox --disable-pip-version-check -ansible-test units --color -v --tox --coverage --python "${version}" +# shellcheck disable=SC2086 +ansible-test units --color -v --tox --python "${version}" --coverage ${CHANGED:+"$CHANGED"} \ diff --git a/test/utils/shippable/windows.sh b/test/utils/shippable/windows.sh index 0276733ddc..34fd22d3d9 100755 --- a/test/utils/shippable/windows.sh +++ b/test/utils/shippable/windows.sh @@ -7,7 +7,8 @@ IFS='/:' read -ra args <<< "${TEST}" job="${args[1]}" -ansible-test windows-integration --explain 2>&1 | { grep ' windows-integration: .* (targeted)$' || true; } > /tmp/windows.txt +# shellcheck disable=SC2086 +ansible-test windows-integration --explain ${CHANGED:+"$CHANGED"} 2>&1 | { grep ' windows-integration: .* (targeted)$' || true; } > /tmp/windows.txt if [ -s /tmp/windows.txt ]; then echo "Detected changes requiring integration tests specific to Windows:" @@ -23,7 +24,7 @@ if [ -s /tmp/windows.txt ]; then target="windows/ci/" # shellcheck disable=SC2086 - ansible-test windows-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} \ + ansible-test windows-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \ --windows 2008-SP2 \ --windows 2008-R2_SP1 \ --windows 2012-RTM \ @@ -36,6 +37,6 @@ else target="windows/ci/group${job}/" # shellcheck disable=SC2086 - ansible-test windows-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} \ + ansible-test windows-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \ --windows 2012-R2_RTM fi |