Add `--failure-ok` option to `ansible-test`. (#22623)

5 files changed, 26 insertions, 16 deletions

diff --git a/test/runner/lib/executor.py b/test/runner/lib/executor.py
index e2c71fdbec..475a92666b 100644
--- a/test/runner/lib/executor.py
+++ b/test/runner/lib/executor.py
@@ -712,8 +712,13 @@ def command_compile(args):
             failed.append('compile --python %s' % version)
 
     if failed:
-        raise ApplicationError('The %d compile test(s) listed below (out of %d) failed. See error output above for details.\n%s' % (
-            len(failed), total, '\n'.join(failed)))
+        message = 'The %d compile test(s) listed below (out of %d) failed. See error output above for details.\n%s' % (
+            len(failed), total, '\n'.join(failed))
+
+        if args.failure_ok:
+            display.error(message)
+        else:
+            raise ApplicationError(message)
 
 
 def compile_version(args, python_version, include, exclude):
diff --git a/test/runner/lib/sanity.py b/test/runner/lib/sanity.py
index 6c2e33a6f0..618d0282ae 100644
--- a/test/runner/lib/sanity.py
+++ b/test/runner/lib/sanity.py
@@ -114,8 +114,13 @@ def command_sanity(args):
                 failed.append(result.test + options)
 
     if failed:
-        raise ApplicationError('The %d sanity test(s) listed below (out of %d) failed. See error output above for details.\n%s' % (
-            len(failed), total, '\n'.join(failed)))
+        message = 'The %d sanity test(s) listed below (out of %d) failed. See error output above for details.\n%s' % (
+            len(failed), total, '\n'.join(failed))
+
+        if args.failure_ok:
+            display.error(message)
+        else:
+            raise ApplicationError(message)
 
 
 def command_sanity_code_smell(args, _, script):
diff --git a/test/runner/lib/test.py b/test/runner/lib/test.py
index 2d42301308..e7edc6841d 100644
--- a/test/runner/lib/test.py
+++ b/test/runner/lib/test.py
@@ -36,6 +36,7 @@ class TestConfig(EnvironmentConfig):
 
         self.lint = args.lint if 'lint' in args else False  # type: bool
         self.junit = args.junit if 'junit' in args else False  # type: bool
+        self.failure_ok = args.failure_ok if 'failure_ok' in args else False  # type: bool
 
 
 class TestResult(object):
@@ -266,7 +267,7 @@ class TestFailure(TestResult):
         if args.explain:
             return
 
-        with open(path, 'wb') as bot_fd:
+        with open(path, 'w') as bot_fd:
             json.dump(bot_data, bot_fd, indent=4, sort_keys=True)
             bot_fd.write('\n')
 
diff --git a/test/runner/test.py b/test/runner/test.py
index 0afc660bb0..494d210119 100755
--- a/test/runner/test.py
+++ b/test/runner/test.py
@@ -383,6 +383,11 @@ def add_lint(parser):
                         action='store_true',
                         help='write test failures to junit xml files')
 
+    parser.add_argument('--failure-ok',
+                        action='store_true',
+                        help='exit successfully on failed tests after saving results')
+
+
 
 def add_changes(parser, argparse):
     """
diff --git a/test/utils/shippable/other.sh b/test/utils/shippable/other.sh
index 01c51907d3..841b9d159d 100755
--- a/test/utils/shippable/other.sh
+++ b/test/utils/shippable/other.sh
@@ -17,17 +17,11 @@ ln -sf x86_64-linux-gnu-gcc-4.9 /usr/bin/x86_64-linux-gnu-gcc
 
 retry.py pip install tox --disable-pip-version-check
 
-errors=0
+ansible-test compile --failure-ok --color -v --junit --requirements
+ansible-test sanity  --failure-ok --color -v --junit --tox --skip-test ansible-doc --python 2.7
+ansible-test sanity  --failure-ok --color -v --junit --tox --test ansible-doc --coverage
 
-set +e
-
-ansible-test compile --color -v --junit --requirements || ((errors++))
-ansible-test sanity --color -v --junit --tox --skip-test ansible-doc --python 2.7 || ((errors++))
-ansible-test sanity --color -v --junit --tox --test ansible-doc --coverage || ((errors++))
-
-set -e
-
-if [ ${errors} -gt 0 ]; then
-    echo "${errors} of the above ansible-test command(s) failed."
+if find test/results/bot/ -mindepth 1 -name '.*' -prune -o -print -quit | grep -q .; then
+    echo "One or more of the above ansible-test commands recorded at least one test failure."
     exit 1
 fi