Replace old shippable.py with new check_matrix.py. (#60022)

This new script does not depend on ansible-test and provides much more robust job matrix testing.

It is also run on every job in the matrix now, to detect issues with jobs being re-run after matrix changes are made.

(cherry picked from commit d3da8e4a5b7c87ea6bb4f1345300ddb0a833a6b2)

5 files changed, 109 insertions, 109 deletions

diff --git a/test/runner/shippable.py b/test/runner/shippable.py
deleted file mode 100755
index 29a485c9ff..0000000000
--- a/test/runner/shippable.py
+++ /dev/null
@@ -1,107 +0,0 @@
-#!/usr/bin/env python
-# PYTHON_ARGCOMPLETE_OK
-"""Verify the current Shippable run has the required number of jobs."""
-
-from __future__ import absolute_import, print_function
-
-# noinspection PyCompatibility
-import argparse
-import errno
-import json
-import os
-import sys
-
-from lib.http import (
-    HttpClient,
-)
-
-from lib.util import (
-    display,
-    ApplicationError,
-    ApplicationWarning,
-    MissingEnvironmentVariable,
-)
-
-
-try:
-    import argcomplete
-except ImportError:
-    argcomplete = None
-
-
-def main():
-    """Main program function."""
-    try:
-        args = parse_args()
-        display.verbosity = args.verbosity
-        display.color = args.color
-
-        try:
-            run_id = os.environ['SHIPPABLE_BUILD_ID']
-        except KeyError as ex:
-            raise MissingEnvironmentVariable(ex.args[0])
-
-        client = HttpClient(args)
-        response = client.get('https://api.shippable.com/jobs?runIds=%s' % run_id)
-        jobs = response.json()
-
-        if not isinstance(jobs, list):
-            raise ApplicationError(json.dumps(jobs, indent=4, sort_keys=True))
-
-        if len(jobs) == 1:
-            raise ApplicationError('Shippable run %s has only one job. Did you use the "Rebuild with SSH" option?' % run_id)
-    except ApplicationWarning as ex:
-        display.warning(str(ex))
-        exit(0)
-    except ApplicationError as ex:
-        display.error(str(ex))
-        exit(1)
-    except KeyboardInterrupt:
-        exit(2)
-    except IOError as ex:
-        if ex.errno == errno.EPIPE:
-            exit(3)
-        raise
-
-
-def parse_args():
-    """Parse command line arguments."""
-    parser = argparse.ArgumentParser()
-
-    parser.add_argument('-e', '--explain',
-                        action='store_true',
-                        help='explain commands that would be executed')
-
-    parser.add_argument('-v', '--verbose',
-                        dest='verbosity',
-                        action='count',
-                        default=0,
-                        help='display more output')
-
-    parser.add_argument('--color',
-                        metavar='COLOR',
-                        nargs='?',
-                        help='generate color output: %(choices)s',
-                        choices=('yes', 'no', 'auto'),
-                        const='yes',
-                        default='auto')
-
-    if argcomplete:
-        argcomplete.autocomplete(parser)
-
-    args = parser.parse_args()
-
-    if args.color == 'yes':
-        args.color = True
-    elif args.color == 'no':
-        args.color = False
-    elif 'SHIPPABLE' in os.environ:
-        args.color = True
-    else:
-        args.color = sys.stdout.isatty()
-
-    return args
-
-
-if __name__ == '__main__':
-    main()
diff --git a/test/sanity/code-smell/replace-urlopen.py b/test/sanity/code-smell/replace-urlopen.py
index 677f9c88a9..0f46ee4493 100755
--- a/test/sanity/code-smell/replace-urlopen.py
+++ b/test/sanity/code-smell/replace-urlopen.py
@@ -12,6 +12,7 @@ def main():
         'lib/ansible/module_utils/urls.py',
         'test/units/module_utils/urls/test_Request.py',
         'test/units/module_utils/urls/test_fetch_url.py',
+        'test/utils/shippable/check_matrix.py',
     ])
 
     for path in sys.argv[1:] or sys.stdin.read().splitlines():
diff --git a/test/utils/shippable/check_matrix.py b/test/utils/shippable/check_matrix.py
new file mode 100755
index 0000000000..99e4ea88ff
--- /dev/null
+++ b/test/utils/shippable/check_matrix.py
@@ -0,0 +1,107 @@
+#!/usr/bin/env python
+"""Verify the currently executing Shippable test matrix matches the one defined in the "shippable.yml" file."""
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import datetime
+import json
+import os
+import re
+import sys
+import time
+
+try:
+    from typing import NoReturn
+except ImportError:
+    NoReturn = None
+
+try:
+    # noinspection PyCompatibility
+    from urllib2 import urlopen  # pylint: disable=ansible-bad-import-from
+except ImportError:
+    # noinspection PyCompatibility
+    from urllib.request import urlopen
+
+
+def main():  # type: () -> None
+    """Main entry point."""
+    with open('shippable.yml', 'rb') as yaml_file:
+        yaml = yaml_file.read().decode('utf-8').splitlines()
+
+    defined_matrix = [match.group(1) for match in [re.search(r'^ *- env: T=(.*)$', line) for line in yaml] if match and match.group(1) != 'none']
+
+    if not defined_matrix:
+        fail('No matrix entries found in the "shippable.yml" file.',
+             'Did you modify the "shippable.yml" file?')
+
+    run_id = os.environ['SHIPPABLE_BUILD_ID']
+    sleep = 1
+    jobs = []
+
+    for attempts_remaining in range(4, -1, -1):
+        try:
+            jobs = json.loads(urlopen('https://api.shippable.com/jobs?runIds=%s' % run_id).read())
+
+            if not isinstance(jobs, list):
+                raise Exception('Shippable run %s data is not a list.' % run_id)
+
+            break
+        except Exception as ex:
+            if not attempts_remaining:
+                fail('Unable to retrieve Shippable run %s matrix.' % run_id,
+                     str(ex))
+
+            sys.stderr.write('Unable to retrieve Shippable run %s matrix: %s\n' % (run_id, ex))
+            sys.stderr.write('Trying again in %d seconds...\n' % sleep)
+            time.sleep(sleep)
+            sleep *= 2
+
+    if len(jobs) != len(defined_matrix):
+        if len(jobs) == 1:
+            hint = '\n\nMake sure you do not use the "Rebuild with SSH" option.'
+        else:
+            hint = ''
+
+        fail('Shippable run %s has %d jobs instead of the expected %d jobs.' % (run_id, len(jobs), len(defined_matrix)),
+             'Try re-running the entire matrix.%s' % hint)
+
+    actual_matrix = dict((job.get('jobNumber'), dict(tuple(line.split('=', 1)) for line in job.get('env', [])).get('T', '')) for job in jobs)
+    errors = [(job_number, test, actual_matrix.get(job_number)) for job_number, test in enumerate(defined_matrix, 1) if actual_matrix.get(job_number) != test]
+
+    if len(errors):
+        error_summary = '\n'.join('Job %s expected "%s" but found "%s" instead.' % (job_number, expected, actual) for job_number, expected, actual in errors)
+
+        fail('Shippable run %s has a job matrix mismatch.' % run_id,
+             'Try re-running the entire matrix.\n\n%s' % error_summary)
+
+
+def fail(message, output):  # type: (str, str) -> NoReturn
+    # Include a leading newline to improve readability on Shippable "Tests" tab.
+    # Without this, the first line becomes indented.
+    output = '\n' + output.strip()
+
+    timestamp = datetime.datetime.utcnow().replace(microsecond=0).isoformat()
+
+    # hack to avoid requiring junit-xml, which isn't pre-installed on Shippable outside our test containers
+    xml = '''
+<?xml version="1.0" encoding="utf-8"?>
+<testsuites disabled="0" errors="1" failures="0" tests="1" time="0.0">
+\t<testsuite disabled="0" errors="1" failures="0" file="None" log="None" name="ansible-test" skipped="0" tests="1" time="0" timestamp="%s" url="None">
+\t\t<testcase classname="timeout" name="timeout">
+\t\t\t<error message="%s" type="error">%s</error>
+\t\t</testcase>
+\t</testsuite>
+</testsuites>
+''' % (timestamp, message, output)
+
+    with open('test/results/junit/check-matrix.xml', 'w') as junit_fd:
+        junit_fd.write(xml.lstrip())
+
+    sys.stderr.write(message + '\n')
+    sys.stderr.write(output + '\n')
+
+    sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/test/utils/shippable/sanity.sh b/test/utils/shippable/sanity.sh
index 734d39c991..ff95d915c1 100755
--- a/test/utils/shippable/sanity.sh
+++ b/test/utils/shippable/sanity.sh
@@ -7,8 +7,6 @@ IFS='/:' read -ra args <<< "$1"
 
 group="${args[1]}"
 
-shippable.py
-
 if [ "${BASE_BRANCH:-}" ]; then
     base_branch="origin/${BASE_BRANCH}"
 else
diff --git a/test/utils/shippable/shippable.sh b/test/utils/shippable/shippable.sh
index 90e0d5f908..795f07bbe9 100755
--- a/test/utils/shippable/shippable.sh
+++ b/test/utils/shippable/shippable.sh
@@ -126,4 +126,5 @@ fi
 
 ansible-test env --dump --show --timeout "${timeout}" --color -v
 
+"test/utils/shippable/check_matrix.py"
 "test/utils/shippable/${script}.sh" "${test}"