Use lint/junit output for compile test. (#22395)

* Refactor sanity classes for use in all tests.
* Use lint/junit output for compile test.
* Add missing options for compile test.
* Fix early bailout on requirements install.

6 files changed, 482 insertions, 314 deletions

diff --git a/test/runner/lib/delegation.py b/test/runner/lib/delegation.py
index d2fb861ff8..c6bb140028 100644
--- a/test/runner/lib/delegation.py
+++ b/test/runner/lib/delegation.py
@@ -14,11 +14,14 @@ from lib.executor import (
     IntegrationConfig,
     SubprocessError,
     ShellConfig,
-    TestConfig,
     SanityConfig,
     create_shell_command,
 )
 
+from lib.test import (
+    TestConfig,
+)
+
 from lib.core_ci import (
     AnsibleCoreCI,
 )
diff --git a/test/runner/lib/executor.py b/test/runner/lib/executor.py
index 9c7b1b44cb..e2c71fdbec 100644
--- a/test/runner/lib/executor.py
+++ b/test/runner/lib/executor.py
@@ -3,6 +3,7 @@
 from __future__ import absolute_import, print_function
 
 import os
+import re
 import tempfile
 import time
 import textwrap
@@ -42,6 +43,10 @@ from lib.util import (
     is_shippable,
 )
 
+from lib.test import (
+    TestConfig,
+)
+
 from lib.ansible_util import (
     ansible_environment,
 )
@@ -70,6 +75,13 @@ from lib.classification import (
     categorize_changes,
 )
 
+from lib.test import (
+    TestMessage,
+    TestSuccess,
+    TestFailure,
+    TestSkipped,
+)
+
 SUPPORTED_PYTHON_VERSIONS = (
     '2.6',
     '2.7',
@@ -124,18 +136,18 @@ def install_command_requirements(args):
     if not args.requirements:
         return
 
-    cmd = generate_pip_install(args.command)
-
-    if not cmd:
-        return
+    packages = []
 
     if isinstance(args, TestConfig):
         if args.coverage:
-            cmd += ['coverage']
-
-    if isinstance(args, SanityConfig):
+            packages.append('coverage')
         if args.junit:
-            cmd += ['junit-xml']
+            packages.append('junit-xml')
+
+    cmd = generate_pip_install(args.command, packages)
+
+    if not cmd:
+        return
 
     try:
         run_command(args, cmd)
@@ -163,18 +175,27 @@ def generate_egg_info(args):
     run_command(args, ['python', 'setup.py', 'egg_info'], capture=args.verbosity < 3)
 
 
-def generate_pip_install(command):
+def generate_pip_install(command, packages=None):
     """
     :type command: str
+    :type packages: list[str] | None
     :rtype: list[str] | None
     """
     constraints = 'test/runner/requirements/constraints.txt'
     requirements = 'test/runner/requirements/%s.txt' % command
 
-    if not os.path.exists(requirements) or not os.path.getsize(requirements):
+    options = []
+
+    if os.path.exists(requirements) and os.path.getsize(requirements):
+        options += ['-r', requirements]
+
+    if packages:
+        options += packages
+
+    if not options:
         return None
 
-    return ['pip', 'install', '--disable-pip-version-check', '-r', requirements, '-c', constraints]
+    return ['pip', 'install', '--disable-pip-version-check', '-c', constraints] + options
 
 
 def command_shell(args):
@@ -672,40 +693,91 @@ def command_compile(args):
 
     install_command_requirements(args)
 
-    version_commands = []
+    total = 0
+    failed = []
 
     for version in COMPILE_PYTHON_VERSIONS:
         # run all versions unless version given, in which case run only that version
         if args.python and version != args.python:
             continue
 
-        # optional list of regex patterns to exclude from tests
-        skip_file = 'test/compile/python%s-skip.txt' % version
+        display.info('Compile with Python %s' % version)
 
-        if os.path.exists(skip_file):
-            with open(skip_file, 'r') as skip_fd:
-                skip_paths = skip_fd.read().splitlines()
-        else:
-            skip_paths = []
+        result = compile_version(args, version, include, exclude)
+        result.write(args)
 
-        # augment file exclusions
-        skip_paths += [e.path for e in exclude]
+        total += 1
 
-        skip_paths = sorted(skip_paths)
+        if isinstance(result, TestFailure):
+            failed.append('compile --python %s' % version)
 
-        python = 'python%s' % version
-        cmd = [python, 'test/compile/compile.py']
+    if failed:
+        raise ApplicationError('The %d compile test(s) listed below (out of %d) failed. See error output above for details.\n%s' % (
+            len(failed), total, '\n'.join(failed)))
 
-        if skip_paths:
-            cmd += ['-x', '|'.join(skip_paths)]
 
-        cmd += [target.path if target.path == '.' else './%s' % target.path for target in include]
+def compile_version(args, python_version, include, exclude):
+    """
+    :type args: CompileConfig
+    :type python_version: str
+    :type include: tuple[CompletionTarget]
+    :param exclude: tuple[CompletionTarget]
+    :rtype: TestResult
+    """
+    command = 'compile'
+    test = ''
 
-        version_commands.append((version, cmd))
+    # optional list of regex patterns to exclude from tests
+    skip_file = 'test/compile/python%s-skip.txt' % python_version
 
-    for version, command in version_commands:
-        display.info('Compile with Python %s' % version)
-        run_command(args, command)
+    if os.path.exists(skip_file):
+        with open(skip_file, 'r') as skip_fd:
+            skip_paths = skip_fd.read().splitlines()
+    else:
+        skip_paths = []
+
+    # augment file exclusions
+    skip_paths += [e.path for e in exclude]
+
+    skip_paths = sorted(skip_paths)
+
+    python = 'python%s' % python_version
+    cmd = [python, 'test/compile/compile.py']
+
+    if skip_paths:
+        cmd += ['-x', '|'.join(skip_paths)]
+
+    cmd += [target.path if target.path == '.' else './%s' % target.path for target in include]
+
+    try:
+        stdout, stderr = run_command(args, cmd, capture=True)
+        status = 0
+    except SubprocessError as ex:
+        stdout = ex.stdout
+        stderr = ex.stderr
+        status = ex.status
+
+    if stderr:
+        raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout)
+
+    if args.explain:
+        return TestSkipped(command, test, python_version=python_version)
+
+    pattern = r'^(?P<path>[^:]*):(?P<line>[0-9]+):(?P<column>[0-9]+): (?P<message>.*)$'
+
+    results = [re.search(pattern, line).groupdict() for line in stdout.splitlines()]
+
+    results = [TestMessage(
+        message=r['message'],
+        path=r['path'].replace('./', ''),
+        line=int(r['line']),
+        column=int(r['column']),
+    ) for r in results]
+
+    if results:
+        return TestFailure(command, test, messages=results, python_version=python_version)
+
+    return TestSuccess(command, test, python_version=python_version)
 
 
 def intercept_command(args, cmd, capture=False, env=None, data=None, cwd=None, python_version=None):
@@ -1013,30 +1085,6 @@ class NoTestsForChanges(ApplicationWarning):
         super(NoTestsForChanges, self).__init__('No tests found for detected changes.')
 
 
-class TestConfig(EnvironmentConfig):
-    """Configuration common to all test commands."""
-    def __init__(self, args, command):
-        """
-        :type args: any
-        :type command: str
-        """
-        super(TestConfig, self).__init__(args, command)
-
-        self.coverage = args.coverage  # type: bool
-        self.include = args.include  # type: list [str]
-        self.exclude = args.exclude  # type: list [str]
-        self.require = args.require  # type: list [str]
-
-        self.changed = args.changed  # type: bool
-        self.tracked = args.tracked  # type: bool
-        self.untracked = args.untracked  # type: bool
-        self.committed = args.committed  # type: bool
-        self.staged = args.staged  # type: bool
-        self.unstaged = args.unstaged  # type: bool
-        self.changed_from = args.changed_from  # type: str
-        self.changed_path = args.changed_path  # type: list [str]
-
-
 class ShellConfig(EnvironmentConfig):
     """Configuration for the shell command."""
     def __init__(self, args):
@@ -1057,8 +1105,6 @@ class SanityConfig(TestConfig):
         self.test = args.test  # type: list [str]
         self.skip_test = args.skip_test  # type: list [str]
         self.list_tests = args.list_tests  # type: bool
-        self.lint = args.lint  # type: bool
-        self.junit = args.junit  # type: bool
 
         if args.base_branch:
             self.base_branch = args.base_branch  # str
diff --git a/test/runner/lib/sanity.py b/test/runner/lib/sanity.py
index 59b1b349e9..1ef62c56aa 100644
--- a/test/runner/lib/sanity.py
+++ b/test/runner/lib/sanity.py
@@ -2,7 +2,6 @@
 
 from __future__ import absolute_import, print_function
 
-import datetime
 import glob
 import json
 import os
@@ -41,6 +40,15 @@ from lib.executor import (
     SanityConfig,
 )
 
+from lib.test import (
+    TestSuccess,
+    TestFailure,
+    TestSkipped,
+    TestMessage,
+)
+
+COMMAND = 'sanity'
+
 PEP8_SKIP_PATH = 'test/sanity/pep8/skip.txt'
 PEP8_LEGACY_PATH = 'test/sanity/pep8/legacy-files.txt'
 
@@ -317,7 +325,7 @@ def command_sanity_pep8(args, targets):
         status = ex.status
 
     if stderr:
-        raise SubprocessError(cmd=cmd, status=status, stderr=stderr)
+        raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout)
 
     if args.explain:
         return SanitySkipped(test)
@@ -444,7 +452,7 @@ def command_sanity_yamllint(args, targets):
         status = ex.status
 
     if stderr:
-        raise SubprocessError(cmd=cmd, status=status, stderr=stderr)
+        raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout)
 
     if args.explain:
         return SanitySkipped(test)
@@ -542,143 +550,27 @@ def sanity_get_tests():
     return SANITY_TESTS
 
 
-class SanityResult(object):
-    """Base class for sanity test results."""
-    def __init__(self, test, python_version=None):
-        """
-        :type test: str
-        :type python_version: str
-        """
-        self.test = test
-        self.python_version = python_version
-
-        try:
-            import junit_xml
-        except ImportError:
-            junit_xml = None
-
-        self.junit = junit_xml
-
-    def write(self, args):
-        """
-        :type args: SanityConfig
-        """
-        self.write_console()
-        self.write_bot(args)
-
-        if args.lint:
-            self.write_lint()
-
-        if args.junit:
-            if self.junit:
-                self.write_junit(args)
-            else:
-                display.warning('Skipping junit xml output because the `junit-xml` python package was not found.', unique=True)
-
-    def write_console(self):
-        """Write results to console."""
-        pass
-
-    def write_lint(self):
-        """Write lint results to stdout."""
-        pass
-
-    def write_bot(self, args):
-        """
-        :type args: SanityConfig
-        """
-        pass
-
-    def write_junit(self, args):
-        """
-        :type args: SanityConfig
-        """
-        pass
-
-    def create_path(self, directory, extension):
-        """
-        :type directory: str
-        :type extension: str
-        :rtype: str
-        """
-        path = 'test/results/%s/ansible-test-%s' % (directory, self.test)
-
-        if self.python_version:
-            path += '-python-%s' % self.python_version
-
-        path += extension
-
-        return path
-
-    def save_junit(self, args, test_case, properties=None):
-        """
-        :type args: SanityConfig
-        :type test_case: junit_xml.TestCase
-        :type properties: dict[str, str] | None
-        :rtype: str | None
-        """
-        path = self.create_path('junit', '.xml')
-
-        test_suites = [
-            self.junit.TestSuite(
-                name='ansible-test',
-                test_cases=[test_case],
-                timestamp=datetime.datetime.utcnow().replace(microsecond=0).isoformat(),
-                properties=properties,
-            ),
-        ]
-
-        report = self.junit.TestSuite.to_xml_string(test_suites=test_suites, prettyprint=True, encoding='utf-8')
-
-        if args.explain:
-            return
-
-        with open(path, 'wb') as xml:
-            xml.write(report.encode('utf-8', 'strict'))
-
-
-class SanitySuccess(SanityResult):
+class SanitySuccess(TestSuccess):
     """Sanity test success."""
     def __init__(self, test, python_version=None):
         """
         :type test: str
         :type python_version: str
         """
-        super(SanitySuccess, self).__init__(test, python_version)
-
-    def write_junit(self, args):
-        """
-        :type args: SanityConfig
-        """
-        test_case = self.junit.TestCase(name=self.test)
+        super(SanitySuccess, self).__init__(COMMAND, test, python_version)
 
-        self.save_junit(args, test_case)
 
-
-class SanitySkipped(SanityResult):
+class SanitySkipped(TestSkipped):
     """Sanity test skipped."""
     def __init__(self, test, python_version=None):
         """
         :type test: str
         :type python_version: str
         """
-        super(SanitySkipped, self).__init__(test, python_version)
-
-    def write_console(self):
-        """Write results to console."""
-        display.info('No tests applicable.', verbosity=1)
-
-    def write_junit(self, args):
-        """
-        :type args: SanityConfig
-        """
-        test_case = self.junit.TestCase(name=self.test)
-        test_case.add_skipped_info('No tests applicable.')
-
-        self.save_junit(args, test_case)
+        super(SanitySkipped, self).__init__(COMMAND, test, python_version)
 
 
-class SanityFailure(SanityResult):
+class SanityFailure(TestFailure):
     """Sanity test failure."""
     def __init__(self, test, python_version=None, messages=None, summary=None):
         """
@@ -687,117 +579,10 @@ class SanityFailure(SanityResult):
         :type messages: list[SanityMessage]
         :type summary: str
         """
-        super(SanityFailure, self).__init__(test, python_version)
-
-        self.messages = messages
-        self.summary = summary
-
-    def write_console(self):
-        """Write results to console."""
-        if self.summary:
-            display.error(self.summary)
-        else:
-            display.error('Found %d %s issue(s) which need to be resolved:' % (len(self.messages), self.test))
-
-            for message in self.messages:
-                display.error(message)
-
-    def write_lint(self):
-        """Write lint results to stdout."""
-        if self.summary:
-            command = self.format_command()
-            message = 'The test `%s` failed. See stderr output for details.' % command
-            path = 'test/runner/ansible-test'
-            message = SanityMessage(message, path)
-            print(message)
-        else:
-            for message in self.messages:
-                print(message)
-
-    def write_junit(self, args):
-        """
-        :type args: SanityConfig
-        """
-        title = self.format_title()
-        output = self.format_block()
-
-        test_case = self.junit.TestCase(classname='sanity', name=self.test)
-
-        # Include a leading newline to improve readability on Shippable "Tests" tab.
-        # Without this, the first line becomes indented.
-        test_case.add_failure_info(message=title, output='\n%s' % output)
-
-        self.save_junit(args, test_case)
-
-    def write_bot(self, args):
-        """
-        :type args: SanityConfig
-        """
-        message = self.format_title()
-        output = self.format_block()
-
-        bot_data = dict(
-            results=[
-                dict(
-                    message=message,
-                    output=output,
-                ),
-            ],
-        )
-
-        path = self.create_path('bot', '.json')
-
-        if args.explain:
-            return
-
-        with open(path, 'wb') as bot_fd:
-            json.dump(bot_data, bot_fd, indent=4, sort_keys=True)
-            bot_fd.write('\n')
-
-    def format_command(self):
-        """
-        :rtype: str
-        """
-        command = 'ansible-test sanity --test %s' % self.test
-
-        if self.python_version:
-            command += ' --python %s' % self.python_version
-
-        return command
-
-    def format_title(self):
-        """
-        :rtype: str
-        """
-        command = self.format_command()
-
-        if self.summary:
-            reason = 'error'
-        else:
-            reason = 'error' if len(self.messages) == 1 else 'errors'
-
-        title = 'The test `%s` failed with the following %s:' % (command, reason)
-
-        return title
-
-    def format_block(self):
-        """
-        :rtype: str
-        """
-        if self.summary:
-            block = self.summary
-        else:
-            block = '\n'.join(str(m) for m in self.messages)
+        super(SanityFailure, self).__init__(COMMAND, test, python_version, messages, summary)
 
-        message = block.strip()
 
-        # Hack to remove ANSI color reset code from SubprocessError messages.
-        message = message.replace(display.clear, '')
-
-        return message
-
-
-class SanityMessage(object):
+class SanityMessage(TestMessage):
     """Single sanity test message for one file."""
     def __init__(self, message, path, line=0, column=0, level='error', code=None):
         """
@@ -808,20 +593,7 @@ class SanityMessage(object):
         :type level: str
         :type code: str | None
         """
-        self.path = path
-        self.line = line
-        self.column = column
-        self.level = level
-        self.code = code
-        self.message = message
-
-    def __str__(self):
-        if self.code:
-            msg = '%s %s' % (self.code, self.message)
-        else:
-            msg = self.message
-
-        return '%s:%s:%s: %s' % (self.path, self.line, self.column, msg)
+        super(SanityMessage, self).__init__(message, path, line, column, level, code)
 
 
 class SanityTargets(object):
diff --git a/test/runner/lib/test.py b/test/runner/lib/test.py
new file mode 100644
index 0000000000..2393a2202c
--- /dev/null
+++ b/test/runner/lib/test.py
@@ -0,0 +1,340 @@
+"""Classes for storing and processing test results."""
+
+from __future__ import absolute_import, print_function
+
+import datetime
+import json
+
+from lib.util import (
+    display,
+    EnvironmentConfig,
+)
+
+
+class TestConfig(EnvironmentConfig):
+    """Configuration common to all test commands."""
+    def __init__(self, args, command):
+        """
+        :type args: any
+        :type command: str
+        """
+        super(TestConfig, self).__init__(args, command)
+
+        self.coverage = args.coverage  # type: bool
+        self.include = args.include  # type: list [str]
+        self.exclude = args.exclude  # type: list [str]
+        self.require = args.require  # type: list [str]
+
+        self.changed = args.changed  # type: bool
+        self.tracked = args.tracked  # type: bool
+        self.untracked = args.untracked  # type: bool
+        self.committed = args.committed  # type: bool
+        self.staged = args.staged  # type: bool
+        self.unstaged = args.unstaged  # type: bool
+        self.changed_from = args.changed_from  # type: str
+        self.changed_path = args.changed_path  # type: list [str]
+
+        self.lint = args.lint if 'lint' in args else False  # type: bool
+        self.junit = args.junit if 'junit' in args else False  # type: bool
+
+
+class TestResult(object):
+    """Base class for test results."""
+    def __init__(self, command, test, python_version=None):
+        """
+        :type command: str
+        :type test: str
+        :type python_version: str
+        """
+        self.command = command
+        self.test = test
+        self.python_version = python_version
+        self.name = self.test or self.command
+
+        if self.python_version:
+            self.name += '-python-%s' % self.python_version
+
+        try:
+            import junit_xml
+        except ImportError:
+            junit_xml = None
+
+        self.junit = junit_xml
+
+    def write(self, args):
+        """
+        :type args: TestConfig
+        """
+        self.write_console()
+        self.write_bot(args)
+
+        if args.lint:
+            self.write_lint()
+
+        if args.junit:
+            if self.junit:
+                self.write_junit(args)
+            else:
+                display.warning('Skipping junit xml output because the `junit-xml` python package was not found.', unique=True)
+
+    def write_console(self):
+        """Write results to console."""
+        pass
+
+    def write_lint(self):
+        """Write lint results to stdout."""
+        pass
+
+    def write_bot(self, args):
+        """
+        :type args: TestConfig
+        """
+        pass
+
+    def write_junit(self, args):
+        """
+        :type args: TestConfig
+        """
+        pass
+
+    def create_path(self, directory, extension):
+        """
+        :type directory: str
+        :type extension: str
+        :rtype: str
+        """
+        path = 'test/results/%s/ansible-test-%s' % (directory, self.command)
+
+        if self.test:
+            path += '-%s' % self.test
+
+        if self.python_version:
+            path += '-python-%s' % self.python_version
+
+        path += extension
+
+        return path
+
+    def save_junit(self, args, test_case, properties=None):
+        """
+        :type args: TestConfig
+        :type test_case: junit_xml.TestCase
+        :type properties: dict[str, str] | None
+        :rtype: str | None
+        """
+        path = self.create_path('junit', '.xml')
+
+        test_suites = [
+            self.junit.TestSuite(
+                name='ansible-test',
+                test_cases=[test_case],
+                timestamp=datetime.datetime.utcnow().replace(microsecond=0).isoformat(),
+                properties=properties,
+            ),
+        ]
+
+        report = self.junit.TestSuite.to_xml_string(test_suites=test_suites, prettyprint=True, encoding='utf-8')
+
+        if args.explain:
+            return
+
+        with open(path, 'wb') as xml:
+            xml.write(report.encode('utf-8', 'strict'))
+
+
+class TestSuccess(TestResult):
+    """Test success."""
+    def __init__(self, command, test, python_version=None):
+        """
+        :type command: str
+        :type test: str
+        :type python_version: str
+        """
+        super(TestSuccess, self).__init__(command, test, python_version)
+
+    def write_junit(self, args):
+        """
+        :type args: TestConfig
+        """
+        test_case = self.junit.TestCase(classname=self.command, name=self.name)
+
+        self.save_junit(args, test_case)
+
+
+class TestSkipped(TestResult):
+    """Test skipped."""
+    def __init__(self, command, test, python_version=None):
+        """
+        :type command: str
+        :type test: str
+        :type python_version: str
+        """
+        super(TestSkipped, self).__init__(command, test, python_version)
+
+    def write_console(self):
+        """Write results to console."""
+        display.info('No tests applicable.', verbosity=1)
+
+    def write_junit(self, args):
+        """
+        :type args: TestConfig
+        """
+        test_case = self.junit.TestCase(classname=self.command, name=self.name)
+        test_case.add_skipped_info('No tests applicable.')
+
+        self.save_junit(args, test_case)
+
+
+class TestFailure(TestResult):
+    """Test failure."""
+    def __init__(self, command, test, python_version=None, messages=None, summary=None):
+        """
+        :type command: str
+        :type test: str
+        :type python_version: str
+        :type messages: list[TestMessage]
+        :type summary: str
+        """
+        super(TestFailure, self).__init__(command, test, python_version)
+
+        self.messages = messages
+        self.summary = summary
+
+    def write_console(self):
+        """Write results to console."""
+        if self.summary:
+            display.error(self.summary)
+        else:
+            if self.python_version:
+                specifier = ' on python %s' % self.python_version
+            else:
+                specifier = ''
+
+            display.error('Found %d %s issue(s)%s which need to be resolved:' % (len(self.messages), self.test or self.command, specifier))
+
+            for message in self.messages:
+                display.error(message)
+
+    def write_lint(self):
+        """Write lint results to stdout."""
+        if self.summary:
+            command = self.format_command()
+            message = 'The test `%s` failed. See stderr output for details.' % command
+            path = 'test/runner/ansible-test'
+            message = TestMessage(message, path)
+            print(message)
+        else:
+            for message in self.messages:
+                print(message)
+
+    def write_junit(self, args):
+        """
+        :type args: TestConfig
+        """
+        title = self.format_title()
+        output = self.format_block()
+
+        test_case = self.junit.TestCase(classname=self.command, name=self.name)
+
+        # Include a leading newline to improve readability on Shippable "Tests" tab.
+        # Without this, the first line becomes indented.
+        test_case.add_failure_info(message=title, output='\n%s' % output)
+
+        self.save_junit(args, test_case)
+
+    def write_bot(self, args):
+        """
+        :type args: TestConfig
+        """
+        message = self.format_title()
+        output = self.format_block()
+
+        bot_data = dict(
+            results=[
+                dict(
+                    message=message,
+                    output=output,
+                ),
+            ],
+        )
+
+        path = self.create_path('bot', '.json')
+
+        if args.explain:
+            return
+
+        with open(path, 'wb') as bot_fd:
+            json.dump(bot_data, bot_fd, indent=4, sort_keys=True)
+            bot_fd.write('\n')
+
+    def format_command(self):
+        """
+        :rtype: str
+        """
+        command = 'ansible-test %s' % self.command
+
+        if self.test:
+            command += ' --test %s' % self.test
+
+        if self.python_version:
+            command += ' --python %s' % self.python_version
+
+        return command
+
+    def format_title(self):
+        """
+        :rtype: str
+        """
+        command = self.format_command()
+
+        if self.summary:
+            reason = 'error'
+        else:
+            reason = 'error' if len(self.messages) == 1 else 'errors'
+
+        title = 'The test `%s` failed with the following %s:' % (command, reason)
+
+        return title
+
+    def format_block(self):
+        """
+        :rtype: str
+        """
+        if self.summary:
+            block = self.summary
+        else:
+            block = '\n'.join(str(m) for m in self.messages)
+
+        message = block.strip()
+
+        # Hack to remove ANSI color reset code from SubprocessError messages.
+        message = message.replace(display.clear, '')
+
+        return message
+
+
+class TestMessage(object):
+    """Single test message for one file."""
+    def __init__(self, message, path, line=0, column=0, level='error', code=None):
+        """
+        :type message: str
+        :type path: str
+        :type line: int
+        :type column: int
+        :type level: str
+        :type code: str | None
+        """
+        self.path = path
+        self.line = line
+        self.column = column
+        self.level = level
+        self.code = code
+        self.message = message
+
+    def __str__(self):
+        if self.code:
+            msg = '%s %s' % (self.code, self.message)
+        else:
+            msg = self.message
+
+        return '%s:%s:%s: %s' % (self.path, self.line, self.column, msg)
diff --git a/test/runner/test.py b/test/runner/test.py
index 4b7a8b6227..0afc660bb0 100755
--- a/test/runner/test.py
+++ b/test/runner/test.py
@@ -260,6 +260,7 @@ def parse_args():
                           choices=COMPILE_PYTHON_VERSIONS,
                           help='python version: %s' % ', '.join(COMPILE_PYTHON_VERSIONS))
 
+    add_lint(compiler)
     add_extra_docker_options(compiler, integration=False)
 
     sanity = subparsers.add_parser('sanity',
@@ -286,14 +287,6 @@ def parse_args():
                         action='store_true',
                         help='list available tests')
 
-    sanity.add_argument('--lint',
-                        action='store_true',
-                        help='write lint output to stdout, everything else stderr')
-
-    sanity.add_argument('--junit',
-                        action='store_true',
-                        help='write test failures to junit xml files')
-
     sanity.add_argument('--python',
                         metavar='VERSION',
                         choices=SUPPORTED_PYTHON_VERSIONS,
@@ -302,6 +295,7 @@ def parse_args():
     sanity.add_argument('--base-branch',
                         help=argparse.SUPPRESS)
 
+    add_lint(sanity)
     add_extra_docker_options(sanity, integration=False)
 
     shell = subparsers.add_parser('shell',
@@ -377,6 +371,19 @@ def parse_args():
     return args
 
 
+def add_lint(parser):
+    """
+    :type parser: argparse.ArgumentParser
+    """
+    parser.add_argument('--lint',
+                        action='store_true',
+                        help='write lint output to stdout, everything else stderr')
+
+    parser.add_argument('--junit',
+                        action='store_true',
+                        help='write test failures to junit xml files')
+
+
 def add_changes(parser, argparse):
     """
     :type parser: argparse.ArgumentParser
diff --git a/test/utils/shippable/other.sh b/test/utils/shippable/other.sh
index 35e6a96125..e4a5acbb94 100755
--- a/test/utils/shippable/other.sh
+++ b/test/utils/shippable/other.sh
@@ -19,7 +19,7 @@ retry.py pip install tox --disable-pip-version-check
 
 errors=0
 
-ansible-test compile --color -v || ((errors++))
+ansible-test compile --color -v --junit --requirements || ((errors++))
 ansible-test sanity --color -v --junit --tox --skip-test ansible-doc --python 2.7 || ((errors++))
 ansible-test sanity --color -v --junit --tox --test ansible-doc --coverage || ((errors++))