SERVER-65581 generate report files for linters

3 files changed, 212 insertions, 1 deletions

diff --git a/buildscripts/simple_report.py b/buildscripts/simple_report.py
new file mode 100644
index 00000000000..4364530eed8
--- /dev/null
+++ b/buildscripts/simple_report.py
@@ -0,0 +1,102 @@
+"""Given a test name, path to log file and exit code, generate/append an Evergreen report.json."""
+import json
+import pathlib
+import os
+from typing import List, Dict, Optional
+from typing_extensions import TypedDict
+import click
+
+
+class Result(TypedDict, total=False):
+    """Evergreen test result."""
+
+    status: str
+    exit_code: int
+    test_file: str
+    start: int
+    end: int
+    elapsed: int
+    log_raw: str
+
+
+class Report(TypedDict):
+    """Evergreen report."""
+
+    failures: int
+    results: List[Result]
+
+
+def _open_and_truncate_log_lines(log_file: pathlib.Path) -> List[str]:
+    with open(log_file) as fh:
+        lines = fh.read().splitlines()
+        for i, line in enumerate(lines):
+            if line == "scons: done reading SConscript files.":
+                offset = i
+                # if possible, also shave off the current and next line
+                # as they contain:
+                # scons: done reading SConscript files.
+                # scons: Building targets ...
+                # which is superfluous.
+                if len(lines) > i + 2:
+                    offset = i + 2
+                return lines[offset:]
+
+        return lines
+
+
+def _clean_log_file(log_file: pathlib.Path, dedup_lines: bool) -> str:
+    lines = _open_and_truncate_log_lines(log_file)
+    if dedup_lines:
+        lines = _dedup_lines(lines)
+    return os.linesep.join(lines)
+
+
+def _make_report(test_name: str, log_file_contents: str, exit_code: int) -> Report:
+    status = "pass" if exit_code == 0 else "fail"
+    return Report({
+        'failures':
+            0 if exit_code == 0 else 1, "results": [
+                Result({
+                    "status": status, "exit_code": exit_code, "test_file": test_name,
+                    "log_raw": log_file_contents
+                })
+            ]
+    })
+
+
+def _try_combine_reports(out: Report):
+    try:
+        with open("report.json") as fh:
+            report = json.load(fh)
+            out["results"] += report["results"]
+            out["failures"] += report["failures"]
+    except NameError:
+        pass
+    except IOError:
+        pass
+
+
+def _dedup_lines(lines: List[str]) -> List[str]:
+    return list(set(lines))
+
+
+def _put_report(out: Report):
+    with open("report.json", "w") as fh:
+        json.dump(out, fh)
+
+
+@click.command()
+@click.option("--test-name", required=True, type=str)
+@click.option("--log-file", required=True, type=pathlib.Path)
+@click.option("--exit-code", required=True, type=int)
+@click.option("--dedup-lines", is_flag=True)
+def main(test_name: str, log_file: pathlib.Path, exit_code: int, dedup_lines: bool):
+    """Given a test name, path to log file and exit code, generate/append an Evergreen report.json."""
+    log_file_contents = _clean_log_file(log_file, dedup_lines)
+    report = _make_report(test_name, log_file_contents, exit_code)
+    _try_combine_reports(report)
+    _put_report(report)
+
+
+if __name__ == "__main__":
+    main()  # pylint: disable=no-value-for-parameter
diff --git a/buildscripts/tests/test_simple_report.py b/buildscripts/tests/test_simple_report.py
new file mode 100644
index 00000000000..456a9d9dc47
--- /dev/null
+++ b/buildscripts/tests/test_simple_report.py
@@ -0,0 +1,109 @@
+# pylint: disable=missing-function-docstring,missing-class-docstring
+"""Simple_report test."""
+import unittest
+import random
+import textwrap
+import sys
+import os
+from unittest.mock import patch, mock_open
+
+from click.testing import CliRunner
+
+import buildscripts.simple_report
+
+
+def ns(name):  # pylint: disable=invalid-name
+    return f"buildscripts.simple_report.{name}"
+
+
+def fix_newlines(string: str) -> str:
+    # These need to be CRLF newlines on Windows, so we split and rejoin
+    # on os.linesep to fix that
+    return os.linesep.join(string.splitlines())
+
+
+class TestSimpleReport(unittest.TestCase):
+    def __init__(self, *args, **kwargs):
+        super(TestSimpleReport, self).__init__(*args, **kwargs)
+        self.seed = random.randrange(sys.maxsize)
+        self.rng = random.Random(self.seed)
+
+    def test_trivial_report(self):
+        """Run test 100x with randomly generated error codes."""
+        print(f"TestSimpleReport.test_trivial_report seed: {self.seed}")
+        for _ in range(0, 100):
+            self._test_trivial_report()  # pylint: disable=no-value-for-parameter
+
+    @patch(ns("_try_combine_reports"))
+    @patch(ns("_clean_log_file"))
+    @patch(ns("_put_report"))
+    def _test_trivial_report(self, mock_put_report, mock_clean_log_file, _mock_try_combine_reports):
+        exit_code = self.rng.randint(0, 254)
+        print(f"Trying exit code: {exit_code}")
+        mock_clean_log_file.return_value = "I'm a little test log, short and stdout."
+        runner = CliRunner()
+        result = runner.invoke(
+            buildscripts.simple_report.main,
+            ["--test-name", "potato", "--log-file", "test.log", "--exit-code",
+             str(exit_code)])
+        report = mock_put_report.call_args[0][0]
+        results = mock_put_report.call_args[0][0]["results"]
+        if exit_code == 0:
+            self.assertEqual(results[0]["status"], "pass")
+            self.assertEqual(report["failures"], 0)
+        else:
+            self.assertEqual(results[0]["status"], "fail")
+            self.assertEqual(report["failures"], 1)
+        self.assertEqual(result.exit_code, 0)
+
+    @patch(ns("_try_combine_reports"))
+    @patch(ns("_put_report"))
+    def test_truncate_scons(self, mock_put_report, _mock_try_combine_reports):
+        exit_code = 0
+        data = fix_newlines(
+            textwrap.dedent("""\
+TO BE TRUNCATED
+TO BE TRUNCATED
+TO BE TRUNCATED
+TO BE TRUNCATED
+scons: done reading SConscript files.
+scons: Building targets ...
+interesting part"""))
+
+        with patch("builtins.open", mock_open(read_data=data)) as _mock_file:
+            runner = CliRunner()
+            result = runner.invoke(
+                buildscripts.simple_report.main,
+                ["--test-name", "potato", "--log-file", "test.log", "--exit-code",
+                 str(exit_code)])
+        report = mock_put_report.call_args[0][0]
+        results = mock_put_report.call_args[0][0]["results"]
+        self.assertEqual(results[0]["status"], "pass")
+        self.assertEqual(results[0]["log_raw"], "interesting part")
+        self.assertEqual(report["failures"], 0)
+        self.assertEqual(result.exit_code, 0)
+
+    @patch(ns("_try_combine_reports"))
+    @patch(ns("_put_report"))
+    def test_non_scons_log(self, mock_put_report, _mock_try_combine_reports):
+        exit_code = 0
+        data = fix_newlines(
+            textwrap.dedent("""\
+*NOT* TO BE TRUNCATED
+*NOT* TO BE TRUNCATED
+*NOT* TO BE TRUNCATED
+*NOT* TO BE TRUNCATED
+interesting part"""))
+
+        with patch("builtins.open", mock_open(read_data=data)) as _mock_file:
+            runner = CliRunner()
+            result = runner.invoke(
+                buildscripts.simple_report.main,
+                ["--test-name", "potato", "--log-file", "test.log", "--exit-code",
+                 str(exit_code)])
+        report = mock_put_report.call_args[0][0]
+        results = mock_put_report.call_args[0][0]["results"]
+        self.assertEqual(results[0]["status"], "pass")
+        self.assertEqual(results[0]["log_raw"], data)
+        self.assertEqual(report["failures"], 0)
+        self.assertEqual(result.exit_code, 0)
diff --git a/buildscripts/yamllinters.sh b/buildscripts/yamllinters.sh
index 0867b61f299..8358ac2b9c7 100755
--- a/buildscripts/yamllinters.sh
+++ b/buildscripts/yamllinters.sh
@@ -3,7 +3,7 @@ set -o errexit
 BASEDIR=$(dirname "$0")
 cd "$BASEDIR/../"
 
-find buildscripts etc jstests -name '*.y*ml' -exec yamllint -c etc/yamllint_config.yml {} +
+yamllint -c etc/yamllint_config.yml buildscripts etc jstests
 
 # TODO: SERVER-64923 re-enable YAML linters.
 #evergreen evaluate ${evergreen_config_file_path} > etc/evaluated_evergreen.yml