SERVER-29061 Add buildscripts/promote_silent_failures.py script to convert silent

test failures into non-silent test failures

3 files changed, 96 insertions, 13 deletions

diff --git a/buildscripts/promote_silent_failures.py b/buildscripts/promote_silent_failures.py
new file mode 100644
index 00000000000..2865fa5aa96
--- /dev/null
+++ b/buildscripts/promote_silent_failures.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+"""
+Converts silent test failures into non-silent failures.
+
+Any test files with at least 2 executions in the report.json file that have a "silentfail" status,
+this script will change the outputted report to have a "fail" status instead.
+"""
+
+from __future__ import absolute_import
+from __future__ import print_function
+
+import collections
+import json
+import optparse
+import os
+import sys
+
+
+# Get relative imports to work when the package is not installed on the PYTHONPATH.
+if __name__ == "__main__" and __package__ is None:
+    sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+    from buildscripts.resmokelib.testing import report
+
+
+def read_json_file(json_file):
+    with open(json_file) as json_data:
+        return json.load(json_data)
+
+
+def main():
+
+    usage = "usage: %prog [options] report.json"
+    parser = optparse.OptionParser(usage=usage)
+    parser.add_option("-o", "--output-file",
+                      dest="outfile",
+                      default="-",
+                      help="If '-', then the report file is written to stdout."
+                           " Any other value is treated as the output file name. By default,"
+                           " output is written to stdout.")
+
+    (options, args) = parser.parse_args()
+
+    if len(args) != 1:
+        parser.error("Requires a single report.json file.")
+
+    report_file_json = read_json_file(args[0])
+    test_report = report.TestReport.from_dict(report_file_json)
+
+    # Count number of "silentfail" per test file.
+    status_dict = collections.defaultdict(int)
+    for test_info in test_report.test_infos:
+        if test_info.status == "silentfail":
+            status_dict[test_info.test_id] += 1
+
+    # For test files with more than 1 "silentfail", convert status to "fail".
+    for test_info in test_report.test_infos:
+        if status_dict[test_info.test_id] >= 2:
+            test_info.status = "fail"
+
+    result_report = test_report.as_dict();
+    if options.outfile != "-":
+        with open(options.outfile, "w") as fp:
+            json.dump(result_report, fp)
+    else:
+        print(json.dumps(result_report))
+
+if __name__ == "__main__":
+    main()
diff --git a/buildscripts/resmokelib/reportfile.py b/buildscripts/resmokelib/reportfile.py
index 716f90d8cae..918e99af306 100644
--- a/buildscripts/resmokelib/reportfile.py
+++ b/buildscripts/resmokelib/reportfile.py
@@ -23,6 +23,6 @@ def write(suites):
     for suite in suites:
         reports.extend(suite.test_group.get_reports())
 
-    combined_report_dict = _report.TestReport.combine(*reports).as_dict()
+    combined_report_dict = _report.TestReport.combine(*reports).as_dict(convert_failures=True)
     with open(config.REPORT_FILE, "w") as fp:
         json.dump(combined_report_dict, fp)
diff --git a/buildscripts/resmokelib/testing/report.py b/buildscripts/resmokelib/testing/report.py
index e4145affc12..088d87782a5 100644
--- a/buildscripts/resmokelib/testing/report.py
+++ b/buildscripts/resmokelib/testing/report.py
@@ -249,7 +249,8 @@ class TestReport(unittest.TestResult):
         """
 
         with self._lock:
-            return [test_info for test_info in self.test_infos if test_info.status == "fail"]
+            return [test_info for test_info in self.test_infos
+                    if test_info.status in ("fail", "silentfail")]
 
     def get_errored(self):
         """
@@ -269,27 +270,31 @@ class TestReport(unittest.TestResult):
         with self._lock:
             return [test_info for test_info in self.test_infos if test_info.status == "timeout"]
 
-    def as_dict(self):
+    def as_dict(self, convert_failures=False):
         """
         Return the test result information as a dictionary.
 
         Used to create the report.json file.
+
+        If 'convert_failures' is true, then "error" and "fail" test statuses are replaced with
+        _config.REPORT_FAILURE_STATUS in the returned dictionary.
         """
 
         results = []
         with self._lock:
             for test_info in self.test_infos:
                 status = test_info.status
-                if status == "error":
-                    # Don't distinguish between failures and errors.
-                    status = _config.REPORT_FAILURE_STATUS
-                elif status == "timeout":
-                    # Until EVG-1536 is completed, we shouldn't distinguish between failures and
-                    # interrupted tests in the report.json file. In Evergreen, the behavior to sort
-                    # tests with the "timeout" test status after tests with the "pass" test status
-                    # effectively hides interrupted tests from the test results sidebar unless
-                    # sorting by the time taken.
-                    status = "fail"
+                if convert_failures:
+                    if status == "error" or status == "fail":
+                        # Don't distinguish between failures and errors.
+                        status = _config.REPORT_FAILURE_STATUS
+                    elif status == "timeout":
+                        # Until EVG-1536 is completed, we shouldn't distinguish between failures and
+                        # interrupted tests in the report.json file. In Evergreen, the behavior to
+                        # sort tests with the "timeout" test status after tests with the "pass" test
+                        # status effectively hides interrupted tests from the test results sidebar
+                        # unless sorting by the time taken.
+                        status = "fail"
 
                 result = {
                     "test_file": test_info.test_id,
@@ -330,6 +335,16 @@ class TestReport(unittest.TestResult):
             test_info.start_time = result["start"]
             test_info.end_time = result["end"]
             report.test_infos.append(test_info)
+
+            if is_dynamic:
+                report.num_dynamic += 1
+
+        # Update cached values for number of successful and failed tests.
+        report.num_failed = len(report.get_failed())
+        report.num_errored = len(report.get_errored())
+        report.num_interrupted = len(report.get_interrupted())
+        report.num_succeeded = len(report.get_successful())
+
         return report
 
     def reset(self):