diff options
| -rw-r--r-- | buildscripts/promote_silent_failures.py | 68 | ||||
| -rw-r--r-- | buildscripts/resmokelib/reportfile.py | 2 | ||||
| -rw-r--r-- | buildscripts/resmokelib/testing/report.py | 39 |
3 files changed, 96 insertions, 13 deletions
diff --git a/buildscripts/promote_silent_failures.py b/buildscripts/promote_silent_failures.py new file mode 100644 index 00000000000..2865fa5aa96 --- /dev/null +++ b/buildscripts/promote_silent_failures.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python +""" +Converts silent test failures into non-silent failures. + +Any test files with at least 2 executions in the report.json file that have a "silentfail" status, +this script will change the outputted report to have a "fail" status instead. +""" + +from __future__ import absolute_import +from __future__ import print_function + +import collections +import json +import optparse +import os +import sys + + +# Get relative imports to work when the package is not installed on the PYTHONPATH. +if __name__ == "__main__" and __package__ is None: + sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + from buildscripts.resmokelib.testing import report + + +def read_json_file(json_file): + with open(json_file) as json_data: + return json.load(json_data) + + +def main(): + + usage = "usage: %prog [options] report.json" + parser = optparse.OptionParser(usage=usage) + parser.add_option("-o", "--output-file", + dest="outfile", + default="-", + help="If '-', then the report file is written to stdout." + " Any other value is treated as the output file name. By default," + " output is written to stdout.") + + (options, args) = parser.parse_args() + + if len(args) != 1: + parser.error("Requires a single report.json file.") + + report_file_json = read_json_file(args[0]) + test_report = report.TestReport.from_dict(report_file_json) + + # Count number of "silentfail" per test file. + status_dict = collections.defaultdict(int) + for test_info in test_report.test_infos: + if test_info.status == "silentfail": + status_dict[test_info.test_id] += 1 + + # For test files with more than 1 "silentfail", convert status to "fail". + for test_info in test_report.test_infos: + if status_dict[test_info.test_id] >= 2: + test_info.status = "fail" + + result_report = test_report.as_dict(); + if options.outfile != "-": + with open(options.outfile, "w") as fp: + json.dump(result_report, fp) + else: + print(json.dumps(result_report)) + +if __name__ == "__main__": + main() diff --git a/buildscripts/resmokelib/reportfile.py b/buildscripts/resmokelib/reportfile.py index 716f90d8cae..918e99af306 100644 --- a/buildscripts/resmokelib/reportfile.py +++ b/buildscripts/resmokelib/reportfile.py @@ -23,6 +23,6 @@ def write(suites): for suite in suites: reports.extend(suite.test_group.get_reports()) - combined_report_dict = _report.TestReport.combine(*reports).as_dict() + combined_report_dict = _report.TestReport.combine(*reports).as_dict(convert_failures=True) with open(config.REPORT_FILE, "w") as fp: json.dump(combined_report_dict, fp) diff --git a/buildscripts/resmokelib/testing/report.py b/buildscripts/resmokelib/testing/report.py index e4145affc12..088d87782a5 100644 --- a/buildscripts/resmokelib/testing/report.py +++ b/buildscripts/resmokelib/testing/report.py @@ -249,7 +249,8 @@ class TestReport(unittest.TestResult): """ with self._lock: - return [test_info for test_info in self.test_infos if test_info.status == "fail"] + return [test_info for test_info in self.test_infos + if test_info.status in ("fail", "silentfail")] def get_errored(self): """ @@ -269,27 +270,31 @@ class TestReport(unittest.TestResult): with self._lock: return [test_info for test_info in self.test_infos if test_info.status == "timeout"] - def as_dict(self): + def as_dict(self, convert_failures=False): """ Return the test result information as a dictionary. Used to create the report.json file. + + If 'convert_failures' is true, then "error" and "fail" test statuses are replaced with + _config.REPORT_FAILURE_STATUS in the returned dictionary. """ results = [] with self._lock: for test_info in self.test_infos: status = test_info.status - if status == "error": - # Don't distinguish between failures and errors. - status = _config.REPORT_FAILURE_STATUS - elif status == "timeout": - # Until EVG-1536 is completed, we shouldn't distinguish between failures and - # interrupted tests in the report.json file. In Evergreen, the behavior to sort - # tests with the "timeout" test status after tests with the "pass" test status - # effectively hides interrupted tests from the test results sidebar unless - # sorting by the time taken. - status = "fail" + if convert_failures: + if status == "error" or status == "fail": + # Don't distinguish between failures and errors. + status = _config.REPORT_FAILURE_STATUS + elif status == "timeout": + # Until EVG-1536 is completed, we shouldn't distinguish between failures and + # interrupted tests in the report.json file. In Evergreen, the behavior to + # sort tests with the "timeout" test status after tests with the "pass" test + # status effectively hides interrupted tests from the test results sidebar + # unless sorting by the time taken. + status = "fail" result = { "test_file": test_info.test_id, @@ -330,6 +335,16 @@ class TestReport(unittest.TestResult): test_info.start_time = result["start"] test_info.end_time = result["end"] report.test_infos.append(test_info) + + if is_dynamic: + report.num_dynamic += 1 + + # Update cached values for number of successful and failed tests. + report.num_failed = len(report.get_failed()) + report.num_errored = len(report.get_errored()) + report.num_interrupted = len(report.get_interrupted()) + report.num_succeeded = len(report.get_successful()) + return report def reset(self): |