diff options
Diffstat (limited to 'buildscripts')
| -rwxr-xr-x | buildscripts/combine_reports.py | 2 | ||||
| -rw-r--r-- | buildscripts/promote_silent_failures.py | 65 | ||||
| -rw-r--r-- | buildscripts/resmokelib/config.py | 6 | ||||
| -rw-r--r-- | buildscripts/resmokelib/configure_resmoke.py | 1 | ||||
| -rw-r--r-- | buildscripts/resmokelib/run/__init__.py | 45 | ||||
| -rw-r--r-- | buildscripts/resmokelib/testing/report.py | 14 | ||||
| -rw-r--r-- | buildscripts/tests/resmokelib/test_parser.py | 1 |
7 files changed, 5 insertions, 129 deletions
diff --git a/buildscripts/combine_reports.py b/buildscripts/combine_reports.py index 8952a564943..dd5564f4454 100755 --- a/buildscripts/combine_reports.py +++ b/buildscripts/combine_reports.py @@ -25,7 +25,7 @@ def report_exit(combined_test_report): """Return report exit code. The exit code of this script is based on the following: - 0: All tests have status "pass", or only non-dynamic tests have status "silentfail". + 0: All tests have status "pass". 31: At least one test has status "fail" or "timeout". Note: A test can be considered dynamic if its name contains a ":" character. """ diff --git a/buildscripts/promote_silent_failures.py b/buildscripts/promote_silent_failures.py deleted file mode 100644 index d8d45872685..00000000000 --- a/buildscripts/promote_silent_failures.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python3 -"""Convert silent test failures into non-silent failures. - -Any test files with at least 2 executions in the report.json file that have a "silentfail" status, -this script will change the outputted report to have a "fail" status instead. -""" - -import collections -import json -import optparse -import os -import sys - -# Get relative imports to work when the package is not installed on the PYTHONPATH. -if __name__ == "__main__" and __package__ is None: - sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - from buildscripts.resmokelib.testing import report - - -def read_json_file(json_file): - """Return contents of a JSON file.""" - with open(json_file) as json_data: - return json.load(json_data) - - -def main(): - """Execute Main program.""" - - usage = "usage: %prog [options] report.json" - parser = optparse.OptionParser(usage=usage) - parser.add_option( - "-o", "--output-file", dest="outfile", default="-", - help=("If '-', then the report file is written to stdout." - " Any other value is treated as the output file name. By default," - " output is written to stdout.")) - - (options, args) = parser.parse_args() - - if len(args) != 1: - parser.error("Requires a single report.json file.") - - report_file_json = read_json_file(args[0]) - test_report = report.TestReport.from_dict(report_file_json) - - # Count number of "silentfail" per test file. - status_dict = collections.defaultdict(int) - for test_info in test_report.test_infos: - if test_info.evergreen_status == "silentfail": - status_dict[test_info.test_id] += 1 - - # For test files with more than 1 "silentfail", convert status to "fail". - for test_info in test_report.test_infos: - if status_dict[test_info.test_id] >= 2: - test_info.evergreen_status = "fail" - - result_report = test_report.as_dict() - if options.outfile != "-": - with open(options.outfile, "w") as fp: - json.dump(result_report, fp) - else: - print(json.dumps(result_report)) - - -if __name__ == "__main__": - main() diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py index f285c6b21de..1da18223f14 100644 --- a/buildscripts/resmokelib/config.py +++ b/buildscripts/resmokelib/config.py @@ -90,7 +90,6 @@ DEFAULTS = { "repeat_tests_min": None, "repeat_tests_secs": None, "replay_file": None, - "report_failure_status": "fail", "report_file": None, "run_all_feature_flag_tests": False, "run_no_feature_flag_tests": False, @@ -169,7 +168,6 @@ _SuiteOptions = collections.namedtuple("_SuiteOptions", [ "num_repeat_tests_max", "num_repeat_tests_min", "time_repeat_tests_secs", - "report_failure_status", ]) @@ -236,7 +234,6 @@ class SuiteOptions(_SuiteOptions): REPEAT_TESTS_MAX, REPEAT_TESTS_MIN, REPEAT_TESTS_SECS, - REPORT_FAILURE_STATUS, ]))) options = self._asdict() @@ -445,9 +442,6 @@ REPEAT_TESTS_MIN = None # If set, then each test is repeated the specified time (seconds) inside the suites. REPEAT_TESTS_SECS = None -# Controls if the test failure status should be reported as failed or be silently ignored. -REPORT_FAILURE_STATUS = None - # If set, then resmoke.py will write out a report file with the status of each test that ran. REPORT_FILE = None diff --git a/buildscripts/resmokelib/configure_resmoke.py b/buildscripts/resmokelib/configure_resmoke.py index cfb856491d0..bfe59e1befa 100644 --- a/buildscripts/resmokelib/configure_resmoke.py +++ b/buildscripts/resmokelib/configure_resmoke.py @@ -331,7 +331,6 @@ or explicitly pass --installDir to the run subcommand of buildscripts/resmoke.py _config.REPEAT_TESTS_MAX = config.pop("repeat_tests_max") _config.REPEAT_TESTS_MIN = config.pop("repeat_tests_min") _config.REPEAT_TESTS_SECS = config.pop("repeat_tests_secs") - _config.REPORT_FAILURE_STATUS = config.pop("report_failure_status") _config.REPORT_FILE = config.pop("report_file") _config.SERVICE_EXECUTOR = config.pop("service_executor") _config.EXPORT_MONGOD_CONFIG = config.pop("export_mongod_config") diff --git a/buildscripts/resmokelib/run/__init__.py b/buildscripts/resmokelib/run/__init__.py index 6fe5537eec9..f85850912bc 100644 --- a/buildscripts/resmokelib/run/__init__.py +++ b/buildscripts/resmokelib/run/__init__.py @@ -490,25 +490,12 @@ class TestRunnerEvg(TestRunner): additional options for running unreliable tests in Evergreen. """ - UNRELIABLE_TAG = _TagInfo( - tag_name="unreliable", - evergreen_aware=True, - suite_options=config.SuiteOptions.ALL_INHERITED._replace( # type: ignore - report_failure_status="silentfail")) - RESOURCE_INTENSIVE_TAG = _TagInfo( tag_name="resource_intensive", evergreen_aware=False, suite_options=config.SuiteOptions.ALL_INHERITED._replace( # type: ignore num_jobs=1)) - RETRY_ON_FAILURE_TAG = _TagInfo( - tag_name="retry_on_failure", - evergreen_aware=True, - suite_options=config.SuiteOptions.ALL_INHERITED._replace( # type: ignore - fail_fast=False, num_repeat_suites=2, num_repeat_tests=1, - report_failure_status="silentfail")) - @staticmethod def _make_evergreen_aware_tags(tag_name): """Return a list of resmoke.py tags. @@ -543,29 +530,8 @@ class TestRunnerEvg(TestRunner): combinations = [] - if config.EVERGREEN_PATCH_BUILD: - combinations.append(("unreliable and resource intensive", - ((cls.UNRELIABLE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, True)))) - combinations.append(("unreliable and not resource intensive", - ((cls.UNRELIABLE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, False)))) - combinations.append(("reliable and resource intensive", - ((cls.UNRELIABLE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, True)))) - combinations.append(("reliable and not resource intensive", - ((cls.UNRELIABLE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, - False)))) - else: - combinations.append(("retry on failure and resource intensive", - ((cls.RETRY_ON_FAILURE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, - True)))) - combinations.append(("retry on failure and not resource intensive", - ((cls.RETRY_ON_FAILURE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, - False)))) - combinations.append(("run once and resource intensive", - ((cls.RETRY_ON_FAILURE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, - True)))) - combinations.append(("run once and not resource intensive", - ((cls.RETRY_ON_FAILURE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, - False)))) + combinations.append(("resource intensive", [(cls.RESOURCE_INTENSIVE_TAG, True)])) + combinations.append(("not resource intensive", [(cls.RESOURCE_INTENSIVE_TAG, False)])) return combinations @@ -999,13 +965,6 @@ class RunPlugin(PluginInterface): help="Writes a JSON file with performance test results.") internal_options.add_argument( - "--reportFailureStatus", action="store", dest="report_failure_status", - choices=("fail", "silentfail"), metavar="STATUS", - help="Controls if the test failure status should be reported as failed" - " or be silently ignored (STATUS=silentfail). Dynamic test failures will" - " never be silently ignored. Defaults to STATUS=%(default)s.") - - internal_options.add_argument( "--reportFile", dest="report_file", metavar="REPORT", help="Writes a JSON file with test status and timing information.") diff --git a/buildscripts/resmokelib/testing/report.py b/buildscripts/resmokelib/testing/report.py index 176fc4bb241..59a866d2c22 100644 --- a/buildscripts/resmokelib/testing/report.py +++ b/buildscripts/resmokelib/testing/report.py @@ -218,12 +218,7 @@ class TestReport(unittest.TestResult): test_info = self.find_test_info(test) test_info.status = "fail" - if test_info.dynamic: - # Dynamic tests are used for data consistency checks, so the failures are never - # silenced. - test_info.evergreen_status = "fail" - else: - test_info.evergreen_status = self.suite_options.report_failure_status + test_info.evergreen_status = "fail" test_info.return_code = test.return_code def setFailure(self, test, return_code=1): @@ -235,12 +230,7 @@ class TestReport(unittest.TestResult): raise ValueError("stopTest was not called on %s" % (test.basename())) test_info.status = "fail" - if test_info.dynamic: - # Dynamic tests are used for data consistency checks, so the failures are never - # silenced. - test_info.evergreen_status = "fail" - else: - test_info.evergreen_status = self.suite_options.report_failure_status + test_info.evergreen_status = "fail" test_info.return_code = return_code # Recompute number of success, failures, and errors. diff --git a/buildscripts/tests/resmokelib/test_parser.py b/buildscripts/tests/resmokelib/test_parser.py index 2e1ff9c9e70..f751f46b416 100644 --- a/buildscripts/tests/resmokelib/test_parser.py +++ b/buildscripts/tests/resmokelib/test_parser.py @@ -231,7 +231,6 @@ class TestLocalCommandLine(unittest.TestCase): cmdline = to_local_args([ "run", "--suites=my_suite", - "--reportFailureStatus=fail", "--reportFile=report.json", "--perfReportFile=perf.json", "--storageEngine=my_storage_engine", |