summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorNamaz Gurbanov <namaz.gurbanov@mongodb.com>2023-04-12 16:17:38 +0000
committerEvergreen Agent <no-reply@evergreen.mongodb.com>2023-04-12 17:15:19 +0000
commita296bbae578b3d5606e284ed71745ac144189000 (patch)
tree20af245fbe3381dd9f2accddd152682806ec2b9e
parent976ed21bb2cf49bebce1e5eb8965f9bb00ecd50c (diff)
downloadmongo-a296bbae578b3d5606e284ed71745ac144189000.tar.gz
SERVER-66927 Remove reference to silentFail from mongo repo
-rwxr-xr-xbuildscripts/combine_reports.py2
-rw-r--r--buildscripts/promote_silent_failures.py65
-rw-r--r--buildscripts/resmokelib/config.py6
-rw-r--r--buildscripts/resmokelib/configure_resmoke.py1
-rw-r--r--buildscripts/resmokelib/run/__init__.py45
-rw-r--r--buildscripts/resmokelib/testing/report.py14
-rw-r--r--buildscripts/tests/resmokelib/test_parser.py1
7 files changed, 5 insertions, 129 deletions
diff --git a/buildscripts/combine_reports.py b/buildscripts/combine_reports.py
index 8952a564943..dd5564f4454 100755
--- a/buildscripts/combine_reports.py
+++ b/buildscripts/combine_reports.py
@@ -25,7 +25,7 @@ def report_exit(combined_test_report):
"""Return report exit code.
The exit code of this script is based on the following:
- 0: All tests have status "pass", or only non-dynamic tests have status "silentfail".
+ 0: All tests have status "pass".
31: At least one test has status "fail" or "timeout".
Note: A test can be considered dynamic if its name contains a ":" character.
"""
diff --git a/buildscripts/promote_silent_failures.py b/buildscripts/promote_silent_failures.py
deleted file mode 100644
index d8d45872685..00000000000
--- a/buildscripts/promote_silent_failures.py
+++ /dev/null
@@ -1,65 +0,0 @@
-#!/usr/bin/env python3
-"""Convert silent test failures into non-silent failures.
-
-Any test files with at least 2 executions in the report.json file that have a "silentfail" status,
-this script will change the outputted report to have a "fail" status instead.
-"""
-
-import collections
-import json
-import optparse
-import os
-import sys
-
-# Get relative imports to work when the package is not installed on the PYTHONPATH.
-if __name__ == "__main__" and __package__ is None:
- sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
- from buildscripts.resmokelib.testing import report
-
-
-def read_json_file(json_file):
- """Return contents of a JSON file."""
- with open(json_file) as json_data:
- return json.load(json_data)
-
-
-def main():
- """Execute Main program."""
-
- usage = "usage: %prog [options] report.json"
- parser = optparse.OptionParser(usage=usage)
- parser.add_option(
- "-o", "--output-file", dest="outfile", default="-",
- help=("If '-', then the report file is written to stdout."
- " Any other value is treated as the output file name. By default,"
- " output is written to stdout."))
-
- (options, args) = parser.parse_args()
-
- if len(args) != 1:
- parser.error("Requires a single report.json file.")
-
- report_file_json = read_json_file(args[0])
- test_report = report.TestReport.from_dict(report_file_json)
-
- # Count number of "silentfail" per test file.
- status_dict = collections.defaultdict(int)
- for test_info in test_report.test_infos:
- if test_info.evergreen_status == "silentfail":
- status_dict[test_info.test_id] += 1
-
- # For test files with more than 1 "silentfail", convert status to "fail".
- for test_info in test_report.test_infos:
- if status_dict[test_info.test_id] >= 2:
- test_info.evergreen_status = "fail"
-
- result_report = test_report.as_dict()
- if options.outfile != "-":
- with open(options.outfile, "w") as fp:
- json.dump(result_report, fp)
- else:
- print(json.dumps(result_report))
-
-
-if __name__ == "__main__":
- main()
diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py
index f285c6b21de..1da18223f14 100644
--- a/buildscripts/resmokelib/config.py
+++ b/buildscripts/resmokelib/config.py
@@ -90,7 +90,6 @@ DEFAULTS = {
"repeat_tests_min": None,
"repeat_tests_secs": None,
"replay_file": None,
- "report_failure_status": "fail",
"report_file": None,
"run_all_feature_flag_tests": False,
"run_no_feature_flag_tests": False,
@@ -169,7 +168,6 @@ _SuiteOptions = collections.namedtuple("_SuiteOptions", [
"num_repeat_tests_max",
"num_repeat_tests_min",
"time_repeat_tests_secs",
- "report_failure_status",
])
@@ -236,7 +234,6 @@ class SuiteOptions(_SuiteOptions):
REPEAT_TESTS_MAX,
REPEAT_TESTS_MIN,
REPEAT_TESTS_SECS,
- REPORT_FAILURE_STATUS,
])))
options = self._asdict()
@@ -445,9 +442,6 @@ REPEAT_TESTS_MIN = None
# If set, then each test is repeated the specified time (seconds) inside the suites.
REPEAT_TESTS_SECS = None
-# Controls if the test failure status should be reported as failed or be silently ignored.
-REPORT_FAILURE_STATUS = None
-
# If set, then resmoke.py will write out a report file with the status of each test that ran.
REPORT_FILE = None
diff --git a/buildscripts/resmokelib/configure_resmoke.py b/buildscripts/resmokelib/configure_resmoke.py
index cfb856491d0..bfe59e1befa 100644
--- a/buildscripts/resmokelib/configure_resmoke.py
+++ b/buildscripts/resmokelib/configure_resmoke.py
@@ -331,7 +331,6 @@ or explicitly pass --installDir to the run subcommand of buildscripts/resmoke.py
_config.REPEAT_TESTS_MAX = config.pop("repeat_tests_max")
_config.REPEAT_TESTS_MIN = config.pop("repeat_tests_min")
_config.REPEAT_TESTS_SECS = config.pop("repeat_tests_secs")
- _config.REPORT_FAILURE_STATUS = config.pop("report_failure_status")
_config.REPORT_FILE = config.pop("report_file")
_config.SERVICE_EXECUTOR = config.pop("service_executor")
_config.EXPORT_MONGOD_CONFIG = config.pop("export_mongod_config")
diff --git a/buildscripts/resmokelib/run/__init__.py b/buildscripts/resmokelib/run/__init__.py
index 6fe5537eec9..f85850912bc 100644
--- a/buildscripts/resmokelib/run/__init__.py
+++ b/buildscripts/resmokelib/run/__init__.py
@@ -490,25 +490,12 @@ class TestRunnerEvg(TestRunner):
additional options for running unreliable tests in Evergreen.
"""
- UNRELIABLE_TAG = _TagInfo(
- tag_name="unreliable",
- evergreen_aware=True,
- suite_options=config.SuiteOptions.ALL_INHERITED._replace( # type: ignore
- report_failure_status="silentfail"))
-
RESOURCE_INTENSIVE_TAG = _TagInfo(
tag_name="resource_intensive",
evergreen_aware=False,
suite_options=config.SuiteOptions.ALL_INHERITED._replace( # type: ignore
num_jobs=1))
- RETRY_ON_FAILURE_TAG = _TagInfo(
- tag_name="retry_on_failure",
- evergreen_aware=True,
- suite_options=config.SuiteOptions.ALL_INHERITED._replace( # type: ignore
- fail_fast=False, num_repeat_suites=2, num_repeat_tests=1,
- report_failure_status="silentfail"))
-
@staticmethod
def _make_evergreen_aware_tags(tag_name):
"""Return a list of resmoke.py tags.
@@ -543,29 +530,8 @@ class TestRunnerEvg(TestRunner):
combinations = []
- if config.EVERGREEN_PATCH_BUILD:
- combinations.append(("unreliable and resource intensive",
- ((cls.UNRELIABLE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, True))))
- combinations.append(("unreliable and not resource intensive",
- ((cls.UNRELIABLE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, False))))
- combinations.append(("reliable and resource intensive",
- ((cls.UNRELIABLE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, True))))
- combinations.append(("reliable and not resource intensive",
- ((cls.UNRELIABLE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG,
- False))))
- else:
- combinations.append(("retry on failure and resource intensive",
- ((cls.RETRY_ON_FAILURE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG,
- True))))
- combinations.append(("retry on failure and not resource intensive",
- ((cls.RETRY_ON_FAILURE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG,
- False))))
- combinations.append(("run once and resource intensive",
- ((cls.RETRY_ON_FAILURE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG,
- True))))
- combinations.append(("run once and not resource intensive",
- ((cls.RETRY_ON_FAILURE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG,
- False))))
+ combinations.append(("resource intensive", [(cls.RESOURCE_INTENSIVE_TAG, True)]))
+ combinations.append(("not resource intensive", [(cls.RESOURCE_INTENSIVE_TAG, False)]))
return combinations
@@ -999,13 +965,6 @@ class RunPlugin(PluginInterface):
help="Writes a JSON file with performance test results.")
internal_options.add_argument(
- "--reportFailureStatus", action="store", dest="report_failure_status",
- choices=("fail", "silentfail"), metavar="STATUS",
- help="Controls if the test failure status should be reported as failed"
- " or be silently ignored (STATUS=silentfail). Dynamic test failures will"
- " never be silently ignored. Defaults to STATUS=%(default)s.")
-
- internal_options.add_argument(
"--reportFile", dest="report_file", metavar="REPORT",
help="Writes a JSON file with test status and timing information.")
diff --git a/buildscripts/resmokelib/testing/report.py b/buildscripts/resmokelib/testing/report.py
index 176fc4bb241..59a866d2c22 100644
--- a/buildscripts/resmokelib/testing/report.py
+++ b/buildscripts/resmokelib/testing/report.py
@@ -218,12 +218,7 @@ class TestReport(unittest.TestResult):
test_info = self.find_test_info(test)
test_info.status = "fail"
- if test_info.dynamic:
- # Dynamic tests are used for data consistency checks, so the failures are never
- # silenced.
- test_info.evergreen_status = "fail"
- else:
- test_info.evergreen_status = self.suite_options.report_failure_status
+ test_info.evergreen_status = "fail"
test_info.return_code = test.return_code
def setFailure(self, test, return_code=1):
@@ -235,12 +230,7 @@ class TestReport(unittest.TestResult):
raise ValueError("stopTest was not called on %s" % (test.basename()))
test_info.status = "fail"
- if test_info.dynamic:
- # Dynamic tests are used for data consistency checks, so the failures are never
- # silenced.
- test_info.evergreen_status = "fail"
- else:
- test_info.evergreen_status = self.suite_options.report_failure_status
+ test_info.evergreen_status = "fail"
test_info.return_code = return_code
# Recompute number of success, failures, and errors.
diff --git a/buildscripts/tests/resmokelib/test_parser.py b/buildscripts/tests/resmokelib/test_parser.py
index 2e1ff9c9e70..f751f46b416 100644
--- a/buildscripts/tests/resmokelib/test_parser.py
+++ b/buildscripts/tests/resmokelib/test_parser.py
@@ -231,7 +231,6 @@ class TestLocalCommandLine(unittest.TestCase):
cmdline = to_local_args([
"run",
"--suites=my_suite",
- "--reportFailureStatus=fail",
"--reportFile=report.json",
"--perfReportFile=perf.json",
"--storageEngine=my_storage_engine",