diff options
author | Daniel Gottlieb <daniel.gottlieb@mongodb.com> | 2020-08-05 13:07:13 -0400 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2020-08-05 20:11:11 +0000 |
commit | 52f4d4d869dca67d7ed22e8956d9fb56a8a79944 (patch) | |
tree | e9ba4971e500055577121a26b4f7aac63788e788 /buildscripts | |
parent | 515bb3ba57dbf775000abafcaa5288569a983f48 (diff) | |
download | mongo-52f4d4d869dca67d7ed22e8956d9fb56a8a79944.tar.gz |
SERVER-47509: Have resmoke better cope with multiple set parameters.
Diffstat (limited to 'buildscripts')
9 files changed, 239 insertions, 21 deletions
diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py index 1e89bcd599d..58b21e4bb4a 100644 --- a/buildscripts/resmokelib/config.py +++ b/buildscripts/resmokelib/config.py @@ -65,9 +65,9 @@ DEFAULTS = { "logger_file": None, "mongo_executable": None, "mongod_executable": None, - "mongod_set_parameters": None, + "mongod_set_parameters": [], "mongos_executable": None, - "mongos_set_parameters": None, + "mongos_set_parameters": [], "mrlog": None, "no_journal": False, "num_clients_per_fixture": 1, @@ -330,13 +330,13 @@ MONGO_EXECUTABLE = None MONGOD_EXECUTABLE = None # The --setParameter options passed to mongod. -MONGOD_SET_PARAMETERS = None +MONGOD_SET_PARAMETERS = [] # The path to the mongos executable used by resmoke.py. MONGOS_EXECUTABLE = None # The --setParameter options passed to mongos. -MONGOS_SET_PARAMETERS = None +MONGOS_SET_PARAMETERS = [] # If true, then all mongod's started by resmoke.py and by the mongo shell will not have journaling # enabled. diff --git a/buildscripts/resmokelib/configure_resmoke.py b/buildscripts/resmokelib/configure_resmoke.py index 622f3cef3ae..b8ab910f716 100644 --- a/buildscripts/resmokelib/configure_resmoke.py +++ b/buildscripts/resmokelib/configure_resmoke.py @@ -1,5 +1,6 @@ """Configure the command line input for the resmoke 'run' subcommand.""" +import collections import configparser import datetime import os @@ -25,7 +26,7 @@ def validate_and_update_config(parser, args): def _validate_options(parser, args): """Do preliminary validation on the options and error on any invalid options.""" - if not 'shell_port' in args or not 'shell_conn_strin' in args: + if not 'shell_port' in args or not 'shell_conn_string' in args: return if args.shell_port is not None and args.shell_conn_string is not None: @@ -42,8 +43,32 @@ def _validate_options(parser, args): "Cannot use --replayFile with additional test files listed on the command line invocation." ) + def get_set_param_errors(process_params): + agg_set_params = collections.defaultdict(list) + for set_param in process_params: + for key, value in utils.load_yaml(set_param).items(): + agg_set_params[key] += [value] -def _validate_config(parser): + errors = [] + for key, values in agg_set_params.items(): + if len(values) > 1: + errors.append(f"setParameter has multiple values. Key: {key} Values: {values}") + + return errors + + config = vars(args) + mongod_set_param_errors = get_set_param_errors(config.get('mongod_set_parameters') or []) + mongos_set_param_errors = get_set_param_errors(config.get('mongos_set_parameters') or []) + error_msgs = {} + if mongod_set_param_errors: + error_msgs["mongodSetParameters"] = mongod_set_param_errors + if mongos_set_param_errors: + error_msgs["mongosSetParameters"] = mongos_set_param_errors + if error_msgs: + parser.error(str(error_msgs)) + + +def _validate_config(parser): # pylint: disable=too-many-branches """Do validation on the config settings.""" if _config.REPEAT_TESTS_MAX: @@ -140,11 +165,18 @@ def _update_config_vars(values): # pylint: disable=too-many-statements,too-many _config.DBTEST_EXECUTABLE = _expand_user(config.pop("dbtest_executable")) _config.MONGO_EXECUTABLE = _expand_user(config.pop("mongo_executable")) + + def _merge_set_params(param_list): + ret = {} + for set_param in param_list: + ret.update(utils.load_yaml(set_param)) + return utils.dump_yaml(ret) + _config.MONGOD_EXECUTABLE = _expand_user(config.pop("mongod_executable")) - _config.MONGOD_SET_PARAMETERS = config.pop("mongod_set_parameters") + _config.MONGOD_SET_PARAMETERS = _merge_set_params(config.pop("mongod_set_parameters")) _config.MONGOS_EXECUTABLE = _expand_user(config.pop("mongos_executable")) + _config.MONGOS_SET_PARAMETERS = _merge_set_params(config.pop("mongos_set_parameters")) - _config.MONGOS_SET_PARAMETERS = config.pop("mongos_set_parameters") _config.MRLOG = config.pop("mrlog") _config.NO_JOURNAL = config.pop("no_journal") _config.NUM_CLIENTS_PER_FIXTURE = config.pop("num_clients_per_fixture") diff --git a/buildscripts/resmokelib/run/__init__.py b/buildscripts/resmokelib/run/__init__.py index cf5a74faddc..9a8d17db6d6 100644 --- a/buildscripts/resmokelib/run/__init__.py +++ b/buildscripts/resmokelib/run/__init__.py @@ -673,7 +673,7 @@ class RunPlugin(PluginInterface): help="The path to the mongod executable for resmoke.py to use.") parser.add_argument( - "--mongodSetParameters", dest="mongod_set_parameters", + "--mongodSetParameters", dest="mongod_set_parameters", action="append", metavar="{key1: value1, key2: value2, ..., keyN: valueN}", help=("Passes one or more --setParameter options to all mongod processes" " started by resmoke.py. The argument is specified as bracketed YAML -" @@ -683,7 +683,7 @@ class RunPlugin(PluginInterface): help="The path to the mongos executable for resmoke.py to use.") parser.add_argument( - "--mongosSetParameters", dest="mongos_set_parameters", + "--mongosSetParameters", dest="mongos_set_parameters", action="append", metavar="{key1: value1, key2: value2, ..., keyN: valueN}", help=("Passes one or more --setParameter options to all mongos processes" " started by resmoke.py. The argument is specified as bracketed YAML -" diff --git a/buildscripts/tests/resmoke_end2end/suites/resmoke_no_mongod.yml b/buildscripts/tests/resmoke_end2end/suites/resmoke_no_mongod.yml index 92af90962dd..5076c1c018a 100644 --- a/buildscripts/tests/resmoke_end2end/suites/resmoke_no_mongod.yml +++ b/buildscripts/tests/resmoke_end2end/suites/resmoke_no_mongod.yml @@ -2,7 +2,8 @@ test_kind: js_test selector: roots: - - buildscripts/tests/resmoke_end2end/testfiles/*.js + - buildscripts/tests/resmoke_end2end/testfiles/one.js + - buildscripts/tests/resmoke_end2end/testfiles/two.js executor: config: diff --git a/buildscripts/tests/resmoke_end2end/suites/resmoke_selftest_set_parameters.yml b/buildscripts/tests/resmoke_end2end/suites/resmoke_selftest_set_parameters.yml new file mode 100644 index 00000000000..45548678628 --- /dev/null +++ b/buildscripts/tests/resmoke_end2end/suites/resmoke_selftest_set_parameters.yml @@ -0,0 +1,21 @@ +test_kind: js_test + +selector: + roots: + - jstests/resmoke_selftest/end2end/**.js + +executor: + archive: + tests: true + config: + shell_options: + global_vars: + TestData: + outputLocation: ./output.json # This is a template value replaced at runtime. + fixture: + class: MongoDFixture + mongod_options: + set_parameters: + enableTestCommands: 1 + logComponentVerbosity: {storage: 2} + testingDiagnosticsEnabled: false diff --git a/buildscripts/tests/resmoke_end2end/suites/resmoke_selftest_set_parameters_sharding.yml b/buildscripts/tests/resmoke_end2end/suites/resmoke_selftest_set_parameters_sharding.yml new file mode 100644 index 00000000000..d5a0f893269 --- /dev/null +++ b/buildscripts/tests/resmoke_end2end/suites/resmoke_selftest_set_parameters_sharding.yml @@ -0,0 +1,19 @@ +test_kind: js_test + +selector: + roots: + - jstests/resmoke_selftest/end2end/**.js + +executor: + archive: + tests: true + config: + shell_options: + global_vars: + TestData: + outputLocation: ./output.json # This is a template value replaced at runtime. + fixture: + class: ShardedClusterFixture + mongod_options: + set_parameters: + enableTestCommands: 1 diff --git a/buildscripts/tests/resmoke_end2end/test_resmoke.py b/buildscripts/tests/resmoke_end2end/test_resmoke.py index a1f6a57e432..48b095f7dc3 100644 --- a/buildscripts/tests/resmoke_end2end/test_resmoke.py +++ b/buildscripts/tests/resmoke_end2end/test_resmoke.py @@ -7,6 +7,7 @@ import os.path import sys import time import unittest +import yaml from buildscripts.resmokelib import core from buildscripts.resmokelib.utils import rmtree @@ -17,16 +18,14 @@ from buildscripts.resmokelib.utils import rmtree class _ResmokeSelftest(unittest.TestCase): @classmethod def setUpClass(cls): + cls.end2end_root = "buildscripts/tests/resmoke_end2end" cls.test_dir = os.path.normpath("/data/db/selftest") cls.resmoke_const_args = ["run", "--dbpathPrefix={}".format(cls.test_dir)] + cls.suites_root = os.path.join(cls.end2end_root, "suites") + cls.testfiles_root = os.path.join(cls.end2end_root, "testfiles") + cls.report_file = os.path.join(cls.test_dir, "reports.json") def setUp(self): - #self.test_dir = os.path.normpath("/data/db/selftest") - self.end2end_root = "buildscripts/tests/resmoke_end2end" - self.suites_root = f"{self.end2end_root}/suites" - self.testfiles_root = f"{self.end2end_root}/testfiles" - self.report_file = os.path.join(self.test_dir, "reports.json") - self.logger = logging.getLogger(self._testMethodName) self.logger.setLevel(logging.DEBUG) handler = logging.StreamHandler(sys.stdout) @@ -248,9 +247,140 @@ class TestTestSelection(_ResmokeSelftest): self.assertEqual(6 * [f"{self.testfiles_root}/two.js"], self.get_tests_run()) def test_disallow_mixing_replay_and_positional(self): + self.create_file_in_test_dir("replay", f"{self.testfiles_root}/two.js\n" * 3) + # Additionally can assert on the error message. - self.assertEqual(1, self.execute_resmoke(["--replay=foo", "jstests/filename.js"]).wait()) - self.assertEqual(1, self.execute_resmoke(["@foo", "jstests/filename.js"]).wait()) + self.assertEqual( + 2, + self.execute_resmoke([f"--replay={self.test_dir}/replay", + "jstests/filename.js"]).wait()) + + # When multiple positional arguments are presented, they're all treated as test files. Technically errors on file `@<testdir>/replay` not existing. It's not a requirement that this invocation errors in this less specific way. + self.assertEqual( + 1, + self.execute_resmoke([f"@{self.test_dir}/replay", "jstests/filename.js"]).wait()) + self.assertEqual( + 1, + self.execute_resmoke([f"{self.testfiles_root}/one.js", + f"@{self.test_dir}/replay"]).wait()) + + +class TestSetParameters(_ResmokeSelftest): + def setUp(self): + self.shell_output_file = f"{self.test_dir}/output.json" + try: + os.remove(self.shell_output_file) + except OSError: + pass + + super().setUp() + + def parse_output_json(self): + # Parses the outputted json. + with open(self.shell_output_file) as fd: + return json.load(fd) - # Technically errors on file `@foo` not existing. Only the first positional argument can be interpreted as a `@replay_file`. This is a limitation not a feature, this test just serves as documentation. - self.assertEqual(1, self.execute_resmoke([f"{self.testfiles_root}/one.js", "@foo"]).wait()) + def generate_suite(self, suite_output_path, template_file): + """Read the template file, substitute the `outputLocation` and rewrite to the file `suite_output_path`.""" + + with open(os.path.normpath(template_file), "r") as template_suite_fd: + suite = yaml.safe_load(template_suite_fd) + + try: + os.remove(suite_output_path) + except FileNotFoundError: + pass + + suite["executor"]["config"]["shell_options"]["global_vars"]["TestData"][ + "outputLocation"] = self.shell_output_file + with open(os.path.normpath(suite_output_path), "w") as fd: + yaml.dump(suite, fd, default_flow_style=False) + + def generate_suite_and_execute_resmoke(self, suite_template, resmoke_args): + """Generates a resmoke suite with the appropriate `outputLocation` and runs resmoke against that suite with the `fixture_info` test. Input `resmoke_args` are appended to the run command.""" + + suite_file = f"{self.test_dir}/suite.yml" + self.generate_suite(suite_file, suite_template) + + self.logger.info( + "Running test. Template suite: {suite_template} Rewritten suite: {self.suite_file} Resmoke Args: {resmoke_args} Test output file: {self.shell_output_file}." + ) + + resmoke_process = core.programs.make_process(self.logger, [ + sys.executable, "buildscripts/resmoke.py", "run", f"--suites={suite_file}", + f"{self.testfiles_root}/fixture_info.js" + ] + resmoke_args) + resmoke_process.start() + + return resmoke_process + + def test_suite_set_parameters(self): + self.generate_suite_and_execute_resmoke( + f"{self.suites_root}/resmoke_selftest_set_parameters.yml", []).wait() + + set_params = self.parse_output_json() + self.assertEqual("1", set_params["enableTestCommands"]) + self.assertEqual("false", set_params["testingDiagnosticsEnabled"]) + self.assertEqual("{'storage': 2}", set_params["logComponentVerbosity"]) + + def test_cli_set_parameters(self): + self.generate_suite_and_execute_resmoke( + f"{self.suites_root}/resmoke_selftest_set_parameters.yml", + ["""--mongodSetParameter={"enableFlowControl": false, "flowControlMaxSamples": 500}""" + ]).wait() + + set_params = self.parse_output_json() + self.assertEqual("1", set_params["enableTestCommands"]) + self.assertEqual("false", set_params["enableFlowControl"]) + self.assertEqual("500", set_params["flowControlMaxSamples"]) + + def test_override_set_parameters(self): + self.generate_suite_and_execute_resmoke( + f"{self.suites_root}/resmoke_selftest_set_parameters.yml", + ["""--mongodSetParameter={"testingDiagnosticsEnabled": true}"""]).wait() + + set_params = self.parse_output_json() + self.assertEqual("true", set_params["testingDiagnosticsEnabled"]) + self.assertEqual("{'storage': 2}", set_params["logComponentVerbosity"]) + + def test_merge_cli_set_parameters(self): + self.generate_suite_and_execute_resmoke( + f"{self.suites_root}/resmoke_selftest_set_parameters.yml", [ + """--mongodSetParameter={"enableFlowControl": false}""", + """--mongodSetParameter={"flowControlMaxSamples": 500}""" + ]).wait() + + set_params = self.parse_output_json() + self.assertEqual("false", set_params["testingDiagnosticsEnabled"]) + self.assertEqual("{'storage': 2}", set_params["logComponentVerbosity"]) + self.assertEqual("false", set_params["enableFlowControl"]) + self.assertEqual("500", set_params["flowControlMaxSamples"]) + + def test_merge_error_cli_set_parameters(self): + self.assertEqual( + 2, + self.generate_suite_and_execute_resmoke( + f"{self.suites_root}/resmoke_selftest_set_parameters.yml", [ + """--mongodSetParameter={"enableFlowControl": false}""", + """--mongodSetParameter={"enableFlowControl": true}""" + ]).wait()) + + def test_mongos_set_parameter(self): + self.generate_suite_and_execute_resmoke( + f"{self.suites_root}/resmoke_selftest_set_parameters_sharding.yml", [ + """--mongosSetParameter={"maxTimeMSForHedgedReads": 100}""", + """--mongosSetParameter={"mongosShutdownTimeoutMillisForSignaledShutdown": 1000}""" + ]).wait() + + set_params = self.parse_output_json() + self.assertEqual("100", set_params["maxTimeMSForHedgedReads"]) + self.assertEqual("1000", set_params["mongosShutdownTimeoutMillisForSignaledShutdown"]) + + def test_merge_error_cli_mongos_set_parameter(self): + self.assertEqual( + 2, + self.generate_suite_and_execute_resmoke( + f"{self.suites_root}/resmoke_selftest_set_parameters_sharding.yml", [ + """--mongosSetParameter={"maxTimeMSForHedgedReads": 100}""", + """--mongosSetParameter={"maxTimeMSForHedgedReads": 1000}""" + ]).wait()) diff --git a/buildscripts/tests/resmoke_end2end/testfiles/fixture_info.js b/buildscripts/tests/resmoke_end2end/testfiles/fixture_info.js new file mode 100644 index 00000000000..fb925299a74 --- /dev/null +++ b/buildscripts/tests/resmoke_end2end/testfiles/fixture_info.js @@ -0,0 +1,6 @@ +(function() { +"use strict"; + +writeFile(TestData.outputLocation, + tojson(db.adminCommand("getCmdLineOpts")["parsed"]["setParameter"])); +}()); diff --git a/buildscripts/tests/resmoke_end2end/testfiles/fixture_info_mongos.js b/buildscripts/tests/resmoke_end2end/testfiles/fixture_info_mongos.js new file mode 100644 index 00000000000..0837b88d06a --- /dev/null +++ b/buildscripts/tests/resmoke_end2end/testfiles/fixture_info_mongos.js @@ -0,0 +1,9 @@ +(function() { +"use strict"; + +print("DEBUG BUILDINFO") +printjson(db.adminCommand("buildInfo")) + +writeFile(TestData.outputLocation, + tojson(db.adminCommand("getCmdLineOpts")["parsed"]["setParameter"])); +}()); |