diff options
author | Robert Guo <robert.guo@10gen.com> | 2017-07-21 10:21:09 -0400 |
---|---|---|
committer | Robert Guo <robert.guo@10gen.com> | 2017-08-21 10:30:59 -0400 |
commit | b136dbce0f79bed6d62df6de34e10bb247a80249 (patch) | |
tree | 28a3e8409f85ecd53e618b44156081ded7c5e942 | |
parent | 6ce6b59a2b0031b212c2bc8ba47ae61b2f81ac46 (diff) | |
download | mongo-b136dbce0f79bed6d62df6de34e10bb247a80249.tar.gz |
SERVER-19541 supporting connecting to an external cluster in resmoke.py
-rwxr-xr-x | buildscripts/resmoke.py | 4 | ||||
-rw-r--r-- | buildscripts/resmokelib/config.py | 6 | ||||
-rw-r--r-- | buildscripts/resmokelib/core/programs.py | 18 | ||||
-rw-r--r-- | buildscripts/resmokelib/parser.py | 47 | ||||
-rw-r--r-- | buildscripts/resmokelib/testing/executor.py | 9 | ||||
-rw-r--r-- | buildscripts/resmokelib/testing/fixtures/interface.py | 10 | ||||
-rw-r--r-- | buildscripts/resmokelib/testing/fixtures/masterslave.py | 6 | ||||
-rw-r--r-- | buildscripts/resmokelib/testing/fixtures/replicaset.py | 22 | ||||
-rw-r--r-- | buildscripts/resmokelib/testing/fixtures/shardedcluster.py | 9 | ||||
-rw-r--r-- | buildscripts/resmokelib/testing/fixtures/standalone.py | 7 | ||||
-rw-r--r-- | buildscripts/resmokelib/testing/hooks/initialsync.py | 9 | ||||
-rw-r--r-- | buildscripts/resmokelib/testing/testcases/cpp_integration_test.py | 2 |
12 files changed, 125 insertions, 24 deletions
diff --git a/buildscripts/resmoke.py b/buildscripts/resmoke.py index 79f3834f0bd..29591ef57a3 100755 --- a/buildscripts/resmoke.py +++ b/buildscripts/resmoke.py @@ -53,8 +53,8 @@ def _execute_suite(suite): logger.info("Skipping %ss, no tests to run", suite.test_kind) return True - suite_config = suite.get_executor_config() - executor = resmokelib.testing.executor.TestSuiteExecutor(logger, suite, **suite_config) + executor_config = suite.get_executor_config() + executor = resmokelib.testing.executor.TestSuiteExecutor(logger, suite, **executor_config) try: executor.run() diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py index 77c88cd9e56..d7d00532ce7 100644 --- a/buildscripts/resmokelib/config.py +++ b/buildscripts/resmokelib/config.py @@ -49,6 +49,8 @@ DEFAULTS = { "mongosSetParameters": None, "nojournal": False, "numClientsPerFixture": 1, + "shellPort": None, + "shellConnString": None, "repeat": 1, "reportFailureStatus": "fail", "reportFile": None, @@ -145,6 +147,10 @@ REPORT_FILE = None # IF set, then mongod/mongos's started by resmoke.py will use the specified service executor SERVICE_EXECUTOR = None +# If set, resmoke will override the default fixture and connect to the fixture specified by this +# connection string instead. +SHELL_CONN_STRING = None + # If set, then mongo shells started by resmoke.py will use the specified read mode. SHELL_READ_MODE = None diff --git a/buildscripts/resmokelib/core/programs.py b/buildscripts/resmokelib/core/programs.py index 29ec5cec0c9..aee4e6cf01a 100644 --- a/buildscripts/resmokelib/core/programs.py +++ b/buildscripts/resmokelib/core/programs.py @@ -12,8 +12,8 @@ import os.path import stat from . import process as _process -from .. import utils from .. import config +from .. import utils def mongod_program(logger, executable=None, process_kwargs=None, **kwargs): @@ -203,9 +203,23 @@ def mongo_shell_program(logger, executable=None, filename=None, process_kwargs=N if config.SHELL_WRITE_MODE is not None: kwargs["writeMode"] = config.SHELL_WRITE_MODE + if config.SHELL_CONN_STRING is not None: + # The --host and --port options are ignored by the mongo shell when an explicit connection + # string is specified. We remove these options to avoid any ambiguity with what server the + # logged mongo shell invocation will connect to. + if "port" in kwargs: + kwargs.pop("port") + + if "host" in kwargs: + kwargs.pop("host") + # Apply the rest of the command line arguments. _apply_kwargs(args, kwargs) + + if config.SHELL_CONN_STRING is not None: + args.append(config.SHELL_CONN_STRING) + # Have the mongos shell run the specified file. args.append(filename) @@ -253,6 +267,7 @@ def dbtest_program(logger, executable=None, suites=None, process_kwargs=None, ** return generic_program(logger, args, process_kwargs=process_kwargs, **kwargs) + def generic_program(logger, args, process_kwargs=None, **kwargs): """ Returns a Process instance that starts an arbitrary executable with @@ -289,6 +304,7 @@ def _format_test_data_set_parameters(set_parameters): params.append("%s=%s" % (param_name, param_value)) return ",".join(params) + def _apply_set_parameters(args, set_parameter): """ Converts key-value pairs from 'kwargs' into --setParameter key=value diff --git a/buildscripts/resmokelib/parser.py b/buildscripts/resmokelib/parser.py index 5071d2c2be9..599623431a7 100644 --- a/buildscripts/resmokelib/parser.py +++ b/buildscripts/resmokelib/parser.py @@ -40,6 +40,8 @@ DEST_TO_CONFIG = { "report_file": "reportFile", "seed": "seed", "service_executor": "serviceExecutor", + "shell_conn_string": "shellConnString", + "shell_port": "shellPort", "shell_read_mode": "shellReadMode", "shell_write_mode": "shellWriteMode", "shuffle": "shuffle", @@ -160,6 +162,18 @@ def parse_command_line(): help=("Enable or disable preallocation of journal files for all mongod" " processes. Defaults to %default.")) + parser.add_option("--shellConnString", dest="shell_conn_string", + metavar="CONN_STRING", + help="Override the default fixture and connect to an existing MongoDB" + " cluster instead. This is useful for connecting to a MongoDB" + " deployment started outside of resmoke.py including one running in a" + " debugger.") + + parser.add_option("--shellPort", dest="shell_port", metavar="PORT", + help="Convenience form of --shellConnString for connecting to an" + " existing MongoDB cluster with the URL mongodb://localhost:[PORT]." + " This is useful for connecting to a server running in a debugger.") + parser.add_option("--repeat", type="int", dest="repeat", metavar="N", help="Repeat the given suite(s) N times, or until one fails.") @@ -238,7 +252,24 @@ def parse_command_line(): shuffle="auto", stagger_jobs="off") - return parser.parse_args() + options, args = parser.parse_args() + + validate_options(parser, options, args) + + return options, args + +def validate_options(parser, options, args): + """ + Do preliminary validation on the options and error on any invalid options. + """ + + if options.shell_port is not None and options.shell_conn_string is not None: + parser.error("Cannot specify both `shellPort` and `shellConnString`") + + if options.executor_file: + parser.error("--executor is superceded by --suites; specify --suites={} {} to run the" + "test(s) under those suite configuration(s)" + .format(options.executor_file, " ".join(args))) def get_logging_config(values): @@ -301,6 +332,15 @@ def update_config_vars(values): else: _config.SHUFFLE = shuffle == "on" + conn_string = config.pop("shellConnString") + port = config.pop("shellPort") + + if port is not None: + conn_string = "mongodb://localhost:" + port + + if conn_string is not None: + _config.SHELL_CONN_STRING = conn_string + if config: raise optparse.OptionValueError("Unknown option(s): %s" % (config.keys())) @@ -337,11 +377,6 @@ def create_test_membership_map(fail_on_missing_selector=False, test_kind=None): def get_suites(values, args): - if values.executor_file: - raise optparse.OptionError( - "superceded by --suites; specify --suites={} {} to run the test(s) under those suite" - " configuration(s)".format(values.executor_file, " ".join(args)), "--executor") - suite_roots = None if args: # Do not change the execution order of the tests passed as args, unless a tag option is diff --git a/buildscripts/resmokelib/testing/executor.py b/buildscripts/resmokelib/testing/executor.py index 7286b05aeb3..776d5753b8c 100644 --- a/buildscripts/resmokelib/testing/executor.py +++ b/buildscripts/resmokelib/testing/executor.py @@ -14,7 +14,6 @@ from . import report as _report from . import testcases from .. import config as _config from .. import errors -from .. import logging from .. import utils from ..utils import queue as _queue @@ -40,7 +39,13 @@ class TestSuiteExecutor(object): """ self.logger = exec_logger - self.fixture_config = fixture + if _config.SHELL_CONN_STRING is not None: + # Specifying the shellConnString command line option should override the fixture + # specified in the YAML configuration to be the no-op fixture. + self.fixture_config = {"class": fixtures.NOOP_FIXTURE_CLASS} + else: + self.fixture_config = fixture + self.hooks_config = utils.default_if_none(hooks, []) self.test_config = utils.default_if_none(config, {}) diff --git a/buildscripts/resmokelib/testing/fixtures/interface.py b/buildscripts/resmokelib/testing/fixtures/interface.py index c44b432890c..2d950461927 100644 --- a/buildscripts/resmokelib/testing/fixtures/interface.py +++ b/buildscripts/resmokelib/testing/fixtures/interface.py @@ -99,7 +99,7 @@ class Fixture(object): """ return True - def get_connection_string(self): + def get_internal_connection_string(self): """ Returns the connection string for this fixture. This is NOT a driver connection string, but a connection string of the format @@ -107,6 +107,14 @@ class Fixture(object): """ raise NotImplementedError("get_connection_string must be implemented by Fixture subclasses") + def get_driver_connection_url(self): + """ + Return the mongodb connection string as defined here: + https://docs.mongodb.com/manual/reference/connection-string/ + """ + raise NotImplementedError( + "get_driver_connection_url must be implemented by Fixture subclasses") + def __str__(self): return "%s (Job #%d)" % (self.__class__.__name__, self.job_num) diff --git a/buildscripts/resmokelib/testing/fixtures/masterslave.py b/buildscripts/resmokelib/testing/fixtures/masterslave.py index c2a5419ca92..bef701ff8ee 100644 --- a/buildscripts/resmokelib/testing/fixtures/masterslave.py +++ b/buildscripts/resmokelib/testing/fixtures/masterslave.py @@ -161,3 +161,9 @@ class MasterSlaveFixture(interface.ReplFixture): mongod_options["source"] = "%s:%d" % (socket.gethostname(), self.port) mongod_options["dbpath"] = os.path.join(self._dbpath_prefix, "slave") return self._new_mongod(mongod_logger, mongod_options) + + def get_internal_connection_string(self): + return self.master.get_internal_connection_string() + + def get_driver_connection_url(self): + return self.master.get_driver_connection_url() diff --git a/buildscripts/resmokelib/testing/fixtures/replicaset.py b/buildscripts/resmokelib/testing/fixtures/replicaset.py index faf26b50cb3..54e5c2fe52c 100644 --- a/buildscripts/resmokelib/testing/fixtures/replicaset.py +++ b/buildscripts/resmokelib/testing/fixtures/replicaset.py @@ -96,7 +96,7 @@ class ReplicaSetFixture(interface.ReplFixture): # Initiate the replica set. members = [] for (i, node) in enumerate(self.nodes): - member_info = {"_id": i, "host": node.get_connection_string()} + member_info = {"_id": i, "host": node.get_internal_connection_string()} if i > 0: member_info["priority"] = 0 if i >= 7 or not self.voting_secondaries: @@ -106,7 +106,7 @@ class ReplicaSetFixture(interface.ReplFixture): members.append(member_info) if self.initial_sync_node: members.append({"_id": self.initial_sync_node_idx, - "host": self.initial_sync_node.get_connection_string(), + "host": self.initial_sync_node.get_internal_connection_string(), "priority": 0, "hidden": 1, "votes": 0}) @@ -285,11 +285,21 @@ class ReplicaSetFixture(interface.ReplFixture): return self.logger.new_fixture_node_logger(node_name) - def get_connection_string(self): + def get_internal_connection_string(self): if self.replset_name is None: - raise ValueError("Must call setup() before calling get_connection_string()") + raise ValueError("Must call setup() before calling get_internal_connection_string()") - conn_strs = [node.get_connection_string() for node in self.nodes] + conn_strs = [node.get_internal_connection_string() for node in self.nodes] if self.initial_sync_node: - conn_strs.append(self.initial_sync_node.get_connection_string()) + conn_strs.append(self.initial_sync_node.get_internal_connection_string()) return self.replset_name + "/" + ",".join(conn_strs) + + def get_driver_connection_url(self): + if self.replset_name is None: + raise ValueError("Must call setup() before calling get_driver_connection_url()") + + conn_strs = [node.get_internal_connection_string() for node in self.nodes] + if self.initial_sync_node: + conn_strs.append(self.initial_sync_node.get_internal_connection_string()) + + return "mongodb://" + ",".join(conn_strs) + "/?replicaSet=" + self.replset_name diff --git a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py index 4aea0bf189f..2c65c70e5a8 100644 --- a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py +++ b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py @@ -180,12 +180,15 @@ class ShardedClusterFixture(interface.Fixture): all(shard.is_running() for shard in self.shards) and self.mongos is not None and self.mongos.is_running()) - def get_connection_string(self): + def get_internal_connection_string(self): if self.mongos is None: - raise ValueError("Must call setup() before calling get_connection_string()") + raise ValueError("Must call setup() before calling get_internal_connection_string()") return "%s:%d" % (socket.gethostname(), self.mongos.port) + def get_driver_connection_url(self): + return "mongodb://" + self.get_internal_connection_string() + def _new_configsvr(self): """ Returns a replicaset.ReplicaSetFixture configured to be used as @@ -283,7 +286,7 @@ class ShardedClusterFixture(interface.Fixture): for more details. """ - connection_string = shard.get_connection_string() + connection_string = shard.get_internal_connection_string() self.logger.info("Adding %s as a shard..." % (connection_string)) client.admin.command({"addShard": "%s" % (connection_string)}) diff --git a/buildscripts/resmokelib/testing/fixtures/standalone.py b/buildscripts/resmokelib/testing/fixtures/standalone.py index e64937cd4a9..a657959259f 100644 --- a/buildscripts/resmokelib/testing/fixtures/standalone.py +++ b/buildscripts/resmokelib/testing/fixtures/standalone.py @@ -147,8 +147,11 @@ class MongoDFixture(interface.Fixture): def is_running(self): return self.mongod is not None and self.mongod.poll() is None - def get_connection_string(self): + def get_internal_connection_string(self): if self.mongod is None: - raise ValueError("Must call setup() before calling get_connection_string()") + raise ValueError("Must call setup() before calling get_internal_connection_string()") return "%s:%d" % (socket.gethostname(), self.port) + + def get_driver_connection_url(self): + return "mongodb://" + self.get_internal_connection_string() diff --git a/buildscripts/resmokelib/testing/hooks/initialsync.py b/buildscripts/resmokelib/testing/hooks/initialsync.py index 013c4c1d380..1b99d766622 100644 --- a/buildscripts/resmokelib/testing/hooks/initialsync.py +++ b/buildscripts/resmokelib/testing/hooks/initialsync.py @@ -13,6 +13,7 @@ import pymongo.errors from . import cleanup from . import jsfile +from ..fixtures import replicaset from ... import errors from ... import utils @@ -35,6 +36,10 @@ class BackgroundInitialSync(jsfile.JsCustomBehavior): DEFAULT_N = cleanup.CleanEveryN.DEFAULT_N def __init__(self, hook_logger, fixture, use_resync=False, n=DEFAULT_N, shell_options=None): + if not isinstance(fixture, replicaset.ReplicaSetFixture): + raise ValueError("`fixture` must be an instance of ReplicaSetFixture, not {}".format( + fixture.__class__.__name__)) + description = "Background Initial Sync" js_filename = os.path.join("jstests", "hooks", "run_initial_sync_node_validation.js") jsfile.JsCustomBehavior.__init__(self, hook_logger, fixture, js_filename, @@ -134,6 +139,10 @@ class IntermediateInitialSync(jsfile.JsCustomBehavior): DEFAULT_N = cleanup.CleanEveryN.DEFAULT_N def __init__(self, hook_logger, fixture, use_resync=False, n=DEFAULT_N): + if not isinstance(fixture, replicaset.ReplicaSetFixture): + raise ValueError("`fixture` must be an instance of ReplicaSetFixture, not {}".format( + fixture.__class__.__name__)) + description = "Intermediate Initial Sync" js_filename = os.path.join("jstests", "hooks", "run_initial_sync_node_validation.js") jsfile.JsCustomBehavior.__init__(self, hook_logger, fixture, js_filename, description) diff --git a/buildscripts/resmokelib/testing/testcases/cpp_integration_test.py b/buildscripts/resmokelib/testing/testcases/cpp_integration_test.py index 46990e79873..12849cc78aa 100644 --- a/buildscripts/resmokelib/testing/testcases/cpp_integration_test.py +++ b/buildscripts/resmokelib/testing/testcases/cpp_integration_test.py @@ -32,7 +32,7 @@ class CPPIntegrationTestCase(interface.TestCase): def configure(self, fixture, *args, **kwargs): interface.TestCase.configure(self, fixture, *args, **kwargs) - self.program_options["connectionString"] = self.fixture.get_connection_string() + self.program_options["connectionString"] = self.fixture.get_internal_connection_string() def run_test(self): try: |