diff options
Diffstat (limited to 'buildscripts')
-rw-r--r-- | buildscripts/resmokelib/config.py | 14 | ||||
-rw-r--r-- | buildscripts/resmokelib/configure_resmoke.py | 4 | ||||
-rw-r--r-- | buildscripts/resmokelib/core/jasper_process.py | 42 | ||||
-rw-r--r-- | buildscripts/resmokelib/core/programs.py | 12 | ||||
-rw-r--r-- | buildscripts/resmokelib/logging/jasper_logger.py | 38 | ||||
-rw-r--r-- | buildscripts/resmokelib/run/__init__.py | 18 | ||||
-rw-r--r-- | buildscripts/resmokelib/testing/report.py | 2 | ||||
-rw-r--r-- | buildscripts/resmokelib/testing/testcases/jstest.py | 3 |
8 files changed, 102 insertions, 31 deletions
diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py index 84bbee8f8bc..5d3eda4275c 100644 --- a/buildscripts/resmokelib/config.py +++ b/buildscripts/resmokelib/config.py @@ -120,6 +120,10 @@ DEFAULTS = { "variant_name": None, "version_id": None, + # Cedar options. + "cedar_url": "cedar.mongodb.com", + "cedar_rpc_port": "7070", + # WiredTiger options. "wt_coll_config": None, "wt_engine_config": None, @@ -249,6 +253,12 @@ BASE_PORT = None # The root url of the buildlogger server. BUILDLOGGER_URL = None +# URL to connect to the Cedar service. +CEDAR_URL = None + +# Cedar gRPC service port. +CEDAR_RPC_PORT = None + # Root directory for where resmoke.py puts directories containing data files of mongod's it starts, # as well as those started by individual tests. DBPATH_PREFIX = None @@ -401,6 +411,10 @@ SHUFFLE = None # or subprocess32 module to spawn threads. If jasper, resmoke uses the jasper module. SPAWN_USING = None +# The connection string to the jasper service, populated when the service is +# initialized in TestRunner. +JASPER_CONNECTION_STR = None + # If true, the launching of jobs is staggered in resmoke.py. STAGGER_JOBS = None diff --git a/buildscripts/resmokelib/configure_resmoke.py b/buildscripts/resmokelib/configure_resmoke.py index 79750d50ce3..afb26d34328 100644 --- a/buildscripts/resmokelib/configure_resmoke.py +++ b/buildscripts/resmokelib/configure_resmoke.py @@ -241,6 +241,10 @@ def _update_config_vars(values): # pylint: disable=too-many-statements,too-many _config.EVERGREEN_VARIANT_NAME = config.pop("variant_name") _config.EVERGREEN_VERSION_ID = config.pop("version_id") + # Cedar options. + _config.CEDAR_URL = config.pop("cedar_url") + _config.CEDAR_RPC_PORT = config.pop("cedar_rpc_port") + # Archival options. Archival is enabled only when running on evergreen. if not _config.EVERGREEN_TASK_ID: _config.ARCHIVE_FILE = None diff --git a/buildscripts/resmokelib/core/jasper_process.py b/buildscripts/resmokelib/core/jasper_process.py index 6198b9d41ed..1e741631397 100644 --- a/buildscripts/resmokelib/core/jasper_process.py +++ b/buildscripts/resmokelib/core/jasper_process.py @@ -8,38 +8,35 @@ try: except ImportError: pass +from buildscripts.resmokelib import config from buildscripts.resmokelib import errors from buildscripts.resmokelib.core import process as _process +from buildscripts.resmokelib.logging.jasper_logger import get_logger_config from buildscripts.resmokelib.testing.fixtures import interface as fixture_interface class Process(_process.Process): """Class for spawning a process using mongodb/jasper.""" - jasper_pb2 = None - jasper_pb2_grpc = None - connection_str = None + pb = None + rpc = None - def __init__(self, logger, args, env=None, env_vars=None): + def __init__(self, logger, args, env=None, env_vars=None, job_num=None, test_id=None): # pylint: disable=too-many-arguments """Initialize the process with the specified logger, arguments, and environment.""" _process.Process.__init__(self, logger, args, env=env, env_vars=env_vars) self._id = None - self._stub = self.jasper_pb2_grpc.JasperProcessManagerStub( - grpc.insecure_channel(self.connection_str)) + self.job_num = job_num + self.test_id = test_id + self._stub = self.rpc.JasperProcessManagerStub( + grpc.insecure_channel(config.JASPER_CONNECTION_STR)) self._return_code = None def start(self): """Start the process and the logger pipes for its stdout and stderr.""" - log_format = self.jasper_pb2.LogFormat.Value("LOGFORMATPLAIN") - log_level = self.jasper_pb2.LogLevel() - buffered = self.jasper_pb2.BufferOptions() - base_opts = self.jasper_pb2.BaseOptions(format=log_format, level=log_level, buffer=buffered) - log_opts = self.jasper_pb2.InheritedLoggerOptions(base=base_opts) - logger = self.jasper_pb2.LoggerConfig() - logger.inherited.CopyFrom(log_opts) - - output_opts = self.jasper_pb2.OutputOptions(loggers=[logger]) - create_options = self.jasper_pb2.CreateOptions( + logger = get_logger_config(group_id=self.job_num, test_id=self.test_id, + process_name=self.args[0]) + output_opts = self.pb.OutputOptions(loggers=[logger]) + create_options = self.pb.CreateOptions( args=self.args, environment=self.env, override_environ=True, @@ -49,7 +46,7 @@ class Process(_process.Process): val = self._stub.Create(create_options) self.pid = val.pid - self._id = self.jasper_pb2.JasperProcessID(value=val.id) + self._id = self.pb.JasperProcessID(value=val.id) self._return_code = None def stop(self, mode=None): @@ -58,16 +55,16 @@ class Process(_process.Process): mode = fixture_interface.TeardownMode.TERMINATE if mode == fixture_interface.TeardownMode.KILL: - signal = self.jasper_pb2.Signals.Value("KILL") + signal = self.pb.Signals.Value("KILL") elif mode == fixture_interface.TeardownMode.TERMINATE: - signal = self.jasper_pb2.Signals.Value("TERMINATE") + signal = self.pb.Signals.Value("TERMINATE") elif mode == fixture_interface.TeardownMode.ABORT: - signal = self.jasper_pb2.Signals.Value("ABRT") + signal = self.pb.Signals.Value("ABRT") else: raise errors.ProcessError("Process wrapper given unrecognized teardown mode: " + mode.value) - signal_process = self.jasper_pb2.SignalProcess(ProcessID=self._id, signal=signal) + signal_process = self.pb.SignalProcess(ProcessID=self._id, signal=signal) val = self._stub.Signal(signal_process) if not val.success \ and "cannot signal a process that has terminated" not in val.text \ @@ -87,8 +84,5 @@ class Process(_process.Process): """Wait until process has terminated and all output has been consumed by the logger pipes.""" if self._return_code is None: wait = self._stub.Wait(self._id) - if not wait.success: - raise OSError("Failed to wait on process with pid {}: {}.".format( - self.pid, wait.text)) self._return_code = wait.exit_code return self._return_code diff --git a/buildscripts/resmokelib/core/programs.py b/buildscripts/resmokelib/core/programs.py index 66f2c47faf6..2c8476eef1e 100644 --- a/buildscripts/resmokelib/core/programs.py +++ b/buildscripts/resmokelib/core/programs.py @@ -58,6 +58,10 @@ def make_process(*args, **kwargs): process_cls = process.Process if config.SPAWN_USING == "jasper": process_cls = jasper_process.Process + else: + # remove jasper process specific args + kwargs.pop("job_num", None) + kwargs.pop("test_id", None) # Add the current working directory and /data/multiversion to the PATH. env_vars = kwargs.get("env_vars", {}).copy() @@ -300,9 +304,9 @@ def mongos_program(logger, executable=None, process_kwargs=None, **kwargs): return make_process(logger, args, **process_kwargs) -def mongo_shell_program( # pylint: disable=too-many-branches,too-many-locals,too-many-statements - logger, executable=None, connection_string=None, filename=None, process_kwargs=None, - **kwargs): +def mongo_shell_program( # pylint: disable=too-many-arguments,too-many-branches,too-many-locals,too-many-statements + logger, job_num=None, test_id=None, executable=None, connection_string=None, filename=None, + process_kwargs=None, **kwargs): """Return a Process instance that starts a mongo shell. The shell is started with the given connection string and arguments constructed from 'kwargs'. @@ -461,6 +465,8 @@ def mongo_shell_program( # pylint: disable=too-many-branches,too-many-locals,to _set_keyfile_permissions(test_data) process_kwargs = utils.default_if_none(process_kwargs, {}) + process_kwargs["job_num"] = job_num + process_kwargs["test_id"] = test_id return make_process(logger, args, **process_kwargs) diff --git a/buildscripts/resmokelib/logging/jasper_logger.py b/buildscripts/resmokelib/logging/jasper_logger.py new file mode 100644 index 00000000000..abc3269b734 --- /dev/null +++ b/buildscripts/resmokelib/logging/jasper_logger.py @@ -0,0 +1,38 @@ +"""Jasper logging handlers and helpers.""" + +import os + +from buildscripts.resmokelib import config + + +def get_logger_config(group_id="", test_id="", process_name=""): + """Return the jasper logger config.""" + + import jasper.jasper_pb2 as pb + + username = os.getenv("CEDAR_USERNAME", default="") + api_key = os.getenv("CEDAR_API_KEY", default="") + + logger_config = pb.LoggerConfig() + log_level = pb.LogLevel(threshold=30, default=30) + log_format = pb.LogFormat.Value("LOGFORMATPLAIN") + + if config.EVERGREEN_TASK_ID and group_id: + buildlogger_info = pb.BuildloggerV3Info( + project=config.EVERGREEN_PROJECT_NAME, version=config.EVERGREEN_VERSION_ID, + variant=config.EVERGREEN_VARIANT_NAME, task_name=config.EVERGREEN_TASK_NAME, + task_id=config.EVERGREEN_TASK_ID, execution=config.EVERGREEN_EXECUTION, + test_name=str(test_id), process_name=process_name, format=log_format, tags=[ + str(group_id) + ], base_address=config.CEDAR_URL, rpc_port=config.CEDAR_RPC_PORT, username=username, + api_key=api_key) + buildlogger_options = pb.BuildloggerV3Options(buildloggerv3=buildlogger_info, + level=log_level) + logger_config.buildloggerv3.CopyFrom(buildlogger_options) + else: + buffered = pb.BufferOptions() + base_opts = pb.BaseOptions(format=log_format, level=log_level, buffer=buffered) + log_opts = pb.DefaultLoggerOptions(base=base_opts) + logger_config.default.CopyFrom(log_opts) + + return logger_config diff --git a/buildscripts/resmokelib/run/__init__.py b/buildscripts/resmokelib/run/__init__.py index eefec02ae0f..a07e73f58d9 100644 --- a/buildscripts/resmokelib/run/__init__.py +++ b/buildscripts/resmokelib/run/__init__.py @@ -39,6 +39,7 @@ from buildscripts.resmokelib.plugin import PluginInterface, Subcommand _INTERNAL_OPTIONS_TITLE = "Internal Options" _BENCHMARK_ARGUMENT_TITLE = "Benchmark/Benchrun test options" _EVERGREEN_ARGUMENT_TITLE = "Evergreen options" +_CEDAR_ARGUMENT_TITLE = "Cedar options" class TestRunner(Subcommand): # pylint: disable=too-many-instance-attributes @@ -395,18 +396,18 @@ class TestRunner(Subcommand): # pylint: disable=too-many-instance-attributes from jasper import jasper_pb2 from jasper import jasper_pb2_grpc - jasper_process.Process.jasper_pb2 = jasper_pb2 - jasper_process.Process.jasper_pb2_grpc = jasper_pb2_grpc + jasper_process.Process.pb = jasper_pb2 + jasper_process.Process.rpc = jasper_pb2_grpc jasper_port = config.BASE_PORT - 1 jasper_conn_str = "localhost:%d" % jasper_port - jasper_process.Process.connection_str = jasper_conn_str jasper_command = [ curator_path, "jasper", "service", "run", "rpc", "--port", str(jasper_port) ] self._jasper_server = process.Process(self._resmoke_logger, jasper_command) self._jasper_server.start() + config.JASPER_CONNECTION_STR = jasper_conn_str channel = grpc.insecure_channel(jasper_conn_str) grpc.channel_ready_future(channel).result() @@ -991,6 +992,17 @@ class RunPlugin(PluginInterface): evergreen_options.add_argument("--versionId", dest="version_id", metavar="VERSION_ID", help="Sets the version ID of the task.") + cedar_options = parser.add_argument_group( + title=_CEDAR_ARGUMENT_TITLE, + description=("Options used to propagate Cedar service connection information.")) + + cedar_options.add_argument("--cedarURL", dest="cedar_url", metavar="CEDAR_URL", + help=("The URL of the Cedar service.")) + + cedar_options.add_argument("--cedarRPCPort", dest="cedar_rpc_port", + metavar="CEDAR_RPC_PORT", + help=("The RPC port of the Cedar service.")) + benchmark_options = parser.add_argument_group( title=_BENCHMARK_ARGUMENT_TITLE, description="Options for running Benchmark/Benchrun tests") diff --git a/buildscripts/resmokelib/testing/report.py b/buildscripts/resmokelib/testing/report.py index 7dc355a4023..d16a37fcf23 100644 --- a/buildscripts/resmokelib/testing/report.py +++ b/buildscripts/resmokelib/testing/report.py @@ -125,6 +125,8 @@ class TestReport(unittest.TestResult): # pylint: disable=too-many-instance-attr test_info.url_endpoint = url_endpoint if self.logging_prefix is not None: test_logger.info(self.logging_prefix) + # Set job_num in test. + test.job_num = self.job_num test.override_logger(test_logger) test_info.start_time = time.time() diff --git a/buildscripts/resmokelib/testing/testcases/jstest.py b/buildscripts/resmokelib/testing/testcases/jstest.py index b79a1c89d2c..1a1658dd452 100644 --- a/buildscripts/resmokelib/testing/testcases/jstest.py +++ b/buildscripts/resmokelib/testing/testcases/jstest.py @@ -102,7 +102,8 @@ class _SingleJSTestCase(interface.ProcessTestCase): def _make_process(self): return core.programs.mongo_shell_program( - self.logger, executable=self.shell_executable, filename=self.js_filename, + self.logger, job_num=self.fixture.job_num, test_id=self._id, + executable=self.shell_executable, filename=self.js_filename, connection_string=self.fixture.get_driver_connection_url(), **self.shell_options) |