diff options
-rw-r--r-- | buildscripts/resmokelib/config.py | 14 | ||||
-rw-r--r-- | buildscripts/resmokelib/configure_resmoke.py | 4 | ||||
-rw-r--r-- | buildscripts/resmokelib/core/jasper_process.py | 39 | ||||
-rw-r--r-- | buildscripts/resmokelib/core/programs.py | 12 | ||||
-rw-r--r-- | buildscripts/resmokelib/logging/jasper_logger.py | 38 | ||||
-rw-r--r-- | buildscripts/resmokelib/run/__init__.py | 18 | ||||
-rw-r--r-- | buildscripts/resmokelib/testing/report.py | 2 | ||||
-rw-r--r-- | buildscripts/resmokelib/testing/testcases/jstest.py | 3 | ||||
-rw-r--r-- | etc/evergreen.yml | 7 | ||||
-rw-r--r-- | etc/pip/components/resmoke.req | 6 |
10 files changed, 31 insertions, 112 deletions
diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py index 05b4ea6ca83..27dfe5a828a 100644 --- a/buildscripts/resmokelib/config.py +++ b/buildscripts/resmokelib/config.py @@ -118,10 +118,6 @@ DEFAULTS = { "variant_name": None, "version_id": None, - # Cedar options. - "cedar_url": "cedar.mongodb.com", - "cedar_rpc_port": "7070", - # WiredTiger options. "wt_coll_config": None, "wt_engine_config": None, @@ -251,12 +247,6 @@ BASE_PORT = None # The root url of the buildlogger server. BUILDLOGGER_URL = None -# URL to connect to the Cedar service. -CEDAR_URL = None - -# Cedar gRPC service port. -CEDAR_RPC_PORT = None - # Root directory for where resmoke.py puts directories containing data files of mongod's it starts, # as well as those started by individual tests. DBPATH_PREFIX = None @@ -406,10 +396,6 @@ SHUFFLE = None # or subprocess32 module to spawn threads. If jasper, resmoke uses the jasper module. SPAWN_USING = None -# The connection string to the jasper service, populated when the service is -# initialized in TestRunner. -JASPER_CONNECTION_STR = None - # If true, the launching of jobs is staggered in resmoke.py. STAGGER_JOBS = None diff --git a/buildscripts/resmokelib/configure_resmoke.py b/buildscripts/resmokelib/configure_resmoke.py index 0660a1cf299..8dc953bb535 100644 --- a/buildscripts/resmokelib/configure_resmoke.py +++ b/buildscripts/resmokelib/configure_resmoke.py @@ -228,10 +228,6 @@ def _update_config_vars(values): # pylint: disable=too-many-statements,too-many _config.EVERGREEN_VARIANT_NAME = config.pop("variant_name") _config.EVERGREEN_VERSION_ID = config.pop("version_id") - # Cedar options. - _config.CEDAR_URL = config.pop("cedar_url") - _config.CEDAR_RPC_PORT = config.pop("cedar_rpc_port") - # Archival options. Archival is enabled only when running on evergreen. if not _config.EVERGREEN_TASK_ID: _config.ARCHIVE_FILE = None diff --git a/buildscripts/resmokelib/core/jasper_process.py b/buildscripts/resmokelib/core/jasper_process.py index 63d5b3e3d03..6198b9d41ed 100644 --- a/buildscripts/resmokelib/core/jasper_process.py +++ b/buildscripts/resmokelib/core/jasper_process.py @@ -8,35 +8,38 @@ try: except ImportError: pass -from buildscripts.resmokelib import config from buildscripts.resmokelib import errors from buildscripts.resmokelib.core import process as _process -from buildscripts.resmokelib.logging.jasper_logger import get_logger_config from buildscripts.resmokelib.testing.fixtures import interface as fixture_interface class Process(_process.Process): """Class for spawning a process using mongodb/jasper.""" - pb = None - rpc = None + jasper_pb2 = None + jasper_pb2_grpc = None + connection_str = None - def __init__(self, logger, args, env=None, env_vars=None, job_num=None, test_id=None): # pylint: disable=too-many-arguments + def __init__(self, logger, args, env=None, env_vars=None): """Initialize the process with the specified logger, arguments, and environment.""" _process.Process.__init__(self, logger, args, env=env, env_vars=env_vars) self._id = None - self.job_num = job_num - self.test_id = test_id - self._stub = self.rpc.JasperProcessManagerStub( - grpc.insecure_channel(config.JASPER_CONNECTION_STR)) + self._stub = self.jasper_pb2_grpc.JasperProcessManagerStub( + grpc.insecure_channel(self.connection_str)) self._return_code = None def start(self): """Start the process and the logger pipes for its stdout and stderr.""" - logger = get_logger_config(group_id=self.job_num, test_id=self.test_id, - process_name=self.args[0]) - output_opts = self.pb.OutputOptions(loggers=[logger]) - create_options = self.pb.CreateOptions( + log_format = self.jasper_pb2.LogFormat.Value("LOGFORMATPLAIN") + log_level = self.jasper_pb2.LogLevel() + buffered = self.jasper_pb2.BufferOptions() + base_opts = self.jasper_pb2.BaseOptions(format=log_format, level=log_level, buffer=buffered) + log_opts = self.jasper_pb2.InheritedLoggerOptions(base=base_opts) + logger = self.jasper_pb2.LoggerConfig() + logger.inherited.CopyFrom(log_opts) + + output_opts = self.jasper_pb2.OutputOptions(loggers=[logger]) + create_options = self.jasper_pb2.CreateOptions( args=self.args, environment=self.env, override_environ=True, @@ -46,7 +49,7 @@ class Process(_process.Process): val = self._stub.Create(create_options) self.pid = val.pid - self._id = self.pb.JasperProcessID(value=val.id) + self._id = self.jasper_pb2.JasperProcessID(value=val.id) self._return_code = None def stop(self, mode=None): @@ -55,16 +58,16 @@ class Process(_process.Process): mode = fixture_interface.TeardownMode.TERMINATE if mode == fixture_interface.TeardownMode.KILL: - signal = self.pb.Signals.Value("KILL") + signal = self.jasper_pb2.Signals.Value("KILL") elif mode == fixture_interface.TeardownMode.TERMINATE: - signal = self.pb.Signals.Value("TERMINATE") + signal = self.jasper_pb2.Signals.Value("TERMINATE") elif mode == fixture_interface.TeardownMode.ABORT: - signal = self.pb.Signals.Value("ABRT") + signal = self.jasper_pb2.Signals.Value("ABRT") else: raise errors.ProcessError("Process wrapper given unrecognized teardown mode: " + mode.value) - signal_process = self.pb.SignalProcess(ProcessID=self._id, signal=signal) + signal_process = self.jasper_pb2.SignalProcess(ProcessID=self._id, signal=signal) val = self._stub.Signal(signal_process) if not val.success \ and "cannot signal a process that has terminated" not in val.text \ diff --git a/buildscripts/resmokelib/core/programs.py b/buildscripts/resmokelib/core/programs.py index 66d78f440ef..0cc8aafa3bc 100644 --- a/buildscripts/resmokelib/core/programs.py +++ b/buildscripts/resmokelib/core/programs.py @@ -58,10 +58,6 @@ def make_process(*args, **kwargs): process_cls = process.Process if config.SPAWN_USING == "jasper": process_cls = jasper_process.Process - else: - # remove jasper process specific args - kwargs.pop("job_num", None) - kwargs.pop("test_id", None) # Add the current working directory and /data/multiversion to the PATH. env_vars = kwargs.get("env_vars", {}).copy() @@ -305,9 +301,9 @@ def mongos_program(logger, executable=None, process_kwargs=None, **kwargs): return make_process(logger, args, **process_kwargs) -def mongo_shell_program( # pylint: disable=too-many-arguments,too-many-branches,too-many-locals,too-many-statements - logger, job_num=None, test_id=None, executable=None, connection_string=None, filename=None, - process_kwargs=None, **kwargs): +def mongo_shell_program( # pylint: disable=too-many-branches,too-many-locals,too-many-statements + logger, executable=None, connection_string=None, filename=None, process_kwargs=None, + **kwargs): """Return a Process instance that starts a mongo shell. The shell is started with the given connection string and arguments constructed from 'kwargs'. @@ -466,8 +462,6 @@ def mongo_shell_program( # pylint: disable=too-many-arguments,too-many-branches _set_keyfile_permissions(test_data) process_kwargs = utils.default_if_none(process_kwargs, {}) - process_kwargs["job_num"] = job_num - process_kwargs["test_id"] = test_id return make_process(logger, args, **process_kwargs) diff --git a/buildscripts/resmokelib/logging/jasper_logger.py b/buildscripts/resmokelib/logging/jasper_logger.py deleted file mode 100644 index abc3269b734..00000000000 --- a/buildscripts/resmokelib/logging/jasper_logger.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Jasper logging handlers and helpers.""" - -import os - -from buildscripts.resmokelib import config - - -def get_logger_config(group_id="", test_id="", process_name=""): - """Return the jasper logger config.""" - - import jasper.jasper_pb2 as pb - - username = os.getenv("CEDAR_USERNAME", default="") - api_key = os.getenv("CEDAR_API_KEY", default="") - - logger_config = pb.LoggerConfig() - log_level = pb.LogLevel(threshold=30, default=30) - log_format = pb.LogFormat.Value("LOGFORMATPLAIN") - - if config.EVERGREEN_TASK_ID and group_id: - buildlogger_info = pb.BuildloggerV3Info( - project=config.EVERGREEN_PROJECT_NAME, version=config.EVERGREEN_VERSION_ID, - variant=config.EVERGREEN_VARIANT_NAME, task_name=config.EVERGREEN_TASK_NAME, - task_id=config.EVERGREEN_TASK_ID, execution=config.EVERGREEN_EXECUTION, - test_name=str(test_id), process_name=process_name, format=log_format, tags=[ - str(group_id) - ], base_address=config.CEDAR_URL, rpc_port=config.CEDAR_RPC_PORT, username=username, - api_key=api_key) - buildlogger_options = pb.BuildloggerV3Options(buildloggerv3=buildlogger_info, - level=log_level) - logger_config.buildloggerv3.CopyFrom(buildlogger_options) - else: - buffered = pb.BufferOptions() - base_opts = pb.BaseOptions(format=log_format, level=log_level, buffer=buffered) - log_opts = pb.DefaultLoggerOptions(base=base_opts) - logger_config.default.CopyFrom(log_opts) - - return logger_config diff --git a/buildscripts/resmokelib/run/__init__.py b/buildscripts/resmokelib/run/__init__.py index 4559d985acf..6026d5951e0 100644 --- a/buildscripts/resmokelib/run/__init__.py +++ b/buildscripts/resmokelib/run/__init__.py @@ -39,7 +39,6 @@ from buildscripts.resmokelib.plugin import PluginInterface, Subcommand _INTERNAL_OPTIONS_TITLE = "Internal Options" _BENCHMARK_ARGUMENT_TITLE = "Benchmark/Benchrun test options" _EVERGREEN_ARGUMENT_TITLE = "Evergreen options" -_CEDAR_ARGUMENT_TITLE = "Cedar options" class TestRunner(Subcommand): # pylint: disable=too-many-instance-attributes @@ -385,18 +384,18 @@ class TestRunner(Subcommand): # pylint: disable=too-many-instance-attributes from jasper import jasper_pb2 from jasper import jasper_pb2_grpc - jasper_process.Process.pb = jasper_pb2 - jasper_process.Process.rpc = jasper_pb2_grpc + jasper_process.Process.jasper_pb2 = jasper_pb2 + jasper_process.Process.jasper_pb2_grpc = jasper_pb2_grpc jasper_port = config.BASE_PORT - 1 jasper_conn_str = "localhost:%d" % jasper_port + jasper_process.Process.connection_str = jasper_conn_str jasper_command = [ curator_path, "jasper", "service", "run", "rpc", "--port", str(jasper_port) ] self._jasper_server = process.Process(self._resmoke_logger, jasper_command) self._jasper_server.start() - config.JASPER_CONNECTION_STR = jasper_conn_str channel = grpc.insecure_channel(jasper_conn_str) grpc.channel_ready_future(channel).result() @@ -975,17 +974,6 @@ class RunPlugin(PluginInterface): evergreen_options.add_argument("--versionId", dest="version_id", metavar="VERSION_ID", help="Sets the version ID of the task.") - cedar_options = parser.add_argument_group( - title=_CEDAR_ARGUMENT_TITLE, - description=("Options used to propagate Cedar service connection information.")) - - cedar_options.add_argument("--cedarURL", dest="cedar_url", metavar="CEDAR_URL", - help=("The URL of the Cedar service.")) - - cedar_options.add_argument("--cedarRPCPort", dest="cedar_rpc_port", - metavar="CEDAR_RPC_PORT", - help=("The RPC port of the Cedar service.")) - benchmark_options = parser.add_argument_group( title=_BENCHMARK_ARGUMENT_TITLE, description="Options for running Benchmark/Benchrun tests") diff --git a/buildscripts/resmokelib/testing/report.py b/buildscripts/resmokelib/testing/report.py index d16a37fcf23..7dc355a4023 100644 --- a/buildscripts/resmokelib/testing/report.py +++ b/buildscripts/resmokelib/testing/report.py @@ -125,8 +125,6 @@ class TestReport(unittest.TestResult): # pylint: disable=too-many-instance-attr test_info.url_endpoint = url_endpoint if self.logging_prefix is not None: test_logger.info(self.logging_prefix) - # Set job_num in test. - test.job_num = self.job_num test.override_logger(test_logger) test_info.start_time = time.time() diff --git a/buildscripts/resmokelib/testing/testcases/jstest.py b/buildscripts/resmokelib/testing/testcases/jstest.py index 1a1658dd452..b79a1c89d2c 100644 --- a/buildscripts/resmokelib/testing/testcases/jstest.py +++ b/buildscripts/resmokelib/testing/testcases/jstest.py @@ -102,8 +102,7 @@ class _SingleJSTestCase(interface.ProcessTestCase): def _make_process(self): return core.programs.mongo_shell_program( - self.logger, job_num=self.fixture.job_num, test_id=self._id, - executable=self.shell_executable, filename=self.js_filename, + self.logger, executable=self.shell_executable, filename=self.js_filename, connection_string=self.fixture.get_driver_connection_url(), **self.shell_options) diff --git a/etc/evergreen.yml b/etc/evergreen.yml index d171d84ff62..e30472fc016 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -1227,7 +1227,6 @@ functions: # Windows path separator toolchain_txt="$pip_dir/toolchain-requirements.txt" ${activate_virtualenv} - python -m pip install --upgrade pip python -m pip install -r "$toolchain_txt" -q python -m pip freeze > pip-requirements.txt @@ -1366,10 +1365,6 @@ functions: working_dir: src shell: bash script: | - # export these before verbose is set to avoid sharing sensitive info - export CEDAR_USERNAME=${cedar_user} - export CEDAR_API_KEY=${cedar_api_key} - set -o errexit set -o verbose @@ -1470,8 +1465,6 @@ functions: # this list of expansions. set +o errexit PATH="$path_value" \ - CEDAR_USERNAME=${cedar_user} \ - CEDAR_PASSWORD AWS_PROFILE=${aws_profile_remote} \ ${gcov_environment} \ ${lang_environment} \ diff --git a/etc/pip/components/resmoke.req b/etc/pip/components/resmoke.req index 3f00b3e623e..29749bbea9b 100644 --- a/etc/pip/components/resmoke.req +++ b/etc/pip/components/resmoke.req @@ -7,7 +7,7 @@ ocspresponder == 0.5.0 flask == 1.1.1 ocspbuilder == 0.10.2 # TODO: uncomment these lines with SERVER-48156 -# grpcio == 1.28.1; platform_machine == "x86_64" -# grpcio-tools == 1.28.1; platform_machine == "x86_64" -# googleapis-common-protos == 1.51.0; platform_machine == "x86_64" +#grpcio == 1.28.1; platform_machine != "x86_64" +#grpcio-tools == 1.28.1; platform_machine != "x86_64" +#googleapis-common-protos == 1.51.0; platform_machine != "x86_64" blackduck == 0.0.51 |