diff options
49 files changed, 611 insertions, 352 deletions
diff --git a/SConstruct b/SConstruct index 5116821ad32..7331ef04e42 100644 --- a/SConstruct +++ b/SConstruct @@ -861,6 +861,11 @@ env_vars.Add('OBJCOPY', help='Sets the path to objcopy', default=WhereIs('objcopy')) + +env_vars.Add('PKGDIR', + help='Directory in which to build packages and archives', + default='$VARIANT_DIR/pkgs') + env_vars.Add('PREFIX', help='Final installation location of files, will be made into a sub dir of $DESTDIR', default='') @@ -1046,15 +1051,15 @@ envDict = dict(BUILD_ROOT=buildDir, ARCHIVE_ADDITIONS=[], PYTHON="$( {} $)".format(sys.executable), SERVER_ARCHIVE='${SERVER_DIST_BASENAME}${DIST_ARCHIVE_SUFFIX}', - UNITTEST_ALIAS='unittests', + UNITTEST_ALIAS='install-unittests', # TODO: Move unittests.txt to $BUILD_DIR, but that requires # changes to MCI. UNITTEST_LIST='$BUILD_ROOT/unittests.txt', - LIBFUZZER_TEST_ALIAS='libfuzzer_tests', + LIBFUZZER_TEST_ALIAS='install-fuzzertests', LIBFUZZER_TEST_LIST='$BUILD_ROOT/libfuzzer_tests.txt', - INTEGRATION_TEST_ALIAS='integration_tests', + INTEGRATION_TEST_ALIAS='install-integration-tests', INTEGRATION_TEST_LIST='$BUILD_ROOT/integration_tests.txt', - BENCHMARK_ALIAS='benchmarks', + BENCHMARK_ALIAS='install-benchmarks', BENCHMARK_LIST='$BUILD_ROOT/benchmarks.txt', CONFIGUREDIR='$BUILD_ROOT/scons/$VARIANT_DIR/sconf_temp', CONFIGURELOG='$BUILD_ROOT/scons/config.log', @@ -1062,6 +1067,13 @@ envDict = dict(BUILD_ROOT=buildDir, LIBDEPS_TAG_EXPANSIONS=[], ) +# TODO: Remove these when hygienic builds are default. +if get_option('install-mode') != 'hygienic': + envDict["UNITTEST_ALIAS"] = "unittests" + envDict["INTEGRATION_TEST_ALIAS"] = "integration_tests" + envDict["LIBFUZZER_TEST_ALIAS"] = "libfuzzer_tests" + envDict["BENCHMARK_ALIAS"] = "benchmarks" + env = Environment(variables=env_vars, **envDict) # Only print the spinner if stdout is a tty @@ -3874,7 +3886,7 @@ if get_option('ninja') == 'true': # TODO: Later, this should live somewhere more graceful. if get_option('install-mode') == 'hygienic': - if get_option('separate-debug') == "on": + if get_option('separate-debug') == "on" or env.TargetOSIs("windows"): env.Tool('separate_debug') env["AIB_TARBALL_SUFFIX"] = "tgz" @@ -3918,6 +3930,19 @@ if get_option('install-mode') == 'hygienic': meta_role="meta", ) + def _aib_debugdir(source, target, env, for_signature): + for s in source: + origin = getattr(s.attributes, "debug_file_for", None) + oentry = env.Entry(origin) + osuf = oentry.get_suffix() + map_entry = env["AIB_SUFFIX_MAP"].get(osuf) + if map_entry: + return map_entry[0] + + return "Unable to find debuginfo for {}".format(str(source)) + + env["PREFIX_DEBUGDIR"] = _aib_debugdir + env.AddSuffixMapping({ "$PROGSUFFIX": env.SuffixMap( directory="$PREFIX_BINDIR", @@ -3926,13 +3951,6 @@ if get_option('install-mode') == 'hygienic': ] ), - "$LIBSUFFIX": env.SuffixMap( - directory="$PREFIX_LIBDIR", - default_roles=[ - "dev", - ] - ), - "$SHLIBSUFFIX": env.SuffixMap( directory="$PREFIX_BINDIR" \ if mongo_platform.get_running_os_name() == "windows" \ @@ -3967,13 +3985,61 @@ if get_option('install-mode') == 'hygienic': env.AddPackageNameAlias( component="dist", role="runtime", - name="${SERVER_DIST_BASENAME}", + name="mongodb-dist", ) env.AddPackageNameAlias( component="dist", role="debug", - name="${SERVER_DIST_BASENAME}-debugsymbols", + name="mongodb-dist-debugsymbols", + ) + + env.AddPackageNameAlias( + component="dist-test", + role="runtime", + name="mongodb-binaries", + ) + + env.AddPackageNameAlias( + component="dist-test", + role="debug", + name="mongo-debugsymbols", + ) + + env.AddPackageNameAlias( + component="dbtest", + role="runtime", + name="dbtest-binary", + ) + + env.AddPackageNameAlias( + component="dbtest", + role="debug", + name="dbtest-debugsymbols", + ) + + env.AddPackageNameAlias( + component="shell", + role="runtime", + name="mongodb-shell", + ) + + env.AddPackageNameAlias( + component="shell", + role="debug", + name="mongodb-shell-debugsymbols", + ) + + env.AddPackageNameAlias( + component="mongocryptd", + role="runtime", + name="mongodb-cryptd", + ) + + env.AddPackageNameAlias( + component="mongocryptd", + role="debug", + name="mongodb-cryptd-debugsymbols", ) env.AddPackageNameAlias( @@ -3982,7 +4048,7 @@ if get_option('install-mode') == 'hygienic': # TODO: we should be able to move this to where the mqlrun binary is # defined when AIB correctly uses environments instead of hooking into # the first environment used. - name="${MH_DIST_BASENAME}-binaries", + name="mh-binaries", ) env.AddPackageNameAlias( @@ -3991,7 +4057,7 @@ if get_option('install-mode') == 'hygienic': # TODO: we should be able to move this to where the mqlrun binary is # defined when AIB correctly uses environments instead of hooking into # the first environment used. - name="${MH_DIST_BASENAME}-debugsymbols", + name="mh-debugsymbols", ) if env['PLATFORM'] == 'posix': @@ -4256,17 +4322,19 @@ if get_option("ninja") == "false": r"$PYTHON buildscripts\make_vcxproj.py " + msvc_version + "mongodb") vcxproj = env.Alias("vcxproj", vcxprojFile) -distSrc = env.DistSrc("mongodb-src-${MONGO_VERSION}.tar") +# TODO: maybe make these work like the other archive- hygienic aliases +# even though they aren't piped through AIB? +distSrc = env.DistSrc("distsrc.tar") env.NoCache(distSrc) env.Alias("distsrc-tar", distSrc) distSrcGzip = env.GZip( - target="mongodb-src-${MONGO_VERSION}.tgz", + target="distsrc.tgz", source=[distSrc]) env.NoCache(distSrcGzip) env.Alias("distsrc-tgz", distSrcGzip) -distSrcZip = env.DistSrc("mongodb-src-${MONGO_VERSION}.zip") +distSrcZip = env.DistSrc("distsrc.zip") env.NoCache(distSrcZip) env.Alias("distsrc-zip", distSrcZip) @@ -4420,18 +4488,7 @@ env.Alias('cache-prune', cachePrune) if get_option('install-mode') == 'hygienic': env.FinalizeInstallDependencies() - # TODO: Remove once hygienic is driving all builds and we can make - # the evergreen.yml make this decision - if env.TargetOSIs("windows"): - env.Alias("archive-dist", "zip-dist") - env.Alias("archive-dist-debug", "zip-dist-debug") - env.Alias("archive-mh", "zip-mh") - env.Alias("archive-mh-debug", "zip-mh-debug") - else: - env.Alias("archive-dist", "tar-dist") - env.Alias("archive-dist-debug", "tar-dist-debug") - env.Alias("archive-mh", "tar-mh") - env.Alias("archive-mh-debug", "tar-mh-debug") + # We don't want installing files to cause them to flow into the cache, # since presumably we can re-install them from the origin if needed. diff --git a/buildscripts/bypass_compile_and_fetch_binaries.py b/buildscripts/bypass_compile_and_fetch_binaries.py index 028c4b37665..4c51a8bfb54 100755 --- a/buildscripts/bypass_compile_and_fetch_binaries.py +++ b/buildscripts/bypass_compile_and_fetch_binaries.py @@ -97,11 +97,15 @@ TargetBuild = namedtuple("TargetBuild", [ ]) -def executable_name(pathname): +def executable_name(pathname, destdir=""): """Return the executable name.""" # Ensure that executable files on Windows have a ".exe" extension. if _IS_WINDOWS and os.path.splitext(pathname)[1] != ".exe": - return "{}.exe".format(pathname) + pathname = "{}.exe".format(pathname) + + if destdir: + return os.path.join(destdir, "bin", pathname) + return pathname @@ -339,9 +343,9 @@ def fetch_artifacts(build: Build, revision: str): raise ValueError("No artifacts were found for the current task") # Need to extract certain files from the pre-existing artifacts.tgz. extract_files = [ - executable_name("mongobridge"), - executable_name("mongotmock"), - executable_name("wt"), + executable_name("mongobridge", destdir=os.getenv("DESTDIR")), + executable_name("mongotmock", destdir=os.getenv("DESTDIR")), + executable_name("wt", destdir=os.getenv("DESTDIR")), ] with tarfile.open(filename, "r:gz") as tar: # The repo/ directory contains files needed by the package task. May diff --git a/buildscripts/hang_analyzer.py b/buildscripts/hang_analyzer.py index ce92c7b1be5..8e812e5f23f 100755 --- a/buildscripts/hang_analyzer.py +++ b/buildscripts/hang_analyzer.py @@ -533,9 +533,7 @@ class DebugExtractor(object): out = [] for ext in ['debug', 'dSYM', 'pdb']: for file in ['mongo', 'mongod', 'mongos']: - # need to glob because it untar's to a directory that looks like - # mongodb-linux-x86_64-enterprise-rhel62-4.3.0-1823-gb9c13fa-patch-5daa05630ae60652f0890f76 - haystack = os.path.join('mongodb*', 'bin', '{file}.{ext}'.format( + haystack = os.path.join('dist-test', 'bin', '{file}.{ext}'.format( file=file, ext=ext)) for needle in glob.glob(haystack): out.append((needle, os.path.join(os.getcwd(), os.path.basename(needle)))) diff --git a/buildscripts/resmokeconfig/suites/integration_tests_replset.yml b/buildscripts/resmokeconfig/suites/integration_tests_replset.yml index 94c51e885d2..16eee75933d 100644 --- a/buildscripts/resmokeconfig/suites/integration_tests_replset.yml +++ b/buildscripts/resmokeconfig/suites/integration_tests_replset.yml @@ -4,6 +4,7 @@ selector: root: build/integration_tests.txt exclude_files: - build/**/mongo/client/client_dbclient_connection_integration_test* # Needs connection to single host. + - build/install/bin/client_dbclient_connection_integration_test* # Needs connection to single host. executor: archive: diff --git a/buildscripts/resmokeconfig/suites/integration_tests_sharded.yml b/buildscripts/resmokeconfig/suites/integration_tests_sharded.yml index eca7475bdc3..1651042bb98 100644 --- a/buildscripts/resmokeconfig/suites/integration_tests_sharded.yml +++ b/buildscripts/resmokeconfig/suites/integration_tests_sharded.yml @@ -4,6 +4,7 @@ selector: root: build/integration_tests.txt exclude_files: - build/**/mongo/client/client_dbclient_connection_integration_test* # Needs sleep command + - build/install/bin/client_dbclient_connection_integration_test* # Needs sleep command executor: archive: diff --git a/buildscripts/resmokelib/core/programs.py b/buildscripts/resmokelib/core/programs.py index c154c8fbb32..edc81b1ee70 100644 --- a/buildscripts/resmokelib/core/programs.py +++ b/buildscripts/resmokelib/core/programs.py @@ -7,6 +7,7 @@ import json import os import os.path import stat +import sys from . import jasper_process from . import process @@ -44,10 +45,20 @@ def make_process(*args, **kwargs): process_cls = process.Process if config.SPAWN_USING == "jasper": process_cls = jasper_process.Process + # Add the current working directory and /data/multiversion to the PATH. env_vars = kwargs.get("env_vars", {}).copy() - path = [env_vars.get("PATH", os.environ.get("PATH", ""))] - path = [os.getcwd(), config.DEFAULT_MULTIVERSION_DIR] + path + path = [ + os.getcwd(), + config.DEFAULT_MULTIVERSION_DIR, + ] + + # If installDir is provided, add it early to the path + if config.INSTALL_DIR is not None: + path.append(config.INSTALL_DIR) + + path.append(env_vars.get("PATH", os.environ.get("PATH", ""))) + env_vars["PATH"] = os.pathsep.join(path) kwargs["env_vars"] = env_vars return process_cls(*args, **kwargs) diff --git a/buildscripts/resmokelib/parser.py b/buildscripts/resmokelib/parser.py index 6093d0f5a2c..a5f5dc88601 100644 --- a/buildscripts/resmokelib/parser.py +++ b/buildscripts/resmokelib/parser.py @@ -627,9 +627,10 @@ def _update_config_vars(values): # pylint: disable=too-many-statements,too-many _config.INSTALL_DIR = config.pop("install_dir") if _config.INSTALL_DIR is not None: - # Inject INSTALL_DIR into the $PATH so RunProgram in the shell - # helpers can find the installed binaries. - os.environ['PATH'] = "{}:{}".format(_expand_user(_config.INSTALL_DIR), os.environ['PATH']) + # Normalize the path so that on Windows dist-test/bin + # translates to .\dist-test\bin then absolutify it since the + # Windows PATH variable requires absolute paths. + _config.INSTALL_DIR = os.path.abspath(_expand_user(os.path.normpath(_config.INSTALL_DIR))) for binary in ["mongo", "mongod", "mongos", "dbtest"]: keyname = binary + "_executable" diff --git a/buildscripts/resmokelib/testing/testcases/jstest.py b/buildscripts/resmokelib/testing/testcases/jstest.py index 12947ef867e..befcd3892dc 100644 --- a/buildscripts/resmokelib/testing/testcases/jstest.py +++ b/buildscripts/resmokelib/testing/testcases/jstest.py @@ -52,18 +52,6 @@ class _SingleJSTestCase(interface.ProcessTestCase): global_vars["MongoRunner.dataDir"] = data_dir global_vars["MongoRunner.dataPath"] = data_path - # Don't set the path to the mongod and mongos executables when the user didn't specify them - # via the command line. The functions in the mongo shell for spawning processes have their - # own logic for determining the default path to use. - if config.MONGOD_EXECUTABLE is not None: - global_vars["MongoRunner.mongodPath"] = config.MONGOD_EXECUTABLE - if config.MONGOS_EXECUTABLE is not None: - global_vars["MongoRunner.mongosPath"] = config.MONGOS_EXECUTABLE - # We provide an absolute path for mongo shell to ensure that programs starting their own - # mongo shell will use the same as specified from resmoke.py. - global_vars["MongoRunner.mongoShellPath"] = os.path.abspath( - utils.default_if_none(self.shell_executable, config.DEFAULT_MONGO_EXECUTABLE)) - test_data = global_vars.get("TestData", {}).copy() test_data["minPort"] = core.network.PortAllocator.min_test_port(self.fixture.job_num) test_data["maxPort"] = core.network.PortAllocator.max_test_port(self.fixture.job_num) diff --git a/buildscripts/resmokelib/testing/testcases/sdam_json_test.py b/buildscripts/resmokelib/testing/testcases/sdam_json_test.py index 27c9e4e94ef..adf9ce1a0f6 100644 --- a/buildscripts/resmokelib/testing/testcases/sdam_json_test.py +++ b/buildscripts/resmokelib/testing/testcases/sdam_json_test.py @@ -3,6 +3,7 @@ import os import os.path from . import interface from ... import core +from ... import config from ... import utils from ...utils import globstar from ... import errors @@ -12,7 +13,10 @@ class SDAMJsonTestCase(interface.ProcessTestCase): """Server Discovery and Monitoring JSON test case.""" REGISTERED_NAME = "sdam_json_test" - EXECUTABLE_BUILD_PATH = "build/**/mongo/client/sdam/sdam_json_test" + if config.INSTALL_DIR is not None: + EXECUTABLE_BUILD_PATH = os.path.join(config.INSTALL_DIR, "sdam_json_test") + else: + EXECUTABLE_BUILD_PATH = "build/**/mongo/client/sdam/sdam_json_test" TEST_DIR = os.path.normpath("src/mongo/client/sdam/json_tests") def __init__(self, logger, json_test_file, program_options=None): diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 4e47173fc67..64efa8ebcba 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -272,8 +272,9 @@ variables: modules: - enterprise expansions: &enterprise-windows-nopush-expansions-template - exe: ".exe" + additional_targets: archive-mongocryptd archive-mongocryptd-debug msi_target: msi + exe: ".exe" content_type: application/zip compile_flags: --ssl MONGO_DISTMOD=windows CPPPATH="c:/sasl/include c:/snmp/include" LIBPATH="c:/sasl/lib c:/snmp/lib" -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) --win-version-min=win10 # We invoke SCons using --jobs = (# of CPUs / 4) to avoid causing out of memory errors due to @@ -536,7 +537,6 @@ functions: script: | set -o errexit ${decompress|tar xzvf} mongo-binaries.tgz - cp mongodb*/bin/* . "check binary version": &check_binary_version command: shell.exec @@ -544,12 +544,7 @@ functions: working_dir: src script: | set -o errexit - mongo_binary=$(find mongodb*/bin -name mongo${exe}) - # There should only be one mongo shell - if [ $(echo $mongo_binary | wc -w) -ne 1 ]; then - echo "There is more than 1 extracted mongo binary: $mongo_binary" - exit 1 - fi + mongo_binary=dist-test/bin/mongo${exe} # For compile bypass we need to skip the binary version check since we can tag a commit # after the base commit binaries were created. This would lead to a mismatch of the binaries # and the version from git describe --abbrev=7 in the compile_expansions.yml. @@ -609,7 +604,6 @@ functions: script: | set -o errexit ${decompress} mh.tgz - cp mongodb*/bin/mqlrun . "archive new corpus": &archive_new_corpus command: archive.targz_pack @@ -849,7 +843,7 @@ functions: optional: true aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: src/mongo-debugsymbols.tgz + local_file: src/mongo-debugsymbols.${ext|tgz} remote_file: ${mongo_debugsymbols} bucket: mciuploads permissions: public-read @@ -1097,7 +1091,7 @@ functions: if [ -n "${burn_in_bypass}" ]; then ${activate_virtualenv} # Evergreen executable is in $HOME, so add that to the path. - PATH=$PATH:$HOME $python buildscripts/burn_in_tags_bypass_compile_and_fetch_binaries.py \ + DESTDIR=${destdir} PATH=$PATH:$HOME $python buildscripts/burn_in_tags_bypass_compile_and_fetch_binaries.py \ --project ${project} \ --build-variant ${burn_in_bypass} \ --revision ${revision} \ @@ -1109,7 +1103,7 @@ functions: elif [[ "${is_patch}" = "true" && "${task_name}" = "compile" ]]; then ${activate_virtualenv} # Evergreen executable is in $HOME, so add that to the path. - PATH=$PATH:$HOME $python buildscripts/bypass_compile_and_fetch_binaries.py \ + DESTDIR=${destdir} PATH=$PATH:$HOME $python buildscripts/bypass_compile_and_fetch_binaries.py \ --project ${project} \ --build-variant ${build_variant} \ --revision ${revision} \ @@ -1193,6 +1187,8 @@ functions: value: ${project}/${build_variant}/${revision}/binaries/mongo-${build_id}.${ext|tgz} - key: mongo_cryptd value: ${project}/${build_variant}/${revision}/binaries/mongo-cryptd-${build_id}.${ext|tgz} + - key: mongo_cryptd_debugsymbols + value: ${project}/${build_variant}/${revision}/binaries/mongo-cryptd-debugsymbols-${build_id}.${ext|tgz} - key: mh_archive value: ${project}/${build_variant}/${revision}/binaries/mh-${build_id}.${ext|tgz} - key: mh_debugsymbols @@ -1201,6 +1197,8 @@ functions: value: ${project}/${build_variant}/${revision}/debugsymbols/debugsymbols-${build_id}.${ext|tgz} - key: mongo_shell value: ${project}/${build_variant}/${revision}/binaries/mongo-shell-${build_id}.${ext|tgz} + - key: mongo_shell_debugsymbols + value: ${project}/${build_variant}/${revision}/binaries/mongo-shell-debugsymbols-${build_id}.${ext|tgz} - key: skip_tests value: skip_test-${build_id} @@ -1433,33 +1431,34 @@ functions: # invocation. It doesn't set any environment variables and should therefore come last in # this list of expansions. set +o errexit - PATH="$path_value" \ - AWS_PROFILE=${aws_profile_remote} \ - RSK_is_asan_build=${is_asan_build|""} \ - ${gcov_environment} \ - ${lang_environment} \ - ${san_options} \ - ${san_symbolizer} \ - ${snmp_config_path} \ - ${resmoke_wrapper} \ - $python buildscripts/evergreen_run_tests.py \ - ${resmoke_args} \ - $extra_args \ - ${test_flags} \ - --log=buildlogger \ - --staggerJobs=on \ - --buildId=${build_id} \ - --distroId=${distro_id} \ - --executionNumber=${execution} \ - --projectName=${project} \ - --gitRevision=${revision} \ - --revisionOrderId=${revision_order_id} \ - --taskId=${task_id} \ - --taskName=${task_name} \ - --variantName=${build_variant} \ - --versionId=${version_id} \ - --archiveFile=archive.json \ - --reportFile=report.json \ + PATH="$path_value" \ + AWS_PROFILE=${aws_profile_remote} \ + RSK_is_asan_build=${is_asan_build|""} \ + ${gcov_environment} \ + ${lang_environment} \ + ${san_options} \ + ${san_symbolizer} \ + ${snmp_config_path} \ + ${resmoke_wrapper} \ + $python buildscripts/evergreen_run_tests.py \ + ${resmoke_args} \ + $extra_args \ + ${test_flags} \ + --log=buildlogger \ + --staggerJobs=on \ + --installDir=${install_dir|dist-test/bin} \ + --buildId=${build_id} \ + --distroId=${distro_id} \ + --executionNumber=${execution} \ + --projectName=${project} \ + --gitRevision=${revision} \ + --revisionOrderId=${revision_order_id} \ + --taskId=${task_id} \ + --taskName=${task_name} \ + --variantName=${build_variant} \ + --versionId=${version_id} \ + --archiveFile=archive.json \ + --reportFile=report.json \ --perfReportFile=perf.json resmoke_exit_code=$? set -o errexit @@ -1796,7 +1795,8 @@ functions: fi rm -rf ${install_directory|/data/mongo-install-directory} - extra_args="" + # Only allow hygienic builds in Evergreen + extra_args="--install-mode=hygienic" if [ -n "${num_scons_link_jobs_available|}" ]; then echo "Changing SCons to run with --jlink=${num_scons_link_jobs_available|}" extra_args="$extra_args --jlink=${num_scons_link_jobs_available|}" @@ -1809,6 +1809,13 @@ functions: # Enable performance debugging extra_args="$extra_args --debug=time" + # Build packages where the upload tasks expect them + if [ -n "${git_project_directory|}" ]; then + extra_args="$extra_args PKGDIR=${git_project_directory}" + else + extra_args="$extra_args PKGDIR=${workdir}/src" + fi + # If we are doing a patch build or we are building a non-push # build on the waterfall, then we don't need the --release # flag. Otherwise, this is potentially a build that "leaves @@ -1830,7 +1837,7 @@ functions: ${compile_env|} $python ./buildscripts/scons.py \ ${compile_flags|} ${task_compile_flags|} ${task_compile_flags_extra|} \ ${scons_cache_args|} $extra_args \ - ${targets} ${additional_targets|} MONGO_VERSION=${version} || exit_status=$? + ${targets} MONGO_VERSION=${version} || exit_status=$? # If compile fails we do not run any tests if [[ $exit_status -ne 0 ]]; then if [[ "${dump_scons_config_on_failure}" == true ]]; then @@ -2185,8 +2192,6 @@ functions: set -o errexit set -o verbose - cp mongodb*/bin/mongod . - git clone --depth 1 git@github.com:10gen/mongo-enterprise-modules.git jstests/enterprise_tests git clone --depth 1 git@github.com:10gen/QA.git jstests/qa_tests @@ -2353,6 +2358,25 @@ functions: echo "Exiting powercycle with code $exit_code" exit $exit_code + "run packager.py": + command: shell.exec + params: + working_dir: src + script: | + set -o errexit + set -o verbose + + if [ "${is_patch}" = "true" ] && [ "${bypass_compile|false}" = "true" ]; then + exit 0 + fi + + ${activate_virtualenv} + if [ "${has_packages|}" = "true" ] ; then + cd buildscripts + $python ${packager_script} --prefix `pwd`/.. --distros ${packager_distro} --tarball `pwd`/../mongodb-dist.tgz -s ${version} -m HEAD -a ${packager_arch} + cd .. + fi + "do snmp setup": command: shell.exec params: @@ -3433,32 +3457,36 @@ functions: - *tar_failed_unittests - *archive_failed_unittests - "detect failed dbtest": &detect_failed_dbtest - command: shell.exec + "archive dbtest": &archive_dbtest + command: s3.put params: - working_dir: "src" - script: | - if [ -f resmoke_error_code ] && [ -f ../dbtest_unstripped.tgz ]; then - echo "Task dbtest failed with error $(cat resmoke_error_code), archiving the unstripped binary" - mv ../dbtest_unstripped.tgz ../dbtest.tgz - fi + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: dbtest-binary.tgz + remote_file: ${project}/${build_variant}/${revision}/dbtest/dbtest-${build_id}-${task_name}-${execution}.tgz + bucket: mciuploads + permissions: public-read + content_type: application/tar + display_name: dbtest binary - Execution ${execution} + optional: true - "archive unstripped dbtest": &archive_unstripped_dbtest + "archive dbtest debugsymbols": &archive_dbtest_debug command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: dbtest.tgz + local_file: dbtest-debugsymbols.tgz remote_file: ${project}/${build_variant}/${revision}/dbtest/dbtest-${build_id}-${task_name}-${execution}.tgz bucket: mciuploads permissions: public-read content_type: application/tar - display_name: Unstripped dbtest binary - Execution ${execution} + display_name: dbtest debugsymbols optional: true "save unstripped dbtest": - *detect_failed_dbtest - - *archive_unstripped_dbtest + - *archive_dbtest + - *archive_dbtest_debug ### Process & archive artifacts from hung processes ### "run hang analyzer": @@ -3704,34 +3732,15 @@ tasks: - func: "scons compile" vars: targets: >- - archive-dist - archive-dist-debug - install-core - install-tools + archive-dist-test + archive-dist-test-debug distsrc-${ext|tgz} ${additional_targets|} - ${msi_target|} ${mh_target|} task_compile_flags: >- --detect-odr-violations - --install-mode=hygienic --separate-debug - --legacy-tarball - - command: shell.exec - type: test - params: - working_dir: src - script: | - set -o errexit - set -o verbose - - if [ "${is_patch}" = "true" ] && [ "${bypass_compile|false}" = "true" ]; then - exit 0 - fi - - mv mongodb-src-*.${ext|tgz} distsrc.${ext|tgz} - mv mongodb-*-debugsymbols.${ext|tgz} mongo-debugsymbols.tgz || true - mv mongodb-*.${ext|tgz} mongodb-binaries.tgz + PREFIX=dist-test # Tar unstripped dbtest, to be archived in case of failure - command: archive.targz_pack @@ -3748,78 +3757,36 @@ tasks: set -o errexit set -o verbose - if [ "${is_patch}" = "true" ] && [ "${bypass_compile|false}" = "true" ]; then - exit 0 - fi - ${activate_virtualenv} - if [ "${has_packages|}" = "true" ] ; then - cd buildscripts - $python ${packager_script} --prefix `pwd`/.. --distros ${packager_distro} --tarball `pwd`/../mongodb-binaries.tgz -s ${version} -m HEAD -a ${packager_arch} - cd .. - fi - - # Create separate shell archive - mkdir -p shell-archive/build - cd shell-archive - ${decompress|tar xzvf} ../mongodb-binaries.tgz - find . -mindepth 3 ! -name "mongo${exe}" -type f -exec rm {} \; # delete bin/* except bin/mongo - $python ../buildscripts/make_archive.py -o mongodb-shell.${ext|tgz} $(find mongodb-* -type f) - cd .. - - # Create separate cryptd archive if mongocryptd is in the tarball - mkdir -p cryptd-archive/build - cd cryptd-archive - ${decompress|tar xzvf} ../mongodb-binaries.tgz - find . -mindepth 3 ! -name "mongocryptd${exe}" -type f -exec rm {} \; # delete bin/* except bin/mongocryptd if [ $(find . -name mongocryptd${exe} | wc -l) -eq 1 ] ; then - $python ../buildscripts/make_archive.py -o mongodb-cryptd.${ext|tgz} $(find mongodb-* -type f) - # Validate that this build_variant is listed as a known enterprise task for mongocryptd PATH=$PATH:$HOME $python ../buildscripts/validate_mongocryptd.py --variant "${build_variant}" ../etc/evergreen.yml fi - cd .. - - - command: archive.targz_pack - params: - target: "packages.tgz" - source_dir: "src" - include: - - "repo/**" - command: archive.targz_pack params: target: "artifacts.tgz" source_dir: "src" include: - - "src/mongo/db/modules/enterprise/jstests/**" + - "./build/**.gcno" + - "./etc/*san.suppressions" + - "./etc/pip/**" + - "./etc/repo_config.yaml" + - "./etc/scons/**" + - "artifacts.json" + - "buildscripts/**" + - "bypass_compile_expansions.yml" - "compile_expansions.yml" - - "src/mongo/db/modules/subscription/jstests/**" - - "src/mongo/db/modules/enterprise/docs/**" - - "*.exe" - "jstests/**" + - "library_dependency_graph.json" + - "patch_files.txt" - "pytests/**" - - "./test*" - - "./mongobridge*" - - "./mongotmock*" - - "./wt*" - - "buildscripts/**" - - "*Example" - - "*Test" - - "./**.pdb" - - "./**.msi" - - "./etc/pip/**" - - "./etc/scons/**" - - "./etc/*san.suppressions" - - "./etc/repo_config.yaml" - - "./build/**.gcno" - - "src/mongo/util/options_parser/test_config_files/**" - "src/mongo/client/sdam/json_tests/**" - - "library_dependency_graph.json" + - "src/mongo/db/modules/enterprise/docs/**" + - "src/mongo/db/modules/enterprise/jstests/**" + - "src/mongo/db/modules/subscription/jstests/**" + - "src/mongo/util/options_parser/test_config_files/**" - "src/third_party/JSON-Schema-Test-Suite/tests/draft4/**" - "src/third_party/mock_ocsp_responder/**" - - "bypass_compile_expansions.yml" - - "patch_files.txt" - - "artifacts.json" exclude_files: - "*_test.pdb" @@ -3829,7 +3796,7 @@ tasks: optional: true aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: src/mongodb-binaries.tgz + local_file: src/mongodb-binaries.${ext|tgz} remote_file: ${mongo_binaries} bucket: mciuploads permissions: public-read @@ -3840,7 +3807,7 @@ tasks: optional: true aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: src/cryptd-archive/mongodb-cryptd.${ext|tgz} + local_file: src/mongodb-cryptd.${ext|tgz} remote_file: ${mongo_cryptd} bucket: mciuploads permissions: public-read @@ -3851,34 +3818,34 @@ tasks: optional: true aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: src/mh-binaries.${ext|tgz} - remote_file: ${mh_archive} + local_file: src/mongodb-cryptd.${ext|tgz} + remote_file: ${mongo_cryptd_debugsymbols} bucket: mciuploads permissions: public-read content_type: ${content_type|application/gzip} - display_name: MH Binaries + display_name: CryptD Debugsymbols - command: s3.put params: optional: true aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: src/mh-debugsymbols.${ext|tgz} - remote_file: ${mh_debugsymbols} + local_file: src/mh-binaries.${ext|tgz} + remote_file: ${mh_archive} bucket: mciuploads permissions: public-read content_type: ${content_type|application/gzip} - display_name: MH Debuginfo + display_name: MH Binaries - command: s3.put params: optional: true aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: src/shell-archive/mongodb-shell.${ext|tgz} - remote_file: ${mongo_shell} + local_file: src/mh-debugsymbols.${ext|tgz} + remote_file: ${mh_debugsymbols} bucket: mciuploads permissions: public-read content_type: ${content_type|application/gzip} - display_name: Shell + display_name: MH Debuginfo - command: s3.put params: aws_key: ${aws_key} @@ -3891,17 +3858,6 @@ tasks: display_name: Artifacts - command: s3.put params: - optional: true - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: packages.tgz - remote_file: ${project}/${build_variant}/${revision}/artifacts/${build_id}-packages.tgz - bucket: mciuploads - permissions: public-read - content_type: application/tar - display_name: Packages - - command: s3.put - params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: src/distsrc.${ext|tgz} @@ -3944,7 +3900,6 @@ tasks: additional_targets: "" task_compile_flags: >- --detect-odr-violations - --install-mode=hygienic --separate-debug - name: compile_ninja @@ -3966,10 +3921,11 @@ tasks: commands: - func: "scons compile" vars: - targets: all + targets: install-all-meta compiling_for_test: true task_compile_flags: >- --detect-odr-violations + --separate-debug - command: s3.put params: aws_key: ${aws_key} @@ -4008,9 +3964,10 @@ tasks: commands: - func: "scons compile" vars: - targets: unittests + targets: install-unittests install-unittests-debug task_compile_flags: >- --detect-odr-violations + --separate-debug compiling_for_test: true - func: "run diskstats" - func: "monitor process threads" @@ -4024,23 +3981,15 @@ tasks: commands: - func: "scons compile" vars: - targets: libfuzzer_tests + targets: archive-fuzzertests task_compile_flags: >- --detect-odr-violations compiling_for_test: true - - command: shell.exec - params: - working_dir: "src" - script: | - set -o errexit - set -o verbose - - tar cfvz libfuzzer-tests.tgz --files-from=build/libfuzzer_tests.txt - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: "src/libfuzzer-tests.tgz" + local_file: "src/fuzzertests-runtime.tgz" remote_file: "${project}/libfuzzer-tests/${build_variant}/${revision}/libfuzzer-tests.tgz" bucket: mciuploads permissions: public-read @@ -4060,7 +4009,7 @@ tasks: commands: - func: "scons compile" vars: - targets: "sdam_json_test_runner" + targets: "install-sdam-json-test" task_compile_flags: >- --detect-odr-violations compiling_for_test: true @@ -4073,23 +4022,18 @@ tasks: commands: - func: "scons compile" vars: - targets: dbtest + targets: archive-dbtest archive-dbtest-debug task_compile_flags: >- --detect-odr-violations + --separate-debug compiling_for_test: true - # Tar unstripped dbtest, to be archived in case of failure - - command: archive.targz_pack - params: - target: "dbtest_unstripped.tgz" - source_dir: "src" - include: - - "./dbtest*" - func: "run diskstats" - func: "monitor process threads" - func: "collect system resource info" - func: "run tests" vars: resmoke_args: --suites=dbtest --storageEngine=wiredTiger + install_dir: build/install/bin ## embedded_sdk_build_and_test_* - build the embedded-dev and embedded-test targets only ## @@ -4179,7 +4123,6 @@ tasks: --dbg=off --enable-free-mon=off --enable-http-client=off - --install-mode=hygienic --js-engine=none --opt=size --separate-debug @@ -4323,7 +4266,6 @@ tasks: vars: targets: install-stitch-support install-stitch-support-debug install-stitch-support-dev task_compile_flags: >- - --install-mode=hygienic --dbg=off --link-model=dynamic-sdk --enable-free-mon=off @@ -4358,7 +4300,6 @@ tasks: targets: install-stitch-support-test compiling_for_test: true task_compile_flags: >- - --install-mode=hygienic --dbg=off --enable-free-mon=off --ssl=off @@ -4396,7 +4337,7 @@ tasks: - func: "apply compile expansions" - func: "scons compile" vars: - targets: benchmarks + targets: install-benchmarks compiling_for_test: true - command: archive.targz_pack params: @@ -5787,7 +5728,7 @@ tasks: - func: "apply compile expansions" - func: "scons compile" vars: - targets: integration_tests + targets: install-integration-tests compiling_for_test: true bypass_compile: false task_compile_flags: >- @@ -5808,7 +5749,7 @@ tasks: - func: "apply compile expansions" - func: "scons compile" vars: - targets: integration_tests + targets: install-integration-tests compiling_for_test: true bypass_compile: false task_compile_flags: >- @@ -5829,7 +5770,7 @@ tasks: - func: "apply compile expansions" - func: "scons compile" vars: - targets: integration_tests + targets: install-integration-tests compiling_for_test: true bypass_compile: false task_compile_flags: >- @@ -5850,7 +5791,7 @@ tasks: - func: "apply compile expansions" - func: "scons compile" vars: - targets: integration_tests + targets: install-integration-tests compiling_for_test: true bypass_compile: false task_compile_flags: >- @@ -7605,11 +7546,11 @@ tasks: resmoke_args: --suites=buildscripts_test resmoke_jobs_max: 1 -- name: package +- name: test_packages depends_on: - - name: compile + - name: package commands: - - func: "fetch artifacts" + - func: "fetch packages" - func: "set up remote credentials" vars: private_key_file: ~/.ssh/kitchen.pem @@ -7618,6 +7559,74 @@ tasks: aws_secret_remote: ${kitchen_aws_secret} - func: "run kitchen" +- name: package + commands: + - func: "scons compile" + vars: + targets: >- + archive-dist + archive-dist-debug + archive-shell + archive-shell-debug + ${msi_target|} + task_compile_flags: >- + --detect-odr-violations + --separate-debug + --legacy-tarball + - func: "run packager.py" + - command: archive.targz_pack + params: + target: "packages.tgz" + source_dir: "src" + include: + - "repo/**" + - "./**.msi" + - command: s3.put + params: + optional: true + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/mongodb-dist.${ext|tgz} + remote_file: ${project}/${build_variant}/${revision}/dist/mongo-${build_id}.${ext|tgz} + bucket: mciuploads + permissions: public-read + content_type: application/tar + display_name: Dist Tarball + - command: s3.put + params: + optional: true + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: packages.tgz + remote_file: ${project}/${build_variant}/${revision}/artifacts/${build_id}-packages.tgz + bucket: mciuploads + permissions: public-read + content_type: application/tar + display_name: Packages + - command: s3.put + params: + optional: true + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/mongodb-shell.${ext|tgz} + remote_file: ${mongo_shell} + bucket: mciuploads + permissions: public-read + content_type: ${content_type|application/gzip} + display_name: Shell + - command: s3.put + params: + optional: true + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/mongodb-shell-debugsymobls.${ext|tgz} + remote_file: ${mongo_shell_debugsymbols} + bucket: mciuploads + permissions: public-read + content_type: ${content_type|application/gzip} + display_name: Shell Debugsymbols + + - name: publish_packages tags: ["publish"] # This should prevent this task from running in patch builds, where we @@ -7627,6 +7636,7 @@ tasks: # Same dependencies as "push" below depends_on: - name: compile + - name: package - name: jsCore - name: dbtest - name: replica_sets_jscore_passthrough @@ -7657,6 +7667,7 @@ tasks: tags: ["publish"] patchable: false depends_on: + - name: package - name: jsCore - name: dbtest - name: replica_sets_jscore_passthrough @@ -8310,6 +8321,10 @@ task_groups: tasks: - compile - <<: *compile_task_group_template + name: compile_packages_TG + tasks: + - package +- <<: *compile_task_group_template name: compile_core_tools_TG tasks: - compile_core_tools @@ -8603,7 +8618,8 @@ buildvariants: - name: watchdog_wiredtiger - name: .ssl - name: .stitch - - name: package + - name: compile_packages_TG + - name: test_packages distros: - ubuntu1604-packer - name: .publish @@ -8616,6 +8632,7 @@ buildvariants: - ubuntu1804-test stepback: false expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug push_path: linux push_bucket: downloads.10gen.com push_name: linux @@ -8656,7 +8673,8 @@ buildvariants: - name: sharding_auth_gen - name: snmp - name: .watchdog - - name: package + - name: compile_packages_TG + - name: test_packages distros: - ubuntu1604-packer - name: .publish @@ -8718,7 +8736,8 @@ buildvariants: - name: .sharding .txns - name: .stitch - name: .ssl - - name: package + - name: compile_packages_TG + - name: test_packages distros: - ubuntu1604-packer - name: .publish @@ -8730,6 +8749,7 @@ buildvariants: run_on: - ubuntu1604-arm64-large expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug push_path: linux push_bucket: downloads.10gen.com push_name: linux @@ -8765,6 +8785,7 @@ buildvariants: - name: sharding_jscore_passthrough - name: .ssl - name: .stitch + - name: compile_packages_TG - name: .publish distros: - ubuntu1604-test @@ -8776,6 +8797,7 @@ buildvariants: run_on: - ubuntu1804-arm64-build expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug push_path: linux push_bucket: downloads.10gen.com push_name: linux @@ -8811,6 +8833,7 @@ buildvariants: - name: sharding_jscore_passthrough - name: .ssl - name: .stitch + - name: compile_packages_TG - name: .publish distros: - ubuntu1804-test @@ -8840,6 +8863,7 @@ buildvariants: - name: free_monitoring - name: jsCore - name: replica_sets_jscore_passthrough + - name: compile_packages_TG - name: .publish distros: - ubuntu1804-test @@ -8851,6 +8875,7 @@ buildvariants: run_on: - ubuntu1804-power8-test expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug push_path: linux push_bucket: downloads.10gen.com push_name: linux @@ -8893,6 +8918,7 @@ buildvariants: - name: sharding_jscore_passthrough - name: .ssl - name: .stitch + - name: compile_packages_TG - name: .publish distros: - ubuntu1804-test @@ -8923,6 +8949,7 @@ buildvariants: - name: jsCore - name: replica_sets_jscore_passthrough - name: ssl_gen + - name: compile_packages_TG - name: .publish distros: - ubuntu1804-test @@ -8936,6 +8963,7 @@ buildvariants: batchtime: 10080 # 7 days stepback: false expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug push_path: linux push_bucket: downloads.10gen.com push_name: linux @@ -8979,6 +9007,7 @@ buildvariants: - name: .sharding .common !.multiversion - name: .sharding .txns - name: snmp + - name: compile_packages_TG - name: .publish distros: - ubuntu1804-test @@ -8990,6 +9019,7 @@ buildvariants: run_on: - amazon1-2018-test expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug push_path: linux push_bucket: downloads.10gen.com push_name: linux @@ -9029,7 +9059,8 @@ buildvariants: - name: slow1_gen - name: snmp - name: .stitch - - name: package + - name: compile_packages_TG + - name: test_packages distros: - ubuntu1604-packer - name: .publish @@ -9079,7 +9110,8 @@ buildvariants: - name: .sharding .txns - name: .ssl - name: .stitch - - name: package + - name: compile_packages_TG + - name: test_packages distros: - ubuntu1604-packer - name: .publish @@ -9091,6 +9123,7 @@ buildvariants: run_on: - amazon2-test expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug test_flags: >- --excludeWithAnyTags=SERVER-34286 push_path: linux @@ -9133,7 +9166,8 @@ buildvariants: - name: slow1_gen - name: snmp - name: .stitch - - name: package + - name: compile_packages_TG + - name: test_packages distros: - ubuntu1604-packer - name: .publish @@ -9187,7 +9221,8 @@ buildvariants: - name: .sharding .txns - name: .ssl - name: .stitch - - name: package + - name: compile_packages_TG + - name: test_packages distros: - ubuntu1604-packer - name: .publish @@ -9206,6 +9241,45 @@ buildvariants: # Windows buildvariants # ########################################### +- name: enterprise-windows-required + display_name: "! Enterprise Windows" + batchtime: 60 # 1 hour + modules: + - enterprise + run_on: + - windows-64-vs2017-test + expansions: &enterprise_windows_expansions + additional_targets: archive-mongocryptd archive-mongocryptd-debug + msi_target: msi + burn_in_tests_build_variant: enterprise-windows + exe: ".exe" + mh_target: archive-mh archive-mh-debug + content_type: application/zip + compile_flags: --ssl MONGO_DISTMOD=windows CPPPATH="c:/sasl/include c:/snmp/include" LIBPATH="c:/sasl/lib c:/snmp/lib" -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) --win-version-min=win10 + # We invoke SCons using --jobs = (# of CPUs / 4) to avoid causing out of memory errors due to + # spawning a large number of linker processes. + num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 )) + python: '/cygdrive/c/python/python37/python.exe' + ext: zip + scons_cache_scope: shared + multiversion_platform: windows + multiversion_edition: enterprise + jstestfuzz_num_generated_files: 35 + target_resmoke_time: 20 + large_distro_name: windows-64-vs2017-compile + tasks: + - name: compile_TG + distros: + - windows-64-vs2017-compile + - name: burn_in_tests_gen + - name: buildscripts_test + - name: unittest_shell_hang_analyzer_gen + - name: server_discovery_and_monitoring_json_test_TG + - name: dbtest_TG + distros: + - windows-64-vs2017-compile + - name: noPassthrough_gen + - name: windows-debug display_name: "* Windows DEBUG" batchtime: 60 # 1 hour @@ -9337,6 +9411,7 @@ buildvariants: - name: sharding_ese_gen - name: snmp - name: unittest_shell_hang_analyzer_gen + - name: compile_packages_TG - name: push distros: - rhel70-small @@ -9369,8 +9444,9 @@ buildvariants: stepback: true batchtime: 10080 # 7 days expansions: - exe: ".exe" + additional_targets: archive-mongocryptd archive-mongocryptd-debug msi_target: msi + exe: ".exe" content_type: application/zip compile_flags: --ssl MONGO_DISTMOD=windows CPPPATH="c:/sasl/include c:/snmp/include" LIBPATH="c:/sasl/lib c:/snmp/lib" -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) --win-version-min=win10 # We invoke SCons using --jobs = (# of CPUs / 4) to avoid causing out of memory errors due to @@ -9398,8 +9474,9 @@ buildvariants: run_on: - windows-64-vs2019-small expansions: - exe: ".exe" + additional_targets: archive-mongocryptd archive-mongocryptd-debug msi_target: msi + exe: ".exe" content_type: application/zip compile_flags: --ssl MONGO_DISTMOD=windows CPPPATH="c:/sasl/include c:/snmp/include" LIBPATH="c:/sasl/lib c:/snmp/lib" -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) --win-version-min=win10 # We invoke SCons using --jobs = (# of CPUs / 4) to avoid causing out of memory errors due to @@ -9444,6 +9521,7 @@ buildvariants: run_on: - windows-64-vs2019-small expansions: + msi_target: msi exe: ".exe" push_path: windows push_bucket: downloads.mongodb.org @@ -9451,7 +9529,6 @@ buildvariants: push_arch: x86_64 multiversion_platform: windows_x86_64-2008plus-ssl multiversion_platform_42_or_later: windows_x86_64-2012plus - msi_target: msi content_type: application/zip compile_flags: --ssl MONGO_DISTMOD=windows -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) --win-version-min=win10 # We invoke SCons using --jobs = (# of CPUs / 4) to avoid causing out of memory errors due to @@ -9494,6 +9571,7 @@ buildvariants: - name: .ssl - name: .stitch - name: .updatefuzzer + - name: compile_packages_TG - name: push distros: - rhel70-small @@ -9505,6 +9583,7 @@ buildvariants: run_on: - windows-64-vs2019-small expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug exe: ".exe" content_type: application/zip compile_flags: --dbg=on --opt=off --ssl MONGO_DISTMOD=windows CPPPATH="c:/sasl/include c:/snmp/include" LIBPATH="c:/sasl/lib c:/snmp/lib" -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) --win-version-min=win10 @@ -9570,6 +9649,7 @@ buildvariants: - name: .sharding .txns - name: .ssl - name: .stitch + - name: compile_packages_TG - name: push distros: - rhel70-small @@ -9605,6 +9685,7 @@ buildvariants: run_on: - macos-1014 expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug push_path: osx push_bucket: downloads.10gen.com push_name: macos @@ -9626,6 +9707,7 @@ buildvariants: - name: replica_sets_auth_gen - name: replica_sets_jscore_passthrough - name: sasl + - name: compile_packages_TG - name: push distros: - rhel70-small @@ -9697,6 +9779,7 @@ buildvariants: run_on: - rhel62-small expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug push_path: linux push_bucket: downloads.10gen.com push_name: linux @@ -9789,7 +9872,8 @@ buildvariants: - name: .stitch - name: .updatefuzzer - name: secondary_reads_passthrough_gen - - name: package + - name: compile_packages_TG + - name: test_packages distros: - ubuntu1604-packer - name: .publish @@ -9803,6 +9887,7 @@ buildvariants: - rhel62-small batchtime: 10080 # 7 days expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug compile_flags: >- --ssl MONGO_DISTMOD=rhel62 @@ -9870,6 +9955,7 @@ buildvariants: run_on: - rhel62-small expansions: &enterprise-rhel-62-64-bit-majority-read-concern-off-expansions + additional_targets: archive-mongocryptd archive-mongocryptd-debug # Ban tests that run prepareTransaction or multi-shard transactions (which use # prepareTransaction). prepareTransaction is rejected on nodes with # enableMajorityReadConcern:"false" (SERVER-37559). @@ -9983,6 +10069,7 @@ buildvariants: - rhel62-small batchtime: 10080 # 7 days expansions: &enterprise-rhel-62-64-bit-flow-control-off + additional_targets: archive-mongocryptd archive-mongocryptd-debug test_flags: >- --flowControl=off --excludeWithAnyTags=requires_flow_control @@ -10059,6 +10146,7 @@ buildvariants: run_on: - rhel62-small expansions: &enterprise-rhel-62-64-bit-single-phase-index-builds + additional_targets: archive-mongocryptd archive-mongocryptd-debug test_flags: >- --mongodSetParameters="{enableTwoPhaseIndexBuild: false}" compile_flags: >- @@ -10129,6 +10217,7 @@ buildvariants: batchtime: 10080 # 7 days stepback: false expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug compile_flags: --dbg=on --gcov --ssl MONGO_DISTMOD=rhel62 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars multiversion_platform: rhel62 multiversion_edition: enterprise @@ -10191,6 +10280,7 @@ buildvariants: run_on: - rhel70-small expansions: &enterprise-rhel-70-64-bit-expansions-template + additional_targets: archive-mongocryptd archive-mongocryptd-debug push_path: linux push_bucket: downloads.10gen.com push_name: linux @@ -10233,7 +10323,8 @@ buildvariants: - name: sharding_auth_gen - name: snmp - name: .stitch - - name: package + - name: compile_packages_TG + - name: test_packages distros: - ubuntu1604-packer - name: .publish @@ -10248,6 +10339,7 @@ buildvariants: run_on: - rhel80-build expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug push_path: linux push_bucket: downloads.10gen.com push_name: linux @@ -10289,7 +10381,8 @@ buildvariants: - name: sharding_auth_gen - name: snmp - name: .stitch - - name: package + - name: compile_packages_TG + - name: test_packages distros: - ubuntu1604-packer - name: .publish @@ -10448,7 +10541,8 @@ buildvariants: - name: .sharding .txns - name: .sharding .common !.op_query !.csrs - name: .ssl - - name: package + - name: compile_packages_TG + - name: test_packages distros: - ubuntu1604-packer - name: .publish @@ -10497,7 +10591,8 @@ buildvariants: - name: .sharding .common !.op_query !.csrs - name: .ssl - name: .stitch - - name: package + - name: compile_packages_TG + - name: test_packages distros: - ubuntu1604-packer - name: .publish @@ -10548,7 +10643,8 @@ buildvariants: - name: .sharding .common !.op_query !.csrs - name: .ssl - name: .stitch - - name: package + - name: compile_packages_TG + - name: test_packages distros: - ubuntu1604-packer - name: .publish @@ -10576,6 +10672,7 @@ buildvariants: - rhel71-power8-test stepback: false expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug # We need to compensate for SMT8 setting the cpu count very high and lower the amount of parallelism down compile_flags: --ssl MONGO_DISTMOD=rhel71 -j$(echo "$(grep -c processor /proc/cpuinfo)/2" | bc) CCFLAGS="-mcpu=power8 -mtune=power8 -mcmodel=medium" --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars resmoke_jobs_factor: 0.25 @@ -10620,6 +10717,7 @@ buildvariants: - name: .sharding .common !.multiversion - name: snmp - name: .stitch + - name: compile_packages_TG - name: .publish distros: - rhel70 @@ -10633,6 +10731,7 @@ buildvariants: batchtime: 10080 # 7 days stepback: false expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug release_buid: true compile_flags: --ssl MONGO_DISTMOD=rhel72 -j3 CCFLAGS="-march=z196 -mtune=zEC12" --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars resmoke_jobs_max: 2 @@ -10676,6 +10775,7 @@ buildvariants: - name: .sharding .common !.multiversion - name: snmp - name: .stitch + - name: compile_packages_TG - name: .publish distros: - rhel70 @@ -10688,6 +10788,7 @@ buildvariants: - rhel67-zseries-test stepback: false expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug compile_flags: --ssl MONGO_DISTMOD=rhel67 -j3 CCFLAGS="-march=z9-109 -mtune=z10" --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --use-hardware-crc32=off has_packages: true packager_script: packager_enterprise.py @@ -10726,6 +10827,7 @@ buildvariants: - name: snmp - name: .stitch - name: secondary_reads_passthrough_gen + - name: compile_packages_TG - name: .publish distros: - rhel62-large @@ -10752,6 +10854,7 @@ buildvariants: - name: jsCore - name: replica_sets_jscore_passthrough - name: ssl_gen + - name: compile_packages_TG - name: .publish distros: - rhel70 @@ -10778,6 +10881,7 @@ buildvariants: - name: jsCore - name: replica_sets_jscore_passthrough - name: ssl_gen + - name: compile_packages_TG - name: .publish distros: - rhel62-large @@ -10793,6 +10897,7 @@ buildvariants: run_on: - ubuntu1604-test expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug push_path: linux push_bucket: downloads.10gen.com push_name: linux @@ -10829,7 +10934,8 @@ buildvariants: - name: snmp - name: .stitch - name: .watchdog - - name: package + - name: compile_packages_TG + - name: test_packages distros: - ubuntu1604-packer - name: .publish @@ -10843,6 +10949,7 @@ buildvariants: run_on: - ubuntu1604-test expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug lang_environment: LANG=C # We need --allocator=system here solely due to SERVER-27675 compile_flags: --link-model=dynamic --allocator=system -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars @@ -10865,6 +10972,7 @@ buildvariants: run_on: - ubuntu1604-build expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug lang_environment: LANG=C # We need --allocator=system here solely due to SERVER-27675 compile_flags: --link-model=dynamic --allocator=system -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars @@ -10903,6 +11011,7 @@ buildvariants: run_on: - suse12-test expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug push_path: linux push_bucket: downloads.10gen.com push_name: linux @@ -10933,7 +11042,8 @@ buildvariants: - name: sharding_auth_gen - name: snmp - name: .stitch - - name: package + - name: compile_packages_TG + - name: test_packages distros: - ubuntu1604-packer - name: .publish @@ -10947,6 +11057,7 @@ buildvariants: batchtime: 10080 # 7 days stepback: false expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug push_path: linux push_bucket: downloads.10gen.com push_name: linux @@ -10988,6 +11099,7 @@ buildvariants: - name: snmp - name: .stitch - name: secondary_reads_passthrough_gen + - name: compile_packages_TG - name: .publish distros: - suse12-test @@ -11019,6 +11131,7 @@ buildvariants: - name: jsCore - name: replica_sets_jscore_passthrough - name: ssl_gen + - name: compile_packages_TG - name: .publish distros: - suse12-test @@ -11067,6 +11180,7 @@ buildvariants: - name: .sharding .common !.op_query !.csrs - name: .ssl - name: .stitch + - name: compile_packages_TG - name: .publish - name: enterprise-suse15-64 @@ -11076,6 +11190,7 @@ buildvariants: run_on: - suse15-test expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug push_path: linux push_bucket: downloads.10gen.com push_name: linux @@ -11104,6 +11219,7 @@ buildvariants: - name: sharding_auth_gen - name: snmp - name: .stitch + - name: compile_packages_TG - name: .publish - name: suse15 @@ -11149,6 +11265,7 @@ buildvariants: - name: .sharding .common !.op_query !.csrs !.multiversion - name: .ssl - name: .stitch + - name: compile_packages_TG - name: .publish ########################################### @@ -11162,6 +11279,7 @@ buildvariants: run_on: - debian92-test expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug push_path: linux push_bucket: downloads.10gen.com push_name: linux @@ -11196,7 +11314,8 @@ buildvariants: - name: sharding_auth_gen - name: snmp - name: .stitch - - name: package + - name: compile_packages_TG + - name: test_packages distros: - ubuntu1604-packer - name: .publish @@ -11248,7 +11367,8 @@ buildvariants: - name: .sharding .common !.op_query !.csrs - name: .ssl - name: .stitch - - name: package + - name: compile_packages_TG + - name: test_packages distros: - ubuntu1604-packer - name: .publish @@ -11260,6 +11380,7 @@ buildvariants: run_on: - debian10-test expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug push_path: linux push_bucket: downloads.10gen.com push_name: linux @@ -11294,7 +11415,8 @@ buildvariants: - name: sharding_auth_gen - name: snmp - name: .stitch - - name: package + - name: compile_packages_TG + - name: test_packages distros: - ubuntu1604-packer - name: .publish @@ -11346,7 +11468,8 @@ buildvariants: - name: .sharding .common !.op_query !.csrs - name: .ssl - name: .stitch - - name: package + - name: compile_packages_TG + - name: test_packages distros: - ubuntu1604-packer - name: .publish @@ -11393,6 +11516,7 @@ buildvariants: run_on: - rhel62-small expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug test_flags: --storageEngine=inMemory --excludeWithAnyTags=requires_persistence,requires_journaling compile_flags: --ssl MONGO_DISTMOD=rhel62 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars multiversion_platform: rhel62 @@ -11454,6 +11578,7 @@ buildvariants: - rhel62-large batchtime: 1440 # 1 day expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug mh_target: archive-mh archive-mh-debug compile_flags: --ssl MONGO_DISTMOD=rhel62 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars CPPDEFINES=MONGO_CONFIG_LOGV2_DEFAULT multiversion_platform: rhel62 @@ -11547,6 +11672,7 @@ buildvariants: - rhel62-large batchtime: 1440 # 1 day expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug mh_target: archive-mh archive-mh-debug compile_flags: --ssl MONGO_DISTMOD=rhel62 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars CPPDEFINES=MONGO_CONFIG_LOGV2_DEFAULT multiversion_platform: rhel62 @@ -11685,6 +11811,7 @@ buildvariants: batchtime: 10080 # 7 days stepback: false expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug # We need to compensate for SMT8 setting the cpu count very high and lower the amount of parallelism down compile_flags: --dbg=on --opt=on --ssl MONGO_DISTMOD=rhel71 -j$(echo "$(grep -c processor /proc/cpuinfo)/2" | bc) CCFLAGS="-mcpu=power8 -mtune=power8 -mcmodel=medium" --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars resmoke_jobs_factor: 0.25 @@ -11726,6 +11853,7 @@ buildvariants: batchtime: 10080 # 7 days stepback: false expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug compile_flags: --dbg=on --opt=on --ssl MONGO_DISTMOD=rhel72 -j3 CCFLAGS="-march=z196 -mtune=zEC12" --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars resmoke_jobs_max: 2 test_flags: --storageEngine=inMemory --excludeWithAnyTags=requires_persistence,requires_journaling @@ -11773,6 +11901,7 @@ buildvariants: - ubuntu1804-build stepback: false expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug # We need llvm-symbolizer in the PATH for ASAN for clang-3.7 or later. variant_path_suffix: /opt/mongodbtoolchain/v3/bin lang_environment: LANG=C @@ -11840,6 +11969,7 @@ buildvariants: - ubuntu1804-build stepback: true expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug # We need llvm-symbolizer in the PATH for ASAN for clang-3.7 or later. variant_path_suffix: /opt/mongodbtoolchain/v3/bin lang_environment: LANG=C @@ -11866,6 +11996,7 @@ buildvariants: - ubuntu1804-build stepback: false expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug # We need llvm-symbolizer in the PATH for UBSAN. variant_path_suffix: /opt/mongodbtoolchain/v3/bin lang_environment: LANG=C @@ -11930,6 +12061,7 @@ buildvariants: - ubuntu1804-build stepback: true expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug # We need llvm-symbolizer in the PATH for ASAN for clang-3.7 or later. variant_path_suffix: /opt/mongodbtoolchain/v3/bin lang_environment: LANG=C @@ -11952,6 +12084,7 @@ buildvariants: - ubuntu1804-build stepback: false expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug # We need llvm-symbolizer in the PATH for ASAN for clang-3.7 or later. variant_path_suffix: /opt/mongodbtoolchain/v3/bin lang_environment: LANG=C @@ -12020,6 +12153,7 @@ buildvariants: modules: - enterprise expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug lang_environment: LANG=C # We need --allocator=system here solely due to SERVER-27675 compile_flags: MONGO_DISTMOD=ubuntu1604 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic --allocator=system diff --git a/jstests/auth/auth_pass_prompt.js b/jstests/auth/auth_pass_prompt.js index 0a1027cadff..6bb98b8f134 100644 --- a/jstests/auth/auth_pass_prompt.js +++ b/jstests/auth/auth_pass_prompt.js @@ -14,7 +14,7 @@ if (!_isWindows()) { "'var database = db.getMongo().getDB(" + database + "); database.auth(" + user + ");'"; const binshell = '/bin/sh'; - const mongo = './mongo'; + const mongo = 'mongo'; const host = conn.host; const port = conn.port; const ret = runProgram( diff --git a/jstests/auth/deleted_recreated_user_base.js b/jstests/auth/deleted_recreated_user_base.js index c64b2e4126f..1a3a8556d91 100644 --- a/jstests/auth/deleted_recreated_user_base.js +++ b/jstests/auth/deleted_recreated_user_base.js @@ -33,7 +33,7 @@ function runTest(s0, s1) { // Perform administrative commands via separate shell. function evalCmd(cmd) { const uri = 'mongodb://admin:pass@localhost:' + s1.port + '/admin'; - const result = runMongoProgram('./mongo', uri, '--eval', cmd); + const result = runMongoProgram('mongo', uri, '--eval', cmd); assert.eq(result, 0, "Command failed"); } evalCmd('db.dropUser("user"); '); diff --git a/jstests/auth/mongoURIAuth.js b/jstests/auth/mongoURIAuth.js index abf518c5a98..fa6a0ad40db 100644 --- a/jstests/auth/mongoURIAuth.js +++ b/jstests/auth/mongoURIAuth.js @@ -44,7 +44,7 @@ const runURIAuthTest = function(userMech, uriMech, authMechanism, regexMechanism uri = "mongodb://user:password@localhost:" + conn.port; } - var shell = runMongoProgram('./mongo', uri, "--eval", "db.getName()"); + var shell = runMongoProgram('mongo', uri, "--eval", "db.getName()"); assert.eq(shell, 0, "Should be able to connect with specified params."); const log = adminDB.runCommand({getLog: "global"}); diff --git a/jstests/auth/pinned_users_clear_pinned_user_list.js b/jstests/auth/pinned_users_clear_pinned_user_list.js index 95119a12fe8..177e2b9e797 100644 --- a/jstests/auth/pinned_users_clear_pinned_user_list.js +++ b/jstests/auth/pinned_users_clear_pinned_user_list.js @@ -1,7 +1,7 @@ (function() { 'use strict'; -jsTest.setOption("enableTestCommands", true); +TestData.enableTestCommands = true; // Start a mongod with the user cache size set to zero, so we know that users who have logged out // always get fetched cleanly from disk. diff --git a/jstests/auth/pinned_users_exclusive_lock_on_admin.js b/jstests/auth/pinned_users_exclusive_lock_on_admin.js index 037b6a5b288..be0f8661d7b 100644 --- a/jstests/auth/pinned_users_exclusive_lock_on_admin.js +++ b/jstests/auth/pinned_users_exclusive_lock_on_admin.js @@ -12,7 +12,7 @@ load("jstests/libs/wait_for_command.js"); -jsTest.setOption("enableTestCommands", true); +TestData.enableTestCommands = true; // Start a mongod with the user cache size set to zero, so we know that users who have logged out // always get fetched cleanly from disk. diff --git a/jstests/auth/pinned_users_remove_user_document_unpins_user.js b/jstests/auth/pinned_users_remove_user_document_unpins_user.js index a09aad8393b..362b11166c4 100644 --- a/jstests/auth/pinned_users_remove_user_document_unpins_user.js +++ b/jstests/auth/pinned_users_remove_user_document_unpins_user.js @@ -6,7 +6,7 @@ (function() { 'use strict'; -jsTest.setOption("enableTestCommands", true); +TestData.enableTestCommands = true; // Start a mongod with the user cache size set to zero, so we know that users who have logged out // always get fetched cleanly from disk. diff --git a/jstests/auth/repl_auth_shell_mechanism.js b/jstests/auth/repl_auth_shell_mechanism.js index acf724e7809..72a00d5204b 100644 --- a/jstests/auth/repl_auth_shell_mechanism.js +++ b/jstests/auth/repl_auth_shell_mechanism.js @@ -29,7 +29,7 @@ const rsName = connString.substr(0, slash); const rsHosts = connString.substr(slash + 1); // Connect with shell using connString. -const csShell = runMongoProgram('./mongo', +const csShell = runMongoProgram('mongo', '--host', connString, '-u', @@ -43,7 +43,7 @@ const csShell = runMongoProgram('./mongo', assert.eq(csShell, 0, 'Failed to connect using connection string'); // Connect with shell explicitly specifying mechanism. -const csShellMech = runMongoProgram('./mongo', +const csShellMech = runMongoProgram('mongo', '--host', connString, '-u', @@ -60,12 +60,12 @@ assert.eq(csShellMech, 0, 'Failed to connect using connection string'); // Connect with shell using URI. const uriString = 'mongodb://admin:password@' + rsHosts + '/admin?replicaSet=' + rsName; -const uriShell = runMongoProgram('./mongo', uriString, '--eval', ';'); +const uriShell = runMongoProgram('mongo', uriString, '--eval', ';'); assert.eq(uriShell, 0, 'Failed to connect using URI'); // Connect with shell using URI and explcit mechanism. const uriShellMech = - runMongoProgram('./mongo', uriString + '&authMechanism=SCRAM-SHA-256', '--eval', ';'); + runMongoProgram('mongo', uriString + '&authMechanism=SCRAM-SHA-256', '--eval', ';'); assert.eq(uriShellMech, 0, 'Failed to connect using URI'); rsTest.stopSet(); diff --git a/jstests/fail_point/set_failpoint_through_set_parameter.js b/jstests/fail_point/set_failpoint_through_set_parameter.js index d081913555a..d35ea87e240 100644 --- a/jstests/fail_point/set_failpoint_through_set_parameter.js +++ b/jstests/fail_point/set_failpoint_through_set_parameter.js @@ -25,14 +25,14 @@ configRS.startSet({configsvr: '', storageEngine: 'wiredTiger'}); configRS.initiate(); // Setting a failpoint via --setParameter fails if enableTestCommands is not on. -jsTest.setOption('enableTestCommands', false); +TestData.enableTestCommands = false; assertStartupFails( MongoRunner.runMongod({setParameter: "failpoint.dummy=" + tojson(validFailpointPayload)})); assertStartupFails(MongoRunner.runMongos({ setParameter: "failpoint.dummy=" + tojson(validFailpointPayload), configdb: configRS.getURL() })); -jsTest.setOption('enableTestCommands', true); +TestData.enableTestCommands = true; // Passing an invalid failpoint payload fails. assertStartupFails( @@ -108,7 +108,7 @@ MongoRunner.stopMongos(mongos); // Failpoint server parameters do not show up in the output of getParameter when not running // with enableTestCommands=1. -jsTest.setOption('enableTestCommands', false); +TestData.enableTestCommands = false; TestData.roleGraphInvalidationIsFatal = false; mongod = MongoRunner.runMongod(); diff --git a/jstests/noPassthrough/bind_ip_all.js b/jstests/noPassthrough/bind_ip_all.js index 216b41b2ca8..e2c4c368781 100644 --- a/jstests/noPassthrough/bind_ip_all.js +++ b/jstests/noPassthrough/bind_ip_all.js @@ -10,7 +10,7 @@ const BINDIPALL = 'jstests/noPassthrough/libs/net.bindIpAll.yaml'; function runTest(config, opt, expectStar, expectLocalhost) { clearRawMongoProgramOutput(); const mongod = - runMongoProgram('./mongod', '--port', port, '--config', config, opt, '--outputConfig'); + runMongoProgram('mongod', '--port', port, '--config', config, opt, '--outputConfig'); assert.eq(mongod, 0); const output = rawMongoProgramOutput(); assert.eq(output.search(/bindIp: "\*"/) >= 0, expectStar, output); diff --git a/jstests/noPassthrough/disabled_test_parameters.js b/jstests/noPassthrough/disabled_test_parameters.js index 9393ea7c018..8dde184b6c5 100644 --- a/jstests/noPassthrough/disabled_test_parameters.js +++ b/jstests/noPassthrough/disabled_test_parameters.js @@ -13,7 +13,7 @@ function assertStarts(opts) { MongoRunner.stopMongod(mongod); } -setJsTestOption('enableTestCommands', false); +TestData.enableTestCommands = false; // enableTestCommands not specified. assertFails({ diff --git a/jstests/noPassthrough/readConcern_atClusterTime.js b/jstests/noPassthrough/readConcern_atClusterTime.js index 24ffc90d214..959de6e4f85 100644 --- a/jstests/noPassthrough/readConcern_atClusterTime.js +++ b/jstests/noPassthrough/readConcern_atClusterTime.js @@ -116,7 +116,7 @@ rst.stopSet(); // readConcern with 'atClusterTime' should succeed regardless of value of 'enableTestCommands'. { - jsTest.setOption('enableTestCommands', false); + TestData.enableTestCommands = false; let rst = new ReplSetTest({nodes: 1}); rst.startSet(); rst.initiate(); @@ -130,7 +130,7 @@ rst.stopSet(); session.endSession(); rst.stopSet(); - jsTest.setOption('enableTestCommands', true); + TestData.enableTestCommands = true; rst = new ReplSetTest({nodes: 1}); rst.startSet(); rst.initiate(); diff --git a/jstests/noPassthrough/readConcern_snapshot_mongos_enable_test_commands.js b/jstests/noPassthrough/readConcern_snapshot_mongos_enable_test_commands.js index 22eaa2fbf89..31b2e513b31 100644 --- a/jstests/noPassthrough/readConcern_snapshot_mongos_enable_test_commands.js +++ b/jstests/noPassthrough/readConcern_snapshot_mongos_enable_test_commands.js @@ -37,10 +37,10 @@ function expectSnapshotReadConcernIsSupported() { } // Snapshot readConcern should succeed when 'enableTestCommands' is set to false. -jsTest.setOption("enableTestCommands", false); +TestData.enableTestCommands = false; expectSnapshotReadConcernIsSupported(); // Snapshot readConcern should succeed when 'enableTestCommands' is set to true. -jsTest.setOption("enableTestCommands", true); +TestData.enableTestCommands = true; expectSnapshotReadConcernIsSupported(); }()); diff --git a/jstests/noPassthrough/server_transaction_metrics_secondary.js b/jstests/noPassthrough/server_transaction_metrics_secondary.js index 9464dd77fc1..5302e0b05c4 100644 --- a/jstests/noPassthrough/server_transaction_metrics_secondary.js +++ b/jstests/noPassthrough/server_transaction_metrics_secondary.js @@ -3,7 +3,7 @@ (function() { "use strict"; -jsTest.setOption("enableTestCommands", false); +TestData.enableTestCommands = false; TestData.authenticationDatabase = "local"; const dbName = "test"; @@ -77,4 +77,4 @@ jsTestLog("Done trying transaction on secondary."); secondarySession.endSession(); replTest.stopSet(); -}());
\ No newline at end of file +}()); diff --git a/jstests/noPassthrough/setshellparameter.js b/jstests/noPassthrough/setshellparameter.js index deed3dc4076..492b9abe4cd 100644 --- a/jstests/noPassthrough/setshellparameter.js +++ b/jstests/noPassthrough/setshellparameter.js @@ -4,7 +4,7 @@ 'use strict'; function test(ssp, succeed) { - const result = runMongoProgram('./mongo', '--setShellParameter', ssp, '--nodb', '--eval', ';'); + const result = runMongoProgram('mongo', '--setShellParameter', ssp, '--nodb', '--eval', ';'); assert.eq( 0 == result, succeed, '--setShellParameter ' + ssp + 'worked/didn\'t-work unexpectedly'); } diff --git a/jstests/noPassthrough/shell_history.js b/jstests/noPassthrough/shell_history.js index 98adac83dc9..18e0b9345c7 100644 --- a/jstests/noPassthrough/shell_history.js +++ b/jstests/noPassthrough/shell_history.js @@ -17,6 +17,7 @@ var env = {}; if (_isWindows()) { args.push("cmd.exe"); args.push("/c"); + cmdline = cmdline.replace("mongo", "mongo.exe"); // Input is set to NUL. The output must also be redirected to NUL, otherwise running the // jstest manually has strange terminal IO behaviour. @@ -40,9 +41,9 @@ if (_isWindows()) { args.push("sh"); args.push("-c"); - // Use the mongo shell from the current dir, same as resmoke.py does. - // Doesn't handle resmoke's --mongo= option. - cmdline = "./" + cmdline; + // Use the mongo shell from the $PATH, Resmoke sets $PATH to + // include all the mongo binaries first. + cmdline = cmdline; // Set umask to 0 prior to running the shell. cmdline = "umask 0 ; " + cmdline; diff --git a/jstests/noPassthrough/shell_interactive.js b/jstests/noPassthrough/shell_interactive.js index ea23099a546..bfcbb0b81c8 100644 --- a/jstests/noPassthrough/shell_interactive.js +++ b/jstests/noPassthrough/shell_interactive.js @@ -6,7 +6,7 @@ if (!_isWindows()) { clearRawMongoProgramOutput(); - var rc = runProgram("./mongo", "--nodb", "--quiet", "--eval", "print(isInteractive())"); + var rc = runProgram("mongo", "--nodb", "--quiet", "--eval", "print(isInteractive())"); assert.eq(rc, 0); var output = rawMongoProgramOutput(); var response = (output.split('\n').slice(-2)[0]).split(' ')[1]; @@ -14,7 +14,7 @@ if (!_isWindows()) { // now try interactive clearRawMongoProgramOutput(); rc = runProgram( - "./mongo", "--nodb", "--quiet", "--shell", "--eval", "print(isInteractive()); quit()"); + "mongo", "--nodb", "--quiet", "--shell", "--eval", "print(isInteractive()); quit()"); assert.eq(rc, 0); output = rawMongoProgramOutput(); response = (output.split('\n').slice(-2)[0]).split(' ')[1]; diff --git a/jstests/noPassthrough/shell_retry_writes_uri.js b/jstests/noPassthrough/shell_retry_writes_uri.js index 56f23981adc..bf5925c0fb8 100644 --- a/jstests/noPassthrough/shell_retry_writes_uri.js +++ b/jstests/noPassthrough/shell_retry_writes_uri.js @@ -58,7 +58,7 @@ function runShellScript(uri, cmdArgs, insertShouldHaveTxnNumber, shellFn) { script += shellFn.toString(); script += ")"; - let args = ["./mongo", uri, "--eval", script].concat(cmdArgs); + let args = ["mongo", uri, "--eval", script].concat(cmdArgs); let exitCode = runMongoProgram(...args); assert.eq(exitCode, 0, `shell script "${shellFn.name}" exited with ${exitCode}`); } diff --git a/jstests/noPassthrough/traffic_recording.js b/jstests/noPassthrough/traffic_recording.js index e748deb7e7b..2021ac65e31 100644 --- a/jstests/noPassthrough/traffic_recording.js +++ b/jstests/noPassthrough/traffic_recording.js @@ -18,13 +18,13 @@ function runTest(client, restartCommand) { mkdir(path); if (!jsTest.isMongos(client)) { - setJsTestOption("enableTestCommands", 0); + TestData.enableTestCommands = false; client = restartCommand({ trafficRecordingDirectory: path, AlwaysRecordTraffic: "notARealPath", enableTestCommands: 0, }); - setJsTestOption("enableTestCommands", 1); + TestData.enableTestCommands = true; assert.eq(null, client, "AlwaysRecordTraffic and not enableTestCommands should fail"); } diff --git a/jstests/noPassthrough/unknown-set-parameter.js b/jstests/noPassthrough/unknown-set-parameter.js index f5e6c2b10b8..91df8149a42 100644 --- a/jstests/noPassthrough/unknown-set-parameter.js +++ b/jstests/noPassthrough/unknown-set-parameter.js @@ -3,9 +3,11 @@ (function() { 'use strict'; +TestData.enableTestCommands = false; + function tryRun(arg) { // runMongoProgram helpfully makes certain that we pass a port when invoking mongod. - return runMongoProgram('./mongod', '--port', 0, '--setParameter', arg, '--outputConfig'); + return runMongoProgram('mongod', '--port', 0, '--setParameter', arg, '--outputConfig'); } // Positive case, valid setparam. diff --git a/jstests/noPassthroughWithMongod/testing_only_commands.js b/jstests/noPassthroughWithMongod/testing_only_commands.js index ad142cc1ecc..329be92da7e 100644 --- a/jstests/noPassthroughWithMongod/testing_only_commands.js +++ b/jstests/noPassthroughWithMongod/testing_only_commands.js @@ -30,7 +30,7 @@ var assertCmdFound = function(db, cmdName) { } }; -jsTest.setOption('enableTestCommands', false); +TestData.enableTestCommands = false; var conn = MongoRunner.runMongod({}); for (i in testOnlyCommands) { @@ -39,7 +39,7 @@ for (i in testOnlyCommands) { MongoRunner.stopMongod(conn); // Now enable the commands -jsTest.setOption('enableTestCommands', true); +TestData.enableTestCommands = true; var conn = MongoRunner.runMongod({}); for (i in testOnlyCommands) { diff --git a/jstests/replsets/disallow_shardsvr_transactions_wcMajorityJournal_false.js b/jstests/replsets/disallow_shardsvr_transactions_wcMajorityJournal_false.js index bb16e3966b9..65d7a04e9e9 100644 --- a/jstests/replsets/disallow_shardsvr_transactions_wcMajorityJournal_false.js +++ b/jstests/replsets/disallow_shardsvr_transactions_wcMajorityJournal_false.js @@ -11,7 +11,7 @@ // A testing exemption was made to allow transactions on shard server even if // writeConcernMajorityJournalDefault = false. So we need to disable the exemption in this test // in order to test the behavior. -jsTest.setOption('enableTestCommands', false); +TestData.enableTestCommands = false; // The following two options by default do not support enableTestCommands=false, change them // accordingly so this test can run. diff --git a/jstests/replsets/transactions_only_allowed_on_primaries.js b/jstests/replsets/transactions_only_allowed_on_primaries.js index 2ca360eca41..910c37740ca 100644 --- a/jstests/replsets/transactions_only_allowed_on_primaries.js +++ b/jstests/replsets/transactions_only_allowed_on_primaries.js @@ -9,7 +9,8 @@ // In 4.0, we allow read-only transactions on secondaries when test commands are enabled, so we // disable them in this test, to test that transactions on secondaries will be disallowed // for production users. -jsTest.setOption('enableTestCommands', false); + +TestData.enableTestCommands = false; TestData.roleGraphInvalidationIsFatal = false; TestData.authenticationDatabase = "local"; diff --git a/jstests/ssl/mongo_uri_secondaries.js b/jstests/ssl/mongo_uri_secondaries.js index eec34153b30..6970015cc12 100644 --- a/jstests/ssl/mongo_uri_secondaries.js +++ b/jstests/ssl/mongo_uri_secondaries.js @@ -64,7 +64,7 @@ const subShellCommandFormatter = function(replSet) { const subShellArgs = [ "env", "SSL_CERT_FILE=jstests/libs/trusted-ca.pem", - './mongo', + 'mongo', '--nodb', '--eval', subShellCommandFormatter(rst) diff --git a/jstests/ssl/repl_ssl_noca.js b/jstests/ssl/repl_ssl_noca.js index 5dea404fb12..258530ccc9a 100644 --- a/jstests/ssl/repl_ssl_noca.js +++ b/jstests/ssl/repl_ssl_noca.js @@ -32,7 +32,7 @@ var nodeList = replTest.nodeList().join(); var checkShellOkay = function(url) { // Should not be able to authenticate with x509. // Authenticate call will return 1 on success, 0 on error. - var argv = ['./mongo', url, '--eval', ('db.runCommand({replSetGetStatus: 1})')]; + var argv = ['mongo', url, '--eval', ('db.runCommand({replSetGetStatus: 1})')]; if (!_isWindows()) { // On Linux we override the default path to the system CA store to point to our // "trusted" CA. On Windows, this CA will have been added to the user's trusted CA list diff --git a/jstests/ssl/repl_ssl_split_horizon.js b/jstests/ssl/repl_ssl_split_horizon.js index fe4f4272098..198c4d4d6f9 100644 --- a/jstests/ssl/repl_ssl_split_horizon.js +++ b/jstests/ssl/repl_ssl_split_horizon.js @@ -117,7 +117,7 @@ var checkExpectedHorizon = function(url, memberIndex, expectedHostname) { 'env', "HOSTALIASES=" + hostsFile, "SSL_CERT_FILE=jstests/libs/splithorizon-ca.pem", - './mongo', + 'mongo', url, '--eval', assertion diff --git a/jstests/ssl/ssl_cert_selector.js b/jstests/ssl/ssl_cert_selector.js index eb4c10b16fb..11c97eff270 100644 --- a/jstests/ssl/ssl_cert_selector.js +++ b/jstests/ssl/ssl_cert_selector.js @@ -28,7 +28,7 @@ requireSSLProvider('windows', function() { const testWithCert = function(certSelector) { jsTest.log(`Testing with SSL cert ${certSelector}`); const argv = [ - './mongo', + 'mongo', '--ssl', '--sslCertificateSelector', certSelector, diff --git a/jstests/ssl/ssl_uri.js b/jstests/ssl/ssl_uri.js index 830bba98984..6a6bfc876f0 100644 --- a/jstests/ssl/ssl_uri.js +++ b/jstests/ssl/ssl_uri.js @@ -30,7 +30,7 @@ shouldSucceed(sslURI + "?ssl=true"); shouldFail(sslURI + "?ssl=false"); var connectWithURI = function(uri) { - return runMongoProgram('./mongo', + return runMongoProgram('mongo', '--ssl', '--sslAllowInvalidCertificates', '--sslCAFile', @@ -56,8 +56,7 @@ shouldNotConnect(sslURI + "?ssl=false"); shouldConnect(sslURI + "?ssl=true"); // Connecting with ssl=true without --ssl will not work -var res = - runMongoProgram('./mongo', sslURI + "?ssl=true", '--eval', 'db.runCommand({ismaster: 1})'); +var res = runMongoProgram('mongo', sslURI + "?ssl=true", '--eval', 'db.runCommand({ismaster: 1})'); assert.eq(res, 1, "should not have been able to connect without --ssl"); // Clean up diff --git a/jstests/ssl/ssl_with_system_ca.js b/jstests/ssl/ssl_with_system_ca.js index 10c4c3ae2b3..4626cceeaa9 100644 --- a/jstests/ssl/ssl_with_system_ca.js +++ b/jstests/ssl/ssl_with_system_ca.js @@ -28,7 +28,7 @@ function testWithCerts(prefix) { {sslMode: 'allowSSL', sslPEMKeyFile: 'jstests/libs/' + prefix + 'server.pem'}); let argv = [ - './mongo', + 'mongo', '--ssl', '--port', conn.port, diff --git a/site_scons/site_tools/auto_install_binaries.py b/site_scons/site_tools/auto_install_binaries.py index 0ff74a32144..d92ba63d241 100644 --- a/site_scons/site_tools/auto_install_binaries.py +++ b/site_scons/site_tools/auto_install_binaries.py @@ -89,6 +89,7 @@ if __name__ == "__main__": archive = zipfile.ZipFile(archive_name, mode='w', compression=zipfile.ZIP_DEFLATED) add_file = archive.write else: + print("WARNING: tar not found in $PATH, install the tar utility to greatly improve archive creation speed.") import tarfile archive = tarfile.open(archive_name, mode='w:gz') add_file = archive.add @@ -476,7 +477,7 @@ def auto_install(env, target, source, **kwargs): setattr(s.attributes, COMPONENTS, components) setattr(s.attributes, ROLES, roles) - # We must do an eearly subst here so that the _aib_debugdir + # We must do an early subst here so that the _aib_debugdir # generator has a chance to run while seeing 'source'. # # TODO: Find a way to not need this early subst. @@ -547,14 +548,13 @@ def finalize_install_dependencies(env): pkg_name = get_package_name(env, component, role) for fmt in ("zip", "tar"): - # TODO: $PKGDIR support if fmt == "zip": pkg_suffix = "$AIB_ZIP_SUFFIX" else: pkg_suffix = "$AIB_TARBALL_SUFFIX" archive = env.__AibArchive( - target="#{}.{}".format(pkg_name, pkg_suffix), + target="$PKGDIR/{}.{}".format(pkg_name, pkg_suffix), source=[make_archive_script] + info.alias, __AIB_ARCHIVE_TYPE=fmt, __AIB_INSTALLED_SET=installed, @@ -562,12 +562,17 @@ def finalize_install_dependencies(env): AIB_ROLE=role, ) - # TODO: perhaps caching of packages / tarballs should be - # configurable? It's possible someone would want to do it. - env.NoCache(archive) + if not env.get("AIB_CACHE_ARCHIVES", False): + env.NoCache(archive) - archive_alias = generate_alias(env, component, role, target=fmt) - env.Alias(archive_alias, archive) + compression_alias = generate_alias(env, component, role, target=fmt) + env.Alias(compression_alias, archive) + + default_fmt = "zip" if env["PLATFORM"] == "win32" else "tar" + archive_alias = generate_alias(env, component, role, target="archive") + default_compression_alias = generate_alias(env, component, role, target=default_fmt) + env.Alias(archive_alias, default_compression_alias) + def auto_install_emitter(target, source, env): @@ -679,15 +684,6 @@ def dest_dir_generator(initial_value=None): return generator -def _aib_debugdir(source, target, env, for_signature): - for s in source: - # TODO: We shouldn't need to reach into the attributes of the debug tool like this. - origin = getattr(s.attributes, "debug_file_for", None) - oentry = env.Entry(origin) - osuf = oentry.get_suffix() - return env[SUFFIX_MAP].get(osuf)[0] - - def exists(_env): """Always activate this tool.""" return True @@ -696,7 +692,7 @@ def exists(_env): def list_components(env, **kwargs): """List registered components for env.""" print("Known AIB components:") - for key in env[ALIAS_MAP]: + for key in sorted(env[ALIAS_MAP]): print("\t", key) @@ -728,7 +724,7 @@ def generate(env): # pylint: disable=too-many-statements env["PREFIX_SHAREDIR"] = env.get("PREFIX_SHAREDIR", "$DESTDIR/share") env["PREFIX_DOCDIR"] = env.get("PREFIX_DOCDIR", "$PREFIX_SHAREDIR/doc") env["PREFIX_INCLUDEDIR"] = env.get("PREFIX_INCLUDEDIR", "$DESTDIR/include") - env["PREFIX_DEBUGDIR"] = env.get("PREFIX_DEBUGDIR", _aib_debugdir) + env["PKGDIR"] = env.get("PKGDIR", "$VARIANT_DIR/pkgs") env[SUFFIX_MAP] = {} env[PACKAGE_ALIAS_MAP] = {} env[ALIAS_MAP] = defaultdict(dict) diff --git a/site_scons/site_tools/mongo_benchmark.py b/site_scons/site_tools/mongo_benchmark.py index 400512e738a..22139bf6d25 100644 --- a/site_scons/site_tools/mongo_benchmark.py +++ b/site_scons/site_tools/mongo_benchmark.py @@ -34,8 +34,6 @@ def build_benchmark(env, target, source, **kwargs): libdeps.append("$BUILD_DIR/mongo/unittest/benchmark_main") kwargs["LIBDEPS"] = libdeps - kwargs["INSTALL_ALIAS"] = ["benchmarks"] - benchmark_test_components = {"tests", "benchmarks"} if "AIB_COMPONENT" in kwargs and not kwargs["AIB_COMPONENT"].endswith("-benchmark"): kwargs["AIB_COMPONENT"] += "-benchmark" diff --git a/site_scons/site_tools/mongo_unittest.py b/site_scons/site_tools/mongo_unittest.py index 3d7e4e13e4e..27a0c9a7b5b 100644 --- a/site_scons/site_tools/mongo_unittest.py +++ b/site_scons/site_tools/mongo_unittest.py @@ -23,7 +23,11 @@ def register_unit_test(env, test): Some SConscripts called RegisterUnitTest directly. """ env.RegisterTest("$UNITTEST_LIST", test) - env.Alias("$UNITTEST_ALIAS", test) + aib_install_actions = getattr(test.attributes, "AIB_INSTALL_ACTIONS", []) + if aib_install_actions: + env.Alias("$UNITTEST_ALIAS", aib_install_actions) + else: + env.Alias("$UNITTEST_ALIAS", test) def exists(env): @@ -36,8 +40,11 @@ def build_cpp_unit_test(env, target, source, **kwargs): kwargs["LIBDEPS"] = libdeps unit_test_components = {"tests", "unittests"} - if "AIB_COMPONENT" in kwargs and not kwargs["AIB_COMPONENT"].endswith("-test"): - kwargs["AIB_COMPONENT"] += "-test" + primary_component = kwargs.get("AIB_COMPONENT", env.get("AIB_COMPONENT", "")) + if primary_component and not primary_component.endswith("-test"): + kwargs["AIB_COMPONENT"] = primary_component + "-test" + elif primary_component: + kwargs["AIB_COMPONENT"] = primary_component if "AIB_COMPONENTS_EXTRA" in kwargs: kwargs["AIB_COMPONENTS_EXTRA"] = set(kwargs["AIB_COMPONENTS_EXTRA"]).union( diff --git a/src/mongo/SConscript b/src/mongo/SConscript index f1d4f64e9e3..64a84eae124 100644 --- a/src/mongo/SConscript +++ b/src/mongo/SConscript @@ -482,6 +482,7 @@ mongod = env.Program( "core", "default", "dist", + "dist-test", "servers", ], ) @@ -587,6 +588,7 @@ mongos = env.Program( AIB_COMPONENTS_EXTRA=[ "core", "dist", + "dist-test", "servers" ] ) @@ -683,6 +685,7 @@ if not has_option('noshell') and usemozjs: AIB_COMPONENTS_EXTRA=[ "core", "dist", + "dist-test", "shell", ], ) @@ -811,7 +814,7 @@ if hygienic: target='$DESTDIR', source=env.get('MODULE_BANNERS', []), AIB_COMPONENT='common', - AIB_COMPONENTS_EXTRA=['dist'], + AIB_COMPONENTS_EXTRA=['dist', 'dist-test'], AIB_ROLE='base', ) diff --git a/src/mongo/client/sdam/SConscript b/src/mongo/client/sdam/SConscript index 8d96a6b2a39..ce12275e3e9 100644 --- a/src/mongo/client/sdam/SConscript +++ b/src/mongo/client/sdam/SConscript @@ -28,6 +28,7 @@ sdam_json_test = env.Program( source=[ 'sdam_json_test_runner.cpp', env.Idlc('sdam_json_test_runner_cli_options.idl')[0]], + AIB_COMPONENT="sdam-json-test", LIBDEPS=[ 'sdam', 'sdam_test', @@ -36,7 +37,6 @@ sdam_json_test = env.Program( '$BUILD_DIR/mongo/util/options_parser/options_parser'], )[0] env.RegisterUnitTest(sdam_json_test) -env.Alias("sdam_json_test_runner", str(sdam_json_test)) env.Library( target='sdam_test', diff --git a/src/mongo/dbtests/SConscript b/src/mongo/dbtests/SConscript index c867767b612..da103f124a9 100644 --- a/src/mongo/dbtests/SConscript +++ b/src/mongo/dbtests/SConscript @@ -172,11 +172,12 @@ if not has_option('noshell') and usemozjs: "mocklib", "testframework", ], - AIB_COMPONENT="dbtests", + AIB_COMPONENT="dbtest", AIB_COMPONENTS_EXTRA=[ "tests", ], ) + hygienic = get_option('install-mode') == 'hygienic' if not hygienic: install_dbtest = env.Install('#/', dbtest) diff --git a/src/mongo/shell/servers.js b/src/mongo/shell/servers.js index 2bd37d0467f..679b42463c9 100644 --- a/src/mongo/shell/servers.js +++ b/src/mongo/shell/servers.js @@ -1079,6 +1079,11 @@ function appendSetParameterArgs(argArray) { // programName includes the version, e.g., mongod-3.2. // baseProgramName is the program name without any version information, e.g., mongod. let programName = argArray[0]; + const separator = _isWindows() ? '\\' : '/'; + if (programName.indexOf(separator) !== -1) { + let pathElements = programName.split(separator); + programName = pathElements[pathElements.length - 1]; + } let [baseProgramName, programVersion] = programName.split("-"); let programMajorMinorVersion = 0; diff --git a/src/mongo/shell/utils.js b/src/mongo/shell/utils.js index 09b12e59dc8..f1a406336a9 100644 --- a/src/mongo/shell/utils.js +++ b/src/mongo/shell/utils.js @@ -264,6 +264,10 @@ var _jsTestOptions = {enableTestCommands: true}; // Test commands should be ena jsTestOptions = function() { if (TestData) { return Object.merge(_jsTestOptions, { + // Test commands should be enabled by default if no enableTestCommands were present in + // TestData + enableTestCommands: + TestData.hasOwnProperty('enableTestCommands') ? TestData.enableTestCommands : true, serviceExecutor: TestData.serviceExecutor, setParameters: TestData.setParameters, setParametersMongos: TestData.setParametersMongos, diff --git a/src/mongo/tools/SConscript b/src/mongo/tools/SConscript index 6e3e8719c85..7a500c2631b 100644 --- a/src/mongo/tools/SConscript +++ b/src/mongo/tools/SConscript @@ -30,7 +30,5 @@ mongobridge = env.Program( '$BUILD_DIR/mongo/util/signal_handlers', ], AIB_COMPONENT='tools', + AIB_COMPONENTS_EXTRA=["dist-test"], ) - -install_mongobridge = env.Install("#/", mongobridge) -env.Alias("install-tools", [install_mongobridge[0]]) diff --git a/src/third_party/IntelRDFPMathLib20U1/SConscript b/src/third_party/IntelRDFPMathLib20U1/SConscript index a0e7482a0c0..e5ba80446d6 100644 --- a/src/third_party/IntelRDFPMathLib20U1/SConscript +++ b/src/third_party/IntelRDFPMathLib20U1/SConscript @@ -5,6 +5,7 @@ import sys Import("env") Import("has_option") Import("debugBuild") +Import("get_option") env = env.Clone() @@ -367,12 +368,30 @@ readtest = env.Program( 'intel_decimal128', ], LIBS=libs, + AIB_COMPONENT="intel-test", + AIB_COMPONENTS_EXTRA=[ + "unittests", + "tests", + ] ) -readtest_input = env.Install( - target='.', - source=['TESTS/readtest.in'], -) +if get_option("install-mode") == "hygienic": + readtest_input = env.AutoInstall( + target="$PREFIX_BINDIR", + source=["TESTS/readtest.in"], + AIB_ROLE="runtime", + AIB_COMPONENT="intel-test", + AIB_COMPONENTS_EXTRA=[ + "unittests", + "tests", + ], + ) +else: + readtest_input = env.Install( + target='.', + source=['TESTS/readtest.in'], + ) + env.Depends(readtest_input, readtest) readtest_dict = { @@ -395,7 +414,36 @@ if env.TargetOSIs('windows'): SUBST_DICT=readtest_dict, ) env.Depends(readtest_wrapper_bat, readtest_wrapper) - env.RegisterUnitTest(readtest_wrapper_bat[0]) + + if get_option("install-mode") == "hygienic": + readtest_wrapper_bat_install = env.AutoInstall( + target="$PREFIX_BINDIR", + source=readtest_wrapper_bat, + AIB_ROLE="runtime", + AIB_COMPONENT="intel-test", + AIB_COMPONENTS_EXTRA=[ + "unittests", + "tests", + ], + ) + env.RegisterUnitTest(readtest_wrapper_bat_install[0]) + else: + env.RegisterUnitTest(readtest_wrapper_bat[0]) else: + + if get_option("install-mode") == "hygienic": + readtest_wrapper_install = env.AutoInstall( + target="$PREFIX_BINDIR", + source=readtest_wrapper, + AIB_ROLE="runtime", + AIB_COMPONENT="intel-test", + AIB_COMPONENTS_EXTRA=[ + "unittests", + "tests", + ], + ) + env.RegisterUnitTest(readtest_wrapper_install[0]) + else: + env.RegisterUnitTest(readtest_wrapper[0]) + env.AddPostAction(readtest_wrapper[0], Chmod(readtest_wrapper[0], 'oug+x')) - env.RegisterUnitTest(readtest_wrapper[0]) diff --git a/src/third_party/wiredtiger/SConscript b/src/third_party/wiredtiger/SConscript index 7b859e0aaaf..8247432fc4e 100644 --- a/src/third_party/wiredtiger/SConscript +++ b/src/third_party/wiredtiger/SConscript @@ -231,10 +231,7 @@ wtbin = wtbinEnv.Program( AIB_COMPONENT="tools", ) -hygienic = get_option("install-mode") == "hygienic" -wtbin_install = wtbinEnv.Install("#/", wtbin) -if hygienic: - # TODO: remove this once hygienic drives all tarball creation - wtbinEnv.Alias("install-tools", wtbin_install[0]) -else: +hygienic = get_option('install-mode') == 'hygienic' +if not hygienic: + wtbin_install = wtbinEnv.Install("#/", wtbin) wtbinEnv.Alias("tools", wtbin_install) |