From e88d29393fa22c857af21e107a3bc20402625b10 Mon Sep 17 00:00:00 2001 From: Andrew Morrow Date: Sat, 28 Apr 2018 12:06:00 -0400 Subject: SERVER-33661 Constrain link parallelism for compile_all and compile_unittests --- SConstruct | 41 +++++++++++++++++++++++++++++++++++++++++ etc/evergreen.yml | 34 ++++++++++++++++------------------ 2 files changed, 57 insertions(+), 18 deletions(-) diff --git a/SConstruct b/SConstruct index 5db0a7bef49..fa85da47db9 100644 --- a/SConstruct +++ b/SConstruct @@ -508,6 +508,11 @@ add_option('msvc-debugging-format', type='choice', ) +add_option('jlink', + help="Limit link concurrency to given value", + nargs=1, + type=int) + try: with open("version.json", "r") as version_fp: version_data = json.load(version_fp) @@ -3413,6 +3418,42 @@ env.Alias("distsrc-tgz", env.GZip( env.Alias("distsrc-zip", env.DistSrc("mongodb-src-${MONGO_VERSION}.zip")) env.Alias("distsrc", "distsrc-tgz") +# Do this as close to last as possible before reading SConscripts, so +# that any tools that may have injected other things via emitters are included +# among the side effect adornments. +# +# TODO: Move this to a tool. +if has_option('jlink'): + jlink = get_option('jlink') + if jlink < 1: + env.FatalError("The argument to jlink must be a positive integer") + + target_builders = ['Program', 'SharedLibrary', 'LoadableModule'] + + # A bound map of stream (as in stream of work) name to side-effect + # file. Since SCons will not allow tasks with a shared side-effect + # to execute concurrently, this gives us a way to limit link jobs + # independently of overall SCons concurrency. + jlink_stream_map = dict() + + def jlink_emitter(target, source, env): + name = str(target[0]) + se_name = "#jlink-stream" + str(hash(name) % jlink) + se_node = jlink_stream_map.get(se_name, None) + if not se_node: + se_node = env.Entry(se_name) + # This may not be necessary, but why chance it + env.NoCache(se_node) + jlink_stream_map[se_name] = se_node + env.SideEffect(se_node, target) + return (target, source) + + for target_builder in target_builders: + builder = env['BUILDERS'][target_builder] + base_emitter = builder.emitter + new_emitter = SCons.Builder.ListEmitter([base_emitter, jlink_emitter]) + builder.emitter = new_emitter + env.SConscript( dirs=[ 'src', diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 6e281f018fe..2ea4d9bbdbd 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -969,9 +969,9 @@ functions: rm -rf ${install_directory|/data/mongo-install-directory} extra_args="" - if [ "${reduce_scons_compile_jobs_available|false}" = "true" ] && [ -n "${num_scons_compile_jobs_available|}" ]; then - echo "Changing SCons to run with --jobs=${num_scons_compile_jobs_available|}" - extra_args="$extra_args --jobs=${num_scons_compile_jobs_available|}" + if [ -n "${num_scons_link_jobs_available|}" ]; then + echo "Changing SCons to run with --jlink=${num_scons_link_jobs_available|}" + extra_args="$extra_args --jlink=${num_scons_link_jobs_available|}" fi ${activate_virtualenv} @@ -2849,7 +2849,6 @@ tasks: - func: "scons compile" vars: targets: all - reduce_scons_compile_jobs_available: true task_compile_flags: >- --use-new-tools --build-mongoreplay="${build_mongoreplay}" @@ -2872,7 +2871,6 @@ tasks: - func: "scons compile" vars: targets: unittests - reduce_scons_compile_jobs_available: true task_compile_flags: >- --detect-odr-violations @@ -7622,7 +7620,7 @@ buildvariants: compile_flags: --ssl MONGO_DISTMOD=amazon2 --release -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_gcc.vars # We invoke SCons using --jobs = (# of CPUs / 4) to avoid causing out of memory errors due to # spawning a large number of linker processes. - num_scons_compile_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 )) + num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 )) num_jobs_available: $(grep -c ^processor /proc/cpuinfo) has_packages: true packager_script: packager_enterprise.py @@ -7699,7 +7697,7 @@ buildvariants: compile_flags: --ssl MONGO_DISTMOD=amazon2 -j$(grep -c ^processor /proc/cpuinfo) --release --variables-files=etc/scons/mongodbtoolchain_gcc.vars # We invoke SCons using --jobs = (# of CPUs / 4) to avoid causing out of memory errors due to # spawning a large number of linker processes. - num_scons_compile_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 )) + num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 )) multiversion_platform: amazon multiversion_edition: targeted num_jobs_available: $(grep -c ^processor /proc/cpuinfo) @@ -7798,7 +7796,7 @@ buildvariants: compile_flags: --dbg=on --opt=on --win-version-min=ws08r2 -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) MONGO_DISTMOD=2008plus # We invoke SCons using --jobs = (# of CPUs / 4) to avoid causing out of memory errors due to # spawning a large number of linker processes. - num_scons_compile_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 )) + num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 )) python: python ext: zip use_scons_cache: true @@ -7940,7 +7938,7 @@ buildvariants: compile_flags: --release --ssl MONGO_DISTMOD=windows-64 CPPPATH="c:/openssl/include c:/sasl/include c:/snmp/include c:/curl/include" LIBPATH="c:/openssl/lib c:/sasl/lib c:/snmp/lib c:/curl/lib" -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) --dynamic-windows --win-version-min=ws08r2 # We invoke SCons using --jobs = (# of CPUs / 4) to avoid causing out of memory errors due to # spawning a large number of linker processes. - num_scons_compile_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 )) + num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 )) python: python num_jobs_available: $(grep -c ^processor /proc/cpuinfo) ext: zip @@ -8312,7 +8310,7 @@ buildvariants: compile_flags: --release --ssl MONGO_DISTMOD=windows-64 CPPPATH="c:/openssl/include c:/sasl/include c:/snmp/include c:/curl/include" LIBPATH="c:/openssl/lib c:/sasl/lib c:/snmp/lib c:/curl/lib" -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) --dynamic-windows --win-version-min=ws08r2 # We invoke SCons using --jobs = (# of CPUs / 4) to avoid causing out of memory errors due to # spawning a large number of linker processes. - num_scons_compile_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 )) + num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 )) python: python num_jobs_available: $(grep -c ^processor /proc/cpuinfo) ext: zip @@ -8336,7 +8334,7 @@ buildvariants: compile_flags: --release --ssl --ssl-provider=openssl MONGO_DISTMOD=windows-64 CPPPATH="c:/openssl/include c:/sasl/include c:/snmp/include c:/curl/include" LIBPATH="c:/openssl/lib c:/sasl/lib c:/snmp/lib c:/curl/lib" -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) --dynamic-windows --win-version-min=ws08r2 # We invoke SCons using --jobs = (# of CPUs / 4) to avoid causing out of memory errors due to # spawning a large number of linker processes. - num_scons_compile_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 )) + num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 )) python: python num_jobs_available: $(grep -c ^processor /proc/cpuinfo) ext: zip @@ -8365,7 +8363,7 @@ buildvariants: compile_flags: --release --ssl MONGO_DISTMOD=windows-64 CPPPATH="c:/openssl/include c:/sasl/include c:/snmp/include c:/curl/include" LIBPATH="c:/openssl/lib c:/sasl/lib c:/snmp/lib c:/curl/lib" -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) --dynamic-windows --win-version-min=ws08r2 # We invoke SCons using --jobs = (# of CPUs / 4) to avoid causing out of memory errors due to # spawning a large number of linker processes. - num_scons_compile_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 )) + num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 )) python: python num_jobs_available: $(grep -c ^processor /proc/cpuinfo) ext: zip @@ -8401,7 +8399,7 @@ buildvariants: compile_flags: --release --ssl MONGO_DISTMOD=windows-64 CPPPATH="c:/openssl/include c:/sasl/include c:/snmp/include c:/curl/include" LIBPATH="c:/openssl/lib c:/sasl/lib c:/snmp/lib c:/curl/lib" -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) --dynamic-windows --win-version-min=ws08r2 # We invoke SCons using --jobs = (# of CPUs / 4) to avoid causing out of memory errors due to # spawning a large number of linker processes. - num_scons_compile_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 )) + num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 )) python: python num_jobs_available: $(grep -c ^processor /proc/cpuinfo) test_flags: --storageEngine=inMemory --excludeWithAnyTags=requires_persistence,requires_journaling,requires_mmapv1 @@ -8471,7 +8469,7 @@ buildvariants: compile_flags: --release --ssl MONGO_DISTMOD=2008plus-ssl CPPPATH="c:/openssl/include" LIBPATH="c:/openssl/lib" -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) --dynamic-windows --win-version-min=ws08r2 # We invoke SCons using --jobs = (# of CPUs / 4) to avoid causing out of memory errors due to # spawning a large number of linker processes. - num_scons_compile_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 )) + num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 )) python: python num_jobs_available: $(grep -c ^processor /proc/cpuinfo) ext: zip @@ -8583,7 +8581,7 @@ buildvariants: compile_flags: --dbg=on --opt=off --ssl MONGO_DISTMOD=2008plus CPPPATH="c:/openssl/include c:/sasl/include c:/snmp/include c:/curl/include" LIBPATH="c:/openssl/lib c:/sasl/lib c:/snmp/lib c:/curl/lib" -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) --dynamic-windows --win-version-min=ws08r2 # We invoke SCons using --jobs = (# of CPUs / 4) to avoid causing out of memory errors due to # spawning a large number of linker processes. - num_scons_compile_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 )) + num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 )) python: python num_jobs_available: $(grep -c ^processor /proc/cpuinfo) ext: zip @@ -11186,7 +11184,7 @@ buildvariants: compile_flags: --ssl MONGO_DISTMOD=debian92 --release -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_gcc.vars # We invoke SCons using --jobs = (# of CPUs / 4) to avoid causing out of memory errors due to # spawning a large number of linker processes. - num_scons_compile_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 )) + num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 )) num_jobs_available: $(grep -c ^processor /proc/cpuinfo) has_packages: true packager_script: packager_enterprise.py @@ -11241,7 +11239,7 @@ buildvariants: compile_flags: --ssl MONGO_DISTMOD=debian92 -j$(grep -c ^processor /proc/cpuinfo) --release --variables-files=etc/scons/mongodbtoolchain_gcc.vars # We invoke SCons using --jobs = (# of CPUs / 4) to avoid causing out of memory errors due to # spawning a large number of linker processes. - num_scons_compile_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 )) + num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 )) multiversion_platform: debian92 multiversion_edition: targeted num_jobs_available: $(grep -c ^processor /proc/cpuinfo) @@ -12560,7 +12558,7 @@ buildvariants: c:/sasl/include c:/snmp/include c:/curl/include" LIBPATH="c:/openssl/lib c:/sasl/lib c:/snmp/lib c:/curl/lib" -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) --dynamic-windows --win-version-min=ws08r2 - num_scons_compile_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 )) + num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 )) python: python num_jobs_available: $(grep -c ^processor /proc/cpuinfo) ext: zip -- cgit v1.2.1