diff options
author | Tausif Rahman <tausif.rahman@mongodb.com> | 2022-06-29 17:59:30 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2022-07-26 17:47:16 +0000 |
commit | a6e5b4986e7bc21080a0fcce8b9ceef89214d48b (patch) | |
tree | b3e9457f02c3ae9b9e5ad5b431b8ec859321c546 | |
parent | cd1400b89f1da5a6d7a85339c836d27584ccc3b9 (diff) | |
download | mongo-a6e5b4986e7bc21080a0fcce8b9ceef89214d48b.tar.gz |
SERVER-66938 Drastically simplify command to generate ninja files
(cherry picked from commit 0dd56eb0358262ee0af4d2b172ec53c62ebb0233)
-rwxr-xr-x[-rw-r--r--] | SConstruct | 54 | ||||
-rw-r--r-- | etc/evergreen_yml_components/definitions.yml | 4 | ||||
-rw-r--r-- | site_scons/mongo/build_profiles.py | 92 | ||||
-rw-r--r-- | site_scons/site_tools/ninja.py | 7 | ||||
-rwxr-xr-x | src/third_party/wiredtiger/test/evergreen.yml | 2 |
5 files changed, 138 insertions, 21 deletions
diff --git a/SConstruct b/SConstruct index d1a1f821d09..30ceaf62baf 100644..100755 --- a/SConstruct +++ b/SConstruct @@ -30,6 +30,7 @@ import mongo.platform as mongo_platform import mongo.toolchain as mongo_toolchain import mongo.generators as mongo_generators import mongo.install_actions as install_actions +from mongo.build_profiles import BUILD_PROFILES EnsurePythonVersion(3, 6) EnsureSConsVersion(3, 1, 1) @@ -104,9 +105,23 @@ SetOption('random', 1) # using the nargs='const' mechanism. # -add_option('ninja', +add_option( + 'build-profile', + choices=list(BUILD_PROFILES.keys()), + default='default', + type='choice', + help='''Short hand for common build options. These profiles are well supported by SDP and are + kept up to date. Unless you need something specific, it is recommended that you only build with + these. san is the recommeneded profile since it exposes bugs before they are found in patch + builds. Check out site_scons/mongo/build_profiles.py to see each profile.''', +) + +build_profile = BUILD_PROFILES[get_option('build-profile')] + +add_option( + 'ninja', choices=['enabled', 'disabled'], - default='disabled', + default=build_profile.ninja, nargs='?', const='enabled', type='choice', @@ -244,7 +259,7 @@ add_option('noshell', add_option('dbg', choices=['on', 'off'], const='on', - default='off', + default=build_profile.dbg, help='Enable runtime debugging checks', nargs='?', type='choice', @@ -311,6 +326,7 @@ add_option('debug-compress', add_option('sanitize', help='enable selected sanitizers', metavar='san1,san2,...sanN', + default=build_profile.sanitize, ) add_option('sanitize-coverage', @@ -320,7 +336,7 @@ add_option('sanitize-coverage', add_option('allocator', choices=["auto", "system", "tcmalloc", "tcmalloc-experimental"], - default="auto", + default=build_profile.allocator, help='allocator to use (use "auto" for best choice for current platform)', type='choice', ) @@ -493,8 +509,9 @@ def find_mongo_custom_variables(): files.append(probe) return files -add_option('variables-files', - default=[], +add_option( + 'variables-files', + default=build_profile.variables_files, action="append", help="Specify variables files to load.", ) @@ -502,7 +519,7 @@ add_option('variables-files', link_model_choices = ['auto', 'object', 'static', 'dynamic', 'dynamic-strict', 'dynamic-sdk'] add_option('link-model', choices=link_model_choices, - default='auto', + default=build_profile.link_model, help='Select the linking model for the project', type='choice' ) @@ -776,8 +793,11 @@ env_vars.Add('ARFLAGS', help='Sets flags for the archiver', converter=variable_shlex_converter) -env_vars.Add('CCACHE', - help='Tells SCons where the ccache binary is') +env_vars.Add( + 'CCACHE', + help='Tells SCons where the ccache binary is', + default=build_profile.CCACHE, +) env_vars.Add( 'CACHE_SIZE', @@ -869,8 +889,11 @@ env_vars.Add('HOST_ARCH', converter=variable_arch_converter, default=None) -env_vars.Add('ICECC', - help='Tells SCons where icecream icecc tool is') +env_vars.Add( + 'ICECC', + help='Tells SCons where icecream icecc tool is', + default=build_profile.ICECC, +) env_vars.Add('ICERUN', help='Tells SCons where icecream icerun tool is') @@ -966,8 +989,9 @@ env_vars.Add('NINJA_BUILDDIR', default="$BUILD_DIR/ninja", ) -env_vars.Add('NINJA_PREFIX', - default="build", +env_vars.Add( + 'NINJA_PREFIX', + default=build_profile.NINJA_PREFIX, help="""A prefix to add to the beginning of generated ninja files. Useful for when compiling multiple build ninja files for different configurations, for instance: @@ -980,7 +1004,7 @@ Will generate the files (respectively): asan.ninja tsan.ninja -Defaults to build. Best used with the generate-ninja alias so you don't have to +Defaults to build. Best used with the --ninja flag so you don't have to reiterate the prefix in the target name and variable. """) @@ -1071,7 +1095,7 @@ env_vars.Add('TOOLS', env_vars.Add('VARIANT_DIR', help='Sets the name (or generator function) for the variant directory', - default=mongo_generators.default_variant_dir_generator, + default=build_profile.VARIANT_DIR, ) env_vars.Add('VERBOSE', diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml index 57a946bcefd..4db63823c3f 100644 --- a/etc/evergreen_yml_components/definitions.yml +++ b/etc/evergreen_yml_components/definitions.yml @@ -2288,8 +2288,6 @@ tasks: default task_compile_flags: >- --ninja - targets: - generate-ninja - *f_expansions_write - command: subprocess.exec params: @@ -2309,8 +2307,6 @@ tasks: task_compile_flags: >- --build-tools=next --ninja - targets: - generate-ninja - *f_expansions_write - command: subprocess.exec params: diff --git a/site_scons/mongo/build_profiles.py b/site_scons/mongo/build_profiles.py new file mode 100644 index 00000000000..37a2520c879 --- /dev/null +++ b/site_scons/mongo/build_profiles.py @@ -0,0 +1,92 @@ +"""Dictionary to store available build profiles.""" +from dataclasses import dataclass +from typing import Any, List, Optional +import mongo.generators as mongo_generators + + +@dataclass +class BuildProfile: + ninja: str + variables_files: List + allocator: str + sanitize: Optional[str] + link_model: str + dbg: str + opt: str + ICECC: Optional[str] + CCACHE: Optional[str] + NINJA_PREFIX: str + VARIANT_DIR: Any + + +BUILD_PROFILES = { + # These options were the default settings before implementing build profiles. + "default": + BuildProfile( + ninja="disabled", + variables_files=[], + allocator="auto", + sanitize=None, + link_model="auto", + dbg="off", + opt="off", + ICECC=None, + CCACHE=None, + NINJA_PREFIX="build", + VARIANT_DIR=mongo_generators.default_variant_dir_generator, + ), + # This build has fast runtime speed & fast build time at the cost of debuggability. + "fast": + BuildProfile( + ninja="enabled", + variables_files=[ + './etc/scons/mongodbtoolchain_stable_clang.vars', + './etc/scons/developer_versions.vars', + ], + allocator="auto", + sanitize=None, + link_model="dynamic", + dbg="off", + opt="on", + ICECC="icecc", + CCACHE="ccache", + NINJA_PREFIX="fast", + VARIANT_DIR="fast", + ), + # This build has fast runtime speed & debuggability at the cost of build time. + "opt": + BuildProfile( + ninja="enabled", + variables_files=[ + './etc/scons/mongodbtoolchain_stable_clang.vars', + './etc/scons/developer_versions.vars', + ], + allocator="auto", + sanitize=None, + link_model="dynamic", + dbg="on", + opt="on", + ICECC="icecc", + CCACHE="ccache", + NINJA_PREFIX="opt", + VARIANT_DIR="opt", + ), + # This build leverages santizers & is the suggested build profile to use for development. + "san": + BuildProfile( + ninja="enabled", + variables_files=[ + './etc/scons/mongodbtoolchain_stable_clang.vars', + './etc/scons/developer_versions.vars', + ], + allocator="system", + sanitize="undefined,address", + link_model="dynamic", + dbg="on", + opt="off", + ICECC="icecc", + CCACHE="ccache", + NINJA_PREFIX="san", + VARIANT_DIR="san", + ), +} diff --git a/site_scons/site_tools/ninja.py b/site_scons/site_tools/ninja.py index 8505ce8e7ba..33fdd8011d4 100644 --- a/site_scons/site_tools/ninja.py +++ b/site_scons/site_tools/ninja.py @@ -1450,7 +1450,6 @@ def generate(env): ninja_file_name = env.subst("${NINJA_PREFIX}.${NINJA_SUFFIX}") ninja_file = env.Ninja(target=ninja_file_name, source=[]) env.AlwaysBuild(ninja_file) - env.Alias("$NINJA_ALIAS_NAME", ninja_file) # TODO: API for getting the SConscripts programmatically # exists upstream: https://github.com/SCons/scons/issues/3625 @@ -1570,6 +1569,12 @@ def generate(env): if not exists(env): return + # There is a target called generate-ninja which needs to be included + # with the --ninja flag in order to generate the ninja file. Because the --ninja + # flag is ONLY used with generate-ninja, we have combined the two by making the --ninja flag + # implicitly build the generate-ninja target. + SCons.Script.BUILD_TARGETS = SCons.Script.TargetList(env.Alias("$NINJA_ALIAS_NAME", ninja_file)) + # Set a known variable that other tools can query so they can # behave correctly during ninja generation. env["GENERATING_NINJA"] = True diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index 1eb0ac2ae5e..d6f096c3764 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -89,7 +89,7 @@ functions: source venv/bin/activate pip3 install requirements_parser pip3 install -r etc/pip/compile-requirements.txt - ./buildscripts/scons.py --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars --link-model=dynamic --ninja generate-ninja ICECC=icecc CCACHE=ccache + ./buildscripts/scons.py --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars --link-model=dynamic --ninja ICECC=icecc CCACHE=ccache ninja -j$(nproc --all) install-mongod "configure wiredtiger": &configure_wiredtiger command: shell.exec |