From 542479adb86d90e80ce4faed167e6848d4107adf Mon Sep 17 00:00:00 2001 From: Mathew Robinson Date: Tue, 19 Nov 2019 17:19:15 +0000 Subject: SERVER-42408 Ensure hygienic builds work with Ninja --- SConstruct | 157 ++-- etc/evergreen.yml | 31 + site_scons/site_tools/auto_install_binaries.py | 11 + site_scons/site_tools/icecream.py | 15 +- site_scons/site_tools/ninja.py | 1137 +++++++++++++++-------- src/third_party/IntelRDFPMathLib20U1/SConscript | 14 +- 6 files changed, 909 insertions(+), 456 deletions(-) diff --git a/SConstruct b/SConstruct index 229280b8390..fd7fd420a91 100644 --- a/SConstruct +++ b/SConstruct @@ -845,6 +845,11 @@ Will generate the files (respectively): install-all-meta.build.ninja.tsan """) +env_vars.Add('__NINJA_NO', + help="Disable the Ninja tool unconditionally. Not intended for human use.", + default=0) + + env_vars.Add('OBJCOPY', help='Sets the path to objcopy', default=WhereIs('objcopy')) @@ -1052,6 +1057,9 @@ envDict = dict(BUILD_ROOT=buildDir, env = Environment(variables=env_vars, **envDict) +# Only print the spinner if stdout is a tty +if sys.stdout.isatty(): + Progress(['-\r', '\\\r', '|\r', '/\r'], interval=5) del envDict @@ -1579,10 +1587,26 @@ if env['_LIBDEPS'] == '$_LIBDEPS_OBJS': fake_lib.write(str(uuid.uuid4())) fake_lib.write('\n') - env['ARCOM'] = write_uuid_to_file - env['ARCOMSTR'] = 'Generating placeholder library $TARGET' - env['RANLIBCOM'] = '' - env['RANLIBCOMSTR'] = 'Skipping ranlib for $TARGET' + # We originally did this by setting ARCOM to write_uuid_to_file, + # this worked more or less by accident. It works when SCons is + # doing the action execution because when it would subst the + # command line subst would execute the function as part of string + # resolution which would have the side effect of writing the + # file. Since it returned None subst would do some special + # handling to make sure it never made it to the command line. This + # breaks Ninja however because we are taking that return value and + # trying to pass it to the command executor (/bin/sh or + # cmd.exe) and end up with the function name as a command. The + # resulting command looks something like `/bin/sh -c + # 'write_uuid_to_file(env, target, source)`. If we instead + # actually do what we want and that is make the StaticLibrary + # builder's action a FunctionAction the Ninja generator will + # correctly dispatch it and not generate an invalid command + # line. This also has the side benefit of being more clear that + # we're expecting a Python function to execute here instead of + # pretending to be a CommandAction that just happens to not run a + # command but instead runs a function. + env["BUILDERS"]["StaticLibrary"].action = SCons.Action.Action(write_uuid_to_file, "Generating placeholder library $TARGET") libdeps.setup_environment(env, emitting_shared=(link_model.startswith("dynamic"))) @@ -3729,64 +3753,84 @@ env["NINJA_SYNTAX"] = "#site_scons/third_party/ninja_syntax.py" env.Tool('icecream') if get_option('ninja') == 'true': - env.Tool("ninja") - def test_txt_writer(alias_name): - """Find all the tests registered to alias_name and write them to a file via ninja.""" - rule_written = False - - def wrapper(env, ninja, node, dependencies): - """Make a Ninja-able version of the test files.""" - rule = alias_name.upper() + "_GENERATOR" - if not rule_written: - ninja.rule( - rule, - description="Generate test list text file", - command="echo $in > $out", - ) - rule_written = True - - alias = env.Alias(alias_name) - paths = [] - children = alias.children() - for child in children: - paths.append('\t' + str(child)) - - ninja.build( - str(node), - rule, - inputs='\n'.join(paths), - implicit=dependencies, + ninja_builder = Tool("ninja") + if ninja_builder.exists(env): + ninja_builder.generate(env) + + # Explicitly add all generated sources to the DAG so NinjaBuilder can + # generate rules for them. SCons if the files don't exist will not wire up + # the dependencies in the DAG because it cannot scan them. The Ninja builder + # does not care about the actual edge here as all generated sources will be + # pushed to the "bottom" of it's DAG via the order_only dependency on + # _generated_sources (an internal phony target) + if get_option('install-mode') == 'hygienic': + env.Alias("install-common-base", env.Alias("generated-sources")) + else: + env.Alias("all", env.Alias("generated-sources")) + env.Alias("core", env.Alias("generated-sources")) + + if get_option("install-mode") == "hygienic": + ninja_build = env.Ninja( + target="install-all-meta.build.ninja", + source=env.Alias("install-all-meta"), + ) + else: + ninja_build = env.Ninja( + target="all.build.ninja", + source=env.Alias("all"), ) - return wrapper - env.NinjaRegisterFunctionHandler("unit_test_list_builder_action", test_txt_writer('$UNITTEST_ALIAS')) - env.NinjaRegisterFunctionHandler("integration_test_list_builder_action", test_txt_writer('$INTEGRATION_TEST_ALIAS')) - env.NinjaRegisterFunctionHandler("benchmark_list_builder_action", test_txt_writer('$BENCHMARK_ALIAS')) - - def fakelib_in_ninja(): - """Generates empty .a files""" - rule_written = False - - def wrapper(env, ninja, node, dependencies): - if not rule_written: - cmd = "touch $out" - if not env.TargetOSIs("posix"): - cmd = "cmd /c copy NUL $out" - ninja.rule( - "FAKELIB", - command=cmd, - ) - rule_written = True + from glob import glob + sconscripts = [env.File(g) for g in glob("**/SConscript", recursive=True)] + sconscripts += [env.File("#SConstruct")] + env.Depends(ninja_build, sconscripts) + - ninja.build(node.get_path(), rule='FAKELIB', implicit=dependencies) + def skip(env, node): + """ + Write an empty test text file. + + Instead of calling SCONS to generate a *test.txt that most + users aren't using (and the current generator skips) we + simply teach Ninja how to make an empty file on the given + platform so as not to break dependency trees. + """ + cmd = "touch {}".format(str(node)) + if env["PLATFORM"] == "win32": + cmd = "copy NUL {}".format(str(node)) + return { + "outputs": [str(node)], + "rule": "CMD", + "variables": { + "cmd": cmd, + } + } - return wrapper + env.NinjaRegisterFunctionHandler("unit_test_list_builder_action", skip) + env.NinjaRegisterFunctionHandler("integration_test_list_builder_action", skip) + env.NinjaRegisterFunctionHandler("benchmark_list_builder_action", skip) - env.NinjaRegisterFunctionHandler("write_uuid_to_file", fakelib_in_ninja()) + # We can create empty files for FAKELIB in Ninja because it + # does not care about content signatures. We have to + # write_uuid_to_file for FAKELIB in SCons because SCons does. + env.NinjaRule( + rule="FAKELIB", + command="cmd /c copy NUL $out" if env["PLATFORM"] == "win32" else "touch $out", + ) - # Load ccache after icecream since order matters when we're both changing CCCOM - if get_option('ccache') == 'true': - env.Tool('ccache') + def fakelib_in_ninja(env, node): + """Generates empty .a files""" + return { + "outputs": [node.get_path()], + "rule": "FAKELIB", + "implicit": [str(s) for s in node.sources], + } + + env.NinjaRegisterFunctionHandler("write_uuid_to_file", fakelib_in_ninja) + + # Load ccache after icecream since order matters when we're both changing CCCOM + if get_option('ccache') == 'true': + env.Tool('ccache') # TODO: Later, this should live somewhere more graceful. if get_option('install-mode') == 'hygienic': @@ -3879,7 +3923,6 @@ if get_option('install-mode') == 'hygienic': "debug" ] ), - }) env.AddPackageNameAlias( diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 77191ee541d..4228fe3bc62 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -4040,6 +4040,20 @@ tasks: --install-mode=hygienic --separate-debug +- name: compile_ninja + commands: + - func: "scons compile" + vars: + task_compile_flags: >- + --ninja + targets: + all.build.ninja + - command: shell.exec + params: + working_dir: src + shell: bash + script: ninja -f all.build.ninja + ## compile_all - build all scons targets ## - name: compile_all commands: @@ -9098,6 +9112,22 @@ task_groups: name: compile_core_tools_TG tasks: - compile_core_tools +- <<: *compile_task_group_template + name: compile_ninja_TG + tasks: + - compile_ninja + teardown_task: + - command: s3.put + params: + optional: true + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/all.build.ninja + remote_file: ${project}/${build_variant}/${revision}/artifacts/all.${build_id}.build.ninja + bucket: mciuploads + permissions: public-read + content_type: text/plain + display_name: build.ninja - <<: *compile_task_group_template name: dbtest_TG tasks: @@ -9409,6 +9439,7 @@ buildvariants: - name: compile_all_run_unittests_TG distros: - ubuntu1804-build + - name: compile_ninja_TG - name: .aggfuzzer .common - name: audit - name: causally_consistent_jscore_txns_passthrough diff --git a/site_scons/site_tools/auto_install_binaries.py b/site_scons/site_tools/auto_install_binaries.py index 984cf5dfac0..bf133852051 100644 --- a/site_scons/site_tools/auto_install_binaries.py +++ b/site_scons/site_tools/auto_install_binaries.py @@ -582,6 +582,16 @@ def auto_install_emitter(target, source, env): suffix = entry.get_suffix() if env.get("AIB_IGNORE", False): continue + + # There is no API for determining if an Entry is operating in + # a SConf context. We obviously do not want to auto tag, and + # install conftest Programs. So we filter them out the only + # way available to us. + # + # We're working with upstream to expose this information. + if 'conftest' in str(entry): + continue + auto_install_mapping = env[SUFFIX_MAP].get(suffix) if auto_install_mapping is not None: env.AutoInstall( @@ -592,6 +602,7 @@ def auto_install_emitter(target, source, env): AIB_ROLES=auto_install_mapping.default_roles, AIB_COMPONENTS_EXTRA=env.get(COMPONENTS), ) + return (target, source) diff --git a/site_scons/site_tools/icecream.py b/site_scons/site_tools/icecream.py index c570c869fd7..bd2690fb93a 100644 --- a/site_scons/site_tools/icecream.py +++ b/site_scons/site_tools/icecream.py @@ -22,7 +22,6 @@ from pkg_resources import parse_version icecream_version_min = '1.1rc2' - def generate(env): if not exists(env): @@ -61,7 +60,19 @@ def generate(env): else: # Make a predictable name for the toolchain icecc_version_target_filename = env.subst('$CC$CXX').replace('/', '_') - icecc_version = env.Dir('$BUILD_ROOT/scons/icecc').File(icecc_version_target_filename) + icecc_version_dir = env.Dir('$BUILD_ROOT/scons/icecc') + icecc_version = icecc_version_dir.File(icecc_version_target_filename) + + # There is a weird ordering problem that occurs when the ninja generator + # is enabled with icecream. Because the modules system runs configure + # checks after the environment is setup and configure checks ignore our + # --no-exec from the Ninja tool they try to create the icecc_env file. + # But since the Ninja tool has reached into the internals of SCons to + # disabled as much of it as possible SCons never creates this directory, + # causing the icecc_create_env call to fail. So we explicitly + # force creation of the directory now so it exists in all + # circumstances. + env.Execute(SCons.Defaults.Mkdir(icecc_version_dir)) # Make an isolated environment so that our setting of ICECC_VERSION in the environment # doesn't appear when executing icecc_create_env diff --git a/site_scons/site_tools/ninja.py b/site_scons/site_tools/ninja.py index 6ff3f7bd001..54e60383e74 100644 --- a/site_scons/site_tools/ninja.py +++ b/site_scons/site_tools/ninja.py @@ -17,351 +17,697 @@ import sys import os import importlib import io +import shutil from glob import glob import SCons -from SCons.Action import _string_from_cmd_list, get_default_ENV +from SCons.Action import _string_from_cmd_list from SCons.Script import COMMAND_LINE_TARGETS NINJA_SYNTAX = "NINJA_SYNTAX" +NINJA_RULES = "__NINJA_CUSTOM_RULES" NINJA_CUSTOM_HANDLERS = "__NINJA_CUSTOM_HANDLERS" -NINJA_ALIAS_PREFIX = "ninja-" -NINJA_CMD = "NINJA_CMD" +NINJA_BUILD = "NINJA_BUILD" +NINJA_OUTPUTS = "__NINJA_OUTPUTS" # These are the types that get_command can do something with -COMMAND_TYPES = (SCons.Action.CommandAction, SCons.Action.CommandGeneratorAction) +COMMAND_TYPES = ( + SCons.Action.CommandAction, + SCons.Action.CommandGeneratorAction, +) -# Global Rules (like SCons Builders) that should always be present. -# -# Primarily we only use "cmd" and pass it the SCons generated compile -# commands. -RULES = { - "cmd": {"command": "$cmd"}, - "INSTALL": {"command": "$COPY $in $out", "description": "INSTALL $out"}, -} -# Global variables that should always be present -VARS = {"COPY": "cmd /c copy" if sys.platform == "win32" else "cp"} +# TODO: make this configurable or improve generated source detection +def is_generated_source(output): + """ + Determine if output indicates this is a generated header file. + """ + for generated in output: + if generated.endswith(".h") or generated.endswith(".hpp"): + return True + return False -class FakePath: - """Make nodes with no get_path method work for dependency finding.""" +def null_function(_env, _node): + """Do nothing for this function action""" + return None - def __init__(self, node): - self.node = node - def get_path(self): - """Return a fake path, as an example Aliases use this as their target name in Ninja.""" - return str(self.node) +def _install_action_function(_env, node): + """Install files using the install or copy commands""" + return { + "outputs": get_outputs(node), + "rule": "INSTALL", + "pool": "install_pool", + "inputs": [get_path(src_file(s)) for s in node.sources], + "implicit": get_dependencies(node), + } -def pathable(node): - """If node is in the build path return it's source file.""" - if hasattr(node, "get_path"): - return node - return FakePath(node) +def _lib_symlink_action_function(_env, node): + """Create shared object symlinks if any need to be created""" + symlinks = getattr(getattr(node, "attributes", None), "shliblinks", None) + if not symlinks or symlinks is None: + return None -def rfile(n): - if hasattr(n, "rfile"): - return n.rfile() - return n + outputs = [link.get_dir().rel_path(linktgt) for link, linktgt in symlinks] + if getattr(node.attributes, NINJA_OUTPUTS, None) is None: + setattr(node.attributes, NINJA_OUTPUTS, outputs) + inputs = [link.get_path() for link, _ in symlinks] -def src_file(node): - """Returns the src code file if it exists.""" - if hasattr(node, "srcnode"): - src = node.srcnode() - if src.stat() is not None: - return src - return pathable(node) + return { + "outputs": outputs, + "inputs": inputs, + "rule": "SYMLINK", + "implicit": get_dependencies(node), + } -def get_command(env, node, action): - """Get the command to execute for node.""" - if node.env: - sub_env = node.env - else: - sub_env = env +def is_valid_dependent_node(node): + """ + Return True if node is not an alias or is an alias that has children - # SCons should not generate resource files since Ninja will - # need to handle the long commands itself. - sub_env["MAXLINELENGTH"] = 10000000000000 + This prevents us from making phony targets that depend on other + phony targets that will never have an associated ninja build + target. - executor = node.get_executor() - if executor is not None: - tlist = executor.get_all_targets() - slist = executor.get_all_sources() - else: - if hasattr(node, "target_peers"): - tlist = node.target_peers + We also have to specify that it's an alias when doing the builder + check because some nodes (like src files) won't have builders but + are valid implicit dependencies. + """ + return not isinstance(node, SCons.Node.Alias.Alias) or node.children() + + +def alias_to_ninja_build(node): + """Convert an Alias node into a Ninja phony target""" + return { + "outputs": get_outputs(node), + "rule": "phony", + "implicit": [ + get_path(n) for n in node.children() if is_valid_dependent_node(n) + ], + } + + +def get_dependencies(node): + """Return a list of dependencies for node.""" + # TODO: test if this is faster as a try except + deps = getattr(node.attributes, "NINJA_DEPS", None) + if deps is None: + deps = [get_path(src_file(child)) for child in node.children()] + setattr(node.attributes, "NINJA_DEPS", deps) + return deps + + +def get_outputs(node): + """Collect the Ninja outputs for node.""" + outputs = getattr(node.attributes, NINJA_OUTPUTS, None) + if outputs is None: + executor = node.get_executor() + if executor is not None: + outputs = executor.get_all_targets() else: - tlist = [node] - slist = node.sources + if hasattr(node, "target_peers"): + outputs = node.target_peers + else: + outputs = [node] + outputs = [get_path(o) for o in outputs] + setattr(node.attributes, NINJA_OUTPUTS, outputs) + return outputs - # Retrieve the repository file for all sources - slist = [rfile(s) for s in slist] - # Generate a real CommandAction - if isinstance(action, SCons.Action.CommandGeneratorAction): - action = action._generate(tlist, slist, sub_env, 1, executor=executor) +class SConsToNinjaTranslator: + """Translates SCons Actions into Ninja build objects.""" - # Actions like CommandAction have a method called process that is - # used by SCons to generate the cmd_line they need to run. So - # check if it's a thing like CommandAction and call it if we can. - if hasattr(action, "process"): - cmd_list, _, _ = action.process(tlist, slist, sub_env, executor=executor) - if cmd_list: - return _string_from_cmd_list(cmd_list[0]) - - # Anything else works with genstring, this is most commonly hit by - # ListActions which essentially call process on all of their - # commands and concatenate it for us. - genstring = action.genstring(tlist, slist, sub_env) - genstring = genstring.replace("\n", " && ").strip() - if genstring.endswith("&&"): - genstring = genstring[0:-2].strip() - if executor is not None: - cmd = sub_env.subst(genstring, executor=executor) - else: - cmd = sub_env.subst(genstring, target=tlist, source=slist) + def __init__(self, env): + self.env = env + self.func_handlers = { - return cmd + # Skip conftest builders + "_createSource": null_function, + # SCons has a custom FunctionAction that just makes sure the + # target isn't static. We let the commands that ninja runs do + # this check for us. + "SharedFlagChecker": null_function, + # The install builder is implemented as a function action. + "installFunc": _install_action_function, + "LibSymlinksActionFunction": _lib_symlink_action_function, -def action_to_ninja_build(env, node, action, dependencies): - """Generate a build arguments for action.""" + } - if isinstance(node, SCons.Node.Alias.Alias): - alias_name = str(node) + self.func_handlers.update(self.env[NINJA_CUSTOM_HANDLERS]) - # Any alias that starts with NINJA_ALIAS_PREFIX builds another - # ninja file and we do not want to add those to the generated - # ninja files. - if not alias_name.startswith(NINJA_ALIAS_PREFIX): - return { - "outputs": str(node), - "rule": "phony", - "implicit": [ - src_file(n).get_path() for n in node.all_children() - if not str(n).startswith(NINJA_ALIAS_PREFIX) - ], - } + # pylint: disable=too-many-return-statements + def action_to_ninja_build(self, node, action=None): + """Generate build arguments dictionary for node.""" + # Use False since None is a valid value for this Attribute + build = getattr(node.attributes, NINJA_BUILD, False) + if build is not False: + return build - # Ideally this should never happen, and we do try to filter - # Ninja builders out of being sources of ninja builders but I - # can't fix every DAG problem so we just skip ninja_builders - # if we find one - elif node.builder == env["BUILDERS"]["Ninja"]: - return None + if node.builder is None: + return None - elif isinstance(action, SCons.Action.FunctionAction): - name = action.function_name() - if name == "installFunc": - return { - "outputs": node.get_path(), - "rule": "INSTALL", - "inputs": [src_file(s).get_path() for s in node.sources], - "implicit": dependencies, - } - else: - custom_handler = env[NINJA_CUSTOM_HANDLERS].get(name, None) - if custom_handler is not None and callable(custom_handler): - print("Using custom handler {} for {}".format(custom_handler.__name__, str(node))) - return custom_handler - else: - # This is the reported name for Substfile and Textfile - # which we still want SCons to generate so don't warn - # for it. - if name != "_action": - print( - "Found unhandled function action {}, " - " generating scons command to build".format(name) - ) - print( - "Note: this is less efficient than Ninja," - " you can write your own ninja build generator for" - " this function using NinjaRegisterFunctionHandler" - ) - return (node, dependencies) - - elif isinstance(action, COMMAND_TYPES): - cmd = getattr(node.attributes, NINJA_CMD, None) - - # If cmd is None it was a node that we didn't hit during print - # time. Ideally everything get hits at that point because the - # most SCons state is still available however, some things do - # not work at this stage so we fall back to trying again here - # in the builder. - if cmd is None: - print("Generating Ninja action for", str(node)) - cmd = get_command(env, node, action) + if action is None: + action = node.builder.action + # Ideally this should never happen, and we do try to filter + # Ninja builders out of being sources of ninja builders but I + # can't fix every DAG problem so we just skip ninja_builders + # if we find one + if node.builder == self.env["BUILDERS"]["Ninja"]: + return None + + if isinstance(action, SCons.Action.FunctionAction): + return self.handle_func_action(node, action) + + if isinstance(action, SCons.Action.LazyAction): + # pylint: disable=protected-access + action = action._generate_cache(node.env if node.env else self.env) + return self.action_to_ninja_build(node, action=action) + + if isinstance(action, SCons.Action.ListAction): + return self.handle_list_action(node, action) + + if isinstance(action, COMMAND_TYPES): + return get_command(node.env if node.env else self.env, node, action) + + # Return the node to indicate that SCons is required return { - "outputs": node.get_path(), - "rule": "cmd", - "variables": {"cmd": cmd}, - "implicit": dependencies, + "rule": "SCONS", + "outputs": get_outputs(node), + "implicit": get_dependencies(node), } - elif isinstance(action, SCons.Action.LazyAction): - generated = action._generate_cache(node.env if node.env else env) - return action_to_ninja_build(env, node, generated, dependencies) + def handle_func_action(self, node, action): + """Determine how to handle the function action.""" + name = action.function_name() + # This is the name given by the Subst/Textfile builders. So return the + # node to indicate that SCons is required + if name == "_action": + return { + "rule": "TEMPLATE", + "outputs": get_outputs(node), + "implicit": get_dependencies(node), + } - # If nothing else works make scons generate the file - return (node, dependencies) + handler = self.func_handlers.get(name, None) + if handler is not None: + return handler(node.env if node.env else self.env, node) + print( + "Found unhandled function action {}, " + " generating scons command to build\n" + "Note: this is less efficient than Ninja," + " you can write your own ninja build generator for" + " this function using NinjaRegisterFunctionHandler".format(name) + ) -def handle_action(env, ninja, node, action, ninja_file=None): - """ - Write a Ninja build for the node's action type. + return { + "rule": "SCONS", + "outputs": get_outputs(node), + "implicit": get_dependencies(node), + } - Returns a tuple of node and dependencies if it couldn't create a Ninja build for it. - """ - children = node.all_children() - dependencies = [src_file(n).get_path() for n in children] - if ninja_file is not None: - dependencies.append(ninja_file) - - build = None - if isinstance(action, SCons.Action.ListAction): - results = [] - for act in action.list: - results.append(action_to_ninja_build(env, node, act, dependencies)) - - # Filter out empty cmdline nodes - results = [r for r in results if r["rule"] == "cmd" and r["variables"]["cmd"]] + # pylint: disable=too-many-branches + def handle_list_action(self, node, action): + """TODO write this comment""" + results = [ + self.action_to_ninja_build(node, action=act) + for act in action.list + if act is not None + ] + results = [result for result in results if result is not None] + if not results: + return None + + # No need to process the results if we only got a single result if len(results) == 1: - build = results[0] + return results[0] - all_outputs = [cmd["outputs"] for cmd in results] + all_outputs = list({output for build in results for output in build["outputs"]}) + setattr(node.attributes, NINJA_OUTPUTS, all_outputs) # If we have no outputs we're done if not all_outputs: return None # Used to verify if all rules are the same - all_install = [r for r in results if isinstance(r, dict) and r["rule"] == "INSTALL"] - all_phony = [r for r in results if isinstance(r, dict) and r["rule"] == "phony"] - all_commands = [r for r in results if isinstance(r, dict) and r["rule"] == "cmd"] + all_one_rule = len( + [ + r + for r in results + if isinstance(r, dict) and r["rule"] == results[0]["rule"] + ] + ) == len(results) + dependencies = get_dependencies(node) + + if not all_one_rule: + # If they aren't all the same rule use scons to generate these + # outputs. At this time nothing hits this case. + return { + "outputs": all_outputs, + "rule": "SCONS", + "implicit": dependencies, + } - if all_commands and len(all_commands) == len(results): + if results[0]["rule"] == "CMD": cmdline = "" - for cmd in all_commands: - if cmdline != "": + for cmd in results: + if not cmd["variables"]["cmd"]: + continue + + if cmdline: cmdline += " && " - if cmd["variables"]["cmd"]: - cmdline += cmd["variables"]["cmd"] + # Skip duplicate commands + if cmd["variables"]["cmd"] in cmdline: + continue + + cmdline += cmd["variables"]["cmd"] + + # Remove all preceding and proceeding whitespace + cmdline = cmdline.strip() # Make sure we didn't generate an empty cmdline if cmdline: - build = { + return { "outputs": all_outputs, - "rule": "cmd", + "rule": "CMD", "variables": {"cmd": cmdline}, "implicit": dependencies, } - elif all_phony and len(all_phony) == len(results): - build = { + elif results[0]["rule"] == "phony": + return { "outputs": all_outputs, "rule": "phony", "implicit": dependencies, } - elif all_install and len(all_install) == len(results): - build = { + elif results[0]["rule"] == "INSTALL": + return { "outputs": all_outputs, "rule": "INSTALL", - "inputs": dependencies, + "pool": "install_pool", + "inputs": [get_path(src_file(s)) for s in node.sources], + "implicit": dependencies, } - - else: - # Else use scons to generate these outputs - build = { + + elif results[0]["rule"] == "SCONS": + return { "outputs": all_outputs, - "rule": "SCONSGEN", - "implicit": dependencies, + "rule": "SCONS", + "inputs": dependencies, } - else: - build = action_to_ninja_build(env, node, action, dependencies) - # If action_to_ninja_build returns a tuple that means SCons is required - if isinstance(build, tuple): - return build - # If action_to_ninja_build returns a function it's a custom Function handler - elif callable(build): - build(env, ninja, node, dependencies) - elif build is not None: - ninja.build(**build) + raise Exception("Unhandled list action with rule: " + results[0]["rule"]) + + +# pylint: disable=too-many-instance-attributes +class NinjaState: + """Maintains state of Ninja build system as it's translated from SCons.""" + + def __init__(self, env, writer_class): + self.env = env + self.writer_class = writer_class + self.__generated = False + self.translator = SConsToNinjaTranslator(env) + self.builds = list() + + # SCons sets this variable to a function which knows how to do + # shell quoting on whatever platform it's run on. Here we use it + # to make the SCONS_INVOCATION variable properly quoted for things + # like CCFLAGS + escape = env.get("ESCAPE", lambda x: x) + + self.built = set() + self.variables = { + "COPY": "cmd.exe /c copy" if sys.platform == "win32" else "cp", + "SCONS_INVOCATION": "{} {} __NINJA_NO=1 $out".format( + sys.executable, + " ".join( + [escape(arg) for arg in sys.argv if arg not in COMMAND_LINE_TARGETS] + ), + ), + "SCONS_INVOCATION_W_TARGETS": "{} {}".format( + sys.executable, + " ".join([escape(arg) for arg in sys.argv]) + ), + } - return None + self.rules = { + "CMD": { + "command": "cmd /c $cmd" if sys.platform == "win32" else "$cmd", + "description": "Building $out", + }, + "SYMLINK": { + "command": ( + "cmd /c mklink $out $in" + if sys.platform == "win32" + else "ln -s $in $out" + ), + "description": "symlink $in -> $out", + }, + "INSTALL": {"command": "$COPY $in $out", "description": "Install $out"}, + "TEMPLATE": { + "command": "$SCONS_INVOCATION $out", + "description": "Rendering $out", + # Console pool restricts to 1 job running at a time, + # it additionally has some special handling about + # passing stdin, stdout, etc to process in this pool + # that we need for SCons to behave correctly when run + # by Ninja. + "pool": "console", + "restat": 1, + }, + "SCONS": { + "command": "$SCONS_INVOCATION $out", + "description": "SCons $out", + "pool": "console", + # restat + # if present, causes Ninja to re-stat the command's outputs + # after execution of the command. Each output whose + # modification time the command did not change will be + # treated as though it had never needed to be built. This + # may cause the output's reverse dependencies to be removed + # from the list of pending build actions. + # + # We use restat any time we execute SCons because + # SCons calls in Ninja typically create multiple + # targets. But since SCons is doing it's own up to + # date-ness checks it may only update say one of + # them. Restat will find out which of the multiple + # build targets did actually change then only rebuild + # those targets which depend specifically on that + # output. + "restat": 1, + }, + "REGENERATE": { + "command": "$SCONS_INVOCATION_W_TARGETS", + "description": "Regenerating $out", + "pool": "console", + # Again we restat in case Ninja thought the + # build.ninja should be regenerated but SCons knew + # better. + "restat": 1, + }, + } + self.rules.update(env.get(NINJA_RULES, {})) + + def generate_builds(self, node): + """Generate a ninja build rule for node and it's children.""" + # Filter out nodes with no builder. They are likely source files + # and so no work needs to be done, it will be used in the + # generation for some real target. + # + # Note that all nodes have a builder attribute but it is sometimes + # set to None. So we cannot use a simpler hasattr check here. + if getattr(node, "builder", None) is None: + return + + stack = [[node]] + self.built = set() + + while stack: + frame = stack.pop() + for child in frame: + outputs = set(get_outputs(child)) + # Check if all the outputs are in self.built, if they + # are we've already seen this node and it's children. + if not outputs.isdisjoint(self.built): + continue + + self.built = self.built.union(outputs) + stack.append(child.children()) + + if isinstance(child, SCons.Node.Alias.Alias): + build = alias_to_ninja_build(child) + elif node.builder is not None: + # Use False since None is a valid value for this attribute + build = getattr(child.attributes, NINJA_BUILD, False) + if build is False: + print("Generating Ninja build for:", str(child)) + build = self.translator.action_to_ninja_build(child) + setattr(child.attributes, NINJA_BUILD, build) + else: + build = None + + # Some things are unbuild-able or need not be built in Ninja + if build is None or build == 0: + continue + + print("Collecting build for:", build["outputs"]) + self.builds.append(build) + + # pylint: disable=too-many-branches,too-many-locals + def generate(self, ninja_file, fallback_default_target=None): + """ + Generate the build.ninja. + + This should only be called once for the lifetime of this object. + """ + if self.__generated: + return + + content = io.StringIO() + ninja = self.writer_class(content, width=100) + + ninja.comment("Generated by scons. DO NOT EDIT.") + + ninja.pool("install_pool", self.env.GetOption("num_jobs") / 2) + + for var, val in self.variables.items(): + ninja.variable(var, val) + + for rule, kwargs in self.rules.items(): + ninja.rule(rule, **kwargs) + + generated_source_files = { + output + + # First find builds which have header files in their outputs. + for build in self.builds + if is_generated_source(build["outputs"]) + for output in build["outputs"] + + # Collect only the header files from the builds with them + # in their output. We do this because is_generated_source + # returns True if it finds a header in any of the outputs, + # here we need to filter so we only have the headers and + # not the other outputs. + if output.endswith(".h") or output.endswith(".hpp") + } + if generated_source_files: + ninja.build( + outputs="_generated_sources", + rule="phony", + implicit=list(generated_source_files), + ) -def handle_exec_node(env, ninja, node, built, scons_required=None, ninja_file=None): - """Write a ninja build rule for node and it's children.""" - if scons_required is None: - scons_required = [] + template_builders = [] - if node in built: - return scons_required + for build in self.builds: + if build["rule"] == "TEMPLATE": + template_builders.append(build) + continue - # Probably a source file and so no work needs to be done, it will - # be used in the generation for some real target. - if getattr(node, "builder", None) is None: - return scons_required + implicit = build.get("implicit", []) + implicit.append(ninja_file) + build["implicit"] = implicit - action = node.builder.action - scons_cmd = handle_action(env, ninja, node, action, ninja_file=ninja_file) - if scons_cmd is not None: - scons_required.append(scons_cmd) + # Don't make generated sources depend on each other. We + # have to check that none of the outputs are generated + # sources and none of the direct implicit dependencies are + # generated sources or else we will create a dependency + # cycle. + if ( + not build["rule"] == "INSTALL" + and not is_generated_source(build["outputs"]) + and set(implicit).isdisjoint(generated_source_files) + ): - built.add(node) - for child in node.all_children(): - scons_required += handle_exec_node(env, ninja, child, built, ninja_file=ninja_file) + # Make all non-generated source targets depend on + # _generated_sources. We use order_only for generated + # sources so that we don't rebuild the world if one + # generated source was rebuilt. We just need to make + # sure that all of these sources are generated before + # other builds. + build["order_only"] = "_generated_sources" + + ninja.build(**build) + + template_builds = dict() + for template_builder in template_builders: + + # Special handling for outputs and implicit since we need to + # aggregate not replace for each builder. + for agg_key in ["outputs", "implicit"]: + new_val = template_builds.get(agg_key, []) + + # Use pop so the key is removed and so the update + # below will not overwrite our aggregated values. + cur_val = template_builder.pop(agg_key, []) + if isinstance(cur_val, list): + new_val += cur_val + else: + new_val.append(cur_val) + template_builds[agg_key] = new_val + + # Collect all other keys + template_builds.update(template_builder) + + if template_builds.get("outputs", []): + ninja.build(**template_builds) + + # We have to glob the SCons files here to teach the ninja file + # how to regenerate itself. We'll never see ourselves in the + # DAG walk so we can't rely on action_to_ninja_build to + # generate this rule even though SCons should know we're + # dependent on SCons files. + ninja.build( + ninja_file, rule="REGENERATE", implicit=glob("**/SCons*", recursive=True), + ) - return scons_required + ninja.build( + "scons-invocation", + rule="CMD", + pool="console", + variables={"cmd": "echo $SCONS_INVOCATION_W_TARGETS"}, + ) + # Look in SCons's list of DEFAULT_TARGETS, find the ones that + # we generated a ninja build rule for. + scons_default_targets = [ + get_path(tgt) + for tgt in SCons.Script.DEFAULT_TARGETS + if get_path(tgt) in self.built + ] -def handle_build_node(env, ninja, node, ninja_file=None): - """Write a ninja build rule for node.""" - # Since the SCons graph looks more like an octopus than a - # Christmas tree we need to make sure we only built nodes once - # with Ninja or else Ninja will be upset we have duplicate - # targets. - built = set() + # If we found an overlap between SCons's list of default + # targets and the targets we created ninja builds for then use + # those as ninja's default as well. + if scons_default_targets: + ninja.default(" ".join(scons_default_targets)) - # handle_exec_node returns a list of tuples of the form (node, - # [stringified_dependencies]) for any node it didn't know what to - # do with. So we will generate scons commands to build those - # files. This is often (and intended) to be Substfile and Textfile - # calls. - scons_required = handle_exec_node(env, ninja, node, built, ninja_file=ninja_file) + # If not then set the default to the fallback_default_target we were given. + # Otherwise we won't create a default ninja target. + elif fallback_default_target is not None: + ninja.default(fallback_default_target) - # Since running SCons is expensive we try to de-duplicate - # Substfile / Textfile calls to make as few SCons calls from - # Ninja as possible. - # - # TODO: attempt to dedupe non-Substfile/Textfile calls - combinable_outputs = [] - combinable_dependencies = [] - for target in scons_required: - builder = target[0].builder - if builder in (env["BUILDERS"]["Substfile"], env["BUILDERS"]["Textfile"]): - if hasattr(target[0], "target_peers"): - combinable_outputs.extend([str(s) for s in target[0].target_peers]) - combinable_outputs.append(str(target[0])) - combinable_dependencies.extend(target[1]) + with open(ninja_file, "w") as build_ninja: + build_ninja.write(content.getvalue()) + + self.__generated = True + + +def get_path(node): + """ + Return a fake path if necessary. + + As an example Aliases use this as their target name in Ninja. + """ + if hasattr(node, "get_path"): + return node.get_path() + return str(node) + + +def rfile(node): + """ + Return the repository file for node if it has one. Otherwise return node + """ + if hasattr(node, "rfile"): + return node.rfile() + return node + + +def src_file(node): + """Returns the src code file if it exists.""" + if hasattr(node, "srcnode"): + src = node.srcnode() + if src.stat() is not None: + return src + return get_path(node) + + +# TODO: Make the Rules smarter. Instead of just using a "cmd" rule +# everywhere we should be smarter about generating CC, CXX, LINK, +# etc. rules +def get_command(env, node, action): # pylint: disable=too-many-branches + """Get the command to execute for node.""" + if node.env: + sub_env = node.env + else: + sub_env = env + + executor = node.get_executor() + if executor is not None: + tlist = executor.get_all_targets() + slist = executor.get_all_sources() + else: + if hasattr(node, "target_peers"): + tlist = node.target_peers else: - ninja.build(str(target[0]), rule="SCONS", implicit=list(set(target[1]))) + tlist = [node] + slist = node.sources - if combinable_outputs: - ninja.build( - list(set(combinable_outputs)), - rule="SCONS", - implicit=list(set(combinable_dependencies)), - ) + # Retrieve the repository file for all sources + slist = [rfile(s) for s in slist] + + # Get the dependencies for all targets + implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)}) + + # Generate a real CommandAction + if isinstance(action, SCons.Action.CommandGeneratorAction): + # pylint: disable=protected-access + action = action._generate(tlist, slist, sub_env, 1, executor=executor) + + # Actions like CommandAction have a method called process that is + # used by SCons to generate the cmd_line they need to run. So + # check if it's a thing like CommandAction and call it if we can. + if hasattr(action, "process"): + cmd_list, _, _ = action.process(tlist, slist, sub_env, executor=executor) + cmd = _string_from_cmd_list(cmd_list[0]) + else: + # Anything else works with genstring, this is most commonly hit by + # ListActions which essentially call process on all of their + # commands and concatenate it for us. + genstring = action.genstring(tlist, slist, sub_env) + if executor is not None: + cmd = sub_env.subst(genstring, executor=executor) + else: + cmd = sub_env.subst(genstring, target=tlist, source=slist) + + # Since we're only enabling Ninja for developer builds right + # now we skip all Manifest related work on Windows as it's not + # necessary. We shouldn't have gotten here but on Windows + # SCons has a ListAction which shows as a + # CommandGeneratorAction for linking. That ListAction ends + # with a FunctionAction (embedManifestExeCheck, + # embedManifestDllCheck) that simply say "does + # target[0].manifest exist?" if so execute the real command + # action underlying me, otherwise do nothing. + # + # Eventually we'll want to find a way to translate this to + # Ninja but for now, and partially because the existing Ninja + # generator does so, we just disable it all together. + cmd = cmd.replace("\n", " && ").strip() + if env["PLATFORM"] == "win32" and ("embedManifestExeCheck" in cmd or "embedManifestDllCheck" in cmd): + cmd = " && ".join(cmd.split(" && ")[0:-1]) + + if cmd.endswith("&&"): + cmd = cmd[0:-2].strip() + + return { + "outputs": get_outputs(node), + "implicit": implicit, + "rule": "CMD", + "variables": {"cmd": cmd}, + } def ninja_builder(env, target, source): @@ -371,15 +717,19 @@ def ninja_builder(env, target, source): if not isinstance(target, list): target = [target] - # Only build if we're building the alias that's actually been - # asked for so we don't generate a bunch of build.ninja files. - alias_name = str(source[0]) - ninja_alias_name = "{}{}".format(NINJA_ALIAS_PREFIX, alias_name) - if ninja_alias_name not in COMMAND_LINE_TARGETS: - return 0 - else: - print("Generating:", str(target[0])) + # We have no COMSTR equivalent so print that we're generating + # here. + print("Generating:", str(target[0])) + # The environment variable NINJA_SYNTAX points to the + # ninja_syntax.py module from the ninja sources found here: + # https://github.com/ninja-build/ninja/blob/master/misc/ninja_syntax.py + # + # This should be vendored into the build sources and it's location + # set in NINJA_SYNTAX. This code block loads the location from + # that variable, gets the absolute path to the vendored file, gets + # it's parent directory then uses importlib to import the module + # dynamically. ninja_syntax_file = env[NINJA_SYNTAX] if isinstance(ninja_syntax_file, str): ninja_syntax_file = env.File(ninja_syntax_file).get_abspath() @@ -388,94 +738,18 @@ def ninja_builder(env, target, source): ninja_syntax_mod_name = os.path.basename(ninja_syntax_file) ninja_syntax = importlib.import_module(ninja_syntax_mod_name.replace(".py", "")) - generated_build_ninja = target[0].get_abspath() - - content = io.StringIO() - ninja = ninja_syntax.Writer(content, width=100) - ninja.comment("Generated by scons for target {}. DO NOT EDIT.".format(alias_name)) - ninja.comment("vim: set textwidth=0 :") - ninja.comment("-*- eval: (auto-fill-mode -1) -*-") - - ninja.variable("PYTHON", sys.executable) - ninja.variable( - "SCONS_INVOCATION", - "{} {} -Q $out".format( - sys.executable, - " ".join( - [ - arg for arg in sys.argv - - # TODO: the "ninja" in arg part of this should be - # handled better, as it stands this is for - # filtering out MongoDB's ninja flag - if arg not in COMMAND_LINE_TARGETS and "ninja" not in arg - ] - ), - ), - ) - - ninja.variable( - "SCONS_INVOCATION_W_TARGETS", - "$SCONS_INVOCATION {}".format( - " ".join([arg for arg in sys.argv if arg in COMMAND_LINE_TARGETS]) - ), - ) - - ninja.rule( - "SCONS", - command="$SCONS_INVOCATION $out", - description="SCONSGEN $out", - - # restat - # if present, causes Ninja to re-stat the command’s outputs - # after execution of the command. Each output whose - # modification time the command did not change will be - # treated as though it had never needed to be built. This - # may cause the output’s reverse dependencies to be removed - # from the list of pending build actions. - restat=1, - ) - - for rule in RULES: - ninja.rule(rule, **RULES[rule]) - - for var, val in VARS.items(): - ninja.variable(var, val) - - environ = get_default_ENV(env) - for var, val in environ.items(): - ninja.variable(var, val) + generated_build_ninja = target[0].get_abspath() + env.get("NINJA_SUFFIX", "") + ninja_state = NinjaState(env, ninja_syntax.Writer) for src in source: - handle_build_node(env, ninja, src, ninja_file=generated_build_ninja) - - ninja.build( - generated_build_ninja, - rule="SCONS", - implicit=glob("**SCons*", recursive=True), - ) - - ninja.build( - "scons-invocation", - rule="cmd", - pool="console", - variables={"cmd": "echo $SCONS_INVOCATION_W_TARGETS"}, - ) - - ninja.default(pathable(source[0]).get_path()) - - with open(generated_build_ninja, "w", encoding="utf-8") as build_ninja: - build_ninja.write(content.getvalue()) - - build_ninja_file = env.File("#build.ninja") - build_ninja_path = build_ninja_file.get_abspath() - print("Linking build.ninja to {}".format(generated_build_ninja)) - if os.path.islink(build_ninja_path): - os.remove(build_ninja_path) - os.symlink(generated_build_ninja, build_ninja_path) + ninja_state.generate_builds(src) + + ninja_state.generate(generated_build_ninja, str(source[0])) + return 0 +# pylint: disable=too-few-public-methods class AlwaysExecAction(SCons.Action.FunctionAction): """Override FunctionAction.__call__ to always execute.""" @@ -484,40 +758,22 @@ class AlwaysExecAction(SCons.Action.FunctionAction): return super().__call__(*args, **kwargs) -def ninja_disguise(alias_func): - """Wrap Alias with a ninja 'emitter'.""" - seen = set() - - def ninja_alias_wrapper(env, target, source=None, action=None, **kw): - """Call the original alias_func and generate a Ninja builder call.""" - alias_name = env.subst(str(target)) - alias = alias_func(env, alias_name, source, action, **kw) - if not alias_name.startswith(NINJA_ALIAS_PREFIX) and not alias_name in seen: - alias_func( - env, - NINJA_ALIAS_PREFIX + alias_name, - env.Ninja(target="#{}.build.ninja${{NINJA_SUFFIX}}".format(alias_name), source=alias), - ) - seen.add(alias_name) - - return alias - - return ninja_alias_wrapper - - def ninja_print(_cmd, target, _source, env): - """Print an accurate generation message and tag the targets with the commands to build them.""" + """Tag targets with the commands to build them.""" if target: for tgt in target: if ( - tgt.builder is not None and - isinstance(tgt.builder.action, COMMAND_TYPES) and - getattr(tgt.attributes, NINJA_CMD, None) is None + tgt.builder is not None + # Use 'is False' because not would still trigger on + # None's which we don't want to regenerate + and getattr(tgt.attributes, NINJA_BUILD, False) is False + and isinstance(tgt.builder.action, COMMAND_TYPES) ): - print("Generating Ninja action for", str(tgt)) - setattr(tgt.attributes, NINJA_CMD, get_command(env, tgt, tgt.builder.action)) - elif getattr(tgt.attributes, NINJA_CMD, None) is None: - print("Deferring Ninja generation for", str(tgt)) + ninja_action = get_command(env, tgt, tgt.builder.action) + setattr(tgt.attributes, NINJA_BUILD, ninja_action) + # Preload the attributes dependencies while we're still running + # multithreaded + get_dependencies(tgt) return 0 @@ -526,41 +782,111 @@ def register_custom_handler(env, name, handler): env[NINJA_CUSTOM_HANDLERS][name] = handler -def ninja_decider(*args, **kwargs): - """Ninja decider skips all calculation in the decision step.""" - return False +def register_custom_rule(env, rule, command, description=""): + """Allows specification of Ninja rules from inside SCons files.""" + env[NINJA_RULES][rule] = { + "command": command, + "description": description if description else "{} $out".format(rule), + } -def exists(_env): +def exists(env): """Enable if called.""" + + # This variable disables the tool when storing the SCons command in the + # generated ninja file to ensure that the ninja tool is not loaded when + # SCons should do actual work as a subprocess of a ninja build. The ninja + # tool is very invasive into the internals of SCons and so should never be + # enabled when SCons needs to build a target. + if env.get("__NINJA_NO", "0") == "1": + return False + return True +def ninja_csig(original): + """Return a dummy csig""" + + def wrapper(self): + name = str(self) + if 'SConscript' in name or 'SConstruct' in name: + return original(self) + return "dummy_ninja_csig" + + return wrapper + + +def ninja_contents(original): + """Return a dummy content without doing IO""" + + def wrapper(self): + name = str(self) + if 'SConscript' in name or 'SConstruct' in name: + return original(self) + return bytes("dummy_ninja_contents", encoding="utf-8") + + return wrapper + + +def ninja_noop(*_args, **_kwargs): + """ + A general purpose no-op function. + + There are many things that happen in SCons that we don't need and + also don't return anything. We use this to disable those functions + instead of creating multiple definitions of the same thing. + """ + pass # pylint: disable=unnecessary-pass + + +def ninja_whereis(thing, *_args, **_kwargs): + """Replace env.WhereIs with a much faster version""" + return shutil.which(thing) + + def generate(env): """Generate the NINJA builders.""" + if not exists(env): + return + env[NINJA_SYNTAX] = env.get(NINJA_SYNTAX, "ninja_syntax.py") - # Add the Ninja builder. This really shouldn't be called by users - # but I guess it could be and will probably Do The Right Thing™ - NinjaAction = AlwaysExecAction(ninja_builder, {}) - Ninja = SCons.Builder.Builder(action=NinjaAction) - env.Append(BUILDERS={"Ninja": Ninja}) + # Add the Ninja builder. + always_exec_ninja_action = AlwaysExecAction(ninja_builder, {}) + ninja_builder_obj = SCons.Builder.Builder(action=always_exec_ninja_action) + env.Append(BUILDERS={"Ninja": ninja_builder_obj}) # Provides a way for users to handle custom FunctionActions they # want to translate to Ninja. env[NINJA_CUSTOM_HANDLERS] = {} env.AddMethod(register_custom_handler, "NinjaRegisterFunctionHandler") + env[NINJA_RULES] = {} + env.AddMethod(register_custom_rule, "NinjaRule") + # Make SCons node walk faster by preventing unnecessary work - env.Decider(ninja_decider) - env.SetOption("max_drift", 1) + env.Decider("timestamp-match") + + # Monkey patch get_csig for some node classes. It slows down the build + # significantly and we don't need content signatures calculated when + # generating a ninja file. + SCons.Node.FS.File.make_ready = ninja_noop + SCons.Node.FS.File.prepare = ninja_noop + SCons.Node.FS.File.push_to_cache = ninja_noop + SCons.Node.FS.File.built = ninja_noop + + SCons.Executor.Executor.get_contents = ninja_contents(SCons.Executor.Executor.get_contents) + SCons.Node.Alias.Alias.get_contents = ninja_contents(SCons.Node.Alias.Alias.get_contents) + SCons.Node.FS.File.get_contents = ninja_contents(SCons.Node.FS.File.get_contents) - # Monkey Patch Alias to use our custom wrapper that generates a - # "hidden" internal alias that corresponds to what the user - # actually passed as an action / source and then generates a new - # alias matching the user provided alias name that just calls the - # ninja builder for that alias. - SCons.Environment.Base.Alias = ninja_disguise(SCons.Environment.Base.Alias) + SCons.Node.FS.File.get_csig = ninja_csig(SCons.Node.FS.File.get_csig) + SCons.Node.FS.Dir.get_csig = ninja_csig(SCons.Node.FS.Dir.get_csig) + SCons.Node.Alias.Alias.get_csig = ninja_csig(SCons.Node.Alias.Alias.get_csig) + + SCons.Util.WhereIs = ninja_whereis + + # pylint: disable=protected-access + SCons.Platform.TempFileMunge._print_cmd_str = ninja_noop # Replace false Compiling* messages with a more accurate output # @@ -581,3 +907,28 @@ def generate(env): # an execution for ninja_builder so this simply effects all other # Builders. env.SetOption("no_exec", True) + + # This makes SCons more aggressively cache MD5 signatures in the + # SConsign file. + env.SetOption("max_drift", 1) + + # We will eventually need to overwrite TempFileMunge to make it + # handle persistent tempfiles or get an upstreamed change to add + # some configurability to it's behavior in regards to tempfiles. + # + # Set all three environment variables that Python's + # tempfile.mkstemp looks at as it behaves differently on different + # platforms and versions of Python. + os.environ["TMPDIR"] = env.Dir("$BUILD_DIR/response_files").get_abspath() + os.environ["TEMP"] = os.environ["TMPDIR"] + os.environ["TMP"] = os.environ["TMPDIR"] + if not os.path.isdir(os.environ["TMPDIR"]): + env.Execute(SCons.Defaults.Mkdir(os.environ["TMPDIR"])) + + # Force the SConsign to be written, we benefit from SCons caching of + # implicit dependencies and conftests. Unfortunately, we have to do this + # using an atexit handler because SCons will not write the file when in a + # no_exec build. + import atexit + + atexit.register(SCons.SConsign.write) diff --git a/src/third_party/IntelRDFPMathLib20U1/SConscript b/src/third_party/IntelRDFPMathLib20U1/SConscript index f23c071c6d2..a0e7482a0c0 100644 --- a/src/third_party/IntelRDFPMathLib20U1/SConscript +++ b/src/third_party/IntelRDFPMathLib20U1/SConscript @@ -355,6 +355,9 @@ env.Library( ] ) +if env["BUILDERS"].get("Ninja", None) is not None: + Return() + readtest = env.Program( target='intel_decimal128_readtest', source=[ @@ -367,9 +370,10 @@ readtest = env.Program( ) readtest_input = env.Install( - '.', + target='.', source=['TESTS/readtest.in'], ) +env.Depends(readtest_input, readtest) readtest_dict = { '@readtest_python_interpreter@' : sys.executable.replace('\\', r'\\'), @@ -378,14 +382,16 @@ readtest_dict = { } readtest_wrapper = env.Substfile( - 'intel_decimal128_readtest_wrapper.py.in', + target='intel_decimal128_readtest_wrapper.py', + source=['intel_decimal128_readtest_wrapper.py.in'], SUBST_DICT=readtest_dict, ) -env.Depends(readtest_wrapper, [readtest, readtest_input]) +env.Depends(readtest_wrapper, readtest_input) if env.TargetOSIs('windows'): readtest_wrapper_bat = env.Substfile( - 'intel_decimal128_readtest_wrapper.bat.in', + target='intel_decimal128_readtest_wrapper.bat', + source=['intel_decimal128_readtest_wrapper.bat.in'], SUBST_DICT=readtest_dict, ) env.Depends(readtest_wrapper_bat, readtest_wrapper) -- cgit v1.2.1