SERVER-46129 Make ninja faster and contain all targets all the time

4 files changed, 157 insertions, 167 deletions

diff --git a/site_scons/site_tools/distsrc.py b/site_scons/site_tools/distsrc.py
index 6fab5559521..0ce88f938be 100644
--- a/site_scons/site_tools/distsrc.py
+++ b/site_scons/site_tools/distsrc.py
@@ -198,8 +198,8 @@ def generate(env, **kwargs):
         generator=distsrc_action_generator,
     )
 
-    def DistSrc(env, target):
-        result = env.__DISTSRC(target=target, source=[])
+    def DistSrc(env, target, **kwargs):
+        result = env.__DISTSRC(target=target, source=[], **kwargs)
         env.AlwaysBuild(result)
         env.NoCache(result)
         return result
diff --git a/site_scons/site_tools/gziptool.py b/site_scons/site_tools/gziptool.py
index dce6062c993..7d7ee7cc418 100644
--- a/site_scons/site_tools/gziptool.py
+++ b/site_scons/site_tools/gziptool.py
@@ -32,8 +32,8 @@ def generate(env, **kwargs):
         "GZIPTOOL_COMSTR", "Compressing $TARGET with gzip"
     )
 
-    def GZipTool(env, target, source):
-        result = env.__GZIPTOOL(target=target, source=source)
+    def GZipTool(env, target, source, **kwargs):
+        result = env.__GZIPTOOL(target=target, source=source, **kwargs)
         env.AlwaysBuild(result)
         return result
 
diff --git a/site_scons/site_tools/ninja.py b/site_scons/site_tools/ninja.py
index 3c7e3f8c824..0f5b89c2a40 100644
--- a/site_scons/site_tools/ninja.py
+++ b/site_scons/site_tools/ninja.py
@@ -837,7 +837,7 @@ def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False):
     def get_response_file_command(env, node, action, targets, sources, executor=None):
         if hasattr(action, "process"):
             cmd_list, _, _ = action.process(targets, sources, env, executor=executor)
-            cmd_list = [str(c) for c in cmd_list[0]]
+            cmd_list = [str(c).replace("$", "$$") for c in cmd_list[0]]
         else:
             command = generate_command(
                 env, node, action, targets, sources, executor=executor
@@ -862,11 +862,7 @@ def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False):
             )
 
         cmd, rsp_content = cmd_list[:tool_idx], cmd_list[tool_idx:]
-
-        # Some commands may have a need to pass a literal $ to a tool
-        # like ld (think $ORIGIN). We need to protect that $ from
-        # Ninja by using $$.
-        rsp_content = " ".join(rsp_content).replace('$', '$$')
+        rsp_content = " ".join(rsp_content)
 
         variables = {"rspc": rsp_content}
         variables[rule] = cmd
diff --git a/site_scons/site_tools/ninja_next.py b/site_scons/site_tools/ninja_next.py
index a865cc6dee3..f4f20d16290 100644
--- a/site_scons/site_tools/ninja_next.py
+++ b/site_scons/site_tools/ninja_next.py
@@ -29,6 +29,7 @@ from SCons.Action import _string_from_cmd_list, get_default_ENV
 from SCons.Util import is_List, flatten_sequence
 from SCons.Script import COMMAND_LINE_TARGETS
 
+NINJA_STATE = None
 NINJA_SYNTAX = "NINJA_SYNTAX"
 NINJA_RULES = "__NINJA_CUSTOM_RULES"
 NINJA_POOLS = "__NINJA_CUSTOM_POOLS"
@@ -100,8 +101,21 @@ def alias_to_ninja_build(node):
     }
 
 
-def get_dependencies(node):
+def get_order_only(node):
+    """Return a list of order only dependencies for node."""
+    if node.prerequisites is None:
+        return []
+    return [get_path(src_file(prereq)) for prereq in node.prerequisites]
+
+
+def get_dependencies(node, skip_sources=False):
     """Return a list of dependencies for node."""
+    if skip_sources:
+        return [
+            get_path(src_file(child))
+            for child in node.children()
+            if child not in node.sources
+        ]
     return [get_path(src_file(child)) for child in node.children()]
 
 
@@ -149,15 +163,14 @@ class SConsToNinjaTranslator:
             "LibSymlinksActionFunction": _lib_symlink_action_function,
         }
 
-        self.func_handlers.update(self.env[NINJA_CUSTOM_HANDLERS])
+        self.loaded_custom = False
 
     # pylint: disable=too-many-return-statements
     def action_to_ninja_build(self, node, action=None):
         """Generate build arguments dictionary for node."""
-        # Use False since None is a valid value for this Attribute
-        build = getattr(node.attributes, NINJA_BUILD, False)
-        if build is not False:
-            return build
+        if not self.loaded_custom:
+            self.func_handlers.update(self.env[NINJA_CUSTOM_HANDLERS])
+            self.loaded_custom = True
 
         if node.builder is None:
             return None
@@ -165,39 +178,49 @@ class SConsToNinjaTranslator:
         if action is None:
             action = node.builder.action
 
+        if node.env and node.env.get("NINJA_SKIP"):
+            return None
+
+        build = {}
+
         # Ideally this should never happen, and we do try to filter
         # Ninja builders out of being sources of ninja builders but I
         # can't fix every DAG problem so we just skip ninja_builders
         # if we find one
         if node.builder == self.env["BUILDERS"]["Ninja"]:
-            return None
-
-        if isinstance(action, SCons.Action.FunctionAction):
-            return self.handle_func_action(node, action)
-
-        if isinstance(action, SCons.Action.LazyAction):
+            build = None
+        elif isinstance(action, SCons.Action.FunctionAction):
+            build = self.handle_func_action(node, action)
+        elif isinstance(action, SCons.Action.LazyAction):
             # pylint: disable=protected-access
             action = action._generate_cache(node.env if node.env else self.env)
-            return self.action_to_ninja_build(node, action=action)
-
-        if isinstance(action, SCons.Action.ListAction):
-            return self.handle_list_action(node, action)
+            build = self.action_to_ninja_build(node, action=action)
+        elif isinstance(action, SCons.Action.ListAction):
+            build = self.handle_list_action(node, action)
+        elif isinstance(action, COMMAND_TYPES):
+            build = get_command(node.env if node.env else self.env, node, action)
+        else:
+            raise Exception("Got an unbuildable ListAction for: {}".format(str(node)))
 
-        if isinstance(action, COMMAND_TYPES):
-            return get_command(node.env if node.env else self.env, node, action)
+        if build is not None:
+            build["order_only"] = get_order_only(node)
 
-        raise Exception("Got an unbuildable ListAction for: {}".format(str(node)))
+        return build
 
     def handle_func_action(self, node, action):
         """Determine how to handle the function action."""
         name = action.function_name()
         # This is the name given by the Subst/Textfile builders. So return the
-        # node to indicate that SCons is required
+        # node to indicate that SCons is required. We skip sources here because
+        # dependencies don't really matter when we're going to shove these to
+        # the bottom of ninja's DAG anyway and Textfile builders can have text
+        # content as their source which doesn't work as an implicit dep in
+        # ninja.
         if name == "_action":
             return {
                 "rule": "TEMPLATE",
                 "outputs": get_outputs(node),
-                "implicit": get_dependencies(node),
+                "implicit": get_dependencies(node, skip_sources=True),
             }
 
         handler = self.func_handlers.get(name, None)
@@ -454,48 +477,21 @@ class NinjaState:
                 self.rules[rule]["deps"] = "gcc"
                 self.rules[rule]["depfile"] = "$out.d"
 
-        self.rules.update(env.get(NINJA_RULES, {}))
-        self.pools.update(env.get(NINJA_POOLS, {}))
+    def add_build(self, node):
+        if not node.has_builder():
+            return False
 
-    def generate_builds(self, node):
-        """Generate a ninja build rule for node and it's children."""
-        # Filter out nodes with no builder. They are likely source files
-        # and so no work needs to be done, it will be used in the
-        # generation for some real target.
-        #
-        # Note that all nodes have a builder attribute but it is sometimes
-        # set to None. So we cannot use a simpler hasattr check here.
-        if getattr(node, "builder", None) is None:
-            return
-
-        stack = [[node]]
-        while stack:
-            frame = stack.pop()
-            for child in frame:
-                outputs = set(get_outputs(child))
-                # Check if all the outputs are in self.built, if they
-                # are we've already seen this node and it's children.
-                if not outputs.isdisjoint(self.built):
-                    continue
-
-                self.built = self.built.union(outputs)
-                stack.append(child.children())
+        if isinstance(node, SCons.Node.Alias.Alias):
+            build = alias_to_ninja_build(node)
+        else:
+            build = self.translator.action_to_ninja_build(node)
 
-                if isinstance(child, SCons.Node.Alias.Alias):
-                    build = alias_to_ninja_build(child)
-                elif node.builder is not None:
-                    # Use False since None is a valid value for this attribute
-                    build = getattr(child.attributes, NINJA_BUILD, False)
-                    if build is False:
-                        build = self.translator.action_to_ninja_build(child)
-                else:
-                    build = None
+        # Some things are unbuild-able or need not be built in Ninja
+        if build is None:
+            return False
 
-                # Some things are unbuild-able or need not be built in Ninja
-                if build is None or build == 0:
-                    continue
-
-                self.builds.append(build)
+        self.builds.append(build)
+        return True
 
     def is_generated_source(self, output):
         """Check if output ends with a known generated suffix."""
@@ -512,7 +508,7 @@ class NinjaState:
         return False
 
     # pylint: disable=too-many-branches,too-many-locals
-    def generate(self, ninja_file, fallback_default_target=None):
+    def generate(self, ninja_file):
         """
         Generate the build.ninja.
 
@@ -521,6 +517,9 @@ class NinjaState:
         if self.__generated:
             return
 
+        self.rules.update(self.env.get(NINJA_RULES, {}))
+        self.pools.update(self.env.get(NINJA_POOLS, {}))
+
         content = io.StringIO()
         ninja = self.writer_class(content, width=100)
 
@@ -585,7 +584,9 @@ class NinjaState:
                 # generated source was rebuilt. We just need to make
                 # sure that all of these sources are generated before
                 # other builds.
-                build["order_only"] = "_generated_sources"
+                order_only = build.get("order_only", [])
+                order_only.append("_generated_sources")
+                build["order_only"] = order_only
 
             # When using a depfile Ninja can only have a single output
             # but SCons will usually have emitted an output for every
@@ -668,13 +669,6 @@ class NinjaState:
             + glob("src/**/SConscript", recursive=True),
         )
 
-        ninja.build(
-            "scons-invocation",
-            rule="CMD",
-            pool="console",
-            variables={"cmd": "echo $SCONS_INVOCATION_W_TARGETS"},
-        )
-
         # If we ever change the name/s of the rules that include
         # compile commands (i.e. something like CC) we will need to
         # update this build to reflect that complete list.
@@ -707,11 +701,6 @@ class NinjaState:
         if scons_default_targets:
             ninja.default(" ".join(scons_default_targets))
 
-        # If not then set the default to the fallback_default_target we were given.
-        # Otherwise we won't create a default ninja target.
-        elif fallback_default_target is not None:
-            ninja.default(fallback_default_target)
-
         with open(ninja_file, "w") as build_ninja:
             build_ninja.write(content.getvalue())
 
@@ -828,7 +817,7 @@ def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False):
     def get_response_file_command(env, node, action, targets, sources, executor=None):
         if hasattr(action, "process"):
             cmd_list, _, _ = action.process(targets, sources, env, executor=executor)
-            cmd_list = [str(c) for c in cmd_list[0]]
+            cmd_list = [str(c).replace("$", "$$") for c in cmd_list[0]]
         else:
             command = generate_command(
                 env, node, action, targets, sources, executor=executor
@@ -853,11 +842,7 @@ def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False):
             )
 
         cmd, rsp_content = cmd_list[:tool_idx], cmd_list[tool_idx:]
-
-        # Some commands may have a need to pass a literal $ to a tool
-        # like ld (think $ORIGIN). We need to protect that $ from
-        # Ninja by using $$.
-        rsp_content = " ".join(rsp_content).replace('$', '$$')
+        rsp_content = " ".join(rsp_content)
 
         variables = {"rspc": rsp_content}
         variables[rule] = cmd
@@ -889,7 +874,8 @@ def generate_command(env, node, action, targets, sources, executor=None):
         if cmd.endswith("&&"):
             cmd = cmd[0:-2].strip()
 
-    return cmd
+    # Escape dollars as necessary
+    return cmd.replace("$", "$$")
 
 
 def get_shell_command(env, node, action, targets, sources, executor=None):
@@ -941,6 +927,7 @@ def get_command(env, node, action):  # pylint: disable=too-many-branches
     implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)})
 
     ninja_build = {
+        "order_only": get_order_only(node),
         "outputs": get_outputs(node),
         "inputs": get_inputs(node),
         "implicit": implicit,
@@ -980,34 +967,8 @@ def ninja_builder(env, target, source):
     # here.
     print("Generating:", str(target[0]))
 
-    # The environment variable NINJA_SYNTAX points to the
-    # ninja_syntax.py module from the ninja sources found here:
-    # https://github.com/ninja-build/ninja/blob/master/misc/ninja_syntax.py
-    #
-    # This should be vendored into the build sources and it's location
-    # set in NINJA_SYNTAX. This code block loads the location from
-    # that variable, gets the absolute path to the vendored file, gets
-    # it's parent directory then uses importlib to import the module
-    # dynamically.
-    ninja_syntax_file = env[NINJA_SYNTAX]
-    if isinstance(ninja_syntax_file, str):
-        ninja_syntax_file = env.File(ninja_syntax_file).get_abspath()
-    ninja_syntax_mod_dir = os.path.dirname(ninja_syntax_file)
-    sys.path.append(ninja_syntax_mod_dir)
-    ninja_syntax_mod_name = os.path.basename(ninja_syntax_file)
-    ninja_syntax = importlib.import_module(ninja_syntax_mod_name.replace(".py", ""))
-
-    suffix = env.get("NINJA_SUFFIX", "")
-    if suffix and not suffix[0] == ".":
-        suffix = "." + suffix
-
-    generated_build_ninja = target[0].get_abspath() + suffix
-    ninja_state = NinjaState(env, ninja_syntax.Writer)
-
-    for src in source:
-        ninja_state.generate_builds(src)
-
-    ninja_state.generate(generated_build_ninja, str(source[0]))
+    generated_build_ninja = target[0].get_abspath()
+    NINJA_STATE.generate(generated_build_ninja)
 
     return 0
 
@@ -1021,25 +982,6 @@ class AlwaysExecAction(SCons.Action.FunctionAction):
         return super().__call__(*args, **kwargs)
 
 
-def ninja_print(_cmd, target, _source, env):
-    """Tag targets with the commands to build them."""
-    if target:
-        for tgt in target:
-            if (
-                tgt.has_builder()
-                # Use 'is False' because not would still trigger on
-                # None's which we don't want to regenerate
-                and getattr(tgt.attributes, NINJA_BUILD, False) is False
-                and isinstance(tgt.builder.action, COMMAND_TYPES)
-            ):
-                ninja_action = get_command(env, tgt, tgt.builder.action)
-                setattr(tgt.attributes, NINJA_BUILD, ninja_action)
-                # Preload the attributes dependencies while we're still running
-                # multithreaded
-                get_dependencies(tgt)
-    return 0
-
-
 def register_custom_handler(env, name, handler):
     """Register a custom handler for SCons function actions."""
     env[NINJA_CUSTOM_HANDLERS][name] = handler
@@ -1202,6 +1144,15 @@ def generate(env):
     ninja_builder_obj = SCons.Builder.Builder(action=always_exec_ninja_action)
     env.Append(BUILDERS={"Ninja": ninja_builder_obj})
 
+    env["NINJA_PREFIX"] = env.get("NINJA_PREFIX", "build")
+    env["NINJA_SUFFIX"] = env.get("NINJA_SUFFIX", "ninja")
+    env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja")
+
+    ninja_file_name = env.subst("${NINJA_PREFIX}.${NINJA_SUFFIX}")
+    ninja_file = env.Ninja(target=ninja_file_name, source=[])
+    env.AlwaysBuild(ninja_file)
+    env.Alias("$NINJA_ALIAS_NAME", ninja_file)
+
     # This adds the required flags such that the generated compile
     # commands will create depfiles as appropriate in the Ninja file.
     if env["PLATFORM"] == "win32":
@@ -1245,8 +1196,12 @@ def generate(env):
         from SCons.Tool.mslink import compositeLinkAction
 
         if env["LINKCOM"] == compositeLinkAction:
-            env["LINKCOM"] = '${TEMPFILE("$LINK $LINKFLAGS /OUT:$TARGET.windows $_LIBDIRFLAGS $_LIBFLAGS $_PDB $SOURCES.windows", "$LINKCOMSTR")}'
-            env["SHLINKCOM"] = '${TEMPFILE("$SHLINK $SHLINKFLAGS $_SHLINK_TARGETS $_LIBDIRFLAGS $_LIBFLAGS $_PDB $_SHLINK_SOURCES", "$SHLINKCOMSTR")}'
+            env[
+                "LINKCOM"
+            ] = '${TEMPFILE("$LINK $LINKFLAGS /OUT:$TARGET.windows $_LIBDIRFLAGS $_LIBFLAGS $_PDB $SOURCES.windows", "$LINKCOMSTR")}'
+            env[
+                "SHLINKCOM"
+            ] = '${TEMPFILE("$SHLINK $SHLINKFLAGS $_SHLINK_TARGETS $_LIBDIRFLAGS $_LIBFLAGS $_PDB $_SHLINK_SOURCES", "$SHLINKCOMSTR")}'
 
     # Normally in SCons actions for the Program and *Library builders
     # will return "${*COM}" as their pre-subst'd command line. However
@@ -1315,6 +1270,7 @@ def generate(env):
     SCons.Node.FS.File.prepare = ninja_noop
     SCons.Node.FS.File.push_to_cache = ninja_noop
     SCons.Executor.Executor.prepare = ninja_noop
+    SCons.Taskmaster.Task.prepare = ninja_noop
     SCons.Node.FS.File.built = ninja_noop
 
     # We make lstat a no-op because it is only used for SONAME
@@ -1347,20 +1303,8 @@ def generate(env):
     SCons.Node.FS.Dir.get_csig = ninja_csig(SCons.Node.FS.Dir.get_csig)
     SCons.Node.Alias.Alias.get_csig = ninja_csig(SCons.Node.Alias.Alias.get_csig)
 
-    # Replace false Compiling* messages with a more accurate output
-    #
-    # We also use this to tag all Nodes with Builders using
-    # CommandActions with the final command that was used to compile
-    # it for passing to Ninja. If we don't inject this behavior at
-    # this stage in the build too much state is lost to generate the
-    # command at the actual ninja_builder execution time for most
-    # commands.
-    #
-    # We do attempt command generation again in ninja_builder if it
-    # hasn't been tagged and it seems to work for anything that
-    # doesn't represent as a non-FunctionAction during the print_func
-    # call.
-    env["PRINT_CMD_LINE_FUNC"] = ninja_print
+    # Replace false action messages with nothing.
+    env["PRINT_CMD_LINE_FUNC"] = ninja_noop
 
     # This reduces unnecessary subst_list calls to add the compiler to
     # the implicit dependencies of targets. Since we encode full paths
@@ -1369,11 +1313,6 @@ def generate(env):
     # where we expect it.
     env["IMPLICIT_COMMAND_DEPENDENCIES"] = False
 
-    # Set build to no_exec, our sublcass of FunctionAction will force
-    # an execution for ninja_builder so this simply effects all other
-    # Builders.
-    env.SetOption("no_exec", True)
-
     # This makes SCons more aggressively cache MD5 signatures in the
     # SConsign file.
     env.SetOption("max_drift", 1)
@@ -1383,6 +1322,69 @@ def generate(env):
     # monkey the Jobs constructor to only use the Serial Job class.
     SCons.Job.Jobs.__init__ = ninja_always_serial
 
+    # The environment variable NINJA_SYNTAX points to the
+    # ninja_syntax.py module from the ninja sources found here:
+    # https://github.com/ninja-build/ninja/blob/master/misc/ninja_syntax.py
+    #
+    # This should be vendored into the build sources and it's location
+    # set in NINJA_SYNTAX. This code block loads the location from
+    # that variable, gets the absolute path to the vendored file, gets
+    # it's parent directory then uses importlib to import the module
+    # dynamically.
+    ninja_syntax_file = env[NINJA_SYNTAX]
+    if isinstance(ninja_syntax_file, str):
+        ninja_syntax_file = env.File(ninja_syntax_file).get_abspath()
+    ninja_syntax_mod_dir = os.path.dirname(ninja_syntax_file)
+    sys.path.append(ninja_syntax_mod_dir)
+    ninja_syntax_mod_name = os.path.basename(ninja_syntax_file)
+    ninja_syntax = importlib.import_module(ninja_syntax_mod_name.replace(".py", ""))
+
+    global NINJA_STATE
+    NINJA_STATE = NinjaState(env, ninja_syntax.Writer)
+
+    # Here we will force every builder to use an emitter which makes the ninja
+    # file depend on it's target. This forces the ninja file to the bottom of
+    # the DAG which is required so that we walk every target, and therefore add
+    # it to the global NINJA_STATE, before we try to write the ninja file.
+    def ninja_file_depends_on_all(target, source, env):
+        env.Depends(ninja_file, target)
+        return target, source
+
+    for _, builder in env["BUILDERS"].items():
+        emitter = builder.emitter
+        if emitter is not None:
+            builder.emitter = SCons.Builder.ListEmitter(
+                [emitter, ninja_file_depends_on_all]
+            )
+        else:
+            builder.emitter = ninja_file_depends_on_all
+
+    # Here we monkey patch the Task.execute method to not do a bunch of
+    # unnecessary work. If a build is a regular builder (i.e not a conftest and
+    # not our own Ninja builder) then we add it to the NINJA_STATE. Otherwise we
+    # build it like normal. This skips all of the caching work that this method
+    # would normally do since we aren't pulling any of these targets from the
+    # cache.
+    #
+    # In the future we may be able to use this to actually cache the build.ninja
+    # file once we have the upstream support for referencing SConscripts as File
+    # nodes.
+    def ninja_execute(self):
+        global NINJA_STATE
+
+        target = self.targets[0]
+        target_name = str(target)
+        if target_name != ninja_file_name and "conftest" not in target_name:
+            NINJA_STATE.add_build(target)
+        else:
+            target.build()
+
+    SCons.Taskmaster.Task.execute = ninja_execute
+
+    # Make needs_execute always return true instead of determining out of
+    # date-ness.
+    # SCons.Script.Main.BuildTask.needs_execute = lambda x: True
+
     # We will eventually need to overwrite TempFileMunge to make it
     # handle persistent tempfiles or get an upstreamed change to add
     # some configurability to it's behavior in regards to tempfiles.
@@ -1397,11 +1399,3 @@ def generate(env):
         env.Execute(SCons.Defaults.Mkdir(os.environ["TMPDIR"]))
 
     env["TEMPFILE"] = NinjaNoResponseFiles
-
-    # Force the SConsign to be written, we benefit from SCons caching of
-    # implicit dependencies and conftests. Unfortunately, we have to do this
-    # using an atexit handler because SCons will not write the file when in a
-    # no_exec build.
-    import atexit
-
-    atexit.register(SCons.SConsign.write)