SERVER-65347 Add testing to ensure that generated ninja files are deterministic

2 files changed, 45 insertions, 31 deletions

diff --git a/site_scons/site_tools/auto_install_binaries.py b/site_scons/site_tools/auto_install_binaries.py
index acc248c5229..fc34fddd8ae 100644
--- a/site_scons/site_tools/auto_install_binaries.py
+++ b/site_scons/site_tools/auto_install_binaries.py
@@ -270,8 +270,8 @@ def scan_for_transitive_install(node, env, _path):
 
     installed_children = set(
         grandchild
-        for child in node.children()
-        for direct_children in child.children()
+        for source in node.sources
+        for direct_children in source.children()
         for grandchild in direct_children.get_executor().get_all_targets()
         if direct_children.get_executor() and grandchild.has_builder()
     )
@@ -286,20 +286,9 @@ def scan_for_transitive_install(node, env, _path):
             child_component = get_component(child)
             child_entry = get_alias_map_entry(env, child_component, child_role)
 
-            # This is where component inheritance happens. We need a default
-            # component for everything so we can store it but if during
-            # transitive scanning we see a child with the default component here
-            # we will move that file to our component. This prevents
-            # over-stepping the DAG bounds since the default component is likely
-            # to be large and an explicitly tagged file is unlikely to depend on
-            # everything in it.
-            if child_component == env.get(DEFAULT_COMPONENT):
-                setattr(node.attributes, COMPONENT, component)
-                for f in auto_installed_files:
-                    child_entry.files.discard(f)
-                entry.files.update(auto_installed_files)
-            elif component != child_component:
-                entry.dependencies.add(child_entry)
+            if component != child_component:
+                if child_component != env.get(DEFAULT_COMPONENT):
+                    entry.dependencies.add(child_entry)
 
             results.update(auto_installed_files)
 
diff --git a/site_scons/site_tools/ninja.py b/site_scons/site_tools/ninja.py
index d069e16382b..2fb4b84a107 100644
--- a/site_scons/site_tools/ninja.py
+++ b/site_scons/site_tools/ninja.py
@@ -31,6 +31,7 @@ import shlex
 import tempfile
 import textwrap
 
+from collections import OrderedDict
 from glob import glob
 from os.path import join as joinpath
 from os.path import splitext
@@ -699,7 +700,8 @@ class NinjaState:
 
             if generated_source_files:
                 generated_sources_alias = "_ninja_generated_sources"
-                ninja.build(
+                ninja_sorted_build(
+                    ninja,
                     outputs=generated_sources_alias,
                     rule="phony",
                     implicit=generated_source_files
@@ -782,8 +784,9 @@ class NinjaState:
                 )
 
                 if remaining_outputs:
-                    ninja.build(
-                        outputs=sorted(remaining_outputs), rule="phony", implicit=first_output,
+                    ninja_sorted_build(
+                        ninja, outputs=sorted(remaining_outputs), rule="phony",
+                        implicit=first_output,
                     )
 
                 build["outputs"] = first_output
@@ -801,7 +804,7 @@ class NinjaState:
             if "inputs" in build:
                 build["inputs"].sort()
 
-            ninja.build(**build)
+            ninja_sorted_build(ninja, **build)
 
         template_builds = {'rule': "TEMPLATE"}
         for template_builder in template_builders:
@@ -821,7 +824,7 @@ class NinjaState:
                 template_builds[agg_key] = new_val
 
         if template_builds.get("outputs", []):
-            ninja.build(**template_builds)
+            ninja_sorted_build(ninja, **template_builds)
 
         # We have to glob the SCons files here to teach the ninja file
         # how to regenerate itself. We'll never see ourselves in the
@@ -845,8 +848,9 @@ class NinjaState:
             self.env['NINJA_REGENERATE_DEPS'],
         )
 
-        ninja.build(
-            ninja_in_file_path,
+        ninja_sorted_build(
+            ninja,
+            outputs=ninja_in_file_path,
             rule="REGENERATE",
             variables={
                 "self": ninja_file_path,
@@ -856,8 +860,9 @@ class NinjaState:
         # This sets up a dependency edge between build.ninja.in and build.ninja
         # without actually taking any action to transform one into the other
         # because we write both files ourselves later.
-        ninja.build(
-            ninja_file_path,
+        ninja_sorted_build(
+            ninja,
+            outputs=ninja_file_path,
             rule="NOOP",
             inputs=[ninja_in_file_path],
             implicit=[__file__],
@@ -866,8 +871,9 @@ class NinjaState:
         # If we ever change the name/s of the rules that include
         # compile commands (i.e. something like CC) we will need to
         # update this build to reflect that complete list.
-        ninja.build(
-            "compile_commands.json",
+        ninja_sorted_build(
+            ninja,
+            outputs="compile_commands.json",
             rule="CMD",
             pool="console",
             implicit=[ninja_file],
@@ -878,8 +884,10 @@ class NinjaState:
             },
         )
 
-        ninja.build(
-            "compiledb", rule="phony", implicit=["compile_commands.json"], order_only=[generated_sources_alias],
+        ninja_sorted_build(
+            ninja, outputs="compiledb", rule="phony",
+            implicit=["compile_commands.json"],
+            order_only=[generated_sources_alias],
         )
 
         # Look in SCons's list of DEFAULT_TARGETS, find the ones that
@@ -948,9 +956,26 @@ def get_comstr(env, action, targets, sources):
     return action.genstring(targets, sources, env)
 
 
+def ninja_recursive_sorted_dict(build):
+    sorted_dict = OrderedDict()
+    for key, val in sorted(build.items()):
+        if isinstance(val, dict):
+            sorted_dict[key] = ninja_recursive_sorted_dict(val)
+        elif isinstance(val, list) and key in ('inputs', 'outputs', 'implicit', 'order_only', 'implicit_outputs'):
+            sorted_dict[key] = sorted(val)
+        else:
+            sorted_dict[key] = val
+    return sorted_dict
+
+
+def ninja_sorted_build(ninja, **build):
+    sorted_dict = ninja_recursive_sorted_dict(build)
+    ninja.build(**sorted_dict)
+
+
 def get_command_env(env):
     """
-    Return a string that sets the enrivonment for any environment variables that
+    Return a string that sets the environment for any environment variables that
     differ between the OS environment and the SCons command ENV.
 
     It will be compatible with the default shell of the operating system.
@@ -973,7 +998,7 @@ def get_command_env(env):
 
     windows = env["PLATFORM"] == "win32"
     command_env = ""
-    for key, value in scons_specified_env.items():
+    for key, value in sorted(scons_specified_env.items()):
         # Ensure that the ENV values are all strings:
         if is_List(value):
             # If the value is a list, then we assume it is a