SERVER-55300 SERVER-55731 Implement expansions handling for standalone shell

3 files changed, 100 insertions, 6 deletions

diff --git a/buildscripts/evergreen_expansions2bash.py b/buildscripts/evergreen_expansions2bash.py
new file mode 100644
index 00000000000..fe6966a0f1d
--- /dev/null
+++ b/buildscripts/evergreen_expansions2bash.py
@@ -0,0 +1,92 @@
+"""Convert Evergreen's expansions.yml to an eval-able shell script."""
+import sys
+import platform
+from shlex import quote
+from typing import Any
+
+import yaml
+import click
+
+
+def _error(msg: str) -> None:
+    print(f"___expansions_error={quote(msg)}")
+    sys.exit(1)
+
+
+def _load_defaults(defaults_file: str) -> dict:
+    with open(defaults_file) as fh:
+        defaults = yaml.safe_load(fh)
+        if not isinstance(defaults, dict):
+            _error("ERROR: expected to read a dictionary. expansions.defaults.yml"
+                   "must be a dictionary. Check the indentation.")
+
+        # expansions MUST be strings. Reject any that are not
+        bad_expansions = set()
+        for key, value in defaults.items():
+            if not isinstance(value, str):
+                bad_expansions.add(key)
+
+        if bad_expansions:
+            _error("ERROR: all default expansions must be strings. You can "
+                   " fix this error by quoting the values in expansions.defaults.yml. "
+                   "Integers, floating points, 'true', 'false', and 'null' "
+                   "must be quoted. The following keys were interpreted as "
+                   f"other types: {bad_expansions}")
+
+        # These values show up if 1. Python's str is used to naively convert
+        # a boolean to str, 2. A human manually entered one of those strings.
+        # Either way, our shell scripts expect 'true' or 'false' (leading
+        # lowercase), and we reject them as errors. This will probably save
+        # someone a lot of time, but if this assumption proves wrong, start
+        # a conversation in #server-testing.
+        risky_boolean_keys = set()
+        for key, value in defaults.items():
+            if value in ("True", "False"):
+                risky_boolean_keys.add(key)
+
+        if risky_boolean_keys:
+            _error("ERROR: Found keys which had 'True' or 'False' as values. "
+                   "Shell scripts assume that booleans are represented as 'true'"
+                   " or 'false' (leading lowercase). If you added a new boolean, "
+                   "ensure that it's represented in lowercase. If not, please report this in "
+                   f"#server-testing. Keys with bad values: {risky_boolean_keys}")
+
+        return defaults
+
+
+def _load_expansions(expansions_file) -> dict:
+    with open(expansions_file) as fh:
+        expansions = yaml.safe_load(fh)
+
+        if not isinstance(expansions, dict):
+            _error("ERROR: expected to read a dictionary. Has the output format "
+                   "of expansions.write changed?")
+
+        if not expansions:
+            _error("ERROR: found 0 expansions. This is almost certainly wrong.")
+
+        return expansions
+
+
+@click.command()
+@click.argument("expansions_file", type=str)
+@click.argument("defaults_file", type=str)
+def _main(expansions_file: str, defaults_file: str):
+    try:
+        defaults = _load_defaults(defaults_file)
+        expansions = _load_expansions(expansions_file)
+
+        # inject defaults into expansions
+        for key, value in defaults.items():
+            if key not in expansions:
+                expansions[key] = value
+
+        for key, value in expansions.items():
+            print(f"{key}={quote(value)}; ", end="")
+
+    except Exception as ex:  # pylint: disable=broad-except
+        _error(ex)
+
+
+if __name__ == "__main__":
+    _main()  # pylint: disable=no-value-for-parameter
diff --git a/buildscripts/generate_compile_expansions.py b/buildscripts/generate_compile_expansions.py
index 653f01502cb..cd3b7c63b0c 100755
--- a/buildscripts/generate_compile_expansions.py
+++ b/buildscripts/generate_compile_expansions.py
@@ -11,6 +11,7 @@ import json
 import os
 import re
 import sys
+import shlex
 import yaml
 
 VERSION_JSON = "version.json"
@@ -94,8 +95,8 @@ def generate_scons_cache_expansions():
                 scons_cache_mode = "nolinked"
 
             if os.getenv("USE_SCONS_CACHE") not in (None, False, "false", ""):
-                expansions["scons_cache_args"] = "--cache={0} --cache-dir='{1}'".format(
-                    scons_cache_mode, default_cache_path)
+                expansions["scons_cache_args"] = "--cache={0} --cache-dir={1}".format(
+                    scons_cache_mode, shlex.quote(default_cache_path))
     return expansions
 
 
diff --git a/buildscripts/generate_compile_expansions_shared_cache.py b/buildscripts/generate_compile_expansions_shared_cache.py
index 685eedb629f..4f31341c806 100755
--- a/buildscripts/generate_compile_expansions_shared_cache.py
+++ b/buildscripts/generate_compile_expansions_shared_cache.py
@@ -11,6 +11,7 @@ import json
 import os
 import re
 import sys
+import shlex
 import yaml
 
 VERSION_JSON = "version.json"
@@ -99,8 +100,8 @@ def generate_scons_cache_expansions():
             shared_mount_root = '/efs'
         default_cache_path = os.path.join(shared_mount_root, system_uuid, "scons-cache")
         expansions["scons_cache_path"] = default_cache_path
-        expansions["scons_cache_args"] = "--cache={0} --cache-dir='{1}'".format(
-            scons_cache_mode, default_cache_path)
+        expansions["scons_cache_args"] = "--cache={0} --cache-dir={1}".format(
+            scons_cache_mode, shlex.quote(default_cache_path))
 
     # Local shared cache - host-based
     elif os.getenv("SCONS_CACHE_SCOPE") == "local":
@@ -112,8 +113,8 @@ def generate_scons_cache_expansions():
 
         default_cache_path = os.path.join(default_cache_path_base, system_uuid)
         expansions["scons_cache_path"] = default_cache_path
-        expansions["scons_cache_args"] = "--cache={0} --cache-dir='{1}'".format(
-            scons_cache_mode, default_cache_path)
+        expansions["scons_cache_args"] = "--cache={0} --cache-dir={1}".format(
+            scons_cache_mode, shlex.quote(default_cache_path))
     # No cache
     else:
         # Anything else is 'none'