SERVER-49298 Add proof targets for out of date test execution

4 files changed, 184 insertions, 3 deletions

diff --git a/site_scons/site_tools/auto_install_binaries.py b/site_scons/site_tools/auto_install_binaries.py
index d42cd3cba7b..38427d8ef41 100644
--- a/site_scons/site_tools/auto_install_binaries.py
+++ b/site_scons/site_tools/auto_install_binaries.py
@@ -583,9 +583,11 @@ def generate(env):  # pylint: disable=too-many-statements
     env[SUFFIX_MAP] = {}
     env[ALIAS_MAP] = defaultdict(dict)
 
-    env[TASKS] = {
-        "install": auto_install_task,
-    }
+    env.AppendUnique(
+        AIB_TASKS={
+            "install": auto_install_task,
+        }
+    )
 
     env.AddMethod(
         scan_for_transitive_install_pseudobuilder, "GetTransitivelyInstalledFiles"
diff --git a/site_scons/site_tools/mongo_integrationtest.py b/site_scons/site_tools/mongo_integrationtest.py
index 9bd5146ac50..cbaadeb610d 100644
--- a/site_scons/site_tools/mongo_integrationtest.py
+++ b/site_scons/site_tools/mongo_integrationtest.py
@@ -54,6 +54,12 @@ def build_cpp_integration_test(env, target, source, **kwargs):
     else:
         kwargs["AIB_COMPONENTS_EXTRA"] = list(integration_test_components)
 
+    # Integration tests are currently undecidable (see
+    # mongo_test_execution.py for details on undecidability) because
+    # we don't correctly express the dependency on the server
+    # components required to run them.
+    kwargs['UNDECIDABLE_TEST'] = True
+
     result = env.Program(target, source, **kwargs)
     env.RegisterTest("$INTEGRATION_TEST_LIST", result[0])
     env.Alias("$INTEGRATION_TEST_ALIAS", result[0])
diff --git a/site_scons/site_tools/mongo_libfuzzer.py b/site_scons/site_tools/mongo_libfuzzer.py
index 3f15032ca56..bcbc0412688 100644
--- a/site_scons/site_tools/mongo_libfuzzer.py
+++ b/site_scons/site_tools/mongo_libfuzzer.py
@@ -60,6 +60,10 @@ def build_cpp_libfuzzer_test(env, target, source, **kwargs):
 
     kwargs["AIB_COMPONENTS_EXTRA"] =  list(libfuzzer_test_components)
 
+    # Fuzzer tests are inherenently undecidable (see
+    # mongo_test_execution.py for details on undecidability).
+    kwargs['UNDECIDABLE_TEST'] = True
+
     result = myenv.Program(target, source, **kwargs)
     myenv.RegisterTest("$LIBFUZZER_TEST_LIST", result[0])
     myenv.Alias("$LIBFUZZER_TEST_ALIAS", result)
diff --git a/site_scons/site_tools/next/mongo_test_execution.py b/site_scons/site_tools/next/mongo_test_execution.py
new file mode 100644
index 00000000000..b7b783c006e
--- /dev/null
+++ b/site_scons/site_tools/next/mongo_test_execution.py
@@ -0,0 +1,169 @@
+# Copyright 2020 MongoDB Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+
+import os
+
+import SCons
+
+import auto_install_binaries
+
+_proof_scanner_cache_key = "proof_scanner_cache"
+_associated_proof = "associated_proof_key"
+
+def proof_generator_command_scanner_func(node, env, path):
+    results = getattr(node.attributes, _proof_scanner_cache_key, None)
+    if results is not None:
+        return results
+    results = env.GetTransitivelyInstalledFiles(node)
+    setattr(node.attributes, _proof_scanner_cache_key, results)
+    return results
+
+proof_generator_command_scanner = SCons.Scanner.Scanner(
+    function=proof_generator_command_scanner_func,
+    path_function=None,
+    recursive=True
+)
+
+def auto_prove_task(env, component, role):
+    entry = auto_install_binaries.get_alias_map_entry(env, component, role)
+    return [
+        getattr(f.attributes, _associated_proof)
+        for f in entry.files
+        if hasattr(f.attributes, _associated_proof)
+    ]
+
+def generate_test_execution_aliases(env, test):
+    installed = [test]
+    if env.get("AUTO_INSTALL_ENABLED", False) and env.GetAutoInstalledFiles(test):
+        installed = env.GetAutoInstalledFiles(test)
+
+    target_name = os.path.basename(installed[0].path)
+
+    target_command = env.Command(
+        target=f"#+{target_name}",
+        source=installed[0],
+        action="$( $ICERUN $) ${SOURCES[0]} $UNITTEST_FLAGS",
+        NINJA_POOL="console",
+    )
+    env.Pseudo(target_command)
+    env.Alias("test-execution-aliases", target_command)
+
+    for source in test.sources:
+        source_base_name = os.path.basename(source.get_path())
+        # Strip suffix
+        dot_idx = source_base_name.rfind(".")
+        suffix = source_base_name[dot_idx:]
+        if suffix in env["TEST_EXECUTION_SUFFIX_BLACKLIST"]:
+            continue
+
+        source_name = source_base_name[:dot_idx]
+        if target_name == source_name:
+            continue
+
+        source_command = env.Command(
+            target=f"#+{source_name}",
+            source=installed[0],
+            action="$( $ICERUN $) ${SOURCES[0]} -fileNameFilter $TEST_SOURCE_FILE_NAME $UNITTEST_FLAGS",
+            TEST_SOURCE_FILE_NAME=source_name,
+            NINJA_POOL="console",
+        )
+        env.Pseudo(source_command)
+        env.Alias('test-execution-aliases', source_command)
+
+    proof_generator_command = env.Command(
+        target=[
+            '${SOURCE}.log',
+            '${SOURCE}.status',
+        ],
+        source=installed[0],
+        action=SCons.Action.Action(
+            "$PROOF_GENERATOR_COMMAND",
+            "$PROOF_GENERATOR_COMSTR"
+        ),
+        source_scanner=proof_generator_command_scanner
+    )
+
+    # We assume tests are provable by default, but some tests may not
+    # be. Such tests can be tagged with UNDECIDABLE_TEST=True. If a
+    # test isn't provable, we disable caching its results and require
+    # it to be always rebuilt.
+    if installed[0].env.get('UNDECIDABLE_TEST', False):
+        env.NoCache(proof_generator_command)
+        env.AlwaysBuild(proof_generator_command)
+
+    proof_analyzer_command = env.Command(
+        target='${SOURCES[1].base}.proof',
+        source=proof_generator_command,
+        action=SCons.Action.Action(
+            "$PROOF_ANALYZER_COMMAND",
+            "$PROOF_ANALYZER_COMSTR"
+        )
+    )
+
+    proof_analyzer_alias = env.Alias(
+        f"prove-{target_name}",
+        proof_analyzer_command,
+    )
+
+    setattr(installed[0].attributes, _associated_proof, proof_analyzer_alias)
+
+    # TODO: Should we enable proof at the file level?
+
+def exists(env):
+    return True
+
+
+def generate(env):
+    # Used for Ninja generator to collect the test execution aliases
+    env.Alias("test-execution-aliases")
+    env.AddMethod(generate_test_execution_aliases, "GenerateTestExecutionAliases")
+
+    env["TEST_EXECUTION_SUFFIX_BLACKLIST"] = env.get(
+        "TEST_EXECUTION_SUFFIX_BLACKLIST", [".in"]
+    )
+
+    env.AppendUnique(
+        AIB_TASKS={
+            "prove": (auto_prove_task, False),
+        }
+    )
+
+    # TODO: Should we have some sort of prefix_xdir for the output location for these? Something like
+    # $PREFIX_VARCACHE and which in our build is pre-populated to $PREFIX/var/cache/mongo or similar?
+
+    if env['PLATFORM'] == 'win32':
+        env["PROOF_GENERATOR_COMMAND"] = "$( $ICERUN $) ${SOURCES[0]} $UNITTEST_FLAGS > ${TARGETS[0]} 2>&1 & call echo %^errorlevel% > ${TARGETS[1]}"
+
+        # Keeping this here for later, but it only works if cmd.exe is
+        # launched with /V, and SCons doesn't do that.
+        #
+        # env["PROOF_ANALYZER_COMMAND"] = "set /p nextErrorLevel=<${SOURCES[1]} & if \"!nextErrorLevel!\"==\"0 \" (type nul > $TARGET) else (exit 1)"
+        #
+        # Instead, use grep! I mean findstr.
+        env["PROOF_ANALYZER_COMMAND"] = "findstr /B /L 0 ${SOURCES[1]} && (type nul > $TARGET) || (exit 1)"
+    else:
+        env["PROOF_GENERATOR_COMMAND"] = "$( $ICERUN $) ${SOURCES[0]} $UNITTEST_FLAGS > ${TARGETS[0]} 2>&1 ; echo $? > ${TARGETS[1]}"
+        env["PROOF_ANALYZER_COMMAND"] = "if $$(exit $$(cat ${SOURCES[1]})) ; then touch $TARGET ; else exit 1 ; fi"
+
+    # TODO: Condition this on verbosity
+    env['PROOF_GENERATOR_COMSTR'] = "Running test ${SOURCES[0]}"
+    env['PROOF_ANALYZER_COMSTR'] = "Analyzing test results in ${SOURCES[1]}"