SERVER-51335 Persist fuzzer corpora for each branch and variant

6 files changed, 124 insertions, 96 deletions

diff --git a/.gitignore b/.gitignore
index aacba7ef7d7..e034398f239 100644
--- a/.gitignore
+++ b/.gitignore
@@ -200,3 +200,8 @@ resmoke.ini
 
 # UndoDB Recordings
 *.undo
+
+# libfuzzer artifacts
+default.profraw
+/corpora
+/corpora-merged
diff --git a/buildscripts/merge_corpus.sh b/buildscripts/merge_corpus.sh
deleted file mode 100755
index a7e0a598c5d..00000000000
--- a/buildscripts/merge_corpus.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/bin/bash
-# merge_corpus.sh
-#
-# Merges the corpus of each libfuzzer tests
-
-set -o verbose
-set -o errexit
-
-input="build/libfuzzer_tests.txt"
-corpus_dir="corpus"
-
-if [ ! -f $input ] || [ ! -d $corpus_dir ]; then
-    echo "Missing corpus information"
-    exit 0
-fi
-
-# For each test, merge in new data.
-while IFS= read -r line; do
-    corpus_file="$corpus_dir/corpus-${line##*/}"
-
-    if [ -d "${corpus_file}-new" ]; then
-        if [ ! -d "${corpus_file}" ]; then
-            # An error in a prior run left old corpus data orphaned, reclaim it.
-            mv -v "${corpus_file}-new" "${corpus_file}"
-        else
-            # Somehow we have multiple corpii, treat non '-new' as official.
-            rm -rv "${corpus_file}-new"
-        fi
-    fi
-
-    # Create a new merge from old corpus and new run.
-    mkdir -v "${corpus_file}-new"
-    ./"$line" "${corpus_file}-new" "$corpus_file" -merge=1
-done < "$input"
-
-# Delete old corpus.
-find corpus/* -not -name '*-new' -type d -exec rm -rv {} +
-
-# Rename new corpus to old corpus.
-for f in "$corpus_dir"/*-new; do
-    mv -v "$f" "${f%-new}"
-done
diff --git a/buildscripts/resmokeconfig/suites/libfuzzer.yml b/buildscripts/resmokeconfig/suites/libfuzzer.yml
index 3460f41ff64..870cfeaec49 100644
--- a/buildscripts/resmokeconfig/suites/libfuzzer.yml
+++ b/buildscripts/resmokeconfig/suites/libfuzzer.yml
@@ -4,4 +4,7 @@ selector:
   root: build/libfuzzer_tests.txt
 
 executor:
-  config: {}
+  config:
+    corpus_directory_stem: corpora
+  hooks:
+  - class: LibfuzzerHook
diff --git a/buildscripts/resmokelib/testing/hooks/cpp_libfuzzer.py b/buildscripts/resmokelib/testing/hooks/cpp_libfuzzer.py
new file mode 100644
index 00000000000..4e59bf97d73
--- /dev/null
+++ b/buildscripts/resmokelib/testing/hooks/cpp_libfuzzer.py
@@ -0,0 +1,57 @@
+"""Test hook that does maintainence tasks for libfuzzer tests."""
+
+import os
+
+from buildscripts.resmokelib import core
+from buildscripts.resmokelib import errors
+from buildscripts.resmokelib import utils
+from buildscripts.resmokelib.testing.fixtures import interface as fixture_interface
+from buildscripts.resmokelib.testing.hooks import interface
+
+
+class LibfuzzerHook(interface.Hook):  # pylint: disable=too-many-instance-attributes
+    """Merges inputs after a fuzzer run."""
+
+    DESCRIPTION = ("Merges inputs after a fuzzer run")
+
+    def __init__(self, hook_logger, fixture):
+        """Initialize the ContinuousStepdown.
+
+        Args:
+            hook_logger: the logger instance for this hook.
+            fixture: the target fixture.
+        """
+        interface.Hook.__init__(self, hook_logger, fixture, LibfuzzerHook.DESCRIPTION)
+
+        self._fixture = fixture
+
+    def before_suite(self, test_report):
+        """Before suite."""
+        pass
+
+    def after_suite(self, test_report):
+        """After suite."""
+        pass
+
+    def before_test(self, test, test_report):
+        """Before test."""
+        pass
+
+    def after_test(self, test, test_report):
+        """After test."""
+        self._merge_corpus(test)
+
+    def _merge_corpus(self, test):
+        self.logger.info(f"Merge for {test.short_name()} libfuzzer test started, "
+                         f"merging to {test.merged_corpus_directory}.")
+        os.makedirs(test.merged_corpus_directory, exist_ok=True)
+        default_args = [
+            test.program_executable,
+            "-merge=1",
+            test.merged_corpus_directory,
+            test.corpus_directory,
+        ]
+        process = core.programs.make_process(self.logger, default_args, **test.program_options)
+        process.start()
+        process.wait()
+        self.logger.info(f"Merge for {test.short_name()} libfuzzer test finished.")
diff --git a/buildscripts/resmokelib/testing/testcases/cpp_libfuzzer_test.py b/buildscripts/resmokelib/testing/testcases/cpp_libfuzzer_test.py
index ab2bc6063ac..98c72993c6c 100644
--- a/buildscripts/resmokelib/testing/testcases/cpp_libfuzzer_test.py
+++ b/buildscripts/resmokelib/testing/testcases/cpp_libfuzzer_test.py
@@ -2,7 +2,6 @@
 
 import datetime
 import os
-import subprocess
 
 from buildscripts.resmokelib import core
 from buildscripts.resmokelib import utils
@@ -16,42 +15,31 @@ class CPPLibfuzzerTestCase(interface.ProcessTestCase):
     REGISTERED_NAME = "cpp_libfuzzer_test"
     DEFAULT_TIMEOUT = datetime.timedelta(hours=1)
 
-    def __init__(self, logger, program_executable, program_options=None):
+    def __init__(  # pylint: disable=too-many-arguments
+            self, logger, program_executable, program_options=None, runs=1000000,
+            corpus_directory_stem="corpora"):
         """Initialize the CPPLibfuzzerTestCase with the executable to run."""
 
         interface.ProcessTestCase.__init__(self, logger, "C++ libfuzzer test", program_executable)
 
         self.program_executable = program_executable
         self.program_options = utils.default_if_none(program_options, {}).copy()
-        self.corpus_directory = "corpus/corpus-" + self.short_name()
+
+        self.runs = runs
+
+        self.corpus_directory = f"{corpus_directory_stem}/corpus-{self.short_name()}"
+        self.merged_corpus_directory = f"{corpus_directory_stem}-merged/corpus-{self.short_name()}"
 
         os.makedirs(self.corpus_directory, exist_ok=True)
 
     def _make_process(self):
         default_args = [
-            self.program_executable, self.corpus_directory, "-max_len=100000", "-rss_limit_mb=5000"
+            self.program_executable,
+            "-max_len=100000",
+            "-rss_limit_mb=5000",
+            f"-runs={self.runs}",
+            self.corpus_directory,
         ]
         self.program_options["job_num"] = self.fixture.job_num
         self.program_options["test_id"] = self._id
         return core.programs.make_process(self.logger, default_args, **self.program_options)
-
-    def _execute(self, process):
-        """Run the specified process."""
-        self.logger.info("Starting Libfuzzer Test %s...\n%s", self.short_description(),
-                         process.as_command())
-        process.start()
-        self.logger.info("%s started with pid %s.", self.short_description(), process.pid)
-        try:
-            self.return_code = process.wait(self.DEFAULT_TIMEOUT.total_seconds())
-        except subprocess.TimeoutExpired:
-            # If the test timeout, then no errors were detected. Thus, the return code should be 0.
-            process.stop(mode=fixture_interface.TeardownMode.KILL)
-            process.wait()
-            self.logger.info("%s timed out. No errors were found.", self.short_description())
-            self.return_code = 0
-
-        if self.return_code != 0:
-            self.logger.info("Failed %s", self.return_code)
-            raise self.failureException("%s failed" % (self.short_description()))
-
-        self.logger.info("%s finished.", self.short_description())
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index c6d906910d1..9aa566aecd6 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -141,7 +141,7 @@ variables:
   name: libfuzzertests!
   execution_tasks:
   - compile_and_archive_libfuzzertests
-  - run_libfuzzertests
+  - fetch_and_run_libfuzzertests
 
 - &compile_task_group_template
   name: compile_task_group_template
@@ -162,18 +162,7 @@ variables:
   - func: "save hang analyzer debugger files"
   - func: "save disk statistics"
   - func: "save system resource information"
-  - command: shell.exec
-    type: setup
-    params:
-      working_dir: src
-      shell: bash
-      script: |
-        set -o verbose
-        set -o errexit
-
-        ./buildscripts/merge_corpus.sh
-  - func: "archive new corpus"
-  - func: "upload new corpus"
+  - func: "save libfuzzertest corpora"
   - func: "remove files"
     vars:
       files: >-
@@ -549,21 +538,25 @@ functions:
     params:
       aws_key: ${s3_access_key_id}
       aws_secret: ${s3_secret_access_key}
-      remote_file: ${project}/corpus/mongo-${build_variant}-latest.tgz
       bucket: fuzzer-artifacts
-      local_file: src/corpus.tgz
+      extract_to: src/corpora
+      remote_file: ${mongo_fuzzer_corpus}
 
-  "extract corpus": &extract_corpus
-    command: archive.auto_extract
+  "fetch legacy corpus": &fetch_legacy_corpus
+    command: s3.get
     params:
-        path: src/corpus.tgz
-        destination: src/corpus
+      aws_key: ${s3_access_key_id}
+      aws_secret: ${s3_secret_access_key}
+      bucket: fuzzer-artifacts
+      # Extract the legacy corpora to the merge directory to synthesize together until we burn in.
+      extract_to: src/corpora-merged
+      remote_file: ${project}/corpus/mongo-${build_variant}-latest.tgz
 
   "archive new corpus": &archive_new_corpus
     command: archive.targz_pack
     params:
-      target: corpus.tgz
-      source_dir: src/corpus
+      target: corpora.tgz
+      source_dir: src/corpora-merged
       include:
         - "**"
 
@@ -572,14 +565,28 @@ functions:
     params:
       aws_key: ${s3_access_key_id}
       aws_secret: ${s3_secret_access_key}
-      local_file: corpus.tgz
-      remote_file: ${project}/corpus/mongo-${build_variant}-latest.tgz
       bucket: fuzzer-artifacts
+      content_type: ${content_type|application/gzip}
+      display_name: "Fuzzer Tests Corpus Tar Archive"
+      local_file: corpora.${ext|tgz}
+      optional: true
       permissions: private
+      remote_file: ${mongo_fuzzer_corpus}
       visibility: signed
+
+  "upload new corpus for mciuploads": &upload_new_corpus_mciuploads
+    command: s3.put
+    params:
+      aws_key: ${aws_key}
+      aws_secret: ${aws_secret}
+      bucket: mciuploads
       content_type: ${content_type|application/gzip}
-      display_name: "Fuzzer Tests Corpus Tar Archive"
+      display_name: Input Corpora
+      local_file: corpora.${ext|tgz}
       optional: true
+      permissions: private
+      remote_file: ${mongo_fuzzer_corpus_mciuploads}
+      visibility: signed
 
   "get buildnumber": &get_buildnumber
     command: keyval.inc
@@ -1109,6 +1116,10 @@ functions:
         value: ${project}/${build_variant}/${revision}/binaries/mongo-shell-${build_id}.${ext|tgz}
       - key: mongo_shell_debugsymbols
         value: ${project}/${build_variant}/${revision}/binaries/mongo-shell-debugsymbols-${build_id}.${ext|tgz}
+      - key: mongo_fuzzer_corpus_mciuploads
+        value: ${project}/${build_variant}/${revision}/libfuzzer-corpora/corpora-${build_id}.${ext|tgz}
+      - key: mongo_fuzzer_corpus
+        value: corpora-${project}-${build_variant}.${ext|tgz}
       - key: skip_tests
         value: skip_test-${build_id}
 
@@ -3130,6 +3141,11 @@ functions:
   - *tar_disk_statistics
   - *archive_disk_statistics
 
+  "save libfuzzertest corpora":
+  - *archive_new_corpus
+  - *upload_new_corpus
+  - *upload_new_corpus_mciuploads
+
   ### Process & archive system resource artifacts ###
   "tar system resource information": &tar_system_resource_information
     command: archive.targz_pack
@@ -3568,6 +3584,7 @@ tasks:
       vars:
         targets: archive-fuzzertests
         compiling_for_test: true
+    # Store the fuzzer executable, which we use to generate and run fuzzer inputs.
     - command: s3.put
       params:
         aws_key: ${aws_key}
@@ -3580,11 +3597,11 @@ tasks:
         content_type: application/tar
         display_name: "LibFuzzer Tests"
 
-## run_libfuzzertests - run libfuzzertests ##
-- name: run_libfuzzertests
+## fetch_and_run_libfuzzertests - get input corpora from s3 and run libfuzzertests ##
+- name: fetch_and_run_libfuzzertests
   commands:
     - func: "fetch corpus"
-    - func: "extract corpus"
+    - func: "fetch legacy corpus"
     - func: "run tests"
       vars:
         resmoke_args: --suites=libfuzzer
@@ -8440,7 +8457,7 @@ task_groups:
   name: compile_archive_and_run_libfuzzertests_TG
   tasks:
   - compile_and_archive_libfuzzertests
-  - run_libfuzzertests
+  - fetch_and_run_libfuzzertests
 
 - <<: *compile_task_group_template
   name: compile_test_and_package_serial_TG