SERVER-24689 Support automatic compile bypass for non-source code changes in Evergreen patch builds

2 files changed, 422 insertions, 21 deletions

diff --git a/buildscripts/bypass_compile_and_fetch_binaries.py b/buildscripts/bypass_compile_and_fetch_binaries.py
new file mode 100644
index 00000000000..cd60bfba44b
--- /dev/null
+++ b/buildscripts/bypass_compile_and_fetch_binaries.py
@@ -0,0 +1,312 @@
+#!/usr/bin/env python
+
+from __future__ import absolute_import
+from __future__ import print_function
+
+import argparse
+import json
+import os
+import re
+import shutil
+import sys
+import tarfile
+
+import urllib
+try:
+    from urlparse import urlparse
+except ImportError:
+    from urllib.parse import urlparse
+
+import requests
+import yaml
+
+_IS_WINDOWS = (sys.platform == "win32" or sys.platform == "cygwin")
+
+
+def executable_name(pathname):
+    # Ensure that executable files on Windows have a ".exe" extension.
+    if _IS_WINDOWS and os.path.splitext(pathname)[1] != ".exe":
+        return "{}.exe".format(pathname)
+    return pathname
+
+
+def archive_name(archive):
+    # Ensure the right archive extension is used for Windows.
+    if _IS_WINDOWS:
+        return "{}.zip".format(archive)
+    return "{}.tgz".format(archive)
+
+
+def requests_get_json(url):
+    response = requests.get(url)
+    response.raise_for_status()
+
+    try:
+        return response.json()
+    except ValueError:
+        print("Invalid JSON object returned with response: {}".format(response.text))
+        raise
+
+
+def read_evg_config():
+    """
+    Attempts to parse the Evergreen configuration from its home location.
+    Returns None if the configuration file wasn't found.
+    """
+    evg_file = os.path.expanduser("~/.evergreen.yml")
+    if os.path.isfile(evg_file):
+        with open(evg_file, "r") as fstream:
+            return yaml.safe_load(fstream)
+
+    return None
+
+
+def write_out_bypass_compile_expansions(patch_file, **expansions):
+    """
+    Write out the macro expansions to given file.
+    """
+    with open(patch_file, "w") as out_file:
+        print("Saving compile bypass expansions to {0}: ({1})".format(patch_file, expansions))
+        yaml.safe_dump(expansions, out_file, default_flow_style=False)
+
+
+def write_out_artifacts(json_file, artifacts):
+    """
+    Write out the JSON file with URLs of artifacts to given file.
+    """
+    with open(json_file, "w") as out_file:
+        print("Generating artifacts.json from pre-existing artifacts {0}".format(
+            json.dumps(artifacts, indent=4)))
+        json.dump(artifacts, out_file)
+
+
+def generate_bypass_expansions(project, build_variant, revision, build_id):
+    expansions = {}
+    # With compile bypass we need to update the URL to point to the correct name of the base commit
+    # binaries.
+    expansions["mongo_binaries"] = (archive_name("{}/{}/{}/binaries/mongo-{}".format(
+            project, build_variant, revision, build_id)))
+
+    # With compile bypass we need to update the URL to point to the correct name of the base commit
+    # debug symbols.
+    expansions["mongo_debugsymbols"] = (archive_name("{}/{}/{}/debugsymbols/debugsymbols-{}".format(
+            project, build_variant, revision, build_id)))
+
+    # With compile bypass we need to update the URL to point to the correct name of the base commit
+    # mongo shell.
+    expansions["mongo_shell"] = (archive_name("{}/{}/{}/binaries/mongo-shell-{}".format(
+            project, build_variant, revision, build_id)))
+
+    # Enable bypass compile
+    expansions["bypass_compile"] = True
+    return expansions
+
+
+def should_bypass_compile():
+    """
+    Based on the modified patch files determine whether the compile stage should be bypassed.
+
+    We use lists of files and directories to more precisely control which modified patch files will
+    lead to compile bypass.
+    """
+
+    # If changes are only from files in the bypass_files list or the bypass_directories list, then
+    # bypass compile, unless they are also found in the requires_compile_directories lists. All
+    # other file changes lead to compile.
+    # Add files to this list that should not cause compilation.
+    bypass_files = [
+        "etc/evergreen.yml",
+    ]
+
+    # Add directories to this list that should not cause compilation.
+    bypass_directories = [
+        "buildscripts/",
+        "jstests/",
+        "pytests/",
+    ]
+
+    # These files are exceptions to any whitelisted directories in bypass_directories. Changes to
+    # any of these files will disable compile bypass. Add files you know should specifically cause
+    # compilation.
+    requires_compile_files = [
+        "buildscripts/errorcodes.py",
+        "buildscripts/make_archive.py",
+        "buildscripts/moduleconfig.py",
+        "buildscripts/msitrim.py",
+        "buildscripts/packager-enterprise.py",
+        "buildscripts/packager.py",
+        "buildscripts/scons.py",
+        "buildscripts/utils.py",
+    ]
+
+    # These directories are exceptions to any whitelisted directories in bypass_directories and will
+    # disable compile bypass. Add directories you know should specifically cause compilation.
+    requires_compile_directories = [
+        "buildscripts/idl/",
+        "src/",
+    ]
+
+    args = parse_args()
+
+    with open(args.patchFile, "r") as pch:
+        for filename in pch:
+            filename = filename.rstrip()
+            # Skip directories that show up in 'git diff HEAD --name-only'.
+            if os.path.isdir(filename):
+                continue
+
+            if (filename in requires_compile_files
+                    or any(filename.startswith(directory)
+                           for directory in requires_compile_directories)):
+                print("Compile bypass disabled after detecting {} as being modified because"
+                      " it is a file known to affect compilation.".format(filename))
+                return False
+
+            if (filename not in bypass_files
+                    and not any(filename.startswith(directory)
+                                for directory in bypass_directories)):
+                print("Compile bypass disabled after detecting {} as being modified because"
+                      " it isn't a file known to not affect compilation.".format(filename))
+                return False
+    return True
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--project",
+                        required=True,
+                        help="The Evergreen project. e.g mongodb-mongo-master")
+
+    parser.add_argument("--buildVariant",
+                        required=True,
+                        help="The build variant. e.g enterprise-rhel-62-64-bit")
+
+    parser.add_argument("--revision",
+                        required=True,
+                        help="The base commit hash.")
+
+    parser.add_argument("--patchFile",
+                        required=True,
+                        help="A list of all files modified in patch build.")
+
+    parser.add_argument("--outFile",
+                        required=True,
+                        help="The YAML file to write out the macro expansions.")
+
+    parser.add_argument("--jsonArtifact",
+                        required=True,
+                        help="The JSON file to write out the metadata of files to attach to task.")
+
+    return parser.parse_args()
+
+
+def main():
+    """
+    From the /rest/v1/projects/{project}/revisions/{revision} endpoint find an existing build id
+    to generate the compile task id to use for retrieving artifacts when bypassing compile.
+
+    We retrieve the URLs to the artifacts from the task info endpoint at
+    /rest/v1/tasks/{build_id}. We only download the artifacts.tgz and extract certain files
+    in order to retain any modified patch files.
+
+    If for any reason bypass compile is false, we do not write out the macro expansion. Only if we
+    determine to bypass compile do we write out the macro expansions.
+    """
+    args = parse_args()
+
+    # Determine if we should bypass compile based on modified patch files.
+    if should_bypass_compile():
+        evg_config = read_evg_config()
+        if evg_config is None:
+            print("Could not find ~/.evergreen.yml config file. Default compile bypass to false.")
+            return
+
+        api_server = "{url.scheme}://{url.netloc}".format(
+            url=urlparse(evg_config.get("api_server_host")))
+        revision_url = "{}/rest/v1/projects/{}/revisions/{}".format(api_server, args.project,
+            args.revision)
+        revisions = requests_get_json(revision_url)
+
+        match = None
+        prefix = "{}_{}_{}_".format(args.project, args.buildVariant, args.revision)
+        # The "project" and "buildVariant" passed in may contain "-", but the "builds" listed from
+        # Evergreen only contain "_". Replace the hyphens before searching for the build.
+        prefix = prefix.replace("-", "_")
+        build_id_pattern = re.compile(prefix)
+        for build_id in revisions["builds"]:
+            # Find a suitable build_id
+            match = build_id_pattern.search(build_id)
+            if match:
+                break
+        else:
+            print("Could not find build id for revision {} on project {}."
+                " Default compile bypass to false.".format(args.revision, args.project))
+            return
+
+        # Generate the compile task id.
+        index = build_id.find(args.revision)
+        compile_task_id = "{}compile_{}".format(build_id[:index], build_id[index:])
+        task_url = "{}/rest/v1/tasks/{}".format(api_server, compile_task_id)
+        # Get info on compile task of base commit.
+        task = requests_get_json(task_url)
+        if task is None or task["status"] != "success":
+            print("Could not retrieve artifacts because the compile task {} for base commit"
+                  " was not available. Default compile bypass to false.".format(compile_task_id))
+            return
+
+        # Get the compile task artifacts from REST API
+        print("Fetching pre-existing artifacts from compile task {}".format(compile_task_id))
+        artifacts = []
+        for artifact in task["files"]:
+            filename = os.path.basename(artifact["url"])
+            if filename.startswith(build_id):
+                print("Retrieving archive {}".format(filename))
+                # This is the artifacts.tgz as referenced in evergreen.yml.
+                try:
+                    urllib.urlretrieve(artifact["url"], filename)
+                except urllib.ContentTooShortError:
+                    print("The artifact {} could not be completely downloaded. Default"
+                          " compile bypass to false.".format(filename))
+                    return
+
+                # Need to extract certain files from the pre-existing artifacts.tgz.
+                extract_files = [executable_name("dbtest"), executable_name("mongobridge"),
+                                 "build/integration_tests.txt"]
+                with tarfile.open(filename, "r:gz") as tar:
+                    # The repo/ directory contains files needed by the package task. May
+                    # need to add other files that would otherwise be generated by SCons
+                    # if we did not bypass compile.
+                    subdir = [tarinfo for tarinfo in tar.getmembers()
+                              if tarinfo.name.startswith("build/integration_tests/")
+                              or tarinfo.name.startswith("repo/")
+                              or tarinfo.name in extract_files]
+                    print("Extracting the following files from {0}...\n{1}".format(
+                        filename, "\n".join(tarinfo.name for tarinfo in subdir)))
+                    tar.extractall(members=subdir)
+            else:
+                print("Linking base artifact {} to this patch build".format(filename))
+                # For other artifacts we just add their URLs to the JSON file to upload.
+                files = {}
+                files["name"] = artifact["name"]
+                files["link"] = artifact["url"]
+                files["visibility"] = "private"
+                # Check the link exists, else raise an exception. Compile bypass is disabled.
+                requests.head(artifact["url"]).raise_for_status()
+                artifacts.append(files)
+
+        # SERVER-21492 related issue where without running scons the jstests/libs/key1
+        # and key2 files are not chmod to 0600. Need to change permissions here since we
+        # bypass SCons.
+        os.chmod("jstests/libs/key1", 0600)
+        os.chmod("jstests/libs/key2", 0600)
+
+        # This is the artifacts.json file.
+        write_out_artifacts(args.jsonArtifact, artifacts)
+
+        # Need to apply these expansions for bypassing SCons.
+        expansions = generate_bypass_expansions(args.project, args.buildVariant, args.revision,
+            build_id)
+        write_out_bypass_compile_expansions(args.outFile, **expansions)
+
+if __name__ == "__main__":
+    main()
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index b0bed5ccf99..712e8c2e293 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -81,7 +81,7 @@ variables:
       MONGO_VERSION=$(git describe)
       # If this is a patch build, we add the patch version id to the version string so we know
       # this build was a patch, and which evergreen task it came from
-      if [ "${is_patch|}" = "true" ]; then
+      if [ "${is_patch}" = "true" ] && [ "${bypass_compile|false}" = "false" ]; then
         MONGO_VERSION="$MONGO_VERSION-patch-${version_id}"
       fi
 
@@ -228,7 +228,7 @@ functions:
     params:
       aws_key: ${aws_key}
       aws_secret: ${aws_secret}
-      remote_file: ${project}/${build_variant}/${revision}/binaries/mongo-${build_id}.${ext|tgz}
+      remote_file: ${mongo_binaries}
       bucket: mciuploads
       local_file: src/mongo-binaries.tgz
 
@@ -253,6 +253,13 @@ functions:
           echo "There is more than 1 extracted mongo binary: $mongo_binary"
           exit 1
         fi
+        # For compile bypass we need to skip the binary version check since we can tag a commit
+        # after the base commit binaries were created. This would lead to a mismatch of the binaries
+        # and the version from git describe in the compile_expansions.yml.
+        if [ "${is_patch}" = "true" ] && [ "${bypass_compile|false}" = "true" ]; then
+          echo "Skipping binary version check since we are bypassing compile in this patch build."
+          exit 0
+        fi
         ${activate_virtualenv}
         bin_ver=$($python -c "import yaml; print(yaml.safe_load(open('compile_expansions.yml'))['version']);" | tr -d '[ \r\n]')
         # Due to SERVER-23810, we cannot use $mongo_binary --quiet --nodb --eval "version();"
@@ -426,10 +433,11 @@ functions:
   "upload debugsymbols" : &upload_debugsymbols
     command: s3.put
     params:
+      optional: true
       aws_key: ${aws_key}
       aws_secret: ${aws_secret}
       local_file: src/mongo-debugsymbols.tgz
-      remote_file: ${project}/${build_variant}/${revision}/debugsymbols/debugsymbols-${build_id}.${ext|tgz}
+      remote_file: ${mongo_debugsymbols}
       bucket: mciuploads
       permissions: public-read
       content_type: ${content_type|application/x-gzip}
@@ -439,7 +447,7 @@ functions:
     params:
       aws_key: ${aws_key}
       aws_secret: ${aws_secret}
-      remote_file: ${project}/${build_variant}/${revision}/debugsymbols/debugsymbols-${build_id}.${ext|tgz}
+      remote_file: ${mongo_debugsymbols}
       bucket: mciuploads
       local_file: src/mongo-debugsymbols.tgz
 
@@ -557,8 +565,57 @@ functions:
              "../../mongo-tools/$i${exe|}" --version
         done
 
+  "get modified patch files" :
+    command: shell.exec
+    params:
+      working_dir: src
+      shell: bash
+      script: |
+        set -o verbose
+        set -o errexit
+
+        # For patch builds gather the modified patch files.
+        if [ "${is_patch}" = "true" ]; then
+          # Get list of patched files
+          git diff HEAD --name-only >> patch_files.txt
+          if [ -d src/mongo/db/modules/enterprise ]; then
+            pushd src/mongo/db/modules/enterprise
+            # Update the patch_files.txt in the mongo repo.
+            git diff HEAD --name-only >> ~1/patch_files.txt
+            popd
+          fi
+        fi
+
+  "update bypass expansions" : &update_bypass_expansions
+    command: expansions.update
+    params:
+      ignore_missing_file: true
+      file: src/bypass_compile_expansions.yml
+
+  "bypass compile and fetch binaries" :
+    command: shell.exec
+    params:
+      continue_on_err: true
+      working_dir: src
+      script: |
+        set -o verbose
+        set -o errexit
+
+        # For patch builds determine if we can bypass compile.
+        if [ "${is_patch}" = "true" ]; then
+          ${activate_virtualenv}
+          $python buildscripts/bypass_compile_and_fetch_binaries.py            \
+            --project ${project}                                               \
+            --buildVariant ${build_variant}                                    \
+            --revision ${revision}                                             \
+            --patchFile patch_files.txt                                        \
+            --outFile bypass_compile_expansions.yml                            \
+            --jsonArtifact artifacts.json
+        fi
+
   "do setup" :
   - *fetch_artifacts
+  - *update_bypass_expansions
   - *fetch_binaries
   - *extract_binaries
   - *check_binary_version
@@ -685,7 +742,7 @@ functions:
             path_value="$path_value:${task_path_suffix}"
           fi
 
-          if [ "${is_patch|}" = "true" ]; then
+          if [ "${is_patch}" = "true" ]; then
             extra_args="$extra_args --tagFile=etc/test_lifecycle.yml --patchBuild"
           else
             extra_args="$extra_args --tagFile=etc/test_retrial.yml"
@@ -1669,9 +1726,9 @@ pre:
   - func: "set up virtualenv"
   - command: expansions.update
     params:
-       updates:
-       - key: activate_virtualenv
-         value: |
+      updates:
+      - key: activate_virtualenv
+        value: |
           # check if virtualenv is set up
           if [ -d "${workdir}/venv" ]; then
             if [ "Windows_NT" = "$OS" ]; then
@@ -1688,16 +1745,16 @@ pre:
             python=${python|/opt/mongodbtoolchain/v2/bin/python2}
           fi
           echo "python set to $(which python)"
-       - key: posix_workdir
-         value: eval 'if [ "Windows_NT" = "$OS" ]; then echo $(cygpath -u "${workdir}"); else echo ${workdir}; fi'
-       # For ssh disable the options GSSAPIAuthentication, CheckHostIP, StrictHostKeyChecking
-       # & UserKnownHostsFile, since these are local connections from one AWS instance to another.
-       - key: ssh_connection_options
-         value: -o GSSAPIAuthentication=no -o CheckHostIP=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=20 -o ConnectionAttempts=20
-       - key: ssh_retries
-         value: "10"
-       - key: set_sudo
-         value: |
+      - key: posix_workdir
+        value: eval 'if [ "Windows_NT" = "$OS" ]; then echo $(cygpath -u "${workdir}"); else echo ${workdir}; fi'
+      # For ssh disable the options GSSAPIAuthentication, CheckHostIP, StrictHostKeyChecking
+      # & UserKnownHostsFile, since these are local connections from one AWS instance to another.
+      - key: ssh_connection_options
+        value: -o GSSAPIAuthentication=no -o CheckHostIP=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=20 -o ConnectionAttempts=20
+      - key: ssh_retries
+        value: "10"
+      - key: set_sudo
+        value: |
           set -o > /tmp/settings.log
           set +o errexit
           grep errexit /tmp/settings.log | grep on
@@ -1714,6 +1771,13 @@ pre:
           if [ $errexit_on -eq 0 ]; then
             set -o errexit
           fi
+      - key: mongo_binaries
+        value: ${project}/${build_variant}/${revision}/binaries/mongo-${build_id}.${ext|tgz}
+      - key: mongo_debugsymbols
+        value: ${project}/${build_variant}/${revision}/debugsymbols/debugsymbols-${build_id}.${ext|tgz}
+      - key: mongo_shell
+        value: ${project}/${build_variant}/${revision}/binaries/mongo-shell-${build_id}.${ext|tgz}
+
   - command: shell.exec
     params:
       system_log: true
@@ -1736,6 +1800,7 @@ post:
       file_location: src/report.json
   - command: attach.artifacts
     params:
+      optional: true
       ignore_artifacts_for_spawn: false
       files:
         - src/archive.json
@@ -2273,6 +2338,10 @@ tasks:
   commands:
     - command: manifest.load
     - *git_get_project
+    - func: "get modified patch files"
+    # NOTE: To disable the compile bypass feature, comment out the next line.
+    - func: "bypass compile and fetch binaries"
+    - func: "update bypass expansions"
     - func: "get buildnumber"
     - func: "set up credentials"
     - func: "build new tools" # noop if ${newtools} is not "true"
@@ -2290,6 +2359,9 @@ tasks:
           set -o errexit
           set -o verbose
 
+          if [ "${is_patch}" = "true" ] && [ "${bypass_compile|false}" = "true" ]; then
+            exit 0
+          fi
           rm -rf ${install_directory|/data/mongo-install-directory}
 
           ${activate_virtualenv}
@@ -2314,6 +2386,9 @@ tasks:
           set -o errexit
           set -o verbose
 
+          if [ "${is_patch}" = "true" ] && [ "${bypass_compile|false}" = "true" ]; then
+            exit 0
+          fi
           ${activate_virtualenv}
           if [ "${has_packages|}" = "true" ] ; then
             cd buildscripts
@@ -2365,25 +2440,30 @@ tasks:
           - "src/mongo/util/options_parser/test_config_files/**"
           - "library_dependency_graph.json"
           - "src/third_party/JSON-Schema-Test-Suite/tests/draft4/**"
+          - "bypass_compile_expansions.yml"
+          - "patch_files.txt"
+          - "artifacts.json"
         exclude_files:
           - "*_test.pdb"
     - func: "upload debugsymbols"
     - command: s3.put
       params:
+        optional: true
         aws_key: ${aws_key}
         aws_secret: ${aws_secret}
         local_file: src/mongodb-binaries.tgz
-        remote_file: ${project}/${build_variant}/${revision}/binaries/mongo-${build_id}.${ext|tgz}
+        remote_file: ${mongo_binaries}
         bucket: mciuploads
         permissions: public-read
         content_type: ${content_type|application/x-gzip}
         display_name: Binaries
     - command: s3.put
       params:
+        optional: true
         aws_key: ${aws_key}
         aws_secret: ${aws_secret}
         local_file: src/shell-archive/mongodb-shell.${ext|tgz}
-        remote_file: ${project}/${build_variant}/${revision}/binaries/mongo-shell-${build_id}.${ext|tgz}
+        remote_file: ${mongo_shell}
         bucket: mciuploads
         permissions: public-read
         content_type: ${content_type|application/x-gzip}
@@ -2411,6 +2491,14 @@ tasks:
         # We only need to upload the source tarball from one of the build variants
         # because it should be the same everywhere, so just use linux-64/windows-64-2k8.
         build_variants: [ linux-64, windows-64-2k8-ssl ]
+    # For patch builds that bypass compile, we upload links to pre-existing tarballs, except for the
+    # artifacts.tgz.
+    - command: attach.artifacts
+      params:
+        optional: true
+        ignore_artifacts_for_spawn: false
+        files:
+          - src/artifacts.json
 
 ## compile_all - build all scons targets including unittests ##
 - name: compile_all
@@ -2508,6 +2596,7 @@ tasks:
   - command: shell.exec
     params:
       working_dir: burn_in_tests_clonedir
+      shell: bash
       script: |
         set -o errexit
         set -o verbose
@@ -4068,7 +4157,7 @@ tasks:
       params:
         aws_key: ${aws_key}
         aws_secret: ${aws_secret}
-        remote_file: ${project}/${build_variant}/${revision}/binaries/mongo-shell-${build_id}.${ext|tgz}
+        remote_file: ${mongo_shell}
         bucket: mciuploads
         local_file: src/mongo-shell.tgz
     - command: s3.get