SERVER-34380 Use variant level depends_on in system_perf.yml

(cherry picked from commit 712e093c2ba468dff5554202f779a03b57a82fbc)

1 files changed, 146 insertions, 100 deletions

diff --git a/etc/system_perf.yml b/etc/system_perf.yml
index 66f6ac75fba..592dbb003aa 100644
--- a/etc/system_perf.yml
+++ b/etc/system_perf.yml
@@ -61,6 +61,109 @@ post:
     - command: shell.cleanup
 
 functions:
+  "compile mongodb":
+    - command: shell.exec
+      params:
+        working_dir: src
+        script: |
+          set -o errexit
+          set -o verbose
+
+          # We get the raw version string (r1.2.3-45-gabcdef) from git
+          MONGO_VERSION=$(git describe)
+          # If we're going to compile the upstream wtdevelop repository for wiredtiger, add
+          # that githash to version string.
+          if [ "${wtdevelop|}" = "-wtdevelop" ]; then
+            WT_VERSION=$(cd src/third_party/wtdevelop; git describe | cut -c 9-)
+            MONGO_VERSION="$MONGO_VERSION-wtdevelop-$WT_VERSION"
+          fi
+          # If this is a patch build, we add the patch version id to the version string so we know
+          # this build was a patch, and which evergreen task it came from
+          if [ "${is_patch|}" = "true" ]; then
+            MONGO_VERSION="$MONGO_VERSION-patch-${version_id}"
+          fi
+
+          # This script converts the generated version string into a sanitized version string for
+          # use by scons and uploading artifacts as well as information about for the scons cache.
+          MONGO_VERSION=$MONGO_VERSION USE_SCONS_CACHE=${use_scons_cache|false} ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/generate_compile_expansions.py --out compile_expansions.yml
+    # Then we load the generated version data into the agent so we can use it in task definitions
+    - command: expansions.update
+      params:
+        file: src/compile_expansions.yml
+    - command: shell.exec
+      params:
+        working_dir: src/src/mongo/gotools
+        script: |
+          set -o verbose
+          set -o errexit
+          # make sure newlines in the scripts are handled correctly by windows
+          if [ "Windows_NT" = "$OS" ]; then
+            set -o igncr
+          fi;
+          sed -i.bak "s/built-without-version-string/$(git describe)/" common/options/options.go
+          sed -i.bak "s/built-without-git-spec/$(git rev-parse HEAD)/" common/options/options.go
+          . ./${set_tools_gopath|set_gopath.sh}
+          build_tools="bsondump mongostat mongofiles mongoexport mongoimport mongorestore mongodump mongotop"
+          if [ "${build_mongoreplay}" = "true" ]; then
+            build_tools="$build_tools mongoreplay"
+          fi
+          for i in $build_tools; do
+            ${gorootvars} go build ${tooltags|} -o "../../mongo-tools/$i${exe|}" $i/main/$i.go
+            "../../mongo-tools/$i${exe|}" --version
+          done
+    - command: shell.exec
+      params:
+        working_dir: src
+        script: |
+          set -o errexit
+          set -o verbose
+          ${python|/opt/mongodbtoolchain/v2/bin/python2} ./buildscripts/scons.py ${compile_flags|} ${scons_cache_args|} mongo${extension} --use-new-tools mongod${extension} mongos${extension} MONGO_VERSION=${version}
+          mkdir -p mongodb/bin
+          mkdir -p mongodb/jstests/hooks
+          mv mongo${extension|} mongodb/bin
+          mv mongod${extension|} mongodb/bin
+          mv mongos${extension|} mongodb/bin
+          mv src/mongo-tools/* mongodb/bin
+          if [ -d jstests/hooks ]
+          then
+            echo "Fetching JS test DB correctness checks from directory jstests"
+            cp -a jstests/* mongodb/jstests
+
+            echo "Now adding our own special run_validate_collections.js wrapper"
+            mv mongodb/jstests/hooks/run_validate_collections.js mongodb/jstests/hooks/run_validate_collections.actual.js
+
+            cat << EOF > mongodb/jstests/hooks/run_validate_collections.js
+            print("NOTE: run_validate_collections.js will skip the oplog!");
+            TestData = { skipValidationNamespaces: ['local.oplog.rs'] };
+            load('jstests/hooks/run_validate_collections.actual.js');
+          EOF
+          fi
+          tar czf mongodb${wtdevelop|}.tar.gz mongodb
+    - command: s3.put
+      params:
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: src/mongodb${wtdevelop|}.tar.gz
+        remote_file: ${project}/${version_id}/${revision}/${platform}/mongodb${wtdevelop|}-${version_id}.tar.gz
+        bucket: mciuploads
+        permissions: public-read
+        content_type: ${content_type|application/x-gzip}
+        display_name: mongodb${wtdevelop|}.tar.gz
+
+  # NOTE: Unlike evergreen.yml, there's no conditional here. If called, this is never a noop!
+  "use WiredTiger develop" :
+    command: shell.exec
+    params:
+      working_dir: src
+      script: |
+        set -o errexit
+        set -o verbose
+        cd src/third_party
+        for wtdir in api dist examples ext lang src test tools ; do
+          rm -rf wiredtiger/$wtdir
+          mv wtdevelop/$wtdir wiredtiger/
+        done
+
   "prepare environment":
     - command: shell.exec
       params:
@@ -86,7 +189,7 @@ functions:
           storageEngine: ${storageEngine}
           test_control: ${test}
           production: true
-          mongodb_binary_archive: "https://s3.amazonaws.com/mciuploads/${project}/${version_id}/${revision}/${platform}/mongod-${version_id}.tar.gz"
+          mongodb_binary_archive: "https://s3.amazonaws.com/mciuploads/${project}/${version_id}/${revision}/${platform}/mongodb${wtdevelop|}-${version_id}.tar.gz"
           workloads_dir: ../src/workloads/workloads
           ycsb_dir: ../src/YCSB/YCSB
           EOF
@@ -228,37 +331,9 @@ tasks:
     - command: git.get_project
       params:
         directory: src
-    - command: git.apply_patch
-      params:
-        directory: src
-    - command: shell.exec
-      params:
-        working_dir: src
-        script: |
-          set -o errexit
-          set -o verbose
-          ${scons|scons} ${compile_flags|} mongo mongod mongos
-          mkdir -p mongodb/bin
-          mv mongo mongodb/bin
-          mv mongod mongodb/bin
-          mv mongos mongodb/bin
-          tar cvf mongodb.tar mongodb
-          gzip mongodb.tar
-    - command: s3.put
-      params:
-        aws_key: ${aws_key}
-        aws_secret: ${aws_secret}
-        local_file: src/mongodb.tar.gz
-        remote_file: ${project}/${version_id}/${revision}/${platform}/mongod-${version_id}.tar.gz
-        bucket: mciuploads
-        permissions: public-read
-        content_type: ${content_type|application/x-gzip}
-        display_name: mongodb.tar.gz
+    - func: "compile mongodb"
 
 - name: industry_benchmarks_WT
-  depends_on:
-    - name: compile
-      variant: linux-standalone
   priority: 5
   commands:
     - func: "prepare environment"
@@ -273,9 +348,6 @@ tasks:
 
 - name: industry_benchmarks_MMAPv1
   priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
   commands:
     - func: "prepare environment"
       vars:
@@ -289,9 +361,6 @@ tasks:
 
 - name: industry_benchmarks_wmajority_WT
   priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
   commands:
     - func: "prepare environment"
       vars:
@@ -305,9 +374,6 @@ tasks:
 
 - name: industry_benchmarks_wmajority_MMAPv1
   priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
   commands:
     - func: "prepare environment"
       vars:
@@ -321,9 +387,6 @@ tasks:
 
 - name: crud_workloads_WT
   priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
   commands:
     - func: "prepare environment"
       vars:
@@ -335,9 +398,6 @@ tasks:
 
 - name: mixed_workloads_WT
   priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
   commands:
     - func: "prepare environment"
       vars:
@@ -349,9 +409,6 @@ tasks:
 
 - name: misc_workloads_WT
   priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
   commands:
     - func: "prepare environment"
       vars:
@@ -363,9 +420,6 @@ tasks:
 
 - name: map_reduce_workloads_WT
   priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
   commands:
     - func: "prepare environment"
       vars:
@@ -377,9 +431,6 @@ tasks:
 
 - name: smoke_test
   priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
   commands:
     - func: "prepare environment"
       vars:
@@ -391,9 +442,6 @@ tasks:
 
 - name: crud_workloads_MMAPv1
   priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
   commands:
     - func: "prepare environment"
       vars:
@@ -405,9 +453,6 @@ tasks:
 
 - name: mixed_workloads_MMAPv1
   priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
   commands:
     - func: "prepare environment"
       vars:
@@ -419,9 +464,6 @@ tasks:
 
 - name: misc_workloads_MMAPv1
   priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
   commands:
     - func: "prepare environment"
       vars:
@@ -433,9 +475,6 @@ tasks:
 
 - name: map_reduce_workloads_MMAPv1
   priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
   commands:
     - func: "prepare environment"
       vars:
@@ -445,15 +484,8 @@ tasks:
     - func: "run test"
     - func: "analyze"
 
-    - func: "deploy cluster"
-    - func: "run test"
-    - func: "analyze"
-
 - name: non_sharded_workloads_WT
   priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
   commands:
     - func: "prepare environment"
       vars:
@@ -465,9 +497,6 @@ tasks:
 
 - name: non_sharded_workloads_MMAPv1
   priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
   commands:
     - func: "prepare environment"
       vars:
@@ -479,9 +508,6 @@ tasks:
 
 - name: mongos_workloads_WT
   priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
   commands:
     - func: "prepare environment"
       vars:
@@ -493,10 +519,6 @@ tasks:
 
 - name: mongos_workloads_MMAPv1
   priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
-  exec_timeout_secs: 32400 # 9 hours
   commands:
     - func: "prepare environment"
       vars:
@@ -508,9 +530,6 @@ tasks:
 
 - name: move_chunk_workloads_WT
   priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
   commands:
     - func: "prepare environment"
       vars:
@@ -522,9 +541,6 @@ tasks:
 
 - name: move_chunk_workloads_MMAPv1
   priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
   commands:
     - func: "prepare environment"
       vars:
@@ -536,9 +552,6 @@ tasks:
 
 - name: initialsync_WT
   priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
   commands:
     - func: "prepare environment"
       vars:
@@ -550,9 +563,6 @@ tasks:
 
 - name: initialsync_MMAPv1
   priority: 5
-  depends_on:
-    - name: compile
-      variant : linux-standalone
   commands:
     - func: "prepare environment"
       vars:
@@ -580,10 +590,33 @@ modules:
 
 
 #######################################
-#         Linux Buildvariants         #
+#         Buildvariants               #
 #######################################
-
 buildvariants:
+
+# We are explicitly tracking the rhel70 variant compile options from evergreen.yml.  If we can get
+# proper artifacts directly from that project, we should do that and remove these tasks.
+- name: compile-rhel70
+  display_name: Compile on rhel70
+  batchtime: 1440 # 24 hours
+  modules:
+    - wtdevelop
+  expansions:
+    compile_flags: --ssl MONGO_DISTMOD=rhel70 -j$(grep -c ^processor /proc/cpuinfo) --release --variables-files=etc/scons/mongodbtoolchain_gcc.vars
+    gorootvars: GOROOT=/opt/go PATH="/opt/go/bin:$PATH"
+    platform: linux
+    project: &project dsi
+    tooltags: ""
+    use_scons_cache: true
+  run_on:
+      - "rhel70"
+  tasks:
+    - name: compile
+
+
+#######################################
+#         Linux Buildvariants         #
+#######################################
 - name: linux-1-node-replSet
   display_name: Linux 1-Node ReplSet
   batchtime: 10080 # 7 days
@@ -595,9 +628,12 @@ buildvariants:
     setup: single-replica
     cluster: single
     platform: linux
-    project: &project dsi-v3.2
+    project: *project
   run_on:
       - "rhel70-perf-single"
+  depends_on:
+      - name: compile
+        variant: compile-rhel70
   tasks:
     - name: industry_benchmarks_WT
     - name: crud_workloads_WT
@@ -618,13 +654,15 @@ buildvariants:
   batchtime: 10080 # 7 days
   modules: *modules
   expansions:
-    compile_flags: -j$(grep -c ^processor /proc/cpuinfo) CC=/opt/mongodbtoolchain/bin/gcc CXX=/opt/mongodbtoolchain/bin/g++ --release
     setup: standalone
     cluster: single
     platform: linux
     project: *project
   run_on:
       - "rhel70-perf-single"
+  depends_on:
+      - name: compile
+        variant: compile-rhel70
   tasks:
     - name: compile
       distros:
@@ -655,6 +693,9 @@ buildvariants:
     project: *project
   run_on:
       - "rhel70-perf-shard"
+  depends_on:
+      - name: compile
+        variant: compile-rhel70
   tasks:
     - name: industry_benchmarks_WT
     - name: crud_workloads_WT
@@ -686,6 +727,9 @@ buildvariants:
     project: *project
   run_on:
       - "rhel70-perf-replset"
+  depends_on:
+      - name: compile
+        variant: compile-rhel70
   tasks:
     - name: industry_benchmarks_WT
     - name: crud_workloads_WT
@@ -715,7 +759,9 @@ buildvariants:
     project: *project
   run_on:
       - "rhel70-perf-replset"
+  depends_on:
+      - name: compile
+        variant: compile-rhel70
   tasks:
     - name: initialsync_WT
     - name: initialsync_MMAPv1
-