diff options
author | Henrik Ingo <henrik.ingo@mongodb.com> | 2018-05-23 15:57:32 +0300 |
---|---|---|
committer | Henrik Ingo <henrik.ingo@mongodb.com> | 2018-05-23 16:23:18 +0300 |
commit | e8e45c12ae4e00d283167f7da0e1648a9d37b187 (patch) | |
tree | de5ae2a0fe3dfc1d1b989ec369b56c415ec3bdd3 | |
parent | a3210c8e070ca82907415c111b0919a82210362c (diff) | |
download | mongo-e8e45c12ae4e00d283167f7da0e1648a9d37b187.tar.gz |
SERVER-34380 Use variant level depends_on in system_perf.yml
(cherry picked from commit 712e093c2ba468dff5554202f779a03b57a82fbc)
-rw-r--r-- | etc/system_perf.yml | 246 |
1 files changed, 146 insertions, 100 deletions
diff --git a/etc/system_perf.yml b/etc/system_perf.yml index 66f6ac75fba..592dbb003aa 100644 --- a/etc/system_perf.yml +++ b/etc/system_perf.yml @@ -61,6 +61,109 @@ post: - command: shell.cleanup functions: + "compile mongodb": + - command: shell.exec + params: + working_dir: src + script: | + set -o errexit + set -o verbose + + # We get the raw version string (r1.2.3-45-gabcdef) from git + MONGO_VERSION=$(git describe) + # If we're going to compile the upstream wtdevelop repository for wiredtiger, add + # that githash to version string. + if [ "${wtdevelop|}" = "-wtdevelop" ]; then + WT_VERSION=$(cd src/third_party/wtdevelop; git describe | cut -c 9-) + MONGO_VERSION="$MONGO_VERSION-wtdevelop-$WT_VERSION" + fi + # If this is a patch build, we add the patch version id to the version string so we know + # this build was a patch, and which evergreen task it came from + if [ "${is_patch|}" = "true" ]; then + MONGO_VERSION="$MONGO_VERSION-patch-${version_id}" + fi + + # This script converts the generated version string into a sanitized version string for + # use by scons and uploading artifacts as well as information about for the scons cache. + MONGO_VERSION=$MONGO_VERSION USE_SCONS_CACHE=${use_scons_cache|false} ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/generate_compile_expansions.py --out compile_expansions.yml + # Then we load the generated version data into the agent so we can use it in task definitions + - command: expansions.update + params: + file: src/compile_expansions.yml + - command: shell.exec + params: + working_dir: src/src/mongo/gotools + script: | + set -o verbose + set -o errexit + # make sure newlines in the scripts are handled correctly by windows + if [ "Windows_NT" = "$OS" ]; then + set -o igncr + fi; + sed -i.bak "s/built-without-version-string/$(git describe)/" common/options/options.go + sed -i.bak "s/built-without-git-spec/$(git rev-parse HEAD)/" common/options/options.go + . ./${set_tools_gopath|set_gopath.sh} + build_tools="bsondump mongostat mongofiles mongoexport mongoimport mongorestore mongodump mongotop" + if [ "${build_mongoreplay}" = "true" ]; then + build_tools="$build_tools mongoreplay" + fi + for i in $build_tools; do + ${gorootvars} go build ${tooltags|} -o "../../mongo-tools/$i${exe|}" $i/main/$i.go + "../../mongo-tools/$i${exe|}" --version + done + - command: shell.exec + params: + working_dir: src + script: | + set -o errexit + set -o verbose + ${python|/opt/mongodbtoolchain/v2/bin/python2} ./buildscripts/scons.py ${compile_flags|} ${scons_cache_args|} mongo${extension} --use-new-tools mongod${extension} mongos${extension} MONGO_VERSION=${version} + mkdir -p mongodb/bin + mkdir -p mongodb/jstests/hooks + mv mongo${extension|} mongodb/bin + mv mongod${extension|} mongodb/bin + mv mongos${extension|} mongodb/bin + mv src/mongo-tools/* mongodb/bin + if [ -d jstests/hooks ] + then + echo "Fetching JS test DB correctness checks from directory jstests" + cp -a jstests/* mongodb/jstests + + echo "Now adding our own special run_validate_collections.js wrapper" + mv mongodb/jstests/hooks/run_validate_collections.js mongodb/jstests/hooks/run_validate_collections.actual.js + + cat << EOF > mongodb/jstests/hooks/run_validate_collections.js + print("NOTE: run_validate_collections.js will skip the oplog!"); + TestData = { skipValidationNamespaces: ['local.oplog.rs'] }; + load('jstests/hooks/run_validate_collections.actual.js'); + EOF + fi + tar czf mongodb${wtdevelop|}.tar.gz mongodb + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/mongodb${wtdevelop|}.tar.gz + remote_file: ${project}/${version_id}/${revision}/${platform}/mongodb${wtdevelop|}-${version_id}.tar.gz + bucket: mciuploads + permissions: public-read + content_type: ${content_type|application/x-gzip} + display_name: mongodb${wtdevelop|}.tar.gz + + # NOTE: Unlike evergreen.yml, there's no conditional here. If called, this is never a noop! + "use WiredTiger develop" : + command: shell.exec + params: + working_dir: src + script: | + set -o errexit + set -o verbose + cd src/third_party + for wtdir in api dist examples ext lang src test tools ; do + rm -rf wiredtiger/$wtdir + mv wtdevelop/$wtdir wiredtiger/ + done + "prepare environment": - command: shell.exec params: @@ -86,7 +189,7 @@ functions: storageEngine: ${storageEngine} test_control: ${test} production: true - mongodb_binary_archive: "https://s3.amazonaws.com/mciuploads/${project}/${version_id}/${revision}/${platform}/mongod-${version_id}.tar.gz" + mongodb_binary_archive: "https://s3.amazonaws.com/mciuploads/${project}/${version_id}/${revision}/${platform}/mongodb${wtdevelop|}-${version_id}.tar.gz" workloads_dir: ../src/workloads/workloads ycsb_dir: ../src/YCSB/YCSB EOF @@ -228,37 +331,9 @@ tasks: - command: git.get_project params: directory: src - - command: git.apply_patch - params: - directory: src - - command: shell.exec - params: - working_dir: src - script: | - set -o errexit - set -o verbose - ${scons|scons} ${compile_flags|} mongo mongod mongos - mkdir -p mongodb/bin - mv mongo mongodb/bin - mv mongod mongodb/bin - mv mongos mongodb/bin - tar cvf mongodb.tar mongodb - gzip mongodb.tar - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: src/mongodb.tar.gz - remote_file: ${project}/${version_id}/${revision}/${platform}/mongod-${version_id}.tar.gz - bucket: mciuploads - permissions: public-read - content_type: ${content_type|application/x-gzip} - display_name: mongodb.tar.gz + - func: "compile mongodb" - name: industry_benchmarks_WT - depends_on: - - name: compile - variant: linux-standalone priority: 5 commands: - func: "prepare environment" @@ -273,9 +348,6 @@ tasks: - name: industry_benchmarks_MMAPv1 priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -289,9 +361,6 @@ tasks: - name: industry_benchmarks_wmajority_WT priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -305,9 +374,6 @@ tasks: - name: industry_benchmarks_wmajority_MMAPv1 priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -321,9 +387,6 @@ tasks: - name: crud_workloads_WT priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -335,9 +398,6 @@ tasks: - name: mixed_workloads_WT priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -349,9 +409,6 @@ tasks: - name: misc_workloads_WT priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -363,9 +420,6 @@ tasks: - name: map_reduce_workloads_WT priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -377,9 +431,6 @@ tasks: - name: smoke_test priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -391,9 +442,6 @@ tasks: - name: crud_workloads_MMAPv1 priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -405,9 +453,6 @@ tasks: - name: mixed_workloads_MMAPv1 priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -419,9 +464,6 @@ tasks: - name: misc_workloads_MMAPv1 priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -433,9 +475,6 @@ tasks: - name: map_reduce_workloads_MMAPv1 priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -445,15 +484,8 @@ tasks: - func: "run test" - func: "analyze" - - func: "deploy cluster" - - func: "run test" - - func: "analyze" - - name: non_sharded_workloads_WT priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -465,9 +497,6 @@ tasks: - name: non_sharded_workloads_MMAPv1 priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -479,9 +508,6 @@ tasks: - name: mongos_workloads_WT priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -493,10 +519,6 @@ tasks: - name: mongos_workloads_MMAPv1 priority: 5 - depends_on: - - name: compile - variant: linux-standalone - exec_timeout_secs: 32400 # 9 hours commands: - func: "prepare environment" vars: @@ -508,9 +530,6 @@ tasks: - name: move_chunk_workloads_WT priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -522,9 +541,6 @@ tasks: - name: move_chunk_workloads_MMAPv1 priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -536,9 +552,6 @@ tasks: - name: initialsync_WT priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -550,9 +563,6 @@ tasks: - name: initialsync_MMAPv1 priority: 5 - depends_on: - - name: compile - variant : linux-standalone commands: - func: "prepare environment" vars: @@ -580,10 +590,33 @@ modules: ####################################### -# Linux Buildvariants # +# Buildvariants # ####################################### - buildvariants: + +# We are explicitly tracking the rhel70 variant compile options from evergreen.yml. If we can get +# proper artifacts directly from that project, we should do that and remove these tasks. +- name: compile-rhel70 + display_name: Compile on rhel70 + batchtime: 1440 # 24 hours + modules: + - wtdevelop + expansions: + compile_flags: --ssl MONGO_DISTMOD=rhel70 -j$(grep -c ^processor /proc/cpuinfo) --release --variables-files=etc/scons/mongodbtoolchain_gcc.vars + gorootvars: GOROOT=/opt/go PATH="/opt/go/bin:$PATH" + platform: linux + project: &project dsi + tooltags: "" + use_scons_cache: true + run_on: + - "rhel70" + tasks: + - name: compile + + +####################################### +# Linux Buildvariants # +####################################### - name: linux-1-node-replSet display_name: Linux 1-Node ReplSet batchtime: 10080 # 7 days @@ -595,9 +628,12 @@ buildvariants: setup: single-replica cluster: single platform: linux - project: &project dsi-v3.2 + project: *project run_on: - "rhel70-perf-single" + depends_on: + - name: compile + variant: compile-rhel70 tasks: - name: industry_benchmarks_WT - name: crud_workloads_WT @@ -618,13 +654,15 @@ buildvariants: batchtime: 10080 # 7 days modules: *modules expansions: - compile_flags: -j$(grep -c ^processor /proc/cpuinfo) CC=/opt/mongodbtoolchain/bin/gcc CXX=/opt/mongodbtoolchain/bin/g++ --release setup: standalone cluster: single platform: linux project: *project run_on: - "rhel70-perf-single" + depends_on: + - name: compile + variant: compile-rhel70 tasks: - name: compile distros: @@ -655,6 +693,9 @@ buildvariants: project: *project run_on: - "rhel70-perf-shard" + depends_on: + - name: compile + variant: compile-rhel70 tasks: - name: industry_benchmarks_WT - name: crud_workloads_WT @@ -686,6 +727,9 @@ buildvariants: project: *project run_on: - "rhel70-perf-replset" + depends_on: + - name: compile + variant: compile-rhel70 tasks: - name: industry_benchmarks_WT - name: crud_workloads_WT @@ -715,7 +759,9 @@ buildvariants: project: *project run_on: - "rhel70-perf-replset" + depends_on: + - name: compile + variant: compile-rhel70 tasks: - name: initialsync_WT - name: initialsync_MMAPv1 - |