diff options
Diffstat (limited to 'etc/system_perf.yml')
-rw-r--r-- | etc/system_perf.yml | 293 |
1 files changed, 149 insertions, 144 deletions
diff --git a/etc/system_perf.yml b/etc/system_perf.yml index b278ca3b75a..8f62a401e5b 100644 --- a/etc/system_perf.yml +++ b/etc/system_perf.yml @@ -60,6 +60,109 @@ post: fi functions: + "compile mongodb": + - command: shell.exec + params: + working_dir: src + script: | + set -o errexit + set -o verbose + + # We get the raw version string (r1.2.3-45-gabcdef) from git + MONGO_VERSION=$(git describe) + # If we're going to compile the upstream wtdevelop repository for wiredtiger, add + # that githash to version string. + if [ "${wtdevelop|}" = "-wtdevelop" ]; then + WT_VERSION=$(cd src/third_party/wtdevelop; git describe | cut -c 9-) + MONGO_VERSION="$MONGO_VERSION-wtdevelop-$WT_VERSION" + fi + # If this is a patch build, we add the patch version id to the version string so we know + # this build was a patch, and which evergreen task it came from + if [ "${is_patch|}" = "true" ]; then + MONGO_VERSION="$MONGO_VERSION-patch-${version_id}" + fi + + # This script converts the generated version string into a sanitized version string for + # use by scons and uploading artifacts as well as information about for the scons cache. + MONGO_VERSION=$MONGO_VERSION USE_SCONS_CACHE=${use_scons_cache|false} ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/generate_compile_expansions.py --out compile_expansions.yml + # Then we load the generated version data into the agent so we can use it in task definitions + - command: expansions.update + params: + file: src/compile_expansions.yml + - command: shell.exec + params: + working_dir: src/src/mongo/gotools + script: | + set -o verbose + set -o errexit + # make sure newlines in the scripts are handled correctly by windows + if [ "Windows_NT" = "$OS" ]; then + set -o igncr + fi; + sed -i.bak "s/built-without-version-string/$(git describe)/" common/options/options.go + sed -i.bak "s/built-without-git-spec/$(git rev-parse HEAD)/" common/options/options.go + . ./${set_tools_gopath|set_gopath.sh} + build_tools="bsondump mongostat mongofiles mongoexport mongoimport mongorestore mongodump mongotop" + if [ "${build_mongoreplay}" = "true" ]; then + build_tools="$build_tools mongoreplay" + fi + for i in $build_tools; do + ${gorootvars} go build ${tooltags|} -o "../../mongo-tools/$i${exe|}" $i/main/$i.go + "../../mongo-tools/$i${exe|}" --version + done + - command: shell.exec + params: + working_dir: src + script: | + set -o errexit + set -o verbose + ${python|/opt/mongodbtoolchain/v2/bin/python2} ./buildscripts/scons.py ${compile_flags|} ${scons_cache_args|} mongo${extension} --use-new-tools mongod${extension} mongos${extension} MONGO_VERSION=${version} + mkdir -p mongodb/bin + mkdir -p mongodb/jstests/hooks + mv mongo${extension|} mongodb/bin + mv mongod${extension|} mongodb/bin + mv mongos${extension|} mongodb/bin + mv src/mongo-tools/* mongodb/bin + if [ -d jstests/hooks ] + then + echo "Fetching JS test DB correctness checks from directory jstests" + cp -a jstests/* mongodb/jstests + + echo "Now adding our own special run_validate_collections.js wrapper" + mv mongodb/jstests/hooks/run_validate_collections.js mongodb/jstests/hooks/run_validate_collections.actual.js + + cat << EOF > mongodb/jstests/hooks/run_validate_collections.js + print("NOTE: run_validate_collections.js will skip the oplog!"); + TestData = { skipValidationNamespaces: ['local.oplog.rs'] }; + load('jstests/hooks/run_validate_collections.actual.js'); + EOF + fi + tar czf mongodb${wtdevelop|}.tar.gz mongodb + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/mongodb${wtdevelop|}.tar.gz + remote_file: ${project}/${version_id}/${revision}/${platform}/mongodb${wtdevelop|}-${version_id}.tar.gz + bucket: mciuploads + permissions: public-read + content_type: ${content_type|application/x-gzip} + display_name: mongodb${wtdevelop|}.tar.gz + + # NOTE: Unlike evergreen.yml, there's no conditional here. If called, this is never a noop! + "use WiredTiger develop" : + command: shell.exec + params: + working_dir: src + script: | + set -o errexit + set -o verbose + cd src/third_party + for wtdir in api dist examples ext lang src test tools ; do + rm -rf wiredtiger/$wtdir + mv wtdevelop/$wtdir wiredtiger/ + done + "prepare environment": - command: shell.exec params: @@ -85,7 +188,7 @@ functions: storageEngine: ${storageEngine} test_control: ${test} production: true - mongodb_binary_archive: "https://s3.amazonaws.com/mciuploads/${project}/${version_id}/${revision}/${platform}/mongod-${version_id}.tar.gz" + mongodb_binary_archive: "https://s3.amazonaws.com/mciuploads/${project}/${version_id}/${revision}/${platform}/mongodb${wtdevelop|}-${version_id}.tar.gz" workloads_dir: ../src/workloads/workloads ycsb_dir: ../src/YCSB/YCSB EOF @@ -228,71 +331,9 @@ tasks: - command: git.get_project params: directory: src - - command: shell.exec - params: - working_dir: src - script: | - set -o errexit - set -o verbose - - # We get the raw version string (r1.2.3-45-gabcdef) from git - MONGO_VERSION=$(git describe) - # If this is a patch build, we add the patch version id to the version string so we know - # this build was a patch, and which evergreen task it came from - if [ "${is_patch|}" = "true" ]; then - MONGO_VERSION="$MONGO_VERSION-patch-${version_id}" - fi - - # This script converts the generated version string into a sanitized version string for - # use by scons and uploading artifacts as well as information about for the scons cache. - echo $MONGO_VERSION | USE_SCONS_CACHE=${use_scons_cache|false} ${python|python} buildscripts/generate_compile_expansions.py | tee compile_expansions.yml - # Then we load the generated version data into the agent so we can use it in task definitions - - command: expansions.update - params: - file: src/compile_expansions.yml - - command: shell.exec - params: - working_dir: src - script: | - set -o errexit - set -o verbose - ${python|python} ./buildscripts/scons.py ${compile_flags|} ${scons_cache_args|} mongo${extension} mongod${extension} mongos${extension} MONGO_VERSION=${version} - mkdir -p mongodb/bin - mkdir -p mongodb/jstests/hooks - mv mongo${extension|} mongodb/bin - mv mongod${extension|} mongodb/bin - mv mongos${extension|} mongodb/bin - if [ -d jstests/hooks ] - then - echo "Fetching JS test DB correctness checks from directory jstests" - cp -a jstests/* mongodb/jstests - - echo "Now adding our own special run_validate_collections.js wrapper" - mv mongodb/jstests/hooks/run_validate_collections.js mongodb/jstests/hooks/run_validate_collections.actual.js - - cat << EOF > mongodb/jstests/hooks/run_validate_collections.js - print("NOTE: run_validate_collections.js will skip the oplog!"); - TestData = { skipValidationNamespaces: ['local.oplog.rs'] }; - load('jstests/hooks/run_validate_collections.actual.js'); - EOF - fi - tar cf mongodb.tar mongodb - gzip mongodb.tar - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: src/mongodb.tar.gz - remote_file: ${project}/${version_id}/${revision}/${platform}/mongod-${version_id}.tar.gz - bucket: mciuploads - permissions: public-read - content_type: ${content_type|application/x-gzip} - display_name: mongodb.tar.gz + - func: "compile mongodb" - name: industry_benchmarks_WT - depends_on: - - name: compile - variant: linux-standalone priority: 5 commands: - func: "prepare environment" @@ -307,9 +348,6 @@ tasks: - name: industry_benchmarks_MMAPv1 priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -323,9 +361,6 @@ tasks: - name: industry_benchmarks_wmajority_WT priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -339,9 +374,6 @@ tasks: - name: industry_benchmarks_wmajority_MMAPv1 priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -355,9 +387,6 @@ tasks: - name: crud_workloads_WT priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -369,9 +398,6 @@ tasks: - name: mixed_workloads_WT priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -383,9 +409,6 @@ tasks: - name: misc_workloads_WT priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -397,9 +420,6 @@ tasks: - name: map_reduce_workloads_WT priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -411,9 +431,6 @@ tasks: - name: smoke_test priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -425,9 +442,6 @@ tasks: - name: crud_workloads_MMAPv1 priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -439,9 +453,6 @@ tasks: - name: mixed_workloads_MMAPv1 priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -453,9 +464,6 @@ tasks: - name: misc_workloads_MMAPv1 priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -467,9 +475,6 @@ tasks: - name: map_reduce_workloads_MMAPv1 priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -479,15 +484,8 @@ tasks: - func: "run test" - func: "analyze" - - func: "deploy cluster" - - func: "run test" - - func: "analyze" - - name: non_sharded_workloads_WT priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -499,9 +497,6 @@ tasks: - name: non_sharded_workloads_MMAPv1 priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -513,9 +508,6 @@ tasks: - name: mongos_workloads_WT priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -527,10 +519,6 @@ tasks: - name: mongos_workloads_MMAPv1 priority: 5 - depends_on: - - name: compile - variant: linux-standalone - exec_timeout_secs: 32400 # 9 hours commands: - func: "prepare environment" vars: @@ -542,9 +530,6 @@ tasks: - name: move_chunk_workloads_WT priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -556,9 +541,6 @@ tasks: - name: move_chunk_workloads_MMAPv1 priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -570,9 +552,6 @@ tasks: - name: secondary_performance_WT priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -586,9 +565,6 @@ tasks: - name: secondary_performance_MMAPv1 priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -602,9 +578,6 @@ tasks: - name: initialsync_WT priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: @@ -616,9 +589,6 @@ tasks: - name: initialsync_MMAPv1 priority: 5 - depends_on: - - name: compile - variant : linux-standalone commands: - func: "prepare environment" vars: @@ -630,9 +600,6 @@ tasks: - name: initialsync-logkeeper_WT priority: 5 - depends_on: - - name: compile - variant: linux-standalone exec_timeout_secs: 216000 # 2.5 days commands: - func: "prepare environment" @@ -661,10 +628,33 @@ modules: ####################################### -# Linux Buildvariants # +# Buildvariants # ####################################### - buildvariants: + +# We are explicitly tracking the rhel70 variant compile options from evergreen.yml. If we can get +# proper artifacts directly from that project, we should do that and remove these tasks. +- name: compile-rhel70 + display_name: Compile on rhel70 + batchtime: 1440 # 24 hours + modules: + - wtdevelop + expansions: + compile_flags: --ssl MONGO_DISTMOD=rhel70 -j$(grep -c ^processor /proc/cpuinfo) --release --variables-files=etc/scons/mongodbtoolchain_gcc.vars + gorootvars: GOROOT=/opt/go PATH="/opt/go/bin:$PATH" + platform: linux + project: &project dsi + tooltags: "" + use_scons_cache: true + run_on: + - "rhel70" + tasks: + - name: compile + + +####################################### +# Linux Buildvariants # +####################################### - name: linux-1-node-replSet display_name: Linux 1-Node ReplSet batchtime: 10080 # 7 days @@ -676,10 +666,12 @@ buildvariants: setup: single-replica cluster: single platform: linux - use_scons_cache: true - project: &project dsi-v3.4 + project: *project run_on: - "rhel70-perf-single" + depends_on: + - name: compile + variant: compile-rhel70 tasks: - name: industry_benchmarks_WT - name: crud_workloads_WT @@ -700,14 +692,15 @@ buildvariants: batchtime: 10080 # 7 days modules: *modules expansions: - compile_flags: -j$(grep -c ^processor /proc/cpuinfo) CC=/opt/mongodbtoolchain/v2/bin/gcc CXX=/opt/mongodbtoolchain/v2/bin/g++ OBJCOPY=/opt/mongodbtoolchain/v2/bin/objcopy setup: standalone cluster: single platform: linux - use_scons_cache: true project: *project run_on: - "rhel70-perf-single" + depends_on: + - name: compile + variant: compile-rhel70 tasks: - name: compile distros: @@ -739,6 +732,9 @@ buildvariants: project: *project run_on: - "rhel70-perf-shard" + depends_on: + - name: compile + variant: compile-rhel70 tasks: - name: industry_benchmarks_WT - name: crud_workloads_WT @@ -770,6 +766,9 @@ buildvariants: project: *project run_on: - "rhel70-perf-replset" + depends_on: + - name: compile + variant: compile-rhel70 tasks: - name: industry_benchmarks_WT - name: crud_workloads_WT @@ -802,6 +801,9 @@ buildvariants: project: *project run_on: - "rhel70-perf-replset" + depends_on: + - name: compile + variant: compile-rhel70 tasks: - name: initialsync_WT - name: initialsync_MMAPv1 @@ -819,5 +821,8 @@ buildvariants: project: *project run_on: - "rhel70-perf-initialsync-logkeeper" + depends_on: + - name: compile + variant: compile-rhel70 tasks: - name: initialsync-logkeeper_WT |