####################################### # Tools Driver Config for MCI # ####################################### # default command type command_type: system # run the same task in the previous revision if the current task fails stepback: true mongo_tools_variables: ## List of tests to run on each buildvariant mongo_tools_task_lists: mac_1012_task_list: &macos_1012_tasks - name: db - name: dist - name: integration - name: integration-auth - name: legacy30 # No SSL on 2.6 for osx # - name: legacy26 - name: qa-tests - name: qa-tests-3.2 - name: qa-tests-3.4 - name: qa-dump-restore-gzip - name: qa-dump-restore-gzip-3.2 - name: qa-dump-restore-archiving - name: qa-dump-restore-archiving-3.2 - name: unit # disabled until BUILD-2273 is done # - name: replay-dist # - name: replay-sanity_check # - name: replay-go_test # - name: replay-auth_test # - name: replay-sharded_test # - name: replay-repl_test # - name: replay-replay_test macos_1012_ssl_task_list: &macos_1012_ssl_tasks - name: dist - name: qa-tests - name: native-cert-ssl - name: unit ubuntu1404_task_list: &ubuntu1404_tasks - name: db - name: dist - name: integration - name: integration-auth - name: legacy30 - name: lint-go - name: lint-js - name: qa-tests - name: qa-tests-3.2 - name: qa-tests-3.4 - name: qa-dump-restore-gzip - name: qa-dump-restore-gzip-3.2 - name: qa-dump-restore-archiving - name: qa-dump-restore-archiving-3.2 - name: unit - name: vet - name: replay-dist - name: replay-sanity_check - name: replay-go_test - name: replay-auth_test - name: replay-sharded_test - name: replay-repl_test - name: replay-replay_test ubuntu1404_ssl_task_list: &ubuntu1404_ssl_tasks - name: dist - name: integration - name: integration-auth - name: legacy30 - name: qa-tests - name: native-cert-ssl ubuntu1404_enterprise_task_list: &ubuntu1404_enterprise_tasks - name: db - name: dist - name: integration - name: integration-auth - name: kerberos - name: legacy30 - name: legacy26 - name: qa-tests - name: native-cert-ssl - name: replay-dist - name: replay-sanity_check - name: replay-go_test - name: replay-auth_test - name: replay-sharded_test - name: replay-repl_test - name: replay-replay_test ubuntu1404_race_task_list: &ubuntu1404_race_tasks - name: db - name: dist - name: integration - name: integration-auth - name: legacy30 - name: legacy26 - name: qa-tests - name: unit - name: replay-dist - name: replay-sanity_check - name: replay-go_test - name: replay-auth_test - name: replay-sharded_test - name: replay-repl_test - name: replay-replay_test windows_64_task_list: &windows_64_tasks - name: db - name: dist - name: integration - name: integration-auth - name: legacy30 distros: - windows-64-vs2013-test - name: qa-tests distros: - windows-64-vs2013-test - name: qa-tests-3.2 distros: - windows-64-vs2013-test - name: qa-tests-3.4 distros: - windows-64-vs2013-test - name: qa-dump-restore-archiving distros: - windows-64-vs2013-test - name: qa-dump-restore-archiving-3.2 distros: - windows-64-vs2013-test - name: qa-dump-restore-gzip distros: - windows-64-vs2013-test - name: qa-dump-restore-gzip-3.2 distros: - windows-64-vs2013-test - name: unit windows_64_ssl_task_list: &windows_64_ssl_tasks - name: dist - name: integration - name: integration-auth - name: legacy30 - name: qa-tests - name: native-cert-ssl - name: unit windows_64_enterprise_task_list: &windows_64_enterprise_tasks - name: db - name: dist - name: integration - name: integration-auth - name: kerberos - name: legacy30 distros: - windows-64-vs2013-test - name: qa-tests distros: - windows-64-vs2013-test - name: unit - name: native-cert-ssl rhel71_enterprise_task_list: &rhel71_enterprise_tasks - name: db - name: dist - name: integration - name: integration-auth #- name: kerberos - name: qa-dump-restore-archiving - name: qa-dump-restore-gzip - name: qa-tests - name: native-cert-ssl # disabled until BUILD-2273 is done # - name: replay-dist # - name: replay-sanity_check # - name: replay-go_test # - name: replay-auth_test # - name: replay-sharded_test # - name: replay-repl_test # - name: replay-replay_test rhel72_enterprise_task_list: &rhel72_enterprise_tasks - name: db - name: dist - name: integration - name: integration-auth - name: kerberos - name: qa-dump-restore-archiving - name: qa-dump-restore-gzip - name: native-cert-ssl # disabled until BUILD-2273 is done # - name: replay-dist # - name: replay-sanity_check # - name: replay-go_test # - name: replay-auth_test # - name: replay-sharded_test # - name: replay-repl_test # - name: replay-replay_test ubuntu1604_ssl_task_list: &ubuntu1604_ssl_tasks - name: db - name: dist - name: integration - name: integration-auth - name: qa-dump-restore-archiving - name: qa-dump-restore-gzip - name: native-cert-ssl ## Common mongodb arguments mongod_arguments: default: &mongod_default_startup_args mongod_args: "" mongod_port: 33333 ssl: &mongod_ssl_startup_args mongod_args: "--sslMode requireSSL --sslCAFile common/db/openssl/testdata/ca-ia.pem --sslPEMKeyFile common/db/openssl/testdata/test-server.pem --bind_ip localhost" mongod_port: 33333 ssl: &mongod_ssl_startup_args_windows mongod_args: "--sslMode requireSSL --sslCAFile common/db/openssl/testdata/ia-ca.pem --sslPEMKeyFile common/db/openssl/testdata/test-server.pem --bind_ip localhost" mongod_port: 33333 # Set storage engine as mmapv1 for 32 bit variants because WiredTiger requires 64 bit support. win32: &mongod_win32_startup_args mongod_args: "--storageEngine=mmapv1" mongod_port: 33333 mongo_arguments: default: &mongo_default_startup_args mongo_args: &mongo_default_startup_args_string "--port 33333" mongod_port: 33333 ssl: &mongo_ssl_startup_args mongo_args: "--port 33333 --ssl --sslCAFile ./common/db/openssl/testdata/ca-ia.pem --sslPEMKeyFile ./common/db/openssl/testdata/test-client.pem --sslAllowInvalidCertificates" mongod_port: 33333 functions: "run legacy tests": command: shell.exec type: test params: working_dir: src/github.com/mongodb/mongo-tools script: | set -x set -v set -e chmod +x bin/* mv bin/* ${test_path}/ cd ${test_path} python buildscripts/smoke.py ${smoke_use_ssl} --nopreallocj --with-cleanbb --mongod ./mongod --mongo ./mongo --report-file ../../report.json --continue-on-failure --buildlogger-builder MCI_${build_variant} --buildlogger-buildnum ${builder_num|} --buildlogger-credentials ./mci.buildlogger --buildlogger-phase ${task_name}_${execution} ${smoke_args} tool "run qa-tests": command: shell.exec type: test params: working_dir: src/github.com/mongodb/mongo-tools script: | set -x set -v set -e chmod +x bin/* mv bin/* test/qa-tests/ cd test/qa-tests rm -rf /data/mci/install /data/mci/multiversion mkdir -p /data/mci/install /data/mci/multiversion if [ "${multiversion_override}" != "skip" ]; then python buildscripts/setup_multiversion_mongodb.py /data/mci/install /data/mci/multiversion ${arch} ${multiversion_override|2.6} --latest ${smoke_use_ssl} --os="${mongo_os}" fi chmod 400 jstests/libs/key* PATH=$PATH:/data/mci/multiversion python buildscripts/resmoke.py --suite=${resmoke_suite} --continueOnFailure --log=buildlogger --reportFile=../../report.json ${resmoke_args} --excludeWithAnyTags="${excludes}" "build tool": command: shell.exec type: test params: working_dir: src/github.com/mongodb/mongo-tools script: | set -x set -v set -e echo "Building ${tool}..." if [ "Windows_NT" = "$OS" ]; then set -o igncr fi; # In RHEL 5.5, /usr/bin/ld can't handle --build-id parameters, so # use a wrapper if it's present on the system # if [ -d /opt/ldwrapper/bin ] then export PATH=/opt/ldwrapper/bin:$PATH fi . ./set_goenv.sh GOROOT="" set_goenv || exit go version env | grep ^GO go build $(buildflags) -ldflags "$(print_ldflags)" ${args} -tags "$(print_tags failpoints ${build_tags})" -o bin/${tool} ${tool}/main/${tool}.go ./bin/${tool} --version "download mongod": command: shell.exec params: working_dir: src/github.com/mongodb/mongo-tools script: | set -x set -v set -e mongotarget=$(if [ "${mongo_target}" ]; then echo "${mongo_target}"; else echo "${mongo_os}"; fi) mongoversion=$(if [ "${mongo_version_always_use_latest}" ]; then echo "latest"; else echo "${mongo_version}"; fi) dlurl=$(python binaryurl.py --edition=${mongo_edition} --target=$mongotarget --version=$mongoversion --arch=${mongo_arch|x86_64}) filename=$(echo $dlurl | sed -e "s_.*/__") mkdir -p bin curl -s $dlurl --output $filename ${decompress} $filename rm $filename if [ "${only_shell}" ]; then mv -f ./mongodb-*/bin/mongo${extension} ./bin/ else mv -f ./mongodb-*/bin/mongo${extension} ./bin/ mv -f ./mongodb-*/bin/mongos${extension} ./bin/ mv -f ./mongodb-*/bin/mongod${extension} ./bin/ fi chmod +x ./bin/* rm -rf ./mongodb-* "fetch tool" : command: s3.get params: bucket: mciuploads aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: src/github.com/mongodb/mongo-tools/bin/${tool}${extension} remote_file: mongo-tools/binaries/${build_id}/${edition|community}/${tool}${extension} "generate coverage html + text": command: shell.exec type: test params: working_dir: src/github.com/mongodb/mongo-tools script: | set -x set -v set -e if [ "${coverage}" = "true" ]; then if [ "Windows_NT" = "$OS" ]; then set -o igncr fi; go tool cover -html=coverage.out -o coverage.html go tool cover -func=coverage.out -o coverage.txt fi; "get buildnumber": command: keyval.inc params: key: "${build_variant}_tools" destination: "builder_num" "move coverage data": command: shell.exec params: working_dir: src/github.com/mongodb/mongo-tools script: | set -x set -v set -e if [ "${coverage}" = "true" ]; then mv ${package}/coverage.out . fi "run unit test": command: shell.exec type: test params: working_dir: src/github.com/mongodb/mongo-tools script: | if [ "Windows_NT" = "$OS" ]; then set -o igncr fi; export MONGODB_KERBEROS_PASSWORD=${kerberos_password} set -x set -v export basedir=$PWD . ./set_goenv.sh GOROOT="" set_goenv || exit cd ${package} go test $(buildflags) -ldflags "$(print_ldflags)" ${coverage_args} ${args} -tags "$(print_tags ${build_tags})" -test.v > unit.suite export exitcode=$? cat unit.suite cp unit.suite $basedir/. exit $exitcode "setup integration test": command: shell.exec params: working_dir: src/github.com/mongodb/mongo-tools # Set up Kerberos stuff: run kinit if necessary, and add KDC to registry # on Windows (see https://wiki.mongodb.com/display/DH/Testing+Kerberos) script: | set -x set -v set -e if [ '${run_kinit}' = 'true' ] then echo "${kerberos_password}" | kinit -p drivers@LDAPTEST.10GEN.CC; fi; if [ "Windows_NT" = "$OS" ]; then cmd /c "REG ADD HKLM\SYSTEM\ControlSet001\Control\Lsa\Kerberos\Domains\LDAPTEST.10GEN.CC /v KdcNames /d ldaptest.10gen.cc /t REG_MULTI_SZ /f" fi; "run tool unit tests": command: shell.exec type: test params: working_dir: src/github.com/mongodb/mongo-tools script: | set -x set -v if [ "Windows_NT" = "$OS" ]; then set -o igncr fi; . ./set_goenv.sh GOROOT="" set_goenv || exit cwd=$(pwd) # run unit tests under common package for i in archive bsonutil db/tlsgo failpoint intents json log options progress text util; do cd $cwd/common/$i COMMON_SUBPKG=$(basename $i) COVERAGE_ARGS="" if [ "${run_coverage}" ]; then COVERAGE_ARGS="-coverprofile=coverage_$COMMON_SUBPKG.out" fi; go test -ldflags "$(print_ldflags)" $COVERAGE_ARGS ${args} -tags "$(print_tags failpoints ${build_tags})" -test.v > $COMMON_SUBPKG.suite if [ $? -ne 0 ]; then echo "suite failed, will exit with error on finish" export exitcode=1 fi cat $COMMON_SUBPKG.suite cp $COMMON_SUBPKG.suite $cwd done #TODO mongotop needs a test for i in mongoimport mongoexport mongostat mongorestore mongodump mongofiles; do cd $cwd/$i COVERAGE_ARGS="" if [ "${run_coverage}" ]; then COVERAGE_ARGS="-coverprofile=coverage_$i.out" fi; go test -ldflags "$(print_ldflags)" $COVERAGE_ARGS ${args} -tags "$(print_tags ${build_tags})" -test.v > $i.suite if [ $? -ne 0 ]; then echo "suite failed, will exit with error on finish" export exitcode=1 fi cat $i.suite cp $i.suite $cwd done exit $exitcode "run tool integration tests": command: shell.exec type: test params: working_dir: src/github.com/mongodb/mongo-tools script: | if [ "Windows_NT" = "$OS" ]; then set -o igncr export MONGODB_KERBEROS_PASSWORD=${kerberos_password} fi; set -x set -v . ./set_goenv.sh GOROOT="" set_goenv || exit if [ "${create_mongod_users_command}" != "" ]; then export AUTH_USERNAME=${auth_username} export AUTH_PASSWORD=${auth_password} echo "${create_mongod_users_command}" | ./bin/mongo${extension} ${mongo_args} admin fi; for i in mongoimport mongoexport mongostat mongorestore mongodump mongofiles; do cd $i COVERAGE_ARGS="" if [ "${run_coverage}" ]; then COVERAGE_ARGS="-coverprofile=coverage_$i.out" fi; go test -ldflags "$(print_ldflags)" $COVERAGE_ARGS -test.v ${args} -tags "$(print_tags ${build_tags})" > $i.suite if [ $? -ne 0 ]; then echo "suite failed, will exit with error on finish" export exitcode=1 fi cat $i.suite cp $i.suite ../. cd .. done exit $exitcode "create coverage reports": command: shell.exec type: test params: working_dir: src/github.com/mongodb/mongo-tools script: | set -x set -v set -e set -o verbose if [ "Windows_NT" = "$OS" ]; then set -o igncr fi; . ./set_goenv.sh GOROOT="" set_goenv || exit # XXX Need to fix up use of 'src' below somehow; for now commented out # because we're using an actual GOPATH instead of a hack and replaced with a copy for i in mongoimport mongoexport mongostat mongorestore mongodump mongofiles; do cd $i # perl -pe 's/.*src/github.com\/mongodb\/mongo-tools/' coverage_$i.out > coverage_$i_rewrite.out go tool cover -html=coverage_$i.out -o coverage_$i.html go tool cover -func=coverage_$i.out -o coverage_$i.txt cd .. done "upload html coverage": command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: src/github.com/mongodb/mongo-tools/${coverage_pkg}/coverage_${coverage_pkg}.html remote_file: mongo-tools/coverage/${coverage_pkg}/${task_id}.html bucket: mciuploads permissions: public-read content_type: text/html build_variants: ["ubuntu", "windows-64"] display_name: ${coverage_pkg}-html "upload text coverage": command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: src/github.com/mongodb/mongo-tools/${coverage_pkg}/coverage_${coverage_pkg}.txt remote_file: mongo-tools/coverage/${coverage_pkg}/${task_id}.txt bucket: mciuploads permissions: public-read content_type: text/plain build_variants: ["ubuntu", "windows-64"] display_name: ${coverage_pkg}-text "setup credentials" : command: shell.exec params: working_dir: src/github.com/mongodb/mongo-tools silent: true script: | set -x set -v set -e cat > mci.buildlogger < ${filename}.suite pre: - command: shell.exec params: silent: true script: | set -x set -v ${killall_mci|pkill -9 mongo; pkill -9 mongodump; pkill -9 mongoexport; pkill -9 mongoimport; pkill -9 mongofiles; pkill -9 mongorestore; pkill -9 mongostat; pkill -9 mongotop; pkill -9 mongod; pkill -9 mongos; pkill -f buildlogger.py; pkill -f smoke.py} >/dev/null 2>&1 rm -rf src /data/db/* exit 0 post: - command: attach.results params: file_location: src/github.com/mongodb/mongo-tools/report.json - command: shell.exec params: silent: true script: | set -x set -v ${killall_mci|pkill -9 mongo; pkill -9 mongodump; pkill -9 mongoexport; pkill -9 mongoimport; pkill -9 mongofiles; pkill -9 mongorestore; pkill -9 mongostat; pkill -9 mongotop; pkill -9 mongod; pkill -9 mongos; pkill -f buildlogger.py; pkill -f smoke.py} >/dev/null 2>&1 exit 0 - command: gotest.parse_files params: files: ["src/github.com/mongodb/mongo-tools/*.suite"] - command: shell.exec params: script: | set -x set -v rm -rf /data/db/* rm -rf /data/mci/multiversion rm -rf /data/mci/install exit 0 timeout: - command: shell.exec params: silent: true script: | set -x set -v # don't attempt to abort on any distro which has a special way of # killing everything (i.e. using taskkill on Windows) if [ "${killall_mci}" = "" ]; then all_tools="bsondump mongodump mongoexport mongofiles mongoimport mongorestore mongostat mongotop" # send SIGABRT to print a stacktrace for any hung tool pkill -ABRT "^($(echo -n $all_tools | tr ' ' '|'))\$" # git the processes a second or two to dump their stacks sleep 10 fi tasks: - name: db commands: - func: "fetch source" - command: expansions.update params: updates: - key: "package" value: "common/db" - key: "args" value: "${args} -test.types=db" - func: "download mongod" vars: mongo_version: "3.6" - func: "start mongod" - func: "wait for mongod to be ready" - func: "setup integration test" - func: "run unit test" - name: dist depends_on: commands: - func: "fetch source" # bsondump - func: "build tool" vars: tool: bsondump - func: "upload tool" vars: tool: bsondump # mongodump - func: "build tool" vars: tool: mongodump - func: "upload tool" vars: tool: mongodump # mongoexport - func: "build tool" vars: tool: mongoexport - func: "upload tool" vars: tool: mongoexport # mongofiles - func: "build tool" vars: tool: mongofiles - func: "upload tool" vars: tool: mongofiles # mongoimport - func: "build tool" vars: tool: mongoimport - func: "upload tool" vars: tool: mongoimport # mongorestore - func: "build tool" vars: tool: mongorestore - func: "upload tool" vars: tool: mongorestore # mongostat - func: "build tool" vars: tool: mongostat - func: "upload tool" vars: tool: mongostat # mongotop - func: "build tool" vars: tool: mongotop - func: "upload tool" vars: tool: mongotop - name: integration commands: - func: "fetch source" - command: expansions.update params: updates: - key: "args" value: "${args} -test.types=${integration_test_args}" - func: "download mongod" vars: mongo_version: "3.6" - func: "start mongod" - func: "wait for mongod to be ready" - func: "run tool integration tests" - name: integration-auth commands: - func: "fetch source" # Concat auth args - command: expansions.update params: updates: - key: "args" value: "${args} -test.types=${integration_test_args},auth" - key: "mongod_args" concat: " --auth" - key: "auth_username" value: "passwordIsTaco" - key: "auth_password" value: "Taco" - key: "create_mongod_users_command" value: "db.createUser({ user: '${auth_username}', pwd: '${auth_password}', roles: [{ role: '__system', db: 'admin' }] });" - func: "download mongod" vars: mongo_version: "3.6" - func: "start mongod" - func: "wait for mongod to be ready" - func: "run tool integration tests" - name: kerberos commands: - func: "fetch source" # Explicitly run ONLY Kerberos tests - command: expansions.update params: updates: - key: "args" value: "${args} -test.types=kerberos" - func: "setup integration test" - func: "run tool integration tests" - name: legacy30 depends_on: - name: dist commands: - func: "fetch source" - func: "get buildnumber" - func: "setup credentials" - func: "download mongod" vars: mongo_version: "3.0" - func: "fetch tool" vars: tool: mongoimport - func: "fetch tool" vars: tool: mongoexport - func: "fetch tool" vars: tool: mongodump - func: "fetch tool" vars: tool: mongostat - func: "fetch tool" vars: tool: mongorestore - func: "fetch tool" vars: tool: mongofiles - func: "run legacy tests" vars: test_path: "test/legacy30" smoke_args: "--authMechanism SCRAM-SHA-1" - name: legacy26 depends_on: - name: dist commands: - func: "fetch source" - func: "get buildnumber" - func: "setup credentials" - func: "download mongod" vars: mongo_version: "2.6" - func: "fetch tool" vars: tool: mongoimport - func: "fetch tool" vars: tool: mongoexport - func: "fetch tool" vars: tool: mongodump - func: "fetch tool" vars: tool: mongostat - func: "fetch tool" vars: tool: mongorestore - func: "fetch tool" vars: tool: mongofiles - func: "run legacy tests" vars: test_path: "test/legacy26" smoke_use_ssl: "" - name: lint-go commands: - func: "fetch source" - command: shell.exec type: test params: working_dir: src/github.com/mongodb/mongo-tools script: | set -x set -v set -e . ./set_goenv.sh GOROOT="" set_goenv || exit retVal=$(go run vendor/github.com/3rf/mongo-lint/golint/golint.go mongo* bson* common/*); if [ "$retVal" = "" ]; then exit 0; else echo $retVal; exit 1; fi; - name: lint-js commands: - func: "fetch source" - command: shell.exec type: test params: working_dir: src/github.com/mongodb/mongo-tools script: | set -x set -v set -e PATH="/opt/node/bin:$PATH" /opt/node/bin/npm install eslint@3.2 /opt/node/bin/node node_modules/eslint/bin/eslint.js test/qa-tests/jstests/**/*.js - name: qa-tests-unstable depends_on: - name: dist commands: - func: "fetch source" - func: "get buildnumber" - func: "setup credentials" - func: "download mongod" vars: mongo_version: "latest" - func: "fetch tool" vars: tool: mongoimport - func: "fetch tool" vars: tool: mongoexport - func: "fetch tool" vars: tool: mongodump - func: "fetch tool" vars: tool: mongorestore - func: "fetch tool" vars: tool: mongostat - func: "fetch tool" vars: tool: mongotop - func: "fetch tool" vars: tool: mongofiles - func: "fetch tool" vars: tool: bsondump - func: "run qa-tests" vars: resmoke_suite: "core${resmoke_use_ssl}" - name: qa-tests depends_on: - name: dist commands: - func: "fetch source" - func: "get buildnumber" - func: "setup credentials" - func: "download mongod" vars: mongo_version: "3.6" - func: "fetch tool" vars: tool: mongoimport - func: "fetch tool" vars: tool: mongoexport - func: "fetch tool" vars: tool: mongodump - func: "fetch tool" vars: tool: mongorestore - func: "fetch tool" vars: tool: mongostat - func: "fetch tool" vars: tool: mongotop - func: "fetch tool" vars: tool: mongofiles - func: "fetch tool" vars: tool: bsondump - func: "run qa-tests" vars: resmoke_suite: "core${resmoke_use_ssl}" excludes: "requires_unstable,${excludes}" - name: qa-tests-3.4 depends_on: - name: dist commands: - func: "fetch source" - func: "get buildnumber" - func: "setup credentials" - func: "download mongod" vars: mongo_version: "3.4" - func: "fetch tool" vars: tool: mongoimport - func: "fetch tool" vars: tool: mongoexport - func: "fetch tool" vars: tool: mongodump - func: "fetch tool" vars: tool: mongorestore - func: "fetch tool" vars: tool: mongostat - func: "fetch tool" vars: tool: mongotop - func: "fetch tool" vars: tool: mongofiles - func: "fetch tool" vars: tool: bsondump - func: "run qa-tests" vars: resmoke_suite: "core${resmoke_use_ssl}" excludes: "requires_unstable,requires_mongo_36,${excludes}" - name: qa-tests-3.2 depends_on: - name: dist commands: - func: "fetch source" - func: "get buildnumber" - func: "setup credentials" - func: "download mongod" vars: mongo_version: "3.2" - func: "fetch tool" vars: tool: mongoimport - func: "fetch tool" vars: tool: mongoexport - func: "fetch tool" vars: tool: mongodump - func: "fetch tool" vars: tool: mongorestore - func: "fetch tool" vars: tool: mongostat - func: "fetch tool" vars: tool: mongotop - func: "fetch tool" vars: tool: mongofiles - func: "fetch tool" vars: tool: bsondump - func: "run qa-tests" vars: resmoke_suite: "core${resmoke_use_ssl}" excludes: "requires_unstable,requires_mongo_34,requires_mongo_36,${excludes}" - name: native-cert-ssl depends_on: - name: dist commands: - func: "fetch source" - func: "get buildnumber" - func: "setup credentials" - func: "download mongod" vars: mongo_version: "3.6" - func: "fetch tool" vars: tool: mongoimport - func: "fetch tool" vars: tool: mongoexport - func: "fetch tool" vars: tool: mongodump - func: "fetch tool" vars: tool: mongorestore - func: "fetch tool" vars: tool: mongostat - func: "fetch tool" vars: tool: mongotop - func: "fetch tool" vars: tool: mongofiles - func: "fetch tool" vars: tool: bsondump - command: shell.exec type: test params: working_dir: src/github.com/mongodb/mongo-tools/test/qa-tests script: | set -x set -v set -e chmod +x ../../bin/* mv ../../bin/* . rm -rf /data/mci/install /data/mci/multiversion mkdir -p /data/mci/install /data/mci/multiversion python buildscripts/setup_multiversion_mongodb.py /data/mci/install /data/mci/multiversion ${arch} "2.6" --latest ${smoke_use_ssl} --os="${mongo_os}" || true chmod 400 jstests/libs/key* PATH=$PATH:/data/mci/multiversion python buildscripts/resmoke.py --suite=native_cert_ssl --continueOnFailure --log=buildlogger --reportFile=../../report.json ${resmoke_args} --excludeWithAnyTags="${excludes}" - name: qa-dump-restore-archiving depends_on: - name: dist commands: - func: "fetch source" - func: "get buildnumber" - func: "setup credentials" - func: "download mongod" vars: mongo_version: "3.6" - func: "fetch tool" vars: tool: mongodump - func: "fetch tool" vars: tool: mongorestore - func: "run qa-tests" vars: resmoke_suite: "restore_archive" excludes: "requires_unstable,${excludes}" - name: qa-dump-restore-archiving-3.2 depends_on: - name: dist commands: - func: "fetch source" - func: "get buildnumber" - func: "setup credentials" - func: "download mongod" vars: mongo_version: "3.2" - func: "fetch tool" vars: tool: mongodump - func: "fetch tool" vars: tool: mongorestore - func: "run qa-tests" vars: resmoke_suite: "restore_archive" excludes: "requires_unstable,requires_mongo_34,requires_mongo_36,${excludes}" - name: qa-dump-restore-gzip depends_on: - name: dist commands: - func: "fetch source" - func: "get buildnumber" - func: "setup credentials" - func: "download mongod" vars: mongo_version: "3.6" - func: "fetch tool" vars: tool: mongodump - func: "fetch tool" vars: tool: mongorestore - func: "run qa-tests" vars: resmoke_suite: "restore_gzip" excludes: "requires_unstable,${excludes}" - name: qa-dump-restore-gzip-3.2 depends_on: - name: dist commands: - func: "fetch source" - func: "get buildnumber" - func: "setup credentials" - func: "download mongod" vars: mongo_version: "3.2" - func: "fetch tool" vars: tool: mongodump - func: "fetch tool" vars: tool: mongorestore - func: "run qa-tests" vars: resmoke_suite: "restore_gzip" excludes: "requires_unstable,requires_mongo_34,requires_mongo_36,${excludes}" - name: unit commands: - command: expansions.update params: updates: - key: "run_coverage" value: "true" - func: "fetch source" - func: "run tool unit tests" - func: "create coverage reports" - command: expansions.update params: updates: - key: "coverage_pkg" value: "mongoimport" - func: "upload html coverage" - func: "upload text coverage" - command: expansions.update params: updates: - key: "coverage_pkg" value: "mongoexport" - func: "upload html coverage" - func: "upload text coverage" - command: expansions.update params: updates: - key: "coverage_pkg" value: "mongostat" - func: "upload html coverage" - func: "upload text coverage" - command: expansions.update params: updates: - key: "coverage_pkg" value: "mongodump" - func: "upload html coverage" - func: "upload text coverage" - command: expansions.update params: updates: - key: "coverage_pkg" value: "mongorestore" - func: "upload html coverage" - func: "upload text coverage" - name: vet commands: - func: "fetch source" - command: shell.exec type: test params: working_dir: src/github.com/mongodb/mongo-tools script: | set -x set -v set -e go tool vet bsondump common mongo* - name: replay-dist commands: - func: "fetch source" - func: "build tool" vars: tool: mongoreplay - func: "upload tool" vars: tool: mongoreplay - func: "build tool" - name: replay-sanity_check depends_on: - name: replay-dist commands: - func: "fetch source" - func: "fetch tool" vars: tool: mongoreplay - func: "download mongod" vars: mongo_version: "3.6" - func: "start mongod" - func: "wait for mongod to be ready" - command: shell.exec params: working_dir: src/github.com/mongodb/mongo-tools script: | set -x set -v set -e chmod +x bin/mongoreplay echo "Running sanity check" PATH=$PATH:$PWD/bin ./mongoreplay/sanity_check.sh -p ${mongod_port} - name: replay-go_test depends_on: - name: replay-dist commands: - func: "fetch source" - func: "fetch tool" vars: tool: mongoreplay - func: "fetch pcap" vars: pcapFname: getmore_multi_channel.pcap - func: "fetch pcap" vars: pcapFname: getmore_single_channel.pcap - func: "download mongod" vars: mongo_version: "3.6" - func: "start mongod" - func: "wait for mongod to be ready" - func: "run go_test" vars: filename: playtest environment_vars: DB_PORT=${mongod_port} additional_args: github.com/mongodb/mongo-tools/mongoreplay - name: replay-sharded_test depends_on: - name: replay-dist commands: - func: "fetch source" - func: "fetch tool" vars: tool: mongoreplay - func: "fetch pcap" vars: pcapFname: getmore_multi_channel.pcap - func: "fetch pcap" vars: pcapFname: getmore_single_channel.pcap - func: "download mongod" vars: mongo_version: "3.6" - func: "create sharded_cluster" - func: "run go_test" vars: filename: sharded environment_vars: DB_PORT=${mongod_port} additional_args: github.com/mongodb/mongo-tools/mongoreplay --run "LiveDB" - name: replay-auth_test depends_on: - name: replay-dist commands: - func: "fetch source" - func: "fetch tool" vars: tool: mongoreplay - func: "fetch pcap" vars: pcapFname: getmore_multi_channel.pcap - func: "fetch pcap" vars: pcapFname: getmore_single_channel.pcap - func: "download mongod" vars: mongo_version: "3.6" - func: "start mongod" vars: additional_args: --auth - func: "wait for mongod to be ready" - func: "create auth_user" vars: mongod_port: ${mongod_port} - func: "run go_test" vars: filename: authtest environment_vars: AUTH=1 DB_PORT=${mongod_port} additional_args: github.com/mongodb/mongo-tools/mongoreplay --run "(LiveDB)|(Authed)" - name: replay-repl_test depends_on: - name: replay-dist commands: - func: "fetch source" - func: "fetch tool" vars: tool: mongoreplay - func: "fetch pcap" vars: pcapFname: getmore_multi_channel.pcap - func: "fetch pcap" vars: pcapFname: getmore_single_channel.pcap - func: "download mongod" vars: mongo_version: "3.6" - func: "create repl_set" vars: mongod_port: ${mongod_port} - func: "run go_test" vars: filename: repltest environment_vars: DB_PORT=${mongod_port} additional_args: github.com/mongodb/mongo-tools/mongoreplay --run "LiveDB" - name: replay-replay_test depends_on: - name: replay-dist commands: - func: "fetch source" - func: "fetch tool" vars: tool: mongoreplay - func: "download mongod" vars: mongo_version: "3.6" - func: "fetch ftdc" - command: shell.exec params: working_dir: src/github.com/mongodb/mongo-tools script: | set -x set -v set -e chmod +x bin/mongoreplay echo "Running replay test" PATH=$PATH:$PWD/bin:$PWD/vendor/bin ./mongoreplay/replay_test.sh --verbose --explicit --keep - command: shell.exec params: working_dir: src/github.com/mongodb/mongo-tools script: | set -x set -v set -e tar czf replay.tar.gz tmp.* - func: "upload archive" vars: filename: replay.tar.gz - func: "create timeseries" - func: "upload timeseries" buildvariants: ####################################### # Amazon Buildvariants # ####################################### - name: amazonlinux64 display_name: Amazon Linux 64 run_on: - linux-64-amzn-test expansions: build_tags: "" tasks: - name: dist - name: replay-dist - name: amazonlinux64-enterprise display_name: Amazon Linux 64 Enterprise run_on: - linux-64-amzn-test expansions: build_tags: "sasl ssl" tasks: - name: dist - name: amazon2 display_name: Amazon Linux 64 v2 run_on: - amazon2-test expansions: build_tags: "" tasks: - name: dist - name: replay-dist - name: amazon2-enterprise display_name: Amazon Linux 64 v2 Enterprise run_on: - amazon2-test expansions: build_tags: "sasl ssl" tasks: - name: dist ####################################### # Debian Buildvariants # ####################################### - name: debian71-enterprise display_name: Debian 7.1 Enterprise run_on: - debian71-test expansions: gorootvars: 'PATH="/opt/go1.11/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.11/go' build_tags: "sasl ssl" tasks: - name: dist - name: debian81 display_name: Debian 8.1 run_on: - debian81-test expansions: build_tags: "" tasks: - name: dist - name: replay-dist - name: debian81-enterprise display_name: Debian 8.1 Enterprise run_on: - debian81-test expansions: build_tags: "sasl ssl" tasks: - name: dist - name: debian92 display_name: Debian 9.2 run_on: - debian92-test expansions: build_tags: "" tasks: - name: dist - name: replay-dist - name: debian92-enterprise display_name: Debian 9.2 Enterprise run_on: - debian92-test expansions: build_tags: "sasl ssl" tasks: - name: dist ####################################### # macOS Buildvariant # ####################################### - name: macOS-1012 display_name: MacOS 10.12 run_on: - macos-1012 expansions: <<: *mongod_default_startup_args <<: *mongo_default_startup_args mongo_os: "osx" mongo_target: "osx-ssl" arch: "osx/x86_64" excludes: requires_many_files tasks: *macos_1012_tasks - name: macOS-1012-ssl display_name: MacOS 10.12 SSL run_on: - macos-1012 expansions: <<: *mongod_ssl_startup_args <<: *mongo_ssl_startup_args mongo_os: "osx" mongo_target: "osx-ssl" arch: "osx/x86_64" build_tags: "ssl" excludes: requires_many_files tasks: *macos_1012_ssl_tasks ####################################### # RHEL Buildvariants # ####################################### - name: rhel62-enterprise display_name: RHEL 6.2 Enterprise run_on: - rhel62-test expansions: build_tags: "sasl ssl" tasks: - name: dist - name: rhel70 display_name: RHEL 7.0 run_on: - rhel70 expansions: build_tags: "" tasks: - name: dist - name: replay-dist - name: rhel70-enterprise display_name: RHEL 7.0 Enterprise run_on: - rhel70 expansions: build_tags: "sasl ssl" tasks: - name: dist ####################################### # SUSE Buildvariants # ####################################### - name: suse12 display_name: SUSE 12 run_on: - suse12-test expansions: build_tags: "" tasks: - name: dist - name: replay-dist - name: suse12-enterprise display_name: SUSE 12 Enterprise run_on: - suse12-test expansions: build_tags: "sasl ssl" tasks: - name: dist ####################################### # Ubuntu Buildvariants # ####################################### - name: ubuntu1404 display_name: Ubuntu 14.04 run_on: - ubuntu1404-test expansions: <<: *mongod_default_startup_args <<: *mongo_default_startup_args mongo_os: "ubuntu1404" mongo_edition: "targeted" build_tags: "" arch: "linux/x86_64" integration_test_args: integration resmoke_args: --jobs $(grep -c ^processor /proc/cpuinfo) tasks: *ubuntu1404_tasks - name: ubuntu1404-ssl display_name: Ubuntu 14.04 SSL run_on: - ubuntu1404-test expansions: <<: *mongod_ssl_startup_args <<: *mongo_ssl_startup_args mongo_os: "ubuntu1404" mongo_edition: "enterprise" build_tags: "ssl" edition: ssl arch: "linux/x86_64" smoke_use_ssl: --use-ssl resmoke_use_ssl: _ssl resmoke_args: --jobs $(grep -c ^processor /proc/cpuinfo) integration_test_args: "integration,ssl" tasks: *ubuntu1404_ssl_tasks - name: ubuntu1404-enterprise display_name: Ubuntu 14.04 Enterprise run_on: - ubuntu1404-test expansions: <<: *mongod_default_startup_args <<: *mongo_default_startup_args mongo_os: "ubuntu1404" mongo_edition: "enterprise" build_tags: "ssl sasl" smoke_use_ssl: --use-ssl resmoke_use_ssl: _ssl arch: "linux/x86_64" edition: enterprise run_kinit: true integration_test_args: integration resmoke_args: --jobs $(grep -c ^processor /proc/cpuinfo) tasks: *ubuntu1404_enterprise_tasks - name: ubuntu1604-enterprise display_name: Ubuntu 16.04 Enterprise run_on: - ubuntu1604-test expansions: build_tags: "sasl ssl" tasks: - name: dist ####################################### # Windows Buildvariants # ####################################### - name: windows-64 display_name: Windows 64-bit run_on: - windows-64-vs2013-test expansions: <<: *mongod_default_startup_args <<: *mongo_default_startup_args mongo_os: "windows-64" mongo_target: "windows_x86_64-2008plus-ssl" resmoke_args: --jobs $(grep -c ^processor /proc/cpuinfo) excludes: requires_large_ram extension: .exe arch: "win32/x86_64" preproc_gpm: "perl -pi -e 's/\\r\\n/\\n/g' " integration_test_args: "integration" tasks: *windows_64_tasks - name: windows-64-ssl display_name: Windows 64-bit SSL run_on: - windows-64-vs2013-compile expansions: <<: *mongod_ssl_startup_args_windows <<: *mongo_ssl_startup_args mongo_os: "windows-64" mongo_target: "windows_x86_64-2008plus-ssl" build_tags: "ssl" edition: ssl smoke_use_ssl: --use-ssl resmoke_use_ssl: _ssl resmoke_args: --jobs $(grep -c ^processor /proc/cpuinfo) excludes: requires_large_ram,requires_mongo_24 multiversion_override: "2.6" extension: .exe arch: "win32/x86_64" preproc_gpm: "perl -pi -e 's/\\r\\n/\\n/g' " integration_test_args: "integration,ssl" tasks: *windows_64_ssl_tasks - name: windows-64-enterprise display_name: Windows 64-bit Enterprise run_on: - windows-64-vs2013-compile expansions: <<: *mongod_default_startup_args <<: *mongo_default_startup_args mongo_os: "windows-64" mongo_edition: "enterprise" mongo_target: "windows" build_tags: "sasl ssl" smoke_use_ssl: --use-ssl resmoke_args: --jobs $(grep -c ^processor /proc/cpuinfo) resmoke_use_ssl: _ssl excludes: requires_large_ram,requires_mongo_24 multiversion_override: "2.6" edition: enterprise extension: .exe arch: "win32/x86_64" preproc_gpm: "perl -pi -e 's/\\r\\n/\\n/g' " integration_test_args: "integration" tasks: *windows_64_enterprise_tasks ####################################### # ARM Buildvariants # ####################################### - name: ubuntu1604-arm64-ssl display_name: ZAP ARM64 Ubuntu 16.04 SSL run_on: - ubuntu1604-arm64-small stepback: false batchtime: 10080 # weekly expansions: <<: *mongod_default_startup_args <<: *mongo_default_startup_args mongo_os: "ubuntu1604" mongo_edition: "targeted" mongo_arch: "arm64" build_tags: "ssl" resmoke_use_ssl: _ssl excludes: requires_mmap_available,requires_large_ram,requires_mongo_24,requires_mongo_26,requires_mongo_30 resmoke_args: -j 2 multiversion_override: "skip" arch: "linux/arm64" edition: ssl integration_test_args: integration tasks: *ubuntu1604_ssl_tasks ####################################### # Power Buildvariants # ####################################### - name: rhel71-ppc64le-enterprise display_name: ZAP PPC64LE RHEL 7.1 Enterprise run_on: - rhel71-power8-test stepback: false batchtime: 10080 # weekly expansions: <<: *mongod_default_startup_args <<: *mongo_default_startup_args mongo_os: "rhel71" mongo_edition: "enterprise" mongo_arch: "ppc64le" build_tags: 'ssl sasl' resmoke_use_ssl: _ssl resmoke_args: -j 4 excludes: requires_mmap_available,requires_large_ram,requires_mongo_24,requires_mongo_26,requires_mongo_30 multiversion_override: "skip" arch: "linux/ppc64le" edition: enterprise run_kinit: true integration_test_args: integration tasks: *rhel71_enterprise_tasks - name: ubuntu1604-ppc64le-enterprise display_name: ZAP PPC64LE Ubuntu 16.04 Enterprise run_on: - ubuntu1604-power8-test stepback: false batchtime: 10080 # weekly expansions: build_tags: 'ssl sasl' tasks: - name: dist ####################################### # Z (s390x) Buildvariants # ####################################### - name: rhel67-s390x-enterprise display_name: ZAP s390x RHEL 6.7 Enterprise run_on: - rhel67-zseries-test stepback: false batchtime: 10080 # weekly expansions: gorootvars: 'PATH="/opt/go1.11/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.11/go CC=/opt/mongodbtoolchain/v2/bin/s390x-mongodb-linux-gcc' build_tags: "sasl ssl" tasks: - name: dist - name: rhel72-s390x-enterprise display_name: ZAP s390x RHEL 7.2 Enterprise run_on: - rhel72-zseries-test stepback: false batchtime: 10080 # weekly expansions: <<: *mongod_default_startup_args <<: *mongo_default_startup_args mongo_os: "rhel72" mongo_edition: "enterprise" mongo_arch: "s390x" build_tags: "sasl ssl" resmoke_use_ssl: _ssl excludes: requires_mmap_available,requires_mongo_24,requires_mongo_26,requires_mongo_30 resmoke_args: -j 2 multiversion_override: "skip" arch: "linux/s390x" edition: enterprise run_kinit: true integration_test_args: integration tasks: *rhel72_enterprise_tasks - name: suse12-s390x-enterprise display_name: ZAP s390x SUSE 12 Enterprise run_on: - suse12-zseries-test stepback: false batchtime: 10080 # weekly expansions: gorootvars: 'PATH="/opt/go1.11/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.11/go CC=/opt/mongodbtoolchain/v2/bin/s390x-mongodb-linux-gcc' build_tags: "sasl ssl" tasks: - name: dist - name: ubuntu1604-s390x-enterprise display_name: ZAP s390x Ubuntu 16.04 Enterprise run_on: - ubuntu1604-zseries-small stepback: false batchtime: 10080 # weekly expansions: build_tags: "sasl ssl" tasks: - name: dist ####################################### # Experimental Buildvariants # ####################################### - name: ubuntu-race stepback: false batchtime: 1440 # daily display_name: z Race Detector Ubuntu 14.04 run_on: - ubuntu1404-test expansions: <<: *mongod_default_startup_args <<: *mongo_default_startup_args mongo_os: "ubuntu1404" mongo_edition: "enterprise" build_tags: "sasl ssl" arch: "linux/x86_64" args: "-buildmode=default -race" excludes: requires_large_ram integration_test_args: integration tasks: *ubuntu1404_race_tasks