##################################################### # A note on expansions # ##################################################### # Expansions usually appear in the form ${key|default} # If 'key' is found in the executor's map of currently known # expansions, the corresponding value is used. If the key can # not be found, the default is used. # # Arbitrary expansions can be specified in the YAML configuration # files in the following places: # - The 'expansions' field for buildvariants (branch file) # - The 'expansions' field for distros (distros file) # # A number of 'built-in' expansions are also available for use; these include: # - environment variables available on the host machine # - 'workdir' (references the executor's work directory). # - 'task_id' (references the task id of the task the executor is working on). # - 'build_variant' (references the executing task's buildvariant). # - 'config_root' (references the root directory for the executor's configuration artifacts). stepback: true command_type: system # Files that match an ignore-list pattern will not trigger a build, if they're the only modified # files in the patch. ignore: - ".*" - "!.clang-format" - "!.eslintrc.yml" - "*.md" - "*.rst" - "*.txt" - "/distsrc/**" - "/docs/**" - "/etc/*.yml" - "!/etc/evergreen.yml" - "README" ## Some variables for convenience: variables: # Used when the tests it runs depend only on mongod, mongos, the mongo shell and the tools. - &task_template name: template depends_on: - name: compile commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --help run_multiple_jobs: false max_jobs: 0 # If set in combination with run_multiple_jobs, will cap the number of jobs used. # Used for tests that invoke resmoke.py and require no additional setup. - &task_depending_on_all_template <<: *task_template depends_on: - name: compile_all # Used to list modules to retrieve from GitHub and apply patches if necessary. - &git_get_project command: git.get_project params: directory: src revisions: # for each module include revision as : ${_rev} enterprise: ${enterprise_rev} rocksdb: ${rocksdb_rev} - &generate_compile_expansions command: shell.exec params: working_dir: src script: | set -o errexit set -o verbose # We get the raw version string (r1.2.3-45-gabcdef) from git MONGO_VERSION=$(git describe) # If this is a patch build, we add the patch version id to the version string so we know # this build was a patch, and which evergreen task it came from if [ "${is_patch|}" = "true" ]; then MONGO_VERSION="$MONGO_VERSION-patch-${version_id}" fi # This script converts the generated version string into a sanitized version string for # use by scons and uploading artifacts as well as information about for the scons cache. MONGO_VERSION=$MONGO_VERSION SCONS_CACHE_MODE=${scons_cache_mode|nolinked} USE_SCONS_CACHE=${use_scons_cache|false} ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/generate_compile_expansions.py --out compile_expansions.yml - &apply_compile_expansions command: expansions.update params: file: src/compile_expansions.yml - &jepsen_config_vars jepsen_key_time_limit: --key-time-limit 15 jepsen_protocol_version: --protocol-version 1 jepsen_read_concern: "" jepsen_read_with_find_and_modify: "" jepsen_storage_engine: "" jepsen_test_name: "" # Empirically, we've had greater success in reproducing the issues found in MongoDB versions # 3.4.0-rc3 and 3.4.0-rc4 when running Jepsen with at least --time-limit=600. jepsen_time_limit: --time-limit 1200 jepsen_write_concern: "" # Template for running Jepsen tests - &run_jepsen_template name: run_jepsen_template depends_on: - name: compile commands: - func: "do setup" - func: "do jepsen setup" - func: "run jepsen test" vars: <<: *jepsen_config_vars - &jstestfuzz_config_vars resmoke_args: --help # resmoke_args needs to be overridden to specify one of the jstestfuzz suites should_shuffle: false continue_on_failure: false - &run_jstestfuzz_tests func: "run tests" # Terminate the function when there has been no output to stdout for 30 minutes. E.g. when something is stuck in an infinite loop. # resmoke.py writes the test output to logkeeper and only writes to stdout when starting the next test. # resmoke.py not producing output on stdout means that the test is still running and presumably not going to finish. # Note that timeout_secs is different from exec_timeout_secs, which applies to a task and times out regardless of whether output has been written to stdout. timeout_secs: 1800 vars: <<: *jstestfuzz_config_vars # Used for tests that invoke 'resmoke.py --suites=jstestfuzz*'. - &jstestfuzz_template name: jstestfuzz_template exec_timeout_secs: 14400 # Time out the task if it runs for more than 4 hours. depends_on: - name: compile commands: - func: "do setup" - func: "run jstestfuzz" - *run_jstestfuzz_tests # Templates used by powercycle - &powercycle_remote_credentials private_key_file: $(${posix_workdir})/powercycle.pem private_key_remote: ${kitchen_private_key} aws_key_remote: ${kitchen_aws_key} aws_secret_remote: ${kitchen_aws_secret} # TODO: Remove psutil from virtualenv_requirements when BUILD-3083 is done - &powercycle_ec2_instance aws_ec2_yml: aws_ec2.yml remote_dir: powercycle ssh_identity: -i ${private_key_file} ssh_key_id: ${kitchen_ssh_key_id} virtualenv_dir: venv_powercycle virtualenv_requirements: psutil - &powercycle_expansions params: updates: - key: backup_path_after value: /data/afterrecovery - key: backup_path_before value: /data/beforerecovery - key: db_path value: /data/db - key: log_path value: ${remote_dir}/mongod.log - &powercycle_test ec2_artifacts: ${log_path} ${backup_path_after} ${backup_path_before} program_options: --logLevel debug --backupPathBefore ${backup_path_before} --backupPathAfter ${backup_path_after} connection_options: --sshUserHost ${ip_address} --sshConnection \"${ssh_identity} ${ssh_connection_options}\" test_options: --testLoops 25 --seedDocNum 10000 --rsync --validate remote --canary remote crash_options: --crashMethod internal --crashWaitTime 30 --jitterForCrashWaitTime 5 mongodb_options: --rootDir ${remote_dir}-${task_id} --mongodbBinDir ${remote_dir} mongod_options: --dbPath ${db_path} --logPath ${log_path} mongod_extra_options: --mongodOptions \"--setParameter enableTestCommands=1\" ####################################### # Functions # ####################################### functions: "fetch artifacts" : &fetch_artifacts command: s3.get params: aws_key: ${aws_key} aws_secret: ${aws_secret} remote_file: mongodb-mongo-master/${build_variant}/${revision}/artifacts/${build_id}.tgz bucket: mciuploads extract_to: src "fetch binaries" : &fetch_binaries command: s3.get params: aws_key: ${aws_key} aws_secret: ${aws_secret} remote_file: mongodb-mongo-master/${build_variant}/${revision}/binaries/mongo-${build_id}.${ext|tgz} bucket: mciuploads local_file: src/mongo-binaries.tgz "extract binaries" : &extract_binaries command: shell.exec params: working_dir: src script: | set -o errexit ${decompress} mongo-binaries.tgz cp mongodb*/bin/* . "check binary version" : &check_binary_version command: shell.exec params: working_dir: src script: | set -o errexit mongo_binary=$(find mongodb*/bin -name mongo${exe}) # There should only be one mongo shell if [ $(echo $mongo_binary | wc -w) -ne 1 ]; then echo "There is more than 1 extracted mongo binary: $mongo_binary" exit 1 fi bin_ver=$(${python|/opt/mongodbtoolchain/v2/bin/python2} -c "import yaml; print(yaml.safe_load(open('compile_expansions.yml'))['version']);" | tr -d '[ \r\n]') # Due to SERVER-23810, we cannot use $mongo_binary --quiet --nodb --eval "version();" mongo_ver=$($mongo_binary --version | perl -pe '/version v(.*)$/; $_ = $1;' | tr -d '[ \r\n]') # The versions must match if [ "$bin_ver" != "$mongo_ver" ]; then echo "The mongo version is $mongo_ver, expected version is $bin_ver" exit 1 fi "get buildnumber" : &get_buildnumber command: keyval.inc params: key: "${build_variant}_master" destination: "builder_num" "run diskstats": &run_diskstats command: shell.exec params: background: true system_log: true script: | set -o errexit set -o verbose # On Windows we can use typeperf.exe to dump performance counters. if [ "Windows_NT" = "$OS" ]; then typeperf -qx PhysicalDisk | grep Disk | grep -v _Total > disk_counters.txt typeperf -cf disk_counters.txt -si 5 -o mongo-diskstats # Linux: iostat -t option for timestamp. elif iostat -tdmx > /dev/null 2>&1; then iostat -tdmx 5 > mongo-diskstats # OSX: Simulate the iostat timestamp. elif iostat -d > /dev/null 2>&1; then iostat -d -w 5 | while IFS= read -r line; do printf '%s %s\n' "$(date +'%m/%d/%Y %H:%M:%S')" "$line" >> mongo-diskstats; done # Check if vmstat -t is available. elif vmstat -td > /dev/null 2>&1; then vmstat -td 5 > mongo-diskstats # Check if vmstat -T d is available. elif vmstat -T d > /dev/null 2>&1; then vmstat -T d 5 > mongo-diskstats else printf "Cannot collect mongo-diskstats on this platform\n" fi "collect system resource info": &collect_system_resource_info command: shell.exec params: working_dir: src background: true system_log: true script: | ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/collect_resource_info.py -o system_resource_info.json -i 5 # Run a monitor process as a background, system task to periodically # display how many threads interesting processes are using. "monitor process threads": &monitor_process_threads command: shell.exec params: background: true system_log: true script: | proc_list="(bsondump|java|lein|mongo|python|_test$|_test\.exe$)" if [ "Windows_NT" = "$OS" ]; then get_pids() { proc_pids=$(tasklist /fo:csv | awk -F'","' '{x=$1; gsub("\"","",x); print $2, x}' | grep -i -E $1 | cut -f1 -d ' '); } get_process_info() { proc_name=""; proc_info=$(wmic process where "ProcessId=\"$1\"" get "Name,ProcessId,ThreadCount" /format:csv 2> /dev/null | grep $1); if [ ! -z $proc_info ]; then proc_name=$(echo $proc_info | cut -f2 -d ','); proc_threads=$(echo $proc_info | cut -f4 -d ','); fi; } else get_pids() { proc_pids=$(pgrep $1); } get_process_info() { proc_name=$(ps -p $1 -o comm=); # /proc is available on Linux platforms if [ -f /proc/$1/status ]; then ${set_sudo} proc_threads=$($sudo grep Threads /proc/$1/status | sed "s/\s//g" | cut -f2 -d ":"); else proc_threads=$(ps -AM $1 | grep -vc PID); fi; } fi while [ 1 ] do get_pids $proc_list if [ ! -z "$proc_pids" ]; then printf "Running process/thread counter\n" printf "PROCESS\tPID\tTHREADS\n" fi for pid in $proc_pids do get_process_info $pid if [ ! -z "$proc_name" ]; then printf "$proc_name\t$pid\t$proc_threads\n" fi done sleep 60 done "setup credentials" : &setup_credentials command: shell.exec params: working_dir: src silent: true script: | cat > mci.buildlogger </dev/null || true done rm -r mongodb*/*.$file_ext 2>/dev/null || true done rm mongo-debugsymbols.tgz 2>/dev/null || true "build rocksdb" : command: shell.exec params: script: | set -o errexit set -o verbose if [ "${build_rocksdb|}" = "true" ]; then rm -rf rocksdb git clone https://github.com/facebook/rocksdb.git cd rocksdb make USE_RTTI=1 CC=/opt/mongodbtoolchain/v2/bin/gcc CXX=/opt/mongodbtoolchain/v2/bin/g++ static_lib fi "build cdriver" : command: shell.exec params: script: | set -o errexit set -o verbose if [ "${build_cdriver|}" = "true" ]; then CDRIVER_VERSION=1.6.3 rm -rf mongo-c-driver-$CDRIVER_VERSION rm -rf mongo-c-driver curl -LO https://github.com/mongodb/mongo-c-driver/releases/download/$CDRIVER_VERSION/mongo-c-driver-$CDRIVER_VERSION.tar.gz tar xzf mongo-c-driver-$CDRIVER_VERSION.tar.gz mv mongo-c-driver-$CDRIVER_VERSION mongo-c-driver cd mongo-c-driver export ${compile_env|} ./configure --prefix=$(pwd)/install --build x86_64-apple-darwin --with-libbson=bundled --disable-automatic-init-and-cleanup --disable-ssl --disable-sasl --disable-examples --disable-man-pages --disable-html-docs --disable-shm-counters --disable-shared --enable-static ${cdriver_configure_flags|} make make install fi "build new tools" : command: shell.exec params: working_dir: src/src/mongo/gotools script: | set -o verbose set -o errexit # make sure newlines in the scripts are handled correctly by windows if [ "Windows_NT" = "$OS" ]; then set -o igncr fi; sed -i.bak "s/built-without-version-string/$(git describe)/" common/options/options.go sed -i.bak "s/built-without-git-spec/$(git rev-parse HEAD)/" common/options/options.go . ./${set_tools_gopath|set_gopath.sh} build_tools="bsondump mongostat mongofiles mongoexport mongoimport mongorestore mongodump mongotop mongooplog" if [ "${build_mongoreplay}" = "true" ]; then build_tools="$build_tools mongoreplay" fi for i in $build_tools; do ${gorootvars} go build ${tooltags|} -o "../../mongo-tools/$i${exe|}" $i/main/$i.go "../../mongo-tools/$i${exe|}" --version done "do setup" : - *fetch_artifacts - *fetch_binaries - *extract_binaries - *check_binary_version - *get_buildnumber - *setup_credentials - *run_diskstats - *monitor_process_threads - *collect_system_resource_info "set up virtualenv" : command: shell.exec type: test params: working_dir: src script: | # exit immediately if virtualenv is not found set -o errexit set -o verbose python_loc=$(which ${python|/opt/mongodbtoolchain/v2/bin/python2}) if [ "Windows_NT" = "$OS" ]; then python_loc=$(cygpath -w $python_loc) fi virtualenv --python $python_loc --system-site-packages ./venv "run tests" : command: shell.exec type: test params: working_dir: src script: | set -o errexit set -o verbose if [ ${disable_unit_tests|false} = "false" ]; then # activate the virtualenv if it has been set up ${activate_virtualenv} if [ -f /proc/self/coredump_filter ]; then # Set the shell process (and its children processes) to dump ELF headers (bit 4), # anonymous shared mappings (bit 1), and anonymous private mappings (bit 0). echo 0x13 > /proc/self/coredump_filter if [ -f /sbin/sysctl ]; then # Check that the core pattern is set explicitly on our distro image instead # of being the OS's default value. This ensures that coredump names are consistent # across distros and can be picked up by Evergreen. core_pattern=$(/sbin/sysctl -n "kernel.core_pattern") if [ "$core_pattern" = "dump_%e.%p.core" ]; then echo "Enabling coredumps" ulimit -c unlimited fi fi fi extra_args="" if [ ${run_multiple_jobs|false} = true ]; then processor_architecture=$(uname -m) num_jobs_available=${num_jobs_available|1} # Reduce the number of available jobs by half when running any sharding*, replica_sets* # and select jsCore passthrough tasks on Windows and ARM to avoid overwhelming test hosts. if [ "Windows_NT" = "$OS" ] || [ "aarch64" = $processor_architecture ]; then case "${task_name}" in replica_sets_initsync_jscore_passthrough \ |replica_sets_initsync_static_jscore_passthrough \ |replica_sets_resync_static_jscore_passthrough \ |replica_sets* \ |sharding* \ ) num_jobs_available=$((${num_jobs_available|2} / 2)) echo "Reducing jobs from ${num_jobs_available|1} to $num_jobs_available" ;; esac fi if [ ${max_jobs|0} -gt 0 ] && [ ${max_jobs|0} -lt $num_jobs_available ]; then extra_args="$extra_args --jobs=${max_jobs}" else extra_args="$extra_args --jobs=$num_jobs_available" fi fi if [ ${should_shuffle|true} = true ]; then extra_args="$extra_args --shuffle" fi if [ ${continue_on_failure|true} = true ]; then extra_args="$extra_args --continueOnFailure" fi # Default storageEngineCacheSizeGB to 1. Override on individual test config if needed. # resmoke will assign to the appropriate parameter on storage engines that support it. set +o errexit echo "${resmoke_args}" | grep -q storageEngineCacheSizeGB if [ $? -eq 1 ]; then extra_args="$extra_args --storageEngineCacheSizeGB=1" fi set -o errexit path_value="$PATH" if [ ${variant_path_suffix} ]; then path_value="$path_value:${variant_path_suffix}" fi if [ ${task_path_suffix} ]; then path_value="$path_value:${task_path_suffix}" fi if [ "${is_patch|}" = "true" ]; then extra_args="$extra_args --tagFile=etc/test_lifecycle.yml --patchBuild" else extra_args="$extra_args --tagFile=etc/test_retrial.yml" fi # The "resmoke_wrapper" expansion is used by the 'burn_in_tests' task to wrap the resmoke.py # invocation. It doesn't set any environment variables and should therefore come last in # this list of expansions. PATH="$path_value" \ ${gcov_environment} \ ${rlp_environment} \ ${san_options} \ ${san_symbolizer} \ ${snmp_config_path} \ ${resmoke_wrapper} \ $python buildscripts/evergreen_run_tests.py \ ${resmoke_args} \ $extra_args \ ${test_flags} \ --log=buildlogger \ --staggerJobs=on \ --taskId=${task_id} \ --taskName=${task_name} \ --variantName=${build_variant} \ --distroId=${distro_id} \ --reportFile=report.json fi # end if ${disable_unit_tests} "do jepsen setup" : - command: shell.exec params: working_dir: src script: | set -o errexit # Build libfaketime. A version of libfaketime at least as new as v0.9.6-9-g75896bd is # required to use the FAKETIME_NO_CACHE and FAKETIME_TIMESTAMP_FILE environment variables. # Additionally, a version of libfaketime containing the changes mentioned in SERVER-29336 # is required to avoid needing to use libfaketimeMT.so.1 and serializing all calls to # fake_clock_gettime() with a mutex. git clone --branch=for-jepsen --depth=1 git@github.com:10gen/libfaketime.git cd libfaketime branch=$(git symbolic-ref --short HEAD) commit=$(git show -s --pretty=format:"%h - %an, %ar: %s") echo "Git branch: $branch, commit: $commit" make PREFIX=$(pwd)/build/ LIBDIRNAME='.' install # Add libfaketime to mongo-binaries.tgz so it gets uploaded to each of the LXC containers. # The build/ prefix of the shared object will get removed when Jepsen extracts the tarball # into the /opt/mongodb/ directory. gzip --stdout --decompress ../mongo-binaries.tgz > mongo-binaries.tar tar --append --file=mongo-binaries.tar build/libfaketime.so.1 gzip --stdout mongo-binaries.tar > ../mongo-binaries.tgz - command: shell.exec params: working_dir: src script: | set -o errexit git clone --branch=jepsen-mongodb-master --depth=1 git@github.com:10gen/jepsen.git jepsen-mongodb cd jepsen-mongodb branch=$(git symbolic-ref --short HEAD) commit=$(git show -s --pretty=format:"%h - %an, %ar: %s") echo "Git branch: $branch, commit: $commit" lein install - command: shell.exec params: working_dir: src script: | # On each LXC host do the following: # Check to see if it's running, if not try to restart it. # Copy the mongodb tarball to /root. numVms=5 exit_failure_code=0 ${set_sudo} for i in $(seq 1 $numVms) do hostn=n$i hostn_log=/var/log/lxc/$hostn.log $sudo lxc-wait -n $hostn -s 'RUNNING' -t 0 lxc_code=$? lxc_restarts=0 # If the LXC host is not running, try 5 times to start it. while [ $lxc_code -ne 0 ] do lxc_restarts=$((lxc_restarts+1)) echo "Restarting host $hostn, attempt $lxc_restarts" # When restarting, increase the logging level to INFO for the LXC log. $sudo lxc-start -n $hostn --logpriority=INFO --logfile=$hostn_log -d $sudo lxc-wait -n $hostn -s 'RUNNING' -t 60 lxc_code=$? if [ $lxc_restarts -eq 5 ]; then break fi done scp -o ConnectTimeout=30 -o ConnectionAttempts=10 mongo-binaries.tgz root@$hostn: exit_code=$? # If the copy fails, then tail the LXC host log. if [ $exit_code -ne 0 ]; then echo "Copy to host $hostn failed!" $sudo tail -n 100 $hostn_log exit_failure_code=$exit_code else # Cleanup any existing logs from previous runs. ssh root@$hostn "rm -f /root/mongod.log /root/stdout.log /opt/mongod/mongod.log /opt/mongod/stdout.log" fi done exit $exit_failure_code "run jepsen test" : - command: shell.exec type: test timeout_secs: 2700 # Timeout test if there is no output for more than 45 minutes. params: working_dir: src/jepsen-mongodb script: | set -o verbose start_time=$(date +%s) lein run test --test ${jepsen_test_name} \ --tarball "file:///root/mongo-binaries.tgz" \ --ssh-private-key ~/.ssh/id_rsa_lxc \ --clock-skew faketime \ --libfaketime-path /opt/mongodb/libfaketime.so.1 \ ${jepsen_key_time_limit} \ ${jepsen_protocol_version} \ ${jepsen_read_concern} \ ${jepsen_read_with_find_and_modify} \ ${jepsen_storage_engine} \ ${jepsen_time_limit} \ ${jepsen_write_concern} \ 2>&1 \ | tee jepsen_${task_name}_${execution}.log end_time=$(date +%s) elapsed_secs=$((end_time-start_time)) # Since we cannot use PIPESTATUS to get the exit code from the "lein run ..." pipe in dash shell, # we will check the output for success, failure or setup error. Note that 'grep' returns with exit code # 0 if it finds a match, and exit code 1 if no match is found. grep -q "Everything looks good" jepsen_${task_name}_${execution}.log grep_exit_code=$? if [ $grep_exit_code -eq 0 ]; then status='"pass"' failures=0 final_exit_code=0 else grep -q "Analysis invalid" jepsen_${task_name}_${execution}.log grep_exit_code=$? if [ $grep_exit_code -eq 0 ]; then status='"fail"' failures=1 final_exit_code=1 else # If the failure is due to setup, then this is considered a system failure. echo $grep_exit_code > jepsen_system_failure_${task_name}_${execution} exit 0 fi fi # Create report.json echo "{\"failures\": $failures, \"results\": [{\"status\": $status, \"exit_code\": $final_exit_code, \"test_file\": \"${task_name}\", \"start\": $start_time, \"end\": $end_time, \"elapsed\": $elapsed_secs}]}" > ../report.json exit $final_exit_code - command: shell.exec params: working_dir: src/jepsen-mongodb script: | set -o verbose # Jepsen system failure if file exists. if [ -f jepsen_system_failure_${task_name}_${execution} ]; then exit $(cat jepsen_system_failure_${task_name}_${execution}) fi "run jstestfuzz": - command: shell.exec params: working_dir: src script: | set -o errexit set -o verbose git clone git@github.com:10gen/jstestfuzz.git cp mongodb*/bin/mongod . - command: shell.exec params: working_dir: src script: | set -o errexit set -o verbose git clone --depth 1 git@github.com:10gen/mongo-enterprise-modules.git jstests/enterprise_tests git clone --depth 1 git@github.com:10gen/QA.git jstests/qa_tests - command: shell.exec type: test params: working_dir: src/jstestfuzz script: | set -o errexit set -o verbose if [ "Windows_NT" = "$OS" ]; then # An "npm" directory might not have been created in %APPDATA% by the Windows installer. # Work around the issue by specifying a different %APPDATA% path. # See: https://github.com/nodejs/node-v0.x-archive/issues/8141 export APPDATA=${workdir}/npm-app-data export PATH="$PATH:/cygdrive/c/Program Files (x86)/nodejs" # Windows location else export PATH="$PATH:/opt/node/bin" fi echo "jstestfuzz self-tests are starting" set +o errexit npm test > npm_test-${task_id}.log 2>&1 if [ $? -ne 0 ]; then which node node --version which npm npm --version cat npm_test-${task_id}.log exit 1 fi rm -f npm_test-${task_id}.log set -o errexit echo "jstestfuzz self-tests finished" npm run ${npm_command|jstestfuzz} -- --jsTestsDir ../jstests ${jstestfuzz_vars} - command: archive.targz_pack params: target: "jstests.tgz" source_dir: "src/jstestfuzz" include: - "out/*.js" - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: jstests.tgz remote_file: mongodb-mongo-master/${build_variant}/${revision}/jstestfuzz/${task_id}-${execution}.tgz bucket: mciuploads permissions: public-read content_type: ${content_type|application/x-gzip} display_name: Generated Tests - Execution ${execution} "run idl tests" : - command: shell.exec type: test params: working_dir: src script: | set -o verbose set -o errexit ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/idl/run_tests.py "run ios sim embedded tests" : command: shell.exec type: test params: working_dir: src script: | set -o verbose set -o errexit if [ ${enable_ios_sim_tests|false} = "true" ]; then ${compile_env|} ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/runiossim.py --test "build/debug/mongo/client/embedded/mongo_embedded_capi_test" ${compile_env|} ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/runiossim.py --test "build/debug/mongo/client/embedded/mongo_embedded_transport_test" fi "run powercycle test" : command: shell.exec params: working_dir: src script: | set -o verbose set -o errexit ${activate_virtualenv} if [ ! -z "${virtualenv_requirements}" ]; then easy_install ${virtualenv_requirements} fi ${set_sudo} if [ ! -z $sudo ]; then remote_sudo="--remoteSudo" fi # The virtaulenv bin_dir is different for Linux and Windows bin_dir=$(find $VIRTUAL_ENV -name activate | sed -e "s,$VIRTUAL_ENV,,;s,activate,,;s,/,,g") cmds="source ${virtualenv_dir|venv}/$bin_dir/activate" cmds="$cmds; python -u" # The remote python operates in a virtualenv remote_python="--remotePython \"$cmds\"" start_time=$(date +%s) status="\"pass\"" failures=0 set +o errexit eval python -u pytests/powertest.py \ "${connection_options} \ ${program_options} \ $remote_sudo \ $remote_python \ ${test_options} \ ${crash_options} \ ${mongodb_options} \ ${mongod_options} \ ${mongod_extra_options}" exit_code=$? # Create report.json end_time=$(date +%s) elapsed_secs=$((end_time-start_time)) if [ $exit_code -ne 0 ]; then status="\"fail\"" failures=1 fi echo "{\"failures\": $failures, \"results\": [{\"status\": $status, \"exit_code\": $exit_code, \"test_file\": \"${task_name}\", \"start\": $start_time, \"end\": $end_time, \"elapsed\": $elapsed_secs}]}" > report.json exit $exit_code "do multiversion setup" : command: shell.exec params: working_dir: src script: | set -o errexit set -o verbose ${activate_virtualenv} # The Windows build variants are running python 2.7.3 and require TLS 1.2 from pyOpenSSL pip install 'pyOpenSSL ; sys_platform == "win32" or sys_platform == "cygwin"' rm -rf /data/install /data/multiversion $python buildscripts/setup_multiversion_mongodb.py \ --installDir /data/install \ --linkDir /data/multiversion \ --edition ${multiversion_edition|base} \ --platformArchitecture ${multiversion_platform_arch|linux/x86_64} \ --useLatest 3.0 3.2 3.4 "do snmp setup" : command: shell.exec params: working_dir: src script: | set -o errexit set -o verbose mkdir -p snmpconf cp -f src/mongo/db/modules/enterprise/docs/mongod.conf.master snmpconf/mongod.conf "do watchdog setup" : command: shell.exec params: working_dir: src script: | set -o errexit set -o verbose bash src/mongo/db/modules/enterprise/jstests/watchdog/charybdefs_setup.sh "cleanup environment" : command: shell.exec params: script: | set -o verbose rm -rf src /data/db/* mongo-diskstats* mongo-*.tgz ~/.aws ~/.boto "kill processes" : command: shell.exec params: silent: true script: | process_kill_list="(^cl\.exe$|bsondump|java|lein|lldb|mongo|python|_test$|_test\.exe$)" # Exclude Evergreen agent processes and other system daemons process_exclude_list="(main|tuned|evergreen)" if [ "Windows_NT" = "$OS" ]; then # Get the list of Windows tasks (tasklist list format): # - Transpose the Image Name and PID # - The first column has the process ID # - The second column (and beyond) has task name # - Grep for the task names of interest while ignoring any names that are in the exclude list processes=$(tasklist /fo:csv | awk -F'","' '{x=$1; gsub("\"","",x); print $2, x}' | grep -i -E "$process_kill_list" | grep -i -v -E "$process_exclude_list") # Kill the Windows process by process ID with force (/f) kill_process () { pid=$(echo $1 | cut -f1 -d ' '); echo "Killing process $1"; taskkill /pid "$pid" /f; } else # Get the list of Unix tasks (pgrep full & long): # - Grep for the task names of interest while ignoring any names that are in the exclude list # - The first column has the process ID # - The second column (and beyond) has task name # There are 2 "styles" of grep, figure out which one works grep=grep echo "a" | $grep -qE "(a)" > /dev/null 2>&1 if [ $? -ne 0 ]; then # Extended grep on Solaris is in the following location grep=/usr/xpg4/bin/grep if [ ! -f $grep ]; then echo "Extended $grep does not exist" return 1 fi fi # There are 2 "styles" of pgrep, figure out which one works. # Due to https://bugs.launchpad.net/ubuntu/+source/procps/+bug/1501916 # we cannot rely on the return status ($?) to detect if the option is supported. pgrep -f --list-full ".*" 2>&1 | $grep -qE "(illegal|invalid|unrecognized) option" if [ $? -ne 0 ]; then pgrep_list=$(pgrep -f --list-full "$process_kill_list") else pgrep_list=$(pgrep -f -l "$process_kill_list") fi # Since a process name might have a CR or LF in it, we need to delete any lines from # pgrep which do not start with space(s) and 1 digit and trim any leading spaces. processes=$(echo "$pgrep_list" | $grep -ivE "$process_exclude_list" | sed -e '/^ *[0-9]/!d; s/^ *//; s/[[:cntrl:]]//g;') # Kill the Unix process ID with signal KILL (9) kill_process () { pid=$(echo $1 | cut -f1 -d ' '); echo "Killing process $1"; kill -9 $pid; } fi # Since a full process name can have spaces, the IFS (internal field separator) # should not include a space, just a LF & CR IFS=$(printf "\n\r") for process in $processes do kill_process "$process" done "set up remote credentials": command: shell.exec params: silent: true script: | set -o errexit # Since the macros 'private_key_remote' and 'private_key_file' are not always defined # we default to /dev/null to avoid syntax errors of an empty expansion. if [ ! -z "${private_key_remote}" ] && [ ! -z "${private_key_file}" ] ; then mkdir -p ~/.ssh echo -n "${private_key_remote}" > ${private_key_file|/dev/null} chmod 0600 ${private_key_file|/dev/null} fi mkdir -p ~/.aws cat < ~/.aws/config [default] region = us-east-1 EOF cat < ~/.aws/credentials [default] aws_access_key_id = ${aws_key_remote} aws_secret_access_key = ${aws_secret_remote} EOF cat < ~/.boto [Boto] https_validate_certificates = False EOF "run kitchen": command: shell.exec type: test params: shell: bash working_dir: src/buildscripts/package_test script: | set -o errexit export KITCHEN_ARTIFACTS_URL="https://s3.amazonaws.com/mciuploads/${project}/${build_variant}/${revision}/artifacts/${build_id}.tgz" export KITCHEN_SECURITY_GROUP="${kitchen_security_group}" export KITCHEN_SSH_KEY_ID="${kitchen_ssh_key_id}" export KITCHEN_SUBNET="${kitchen_subnet}" # set expiration tag 2 hours in the future, since no test should take this long export KITCHEN_EXPIRE="$(${python|/opt/mongodbtoolchain/v2/bin/python2} -c 'import datetime; print((datetime.datetime.utcnow() + datetime.timedelta(hours=2)).strftime("%Y-%m-%d %H:%M:%S"))')" for i in {1..3} do if ! kitchen verify "${packager_distro}"; then verified="false" kitchen destroy "${packager_distro}" || true sleep 30 else verified="true" break fi done kitchen destroy "${packager_distro}" || true test "$verified" = "true" "fetch test_lifecycle.yml": - command: shell.exec type: test params: working_dir: src script: | set -o verbose ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/fetch_test_lifecycle.py \ --metadataRepo git@github.com:mongodb/mongo-test-metadata.git \ --lifecycleFile etc/test_lifecycle.yml \ --referencesFile references.yml \ --destinationFile etc/test_lifecycle.yml \ --revision ${revision} \ ${project} exit_code=$? if [ ${fail_task_on_error|false} = true ]; then exit $exit_code else exit 0 fi "set up EC2 instance": - command: shell.exec params: working_dir: src script: | set -o errexit ${activate_virtualenv} easy_install --upgrade boto3 # Clone another instance of this host in EC2. buildscripts/launch_evergreen_ec2_instance.sh \ -k ${ssh_key_id} \ -t "AMI Evergreen ${task_id}" \ -y ${aws_ec2_yml} - command: expansions.update params: file: src/${aws_ec2_yml} - command: shell.exec params: shell: bash working_dir: src script: | set -o errexit # Copy mount_data_drive.sh script to remote host. ssh_connection_options="${ssh_identity} ${ssh_connection_options}" ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/remote_operations.py \ --verbose \ --userHost $USER@${ip_address} \ --operation "copy_to" \ --sshConnectionOptions "$ssh_connection_options" \ --retries ${ssh_retries|0} \ --file buildscripts/mount_data_drive.sh - command: shell.exec params: shell: bash working_dir: src script: | set -o errexit # Mount /data on the attached drive(s), more than 1 indicates a RAID set. ${set_sudo} script_opts="-d '${device_names}'" if [ ! -z "${raid_device_name}" ]; then script_opts="$script_opts -r ${raid_device_name}" fi if [ ! -z "${fstype}" ]; then script_opts="$script_opts -t ${fstype}" fi if [ ! -z "${fs_options}" ]; then script_opts="$script_opts -o '${fs_options}'" fi group=$(id -Gn $USER | cut -f1 -d ' ') || true user_group="$USER:$group" script_opts="$script_opts -u $user_group" data_db=/data/db set_permission="chmod 777 $data_db" if [ "Windows_NT" = "$OS" ]; then set_permission="setfacl -s user::rwx,group::rwx,other::rwx $data_db" fi cmds="$sudo bash mount_data_drive.sh $script_opts; $set_permission; ls -ld $data_db; df; mount" ssh_connection_options="${ssh_identity} ${ssh_connection_options}" ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/remote_operations.py \ --verbose \ --userHost $USER@${ip_address} \ --sshConnectionOptions "$ssh_connection_options" \ --retries ${ssh_retries|0} \ --commands "$cmds" - command: shell.exec params: shell: bash working_dir: src script: | set -o errexit # Create remote_dir, if specified as expansion macro and is not '.' (pwd). if [[ -z "${remote_dir|}" || ${remote_dir} == "." ]]; then exit 0 fi ${set_sudo} ssh_connection_options="${ssh_identity} ${ssh_connection_options}" group=$(id -Gn $USER | cut -f1 -d ' ') || true user_group="$USER:$group" set_permission="chmod 777 ${remote_dir}" if [ "Windows_NT" = "$OS" ]; then set_permission="setfacl -s user::rwx,group::rwx,other::rwx ${remote_dir}" fi cmds="$sudo mkdir -p ${remote_dir}; $sudo chown $user_group ${remote_dir}; $set_permission; ls -ld ${remote_dir}" ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/remote_operations.py \ --verbose \ --userHost $USER@${ip_address} \ --sshConnectionOptions "$ssh_connection_options" \ --retries ${ssh_retries|0} \ --commands "$cmds" - command: shell.exec params: shell: bash working_dir: src script: | set -o errexit # Copy buildscripts, mongoDB executables and debug symbols to the remote host. file_param="--file buildscripts" mongo_executables="mongo mongod mongos" file_extensions=".debug .dSYM .pdb" for executable in $mongo_executables do file_param="$file_param --file $executable${exe}" for extension in $file_extensions do file=$executable$extension if [[ -f $file || -d $file ]]; then file_param="$file_param --file $file" fi done done ssh_connection_options="${ssh_identity} ${ssh_connection_options}" ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/remote_operations.py \ --verbose \ --userHost $USER@${ip_address} \ --operation "copy_to" \ --sshConnectionOptions "$ssh_connection_options" \ --retries ${ssh_retries|0} \ $file_param \ --remoteDir ${remote_dir} - command: shell.exec params: shell: bash working_dir: src script: | set -o errexit # Setup virtualenv and install python modules on remote, if required if [ ! -z "${virtualenv_requirements}" ]; then cmds="python_loc=\$(which ${python|/opt/mongodbtoolchain/v2/bin/python2})" cmds="$cmds; if [ \"Windows_NT\" = \"$OS\" ]; then python_loc=\$(cygpath -w \$python_loc); fi" cmds="$cmds; virtualenv --python \$python_loc --system-site-packages ${virtualenv_dir|venv}" cmds="$cmds; activate=\$(find ${virtualenv_dir|venv} -name 'activate')" cmds="$cmds; source \$activate" cmds="$cmds; easy_install ${virtualenv_requirements}" ssh_connection_options="${ssh_identity} ${ssh_connection_options}" ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/remote_operations.py \ --verbose \ --userHost $USER@${ip_address} \ --sshConnectionOptions "$ssh_connection_options" \ --retries ${ssh_retries|0} \ --commands "$cmds" fi "copy EC2 artifacts": - command: shell.exec params: shell: bash working_dir: src script: | # Tar/zip artifacts on remote host. if [ -z "${ec2_artifacts}" ]; then exit 0 fi cmd="${tar|tar} czvf ec2_artifacts.tgz ${ec2_artifacts}" ssh_connection_options="${ssh_identity} ${ssh_connection_options}" ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/remote_operations.py \ --verbose \ --userHost $USER@${ip_address} \ --sshConnectionOptions "$ssh_connection_options" \ --retries ${ssh_retries|0} \ --commands "$cmd" - command: shell.exec params: shell: bash working_dir: src script: | # Copy remote artifacts. if [ -z "${ec2_artifacts}" ]; then exit 0 fi ssh_connection_options="${ssh_identity} ${ssh_connection_options}" ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/remote_operations.py \ --verbose \ --userHost $USER@${ip_address} \ --operation "copy_from" \ --sshConnectionOptions "$ssh_connection_options" \ --retries ${ssh_retries|0} \ --file ec2_artifacts.tgz "cleanup EC2 instance": command: shell.exec params: working_dir: src script: | if [ -z ${instance_id} ]; then exit 0 fi ${activate_virtualenv} echo "Terminating $instance_id" aws_ec2=$(python buildscripts/aws_ec2.py --imageId ${instance_id} --mode terminate) echo "Terminated AMI EC2 instance: $aws_ec2" pre: - func: "kill processes" - func: "cleanup environment" - command: expansions.update params: updates: - key: activate_virtualenv value: | # check if virtualenv is set up if [ -d "venv" ]; then if [ "Windows_NT" = "$OS" ]; then . ./venv/Scripts/activate else . ./venv/bin/activate fi python=python else python=${python|/opt/mongodbtoolchain/v2/bin/python2} fi - key: posix_workdir value: eval 'if [ "Windows_NT" = "$OS" ]; then echo $(cygpath -u "${workdir}"); else echo ${workdir}; fi' # For ssh disable the options GSSAPIAuthentication, CheckHostIP, StrictHostKeyChecking # & UserKnownHostsFile, since these are local connections from one AWS instance to another. - key: ssh_connection_options value: -o GSSAPIAuthentication=no -o CheckHostIP=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=20 -o ConnectionAttempts=20 - key: ssh_retries value: "10" - key: set_sudo value: | set -o > /tmp/settings.log set +o errexit grep errexit /tmp/settings.log | grep on errexit_on=$? # Set errexit "off". set +o errexit sudo= # Use sudo, if it is supported. sudo date > /dev/null 2>&1 if [ $? -eq 0 ]; then sudo=sudo fi # Set errexit "on", if previously enabled. if [ $errexit_on -eq 0 ]; then set -o errexit fi - command: shell.exec params: system_log: true script: | ulimit -a # Clear the dmesg ring buffer. The "post" phase will check dmesg for OOM messages. ${set_sudo} $sudo dmesg -c > /dev/null 2>&1 if [ $? -eq 0 ]; then echo "Cleared the dmesg ring buffer" else echo "Could not clear the dmesg ring buffer" fi post: - command: attach.results params: file_location: src/report.json - func: "copy EC2 artifacts" - func: "cleanup EC2 instance" - func: "kill processes" # Print out any Out of Memory killed process messages. - command: shell.exec params: system_log: true working_dir: src # Temporary files created in src will be cleaned up in "pre". script: | ${set_sudo} # Use dmesg -T option, if supported, to display timestamps. dmesg=dmesg $sudo dmesg -T > /dev/null 2>&1 if [ $? -eq 0 ]; then dmesg="dmesg -T" fi $sudo $dmesg 2> /dev/null > dmesg.txt if [ $? -ne 0 ]; then echo "Cannot check for OOM (Out of memory) killed processes on this platform" exit 0 fi egrep -i '(Out of memory|OOM[- ]killer|Killed process)' dmesg.txt > oom.txt if [ -s oom.txt ]; then echo "OOM (Out of memory) killed processes detected" cat oom.txt else echo "No OOM (Out of memory) killed processes detected" fi # Cleanup after the watchdog FUSE testing - command: shell.exec params: working_dir: src script: | if [ -d /data/thrift ]; then rm -rf /data/thrift fi if [ -d /data/charybdefs ]; then rm -rf /data/charybdefs fi # Gather and archive FTDC data. - command: shell.exec params: working_dir: src script: | # Using shell and tar to recurse properly to all possible diagnostic.data subdirectories. # The archive.targz_pack command is not being used here because the command's glob support # did not allow us to gather all directories. if [ -d /data/db ]; then file_list=$(cd /data/db && find . -type d -name diagnostic.data) if [ -n "$file_list" ]; then ${tar|tar} cvzf diagnostic-data.tgz -C /data/db $file_list fi fi - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: src/diagnostic-data.tgz remote_file: mongodb-mongo-master/${build_variant}/${revision}/ftdc/mongo-diagnostic-data-${task_id}-${execution}.tgz bucket: mciuploads permissions: public-read content_type: ${content_type|application/x-gzip} display_name: FTDC Diagnostic Data - Execution ${execution} optional: true # Archive remote EC2 artifacts. - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: src/ec2_artifacts.tgz remote_file: mongodb-mongo-master/${build_variant}/${revision}/remote_ec2/remote_ec2_artifacts-${task_id}-${execution}.tgz bucket: mciuploads permissions: public-read content_type: ${content_type|application/x-gzip} display_name: Remote EC2 Artifacts - Execution ${execution} optional: true # Process and save coverage data. - command: shell.exec params: working_dir: src script: | if [ -d "./build" ]; then file_list=$(find ./build -type f -name "*.gcda") if [ -n "$file_list" ]; then for gcda_file in $file_list; do /opt/mongodbtoolchain/v2/bin/gcov -i $gcda_file done fi fi - command: archive.targz_pack params: target: "src/gcov-intermediate-files.tgz" source_dir: "src" include: - "*.gcda.gcov" - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: "src/gcov-intermediate-files.tgz" remote_file: mongodb-mongo-master/${build_variant}/${revision}/gcov/gcov-intermediate-files-${task_id}-${execution}.tgz bucket: mciuploads permissions: public-read content_type: ${content_type|application/x-gzip} display_name: gcov intermediate files - Execution ${execution} optional: true # Gather and archive the Jepsen mongod logs. - command: shell.exec params: working_dir: src script: | # For Jepsen tasks, archive the mongod log files. if [ ${distro_id} = "ubuntu1404-jepsen" ]; then # Copy mongod.log & stdout.log from each LXC host for archive purposes. Note that # stdout.log is also generated as a result of starting mongod. # The logs can reside in 1 of 2 places: # If the task completed: /root # If the task failed to complete: /opt/mongodb # There may be logs in both places, but the ones in /opt/mongodb should override # the ones in /root, since the logs in /root could be from a previous successful run. numVms=5 for i in $(seq 1 $numVms) do hostn=n$i for log_dir in /root /opt/mongodb do for log in mongod.log stdout.log do outlog="$(echo $log | cut -f1 -d '.')-n$i.log" scp -q root@$hostn:$log_dir/$log $outlog 2> /dev/null if [ -f $outlog ]; then echo "Copied $log_dir/$log from $hostn to $outlog" fi done done done fi - command: archive.targz_pack params: target: "src/jepsen-mongod-logs.tgz" source_dir: "src" include: - "mongod*.log" - "stdout*.log" - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: src/jepsen-mongod-logs.tgz remote_file: mongodb-mongo-master/${build_variant}/${revision}/jepsen/jepsen-mongod-logs-${task_id}-${execution}.tgz bucket: mciuploads permissions: public-read content_type: ${content_type|application/x-gzip} display_name: Jepsen mongod Logs - ${execution} optional: true # Gather and archive the Jepsen results. - command: archive.targz_pack params: target: "src/jepsen-results.tgz" source_dir: "src/jepsen-mongodb/store" include: - "./**" - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: src/jepsen-results.tgz remote_file: mongodb-mongo-master/${build_variant}/${revision}/jepsen/jepsen-results-${task_id}-${execution}.tgz bucket: mciuploads permissions: public-read content_type: ${content_type|application/x-gzip} display_name: Jepsen Test Results - ${execution} optional: true # Gather and archive mongo coredumps. - command: shell.exec params: working_dir: "src" script: | # Find all core files and move to src core_files=$(/usr/bin/find -H .. \( -name "*.core" -o -name "*.mdmp" \) 2> /dev/null) for core_file in $core_files do base_name=$(echo $core_file | sed "s/.*\///") # Move file if it does not already exist if [ ! -f $base_name ]; then mv $core_file . fi done - command: archive.targz_pack params: target: "mongo-coredumps.tgz" source_dir: "src" include: - "./**.core" - "./**.mdmp" # Windows: minidumps - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: mongo-coredumps.tgz remote_file: mongodb-mongo-master/${build_variant}/${revision}/coredumps/mongo-coredumps-${build_id}-${task_name}-${execution}.tgz bucket: mciuploads permissions: public-read content_type: ${content_type|application/x-gzip} display_name: Core Dumps - Execution ${execution} optional: true # Gather and archive failed unit test binaries. - command: shell.exec params: working_dir: "src" script: | mkdir unittest_binaries # Find all core files core_files=$(/usr/bin/find -H . \( -name "dump_*.core" -o -name "*.mdmp" \) 2> /dev/null) for core_file in $core_files do # A core file name does not always have the executable name that generated it. # See http://stackoverflow.com/questions/34801353/core-dump-filename-gets-thread-name-instead-of-executable-name-with-core-pattern # On platforms with GDB, we get the binary name from core file gdb=/opt/mongodbtoolchain/gdb/bin/gdb if [ -f $gdb ]; then binary_file=$($gdb -batch --quiet -ex "core $core_file" 2> /dev/null | grep "Core was generated" | cut -f2 -d "\`" | cut -f1 -d "'" | cut -f1 -d " ") binary_file_locations=$binary_file else # Find the base file name from the core file name, note it may be truncated. # Remove leading 'dump_' and trailing '..core' or '..mdmp' binary_file=$(echo $core_file | sed "s/.*\///;s/dump_//;s/\..*\.core//;s/\..*\.mdmp//") # Locate the binary file. Since the base file name might be truncated, the find # may return more than 1 file. binary_file_locations=$(/usr/bin/find -H . -name "$binary_file*${exe}" 2> /dev/null) fi if [ -z "$binary_file_locations" ]; then echo "Cannot locate the unittest binary file ($binary_file) that generated the core file $core_file" fi for binary_file_location in $binary_file_locations do new_binary_file=unittest_binaries/$(echo $binary_file_location | sed "s/.*\///") if [ ! -f $new_binary_file ]; then mv $binary_file_location $new_binary_file fi # On Windows if a .pdb symbol file exists, include it in the archive. pdb_file=$(echo $binary_file_location | sed "s/\.exe/.pdb/") if [ -f $pdb_file ]; then new_pdb_file=unittest_binaries/$(echo $pdb_file | sed "s/.*\///") mv $pdb_file $new_pdb_file fi done done - command: archive.targz_pack params: target: "mongo-unittests.tgz" source_dir: "src/unittest_binaries" include: - "./*_test${exe}" - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: mongo-unittests.tgz remote_file: mongodb-mongo-master/${build_variant}/${revision}/unittests/mongo-unittests-${build_id}-${task_name}-${execution}.tgz bucket: mciuploads permissions: public-read content_type: ${content_type|application/x-gzip} display_name: Unit tests - Execution ${execution} optional: true # Gather and archive hang_analyzer debugger files. - command: archive.targz_pack params: target: "src/mongo-hanganalyzer.tgz" source_dir: "src" include: - "./debugger*.*" - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: src/mongo-hanganalyzer.tgz remote_file: mongodb-mongo-master/${build_variant}/${revision}/hanganalyzer/mongo-hanganalyzer-${build_id}-${task_name}-${execution}.tgz bucket: mciuploads permissions: public-read content_type: ${content_type|application/x-gzip} display_name: Hang Analyzer Output - Execution ${execution} optional: true # Gather and archive disk statistics. - command: archive.targz_pack params: target: "diskstats.tgz" source_dir: "./" include: - "./mongo-diskstats*" - "./mongo-diskstats*.csv" - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: diskstats.tgz remote_file: mongodb-mongo-master/${build_variant}/${revision}/diskstats/mongo-diskstats-${task_id}-${execution}.tgz bucket: mciuploads permissions: public-read content_type: ${content_type|application/x-gzip} display_name: Disk Stats - Execution ${execution} optional: true # Gather and archive system resource information. - command: archive.targz_pack params: target: "system-resource-info.tgz" source_dir: src include: - "./system_resource_info*" - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: system-resource-info.tgz remote_file: mongodb-mongo-master/${build_variant}/${revision}/systemresourceinfo/mongo-system-resource-info-${task_id}-${execution}.tgz bucket: mciuploads permissions: public-read content_type: ${content_type|application/x-gzip} display_name: System Resource Info - Execution ${execution} optional: true # Cleanup steps. - command: shell.exec params: working_dir: src script: | # removes files from the (local) scons cache when it's over a # threshold, to the $prune_ratio percentage. Ideally override # these default values in the distro config in evergreen. if [ -d "${scons_cache_path}" ]; then ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/scons_cache_prune.py --cache-dir '${scons_cache_path}' --cache-size ${scons_cache_size|200} --prune-ratio ${scons_prune_ratio|0.8} fi - func: "cleanup environment" timeout: - func: "fetch debugsymbols archive" - func: "extract debugsymbols" - command: shell.exec params: working_dir: src script: | set -o verbose # We run the hang_analyzer.py script with root privileges when running the Jepsen tests in # order to be able to attach to the mongod processes inside the LXC containers. sudo= if [ ${distro_id} = "ubuntu1404-jepsen" ]; then ${set_sudo} fi hang_analyzer_option="-o file -o stdout -p ${hang_analyzer_processes|dbtest,java,mongo,mongod,mongos,python,_test} -g bsondump,mongodump,mongoexport,mongofiles,mongoimport,mongooplog,mongoreplay,mongorestore,mongostat,mongotop" if [ ${hang_analyzer_dump_core|true} = true ]; then hang_analyzer_option="-c $hang_analyzer_option" fi echo "Calling the hang analyzer: PATH=\"/opt/mongodbtoolchain/gdb/bin:$PATH\" $sudo ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/hang_analyzer.py $hang_analyzer_option" PATH="/opt/mongodbtoolchain/gdb/bin:$PATH" $sudo ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/hang_analyzer.py $hang_analyzer_option # Call hang_analyzer.py script for tasks that are running remote mongo processes # The file ${aws_ec2_yml} will define the remote's ${ip_address}. # Any task which uses remote processes will have previously loaded this file to set these # expansions macros. if [ -f ${aws_ec2_yml|""} ]; then core_ext=core if [ "Windows_NT" = "$OS" ]; then core_ext=mdmp fi ssh_connection_options="${ssh_identity} ${ssh_connection_options}" # buildscripts must be installed in ${remote_dir} on the remote host. remote_dir=${remote_dir|.} cmds="PATH=\"/opt/mongodbtoolchain/gdb/bin:\$PATH\" $sudo ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/hang_analyzer.py $hang_analyzer_option" ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/remote_operations.py \ --verbose \ --userHost $USER@${ip_address} \ --sshConnectionOptions "$ssh_connection_options" \ --retries ${ssh_retries} \ --commands "$cmds" \ --commandDir $remote_dir ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/remote_operations.py \ --verbose \ --userHost $USER@${ip_address} \ --operation "copy_from" \ --sshConnectionOptions "$ssh_connection_options" \ --retries ${ssh_retries} \ --file "$remote_dir/debugger*.*" \ --file "$remote_dir/*.$core_ext" fi ####################################### # Tasks # ####################################### ## The test_lifecycle_excluded_tasks are a list of tasks names which include: ## - Non jstests, i.e., compile, unittests ## - Non standard jstests, i.e., jstestfuzz ## Note that the task name supports glob patterns. test_lifecycle_excluded_tasks: - burn_in_tests - compile* - dbtest* - generational_jstestfuzz* - idl_tests - integration* - jepsen* - jstestfuzz* - lint - mongos* - package - push - rollback_fuzzer* - unittests* tasks: ## compile - build all scons targets except unittests ## - name: compile depends_on: [] commands: - command: manifest.load - *git_get_project - func: "get buildnumber" - func: "setup credentials" - func: "build new tools" # noop if ${newtools} is not "true" - func: "build rocksdb" # noop if ${build_rocksdb} is not "true" - *generate_compile_expansions # Then we load the generated version data into the agent so we can use it in task definitions - *apply_compile_expansions - command: shell.exec type: test params: working_dir: src script: | set -o errexit set -o verbose rm -rf ${install_directory|/data/mongo-install-directory} ${compile_env|} ${python|/opt/mongodbtoolchain/v2/bin/python2} ./buildscripts/scons.py ${compile_flags|} ${scons_cache_args|} --use-new-tools --build-mongoreplay="${build_mongoreplay}" core tools dbtest integration_tests dist dist-debugsymbols distsrc-${ext|tgz} ${msi_target|} MONGO_VERSION=${version} mv mongodb-src-*.${ext|tgz} distsrc.${ext|tgz} mv mongodb-*-debugsymbols.${ext|tgz} mongo-debugsymbols.tgz || true mv mongodb-*.${ext|tgz} mongodb-binaries.tgz # If strip is on the path (everywhere except windows) then we should strip the test binaries # before tarring them up if [ -x ${strip_path|/usr/bin/strip} ]; then cat build/integration_tests.txt | xargs ${strip_command|/usr/bin/strip} ${strip_command|/usr/bin/strip} dbtest ${strip_command|/usr/bin/strip} mongobridge fi - command: shell.exec params: working_dir: src script: | set -o errexit set -o verbose if [ "${has_packages|}" = "true" ] ; then cd buildscripts ${python|/opt/mongodbtoolchain/v2/bin/python2} ${packager_script} --prefix `pwd`/.. --distros ${packager_distro} --tarball `pwd`/../mongodb-binaries.tgz -s ${version} -m HEAD -a ${packager_arch} cd .. fi # Create separate shell archive mkdir -p shell-archive/build cd shell-archive ${platform_decompress|tar xzvf} ../mongodb-binaries.tgz find . -mindepth 3 ! -name "mongo${exe}" -type f -exec rm {} \; # delete bin/* except bin/mongo ${python|/opt/mongodbtoolchain/v2/bin/python2} ../buildscripts/make_archive.py -o mongodb-shell.${ext|tgz} $(find mongodb-* -type f) cd .. - func: "fetch test_lifecycle.yml" vars: # Do not fail the task if we fail to fetch the test_lifecycle.yml file fail_task_on_error: false - command: archive.targz_pack params: target: "artifacts.tgz" source_dir: "src" include: - "src/mongo/db/modules/enterprise/jstests/**" - "compile_expansions.yml" - "src/mongo/db/modules/subscription/jstests/**" - "src/mongo/db/modules/enterprise/docs/**" - "*.exe" - "jstests/**" - "pytests/**" - "./test*" - "./dbtest*" - "./mongobridge*" - "buildscripts/**" - "*Example" - "*Test" - "./**.pdb" - "./**.msi" - "./etc/*san.suppressions" - "./etc/repo_config.yaml" - "./etc/test_lifecycle.yml" - "./etc/test_retrial.yml" - "./build/integration_tests/**" - "./build/integration_tests.txt" - "./build/**.gcno" - "repo/**" - "src/mongo/util/options_parser/test_config_files/**" - "library_dependency_graph.json" - "src/third_party/JSON-Schema-Test-Suite/tests/draft4/**" exclude_files: - "*_test.pdb" - func: "upload debugsymbols" - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: src/mongodb-binaries.tgz remote_file: mongodb-mongo-master/${build_variant}/${revision}/binaries/mongo-${build_id}.${ext|tgz} bucket: mciuploads permissions: public-read content_type: ${content_type|application/x-gzip} display_name: Binaries - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: src/shell-archive/mongodb-shell.${ext|tgz} remote_file: mongodb-mongo-master/${build_variant}/${revision}/binaries/mongo-shell-${build_id}.${ext|tgz} bucket: mciuploads permissions: public-read content_type: ${content_type|application/x-gzip} display_name: Shell - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: artifacts.tgz remote_file: mongodb-mongo-master/${build_variant}/${revision}/artifacts/${build_id}.tgz bucket: mciuploads permissions: public-read content_type: application/tar display_name: Artifacts - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: src/distsrc.${ext|tgz} remote_file: mongodb-mongo-master/${build_variant}/${revision}/sources/mongo-src-${build_id}.${ext|tgz} bucket: mciuploads permissions: public-read content_type: ${content_type|application/x-gzip} display_name: Source tarball # We only need to upload the source tarball from one of the build variants # because it should be the same everywhere, so just use linux-64/windows-64-2k8. build_variants: [ linux-64, windows-64-2k8-ssl ] ## compile_all - build all scons targets including unittests ## - name: compile_all commands: - command: manifest.load - *git_get_project - func: "get buildnumber" - func: "setup credentials" - func: "build new tools" # noop if ${newtools} is not "true" - func: "build rocksdb" # noop if ${build_rocksdb} is not "true" - func: "build cdriver" # noop if ${build_cdriver} is not "true" - *generate_compile_expansions # Then we load the generated version data into the agent so we can use it in task definitions. - *apply_compile_expansions - command: shell.exec type: test params: working_dir: src script: | set -o errexit set -o verbose rm -rf ${install_directory|/data/mongo-install-directory} extra_args="" if [ -n "${num_scons_compile_all_jobs_available|}" ]; then echo "Changing SCons to run with --jobs=${num_scons_compile_all_jobs_available|}" extra_args="$extra_args --jobs=${num_scons_compile_all_jobs_available|}" fi ${compile_env|} ${python|/opt/mongodbtoolchain/v2/bin/python2} ./buildscripts/scons.py ${compile_flags|} ${scons_cache_args|} --use-new-tools --build-mongoreplay="${build_mongoreplay}" $extra_args all ${additional_targets|} MONGO_VERSION=${version} - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: src/library_dependency_graph.json remote_file: mongodb-mongo-master/${build_variant}/${revision}/library_dependency_graph.${build_id}.json bucket: mciuploads permissions: public-read content_type: application/json display_name: Library Dependency Graph (library_dependency_graph.json) build_variants: [enterprise-ubuntu-dynamic-1604-64-bit] # Run the C++ unittests as part of compile_all. The compiled binaries are automatically # installed into the top-level directory by SCons. # noop if ${disable_unit_tests} is "true" - func: "run tests" vars: resmoke_args: --suites=unittests run_multiple_jobs: true # Run the iOS embedded CAPI tests in the iOS simulator # noop if ${run_ios_embedded_tests} is not "true" - func: "run ios sim embedded tests" ## lint ## - name: lint depends_on: [] commands: - command: manifest.load - *git_get_project - command: shell.exec type: test params: working_dir: src script: | set -o errexit set -o verbose ${compile_env|} ${python|/opt/mongodbtoolchain/v2/bin/python2} ./buildscripts/scons.py ${compile_flags|} --stack-size=1024 lint - <<: *task_template name: burn_in_tests depends_on: - name: compile commands: - *git_get_project - command: shell.exec params: working_dir: src script: | # Remove the empty directory path left over by the git.get_project command as it can cause # a name conflict when copying the binaries over during 'do setup'. See EVG-1954. rmdir -p mongo/db/modules/* 2>/dev/null || true - func: "do setup" - command: shell.exec params: working_dir: src script: | ${activate_virtualenv} # If this is a scheduled build, we check for changes against the last scheduled commit. if [ "${is_patch}" != "true" ]; then burn_in_args="--checkEvergreen" fi # Capture a list of new and modified tests. $python buildscripts/burn_in_tests.py --branch=${branch_name} --buildVariant=${build_variant} --testListOutfile=jstests/new_tests.json --noExec $burn_in_args - func: "do multiversion setup" - func: "run tests" vars: task_path_suffix: /data/multiversion resmoke_wrapper: $python buildscripts/burn_in_tests.py --testListFile=jstests/new_tests.json resmoke_args: --repeat=2 run_multiple_jobs: true - <<: *run_jepsen_template name: jepsen_register_findAndModify_WT commands: - func: "do setup" - func: "do jepsen setup" - func: "run jepsen test" vars: <<: *jepsen_config_vars jepsen_read_with_find_and_modify: --read-with-find-and-modify jepsen_storage_engine: --storage-engine wiredTiger jepsen_test_name: register - <<: *run_jepsen_template name: jepsen_register_findAndModify_pv0 commands: - func: "do setup" - func: "do jepsen setup" - func: "run jepsen test" vars: <<: *jepsen_config_vars jepsen_protocol_version: --protocol-version 0 jepsen_read_with_find_and_modify: --read-with-find-and-modify jepsen_storage_engine: --storage-engine mmapv1 jepsen_test_name: register - <<: *run_jepsen_template name: jepsen_register_findAndModify_pv0_WT commands: - func: "do setup" - func: "do jepsen setup" - func: "run jepsen test" vars: <<: *jepsen_config_vars jepsen_protocol_version: --protocol-version 0 jepsen_read_with_find_and_modify: --read-with-find-and-modify jepsen_storage_engine: --storage-engine wiredTiger jepsen_test_name: register - <<: *run_jepsen_template name: jepsen_register_linearizableRead commands: - func: "do setup" - func: "do jepsen setup" - func: "run jepsen test" vars: <<: *jepsen_config_vars jepsen_read_concern: --read-concern linearizable jepsen_storage_engine: --storage-engine mmapv1 jepsen_test_name: register # Running Jepsen when using the MMAPv1 storage engine for longer periods of time can lead to # out of memory issues and task timeouts during the analysis phase. jepsen_time_limit: --time-limit 300 - <<: *run_jepsen_template name: jepsen_register_linearizableRead_WT commands: - func: "do setup" - func: "do jepsen setup" - func: "run jepsen test" vars: <<: *jepsen_config_vars jepsen_read_concern: --read-concern linearizable jepsen_storage_engine: --storage-engine wiredTiger jepsen_test_name: register - <<: *run_jepsen_template name: jepsen_set_linearizableRead commands: - func: "do setup" - func: "do jepsen setup" - func: "run jepsen test" vars: <<: *jepsen_config_vars jepsen_read_concern: --read-concern linearizable jepsen_storage_engine: --storage-engine mmapv1 jepsen_test_name: set # Running Jepsen when using the MMAPv1 storage engine for longer periods of time can lead to # out of memory issues and task timeouts during the analysis phase. jepsen_time_limit: --time-limit 300 - <<: *run_jepsen_template name: jepsen_set_linearizableRead_WT commands: - func: "do setup" - func: "do jepsen setup" - func: "run jepsen test" vars: <<: *jepsen_config_vars jepsen_read_concern: --read-concern linearizable jepsen_storage_engine: --storage-engine wiredTiger jepsen_test_name: set - <<: *run_jepsen_template # We only run this test on wiredTiger since MMAPv1 doesn't support majority read concern. name: jepsen_read-concern-majority_WT commands: - func: "do setup" - func: "do jepsen setup" - func: "run jepsen test" vars: <<: *jepsen_config_vars jepsen_storage_engine: --storage-engine wiredTiger jepsen_test_name: read-concern-majority - <<: *run_jepsen_template # We only run this test on wiredTiger since MMAPv1 doesn't support majority read concern. name: jepsen_read-concern-majority_w1_WT commands: - func: "do setup" - func: "do jepsen setup" - func: "run jepsen test" vars: <<: *jepsen_config_vars jepsen_storage_engine: --storage-engine wiredTiger jepsen_test_name: read-concern-majority jepsen_write_concern: --write-concern w1 ## jstestfuzz standalone generational fuzzer mmapv1 ## - <<: *jstestfuzz_template name: generational_jstestfuzz commands: - func: "do setup" - func: "run jstestfuzz" vars: jstestfuzz_vars: -g -f 50 - <<: *run_jstestfuzz_tests vars: <<: *jstestfuzz_config_vars resmoke_args: --suites=generational_jstestfuzz --storageEngine=mmapv1 ## jstestfuzz standalone generational fuzzer WT ## - <<: *jstestfuzz_template name: generational_jstestfuzz_WT commands: - func: "do setup" - func: "run jstestfuzz" vars: jstestfuzz_vars: -g -f 50 - <<: *run_jstestfuzz_tests vars: <<: *jstestfuzz_config_vars resmoke_args: --suites=generational_jstestfuzz --storageEngine=wiredTiger ## rollback fuzzer ## - <<: *jstestfuzz_template name: rollback_fuzzer commands: - func: "do setup" - func: "run jstestfuzz" vars: jstestfuzz_vars: --numGeneratedFiles 15 npm_command: rollback-fuzzer - <<: *run_jstestfuzz_tests vars: <<: *jstestfuzz_config_vars resmoke_args: --suites=rollback_fuzzer --storageEngine=mmapv1 ## rollback fuzzer WT ## - <<: *jstestfuzz_template name: rollback_fuzzer_WT commands: - func: "do setup" - func: "run jstestfuzz" vars: jstestfuzz_vars: --numGeneratedFiles 15 npm_command: rollback-fuzzer - <<: *run_jstestfuzz_tests vars: <<: *jstestfuzz_config_vars resmoke_args: --suites=rollback_fuzzer --storageEngine=wiredTiger ## jstestfuzz mmapv1 ## - <<: *jstestfuzz_template name: jstestfuzz commands: - func: "do setup" - func: "run jstestfuzz" - <<: *run_jstestfuzz_tests vars: <<: *jstestfuzz_config_vars resmoke_args: --suites=jstestfuzz --storageEngine=mmapv1 ## jstestfuzz WT ## - <<: *jstestfuzz_template name: jstestfuzz_WT commands: - func: "do setup" - func: "run jstestfuzz" - <<: *run_jstestfuzz_tests vars: <<: *jstestfuzz_config_vars resmoke_args: --suites=jstestfuzz --storageEngine=wiredTiger ## jstestfuzz concurrent mmapv1 ## - <<: *jstestfuzz_template name: jstestfuzz_concurrent commands: - func: "do setup" - func: "run jstestfuzz" vars: jstestfuzz_vars: --numGeneratedFiles 75 - <<: *run_jstestfuzz_tests vars: <<: *jstestfuzz_config_vars resmoke_args: --suites=jstestfuzz --storageEngine=mmapv1 --numClientsPerFixture=10 ## jstestfuzz concurrent WT ## - <<: *jstestfuzz_template name: jstestfuzz_concurrent_WT commands: - func: "do setup" - func: "run jstestfuzz" vars: jstestfuzz_vars: --numGeneratedFiles 75 - <<: *run_jstestfuzz_tests vars: <<: *jstestfuzz_config_vars resmoke_args: --suites=jstestfuzz --storageEngine=wiredTiger --numClientsPerFixture=10 ## jstestfuzz concurrent replica set mmapv1 ## - <<: *jstestfuzz_template name: jstestfuzz_concurrent_replication commands: - func: "do setup" - func: "run jstestfuzz" vars: jstestfuzz_vars: --numGeneratedFiles 75 - <<: *run_jstestfuzz_tests vars: <<: *jstestfuzz_config_vars resmoke_args: --suites=jstestfuzz_replication --storageEngine=mmapv1 --numClientsPerFixture=10 ## jstestfuzz concurrent replica set WT ## - <<: *jstestfuzz_template name: jstestfuzz_concurrent_replication_WT commands: - func: "do setup" - func: "run jstestfuzz" vars: jstestfuzz_vars: --numGeneratedFiles 75 - <<: *run_jstestfuzz_tests vars: <<: *jstestfuzz_config_vars resmoke_args: --suites=jstestfuzz_replication --storageEngine=wiredTiger --numClientsPerFixture=10 ## jstestfuzz concurrent replica set with logical session mmapv1 ## - <<: *jstestfuzz_template name: jstestfuzz_concurrent_replication_session commands: - func: "do setup" - func: "run jstestfuzz" vars: jstestfuzz_vars: --numGeneratedFiles 75 - <<: *run_jstestfuzz_tests vars: <<: *jstestfuzz_config_vars resmoke_args: --suites=jstestfuzz_replication_session --storageEngine=mmapv1 --numClientsPerFixture=10 ## jstestfuzz concurrent replica set with logical session WT ## - <<: *jstestfuzz_template name: jstestfuzz_concurrent_replication_session_WT commands: - func: "do setup" - func: "run jstestfuzz" vars: jstestfuzz_vars: --numGeneratedFiles 75 - <<: *run_jstestfuzz_tests vars: <<: *jstestfuzz_config_vars resmoke_args: --suites=jstestfuzz_replication_session --storageEngine=wiredTiger --numClientsPerFixture=10 ## jstestfuzz concurrent sharded cluster mmapv1 ## - <<: *jstestfuzz_template name: jstestfuzz_concurrent_sharded commands: - func: "do setup" - func: "run jstestfuzz" vars: jstestfuzz_vars: --numGeneratedFiles 75 - <<: *run_jstestfuzz_tests vars: <<: *jstestfuzz_config_vars resmoke_args: --suites=jstestfuzz_sharded --storageEngine=mmapv1 --numClientsPerFixture=10 ## jstestfuzz concurrent sharded cluster WT ## - <<: *jstestfuzz_template name: jstestfuzz_concurrent_sharded_WT commands: - func: "do setup" - func: "run jstestfuzz" vars: jstestfuzz_vars: --numGeneratedFiles 75 - <<: *run_jstestfuzz_tests vars: <<: *jstestfuzz_config_vars resmoke_args: --suites=jstestfuzz_sharded --storageEngine=wiredTiger --numClientsPerFixture=10 ## jstestfuzz concurrent sharded cluster causal consistency mmapv1 ## - <<: *jstestfuzz_template name: jstestfuzz_concurrent_sharded_causal_consistency commands: - func: "do setup" - func: "run jstestfuzz" vars: jstestfuzz_vars: --numGeneratedFiles 75 - <<: *run_jstestfuzz_tests vars: <<: *jstestfuzz_config_vars resmoke_args: --suites=jstestfuzz_sharded_causal_consistency --storageEngine=mmapv1 --numClientsPerFixture=10 ## jstestfuzz concurrent sharded cluster causal consistency WT ## - <<: *jstestfuzz_template name: jstestfuzz_concurrent_sharded_causal_consistency_WT commands: - func: "do setup" - func: "run jstestfuzz" vars: jstestfuzz_vars: --numGeneratedFiles 75 - <<: *run_jstestfuzz_tests vars: <<: *jstestfuzz_config_vars resmoke_args: --suites=jstestfuzz_sharded_causal_consistency --storageEngine=wiredTiger --numClientsPerFixture=10 ## jstestfuzz concurrent sharded cluster with logical session mmapv1 ## - <<: *jstestfuzz_template name: jstestfuzz_concurrent_sharded_session commands: - func: "do setup" - func: "run jstestfuzz" vars: jstestfuzz_vars: --numGeneratedFiles 75 - <<: *run_jstestfuzz_tests vars: <<: *jstestfuzz_config_vars resmoke_args: --suites=jstestfuzz_sharded_session --storageEngine=mmapv1 --numClientsPerFixture=10 ## jstestfuzz concurrent sharded cluster with logical session WT ## - <<: *jstestfuzz_template name: jstestfuzz_concurrent_sharded_session_WT commands: - func: "do setup" - func: "run jstestfuzz" vars: jstestfuzz_vars: --numGeneratedFiles 75 - <<: *run_jstestfuzz_tests vars: <<: *jstestfuzz_config_vars resmoke_args: --suites=jstestfuzz_sharded_session --storageEngine=wiredTiger --numClientsPerFixture=10 ## jstestfuzz replica set mmapv1 ## - <<: *jstestfuzz_template name: jstestfuzz_replication commands: - func: "do setup" - func: "run jstestfuzz" - <<: *run_jstestfuzz_tests vars: <<: *jstestfuzz_config_vars resmoke_args: --suites=jstestfuzz_replication --storageEngine=mmapv1 ## jstestfuzz replica set WT ## - <<: *jstestfuzz_template name: jstestfuzz_replication_WT commands: - func: "do setup" - func: "run jstestfuzz" - <<: *run_jstestfuzz_tests vars: <<: *jstestfuzz_config_vars resmoke_args: --suites=jstestfuzz_replication --storageEngine=wiredTiger ## jstestfuzz initial sync replica set mmapv1 ## - <<: *jstestfuzz_template name: jstestfuzz_replication_initsync commands: - func: "do setup" - func: "run jstestfuzz" vars: jstestfuzz_vars: --numGeneratedFiles 75 - <<: *run_jstestfuzz_tests vars: <<: *jstestfuzz_config_vars resmoke_args: --suites=jstestfuzz_replication_initsync --storageEngine=mmapv1 ## jstestfuzz initial sync replica set WT ## - <<: *jstestfuzz_template name: jstestfuzz_replication_initsync_WT commands: - func: "do setup" - func: "run jstestfuzz" vars: jstestfuzz_vars: --numGeneratedFiles 75 - <<: *run_jstestfuzz_tests vars: <<: *jstestfuzz_config_vars resmoke_args: --suites=jstestfuzz_replication_initsync --storageEngine=wiredTiger ## jstestfuzz initial sync replica set mmapv1 ## - <<: *jstestfuzz_template name: jstestfuzz_replication_resync commands: - func: "do setup" - func: "run jstestfuzz" vars: jstestfuzz_vars: --numGeneratedFiles 75 - <<: *run_jstestfuzz_tests vars: <<: *jstestfuzz_config_vars resmoke_args: --suites=jstestfuzz_replication_resync --storageEngine=mmapv1 ## jstestfuzz initial sync replica set WT ## - <<: *jstestfuzz_template name: jstestfuzz_replication_resync_WT commands: - func: "do setup" - func: "run jstestfuzz" vars: jstestfuzz_vars: --numGeneratedFiles 75 - <<: *run_jstestfuzz_tests vars: <<: *jstestfuzz_config_vars resmoke_args: --suites=jstestfuzz_replication_resync --storageEngine=wiredTiger ## jstestfuzz replica set with logical session mmapv1 ## - <<: *jstestfuzz_template name: jstestfuzz_replication_session commands: - func: "do setup" - func: "run jstestfuzz" - <<: *run_jstestfuzz_tests vars: <<: *jstestfuzz_config_vars resmoke_args: --suites=jstestfuzz_replication_session --storageEngine=mmapv1 ## jstestfuzz replica set with logical session WT ## - <<: *jstestfuzz_template name: jstestfuzz_replication_session_WT commands: - func: "do setup" - func: "run jstestfuzz" - <<: *run_jstestfuzz_tests vars: <<: *jstestfuzz_config_vars resmoke_args: --suites=jstestfuzz_replication_session --storageEngine=wiredTiger ## jstestfuzz sharded cluster mmapv1 ## - <<: *jstestfuzz_template name: jstestfuzz_sharded commands: - func: "do setup" - func: "run jstestfuzz" - <<: *run_jstestfuzz_tests vars: <<: *jstestfuzz_config_vars resmoke_args: --suites=jstestfuzz_sharded --storageEngine=mmapv1 ## jstestfuzz sharded cluster WT ## - <<: *jstestfuzz_template name: jstestfuzz_sharded_WT commands: - func: "do setup" - func: "run jstestfuzz" - <<: *run_jstestfuzz_tests vars: <<: *jstestfuzz_config_vars resmoke_args: --suites=jstestfuzz_sharded --storageEngine=wiredTiger ## jstestfuzz sharded cluster causal consistency mmapv1 ## - <<: *jstestfuzz_template name: jstestfuzz_sharded_causal_consistency commands: - func: "do setup" - func: "run jstestfuzz" - <<: *run_jstestfuzz_tests vars: <<: *jstestfuzz_config_vars resmoke_args: --suites=jstestfuzz_sharded_causal_consistency --storageEngine=mmapv1 ## jstestfuzz sharded cluster causal consistency WT ## - <<: *jstestfuzz_template name: jstestfuzz_sharded_causal_consistency_WT commands: - func: "do setup" - func: "run jstestfuzz" - <<: *run_jstestfuzz_tests vars: <<: *jstestfuzz_config_vars resmoke_args: --suites=jstestfuzz_sharded_causal_consistency --storageEngine=wiredTiger ## jstestfuzz sharded cluster with logical session mmapv1 ## - <<: *jstestfuzz_template name: jstestfuzz_sharded_session commands: - func: "do setup" - func: "run jstestfuzz" - <<: *run_jstestfuzz_tests vars: <<: *jstestfuzz_config_vars resmoke_args: --suites=jstestfuzz_sharded_session --storageEngine=mmapv1 ## jstestfuzz sharded cluster with logical session WT ## - <<: *jstestfuzz_template name: jstestfuzz_sharded_session_WT commands: - func: "do setup" - func: "run jstestfuzz" - <<: *run_jstestfuzz_tests vars: <<: *jstestfuzz_config_vars resmoke_args: --suites=jstestfuzz_sharded_session --storageEngine=wiredTiger ## integration test suites ## - <<: *task_template name: aggregation commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=aggregation --storageEngine=mmapv1 run_multiple_jobs: true - <<: *task_template name: aggregation_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=aggregation --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: aggregation_WT_ese depends_on: - name: jsCore_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=aggregation_ese --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: aggregation_auth depends_on: - name: aggregation_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=aggregation_auth run_multiple_jobs: true - <<: *task_template name: aggregation_facet_unwind_passthrough_WT depends_on: - name: aggregation_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=aggregation_facet_unwind_passthrough --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: aggregation_read_concern_majority_passthrough_WT depends_on: - name: aggregation_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=aggregation_read_concern_majority_passthrough --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: aggregation_sharded_collections_passthrough_WT depends_on: - name: aggregation_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=aggregation_sharded_collections_passthrough --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: audit commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=audit --storageEngine=mmapv1 run_multiple_jobs: true - <<: *task_template name: audit_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=audit --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: auth commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=auth --storageEngine=mmapv1 run_multiple_jobs: true - <<: *task_template name: auth_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=auth --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: auth_audit commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=auth_audit --storageEngine=mmapv1 run_multiple_jobs: true - <<: *task_template name: auth_audit_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=auth_audit --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: change_streams_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=change_streams --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: change_streams_mongos_passthrough_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=change_streams_mongos_passthrough --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: dbtest commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=dbtest --storageEngine=mmapv1 run_multiple_jobs: true - <<: *task_template name: dbtest_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=dbtest --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: disk commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=disk_mmapv1 run_multiple_jobs: false - <<: *task_template name: disk_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=disk_wiredtiger --excludeWithAnyTags=requires_mmapv1 run_multiple_jobs: false - <<: *task_template name: durability commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=durability run_multiple_jobs: true - <<: *task_template name: dur_jscore_passthrough depends_on: - name: jsCore commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=dur_jscore_passthrough run_multiple_jobs: true - <<: *task_template name: ese_WT commands: - func: "do setup" - func: "set up virtualenv" - command: shell.exec type: test params: working_dir: src script: | set -o errexit set -o verbose ${activate_virtualenv} # we should go back to using pip when it works on SLES easy_install --upgrade `cat src/mongo/db/modules/enterprise/jstests/encryptdb/libs/requirements.txt` - func: "run tests" vars: resmoke_args: --suites=ese --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: failpoints commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=failpoints run_multiple_jobs: true - <<: *task_template name: failpoints_auth commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=failpoints_auth run_multiple_jobs: true - <<: *task_template name: gle_auth commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=gle_auth --shellWriteMode=legacy --shellReadMode=legacy --storageEngine=mmapv1 --excludeWithAnyTags=requires_find_command run_multiple_jobs: true - <<: *task_template name: gle_auth_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=gle_auth --shellWriteMode=legacy --shellReadMode=legacy --storageEngine=wiredTiger --excludeWithAnyTags=requires_find_command run_multiple_jobs: true - <<: *task_template name: gle_auth_write_cmd commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=gle_auth --shellWriteMode=commands --storageEngine=mmapv1 run_multiple_jobs: true - <<: *task_template name: gle_auth_write_cmd_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=gle_auth --shellWriteMode=commands --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: gle_auth_basics_passthrough commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=gle_auth_basics_passthrough --shellWriteMode=legacy --shellReadMode=legacy --storageEngine=mmapv1 --excludeWithAnyTags=requires_find_command run_multiple_jobs: true - <<: *task_template name: gle_auth_basics_passthrough_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=gle_auth_basics_passthrough --shellWriteMode=legacy --shellReadMode=legacy --storageEngine=wiredTiger --excludeWithAnyTags=requires_find_command run_multiple_jobs: true - <<: *task_template name: gle_auth_basics_passthrough_write_cmd commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=gle_auth_basics_passthrough --shellWriteMode=commands --storageEngine=mmapv1 run_multiple_jobs: true - <<: *task_template name: gle_auth_basics_passthrough_write_cmd_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=gle_auth_basics_passthrough --shellWriteMode=commands --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: integration_tests_standalone commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=integration_tests_standalone run_multiple_jobs: true - <<: *task_template name: integration_tests_replset commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=integration_tests_replset run_multiple_jobs: true - <<: *task_template name: integration_tests_sharded commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=integration_tests_sharded run_multiple_jobs: true - <<: *task_template name: external_auth_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=external_auth --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: sharding_gle_auth_basics_passthrough depends_on: - name: gle_auth_basics_passthrough commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=sharding_gle_auth_basics_passthrough --shellWriteMode=legacy --shellReadMode=legacy --storageEngine=mmapv1 --excludeWithAnyTags=requires_find_command run_multiple_jobs: true - <<: *task_template name: sharding_gle_auth_basics_passthrough_WT depends_on: - name: gle_auth_basics_passthrough_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=sharding_gle_auth_basics_passthrough --shellWriteMode=legacy --shellReadMode=legacy --storageEngine=wiredTiger --excludeWithAnyTags=requires_find_command run_multiple_jobs: true - <<: *task_template name: sharding_gle_auth_basics_passthrough_write_cmd depends_on: - name: gle_auth_basics_passthrough_write_cmd commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=sharding_gle_auth_basics_passthrough --shellWriteMode=commands --storageEngine=mmapv1 run_multiple_jobs: true - <<: *task_template name: sharding_gle_auth_basics_passthrough_write_cmd_WT depends_on: - name: gle_auth_basics_passthrough_write_cmd_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=sharding_gle_auth_basics_passthrough --shellWriteMode=commands --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: jsCore commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=core --storageEngine=mmapv1 run_multiple_jobs: true - <<: *task_template name: jsCore_inMem commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=core --storageEngine=inMemory run_multiple_jobs: true - <<: *task_template name: jsCore_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=core --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: jsCore_WT_ese depends_on: - name: jsCore_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=core_ese --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: jsCore_compatibility depends_on: - name: jsCore commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=core --shellReadMode=legacy --shellWriteMode=compatibility --storageEngine=mmapv1 --excludeWithAnyTags=requires_find_command run_multiple_jobs: true - <<: *task_template name: jsCore_compatibility_WT depends_on: - name: jsCore_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=core --shellReadMode=legacy --shellWriteMode=compatibility --storageEngine=wiredTiger --excludeWithAnyTags=requires_find_command run_multiple_jobs: true - <<: *task_template name: jsCore_auth depends_on: - name: jsCore commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=core_auth run_multiple_jobs: true - <<: *task_template name: jsCore_minimum_batch_size_WT depends_on: - name: jsCore commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=core_minimum_batch_size --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: jsCore_op_query depends_on: - name: jsCore commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=core_op_query --storageEngine=mmapv1 run_multiple_jobs: true - <<: *task_template name: jsCore_op_query_WT depends_on: - name: jsCore_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=core_op_query --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: master_slave_jscore_passthrough depends_on: - name: jsCore commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=master_slave_jscore_passthrough --storageEngine=mmapv1 run_multiple_jobs: true - <<: *task_template name: master_slave_jscore_passthrough_WT depends_on: - name: jsCore_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=master_slave_jscore_passthrough --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: replica_sets_jscore_passthrough depends_on: - name: jsCore commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=replica_sets_jscore_passthrough --storageEngine=mmapv1 run_multiple_jobs: true - <<: *task_template name: replica_sets_jscore_passthrough_WT depends_on: - name: jsCore_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=replica_sets_jscore_passthrough --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: replica_sets_initsync_jscore_passthrough depends_on: - name: jsCore commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=replica_sets_initsync_jscore_passthrough --storageEngine=mmapv1 run_multiple_jobs: true - <<: *task_template name: replica_sets_initsync_jscore_passthrough_WT depends_on: - name: jsCore_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=replica_sets_initsync_jscore_passthrough --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: replica_sets_resync_jscore_passthrough depends_on: - name: jsCore commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=replica_sets_resync_jscore_passthrough --storageEngine=mmapv1 run_multiple_jobs: true - <<: *task_template name: replica_sets_resync_jscore_passthrough_WT depends_on: - name: jsCore_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=replica_sets_resync_jscore_passthrough --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: replica_sets_initsync_static_jscore_passthrough depends_on: - name: jsCore commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=replica_sets_initsync_static_jscore_passthrough --storageEngine=mmapv1 run_multiple_jobs: true - <<: *task_template name: replica_sets_initsync_static_jscore_passthrough_WT depends_on: - name: jsCore_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=replica_sets_initsync_static_jscore_passthrough --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: replica_sets_resync_static_jscore_passthrough depends_on: - name: jsCore commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=replica_sets_resync_static_jscore_passthrough --storageEngine=mmapv1 run_multiple_jobs: true - <<: *task_template name: replica_sets_resync_static_jscore_passthrough_WT depends_on: - name: jsCore_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=replica_sets_resync_static_jscore_passthrough --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: replica_sets_kill_secondaries_jscore_passthrough_WT depends_on: - name: jsCore_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=replica_sets_kill_secondaries_jscore_passthrough --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: mongosTest commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=mongos_test run_multiple_jobs: true - <<: *task_template name: mmap commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=mmap run_multiple_jobs: true - <<: *task_template name: multiversion commands: - func: "do setup" - func: "set up virtualenv" - func: "do multiversion setup" - func: "run tests" vars: task_path_suffix: /data/multiversion resmoke_args: --suites=multiversion --storageEngine=mmapv1 --excludeWithAnyTags=requires_wiredtiger run_multiple_jobs: true - <<: *task_template name: multiversion_auth commands: - func: "do setup" - func: "set up virtualenv" - func: "do multiversion setup" - func: "run tests" vars: task_path_suffix: /data/multiversion resmoke_args: --suites=multiversion_auth --storageEngine=mmapv1 --excludeWithAnyTags=requires_wiredtiger run_multiple_jobs: true - <<: *task_template name: multiversion_WT commands: - func: "do setup" - func: "set up virtualenv" - func: "do multiversion setup" - func: "run tests" vars: task_path_suffix: /data/multiversion resmoke_args: --suites=multiversion --storageEngine=wiredTiger --excludeWithAnyTags=requires_mmapv1 run_multiple_jobs: true - <<: *task_template name: multiversion_multistorage_engine commands: - func: "do setup" - func: "set up virtualenv" - func: "do multiversion setup" - func: "run tests" vars: task_path_suffix: /data/multiversion resmoke_args: --suites=multiversion_multistorage_engine run_multiple_jobs: true - <<: *task_template name: noPassthrough commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=no_passthrough --storageEngine=mmapv1 run_multiple_jobs: true - <<: *task_template name: noPassthrough_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=no_passthrough --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: noPassthroughWithMongod commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=no_passthrough_with_mongod --storageEngine=mmapv1 run_multiple_jobs: true - <<: *task_template name: noPassthroughWithMongod_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=no_passthrough_with_mongod --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: bulk_gle_passthrough commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=bulk_gle_passthrough --storageEngine=mmapv1 run_multiple_jobs: true - <<: *task_template name: bulk_gle_passthrough_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=bulk_gle_passthrough --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: slow1 commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=slow1 --storageEngine=mmapv1 run_multiple_jobs: false - <<: *task_template name: slow1_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=slow1 --storageEngine=wiredTiger run_multiple_jobs: false - <<: *task_template name: serial_run commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=serial_run --storageEngine=mmapv1 run_multiple_jobs: false - <<: *task_template name: serial_run_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=serial_run --storageEngine=wiredTiger run_multiple_jobs: false - <<: *task_template name: sharded_collections_jscore_passthrough depends_on: - name: jsCore commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=sharded_collections_jscore_passthrough --storageEngine=mmapv1 run_multiple_jobs: true - <<: *task_template name: sharded_collections_jscore_passthrough_WT depends_on: - name: jsCore_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=sharded_collections_jscore_passthrough --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: sharding_jscore_passthrough depends_on: - name: jsCore commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=sharding_jscore_passthrough --storageEngine=mmapv1 run_multiple_jobs: true - <<: *task_template name: sharding_jscore_passthrough_WT depends_on: - name: jsCore_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=sharding_jscore_passthrough --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: sharding_jscore_op_query_passthrough_WT depends_on: - name: jsCore_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=sharding_jscore_op_query_passthrough --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: sharding_jscore_passthrough_wire_ops_WT depends_on: - name: jsCore_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=sharding_jscore_passthrough --storageEngine=wiredTiger --shellReadMode=legacy --shellWriteMode=compatibility --excludeWithAnyTags=requires_find_command run_multiple_jobs: true - <<: *task_template name: parallel depends_on: - name: jsCore commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=parallel --storageEngine=mmapv1 - <<: *task_template name: parallel_WT depends_on: - name: jsCore_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=parallel --storageEngine=wiredTiger - <<: *task_template name: parallel_compatibility depends_on: - name: jsCore_compatibility commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=parallel --shellReadMode=legacy --shellWriteMode=compatibility --storageEngine=mmapv1 --excludeWithAnyTags=requires_find_command - <<: *task_template name: parallel_compatibility_WT depends_on: - name: jsCore_compatibility_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=parallel --shellReadMode=legacy --shellWriteMode=compatibility --storageEngine=wiredTiger --excludeWithAnyTags=requires_find_command - <<: *task_template name: concurrency commands: - func: "do setup" - func: "run tests" timeout_secs: 21600 # 6 hour timeout vars: resmoke_args: --suites=concurrency --storageEngine=mmapv1 - <<: *task_template name: concurrency_WT commands: - func: "do setup" - func: "run tests" timeout_secs: 21600 # 6 hour timeout vars: resmoke_args: --suites=concurrency --storageEngine=wiredTiger - <<: *task_template name: concurrency_replication commands: - func: "do setup" - func: "run tests" timeout_secs: 21600 # 6 hour timeout vars: resmoke_args: --suites=concurrency_replication --storageEngine=mmapv1 - <<: *task_template name: concurrency_replication_WT commands: - func: "do setup" - func: "run tests" timeout_secs: 21600 # 6 hour timeout vars: resmoke_args: --suites=concurrency_replication --storageEngine=wiredTiger - <<: *task_template name: concurrency_sharded exec_timeout_secs: 43200 # 12 hour timeout for the task overall commands: - func: "do setup" - func: "run tests" timeout_secs: 21600 # 6 hour timeout for each test vars: resmoke_args: --suites=concurrency_sharded --storageEngine=mmapv1 - <<: *task_template name: concurrency_sharded_WT exec_timeout_secs: 43200 # 12 hour timeout for the task overall commands: - func: "do setup" - func: "run tests" timeout_secs: 21600 # 6 hour timeout for each test vars: resmoke_args: --suites=concurrency_sharded --storageEngine=wiredTiger - <<: *task_template name: concurrency_sharded_causal_consistency commands: - func: "do setup" - func: "run tests" timeout_secs: 21600 # 6 hour timeout for each test vars: resmoke_args: --suites=concurrency_sharded_causal_consistency --storageEngine=mmapv1 - <<: *task_template name: concurrency_sharded_causal_consistency_WT commands: - func: "do setup" - func: "run tests" timeout_secs: 21600 # 6 hour timeout for each test vars: resmoke_args: --suites=concurrency_sharded_causal_consistency --storageEngine=wiredTiger - <<: *task_template name: concurrency_sharded_causal_consistency_and_balancer commands: - func: "do setup" - func: "run tests" timeout_secs: 21600 # 6 hour timeout for each test vars: resmoke_args: --suites=concurrency_sharded_causal_consistency_and_balancer --storageEngine=mmapv1 - <<: *task_template name: concurrency_sharded_causal_consistency_and_balancer_WT commands: - func: "do setup" - func: "run tests" timeout_secs: 21600 # 6 hour timeout for each test vars: resmoke_args: --suites=concurrency_sharded_causal_consistency_and_balancer --storageEngine=wiredTiger - <<: *task_template name: concurrency_sharded_with_stepdowns commands: - func: "do setup" - func: "run tests" timeout_secs: 21600 # 6 hour timeout for each test vars: resmoke_args: --suites=concurrency_sharded_with_stepdowns --storageEngine=mmapv1 - <<: *task_template name: concurrency_sharded_with_stepdowns_WT commands: - func: "do setup" - func: "run tests" timeout_secs: 21600 # 6 hour timeout for each test vars: resmoke_args: --suites=concurrency_sharded_with_stepdowns --storageEngine=wiredTiger - <<: *task_template name: concurrency_sharded_with_stepdowns_and_balancer commands: - func: "do setup" - func: "run tests" timeout_secs: 21600 # 6 hour timeout for each test vars: resmoke_args: --suites=concurrency_sharded_with_stepdowns_and_balancer --storageEngine=mmapv1 - <<: *task_template name: concurrency_sharded_with_stepdowns_and_balancer_WT commands: - func: "do setup" - func: "run tests" timeout_secs: 21600 # 6 hour timeout for each test vars: resmoke_args: --suites=concurrency_sharded_with_stepdowns_and_balancer --storageEngine=wiredTiger - <<: *task_template name: concurrency_simultaneous commands: - func: "do setup" - func: "run tests" timeout_secs: 21600 # 6 hour timeout vars: resmoke_args: --suites=concurrency jstests/concurrency/fsm_all_simultaneous.js --storageEngine=mmapv1 - <<: *task_template name: concurrency_simultaneous_WT commands: - func: "do setup" - func: "run tests" timeout_secs: 21600 # 6 hour timeout vars: resmoke_args: --suites=concurrency jstests/concurrency/fsm_all_simultaneous.js --storageEngine=wiredTiger - <<: *task_template name: rlp commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=rlp --storageEngine=mmapv1 run_multiple_jobs: true - <<: *task_template name: rlp_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=rlp --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: read_concern_linearizable_passthrough depends_on: - name: jsCore commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=read_concern_linearizable_passthrough --storageEngine=mmapv1 run_multiple_jobs: true - <<: *task_template name: read_concern_linearizable_passthrough_WT depends_on: - name: jsCore_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=read_concern_linearizable_passthrough --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: read_concern_majority_passthrough_WT depends_on: - name: jsCore_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=read_concern_majority_passthrough --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: replica_sets commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=replica_sets --storageEngine=mmapv1 run_multiple_jobs: true - <<: *task_template name: replica_sets_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=replica_sets --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: replica_sets_WT_ese commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=replica_sets_ese --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: replica_sets_auth commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=replica_sets_auth run_multiple_jobs: true - <<: *task_template name: replica_sets_pv0 commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=replica_sets_pv0 run_multiple_jobs: true - <<: *task_template name: replica_sets_rollback_refetch_no_uuid commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=replica_sets_rollback_refetch_no_uuid run_multiple_jobs: true - <<: *task_template name: master_slave commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=master_slave --storageEngine=mmapv1 run_multiple_jobs: true - <<: *task_template name: master_slave_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=master_slave --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: master_slave_auth depends_on: - name: master_slave_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=master_slave_auth run_multiple_jobs: true - <<: *task_template name: sasl commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=sasl run_multiple_jobs: true - <<: *task_template name: sharding commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=sharding --storageEngine=mmapv1 run_multiple_jobs: true - <<: *task_template name: sharding_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=sharding --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: sharding_csrs_continuous_config_stepdown_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=sharding_continuous_config_stepdown --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: sharding_WT_ese commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=sharding_ese --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: sharding_op_query_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=sharding --shellReadMode=legacy --storageEngine=wiredTiger --excludeWithAnyTags=requires_find_command run_multiple_jobs: true - <<: *task_template name: sharding_auth commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=sharding_auth run_multiple_jobs: true - <<: *task_template name: sharding_auth_audit_WT depends_on: - name: sharding_auth commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=sharding_auth_audit --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: sharding_last_stable_mongos_and_mixed_shards commands: - func: "do setup" - func: "set up virtualenv" - func: "do multiversion setup" - func: "run tests" vars: task_path_suffix: /data/multiversion resmoke_args: --suites=sharding_last_stable_mongos_and_mixed_shards run_multiple_jobs: true - <<: *task_template name: snmp commands: - func: "do setup" - func: "do snmp setup" - func: "run tests" vars: snmp_config_path: SNMPCONFPATH=snmpconf resmoke_args: --suites=snmp --storageEngine=mmapv1 run_multiple_jobs: true - <<: *task_template name: snmp_WT commands: - func: "do setup" - func: "do snmp setup" - func: "run tests" vars: snmp_config_path: SNMPCONFPATH=snmpconf resmoke_args: --suites=snmp --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: ssl commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=ssl run_multiple_jobs: true - <<: *task_template name: sslSpecial commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=ssl_special run_multiple_jobs: true - <<: *task_template name: tool commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=tool --storageEngine=mmapv1 run_multiple_jobs: true - <<: *task_template name: tool_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=tool --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: jsCore_decimal commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=decimal --storageEngine=mmapv1 run_multiple_jobs: true - <<: *task_template name: jsCore_decimal_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=decimal --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: read_only commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=read_only --storageEngine=mmapv1 run_multiple_jobs: true - <<: *task_template name: read_only_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=read_only --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: read_only_sharded commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=read_only_sharded --storageEngine=mmapv1 run_multiple_jobs: true - <<: *task_template name: read_only_sharded_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=read_only_sharded --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: session_jscore_passthrough_WT depends_on: - name: jsCore_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=session_jscore_passthrough --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: causally_consistent_jscore_passthrough_WT depends_on: - name: jsCore_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=causally_consistent_jscore_passthrough --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: causally_consistent_jscore_passthrough_auth_WT depends_on: - name: jsCore_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=causally_consistent_jscore_passthrough_auth --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: sharded_causally_consistent_jscore_passthrough_WT depends_on: - name: jsCore_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=sharded_causally_consistent_jscore_passthrough --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: retryable_writes_jscore_passthrough depends_on: - name: jsCore commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=retryable_writes_jscore_passthrough --storageEngine=mmapv1 run_multiple_jobs: true - <<: *task_template name: retryable_writes_jscore_passthrough_WT depends_on: - name: jsCore_WT commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=retryable_writes_jscore_passthrough --storageEngine=wiredTiger run_multiple_jobs: true - <<: *task_template name: watchdog commands: - func: "do setup" - func: "do watchdog setup" - func: "run tests" vars: resmoke_args: --suites=watchdog --storageEngine=mmapv1 run_multiple_jobs: false - <<: *task_template name: watchdog_WT commands: - func: "do setup" - func: "do watchdog setup" - func: "run tests" vars: resmoke_args: --suites=watchdog --storageEngine=wiredTiger run_multiple_jobs: false - <<: *task_template name: jsonSchema commands: - func: "do setup" - func: "run tests" vars: resmoke_args: --suites=json_schema --storageEngine=wiredTiger run_multiple_jobs: false - name: powercycle depends_on: - name: compile commands: - func: "do setup" - func: "set up remote credentials" vars: <<: *powercycle_remote_credentials - func: "set up virtualenv" - func: "set up EC2 instance" vars: <<: *powercycle_ec2_instance - command: expansions.update <<: *powercycle_expansions - func: "run powercycle test" vars: <<: *powercycle_test mongod_extra_options: --mongodOptions \"--setParameter enableTestCommands=1 --storageEngine mmapv1\" - name: powercycle_WT depends_on: - name: compile commands: - func: "do setup" - func: "set up remote credentials" vars: <<: *powercycle_remote_credentials - func: "set up virtualenv" - func: "set up EC2 instance" vars: <<: *powercycle_ec2_instance - command: expansions.update <<: *powercycle_expansions - func: "run powercycle test" vars: <<: *powercycle_test mongod_extra_options: --mongodOptions \"--setParameter enableTestCommands=1 --storageEngine wiredTiger\" - name: idl_tests depends_on: - name: compile commands: - func: "do setup" - func: "run idl tests" - name: package depends_on: - name: compile commands: - func: "fetch artifacts" - func: "set up remote credentials" vars: private_key_file: ~/.ssh/kitchen.pem private_key_remote: ${kitchen_private_key} aws_key_remote: ${kitchen_aws_key} aws_secret_remote: ${kitchen_aws_secret} - func: "run kitchen" - name: push patchable: false depends_on: - name: "*" stepback: false commands: - func: "fetch artifacts" - func: "fetch binaries" - command: s3.get params: aws_key: ${aws_key} aws_secret: ${aws_secret} remote_file: mongodb-mongo-master/${build_variant}/${revision}/binaries/mongo-shell-${build_id}.${ext|tgz} bucket: mciuploads local_file: src/mongo-shell.tgz - command: s3.get params: aws_key: ${aws_key} aws_secret: ${aws_secret} remote_file: mongodb-mongo-master/${build_variant}/${revision}/sources/mongo-src-${build_id}.${ext|tgz} bucket: mciuploads local_file: src/distsrc.${ext|tgz} build_variants: [ linux-64, windows-64-2k8-ssl ] - *apply_compile_expansions - func: "fetch debugsymbols archive" - func: "set up remote credentials" vars: aws_key_remote: ${repo_aws_key} aws_secret_remote: ${repo_aws_secret} - command: shell.exec params: working_dir: src silent: true script: | set -o errexit cat < notary_env.sh export NOTARY_TOKEN=${signing_auth_token_36} EOF echo "${signing_auth_token_36}" > signing_auth_token - command: shell.exec params: working_dir: src script: | . ./notary_env.sh set -o errexit set -o verbose mv mongo-binaries.tgz mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz} mv mongo-shell.tgz mongodb-shell-${push_name}-${push_arch}-${suffix}.${ext|tgz} mv mongo-debugsymbols.tgz mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz} || true mv distsrc.${ext|tgz} mongodb-src-${src_suffix}.${ext|tar.gz} || true /usr/bin/find build/ -type f | grep msi$ | xargs -I original_filename cp original_filename mongodb-win32-${push_arch}-${suffix}.msi || true /usr/local/bin/notary-client.py --key-name "server-3.6" --auth-token-file ${workdir}/src/signing_auth_token --comment "Evergreen Automatic Signing ${revision} - ${build_variant} - ${branch_name}" --notary-url http://notary-service.build.10gen.cc:5000 --skip-missing mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz} mongodb-shell-${push_name}-${push_arch}-${suffix}.${ext|tgz} mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz} mongodb-win32-${push_arch}-${suffix}.msi mongodb-src-${src_suffix}.${ext|tar.gz} if [ "${has_packages|}" = "true" ]; then CURATOR_RELEASE="ea8d75dcc1a587111e7418e2428fb67e267af9fe" curl -L -O http://boxes.10gen.com/build/curator/curator-dist-rhel70-$CURATOR_RELEASE.tar.gz tar -zxvf curator-dist-rhel70-$CURATOR_RELEASE.tar.gz ./curator repo --config ./etc/repo_config.yaml --distro ${packager_distro} --edition ${repo_edition} --version ${version} --arch ${packager_arch} --packages repo fi # Put the binaries tarball/zipfile - command: s3.put params: aws_secret: ${aws_secret} local_file: src/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz} aws_key: ${aws_key} bucket: build-push-testing permissions: public-read content_type: ${content_type|application/x-gzip} remote_file: ${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-${suffix}-${task_id}.${ext|tgz} # Put the shell tarball/zipfile - command: s3.put params: aws_secret: ${aws_secret} local_file: src/mongodb-shell-${push_name}-${push_arch}-${suffix}.${ext|tgz} aws_key: ${aws_key} bucket: build-push-testing permissions: public-read content_type: ${content_type|application/x-gzip} remote_file: ${push_path}-STAGE/${push_name}/mongodb-shell-${push_name}-${push_arch}-${suffix}-${task_id}.${ext|tgz} # Put the source tarball - command: s3.put params: aws_secret: ${aws_secret} local_file: src/mongodb-src-${src_suffix}.${ext|tar.gz} aws_key: ${aws_key} bucket: build-push-testing permissions: public-read content_type: ${content_type|application/x-gzip} remote_file: ${push_path}-STAGE/${push_name}/mongodb-src-${src_suffix}-${task_id}.${ext|tar.gz} build_variants: [ linux-64, windows-64-2k8-ssl ] # Put the debug symbols - command: s3.put params: aws_secret: ${aws_secret} aws_key: ${aws_key} permissions: public-read local_file: src/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz} bucket: build-push-testing content_type: ${content_type|application/x-gzip} remote_file: ${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}-${task_id}.${ext|tgz} optional: true # Put the binaries tarball signature - command: s3.put params: aws_secret: ${aws_secret} local_file: src/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sig aws_key: ${aws_key} bucket: build-push-testing permissions: public-read content_type: ${content_type|application/x-gzip} remote_file: ${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-${suffix}-${task_id}.${ext|tgz}.sig # Put the shell tarball signature - command: s3.put params: aws_secret: ${aws_secret} local_file: src/mongodb-shell-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sig aws_key: ${aws_key} bucket: build-push-testing permissions: public-read content_type: ${content_type|application/x-gzip} remote_file: ${push_path}-STAGE/${push_name}/mongodb-shell-${push_name}-${push_arch}-${suffix}-${task_id}.${ext|tgz}.sig # Put the source tarball signature - command: s3.put params: aws_secret: ${aws_secret} local_file: src/mongodb-src-${src_suffix}.${ext|tar.gz}.sig aws_key: ${aws_key} bucket: build-push-testing permissions: public-read content_type: ${content_type|application/x-gzip} remote_file: ${push_path}-STAGE/${push_name}/mongodb-src-${src_suffix}-${task_id}.${ext|tar.gz}.sig build_variants: [ linux-64, windows-64-2k8-ssl ] # Put the debug symbols signature - command: s3.put params: aws_secret: ${aws_secret} aws_key: ${aws_key} permissions: public-read local_file: src/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.sig bucket: build-push-testing content_type: ${content_type|application/x-gzip} remote_file: ${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}-${task_id}.${ext|tgz}.sig optional: true # Put the signed MSI file - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} permissions: public-read build_variants: ["enterprise-windows-64-2k8", "windows-64-2k8-ssl"] local_file: src/mongodb-win32-${push_arch}-${suffix}-signed.msi bucket: build-push-testing content_type: application/x-msi remote_file: ${push_path}-STAGE/${push_name}/mongodb-win32-${push_arch}-${suffix}-${task_id}-signed.msi # Put the binaries tarball sha1 - command: s3.put params: aws_secret: ${aws_secret} local_file: src/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha1 aws_key: ${aws_key} permissions: public-read bucket: build-push-testing content_type: text/plain remote_file: ${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-${suffix}-${task_id}.${ext|tgz}.sha1 # Put the shell tarball sha1 - command: s3.put params: aws_secret: ${aws_secret} local_file: src/mongodb-shell-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha1 aws_key: ${aws_key} permissions: public-read bucket: build-push-testing content_type: text/plain remote_file: ${push_path}-STAGE/${push_name}/mongodb-shell-${push_name}-${push_arch}-${suffix}-${task_id}.${ext|tgz}.sha1 # Put the source tarball sha1 - command: s3.put params: aws_secret: ${aws_secret} local_file: src/mongodb-src-${src_suffix}.${ext|tar.gz}.sha1 aws_key: ${aws_key} permissions: public-read bucket: build-push-testing content_type: text/plain remote_file: ${push_path}-STAGE/${push_name}/mongodb-src-${src_suffix}-${task_id}.${ext|tar.gz}.sha1 build_variants: [ linux-64, windows-64-2k8-ssl ] # Put the debug symbols sha1 - command: s3.put params: aws_secret: ${aws_secret} aws_key: ${aws_key} permissions: public-read local_file: src/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.sha1 bucket: build-push-testing content_type: text/plain remote_file: ${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}-${task_id}.${ext|tgz}.sha1 optional: true # Push the signed MSI sha1 - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} permissions: public-read build_variants: ["enterprise-windows-64-2k8", "windows-64-2k8-ssl"] local_file: src/mongodb-win32-${push_arch}-${suffix}-signed.msi.sha1 bucket: build-push-testing content_type: text/plain remote_file: ${push_path}-STAGE/${push_name}/mongodb-win32-${push_arch}-${suffix}-${task_id}-signed.msi.sha1 # Put the binaries tarball sha256 - command: s3.put params: aws_secret: ${aws_secret} local_file: src/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha256 permissions: public-read aws_key: ${aws_key} bucket: build-push-testing content_type: text/plain remote_file: ${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-${suffix}-${task_id}.${ext|tgz}.sha256 # Put the shell tarball sha256 - command: s3.put params: aws_secret: ${aws_secret} local_file: src/mongodb-shell-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha256 permissions: public-read aws_key: ${aws_key} bucket: build-push-testing content_type: text/plain remote_file: ${push_path}-STAGE/${push_name}/mongodb-shell-${push_name}-${push_arch}-${suffix}-${task_id}.${ext|tgz}.sha256 # Put the source tarball sha256 - command: s3.put params: aws_secret: ${aws_secret} local_file: src/mongodb-src-${src_suffix}.${ext|tar.gz}.sha256 permissions: public-read aws_key: ${aws_key} bucket: build-push-testing content_type: text/plain remote_file: ${push_path}-STAGE/${push_name}/mongodb-src-${src_suffix}-${task_id}.${ext|tar.gz}.sha256 build_variants: [ linux-64, windows-64-2k8-ssl ] # Put the debug symbols sha256 - command: s3.put params: aws_secret: ${aws_secret} local_file: src/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.sha256 aws_key: ${aws_key} bucket: build-push-testing permissions: public-read content_type: text/plain remote_file: ${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}-${task_id}.${ext|tgz}.sha256 optional: true # Put the signed MSI sha256 - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} build_variants: ["enterprise-windows-64-2k8", "windows-64-2k8-ssl"] local_file: src/mongodb-win32-${push_arch}-${suffix}-signed.msi.sha256 bucket: build-push-testing permissions: public-read remote_file: ${push_path}-STAGE/${push_name}/mongodb-win32-${push_arch}-${suffix}-${task_id}-signed.msi.sha256 content_type: text/plain # Put the binaries tarball md5 - command: s3.put params: aws_secret: ${aws_secret} local_file: src/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}.md5 aws_key: ${aws_key} bucket: build-push-testing permissions: public-read content_type: text/plain remote_file: ${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-${suffix}-${task_id}.${ext|tgz}.md5 # Put the shell tarball md5 - command: s3.put params: aws_secret: ${aws_secret} local_file: src/mongodb-shell-${push_name}-${push_arch}-${suffix}.${ext|tgz}.md5 aws_key: ${aws_key} bucket: build-push-testing permissions: public-read content_type: text/plain remote_file: ${push_path}-STAGE/${push_name}/mongodb-shell-${push_name}-${push_arch}-${suffix}-${task_id}.${ext|tgz}.md5 # Put the source tarball md5 - command: s3.put params: aws_secret: ${aws_secret} local_file: src/mongodb-src-${src_suffix}.${ext|tar.gz}.md5 aws_key: ${aws_key} bucket: build-push-testing permissions: public-read content_type: text/plain remote_file: ${push_path}-STAGE/${push_name}/mongodb-src-${src_suffix}-${task_id}.${ext|tar.gz}.md5 build_variants: [ linux-64, windows-64-2k8-ssl ] # Put the debug symbols md5 - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: src/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.md5 bucket: build-push-testing content_type: text/plain permissions: public-read remote_file: ${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}-${task_id}.${ext|tgz}.md5 optional: true # Put the signed MSI md5 - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} build_variants: ["enterprise-windows-64-2k8", "windows-64-2k8-ssl"] local_file: src/mongodb-win32-${push_arch}-${suffix}-signed.msi.md5 bucket: build-push-testing permissions: public-read content_type: text/plain remote_file: ${push_path}-STAGE/${push_name}/mongodb-win32-${push_arch}-${suffix}-${task_id}-signed.msi.md5 - command: s3Copy.copy params: aws_key: ${aws_key} aws_secret: ${aws_secret} s3_copy_files: #Binaries - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-${suffix}-${task_id}.${ext|tgz}', 'bucket': 'build-push-testing'}, 'destination': {'path': '${push_path}/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}', 'bucket': '${push_bucket}'}} #Shell - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-shell-${push_name}-${push_arch}-${suffix}-${task_id}.${ext|tgz}', 'bucket': 'build-push-testing'}, 'destination': {'path': '${push_path}/mongodb-shell-${push_name}-${push_arch}-${suffix}.${ext|tgz}', 'bucket': '${push_bucket}'}} #Source tarball - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-src-${src_suffix}-${task_id}.${ext|tar.gz}', 'bucket': 'build-push-testing'}, 'destination': {'path': 'src/mongodb-src-${src_suffix}.${ext|tar.gz}', 'bucket': '${push_bucket}'}, 'build_variants': [ 'linux-64', 'windows-64-2k8-ssl' ]} #MSI (Windows only) - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-win32-${push_arch}-${suffix}-${task_id}-signed.msi', 'bucket': 'build-push-testing'}, 'destination': {'path': '${push_path}/mongodb-win32-${push_arch}-${suffix}-signed.msi', 'bucket': '${push_bucket}'}, 'build_variants': [ 'enterprise-windows-64-2k8', 'windows-64-2k8-ssl' ]} #Binaries Signature - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-${suffix}-${task_id}.${ext|tgz}.sig', 'bucket': 'build-push-testing'}, 'destination': {'path': '${push_path}/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sig', 'bucket': '${push_bucket}'}} #Shell Signature - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-shell-${push_name}-${push_arch}-${suffix}-${task_id}.${ext|tgz}.sig', 'bucket': 'build-push-testing'}, 'destination': {'path': '${push_path}/mongodb-shell-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sig', 'bucket': '${push_bucket}'}} #Source tarball signature - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-src-${src_suffix}-${task_id}.${ext|tar.gz}.sig', 'bucket': 'build-push-testing'}, 'destination': {'path': 'src/mongodb-src-${src_suffix}.${ext|tar.gz}.sig', 'bucket': '${push_bucket}'}, 'build_variants': [ 'linux-64', 'windows-64-2k8-ssl' ]} #SHA1 for binaries - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-${suffix}-${task_id}.${ext|tgz}.sha1', 'bucket': 'build-push-testing'}, 'destination': {'path': '${push_path}/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha1', 'bucket': '${push_bucket}'}} #SHA1 for shell - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-shell-${push_name}-${push_arch}-${suffix}-${task_id}.${ext|tgz}.sha1', 'bucket': 'build-push-testing'}, 'destination': {'path': '${push_path}/mongodb-shell-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha1', 'bucket': '${push_bucket}'}} #SHA1 for source tarball - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-src-${src_suffix}-${task_id}.${ext|tar.gz}.sha1', 'bucket': 'build-push-testing'}, 'destination': {'path': 'src/mongodb-src-${src_suffix}.${ext|tar.gz}.sha1', 'bucket': '${push_bucket}'}, 'build_variants': [ 'linux-64', 'windows-64-2k8-ssl' ]} #SHA1 for MSI - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-win32-${push_arch}-${suffix}-${task_id}-signed.msi.sha1', 'bucket': 'build-push-testing'}, 'destination': {'path': '${push_path}/mongodb-win32-${push_arch}-${suffix}-signed.msi.sha1', 'bucket': '${push_bucket}'}, 'build_variants': ['enterprise-windows-64-2k8', 'windows-64-2k8-ssl']} #SHA256 for binaries - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-${suffix}-${task_id}.${ext|tgz}.sha256', 'bucket': 'build-push-testing'}, 'destination': {'path': '${push_path}/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha256', 'bucket': '${push_bucket}'}} #SHA256 for shell - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-shell-${push_name}-${push_arch}-${suffix}-${task_id}.${ext|tgz}.sha256', 'bucket': 'build-push-testing'}, 'destination': {'path': '${push_path}/mongodb-shell-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha256', 'bucket': '${push_bucket}'}} #SHA256 for source tarball - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-src-${src_suffix}-${task_id}.${ext|tar.gz}.sha256', 'bucket': 'build-push-testing'}, 'destination': {'path': 'src/mongodb-src-${src_suffix}.${ext|tar.gz}.sha256', 'bucket': '${push_bucket}'}, 'build_variants': [ 'linux-64', 'windows-64-2k8-ssl' ]} #SHA256 for MSI files - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-win32-${push_arch}-${suffix}-${task_id}-signed.msi.sha256', 'bucket': 'build-push-testing'}, 'destination': {'path': '${push_path}/mongodb-win32-${push_arch}-${suffix}-signed.msi.sha256', 'bucket': '${push_bucket}'}, 'build_variants': ['enterprise-windows-64-2k8', 'windows-64-2k8-ssl']} #MD5 for binaries - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-${suffix}-${task_id}.${ext|tgz}.md5', 'bucket': 'build-push-testing'}, 'destination': {'path': '${push_path}/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}.md5', 'bucket': '${push_bucket}'}} #MD5 for shell - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-shell-${push_name}-${push_arch}-${suffix}-${task_id}.${ext|tgz}.md5', 'bucket': 'build-push-testing'}, 'destination': {'path': '${push_path}/mongodb-shell-${push_name}-${push_arch}-${suffix}.${ext|tgz}.md5', 'bucket': '${push_bucket}'}} #MD5 for source tarball - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-src-${src_suffix}-${task_id}.${ext|tar.gz}.md5', 'bucket': 'build-push-testing'}, 'destination': {'path': 'src/mongodb-src-${src_suffix}.${ext|tar.gz}.md5', 'bucket': '${push_bucket}'}, 'build_variants': [ 'linux-64', 'windows-64-2k8-ssl' ]} #MD5 for MSIs - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-win32-${push_arch}-${suffix}-${task_id}-signed.msi.md5', 'bucket': 'build-push-testing'}, 'destination': {'path': '${push_path}/mongodb-win32-${push_arch}-${suffix}-signed.msi.md5', 'bucket': '${push_bucket}'}, 'build_variants': ['enterprise-windows-64-2k8', 'windows-64-2k8-ssl'], } # Debug symbols are not created for all variants and the copy is optional. - command: s3Copy.copy params: aws_key: ${aws_key} aws_secret: ${aws_secret} optional: true s3_copy_files: #Debug Symbols - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}-${task_id}.${ext|tgz}', 'bucket': 'build-push-testing'}, 'destination': {'path': '${push_path}/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}', 'bucket': '${push_bucket}'}} #Debug Symbols Signature - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}-${task_id}.${ext|tgz}.sig', 'bucket': 'build-push-testing'}, 'destination': {'path': '${push_path}/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.sig', 'bucket': '${push_bucket}'}} #SHA1 for debug symbols - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}-${task_id}.${ext|tgz}.sha1', 'bucket': 'build-push-testing'}, 'destination': {'path': '${push_path}/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.sha1', 'bucket': '${push_bucket}'}} #SHA256 for debugsymbols - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}-${task_id}.${ext|tgz}.sha256', 'bucket': 'build-push-testing'}, 'destination': {'path': '${push_path}/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.sha256', 'bucket': '${push_bucket}'}} #MD5 for debugsymbols - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}-${task_id}.${ext|tgz}.md5', 'bucket': 'build-push-testing'}, 'destination': {'path': '${push_path}/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.md5', 'bucket': '${push_bucket}'}} - name: fetch_test_lifecycle depends_on: [] commands: - *git_get_project - func: "fetch test_lifecycle.yml" vars: # This task is meant to fail if there is an error while fetching test_lifecycle.yml since # the compile task won't fail. fail_task_on_error: true - name: update_test_lifecycle exec_timeout_secs: 21600 # 6 hour timeout for the task overall depends_on: [] commands: - *git_get_project - func: "set up virtualenv" - command: shell.exec timeout_secs: 14400 # Timeout if there is no output for 4 hours type: test params: working_dir: src script: | set -o errexit # Setup the git ssh private key mkdir -p ~/.ssh echo -n "${testlifecycle_ssh_key}" > ~/.ssh/test_lifecycle.pem chmod 0600 ~/.ssh/test_lifecycle.pem export GIT_SSH_COMMAND="ssh -i ~/.ssh/test_lifecycle.pem" # Create the jira credentials configuration file cat > ~/.jira.yml <