##################################################### # Drivers Nightly Tests # ##################################################### ### Instructions for Adding a Driver ### # The Major sections are: # VARIABLES # FUNCTIONS (MongoDB and Drivers) # TASKS (MongoDB and Drivers) # AXES (Drivers) # MODULES (MongoDB and Drivers) # BUILDVARIANTS (MongoDB and Drivers) # The Major sections of this config file have a space that handles the Mongodb setup, compile and test. # And a section that handles the same for drivers after MongoDB and compiled, tested, and pushed. # To Add a Driver: # Add your repo to the MODULES section # Add your Driver-specific AXES and variables # Add your Driver-specific Matrix at the bottom of this file # Update the Driver-specific variables in the 'run driver tests' function if needed # ### Instructions for updating for future branches # 1. File a BUILD ticket to add permissions to the project's AWS keys so they # can write to the mciuploads bucket, at the root folder that has the same # name as the drivers-nightly project for the branch you are adding. # ex: drivers-nightly-4.4 # # 2. Make a git branch off a known good commit, where compile and unittests are # green # # 2. Synchronize this file with etc/evergreen.yml # - venv and pip setup (&setup_venv) # - compile # - Make sure the compile_flags and extra_flags are synchronized, and that # you are performing a release build # - "run mongo tests" # # 3. Run patch builds, ensuring that issues with compile are resolved # - Inspect the drivers tests failures, and make sure that the errors are # caused by drivers tests, and not by setup or s3 failures # # 4. Update this block with any pain points for the next person here stepback: true command_type: system # Files that match an ignore-list pattern will not trigger a build, if they're the only modified # files in the patch. ignore: - ".*" - "!.clang-format" - "!.eslintrc.yml" - "*.md" - "*.rst" - "*.txt" - "/distsrc/**" - "/docs/**" - "/etc/*.yml" - "!/etc/drivers_nightly.yml" - "README" ####################################### # VARIABLES # ####################################### variables: # Used when the tests it runs depend only on mongod, mongos, the mongo shell and the tools. - &task_template name: template depends_on: - name: compile commands: - func: "do setup" - func: "run mongo tests" vars: resmoke_args: --help run_multiple_jobs: false max_jobs: 0 # If set in combination with run_multiple_jobs, will cap the number of jobs used. # Used to list modules to retrieve from GitHub and apply patches if necessary. - &git_get_project command: git.get_project params: directory: src revisions: # for each module include revision as : ${_rev} enterprise: ${enterprise_rev} mongo-java-driver: ${mongo-java-driver_rev} mongo-python-driver: ${mongo-python-driver_rev} drivers-evergreen-tools: ${drivers-evergreen-tools_rev} - &generate_compile_expansions command: shell.exec params: working_dir: src script: | set -o errexit set -o verbose # We get the raw version string (r1.2.3-45-gabcdef) from git MONGO_VERSION=$(git describe) # If this is a patch build, we add the patch version id to the version string so we know # this build was a patch, and which evergreen task it came from if [ "${is_patch|}" = "true" ]; then MONGO_VERSION="$MONGO_VERSION-patch-${version_id}" fi ${activate_venv} # This script converts the generated version string into a sanitized version string for # use by scons and uploading artifacts as well as information about for the scons cache. MONGO_VERSION=$MONGO_VERSION \ SCONS_CACHE_MODE=${scons_cache_mode|nolinked} \ USE_SCONS_CACHE=${use_scons_cache|false} \ $python buildscripts/generate_compile_expansions.py --out compile_expansions.yml - &apply_compile_expansions command: expansions.update params: file: src/compile_expansions.yml ####################################### # FUNCTIONS # ####################################### ### MONGODB COMPILE & TEST FUNCTIONS ### functions: "set task expansion macros": &set_task_expansion_macros command: expansions.update params: updates: - key: activate_venv value: | # check if virtualenv is set up if [ -d "${workdir}/venv" ]; then if [ "Windows_NT" = "$OS" ]; then # Need to quote the path on Windows to preserve the separator. . "${workdir}/venv/Scripts/activate" 2> /tmp/activate_error.log else . ${workdir}/venv/bin/activate 2> /tmp/activate_error.log fi if [ $? -ne 0 ]; then echo "Failed to activate venv: $(cat /tmp/activate_error.log)" fi python=python else python=${python|/opt/mongodbtoolchain/v3/bin/python3} fi if [ "Windows_NT" = "$OS" ]; then export PYTHONPATH="$PYTHONPATH;$(cygpath -w ${workdir}/src)" else export PYTHONPATH="$PYTHONPATH:${workdir}/src" fi echo "python set to $(which $python)" "fetch binaries": &fetch_binaries command: s3.get params: aws_key: ${aws_key} aws_secret: ${aws_secret} remote_file: drivers-nightly/${platform}/${revision}/binaries/mongo-${revision}.${ext|tgz} bucket: mciuploads local_file: src/mongodb-binaries.${ext|tgz} "extract binaries": &extract_binaries command: shell.exec params: working_dir: src script: | set -o errexit set -o verbose ${decompress|tar xzvf} mongodb-binaries.${ext|tgz} chmod +x mongodb*/bin/* mv mongodb*/bin/* . "get buildnumber": &get_buildnumber command: keyval.inc params: key: "v4.4" destination: "builder_num" "setup credentials": &setup_credentials command: shell.exec params: working_dir: src silent: true script: | cat > mci.buildlogger < pip-requirements.txt "run mongo tests": command: shell.exec type: test params: working_dir: src script: | set -o errexit set -o verbose ${activate_venv} extra_args="" processor_architecture=$(uname -m) num_jobs_available=${num_jobs_available|1} if [ ${max_jobs|0} -gt 0 ] && [ ${max_jobs|0} -lt $num_jobs_available ]; then extra_args="$extra_args --jobs=${max_jobs}" else extra_args="$extra_args --jobs=$num_jobs_available" fi $python buildscripts/resmoke.py run ${resmoke_args} $extra_args ${test_flags} --log=buildlogger --reportFile=report.json "cleanup environment": command: shell.exec params: script: | set -o errexit set -o verbose rm -rf src /data/db/* mongo-diskstats* mongo-coredumps.tgz ~/.aws pkill -9 orchest ### DRIVERS FUNCTIONS ### "bootstrap mongo-orchestration": - command: shell.exec params: script: | ${PREPARE_SHELL} echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" > $MONGO_ORCHESTRATION_HOME/orchestration.config MONGODB_VERSION="nightly" MONGODB_DOWNLOAD_URL=${MONGODB_DOWNLOAD_URL} TOPOLOGY=${TOPOLOGY} AUTH=${AUTH} SSL=${SSL} sh ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh # run-orchestration generates expansion file with the MONGODB_URI for the cluster - command: expansions.update params: file: mo-expansion.yml "run driver tests": - command: shell.exec type: test params: working_dir: "src" script: | cd ${DRIVER_WORKING_DIRECTORY} ${PREPARE_SHELL} echo $DRIVERS_TOOLS echo $MONGO_ORCHESTRATION_HOME echo $MONGODB_BINARIES echo $UPLOAD_BUCKET echo $PROJECT_DIRECTORY echo $TMPDIR echo $PATH echo $PROJECT # If there are extra dependencies, we set them up here first file="${PROJECT_DIRECTORY}/.evergreen/install-dependencies.sh" # Don't use ${file} syntax here because evergreen treats it as an empty expansion. [ -f "$file" ] && sh $file || echo "$file not available, skipping" ### DRIVER-SPECIFIC VARIABLES HERE PYTHON_BINARY="${PYTHON_BINARY}" \ JDK="${JDK}" \ SAFE_FOR_MULTI_MONGOS="true" \ AUTH="${AUTH}" SSL="${SSL}" MONGODB_URI="${MONGODB_URI}" TOPOLOGY="${TOPOLOGY}" .evergreen/run-tests.sh "fix absolute paths": - command: shell.exec params: script: | ${PREPARE_SHELL} for filename in $(find ${DRIVERS_TOOLS} -name \*.json); do perl -p -i -e "s|ABSOLUTE_PATH_REPLACEMENT_TOKEN|${DRIVERS_TOOLS}|g" $filename done "windows fix": - command: shell.exec params: script: | # Make sure we are on Windows, otherwise don't run if [ "Windows_NT" == "$OS" ]; then # Magic variable in cygwin ${PREPARE_SHELL} for i in $(find ${DRIVERS_TOOLS}/.evergreen ${PROJECT_DIRECTORY}/.evergreen -name \*.sh); do cat $i | tr -d '\r' > $i.new mv $i.new $i done # Copy client certificate because symlinks do not work on Windows. cp ${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem ${MONGO_ORCHESTRATION_HOME}/lib/client.pem fi "make files executable": - command: shell.exec params: script: | ${PREPARE_SHELL} for i in $(find ${DRIVERS_TOOLS}/.evergreen ${PROJECT_DIRECTORY}/.evergreen -name \*.sh); do chmod +x $i done "init test-results": - command: shell.exec params: script: | ${PREPARE_SHELL} echo '{"results": [{ "status": "FAIL", "test_file": "Build", "log_raw": "No test-results.json found was created" } ]}' > ${PROJECT_DIRECTORY}/test-results.json "upload xml test results": - command: attach.xunit_results params: file: "${DRIVER_TEST_UPLOAD_DIRECTORY}" ### UPLOAD MONGODB AND ORCHESTRATION LOGS ### "upload mo artifacts": - command: shell.exec params: script: | ${PREPARE_SHELL} find $MONGO_ORCHESTRATION_HOME -name \*.log | xargs tar czf ${PROJECT_DIRECTORY}/mongodb-logs.tar.gz - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: ${PROJECT_DIRECTORY}/mongodb-logs.tar.gz remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-mongodb-logs.tar.gz bucket: mciuploads permissions: public-read content_type: ${content_type|application/x-gzip} display_name: "mongodb-logs.tar.gz" - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: ${DRIVERS_TOOLS}/.evergreen/orchestration/server.log remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-orchestration.log bucket: mciuploads permissions: public-read content_type: ${content_type|text/plain} display_name: "orchestration.log" "setup drivers environment": - *set_task_expansion_macros - command: shell.exec params: working_dir: "src" script: | cd ${DRIVER_WORKING_DIRECTORY} # Get the current unique version of this checkout if [ "${is_patch}" = "true" ]; then CURRENT_VERSION=$(git describe)-patch-${version_id} else CURRENT_VERSION=latest fi export DRIVERS_TOOLS="$(pwd)/../drivers-evergreen-tools" # Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory if [ "Windows_NT" == "$OS" ]; then # Magic variable in cygwin export DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS) fi echo "TESTING PLATFORM ${PLATFORM}" export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration" export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" export UPLOAD_BUCKET="${project}" export PROJECT_DIRECTORY="$(pwd)" export MONGODB_DOWNLOAD_URL="https://s3.amazonaws.com/mciuploads/drivers-nightly/${PLATFORM}/${revision}/binaries/mongo-${revision}.${FILE_EXT}" cat < expansion.yml CURRENT_VERSION: "$CURRENT_VERSION" DRIVERS_TOOLS: "$DRIVERS_TOOLS" MONGO_ORCHESTRATION_HOME: "$MONGO_ORCHESTRATION_HOME" MONGODB_BINARIES: "$MONGODB_BINARIES" UPLOAD_BUCKET: "$UPLOAD_BUCKET" PROJECT_DIRECTORY: "$PROJECT_DIRECTORY" MONGODB_DOWNLOAD_URL: "$MONGODB_DOWNLOAD_URL" PREPARE_SHELL: | set -o errexit set -o xtrace export DRIVERS_TOOLS="$DRIVERS_TOOLS" export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME" export MONGODB_BINARIES="$MONGODB_BINARIES" export UPLOAD_BUCKET="$UPLOAD_BUCKET" export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" export MONGODB_DOWNLOAD_URL="$MONGODB_DOWNLOAD_URL" export TMPDIR="$MONGO_ORCHESTRATION_HOME/db" export PATH="$MONGODB_BINARIES:$PATH" export PROJECT="${project}" EOT # See what we've done cat expansion.yml # Load the expansion file to make an evergreen variable with the current unique version - command: expansions.update params: file: src/${DRIVER_WORKING_DIRECTORY}/expansion.yml ### PRE ### pre: - func: "cleanup environment" ### POST ### post: - command: attach.results params: file_location: src/report.json - func: "upload mo artifacts" - func: "upload xml test results" # Cleanup steps. - command: shell.exec params: working_dir: src script: | # removes files from the (local) scons cache when it's over a # threshold, to the $prune_ratio percentage. Ideally override # these default values in the distro config in evergreen. ${activate_venv} if [ -d "${scons_cache_path}" ]; then $python buildscripts/scons_cache_prune.py --cache-dir '${scons_cache_path}' --cache-size ${scons_cache_size|200} --prune-ratio ${scons_prune_ratio|0.8} fi - func: "cleanup environment" ####################################### # TASKS # ####################################### tasks: ### MONGO COMPILE & TEST TASKS ### - name: compile depends_on: [] commands: - *set_task_expansion_macros - command: manifest.load - *git_get_project - *setup_venv - func: "get buildnumber" - func: "setup credentials" - *generate_compile_expansions # Then we load the generated version data into the agent so we can use it in task definitions - *apply_compile_expansions - command: shell.exec type: test params: working_dir: src script: | set -o errexit set -o verbose # extra_args from etc/evergreen.yml # Only allow hygienic builds in Evergreen, and use hardlinks to # reduce the disk space impact of installing all of the binaries and # associated debug info. # The expansion here is a workaround to let us set a different install-action # for tasks that don't support the one we set here. A better plan would be # to support install-action for Ninja builds directly. # TODO: https://jira.mongodb.org/browse/SERVER-48203 extra_args="--install-mode=hygienic --install-action=${task_install_action|hardlink}" # By default, limit link jobs to one quarter of our overall -j # concurrency unless locally overridden. We do this because in # static link environments, the memory consumption of each # link job is so high that without constraining the number of # links we are likely to OOM or thrash the machine. Dynamic # builds, where htis is not a concern, override this value. echo "Changing SCons to run with --jlink=${num_scons_link_jobs_available|0.25}" extra_args="$extra_args --jlink=${num_scons_link_jobs_available|0.25}" if [ "${scons_cache_scope|}" = "shared" ]; then extra_args="$extra_args --cache-debug=scons_cache.log" fi # Enable performance debugging extra_args="$extra_args --debug=time" # Build packages where the upload tasks expect them extra_args="$extra_args PKGDIR=${workdir}/src" extra_args="$extra_args --release --detect-odr-violations --separate-debug" rm -rf ${install_directory|/data/mongo-install-directory} ${activate_venv} ${compile_env|} $python ./buildscripts/scons.py \ ${compile_flags|} ${scons_cache_args|} $extra_args \ archive-dist-test install-unittests install-unittests-debug \ MONGO_VERSION=${version} # Note to next reader: mongo drivers tests expect the binaries to be # packed differently than archive-dist-test. Don't try and use # it for this purpose # Pack up the binaries mkdir -p mongodb-binaries/bin find build/install/bin -maxdepth 1 -type f -iname "mongo*" ! \( -iname "*_*" -o -iname ".debug" \) -exec mv {} ./mongodb-binaries/bin/ \; ${compress} mongodb-binaries.${ext|tgz} mongodb-binaries/ ### Run the unittests after the compile - func: "run mongo tests" vars: resmoke_args: --suites=unittests run_multiple_jobs: true - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: src/mongodb-binaries.${ext|tgz} remote_file: drivers-nightly/${platform}/${revision}/binaries/mongo-${revision}.${ext|tgz} bucket: mciuploads permissions: public-read content_type: ${content_type|application/x-gzip} display_name: Binaries - <<: *task_template name: jsCore_WT commands: - *set_task_expansion_macros - func: "do setup" - func: "run mongo tests" vars: resmoke_args: --suites=core --storageEngine=wiredTiger run_multiple_jobs: true ### DRIVER TEST TASKS ### - name: "test-nightly-standalone" depends_on: - variant: ".compile-variant" name: "jsCore_WT" tags: ["nightly", "standalone"] commands: - *git_get_project - func: "setup drivers environment" vars: VERSION: "nightly" TOPOLOGY: "server" - func: "windows fix" - func: "fix absolute paths" - func: "init test-results" - func: "make files executable" - func: "bootstrap mongo-orchestration" - func: "run driver tests" - name: "test-nightly-replica_set" depends_on: - variant: ".compile-variant" name: "jsCore_WT" tags: ["nightly", "replica_set"] commands: - *git_get_project - func: "setup drivers environment" vars: VERSION: "nightly" TOPOLOGY: "replica_set" - func: "windows fix" - func: "fix absolute paths" - func: "init test-results" - func: "make files executable" - func: "bootstrap mongo-orchestration" - func: "run driver tests" - name: "test-nightly-sharded_cluster" depends_on: - variant: ".compile-variant" name: "jsCore_WT" tags: ["nightly", "sharded_cluster"] commands: - *git_get_project - func: "setup drivers environment" vars: VERSION: "nightly" TOPOLOGY: "sharded_cluster" - func: "windows fix" - func: "fix absolute paths" - func: "init test-results" - func: "make files executable" - func: "bootstrap mongo-orchestration" - func: "run driver tests" ####################################### # AXES # ####################################### axes: ### GENERAL FOR ALL DRIVERS TESTS ### - id: os display_name: OS values: - id: "rhel70" display_name: "RHEL 7.0" run_on: rhel70-small variables: PLATFORM: "linux" FILE_EXT: "tgz" - id: "windows64" display_name: "Windows 64" run_on: windows-64-vs2019-small variables: PLATFORM: "windows64" FILE_EXT: "zip" - id: auth display_name: Authentication values: - id: "auth" display_name: Auth variables: AUTH: "auth" - id: ssl display_name: SSL values: - id: "ssl" display_name: SSL variables: SSL: "ssl" ### DRIVER-SPECIFIC AXES ### ### DRIVERS TEAM, ADD YOUR AXES VARIABLES HERE ### ### Java ### - id: jdk display_name: JDK values: - id: "jdk8" display_name: JDK8 variables: JDK: "jdk8" DRIVER_WORKING_DIRECTORY: "mongo-java-driver" DRIVER_TEST_UPLOAD_DIRECTORY: "src/mongo-java-driver/*/build/test-results/*/TEST-*.xml" ### Python ### - id: python-linux display_name: "Python" values: - id: "3.6" display_name: "Python 3.6" variables: PYTHON_BINARY: "/opt/mongodbtoolchain/v2/bin/python3" DRIVER_WORKING_DIRECTORY: "mongo-python-driver" DRIVER_TEST_UPLOAD_DIRECTORY: "src/mongo-python-driver/xunit-results/TEST-*.xml" - id: python-windows display_name: "Windows Visual Studio 2010 Python" values: - id: "3.6" display_name: "Python 3.6" variables: PYTHON_BINARY: "/cygdrive/c/python/Python36/python.exe" DRIVER_WORKING_DIRECTORY: "mongo-python-driver" DRIVER_TEST_UPLOAD_DIRECTORY: "src/mongo-python-driver/xunit-results/TEST-*.xml" ####################################### # MODULES # ####################################### # If a module is added it must be added to the manifest # be sure to add the module to 'git.get_project' revisions parameter # DRIVERS add your own git repo / branch here and add up top in 'git.get_project' modules: - name: enterprise repo: git@github.com:10gen/mongo-enterprise-modules.git prefix: src/mongo/db/modules branch: v4.4 # Shared by all drivers projects - name: drivers-evergreen-tools repo: git@github.com:mongodb-labs/drivers-evergreen-tools.git branch: master ### DRIVER REPOS ### ### DRIVERS ADD YOUR GIT REPO HERE ### - name: mongo-java-driver repo: git@github.com:mongodb/mongo-java-driver.git branch: master - name: mongo-python-driver repo: git@github.com:mongodb/mongo-python-driver.git branch: master ####################################### # BUILDVARIANTS # ####################################### buildvariants: ### MONGO COMPILE VARIANTS ### ### Linux Compile Variant ### - name: rhel70 display_name: RHEL 7.0 MongoDB Compile modules: - enterprise run_on: - rhel70-small expansions: platform: linux compile_flags: --ssl MONGO_DISTMOD=rhel70 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars multiversion_platform_arch: "rhel70" multiversion_edition: "targeted" num_jobs_available: $(grep -c ^processor /proc/cpuinfo) use_scons_cache: true ext: tgz compress: tar czf tags: ["compile-variant"] tasks: - name: compile distros: - rhel70-large - name: jsCore_WT ### Windows compile variant ### - name: enterprise-windows-64 display_name: "Enterprise Windows MongoDB Compile" modules: - enterprise run_on: - windows-64-vs2019-small expansions: platform: "windows64" exe: ".exe" content_type: application/zip compile_flags: --ssl MONGO_DISTMOD=windows CPPPATH="c:/sasl/include c:/snmp/include" LIBPATH="c:/sasl/lib c:/snmp/lib" -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) --win-version-min=win10 num_jobs_available: $(grep -c ^processor /proc/cpuinfo) ext: zip use_scons_cache: true python: '/cygdrive/c/python/Python37/python.exe' compress: 7z a tags: ["compile-variant"] tasks: - name: compile distros: - windows-64-vs2019-large - name: jsCore_WT ### DRIVER Test Variants ### DRIVERS add your matrix or test variant here ### Python Matrix Linux - matrix_name: "python-driver-linux" matrix_spec: {auth: "*", ssl: "*", os: "rhel70", python-linux: "3.6"} display_name: "Python Driver ${os} ${python-linux} ${auth} ${ssl}" modules: ["mongo-python-driver", "drivers-evergreen-tools"] tasks: - name: "test-nightly-replica_set" - name: "test-nightly-sharded_cluster" - name: "test-nightly-standalone" ### Python Matrix Windows - matrix_name: "python-driver-windows" matrix_spec: {auth: "*", ssl: "*", os: "windows64", python-windows: "3.6"} display_name: "Python Driver ${os} ${python-windows} ${auth} ${ssl}" modules: ["mongo-python-driver", "drivers-evergreen-tools"] tasks: - name: "test-nightly-replica_set" - name: "test-nightly-sharded_cluster" - name: "test-nightly-standalone" ### Java Matrix Linux - matrix_name: "java-driver-linux" matrix_spec: {auth: "*", ssl: "*", os: "rhel70", jdk: "jdk8"} display_name: "Java Driver ${os} ${jdk} ${auth} ${ssl}" modules: ["mongo-java-driver", "drivers-evergreen-tools"] tasks: - name: "test-nightly-replica_set" - name: "test-nightly-sharded_cluster" - name: "test-nightly-standalone"