##################################################### # Drivers Nightly Tests # ##################################################### ### Instructions for Adding a Driver ### # The Major sections are: # VARIABLES # FUNCTIONS (MongoDB and Drivers) # TASKS (MongoDB and Drivers) # AXES (Drivers) # MODULES (MongoDB and Drivers) # BUILDVARIANTS (MongoDB and Drivers) # The Major sections of this config file have a space that handles the Mongodb setup, compile and test. # And a section that handles the same for drivers after MongoDB and compiled, tested, and pushed. # To Add a Driver: # Add your repo to the MODULES section # Add your Driver-specific AXES and variables # Add your Driver-specific Matrix at the bottom of this file # Update the Driver-specific variables in the 'run driver tests' function if needed stepback: true command_type: system # Files that match an ignore-list pattern will not trigger a build, if they're the only modified # files in the patch. ignore: - ".*" - "!.clang-format" - "!.eslintrc.yml" - "*.md" - "*.rst" - "*.txt" - "/distsrc/**" - "/docs/**" - "/etc/*.yml" - "!/etc/drivers_nightly.yml" - "README" ####################################### # VARIABLES # ####################################### variables: # Used when the tests it runs depend only on mongod, mongos, the mongo shell and the tools. - &task_template name: template depends_on: - name: compile commands: - func: "do setup" - func: "run mongo tests" vars: resmoke_args: --help run_multiple_jobs: false max_jobs: 0 # If set in combination with run_multiple_jobs, will cap the number of jobs used. # Used to list modules to retrieve from GitHub and apply patches if necessary. - &git_get_project command: git.get_project params: directory: src revisions: # for each module include revision as : ${_rev} enterprise: ${enterprise_rev} mongo-java-driver: ${mongo-java-driver_rev} mongo-python-driver: ${mongo-python-driver_rev} drivers-evergreen-tools: ${drivers-evergreen-tools_rev} - &generate_compile_expansions command: shell.exec params: working_dir: src script: | set -o errexit set -o verbose # We get the raw version string (r1.2.3-45-gabcdef) from git MONGO_VERSION=$(git describe) # If this is a patch build, we add the patch version id to the version string so we know # this build was a patch, and which evergreen task it came from if [ "${is_patch|}" = "true" ]; then MONGO_VERSION="$MONGO_VERSION-patch-${version_id}" fi # This script converts the generated version string into a sanitized version string for # use by scons and uploading artifacts as well as information about for the scons cache. MONGO_VERSION=$MONGO_VERSION SCONS_CACHE_MODE=${scons_cache_mode|nolinked} USE_SCONS_CACHE=${use_scons_cache|false} ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/generate_compile_expansions.py --out compile_expansions.yml - &apply_compile_expansions command: expansions.update params: file: src/compile_expansions.yml ####################################### # FUNCTIONS # ####################################### ### MONGODB COMPILE & TEST FUNCTIONS ### functions: "fetch and extract binaries" : &fetch_extract_binaries command: shell.exec params: working_dir: src script: | set -o errexit set -o verbose curl -s https://s3.amazonaws.com/mciuploads/drivers-nightly/${platform}/${revision}/binaries/mongo-${revision}.${ext} --output mongodb-binaries.${ext} # When we call git_get_project and check out our source code, it creates a directory called 'mongo', # this isn't needed and interferes with our shell binary 'mongo' when we try to copy it. # To prevent this, we just remove the 'mongo' directory first, then we do our copy. mv mongo mongo-old || true ${decompress} mongodb-binaries.${ext|tgz} chmod +x mongodb*/bin/* mv mongodb*/bin/* . "get buildnumber" : &get_buildnumber command: keyval.inc params: key: "master" destination: "builder_num" "setup credentials" : &setup_credentials command: shell.exec params: working_dir: src silent: true script: | cat > mci.buildlogger < $MONGO_ORCHESTRATION_HOME/orchestration.config MONGODB_VERSION="nightly" MONGODB_DOWNLOAD_URL=${MONGODB_DOWNLOAD_URL} TOPOLOGY=${TOPOLOGY} AUTH=${AUTH} SSL=${SSL} sh ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh # run-orchestration generates expansion file with the MONGODB_URI for the cluster - command: expansions.update params: file: mo-expansion.yml "run driver tests": - command: shell.exec type: test params: working_dir: "src" script: | cd ${DRIVER_WORKING_DIRECTORY} ${PREPARE_SHELL} echo $DRIVERS_TOOLS echo $MONGO_ORCHESTRATION_HOME echo $MONGODB_BINARIES echo $UPLOAD_BUCKET echo $PROJECT_DIRECTORY echo $TMPDIR echo $PATH echo $PROJECT # If there are extra dependencies, we set them up here first file="${PROJECT_DIRECTORY}/.evergreen/install-dependencies.sh" # Don't use ${file} syntax here because evergreen treats it as an empty expansion. [ -f "$file" ] && sh $file || echo "$file not available, skipping" ### DRIVER-SPECIFIC VARIABLES HERE PYTHON_BINARY="${PYTHON_BINARY}" \ JDK="${JDK}" \ AUTH="${AUTH}" SSL="${SSL}" MONGODB_URI="${MONGODB_URI}" TOPOLOGY="${TOPOLOGY}" .evergreen/run-tests.sh "fix absolute paths": - command: shell.exec params: script: | ${PREPARE_SHELL} for filename in $(find ${DRIVERS_TOOLS} -name \*.json); do perl -p -i -e "s|ABSOLUTE_PATH_REPLACEMENT_TOKEN|${DRIVERS_TOOLS}|g" $filename done "windows fix": - command: shell.exec params: script: | # Make sure we are on Windows, otherwise don't run if [ "Windows_NT" == "$OS" ]; then # Magic variable in cygwin ${PREPARE_SHELL} for i in $(find ${DRIVERS_TOOLS}/.evergreen ${PROJECT_DIRECTORY}/.evergreen -name \*.sh); do cat $i | tr -d '\r' > $i.new mv $i.new $i done # Copy client certificate because symlinks do not work on Windows. cp ${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem ${MONGO_ORCHESTRATION_HOME}/lib/client.pem fi "make files executable": - command: shell.exec params: script: | ${PREPARE_SHELL} for i in $(find ${DRIVERS_TOOLS}/.evergreen ${PROJECT_DIRECTORY}/.evergreen -name \*.sh); do chmod +x $i done "init test-results": - command: shell.exec params: script: | ${PREPARE_SHELL} echo '{"results": [{ "status": "FAIL", "test_file": "Build", "log_raw": "No test-results.json found was created" } ]}' > ${PROJECT_DIRECTORY}/test-results.json "upload xml test results": - command: attach.xunit_results params: file: "${DRIVER_TEST_UPLOAD_DIRECTORY}" ### UPLOAD MONGODB AND ORCHESTRATION LOGS ### "upload mo artifacts": - command: shell.exec params: script: | ${PREPARE_SHELL} find $MONGO_ORCHESTRATION_HOME -name \*.log | xargs tar czf ${PROJECT_DIRECTORY}/mongodb-logs.tar.gz - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: ${PROJECT_DIRECTORY}/mongodb-logs.tar.gz remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-mongodb-logs.tar.gz bucket: mciuploads permissions: public-read content_type: ${content_type|application/x-gzip} display_name: "mongodb-logs.tar.gz" - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: ${DRIVERS_TOOLS}/.evergreen/orchestration/server.log remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-orchestration.log bucket: mciuploads permissions: public-read content_type: ${content_type|text/plain} display_name: "orchestration.log" "setup drivers environment": - command: shell.exec params: working_dir: "src" script: | cd ${DRIVER_WORKING_DIRECTORY} # Get the current unique version of this checkout if [ "${is_patch}" = "true" ]; then CURRENT_VERSION=$(git describe)-patch-${version_id} else CURRENT_VERSION=latest fi export DRIVERS_TOOLS="$(pwd)/../drivers-evergreen-tools" # Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory if [ "Windows_NT" == "$OS" ]; then # Magic variable in cygwin export DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS) fi echo "TESTING PLATFORM ${PLATFORM}" export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration" export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" export UPLOAD_BUCKET="${project}" export PROJECT_DIRECTORY="$(pwd)" export MONGODB_DOWNLOAD_URL="https://s3.amazonaws.com/mciuploads/drivers-nightly/${PLATFORM}/${revision}/binaries/mongo-${revision}.${FILE_EXT}" cat < expansion.yml CURRENT_VERSION: "$CURRENT_VERSION" DRIVERS_TOOLS: "$DRIVERS_TOOLS" MONGO_ORCHESTRATION_HOME: "$MONGO_ORCHESTRATION_HOME" MONGODB_BINARIES: "$MONGODB_BINARIES" UPLOAD_BUCKET: "$UPLOAD_BUCKET" PROJECT_DIRECTORY: "$PROJECT_DIRECTORY" MONGODB_DOWNLOAD_URL: "$MONGODB_DOWNLOAD_URL" PREPARE_SHELL: | set -o errexit set -o xtrace export DRIVERS_TOOLS="$DRIVERS_TOOLS" export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME" export MONGODB_BINARIES="$MONGODB_BINARIES" export UPLOAD_BUCKET="$UPLOAD_BUCKET" export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" export MONGODB_DOWNLOAD_URL="$MONGODB_DOWNLOAD_URL" export TMPDIR="$MONGO_ORCHESTRATION_HOME/db" export PATH="$MONGODB_BINARIES:$PATH" export PROJECT="${project}" EOT # See what we've done cat expansion.yml # Load the expansion file to make an evergreen variable with the current unique version - command: expansions.update params: file: src/${DRIVER_WORKING_DIRECTORY}/expansion.yml ### PRE ### pre: - func: "cleanup environment" ### POST ### post: - command: attach.results params: file_location: src/report.json - func: "upload mo artifacts" - func: "upload xml test results" # Cleanup steps. - command: shell.exec params: working_dir: src script: | # removes files from the (local) scons cache when it's over a # threshold, to the $prune_ratio percentage. Ideally override # these default values in the distro config in evergreen. if [ -d "${scons_cache_path}" ]; then ${python|python} buildscripts/scons_cache_prune.py --cache-dir '${scons_cache_path}' --cache-size ${scons_cache_size|200} --prune-ratio ${scons_prune_ratio|0.8} fi - func: "cleanup environment" ####################################### # TASKS # ####################################### tasks: ### MONGO COMPILE & TEST TASKS ### - name: compile depends_on: [] commands: - command: manifest.load - *git_get_project - func: "get buildnumber" - func: "setup credentials" - *generate_compile_expansions # Then we load the generated version data into the agent so we can use it in task definitions - *apply_compile_expansions - command: shell.exec type: test params: working_dir: src script: | set -o errexit set -o verbose rm -rf ${install_directory|/data/mongo-install-directory} ${compile_env|} ${python|/opt/mongodbtoolchain/v2/bin/python2} ./buildscripts/scons.py ${compile_flags|} ${scons_cache_args|} core unittests MONGO_VERSION=${version} # Pack up the binaries mkdir -p mongodb-binaries/bin if [ `which strip` ]; then echo "found strip" find . -maxdepth 1 -type f -iname "mongo*" -exec strip {} \; fi find . -maxdepth 1 -type f -iname "mongo*" -exec mv {} ./mongodb-binaries/bin/ \; ${compress} mongodb-binaries.${ext|tgz} mongodb-binaries/ ### Run the unittests after the compile - func: "run mongo tests" vars: resmoke_args: --suites=unittests run_multiple_jobs: true - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: src/mongodb-binaries.${ext|tgz} remote_file: drivers-nightly/${platform}/${revision}/binaries/mongo-${revision}.${ext|tgz} bucket: mciuploads permissions: public-read content_type: ${content_type|application/x-gzip} display_name: Binaries - <<: *task_template name: jsCore_WT commands: - func: "do setup" - func: "run mongo tests" vars: resmoke_args: --suites=core --storageEngine=wiredTiger run_multiple_jobs: true ### DRIVER TEST TASKS ### - name: "test-nightly-standalone" depends_on: - variant: ".compile-variant" name: "jsCore_WT" tags: ["nightly", "standalone"] commands: - *git_get_project - func: "setup drivers environment" vars: VERSION: "nightly" TOPOLOGY: "server" - func: "windows fix" - func: "fix absolute paths" - func: "init test-results" - func: "make files executable" - func: "bootstrap mongo-orchestration" - func: "run driver tests" - name: "test-nightly-replica_set" depends_on: - variant: ".compile-variant" name: "jsCore_WT" tags: ["nightly", "replica_set"] commands: - *git_get_project - func: "setup drivers environment" vars: VERSION: "nightly" TOPOLOGY: "replica_set" - func: "windows fix" - func: "fix absolute paths" - func: "init test-results" - func: "make files executable" - func: "bootstrap mongo-orchestration" - func: "run driver tests" - name: "test-nightly-sharded_cluster" depends_on: - variant: ".compile-variant" name: "jsCore_WT" tags: ["nightly", "sharded_cluster"] commands: - *git_get_project - func: "setup drivers environment" vars: VERSION: "nightly" TOPOLOGY: "sharded_cluster" - func: "windows fix" - func: "fix absolute paths" - func: "init test-results" - func: "make files executable" - func: "bootstrap mongo-orchestration" - func: "run driver tests" ####################################### # AXES # ####################################### axes: ### GENERAL FOR ALL DRIVERS TESTS ### - id: os display_name: OS values: - id: "rhel62" display_name: "RHEL 6.2" run_on: rhel62-test variables: PLATFORM: "linux" FILE_EXT: "tgz" - id: "windows64" display_name: "Windows 64" run_on: windows-64-vs2015-test variables: PLATFORM: "windows64" FILE_EXT: "zip" - id: auth display_name: Authentication values: - id: "auth" display_name: Auth variables: AUTH: "auth" - id: "noauth" display_name: NoAuth variables: AUTH: "noauth" - id: ssl display_name: SSL values: - id: "ssl" display_name: SSL variables: SSL: "ssl" - id: "nossl" display_name: NoSSL variables: SSL: "nossl" ### DRIVER-SPECIFIC AXES ### ### DRIVERS TEAM, ADD YOUR AXES VARIABLES HERE ### ### Java ### - id: jdk display_name: JDK values: - id: "jdk8" display_name: JDK8 variables: JDK: "jdk8" DRIVER_WORKING_DIRECTORY: "mongo-java-driver" DRIVER_TEST_UPLOAD_DIRECTORY: "src/mongo-java-driver/*/build/test-results/TEST-*.xml" ### Python ### - id: python-linux display_name: "Python" values: - id: "2.6" display_name: "Python 2.6" variables: PYTHON_BINARY: "/opt/python/2.6/bin/python" DRIVER_WORKING_DIRECTORY: "mongo-python-driver" DRIVER_TEST_UPLOAD_DIRECTORY: "src/mongo-python-driver/xunit-results/TEST-*.xml" - id: "2.7" display_name: "Python 2.7" variables: PYTHON_BINARY: "/opt/python/2.7/bin/python" DRIVER_WORKING_DIRECTORY: "mongo-python-driver" - id: python-windows display_name: "Windows Visual Studio 2010 Python" values: - id: "3.6" display_name: "Python 3.6" variables: PYTHON_BINARY: "/cygdrive/c/python/Python36/python.exe" DRIVER_WORKING_DIRECTORY: "mongo-python-driver" DRIVER_TEST_UPLOAD_DIRECTORY: "src/mongo-python-driver/xunit-results/TEST-*.xml" ####################################### # MODULES # ####################################### # If a module is added it must be added to the manifest # be sure to add the module to 'git.get_project' revisions parameter # DRIVERS add your own git repo / branch here and add up top in 'git.get_project' modules: - name: enterprise repo: git@github.com:10gen/mongo-enterprise-modules.git prefix: src/mongo/db/modules branch: master # Shared by all drivers projects - name: drivers-evergreen-tools repo: git@github.com:mongodb-labs/drivers-evergreen-tools.git branch: master ### DRIVER REPOS ### ### DRIVERS ADD YOUR GIT REPO HERE ### - name: mongo-java-driver repo: git@github.com:mongodb/mongo-java-driver.git branch: master - name: mongo-python-driver repo: git@github.com:mongodb/mongo-python-driver.git branch: master ####################################### # BUILDVARIANTS # ####################################### buildvariants: ### MONGO COMPILE VARIANTS ### ### Linux Compile Variant ### - name: rhel62 display_name: RHEL 6.2 MongoDB Compile modules: - enterprise run_on: - rhel62-small expansions: rlp_environment: MONGOD_UNITTEST_RLP_LANGUAGE_TEST_BTROOT=/opt/basis platform: linux compile_flags: --ssl MONGO_DISTMOD=rhel62 -j$(grep -c ^processor /proc/cpuinfo) --release --variables-files=etc/scons/mongodbtoolchain_gcc.vars CPPPATH="/opt/basis/rlp/rlp/include /opt/basis/rlp/utilities/include" multiversion_platform_arch: "rhel62" multiversion_edition: "targeted" num_jobs_available: $(grep -c ^processor /proc/cpuinfo) use_scons_cache: true ### build_mongoreplay: true ext: tgz tags: ["compile-variant"] tasks: - name: compile distros: - rhel62-large - name: jsCore_WT ### Windows compile variant ### - name: enterprise-windows-64 display_name: "Enterprise Windows MongoDB Compile" modules: - enterprise run_on: - windows-64-vs2015-compile expansions: platform: "windows64" exe: ".exe" content_type: application/zip compile_flags: --ssl MONGO_DISTMOD=windows-64 --release CPPPATH="c:/openssl/include c:/sasl/include c:/snmp/include c:/curl/include" LIBPATH="c:/openssl/lib c:/sasl/lib c:/snmp/lib c:/curl/lib" -j$(( $(grep -c ^processor /proc/cpuinfo) / 4 )) --dynamic-windows --win-version-min=ws08r2 VARIANT_DIR=win32 num_jobs_available: $(grep -c ^processor /proc/cpuinfo) ext: zip use_scons_cache: true python: python tags: ["compile-variant"] tasks: - name: compile distros: - windows-64-vs2015-large - name: jsCore_WT ### DRIVER Test Variants ### DRIVERS add your matrix or test variant here ### Python Matrix Linux - matrix_name: "python-driver-linux" matrix_spec: {auth: "*", ssl: "*", os: "rhel62", python-linux: "2.6" } display_name: "Python Driver ${os} ${python-linux} ${auth} ${ssl}" modules: [ "mongo-python-driver", "drivers-evergreen-tools" ] tasks: - name: "test-nightly-replica_set" - name: "test-nightly-sharded_cluster" - name: "test-nightly-standalone" ### Python Matrix Windows - matrix_name: "python-driver-windows" matrix_spec: {auth: "*", ssl: "*", os: "windows64", python-windows: "3.6" } display_name: "Python Driver ${os} ${python-windows} ${auth} ${ssl}" modules: [ "mongo-python-driver", "drivers-evergreen-tools" ] tasks: - name: "test-nightly-replica_set" - name: "test-nightly-sharded_cluster" - name: "test-nightly-standalone" ### Java Matrix Linux - matrix_name: "java-driver-linux" matrix_spec: {auth: "*", ssl: "*", os: "rhel62", jdk: "jdk8" } display_name: "Java Driver ${os} ${jdk} ${auth} ${ssl}" modules: [ "mongo-java-driver", "drivers-evergreen-tools" ] tasks: - name: "test-nightly-replica_set" - name: "test-nightly-sharded_cluster" - name: "test-nightly-standalone"