diff options
author | Brian Samek <brian.samek@mongodb.com> | 2020-03-04 10:48:29 -0500 |
---|---|---|
committer | Brian Samek <brian.samek@mongodb.com> | 2020-03-04 10:48:29 -0500 |
commit | 469827589450d4444217fec1d29ddb56f4c2e7b7 (patch) | |
tree | 83c9924219aebb625fddb2992024d001e6229c1f /etc | |
parent | 73880e8b641167a779e4697029bec8983a0b4491 (diff) | |
download | mongo-469827589450d4444217fec1d29ddb56f4c2e7b7.tar.gz |
SERVER-46320 Add drivers nightly testing to 4.4 branch
Diffstat (limited to 'etc')
-rw-r--r-- | etc/drivers_nightly.yml | 711 |
1 files changed, 711 insertions, 0 deletions
diff --git a/etc/drivers_nightly.yml b/etc/drivers_nightly.yml new file mode 100644 index 00000000000..6e1e218083d --- /dev/null +++ b/etc/drivers_nightly.yml @@ -0,0 +1,711 @@ +##################################################### +# Drivers Nightly Tests # +##################################################### + +### Instructions for Adding a Driver ### +# The Major sections are: + +# VARIABLES +# FUNCTIONS (MongoDB and Drivers) +# TASKS (MongoDB and Drivers) +# AXES (Drivers) +# MODULES (MongoDB and Drivers) +# BUILDVARIANTS (MongoDB and Drivers) + +# The Major sections of this config file have a space that handles the Mongodb setup, compile and test. +# And a section that handles the same for drivers after MongoDB and compiled, tested, and pushed. + +# To Add a Driver: + +# Add your repo to the MODULES section +# Add your Driver-specific AXES and variables +# Add your Driver-specific Matrix at the bottom of this file +# Update the Driver-specific variables in the 'run driver tests' function if needed + +stepback: true +command_type: system + +# Files that match an ignore-list pattern will not trigger a build, if they're the only modified +# files in the patch. +ignore: + - ".*" + - "!.clang-format" + - "!.eslintrc.yml" + - "*.md" + - "*.rst" + - "*.txt" + - "/distsrc/**" + - "/docs/**" + - "/etc/*.yml" + - "!/etc/drivers_nightly.yml" + - "README" + +####################################### +# VARIABLES # +####################################### +variables: + +# Used when the tests it runs depend only on mongod, mongos, the mongo shell and the tools. +- &task_template + name: template + depends_on: + - name: compile + commands: + - func: "do setup" + - func: "run mongo tests" + vars: + resmoke_args: --help + run_multiple_jobs: false + max_jobs: 0 # If set in combination with run_multiple_jobs, will cap the number of jobs used. + +# Used to list modules to retrieve from GitHub and apply patches if necessary. +- &git_get_project + command: git.get_project + params: + directory: src + revisions: # for each module include revision as <module_name>: ${<module_name>_rev} + enterprise: ${enterprise_rev} + mongo-java-driver: ${mongo-java-driver_rev} + mongo-python-driver: ${mongo-python-driver_rev} + drivers-evergreen-tools: ${drivers-evergreen-tools_rev} + +- &generate_compile_expansions + command: shell.exec + params: + working_dir: src + script: | + set -o errexit + set -o verbose + + # We get the raw version string (r1.2.3-45-gabcdef) from git + MONGO_VERSION=$(git describe) + # If this is a patch build, we add the patch version id to the version string so we know + # this build was a patch, and which evergreen task it came from + if [ "${is_patch|}" = "true" ]; then + MONGO_VERSION="$MONGO_VERSION-patch-${version_id}" + fi + + # This script converts the generated version string into a sanitized version string for + # use by scons and uploading artifacts as well as information about for the scons cache. + MONGO_VERSION=$MONGO_VERSION SCONS_CACHE_MODE=${scons_cache_mode|nolinked} USE_SCONS_CACHE=${use_scons_cache|false} ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/generate_compile_expansions.py --out compile_expansions.yml + +- &apply_compile_expansions + command: expansions.update + params: + file: src/compile_expansions.yml + +####################################### +# FUNCTIONS # +####################################### + +### MONGODB COMPILE & TEST FUNCTIONS ### + +functions: + + "fetch and extract binaries": &fetch_extract_binaries + command: shell.exec + params: + working_dir: src + script: | + set -o errexit + set -o verbose + + curl -s https://s3.amazonaws.com/mciuploads/drivers-nightly/${platform}/${revision}/binaries/mongo-${revision}.${ext} --output mongodb-binaries.${ext} + # When we call git_get_project and check out our source code, it creates a directory called 'mongo', + # this isn't needed and interferes with our shell binary 'mongo' when we try to copy it. + # To prevent this, we just remove the 'mongo' directory first, then we do our copy. + mv mongo mongo-old || true + ${decompress} mongodb-binaries.${ext|tgz} + chmod +x mongodb*/bin/* + mv mongodb*/bin/* . + + "get buildnumber": &get_buildnumber + command: keyval.inc + params: + key: "v4.4" + destination: "builder_num" + + "setup credentials": &setup_credentials + command: shell.exec + params: + working_dir: src + silent: true + script: | + cat > mci.buildlogger <<END_OF_CREDS + slavename='${slave}' + passwd='${passwd}' + builder='MCI_${build_variant}' + build_num=${builder_num} + build_phase='${task_name}_${execution}' + END_OF_CREDS + + "do setup": + - *git_get_project + - *fetch_extract_binaries + - *get_buildnumber + - *setup_credentials + + + ### FOR MONGODB TESTS ### + "set up virtualenv": + command: shell.exec + type: test + params: + working_dir: src + script: | + # exit immediately if virtualenv is not found + set -o errexit + set -o verbose + + virtualenv --system-site-packages ./venv + + "run mongo tests": + command: shell.exec + type: test + params: + working_dir: src + script: | + set -o errexit + set -o verbose + + # check if virtualenv is set up + if [ -d "venv" ]; then + if [ "Windows_NT" = "$OS" ]; then + . ./venv/Scripts/activate + else + . ./venv/bin/activate + fi + fi + + extra_args="" + processor_architecture=$(uname -m) + num_jobs_available=${num_jobs_available|1} + + if [ ${max_jobs|0} -gt 0 ] && [ ${max_jobs|0} -lt $num_jobs_available ]; then + extra_args="$extra_args --jobs=${max_jobs}" + else + extra_args="$extra_args --jobs=$num_jobs_available" + fi + + ${rlp_environment} \ + ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/resmoke.py ${resmoke_args} $extra_args ${test_flags} --log=buildlogger --reportFile=report.json + + "cleanup environment": + command: shell.exec + params: + script: | + set -o errexit + set -o verbose + + rm -rf src /data/db/* mongo-diskstats* mongo-coredumps.tgz ~/.aws + pkill -9 orchest + + ### DRIVERS FUNCTIONS ### + + "bootstrap mongo-orchestration": + - command: shell.exec + params: + script: | + ${PREPARE_SHELL} + echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" > $MONGO_ORCHESTRATION_HOME/orchestration.config + MONGODB_VERSION="nightly" MONGODB_DOWNLOAD_URL=${MONGODB_DOWNLOAD_URL} TOPOLOGY=${TOPOLOGY} AUTH=${AUTH} SSL=${SSL} sh ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh + # run-orchestration generates expansion file with the MONGODB_URI for the cluster + - command: expansions.update + params: + file: mo-expansion.yml + + "run driver tests": + - command: shell.exec + type: test + params: + working_dir: "src" + script: | + cd ${DRIVER_WORKING_DIRECTORY} + ${PREPARE_SHELL} + echo $DRIVERS_TOOLS + echo $MONGO_ORCHESTRATION_HOME + echo $MONGODB_BINARIES + echo $UPLOAD_BUCKET + echo $PROJECT_DIRECTORY + + echo $TMPDIR + echo $PATH + echo $PROJECT + + # If there are extra dependencies, we set them up here first + file="${PROJECT_DIRECTORY}/.evergreen/install-dependencies.sh" + # Don't use ${file} syntax here because evergreen treats it as an empty expansion. + [ -f "$file" ] && sh $file || echo "$file not available, skipping" + + ### DRIVER-SPECIFIC VARIABLES HERE + PYTHON_BINARY="${PYTHON_BINARY}" \ + JDK="${JDK}" \ + AUTH="${AUTH}" SSL="${SSL}" MONGODB_URI="${MONGODB_URI}" TOPOLOGY="${TOPOLOGY}" .evergreen/run-tests.sh + + "fix absolute paths": + - command: shell.exec + params: + script: | + ${PREPARE_SHELL} + for filename in $(find ${DRIVERS_TOOLS} -name \*.json); do + perl -p -i -e "s|ABSOLUTE_PATH_REPLACEMENT_TOKEN|${DRIVERS_TOOLS}|g" $filename + done + + "windows fix": + - command: shell.exec + params: + script: | + # Make sure we are on Windows, otherwise don't run + if [ "Windows_NT" == "$OS" ]; then # Magic variable in cygwin + ${PREPARE_SHELL} + for i in $(find ${DRIVERS_TOOLS}/.evergreen ${PROJECT_DIRECTORY}/.evergreen -name \*.sh); do + cat $i | tr -d '\r' > $i.new + mv $i.new $i + done + # Copy client certificate because symlinks do not work on Windows. + cp ${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem ${MONGO_ORCHESTRATION_HOME}/lib/client.pem + fi + + "make files executable": + - command: shell.exec + params: + script: | + ${PREPARE_SHELL} + for i in $(find ${DRIVERS_TOOLS}/.evergreen ${PROJECT_DIRECTORY}/.evergreen -name \*.sh); do + chmod +x $i + done + + "init test-results": + - command: shell.exec + params: + script: | + ${PREPARE_SHELL} + echo '{"results": [{ "status": "FAIL", "test_file": "Build", "log_raw": "No test-results.json found was created" } ]}' > ${PROJECT_DIRECTORY}/test-results.json + + "upload xml test results": + - command: attach.xunit_results + params: + file: "${DRIVER_TEST_UPLOAD_DIRECTORY}" + + + ### UPLOAD MONGODB AND ORCHESTRATION LOGS ### + "upload mo artifacts": + - command: shell.exec + params: + script: | + ${PREPARE_SHELL} + find $MONGO_ORCHESTRATION_HOME -name \*.log | xargs tar czf ${PROJECT_DIRECTORY}/mongodb-logs.tar.gz + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: ${PROJECT_DIRECTORY}/mongodb-logs.tar.gz + remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-mongodb-logs.tar.gz + bucket: mciuploads + permissions: public-read + content_type: ${content_type|application/x-gzip} + display_name: "mongodb-logs.tar.gz" + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: ${DRIVERS_TOOLS}/.evergreen/orchestration/server.log + remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-orchestration.log + bucket: mciuploads + permissions: public-read + content_type: ${content_type|text/plain} + display_name: "orchestration.log" + + "setup drivers environment": + - command: shell.exec + params: + working_dir: "src" + script: | + cd ${DRIVER_WORKING_DIRECTORY} + # Get the current unique version of this checkout + if [ "${is_patch}" = "true" ]; then + CURRENT_VERSION=$(git describe)-patch-${version_id} + else + CURRENT_VERSION=latest + fi + + export DRIVERS_TOOLS="$(pwd)/../drivers-evergreen-tools" + + # Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory + if [ "Windows_NT" == "$OS" ]; then # Magic variable in cygwin + export DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS) + fi + + echo "TESTING PLATFORM ${PLATFORM}" + + export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration" + export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" + export UPLOAD_BUCKET="${project}" + export PROJECT_DIRECTORY="$(pwd)" + export MONGODB_DOWNLOAD_URL="https://s3.amazonaws.com/mciuploads/drivers-nightly/${PLATFORM}/${revision}/binaries/mongo-${revision}.${FILE_EXT}" + + cat <<EOT > expansion.yml + CURRENT_VERSION: "$CURRENT_VERSION" + DRIVERS_TOOLS: "$DRIVERS_TOOLS" + MONGO_ORCHESTRATION_HOME: "$MONGO_ORCHESTRATION_HOME" + MONGODB_BINARIES: "$MONGODB_BINARIES" + UPLOAD_BUCKET: "$UPLOAD_BUCKET" + PROJECT_DIRECTORY: "$PROJECT_DIRECTORY" + MONGODB_DOWNLOAD_URL: "$MONGODB_DOWNLOAD_URL" + PREPARE_SHELL: | + set -o errexit + set -o xtrace + export DRIVERS_TOOLS="$DRIVERS_TOOLS" + export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME" + export MONGODB_BINARIES="$MONGODB_BINARIES" + export UPLOAD_BUCKET="$UPLOAD_BUCKET" + export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" + export MONGODB_DOWNLOAD_URL="$MONGODB_DOWNLOAD_URL" + + export TMPDIR="$MONGO_ORCHESTRATION_HOME/db" + export PATH="$MONGODB_BINARIES:$PATH" + export PROJECT="${project}" + EOT + # See what we've done + cat expansion.yml + + # Load the expansion file to make an evergreen variable with the current unique version + - command: expansions.update + params: + file: src/${DRIVER_WORKING_DIRECTORY}/expansion.yml + +### PRE ### +pre: + - func: "cleanup environment" + +### POST ### +post: + - command: attach.results + params: + file_location: src/report.json + - func: "upload mo artifacts" + - func: "upload xml test results" + # Cleanup steps. + - command: shell.exec + params: + working_dir: src + script: | + # removes files from the (local) scons cache when it's over a + # threshold, to the $prune_ratio percentage. Ideally override + # these default values in the distro config in evergreen. + + if [ -d "${scons_cache_path}" ]; then + ${python|python} buildscripts/scons_cache_prune.py --cache-dir '${scons_cache_path}' --cache-size ${scons_cache_size|200} --prune-ratio ${scons_prune_ratio|0.8} + fi + - func: "cleanup environment" + +####################################### +# TASKS # +####################################### + +tasks: + +### MONGO COMPILE & TEST TASKS ### + +- name: compile + depends_on: [] + commands: + - command: manifest.load + - *git_get_project + - func: "get buildnumber" + - func: "setup credentials" + - *generate_compile_expansions + # Then we load the generated version data into the agent so we can use it in task definitions + - *apply_compile_expansions + + - command: shell.exec + type: test + params: + working_dir: src + script: | + set -o errexit + set -o verbose + + rm -rf ${install_directory|/data/mongo-install-directory} + + ${compile_env|} ${python|/opt/mongodbtoolchain/v2/bin/python2} ./buildscripts/scons.py ${compile_flags|} ${scons_cache_args|} core unittests MONGO_VERSION=${version} + + # Pack up the binaries + mkdir -p mongodb-binaries/bin + if [ `which strip` ]; then + echo "found strip" + find . -maxdepth 1 -type f -iname "mongo*" -exec strip {} \; + fi + find . -maxdepth 1 -type f -iname "mongo*" -exec mv {} ./mongodb-binaries/bin/ \; + ${compress} mongodb-binaries.${ext|tgz} mongodb-binaries/ + + ### Run the unittests after the compile + - func: "run mongo tests" + vars: + resmoke_args: --suites=unittests + run_multiple_jobs: true + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/mongodb-binaries.${ext|tgz} + remote_file: drivers-nightly/${platform}/${revision}/binaries/mongo-${revision}.${ext|tgz} + bucket: mciuploads + permissions: public-read + content_type: ${content_type|application/x-gzip} + display_name: Binaries + +- <<: *task_template + name: jsCore_WT + commands: + - func: "do setup" + - func: "run mongo tests" + vars: + resmoke_args: --suites=core --storageEngine=wiredTiger + run_multiple_jobs: true + +### DRIVER TEST TASKS ### +- name: "test-nightly-standalone" + depends_on: + - variant: ".compile-variant" + name: "jsCore_WT" + tags: ["nightly", "standalone"] + commands: + - *git_get_project + - func: "setup drivers environment" + vars: + VERSION: "nightly" + TOPOLOGY: "server" + - func: "windows fix" + - func: "fix absolute paths" + - func: "init test-results" + - func: "make files executable" + - func: "bootstrap mongo-orchestration" + - func: "run driver tests" + +- name: "test-nightly-replica_set" + depends_on: + - variant: ".compile-variant" + name: "jsCore_WT" + tags: ["nightly", "replica_set"] + commands: + - *git_get_project + - func: "setup drivers environment" + vars: + VERSION: "nightly" + TOPOLOGY: "replica_set" + - func: "windows fix" + - func: "fix absolute paths" + - func: "init test-results" + - func: "make files executable" + - func: "bootstrap mongo-orchestration" + - func: "run driver tests" + +- name: "test-nightly-sharded_cluster" + depends_on: + - variant: ".compile-variant" + name: "jsCore_WT" + tags: ["nightly", "sharded_cluster"] + commands: + - *git_get_project + - func: "setup drivers environment" + vars: + VERSION: "nightly" + TOPOLOGY: "sharded_cluster" + - func: "windows fix" + - func: "fix absolute paths" + - func: "init test-results" + - func: "make files executable" + - func: "bootstrap mongo-orchestration" + - func: "run driver tests" + +####################################### +# AXES # +####################################### +axes: + ### GENERAL FOR ALL DRIVERS TESTS ### + - id: os + display_name: OS + values: + - id: "rhel62" + display_name: "RHEL 6.2" + run_on: rhel62-test + variables: + PLATFORM: "linux" + FILE_EXT: "tgz" + - id: "windows64" + display_name: "Windows 64" + run_on: windows-64-vs2015-test + variables: + PLATFORM: "windows64" + FILE_EXT: "zip" + - id: auth + display_name: Authentication + values: + - id: "auth" + display_name: Auth + variables: + AUTH: "auth" + - id: "noauth" + display_name: NoAuth + variables: + AUTH: "noauth" + - id: ssl + display_name: SSL + values: + - id: "ssl" + display_name: SSL + variables: + SSL: "ssl" + - id: "nossl" + display_name: NoSSL + variables: + SSL: "nossl" + + ### DRIVER-SPECIFIC AXES ### + ### DRIVERS TEAM, ADD YOUR AXES VARIABLES HERE ### + + ### Java ### + - id: jdk + display_name: JDK + values: + - id: "jdk8" + display_name: JDK8 + variables: + JDK: "jdk8" + DRIVER_WORKING_DIRECTORY: "mongo-java-driver" + DRIVER_TEST_UPLOAD_DIRECTORY: "src/mongo-java-driver/*/build/test-results/TEST-*.xml" + ### Python ### + - id: python-linux + display_name: "Python" + values: + - id: "2.7" + display_name: "Python 2.7" + variables: + PYTHON_BINARY: "/opt/python/2.7/bin/python" + DRIVER_WORKING_DIRECTORY: "mongo-python-driver" + - id: python-windows + display_name: "Windows Visual Studio 2010 Python" + values: + - id: "3.6" + display_name: "Python 3.6" + variables: + PYTHON_BINARY: "/cygdrive/c/python/Python36/python.exe" + DRIVER_WORKING_DIRECTORY: "mongo-python-driver" + DRIVER_TEST_UPLOAD_DIRECTORY: "src/mongo-python-driver/xunit-results/TEST-*.xml" + +####################################### +# MODULES # +####################################### + +# If a module is added it must be added to the manifest +# be sure to add the module to 'git.get_project' revisions parameter +# DRIVERS add your own git repo / branch here and add up top in 'git.get_project' +modules: +- name: enterprise + repo: git@github.com:10gen/mongo-enterprise-modules.git + prefix: src/mongo/db/modules + branch: v4.4 + +# Shared by all drivers projects +- name: drivers-evergreen-tools + repo: git@github.com:mongodb-labs/drivers-evergreen-tools.git + branch: master + +### DRIVER REPOS ### +### DRIVERS ADD YOUR GIT REPO HERE ### +- name: mongo-java-driver + repo: git@github.com:mongodb/mongo-java-driver.git + branch: master + +- name: mongo-python-driver + repo: git@github.com:mongodb/mongo-python-driver.git + branch: master + +####################################### +# BUILDVARIANTS # +####################################### + +buildvariants: + +### MONGO COMPILE VARIANTS ### + +### Linux Compile Variant ### +- name: rhel62 + display_name: RHEL 6.2 MongoDB Compile + modules: + - enterprise + run_on: + - rhel62-small + expansions: + rlp_environment: MONGOD_UNITTEST_RLP_LANGUAGE_TEST_BTROOT=/opt/basis + platform: linux + compile_flags: --ssl MONGO_DISTMOD=rhel62 -j$(grep -c ^processor /proc/cpuinfo) --release --variables-files=etc/scons/mongodbtoolchain_gcc.vars CPPPATH="/opt/basis/rlp/rlp/include /opt/basis/rlp/utilities/include" + multiversion_platform_arch: "rhel62" + multiversion_edition: "targeted" + num_jobs_available: $(grep -c ^processor /proc/cpuinfo) + use_scons_cache: true + ### build_mongoreplay: true + ext: tgz + tags: ["compile-variant"] + tasks: + - name: compile + distros: + - rhel62-large + - name: jsCore_WT + +### Windows compile variant ### +- name: enterprise-windows-64 + display_name: "Enterprise Windows MongoDB Compile" + modules: + - enterprise + run_on: + - windows-64-vs2015-compile + expansions: + platform: "windows64" + exe: ".exe" + content_type: application/zip + compile_flags: --ssl MONGO_DISTMOD=windows-64 --release CPPPATH="c:/openssl/include c:/sasl/include c:/snmp/include c:/curl/include" LIBPATH="c:/openssl/lib c:/sasl/lib c:/snmp/lib c:/curl/lib" -j$(( $(grep -c ^processor /proc/cpuinfo) / 4 )) --dynamic-windows --win-version-min=ws08r2 VARIANT_DIR=win32 + num_jobs_available: $(grep -c ^processor /proc/cpuinfo) + ext: zip + use_scons_cache: true + python: python + tags: ["compile-variant"] + tasks: + - name: compile + distros: + - windows-64-vs2015-large + - name: jsCore_WT + +### DRIVER Test Variants +### DRIVERS add your matrix or test variant here + +### Python Matrix Linux +- matrix_name: "python-driver-linux" + matrix_spec: {auth: "*", ssl: "*", os: "rhel62", python-linux: "2.7"} + display_name: "Python Driver ${os} ${python-linux} ${auth} ${ssl}" + modules: ["mongo-python-driver", "drivers-evergreen-tools"] + tasks: + - name: "test-nightly-replica_set" + - name: "test-nightly-sharded_cluster" + - name: "test-nightly-standalone" + +### Python Matrix Windows +- matrix_name: "python-driver-windows" + matrix_spec: {auth: "*", ssl: "*", os: "windows64", python-windows: "3.6"} + display_name: "Python Driver ${os} ${python-windows} ${auth} ${ssl}" + modules: ["mongo-python-driver", "drivers-evergreen-tools"] + tasks: + - name: "test-nightly-replica_set" + - name: "test-nightly-sharded_cluster" + - name: "test-nightly-standalone" + +### Java Matrix Linux +- matrix_name: "java-driver-linux" + matrix_spec: {auth: "*", ssl: "*", os: "rhel62", jdk: "jdk8"} + display_name: "Java Driver ${os} ${jdk} ${auth} ${ssl}" + modules: ["mongo-java-driver", "drivers-evergreen-tools"] + tasks: + - name: "test-nightly-replica_set" + - name: "test-nightly-sharded_cluster" + - name: "test-nightly-standalone" |