diff options
| -rw-r--r-- | .gitignore | 6 | ||||
| -rw-r--r-- | buildscripts/antithesis/Dockerfile | 6 | ||||
| -rw-r--r-- | buildscripts/antithesis/README.md | 15 | ||||
| -rw-r--r-- | buildscripts/antithesis/database/Dockerfile | 25 | ||||
| -rw-r--r-- | buildscripts/antithesis/docker-compose.yml | 63 | ||||
| -rw-r--r-- | buildscripts/antithesis/workload/Dockerfile | 37 | ||||
| -rwxr-xr-x | buildscripts/antithesis/workload/init.sh | 7 | ||||
| -rw-r--r-- | buildscripts/antithesis/workload/preseed.txt | 2 | ||||
| -rwxr-xr-x | buildscripts/antithesis_suite.py | 99 | ||||
| -rw-r--r-- | etc/evergreen.yml | 47 | ||||
| -rw-r--r-- | evergreen/antithesis_image_build.sh | 51 |
11 files changed, 358 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore index 7c2303af13a..0146ad41d1b 100644 --- a/.gitignore +++ b/.gitignore @@ -222,3 +222,9 @@ default.profraw /rpmbuild /repo /rpm/tmp + +# antithesis image building assets +/buildscripts/antithesis/database/dist-test +/buildscripts/antithesis/workload/src +/buildscripts/antithesis/workload/mongo +/buildscripts/resmokeconfig/suites/antithesis_*.yml diff --git a/buildscripts/antithesis/Dockerfile b/buildscripts/antithesis/Dockerfile new file mode 100644 index 00000000000..58681a1dee7 --- /dev/null +++ b/buildscripts/antithesis/Dockerfile @@ -0,0 +1,6 @@ +FROM scratch +COPY docker-compose.yml / +#COPY .env / +ADD workload /workload +ADD database /database +ADD logs /logs diff --git a/buildscripts/antithesis/README.md b/buildscripts/antithesis/README.md new file mode 100644 index 00000000000..bfbebb0cbb5 --- /dev/null +++ b/buildscripts/antithesis/README.md @@ -0,0 +1,15 @@ +# Antithesis Docker Image Building + +This directory contains `Dockerfile`s and other resources for constructing +the workload, SUT, and config docker images for use with Antithesis. + +- workload: the workload image that runs resmoke +- database: the System Under Test, spawned multiple times to form the + appropriate topology +- config: this directory, contains the above and a docker-compose.yml file for + spawning the system +- logs: the log folders of each database node will be mounted as a subdirectory + in logs. + +Please also see evergreen/antithesis_image_build.sh, which prepares the build +contexts for building these images, and uploads them to the docker registry diff --git a/buildscripts/antithesis/database/Dockerfile b/buildscripts/antithesis/database/Dockerfile new file mode 100644 index 00000000000..11ca1b2e8bc --- /dev/null +++ b/buildscripts/antithesis/database/Dockerfile @@ -0,0 +1,25 @@ +FROM ubuntu:18.04 + +EXPOSE 27017 +EXPOSE 27018 +EXPOSE 27019 + +# prep the environment +RUN mkdir -p /data/db +RUN mkdir -p /var/log/mongodb + +# Install dependencies of MongoDB Server +RUN apt-get update +RUN apt-get install -qy libcurl4 libgssapi-krb5-2 libldap-2.4-2 libwrap0 libsasl2-2 libsasl2-modules libsasl2-modules-gssapi-mit snmp openssl liblzma5 + +# ------------------- +# Everything above this line should be common image setup +# Everything below should be specific to the version of mongodb being installed + +COPY dist-test/bin/* /usr/bin/ +RUN chmod +x /usr/bin/* +COPY libvoidstar.so /usr/lib/libvoidstar.so + +RUN /usr/bin/mongo --version + +CMD /usr/bin/mongod --bind_ip 0.0.0.0 --replSet RollbackFuzzerTest --logpath /var/log/mongodb/mongodb.log diff --git a/buildscripts/antithesis/docker-compose.yml b/buildscripts/antithesis/docker-compose.yml new file mode 100644 index 00000000000..041faed007f --- /dev/null +++ b/buildscripts/antithesis/docker-compose.yml @@ -0,0 +1,63 @@ +version: '3.0' + +services: + database1: + container_name: database1 + hostname: database1 + build: ./database + image: database:evergreen-latest + volumes: + - ./logs/database1:/var/log/mongodb/ + networks: + antithesis-net: + ipv4_address: 10.20.20.3 + # Set the an IPv4 with an address of 10.20.20.130 or higher + # to be ignored by the fault injector + # + database2: + container_name: database2 + hostname: database2 + build: ./database + image: database:evergreen-latest + volumes: + - ./logs/database2:/var/log/mongodb/ + networks: + antithesis-net: + ipv4_address: 10.20.20.4 + # Set the an IPv4 with an address of 10.20.20.130 or higher + # to be ignored by the fault injector + # + database3: + container_name: database3 + hostname: database3 + build: ./database + image: database:evergreen-latest + volumes: + - ./logs/database3:/var/log/mongodb/ + networks: + antithesis-net: + ipv4_address: 10.20.20.5 + # Set the an IPv4 with an address of 10.20.20.130 or higher + # to be ignored by the fault injector + + workload: + container_name: workload + hostname: workload + build: ./workload + image: workload:evergreen-latest + depends_on: + - "database1" + - "database2" + - "database3" + networks: + antithesis-net: + ipv4_address: 10.20.20.130 + # The subnet provided here is an example + # An alternative subnet can be used + +networks: + antithesis-net: + driver: bridge + ipam: + config: + - subnet: 10.20.20.0/24 diff --git a/buildscripts/antithesis/workload/Dockerfile b/buildscripts/antithesis/workload/Dockerfile new file mode 100644 index 00000000000..7c345658223 --- /dev/null +++ b/buildscripts/antithesis/workload/Dockerfile @@ -0,0 +1,37 @@ +FROM ubuntu:18.04 + +EXPOSE 27017 +EXPOSE 27018 +EXPOSE 27019 + +# tzdata will block waiting for interactive input. Ensure that dpkg gets run +# in a non-interactive fashion by preseeding tzdata +COPY preseed.txt /tmp/preseed.txt +RUN debconf-set-selections /tmp/preseed.txt +RUN rm /tmp/preseed.txt + +RUN apt-get update +RUN apt-get install -qy libcurl4 libgssapi-krb5-2 libldap-2.4-2 libwrap0 libsasl2-2 libsasl2-modules libsasl2-modules-gssapi-mit snmp openssl liblzma5 libssl-dev build-essential software-properties-common +RUN add-apt-repository ppa:deadsnakes/ppa +RUN apt-get update + +# python 3.9 +RUN DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true apt-get install -qy python3.9 python3.9-dev python3.9-venv + +# ------------------- +# Everything above this line should be common image setup +# Everything below should be specific to the version of mongodb being installed + +# copy resmoke, make the venv, and pip install +COPY src /resmoke + +RUN bash -c "cd /resmoke && python3.9 -m venv python3-venv && . python3-venv/bin/activate && pip install --upgrade pip wheel && pip install -r ./buildscripts/requirements.txt && ./buildscripts/antithesis_suite.py generate-all" + +COPY init.sh /resmoke/init.sh +COPY mongo /usr/bin +RUN chmod +x /usr/bin/mongo +COPY libvoidstar.so /usr/lib/libvoidstar.so + +RUN /usr/bin/mongo --version + +ENTRYPOINT /resmoke/init.sh diff --git a/buildscripts/antithesis/workload/init.sh b/buildscripts/antithesis/workload/init.sh new file mode 100755 index 00000000000..57c90d17176 --- /dev/null +++ b/buildscripts/antithesis/workload/init.sh @@ -0,0 +1,7 @@ +sleep 5s +mongo --host database1 --port 27017 --eval "config={\"_id\" : \"RollbackFuzzerTest\",\"protocolVersion\" : 1,\"members\" : [{\"_id\" : 0,\"host\" : \"database1:27017\"}, {\"_id\" : 1,\"host\" : \"database2:27017\"}, {\"_id\" : 2,\"host\" : \"database3:27017\"} ],\"settings\" : {\"chainingAllowed\" : false,\"electionTimeoutMillis\" : 500, \"heartbeatTimeoutSecs\" : 700, \"catchUpTimeoutMillis\": 700}}; rs.initiate(config)" + +mongo --host database1 --eval "rs.config()" + +# this cryptic statement keeps the workload container running. +tail -f /dev/null diff --git a/buildscripts/antithesis/workload/preseed.txt b/buildscripts/antithesis/workload/preseed.txt new file mode 100644 index 00000000000..5b20c6f355c --- /dev/null +++ b/buildscripts/antithesis/workload/preseed.txt @@ -0,0 +1,2 @@ +tzdata tzdata/Areas select America +tzdata tzdata/Zones/America select New_York diff --git a/buildscripts/antithesis_suite.py b/buildscripts/antithesis_suite.py new file mode 100755 index 00000000000..25114fac786 --- /dev/null +++ b/buildscripts/antithesis_suite.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +"""Command line utility for generating suites for targeting antithesis.""" + +import os.path +import sys +import pathlib + +import click +import yaml + +# Get relative imports to work when the package is not installed on the PYTHONPATH. +if __name__ == "__main__" and __package__ is None: + sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +SUITE_BLACKLIST = [ + "CheckReplDBHash", + "CheckReplOplogs", + "CleanEveryN", + "ContinuousStepdown", +] + + +def _sanitize_hooks(hooks): + if len(hooks) == 0: + return hooks + # it's either a list of strings, or a list of dicts, each with key 'class' + if isinstance(hooks[0], str): + return list(filter(lambda x: x not in SUITE_BLACKLIST, hooks)) + elif isinstance(hooks[0], dict): + return list(filter(lambda x: x['class'] not in SUITE_BLACKLIST, hooks)) + else: + raise RuntimeError('Unknown structure in hook. File a TIG ticket.') + + +_SUITES_PATH = os.path.join("buildscripts", "resmokeconfig", "suites") + + +@click.group() +def cli(): + """CLI Entry point.""" + pass + + +def _generate(suite_name: str) -> None: + with open(os.path.join(_SUITES_PATH, "{}.yml".format(suite_name))) as fstream: + suite = yaml.safe_load(fstream) + + try: + suite["archive"]["hooks"] = _sanitize_hooks(suite["archive"]["hooks"]) + except KeyError: + # pass, don't care + pass + except TypeError: + pass + + try: + suite["executor"]["archive"]["hooks"] = _sanitize_hooks( + suite["executor"]["archive"]["hooks"]) + except KeyError: + # pass, don't care + pass + except TypeError: + pass + + try: + suite["executor"]["hooks"] = _sanitize_hooks(suite["executor"]["hooks"]) + except KeyError: + # pass, don't care + pass + except TypeError: + pass + + out = yaml.dump(suite) + with open(os.path.join(_SUITES_PATH, "antithesis_{}.yml".format(suite_name)), "w") as fstream: + fstream.write( + "# this file was generated by buildscripts/antithesis_suite.py generate {}\n".format( + suite_name)) + fstream.write("# Do not modify by hand\n") + fstream.write(out) + + +@cli.command() +@click.argument('suite_name') +def generate(suite_name: str) -> None: + """Generate a single suite.""" + _generate(suite_name) + + +@cli.command('generate-all') +def generate_all(): + """Generate all suites.""" + for path in os.listdir(_SUITES_PATH): + if os.path.isfile(os.path.join(_SUITES_PATH, path)): + suite = path.split(".")[0] + _generate(suite) + + +if __name__ == "__main__": + cli() diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 44c6f063156..a8340e52afa 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -7193,6 +7193,30 @@ tasks: args: - "./src/evergreen/feature_flag_tags_check.sh" +- name: antithesis_image_build + tags: ["antithesis"] + # this is not patchable to avoid hitting the docker registry excessively. + # When iterating on this task, feel free to make this patchable for + # testing purposes. Your image changes will be pushed with the + # evergreen-patch tag, so as to not clobber the waterfall + patchable: false + depends_on: + - name: archive_dist_test_debug + commands: + - *f_expansions_write + - func: "git get project no modules" + - func: "f_expansions_write" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - func: "do setup" + # TODO SERVER-60619 download debug symbols and put them where needed. + - command: subprocess.exec + params: + binary: bash + args: + - "./src/evergreen/antithesis_image_build.sh" + ####################################### # Task Groups # ####################################### @@ -7697,6 +7721,9 @@ buildvariants: - name: compile_ninja_TG distros: - ubuntu1804-build + - name: .antithesis + distros: + - ubuntu1804-large - name: build_variant_gen - name: .aggfuzzer .common - name: audit @@ -12283,3 +12310,23 @@ buildvariants: test_flags: >- --runAllFeatureFlagTests --excludeWithAnyTags=resource_intensive + +- name: enterprise-ubuntu1804-64-libvoidstar + display_name: ~ Enterprise Ubuntu 18.04 w/ libvoidstar + cron: "0 0 * * FRI" # Every week starting 00:00 on Friday + modules: + - enterprise + run_on: + - ubuntu1804-large + stepback: false + expansions: + compile_flags: --ssl --ocsp-stapling=off MONGO_DISTMOD=ubuntu1804 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars CCFLAGS="-fsanitize-coverage=trace-pc-guard" LIBS="voidstar" + multiversion_platform: ubuntu1804 + multiversion_edition: enterprise + repo_edition: enterprise + large_distro_name: ubuntu1804-build + use_scons_cache: false + scons_cache_scope: "none" + tasks: + - name: compile_and_archive_dist_test_TG + - name: .antithesis diff --git a/evergreen/antithesis_image_build.sh b/evergreen/antithesis_image_build.sh new file mode 100644 index 00000000000..eecd77a8ea0 --- /dev/null +++ b/evergreen/antithesis_image_build.sh @@ -0,0 +1,51 @@ +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +. "$DIR/prelude.sh" + +set -euo pipefail + +# prepare the image building environment +cp -rf src/buildscripts/antithesis/ antithesis +# due to gitignore, we can't commit a folder called logs, so make it here +mkdir -p antithesis/logs/database{1,2,3} + +# copy ... to the build context +# resmoke +cp -rf src antithesis/workload/src +# mongo binary +cp src/dist-test/bin/mongo antithesis/workload +# libvoidstar +cp /usr/lib/libvoidstar.so antithesis/workload/ +# these aren't needed for the workload image, so get rid of them +rm -rf antithesis/workload/src/dist-test +# all mongodb binaries +cp -rf src/dist-test antithesis/database +cp /usr/lib/libvoidstar.so antithesis/database/ + +cd antithesis/workload +# push images as evergreen-latest-${branch_name}, unless it's a patch +tag="evergreen-latest-${branch_name}" +if [ "${is_patch}" = "true" ]; then + tag="evergreen-patch" +fi +docker build . -t workload:$tag +cd ../database +docker build . -t database:$tag +cd .. +docker build . -t config:$tag + +# login, push, and logout +echo "${antithesis_repo_key}" > mongodb.key.json +cat mongodb.key.json | docker login -u _json_key https://us-central1-docker.pkg.dev --password-stdin +rm mongodb.key.json + +# tag and push to the registry +docker tag workload:$tag us-central1-docker.pkg.dev/molten-verve-216720/mongodb-repository/workload:$tag +docker push us-central1-docker.pkg.dev/molten-verve-216720/mongodb-repository/workload:$tag + +docker tag database:$tag us-central1-docker.pkg.dev/molten-verve-216720/mongodb-repository/database:$tag +docker push us-central1-docker.pkg.dev/molten-verve-216720/mongodb-repository/database:$tag + +docker tag config:$tag us-central1-docker.pkg.dev/molten-verve-216720/mongodb-repository/config:$tag +docker push us-central1-docker.pkg.dev/molten-verve-216720/mongodb-repository/config:$tag + +docker logout https://us-central1-docker.pkg.dev |