SERVER-58836 Integrate list-append into Evergreen

7 files changed, 171 insertions, 25 deletions

diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 468bca82a2e..d706fbd9de4 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -1257,6 +1257,62 @@ functions:
         args:
           - "./src/evergreen/do_jepsen_setup/move_binaries.sh"
 
+  "do jepsen docker setup":
+    - *f_expansions_write
+    - command: subprocess.exec
+      type: system
+      params:
+        binary: bash
+        args:
+          - "./src/evergreen/jepsen_docker/setup.sh"
+  "run jepsen docker test":
+    - *f_expansions_write
+    - command: subprocess.exec
+      type: setup
+      params:
+        binary: bash
+        args:
+          - "./src/evergreen/jepsen_docker/docker-up.sh"
+    - command: archive.targz_pack
+      params:
+        target: jepsen-docker-log.tgz
+        source_dir: jepsen/docker
+        include:
+          - "docker.log"
+    - command: s3.put
+      params:
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: jepsen-docker-log.tgz
+        remote_file: ${project}/${build_variant}/${revision}/jstestfuzz/jepsen-docker-log-${task_id}-${execution}.tgz
+        bucket: mciuploads
+        permissions: private
+        visibility: signed
+        content_type: application/gzip
+        display_name: Jepsen Docker Build Log - Execution ${execution}
+
+    - command: subprocess.exec
+      type: test
+      timeout_secs: 2700 # Timeout test if there is no output for more than 45 minutes.
+      params:
+        binary: bash
+        args:
+          - "./src/evergreen/jepsen_docker/list-append.sh"
+    - command: subprocess.exec
+      params:
+        binary: bash
+        args:
+          - "./src/evergreen/jepsen_test_fail.sh"
+
+  "cleanup jepsen docker test":
+    - *f_expansions_write
+    - command: subprocess.exec
+      type: system
+      params:
+        binary: bash
+        args:
+          - "./src/evergreen/jepsen_docker/cleanup.sh"
+
   "run jepsen test":
     - *f_expansions_write
     - command: subprocess.exec
@@ -1943,6 +1999,7 @@ post:
   - func: "umount shared scons directory"
   - func: "cleanup FUSE watchdog"
   - func: "cleanup environment"
+  - func: "cleanup jepsen docker test"
 
 # Timeout steps
 timeout:
@@ -3012,6 +3069,14 @@ tasks:
       jepsen_test_name: read-concern-majority
       jepsen_write_concern: --write-concern w1
 
+- <<: *run_jepsen_template
+  name: jepsen_list-append
+  tags: ["jepsen_docker"]
+  commands:
+  - func: "do setup"
+  - func: "do jepsen docker setup"
+  - func: "run jepsen docker test"
+
 ## initial sync multiversion fuzzer ##
 - <<: *jstestfuzz_template
   name: initial_sync_multiversion_fuzzer_gen
@@ -10821,6 +10886,9 @@ buildvariants:
   - name: audit
   - name: causally_consistent_jscore_txns_passthrough
   - name: .encrypt !.replica_sets !.aggregation !.sharding !.jscore
+  - name: .jepsen_docker
+    distros:
+    - debian10-large
   - name: .jscore .common !.decimal !.sharding
   - name: .jstestfuzz .common
   - name: .logical_session_cache .one_sec
diff --git a/evergreen/jepsen_docker/cleanup.sh b/evergreen/jepsen_docker/cleanup.sh
new file mode 100644
index 00000000000..9a4c06d8ef5
--- /dev/null
+++ b/evergreen/jepsen_docker/cleanup.sh
@@ -0,0 +1,8 @@
+set -euo pipefail
+
+if [ -d jepsen ]; then
+  echo "Cleanup docker containers"
+  # docker ps -q fails when no containers are running
+  sudo docker container kill $(docker ps -q) || true
+  sudo docker system prune -f
+fi
diff --git a/evergreen/jepsen_docker/docker-up.sh b/evergreen/jepsen_docker/docker-up.sh
new file mode 100644
index 00000000000..0a9bba00cc2
--- /dev/null
+++ b/evergreen/jepsen_docker/docker-up.sh
@@ -0,0 +1,4 @@
+set -euo pipefail
+
+cd jepsen/docker
+./bin/up -n 9 -d 2>&1 > docker.log
diff --git a/evergreen/jepsen_docker/list-append.sh b/evergreen/jepsen_docker/list-append.sh
new file mode 100644
index 00000000000..9b01183f9cc
--- /dev/null
+++ b/evergreen/jepsen_docker/list-append.sh
@@ -0,0 +1,22 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+set -euo pipefail
+
+cd jepsen/docker
+
+# actually run the tests
+start_time=$(date +%s)
+sudo docker exec jepsen-control bash --login -c "cd /jepsen/mongodb && lein run test-all -w list-append -n n1 -n n2 -n n3 -n n4 -n n5 -n n6 -n n7 -n n8 -n n9 -r 1000 --concurrency 3n --time-limit 120 --max-writes-per-key 128 --read-concern majority --write-concern majority --txn-read-concern snapshot --txn-write-concern majority --nemesis-interval 1 --nemesis partition --test-count 30" | tee jepsen_${task_name}_${execution}.log
+end_time=$(date +%s)
+elapsed_secs=$((end_time - start_time))
+
+# copy files to expected locations for archiving
+cd ../../
+mkdir -p src/jepsen-mongodb
+sudo docker cp jepsen-control:/jepsen/mongodb/store src/jepsen-mongodb/store
+cp jepsen/docker/jepsen_${task_name}_${execution}.log src/jepsen-mongodb
+sudo docker cp jepsen-control:/jepsen/mongodb src/jepsen-workdir
+
+cd src/jepsen-mongodb
+. ../evergreen/jepsen_report.sh
diff --git a/evergreen/jepsen_docker/setup.sh b/evergreen/jepsen_docker/setup.sh
new file mode 100644
index 00000000000..cc76e20e541
--- /dev/null
+++ b/evergreen/jepsen_docker/setup.sh
@@ -0,0 +1,43 @@
+set -euo pipefail
+
+if [ ! $(which docker) ]; then
+  sudo apt-get update
+  sudo apt-get install -yq \
+    apt-transport-https \
+    ca-certificates \
+    curl \
+    gnupg \
+    lsb-release
+
+  if [ ! -f "/usr/share/keyrings/docker-archive-keyring.gpg" ]; then
+    curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+  fi
+
+  set +e
+  if ! grep "https://download.docker.com/linux/debian" "/etc/apt/sources.list.d/docker.list"; then
+    echo \
+      "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian \
+            $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+  fi
+  set -e
+
+  sudo apt-get update
+  sudo apt-get install -yq docker-ce docker-ce-cli containerd.io
+  sudo docker run hello-world
+fi
+
+if [ ! $(which docker-compose) ]; then
+  sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/bin/docker-compose
+  sudo chmod +x /usr/bin/docker-compose
+fi
+
+sudo chmod 777 /var/run/docker.sock
+
+git clone --branch=evergreen-master git@github.com:10gen/jepsen.git jepsen
+cp -rf src/dist-test jepsen/docker/node
+# place the mongodb jepsen test adjacent to the control node's Dockerfile.
+# it'll get copied into the image during the build process
+git clone --branch=no-download-master git@github.com:10gen/jepsen-io-mongodb.git jepsen/docker/control/mongodb
+
+# kill any running containers
+sudo docker container kill $(docker ps -q) || true
diff --git a/evergreen/jepsen_report.sh b/evergreen/jepsen_report.sh
new file mode 100644
index 00000000000..d6ec19e9577
--- /dev/null
+++ b/evergreen/jepsen_report.sh
@@ -0,0 +1,25 @@
+# Since we cannot use PIPESTATUS to get the exit code from the "lein run ..." pipe in dash shell,
+# we will check the output for success, failure or setup error. Note that 'grep' returns with exit code
+# 0 if it finds a match, and exit code 1 if no match is found.
+grep -q "Everything looks good" jepsen_${task_name}_${execution}.log
+grep_exit_code=$?
+if [ $grep_exit_code -eq 0 ]; then
+  status='"pass"'
+  failures=0
+  final_exit_code=0
+else
+  grep -q "Analysis invalid" jepsen_${task_name}_${execution}.log
+  grep_exit_code=$?
+  if [ $grep_exit_code -eq 0 ]; then
+    status='"fail"'
+    failures=1
+    final_exit_code=1
+  else
+    # If the failure is due to setup, then this is considered a system failure.
+    echo $grep_exit_code > jepsen_system_failure_${task_name}_${execution}
+    exit 0
+  fi
+fi
+# Create report.json
+echo "{\"failures\": $failures, \"results\": [{\"status\": $status, \"exit_code\": $final_exit_code, \"test_file\": \"${task_name}\", \"start\": $start_time, \"end\": $end_time, \"elapsed\": $elapsed_secs}]}" > ../report.json
+exit $final_exit_code
diff --git a/evergreen/jepsen_test_run.sh b/evergreen/jepsen_test_run.sh
index 2729661d055..2cf8b0ed387 100644
--- a/evergreen/jepsen_test_run.sh
+++ b/evergreen/jepsen_test_run.sh
@@ -35,28 +35,4 @@ lein run test --test ${jepsen_test_name} \
   | tee jepsen_${task_name}_${execution}.log
 end_time=$(date +%s)
 elapsed_secs=$((end_time - start_time))
-# Since we cannot use PIPESTATUS to get the exit code from the "lein run ..." pipe in dash shell,
-# we will check the output for success, failure or setup error. Note that 'grep' returns with exit code
-# 0 if it finds a match, and exit code 1 if no match is found.
-grep -q "Everything looks good" jepsen_${task_name}_${execution}.log
-grep_exit_code=$?
-if [ $grep_exit_code -eq 0 ]; then
-  status='"pass"'
-  failures=0
-  final_exit_code=0
-else
-  grep -q "Analysis invalid" jepsen_${task_name}_${execution}.log
-  grep_exit_code=$?
-  if [ $grep_exit_code -eq 0 ]; then
-    status='"fail"'
-    failures=1
-    final_exit_code=1
-  else
-    # If the failure is due to setup, then this is considered a system failure.
-    echo $grep_exit_code > jepsen_system_failure_${task_name}_${execution}
-    exit 0
-  fi
-fi
-# Create report.json
-echo "{\"failures\": $failures, \"results\": [{\"status\": $status, \"exit_code\": $final_exit_code, \"test_file\": \"${task_name}\", \"start\": $start_time, \"end\": $end_time, \"elapsed\": $elapsed_secs}]}" > ../report.json
-exit $final_exit_code
+. ../evergreen/jepsen_report.sh