diff options
-rw-r--r-- | src/third_party/wiredtiger/import.data | 2 | ||||
-rwxr-xr-x | src/third_party/wiredtiger/test/checkpoint/recovery-test.sh | 17 | ||||
-rwxr-xr-x | src/third_party/wiredtiger/test/evergreen.yml | 121 | ||||
-rwxr-xr-x | src/third_party/wiredtiger/tools/run_parallel.sh | 55 |
4 files changed, 192 insertions, 3 deletions
diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index cce373af92b..33e37e711ba 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-master", - "commit": "c8a0136f4d6816cf350571ce7ad566f4ab4ea0cd" + "commit": "0a709818aaeda3d9421c944a5df3d1df44f66c8b" } diff --git a/src/third_party/wiredtiger/test/checkpoint/recovery-test.sh b/src/third_party/wiredtiger/test/checkpoint/recovery-test.sh index fc98ff2f463..1cc0653aef9 100755 --- a/src/third_party/wiredtiger/test/checkpoint/recovery-test.sh +++ b/src/third_party/wiredtiger/test/checkpoint/recovery-test.sh @@ -2,12 +2,25 @@ set -x -home=${1:-WT_TEST} +usage () { + cat << EOF +Usage: recovery_test.sh {config} {home directory} +EOF +} + +if [ "$#" -ne 2 ]; then + echo "Illegal number of parameters." + usage + exit 1 +fi + +config=$1 +home=$2 backup=$home.backup recovery=$home.recovery #./t -t r -W 3 -D -X -n 100000 -k 100000 -C cache_size=100MB -h $home > $home.out 2>&1 & -./t -t r -s 2 -m -W 3 -D -p -x -n 100000 -k 100000 -C cache_size=100MB -h $home > $home.out 2>&1 & +./t ${config} -h ${home} > $home.out 2>&1 & pid=$! trap "kill -9 $pid" 0 1 2 3 13 15 diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index f04b6660edd..ba9f33806d5 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -2421,6 +2421,118 @@ tasks: cp ../../../bench/wtperf/split_heavy.wtperf . ./wtperf -O ./split_heavy.wtperf -o verbose=2 + - name: data-validation-stress-test-checkpoint + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix/test/checkpoint" + shell: bash + script: | + set -o errexit + set -o verbose + ../../../tools/run_parallel.sh 'nice ../../../test/checkpoint/recovery-test.sh "${data_validation_stress_test_args}" WT_TEST.$t' 120 + + - name: data-validation-stress-test-checkpoint-fp-hs-insert-s1 + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix/test/checkpoint" + shell: bash + script: | + set -o errexit + set -o verbose + ../../../tools/run_parallel.sh 'nice ../../../test/checkpoint/recovery-test.sh "${data_validation_stress_test_args} -s 1" WT_TEST.$t' 120 + + - name: data-validation-stress-test-checkpoint-fp-hs-insert-s2 + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix/test/checkpoint" + shell: bash + script: | + set -o errexit + set -o verbose + ../../../tools/run_parallel.sh 'nice ../../../test/checkpoint/recovery-test.sh "${data_validation_stress_test_args} -s 2" WT_TEST.$t' 120 + + - name: data-validation-stress-test-checkpoint-fp-hs-insert-s3 + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix/test/checkpoint" + shell: bash + script: | + set -o errexit + set -o verbose + ../../../tools/run_parallel.sh 'nice ../../../test/checkpoint/recovery-test.sh "${data_validation_stress_test_args} -s 3" WT_TEST.$t' 120 + + - name: data-validation-stress-test-checkpoint-fp-hs-insert-s4 + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix/test/checkpoint" + shell: bash + script: | + set -o errexit + set -o verbose + ../../../tools/run_parallel.sh 'nice ../../../test/checkpoint/recovery-test.sh "${data_validation_stress_test_args} -s 4" WT_TEST.$t' 120 + + - name: data-validation-stress-test-checkpoint-fp-hs-insert-s5 + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix/test/checkpoint" + shell: bash + script: | + set -o errexit + set -o verbose + ../../../tools/run_parallel.sh 'nice ../../../test/checkpoint/recovery-test.sh "${data_validation_stress_test_args} -s 5" WT_TEST.$t' 120 + + - name: data-validation-stress-test-checkpoint-fp-hs-insert-s6 + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix/test/checkpoint" + shell: bash + script: | + set -o errexit + set -o verbose + ../../../tools/run_parallel.sh 'nice ../../../test/checkpoint/recovery-test.sh "${data_validation_stress_test_args} -s 6" WT_TEST.$t' 120 + + - name: data-validation-stress-test-checkpoint-fp-hs-insert-s7 + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix/test/checkpoint" + shell: bash + script: | + set -o errexit + set -o verbose + ../../../tools/run_parallel.sh 'nice ../../../test/checkpoint/recovery-test.sh "${data_validation_stress_test_args} -s 7" WT_TEST.$t' 120 + - name: format-failure-configs-test depends_on: - name: compile @@ -2747,6 +2859,7 @@ buildvariants: posix_configure_flags: --enable-silent-rules --enable-diagnostic --enable-python --enable-zlib --enable-snappy --enable-strict --enable-static --prefix=$(pwd)/LOCAL_INSTALL python_binary: '/opt/mongodbtoolchain/v3/bin/python3' make_command: PATH=/opt/mongodbtoolchain/v3/bin:$PATH make + data_validation_stress_test_args: -t r -m -W 3 -D -p -x -n 100000 -k 100000 -C cache_size=100MB tasks: - name: ".pull_request !.windows_only !.pull_request_compilers" - name: linux-directio @@ -2766,6 +2879,14 @@ buildvariants: - name: long-test - name: static-wt-build-test - name: format-failure-configs-test + - name: data-validation-stress-test-checkpoint + - name: data-validation-stress-test-checkpoint-fp-hs-insert-s1 + - name: data-validation-stress-test-checkpoint-fp-hs-insert-s2 + - name: data-validation-stress-test-checkpoint-fp-hs-insert-s3 + - name: data-validation-stress-test-checkpoint-fp-hs-insert-s4 + - name: data-validation-stress-test-checkpoint-fp-hs-insert-s5 + - name: data-validation-stress-test-checkpoint-fp-hs-insert-s6 + - name: data-validation-stress-test-checkpoint-fp-hs-insert-s7 - name: ubuntu2004-cmake display_name: "* Ubuntu 20.04 CMake" diff --git a/src/third_party/wiredtiger/tools/run_parallel.sh b/src/third_party/wiredtiger/tools/run_parallel.sh new file mode 100755 index 00000000000..526dc5e65f0 --- /dev/null +++ b/src/third_party/wiredtiger/tools/run_parallel.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +usage () { + cat << EOF +Usage: run_parallel.sh {command} {num_iter} {num_parallel} +Where: + {command} is a string containing the command, including parameters, to run + {num_iter} is an positive integer indicating how many iterations to execute + [num_parallel] is an (optional) positive integer indicating how many parallel commands should be executed + in each iteration. If not provided, the default is half the number of available CPU cores. +EOF +} + +if [ "$#" -lt 2 ]; then + echo "Illegal number of parameters." + usage + exit 1 +fi + +# Determine the number of CPU cores. This code is Linux specific at the moment. +NCORES=$(grep -c ^processor /proc/cpuinfo) + +command=$1 +num_iter=$2 + +if [ "$#" -eq 3 ]; then + num_parallel=$3 +else + # Use half the number of processor cores + num_parallel="$(($NCORES / 2))" +fi + +echo "run_parallel:" +echo " number of cores: $NCORES" +echo " command: $command" +echo " num_parallel: $num_parallel" +echo " num_iter: $num_iter" + +outf=./outfile.txt + +for i in $(seq $num_iter); do + echo "Starting iteration $i" >> $outf + echo "Starting iteration $i" + + # start the commands in parallel + for((t=1; t<=num_parallel; t++)); do + echo "Starting parallel command $t (of $num_parallel) in iteration $i (of $num_iter)" >> nohup.out.$t + eval nohup $command >> nohup.out.$t 2>&1 & + done + + # Wait for the commands to all complete + for((t=1; t<=num_parallel; t++)); do + wait || exit $? + done +done |