From 816ba22112096f783ecbfd1f76b386942d5d9ee7 Mon Sep 17 00:00:00 2001 From: Luke Chen Date: Tue, 9 Jun 2020 15:58:09 +1000 Subject: Import wiredtiger: bf1d78126c2ff448b868f199aef22b6c528dd45d from branch mongodb-4.4 ref: 4eb9f719ee..bf1d78126c for: 4.4.0-rc9 WT-6353 Print out failing values and value history of prealloc WT-6375 Test to verify RTS fixes the inconsistent checkpoint data WT-6379 Split Evergreen stress test job into multiple shorter running jobs WT-6381 RTS to remove hs record that is appended to the update chain WT-6382 Use history store btree when searching --- src/third_party/wiredtiger/import.data | 2 +- src/third_party/wiredtiger/src/btree/bt_cursor.c | 1 + src/third_party/wiredtiger/src/btree/col_srch.c | 6 + src/third_party/wiredtiger/src/btree/row_srch.c | 6 + src/third_party/wiredtiger/src/history/hs.c | 18 +- .../wiredtiger/src/txn/txn_rollback_to_stable.c | 7 +- src/third_party/wiredtiger/test/evergreen.yml | 421 ++++++++++++--------- .../wiredtiger/test/suite/test_bug019.py | 5 + .../test/suite/test_rollback_to_stable10.py | 183 +++++++++ 9 files changed, 473 insertions(+), 176 deletions(-) create mode 100755 src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 049419a81f9..942a0437e0e 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-4.4", - "commit": "4eb9f719eed3e24a9e49a6fa60d8d56eadb8b189" + "commit": "bf1d78126c2ff448b868f199aef22b6c528dd45d" } diff --git a/src/third_party/wiredtiger/src/btree/bt_cursor.c b/src/third_party/wiredtiger/src/btree/bt_cursor.c index 69fd38f3b4b..66f525ff862 100644 --- a/src/third_party/wiredtiger/src/btree/bt_cursor.c +++ b/src/third_party/wiredtiger/src/btree/bt_cursor.c @@ -1822,6 +1822,7 @@ __wt_btcur_init(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) memset(cbt, 0, sizeof(WT_CURSOR_BTREE)); cbt->iface.session = &session->iface; + cbt->dhandle = session->dhandle; } /* diff --git a/src/third_party/wiredtiger/src/btree/col_srch.c b/src/third_party/wiredtiger/src/btree/col_srch.c index e98cf094421..88a7b48ba88 100644 --- a/src/third_party/wiredtiger/src/btree/col_srch.c +++ b/src/third_party/wiredtiger/src/btree/col_srch.c @@ -78,6 +78,12 @@ __wt_col_search( btree = S2BT(session); current = NULL; + /* + * Assert the session and cursor have the right relationship (not search specific, but search is + * a convenient place to check given any operation on a cursor will likely search a page). + */ + WT_ASSERT(session, session->dhandle == cbt->dhandle); + __cursor_pos_clear(cbt); /* diff --git a/src/third_party/wiredtiger/src/btree/row_srch.c b/src/third_party/wiredtiger/src/btree/row_srch.c index 917705f6f9c..05516281d59 100644 --- a/src/third_party/wiredtiger/src/btree/row_srch.c +++ b/src/third_party/wiredtiger/src/btree/row_srch.c @@ -230,6 +230,12 @@ __wt_row_search(WT_CURSOR_BTREE *cbt, WT_ITEM *srch_key, bool insert, WT_REF *le item = cbt->tmp; current = NULL; + /* + * Assert the session and cursor have the right relationship (not search specific, but search is + * a convenient place to check given any operation on a cursor will likely search a page). + */ + WT_ASSERT(session, session->dhandle == cbt->dhandle); + __cursor_pos_clear(cbt); /* diff --git a/src/third_party/wiredtiger/src/history/hs.c b/src/third_party/wiredtiger/src/history/hs.c index 398b871ea57..b55e1a77171 100644 --- a/src/third_party/wiredtiger/src/history/hs.c +++ b/src/third_party/wiredtiger/src/history/hs.c @@ -278,6 +278,20 @@ __wt_hs_cursor_close(WT_SESSION_IMPL *session, uint32_t session_flags, bool is_o return (0); } +/* + * __hs_row_search -- + * Search the history store for a given key and position the cursor on it. + */ +static int +__hs_row_search(WT_CURSOR_BTREE *hs_cbt, WT_ITEM *srch_key, bool insert) +{ + WT_DECL_RET; + + WT_WITH_BTREE(CUR2S(hs_cbt), CUR2BT(hs_cbt), + ret = __wt_row_search(hs_cbt, srch_key, insert, NULL, false, NULL)); + return (ret); +} + /* * __wt_hs_modify -- * Make an update to the history store. @@ -406,7 +420,7 @@ __hs_insert_record_with_btree_int(WT_SESSION_IMPL *session, WT_CURSOR *cursor, W * Search the page and insert the updates. We expect there will be no existing data: assert that * we don't find a matching key. */ - WT_WITH_PAGE_INDEX(session, ret = __wt_row_search(cbt, &cursor->key, true, NULL, false, NULL)); + WT_WITH_PAGE_INDEX(session, ret = __hs_row_search(cbt, &cursor->key, true)); WT_ERR(ret); WT_ERR(__wt_hs_modify(cbt, hs_upd)); @@ -1354,7 +1368,7 @@ __verify_history_store_id(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, uint32 WT_ERR(__wt_compare(session, NULL, &hs_key, prev_hs_key, &cmp)); if (cmp == 0) continue; - WT_WITH_PAGE_INDEX(session, ret = __wt_row_search(cbt, &hs_key, false, NULL, false, NULL)); + WT_WITH_PAGE_INDEX(session, ret = __hs_row_search(cbt, &hs_key, false)); WT_ERR(ret); found = cbt->compare == 0; diff --git a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c index d9f6afe3477..48dc34a2cf7 100644 --- a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c +++ b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c @@ -247,10 +247,11 @@ __rollback_row_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_PAGE *page, WT_ROW /* * Stop processing when we find the newer version value of this key is stable according to - * the current version stop timestamp. Also it confirms that history store doesn't contains - * any newer version than the current version for the key. + * the current version stop timestamp when it is not appending the selected update to the + * update chain. Also it confirms that history store doesn't contains any newer version than + * the current version for the key. */ - if (hs_stop_ts <= rollback_timestamp) { + if (!replace && hs_stop_ts <= rollback_timestamp) { __wt_verbose(session, WT_VERB_RTS, "history store update valid with stop timestamp: %s and stable timestamp: %s", __wt_timestamp_to_string(hs_stop_ts, ts_string[0]), diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index 4cc34b03a89..07f3d98502c 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -308,6 +308,88 @@ functions: done done +######################################################################################### +# VARIABLES +# +# The following stress tests are configured to run for six hours via the "-t 360" +# argument to format.sh: format-stress-test, format-stress-sanitizer-test, and +# race-condition-stress-sanitizer-test. The smoke and recovery tests run in a loop, +# with the number of runs adjusted to provide aproximately six hours of testing. +######################################################################################### +variables: + + - &format-stress-test + exec_timeout_secs: 25200 + commands: + - func: "get project" + - func: "compile wiredtiger with builtins" + - func: "format test script" + vars: + format_test_script_args: -e "SEGFAULT_SIGNALS=all" -b "catchsegv ./t" -t 360 + + - &format-stress-sanitizer-test + exec_timeout_secs: 25200 + commands: + - func: "get project" + - func: "compile wiredtiger address sanitizer" + - func: "format test script" + vars: + format_test_script_args: -t 360 + test_env_vars: + ASAN_OPTIONS="detect_leaks=1:abort_on_error=1:disable_coredump=0" + ASAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer + + - &format-stress-smoke-test + exec_timeout_secs: 25200 + commands: + - func: "get project" + - func: "compile wiredtiger with builtins" + - func: "format test script" + vars: + format_test_script_args: -e "SEGFAULT_SIGNALS=all" -b "catchsegv ./t" -S + times: 30 + + - &format-stress-sanitizer-smoke-test + exec_timeout_secs: 25200 + commands: + - func: "get project" + - func: "compile wiredtiger address sanitizer" + - func: "format test script" + vars: + test_env_vars: + ASAN_OPTIONS="detect_leaks=1:abort_on_error=1:disable_coredump=0" + ASAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer + format_test_script_args: -S + times: 20 + + - &race-condition-stress-sanitizer-test + exec_timeout_secs: 25200 + commands: + - func: "get project" + - func: "compile wiredtiger" + vars: + configure_env_vars: + CC="/opt/mongodbtoolchain/v3/bin/clang -fsanitize=address" + PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-ggdb -fPIC -fno-omit-frame-pointer" + posix_configure_flags: --enable-diagnostic --with-builtins=lz4,snappy,zlib + - func: "format test script" + vars: + format_test_script_args: -R -t 360 + test_env_vars: + ASAN_OPTIONS="detect_leaks=1:abort_on_error=1:disable_coredump=0" + ASAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer + + - &recovery-stress-test + exec_timeout_secs: 25200 + commands: + - func: "get project" + - func: "compile wiredtiger" + vars: + posix_configure_flags: --enable-strict --enable-diagnostic --with-builtins=lz4,snappy,zlib + - func: "recovery stress test script" + vars: + times: 40 + pre: - func: "cleanup" post: @@ -1981,8 +2063,6 @@ tasks: - name: time-shift-sensitivity-test depends_on: - name: compile - vars: - posix_configure_flags: --enable-strict commands: - func: "fetch artifacts" vars: @@ -1996,140 +2076,6 @@ tasks: ./time_shift_test.sh /usr/local/lib/faketime/libfaketimeMT.so.1 0-1 2>&1 - - name: format-stress-sanitizer-test - #set a 25 hours (25*60*60 = 90000 seconds) timeout - exec_timeout_secs: 90000 - commands: - - func: "get project" - - func: "compile wiredtiger address sanitizer" - - func: "format test script" - vars: - test_env_vars: ASAN_OPTIONS="detect_leaks=1:abort_on_error=1:disable_coredump=0" ASAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer - # run for 24 hours ( 24 * 60 = 1440 minutes), skip known errors, don't stop at failed tests, use default config - format_test_script_args: -E -t 1440 - - - name: format-stress-sanitizer-lsm-test - commands: - - func: "get project" - - func: "compile wiredtiger address sanitizer" - - func: "format test script" - vars: - test_env_vars: ASAN_OPTIONS="detect_leaks=1:abort_on_error=1:disable_coredump=0" ASAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer - # Run for 30 mins, and explicitly set data_source to LSM with a large cache - format_test_script_args: -t 30 data_source=lsm cache_minimum=5000 - - - name: format-stress-sanitizer-smoke-test - #set a 7 hours timeout - exec_timeout_secs: 25200 - commands: - - func: "get project" - - func: "compile wiredtiger address sanitizer" - - func: "format test script" - # to emulate the original Jenkins job's test coverage, we are running the smoke test 16 times - # run smoke tests, skip known errors, don't stop at failed tests, use default config - vars: - test_env_vars: ASAN_OPTIONS="detect_leaks=1:abort_on_error=1:disable_coredump=0" ASAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer - format_test_script_args: -E -S - times: 16 - - - name: race-condition-stress-sanitizer-test - # set an evergreen timeout of 25 hours - exec_timeout_secs: 90000 - commands: - - func: "get project" - - func: "compile wiredtiger" - vars: - configure_env_vars: CC="/opt/mongodbtoolchain/v3/bin/clang -fsanitize=address" PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-ggdb -fPIC -fno-omit-frame-pointer" - posix_configure_flags: --enable-diagnostic --with-builtins=lz4,snappy,zlib - - func: "format test script" - vars: - # set a format.sh timeout of 24 hours, skip known errors, don't stop at failed tests, use default config - format_test_script_args: -E -R -t 1440 - test_env_vars: ASAN_OPTIONS="detect_leaks=1:abort_on_error=1:disable_coredump=0" ASAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer - - - name: recovery-stress-test - #set a 25 hours timeout - exec_timeout_secs: 90000 - commands: - - func: "get project" - - func: "compile wiredtiger" - vars: - posix_configure_flags: --enable-strict --enable-diagnostic --with-builtins=lz4,snappy,zlib - - func: "recovery stress test script" - vars: - # Repeat this script enough times to make this task run for 24 hours - # At the time of writing this script, one call to underlying scripts takes about ~15 mins to finish in worst case. - # We are giving an extra ~20% room for vairance in execution time. - times: 80 - - - name: split-stress-test - commands: - - func: "get project" - - func: "compile wiredtiger" - vars: - configure_env_vars: CXX=/opt/mongodbtoolchain/v3/bin/g++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-ggdb -fPIC" - - command: shell.exec - params: - working_dir: "wiredtiger/bench/workgen/runner" - script: | - set -o errexit - set -o verbose - for i in {1..10}; do ${python_binary|python} split_stress.py; done - - - name: format-stress-test - # Set 25 hours timeout - exec_timeout_secs: 90000 - commands: - - func: "get project" - - func: "compile wiredtiger with builtins" - - func: "format test script" - vars: - #run for 24 hours ( 24 * 60 = 1440 minutes), use default config - format_test_script_args: -e "SEGFAULT_SIGNALS=all" -b "catchsegv ./t" -t 1440 - - - name: format-stress-smoke-test - # Set 7 hours timeout - exec_timeout_secs: 25200 - commands: - - func: "get project" - - func: "compile wiredtiger with builtins" - - func: "format test script" - vars: - # to emulate the original Jenkins job's test coverage, we are running the smoke test 16 times - # run smoke tests, use default config (-S) - format_test_script_args: -e "SEGFAULT_SIGNALS=all" -b "catchsegv ./t" -S - times: 16 - - - name: checkpoint-stress-test - commands: - - command: timeout.update - params: - # set 24 hour timeout for the task and command - exec_timeout_secs: 86400 - timeout_secs: 86400 - - func: "get project" - - func: "compile wiredtiger with builtins" - - func: "checkpoint stress test" - vars: - # No of times to run the loop - times: 1 - # No of processes to run in the background - no_of_procs: 8 - - # The task name is ppc-zseries because this task will be used in both buildVariants - - name: format-stress-ppc-zseries-test - # Set 2.5 hours timeout (60 * 60 * 2.5) - exec_timeout_secs: 9000 - commands: - - func: "get project" - - func: "compile wiredtiger with builtins" - - func: "format test script" - vars: - # Make sure we dump core on failure - format_test_setting: ulimit -c unlimited - #run for 2 hours ( 2 * 60 = 120 minutes), use default config - format_test_script_args: -e "SEGFAULT_SIGNALS=all" -b "catchsegv ./t" -t 120 - - name: format-wtperf-test commands: - func: "get project" @@ -2177,41 +2123,175 @@ tasks: exit 1 fi + - name: format-stress-sanitizer-lsm-test + # Temporarily disabled (WT-6255) + # tags: ["stress-test-1"] + commands: + - func: "get project" + - func: "compile wiredtiger address sanitizer" + - func: "format test script" + vars: + test_env_vars: + ASAN_OPTIONS="detect_leaks=1:abort_on_error=1:disable_coredump=0" + ASAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer + # Run for 30 mins, and explicitly set data_source to LSM with a large cache + format_test_script_args: -t 30 data_source=lsm cache_minimum=5000 + + - name: checkpoint-stress-test + tags: ["stress-test-1"] + exec_timeout_secs: 86400 + commands: + - command: timeout.update + params: + timeout_secs: 86400 + - func: "get project" + - func: "compile wiredtiger with builtins" + - func: "checkpoint stress test" + vars: + times: 1 # No of times to run the loop + no_of_procs: 8 # No of processes to run in the background + + - name: split-stress-test + tags: ["stress-test-1", "stress-test-zseries-1"] + commands: + - func: "get project" + - func: "compile wiredtiger" + vars: + configure_env_vars: + CXX=/opt/mongodbtoolchain/v3/bin/g++ + PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-ggdb -fPIC" + - command: shell.exec + params: + working_dir: "wiredtiger/bench/workgen/runner" + script: | + set -o errexit + set -o verbose + for i in {1..10}; do ${python_binary|python} split_stress.py; done + + # The task name is ppc-zseries because this task will be used in both buildVariants + - name: format-stress-ppc-zseries-test + tags: ["stress-test-ppc-1", "stress-test-zseries-1"] + # Set 2.5 hours timeout (60 * 60 * 2.5) + exec_timeout_secs: 9000 + commands: + - func: "get project" + - func: "compile wiredtiger with builtins" + - func: "format test script" + vars: + # Make sure we dump core on failure + format_test_setting: ulimit -c unlimited + #run for 2 hours ( 2 * 60 = 120 minutes), use default config + format_test_script_args: -e "SEGFAULT_SIGNALS=all" -b "catchsegv ./t" -t 120 + + # Replace this test with format-stress-sanitizer-test after BUILD-10248 fix. - name: format-stress-sanitizer-ppc-test + tags: ["stress-test-ppc-1"] # Set 2.5 hours timeout (60 * 60 * 2.5) exec_timeout_secs: 9000 commands: - func: "get project" - func: "compile wiredtiger" vars: - # CC is set to the system default "clang" binary here as a workaround. Change it back to mongodbtoolchain "clang" binary after BUILD-10248 fix. - configure_env_vars: CCAS=/opt/mongodbtoolchain/v3/bin/gcc CC=/usr/bin/clang CXX=/opt/mongodbtoolchain/v3/bin/clang++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-ggdb -fPIC -fsanitize=address -fno-omit-frame-pointer -I/opt/mongodbtoolchain/v3/lib/gcc/ppc64le-mongodb-linux/8.2.0/include" + # CC is set to the system default "clang" binary here as a workaround. Change it back + # to mongodbtoolchain "clang" binary after BUILD-10248 fix. + configure_env_vars: + CCAS=/opt/mongodbtoolchain/v3/bin/gcc CC=/usr/bin/clang + CXX=/opt/mongodbtoolchain/v3/bin/clang++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH + CFLAGS="-ggdb -fPIC -fsanitize=address -fno-omit-frame-pointer + -I/opt/mongodbtoolchain/v3/lib/gcc/ppc64le-mongodb-linux/8.2.0/include" posix_configure_flags: --enable-diagnostic --with-builtins=lz4,snappy,zlib - func: "format test script" vars: - test_env_vars: ASAN_OPTIONS="detect_leaks=1:abort_on_error=1:disable_coredump=0" ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-6.0/bin/llvm-symbolizer - # run for 2 hours ( 2 * 60 = 120 minutes), don't stop at failed tests, use default config + test_env_vars: + ASAN_OPTIONS="detect_leaks=1:abort_on_error=1:disable_coredump=0" + ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-6.0/bin/llvm-symbolizer + # Run for 2 hours (2 * 60 = 120 minutes), don't stop at failed tests, use default config format_test_script_args: -t 120 + # Replace this test with format-stress-sanitizer-smoke-test after BUILD-10248 fix. - name: format-stress-sanitizer-smoke-ppc-test + tags: ["stress-test-ppc-1"] # Set 7 hours timeout (60 * 60 * 7) exec_timeout_secs: 25200 commands: - func: "get project" - func: "compile wiredtiger" vars: - # CC is set to the system default "clang" binary here as a workaround. Change it back to mongodbtoolchain "clang" binary after BUILD-10248 fix. - configure_env_vars: CCAS=/opt/mongodbtoolchain/v3/bin/gcc CC=/usr/bin/clang CXX=/opt/mongodbtoolchain/v3/bin/clang++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-ggdb -fPIC -fsanitize=address -fno-omit-frame-pointer -I/opt/mongodbtoolchain/v3/lib/gcc/ppc64le-mongodb-linux/8.2.0/include" + # CC is set to the system default "clang" binary here as a workaround. Change it back + # to mongodbtoolchain "clang" binary after BUILD-10248 fix. + configure_env_vars: + CCAS=/opt/mongodbtoolchain/v3/bin/gcc CC=/usr/bin/clang + CXX=/opt/mongodbtoolchain/v3/bin/clang++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH + CFLAGS="-ggdb -fPIC -fsanitize=address -fno-omit-frame-pointer + -I/opt/mongodbtoolchain/v3/lib/gcc/ppc64le-mongodb-linux/8.2.0/include" posix_configure_flags: --enable-diagnostic --with-builtins=lz4,snappy,zlib - func: "format test script" - # to emulate the original Jenkins job's test coverage, we are running the smoke test 16 times - # run smoke tests, don't stop at failed tests, use default config + # To emulate the original Jenkins job's test coverage, we are running the smoke test 16 times + # Run smoke tests, don't stop at failed tests, use default config vars: - test_env_vars: ASAN_OPTIONS="detect_leaks=1:abort_on_error=1:disable_coredump=0" ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-6.0/bin/llvm-symbolizer + test_env_vars: + ASAN_OPTIONS="detect_leaks=1:abort_on_error=1:disable_coredump=0" + ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-6.0/bin/llvm-symbolizer format_test_script_args: -S times: 16 + - <<: *format-stress-test + name: format-stress-test-1 + tags: ["stress-test-1"] + - <<: *format-stress-test + name: format-stress-test-2 + tags: ["stress-test-2"] + - <<: *format-stress-test + name: format-stress-test-3 + tags: ["stress-test-3"] + - <<: *format-stress-test + name: format-stress-test-4 + tags: ["stress-test-4"] + - <<: *format-stress-sanitizer-test + name: format-stress-sanitizer-test-1 + tags: ["stress-test-1"] + - <<: *format-stress-sanitizer-test + name: format-stress-sanitizer-test-2 + tags: ["stress-test-2"] + - <<: *format-stress-sanitizer-test + name: format-stress-sanitizer-test-3 + tags: ["stress-test-3"] + - <<: *format-stress-sanitizer-test + name: format-stress-sanitizer-test-4 + tags: ["stress-test-4"] + - <<: *format-stress-smoke-test + name: format-stress-smoke-test-1 + tags: ["stress-test-1", "stress-test-ppc-1", "stress-test-zseries-1"] + - <<: *format-stress-smoke-test + name: format-stress-smoke-test-2 + tags: ["stress-test-2", "stress-test-ppc-2", "stress-test-zseries-2"] + - <<: *format-stress-sanitizer-smoke-test + name: format-stress-sanitizer-smoke-test-1 + tags: ["stress-test-1"] + - <<: *format-stress-sanitizer-smoke-test + name: format-stress-sanitizer-smoke-test-2 + tags: ["stress-test-2"] + - <<: *race-condition-stress-sanitizer-test + name: race-condition-stress-sanitizer-test-1 + tags: ["stress-test-1"] + - <<: *race-condition-stress-sanitizer-test + name: race-condition-stress-sanitizer-test-2 + tags: ["stress-test-2"] + - <<: *race-condition-stress-sanitizer-test + name: race-condition-stress-sanitizer-test-3 + tags: ["stress-test-3"] + - <<: *race-condition-stress-sanitizer-test + name: race-condition-stress-sanitizer-test-4 + tags: ["stress-test-4"] + - <<: *recovery-stress-test + name: recovery-stress-test-1 + tags: ["stress-test-1", "stress-test-zseries-1"] + - <<: *recovery-stress-test + name: recovery-stress-test-2 + tags: ["stress-test-2", "stress-test-zseries-2"] + buildvariants: + - name: ubuntu1804 display_name: "! Ubuntu 18.04" run_on: @@ -2275,21 +2355,19 @@ buildvariants: run_on: - ubuntu1804-test expansions: - test_env_vars: LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libeatmydata.so PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs top_srcdir=$(pwd)/.. top_builddir=$(pwd) smp_command: -j $(grep -c ^processor /proc/cpuinfo) - posix_configure_flags: --enable-silent-rules --enable-diagnostic --enable-python --enable-zlib --enable-snappy --enable-strict --enable-static --prefix=$(pwd)/LOCAL_INSTALL make_command: PATH=/opt/mongodbtoolchain/v3/bin:$PATH make + test_env_vars: + LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libeatmydata.so LD_LIBRARY_PATH=$(pwd)/.libs + PATH=/opt/mongodbtoolchain/v3/bin:$PATH top_srcdir=$(pwd)/.. top_builddir=$(pwd) + posix_configure_flags: + --enable-silent-rules --enable-diagnostic --enable-python --enable-zlib --enable-snappy + --enable-strict --enable-static --prefix=$(pwd)/LOCAL_INSTALL tasks: - - name: recovery-stress-test - - name: format-stress-sanitizer-test - - name: format-stress-sanitizer-smoke-test - # Temporarily disabled (WT-6255) - # - name: format-stress-sanitizer-lsm-test - - name: split-stress-test - - name: format-stress-test - - name: format-stress-smoke-test - - name: race-condition-stress-sanitizer-test - - name: checkpoint-stress-test + - name: ".stress-test-1" + - name: ".stress-test-2" + - name: ".stress-test-3" + - name: ".stress-test-4" - name: package display_name: "~ Package" @@ -2433,18 +2511,19 @@ buildvariants: - ubuntu1804-power8-test expansions: smp_command: -j $(grep -c ^processor /proc/cpuinfo) - posix_configure_flags: --enable-silent-rules --enable-diagnostic --enable-python --enable-zlib --enable-snappy --enable-strict --enable-static --prefix=$(pwd)/LOCAL_INSTALL make_command: PATH=/opt/mongodbtoolchain/v3/bin:$PATH make - test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs top_srcdir=$(pwd)/.. top_builddir=$(pwd) + test_env_vars: + PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs + top_srcdir=$(pwd)/.. top_builddir=$(pwd) + posix_configure_flags: + --enable-silent-rules --enable-diagnostic --enable-python --enable-zlib --enable-snappy + --enable-strict --enable-static --prefix=$(pwd)/LOCAL_INSTALL tasks: - name: compile - name: unit-test - - name: format-stress-ppc-zseries-test - - name: format-stress-smoke-test - name: format-wtperf-test - # Replace the below two tests with format-stress-sanitizer-ppc-test and format-stress-sanitizer-smoke-test after BUILD-10248 fix. - - name: format-stress-sanitizer-ppc-test - - name: format-stress-sanitizer-smoke-ppc-test + - name: ".stress-test-ppc-1" + - name: ".stress-test-ppc-2" - name: ubuntu1804-zseries display_name: "~ Ubuntu 18.04 zSeries" @@ -2452,13 +2531,15 @@ buildvariants: - ubuntu1804-zseries-test expansions: smp_command: -j $(grep -c ^processor /proc/cpuinfo) - posix_configure_flags: --enable-silent-rules --enable-diagnostic --enable-python --enable-zlib --enable-snappy --enable-strict --enable-static --prefix=$(pwd)/LOCAL_INSTALL make_command: PATH=/opt/mongodbtoolchain/v3/bin:$PATH make - test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs:$(pwd)/lang/python top_srcdir=$(pwd)/.. top_builddir=$(pwd) + test_env_vars: + PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs:$(pwd)/lang/python + top_srcdir=$(pwd)/.. top_builddir=$(pwd) + posix_configure_flags: + --enable-silent-rules --enable-diagnostic --enable-python --enable-zlib --enable-snappy + --enable-strict --enable-static --prefix=$(pwd)/LOCAL_INSTALL tasks: - name: compile - name: unit-test - - name: recovery-stress-test - - name: split-stress-test - - name: format-stress-ppc-zseries-test - - name: format-stress-smoke-test + - name: ".stress-test-zseries-1" + - name: ".stress-test-zseries-2" diff --git a/src/third_party/wiredtiger/test/suite/test_bug019.py b/src/third_party/wiredtiger/test/suite/test_bug019.py index a11581fbc0f..0cfcc2b13cb 100644 --- a/src/third_party/wiredtiger/test/suite/test_bug019.py +++ b/src/third_party/wiredtiger/test/suite/test_bug019.py @@ -57,6 +57,8 @@ class test_bug019(wttest.WiredTigerTestCase): if i % 500 == 0: prealloc = self.get_prealloc_stat() if prealloc > self.max_prealloc: + self.pr("Updating max_prealloc from " + str(self.max_prealloc)) + self.pr(" to new prealloc " + str(prealloc)) self.max_prealloc = prealloc c.close() @@ -78,6 +80,9 @@ class test_bug019(wttest.WiredTigerTestCase): start_prealloc = self.get_prealloc_stat() self.populate(self.entries) self.session.checkpoint() + if self.max_prealloc <= start_prealloc: + self.pr("FAILURE: max_prealloc " + str(self.max_prealloc)) + self.pr("FAILURE: start_prealloc " + str(start_prealloc)) self.assertTrue(self.max_prealloc > start_prealloc) # Loop, making sure pre-allocation is working and the range is moving. diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py new file mode 100755 index 00000000000..e79c55203bf --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2020 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import fnmatch, os, shutil, threading, time +from helper import copy_wiredtiger_home +from test_rollback_to_stable01 import test_rollback_to_stable_base +from wiredtiger import stat +from wtdataset import SimpleDataSet +from wtscenario import make_scenarios +from wtthread import checkpoint_thread, op_thread + +def timestamp_str(t): + return '%x' % t + +# test_rollback_to_stable10.py +# Test the rollback to stable operation performs sweeping history store. +class test_rollback_to_stable10(test_rollback_to_stable_base): + session_config = 'isolation=snapshot' + + prepare_values = [ + ('no_prepare', dict(prepare=False)), + ('prepare', dict(prepare=True)) + ] + + scenarios = make_scenarios(prepare_values) + + def conn_config(self): + config = 'cache_size=5MB,statistics=(all),statistics_log=(json,on_close,wait=1),log=(enabled=true),timing_stress_for_test=[history_store_checkpoint_delay]' + return config + + def simulate_crash_restart(self, olddir, newdir): + ''' Simulate a crash from olddir and restart in newdir. ''' + # with the connection still open, copy files to new directory + shutil.rmtree(newdir, ignore_errors=True) + os.mkdir(newdir) + for fname in os.listdir(olddir): + fullname = os.path.join(olddir, fname) + # Skip lock file on Windows since it is locked + if os.path.isfile(fullname) and \ + "WiredTiger.lock" not in fullname and \ + "Tmplog" not in fullname and \ + "Preplog" not in fullname: + shutil.copy(fullname, newdir) + # + # close the original connection and open to new directory + # NOTE: This really cannot test the difference between the + # write-no-sync (off) version of log_flush and the sync + # version since we're not crashing the system itself. + # + self.close_conn() + self.conn = self.setUpConnectionOpen(newdir) + self.session = self.setUpSessionOpen(self.conn) + + def test_rollback_to_stable(self): + nrows = 1000 + + # Create a table without logging. + uri_1 = "table:rollback_to_stable10_1" + ds_1 = SimpleDataSet( + self, uri_1, 0, key_format="i", value_format="S", config='log=(enabled=false)') + ds_1.populate() + + # Create another table without logging. + uri_2 = "table:rollback_to_stable10_2" + ds_2 = SimpleDataSet( + self, uri_2, 0, key_format="i", value_format="S", config='log=(enabled=false)') + ds_2.populate() + + # Pin oldest and stable to timestamp 10. + self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + + ',stable_timestamp=' + timestamp_str(10)) + + value_a = "aaaaa" * 100 + value_b = "bbbbb" * 100 + value_c = "ccccc" * 100 + value_d = "ddddd" * 100 + value_e = "eeeee" * 100 + value_f = "fffff" * 100 + + # Perform several updates. + self.large_updates(uri_1, value_d, ds_1, nrows, 20) + self.large_updates(uri_1, value_c, ds_1, nrows, 30) + self.large_updates(uri_1, value_b, ds_1, nrows, 40) + self.large_updates(uri_1, value_a, ds_1, nrows, 50) + + self.large_updates(uri_2, value_d, ds_2, nrows, 20) + self.large_updates(uri_2, value_c, ds_2, nrows, 30) + self.large_updates(uri_2, value_b, ds_2, nrows, 40) + self.large_updates(uri_2, value_a, ds_2, nrows, 50) + + # Verify data is visible and correct. + self.check(value_d, uri_1, nrows, 20) + self.check(value_c, uri_1, nrows, 30) + self.check(value_b, uri_1, nrows, 40) + self.check(value_a, uri_1, nrows, 50) + + self.check(value_d, uri_2, nrows, 20) + self.check(value_c, uri_2, nrows, 30) + self.check(value_b, uri_2, nrows, 40) + self.check(value_a, uri_2, nrows, 50) + + # Pin stable to timestamp 60 if prepare otherwise 50. + if self.prepare: + self.conn.set_timestamp('stable_timestamp=' + timestamp_str(60)) + else: + self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50)) + + # Create a checkpoint thread + done = threading.Event() + ckpt = checkpoint_thread(self.conn, done) + ckpt.start() + + # Perform several updates in parallel with checkpoint. + self.large_updates(uri_1, value_e, ds_1, nrows, 70) + self.large_updates(uri_2, value_e, ds_2, nrows, 70) + self.large_updates(uri_1, value_f, ds_1, nrows, 80) + self.large_updates(uri_2, value_f, ds_2, nrows, 80) + + done.set() + ckpt.join() + + # Simulate a server crash and restart. + self.simulate_crash_restart(".", "RESTART") + + # Check that the correct data is seen at and after the stable timestamp. + self.check(value_a, uri_1, nrows, 50) + self.check(value_a, uri_1, nrows, 80) + self.check(value_b, uri_1, nrows, 40) + self.check(value_c, uri_1, nrows, 30) + self.check(value_d, uri_1, nrows, 20) + + # Check that the correct data is seen at and after the stable timestamp. + self.check(value_a, uri_2, nrows, 50) + self.check(value_a, uri_2, nrows, 80) + self.check(value_b, uri_2, nrows, 40) + self.check(value_c, uri_2, nrows, 30) + self.check(value_d, uri_2, nrows, 20) + + stat_cursor = self.session.open_cursor('statistics:', None, None) + calls = stat_cursor[stat.conn.txn_rts][2] + hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2] + hs_sweep = stat_cursor[stat.conn.txn_rts_sweep_hs_keys][2] + keys_removed = stat_cursor[stat.conn.txn_rts_keys_removed][2] + keys_restored = stat_cursor[stat.conn.txn_rts_keys_restored][2] + pages_visited = stat_cursor[stat.conn.txn_rts_pages_visited][2] + upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2] + stat_cursor.close() + + self.assertEqual(calls, 0) + self.assertEqual(keys_removed, 0) + self.assertEqual(keys_restored, 0) + self.assertGreaterEqual(upd_aborted, 0) + self.assertGreater(pages_visited, 0) + self.assertGreaterEqual(hs_removed, 0) + self.assertGreater(hs_sweep, 0) + +if __name__ == '__main__': + wttest.run() -- cgit v1.2.1