From bbdf0a11d1c61be0760a829e82799129beac7be0 Mon Sep 17 00:00:00 2001 From: Luke Chen Date: Fri, 8 May 2020 19:05:30 +1000 Subject: Import wiredtiger: 404b4a70af14e7d3aecf7f206380884af5d06786 from branch mongodb-4.4 ref: 18dfb9e58e..404b4a70af for: 4.4.0-rc5 WT-6093 Fix and reenable test_random_abort in recovery-stress-test WT-6097 Let the Python test suite runner allocate tests to buckets for evergreen WT-6128 Reduce the data size in test_checkpoint04 so it completes quickly WT-6131 Add a test to verify checkpointing truncation in durable history WT-6137 Fix calculation of bits versus bytes for incremental bitmap WT-6138 Turn off non-timestamp testing in schema_abort WT-6141 Disable checkpoint deletion during backup WT-6142 Fix the assert with no more than one aborted update in chain WT-6144 Avoid configures verify_metadata in test/format when running on historic builds WT-6152 Fix accessing checkpoint durable timestamp including older version WT-6156 Enable format to select "backup.incremental=log" --- src/third_party/wiredtiger/import.data | 2 +- src/third_party/wiredtiger/src/block/block_ckpt.c | 23 ++++--- src/third_party/wiredtiger/src/btree/bt_sync.c | 6 +- src/third_party/wiredtiger/src/include/meta.h | 2 +- src/third_party/wiredtiger/src/meta/meta_ckpt.c | 20 ++++++ .../wiredtiger/src/reconcile/rec_visibility.c | 30 ++++++--- src/third_party/wiredtiger/src/txn/txn.c | 3 +- src/third_party/wiredtiger/src/txn/txn_ckpt.c | 15 +++-- .../wiredtiger/test/csuite/random_abort/smoke.sh | 4 -- .../wiredtiger/test/csuite/schema_abort/smoke.sh | 3 +- src/third_party/wiredtiger/test/evergreen.yml | 55 ++++++---------- src/third_party/wiredtiger/test/format/config.c | 8 ++- src/third_party/wiredtiger/test/format/wts.c | 2 + src/third_party/wiredtiger/test/suite/run.py | 53 +++++++++++++-- .../wiredtiger/test/suite/test_backup01.py | 13 ++-- .../wiredtiger/test/suite/test_checkpoint04.py | 4 +- .../wiredtiger/test/suite/test_checkpoint05.py | 5 +- .../wiredtiger/test/suite/test_checkpoint06.py | 77 ++++++++++++++++++++++ src/third_party/wiredtiger/test/suite/test_hs12.py | 2 +- 19 files changed, 241 insertions(+), 86 deletions(-) mode change 100644 => 100755 src/third_party/wiredtiger/test/suite/test_checkpoint04.py create mode 100644 src/third_party/wiredtiger/test/suite/test_checkpoint06.py diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 4fcc4a17180..a08ef7d90d9 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-4.4", - "commit": "18dfb9e58e39927696affcd8e362364e23e1aa59" + "commit": "404b4a70af14e7d3aecf7f206380884af5d06786" } diff --git a/src/third_party/wiredtiger/src/block/block_ckpt.c b/src/third_party/wiredtiger/src/block/block_ckpt.c index 292917be5d6..7480ddfca2b 100644 --- a/src/third_party/wiredtiger/src/block/block_ckpt.c +++ b/src/third_party/wiredtiger/src/block/block_ckpt.c @@ -661,14 +661,20 @@ static int __ckpt_add_blkmod_entry( WT_SESSION_IMPL *session, WT_BLOCK_MODS *blk_mod, wt_off_t offset, wt_off_t len) { - uint64_t end, start; - uint32_t end_buf_bytes, end_rdup_bytes; + uint64_t end_bit, start_bit; + uint32_t end_buf_bytes, end_rdup_bits, end_rdup_bytes; WT_ASSERT(session, blk_mod->granularity != 0); - start = (uint64_t)offset / blk_mod->granularity; - end = (uint64_t)(offset + len) / blk_mod->granularity; - WT_ASSERT(session, end < UINT32_MAX); - end_rdup_bytes = WT_MAX(__wt_rduppo2((uint32_t)end, 8), WT_BLOCK_MODS_LIST_MIN); + /* + * Figure out how the starting and ending bits based on the granularity and our offset and + * length. + */ + start_bit = (uint64_t)offset / blk_mod->granularity; + end_bit = (uint64_t)(offset + len - 1) / blk_mod->granularity; + WT_ASSERT(session, end_bit < UINT32_MAX); + /* We want to grow the bitmap by 64 bits, or 8 bytes at a time. */ + end_rdup_bits = WT_MAX(__wt_rduppo2((uint32_t)end_bit, 64), WT_BLOCK_MODS_LIST_MIN); + end_rdup_bytes = end_rdup_bits >> 3; end_buf_bytes = (uint32_t)blk_mod->nbits >> 3; /* * We are doing a lot of shifting. Make sure that the number of bytes we end up with is a @@ -687,11 +693,10 @@ __ckpt_add_blkmod_entry( memset( (uint8_t *)blk_mod->bitstring.mem + end_buf_bytes, 0, end_rdup_bytes - end_buf_bytes); } - blk_mod->nbits = end_rdup_bytes << 3; + blk_mod->nbits = end_rdup_bits; } - /* Set all the bits needed to record this offset/length pair. */ - __bit_nset(blk_mod->bitstring.mem, start, end); + __bit_nset(blk_mod->bitstring.mem, start_bit, end_bit); return (0); } diff --git a/src/third_party/wiredtiger/src/btree/bt_sync.c b/src/third_party/wiredtiger/src/btree/bt_sync.c index fd36f6b24f9..100aeed5105 100644 --- a/src/third_party/wiredtiger/src/btree/bt_sync.c +++ b/src/third_party/wiredtiger/src/btree/bt_sync.c @@ -373,9 +373,9 @@ __wt_sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) /* * Skip all deleted pages. For a page to be marked deleted, it must have been evicted from cache * and marked clean. Checkpoint should never instantiate deleted pages: if a truncate is not - * visible to the checkpoint, the on-disk version is correct. If the truncate is visible, we - * skip over the child page when writing its parent. We check whether a truncate is visible in - * the checkpoint as part of reconciling internal pages (specifically in __rec_child_modify). + * visible to the checkpoint, the on-disk version is correct. If the truncate is visible to + * checkpoint, we write a proxy cell to its parent. We check whether a truncate is visible in + * the checkpoint as part of reconciling internal pages (specifically in __rec_child_deleted). */ LF_SET(WT_READ_DELETED_SKIP); diff --git a/src/third_party/wiredtiger/src/include/meta.h b/src/third_party/wiredtiger/src/include/meta.h index f9160c6b28c..0b0d436f146 100644 --- a/src/third_party/wiredtiger/src/include/meta.h +++ b/src/third_party/wiredtiger/src/include/meta.h @@ -85,7 +85,7 @@ struct __wt_blkincr { /* * At the default granularity, this is enough for blocks in a 2G file. */ -#define WT_BLOCK_MODS_LIST_MIN 16 /* Initial bytes for bitmap. */ +#define WT_BLOCK_MODS_LIST_MIN 128 /* Initial bits for bitmap. */ struct __wt_block_mods { const char *id_str; diff --git a/src/third_party/wiredtiger/src/meta/meta_ckpt.c b/src/third_party/wiredtiger/src/meta/meta_ckpt.c index 21acec991c6..9742d16a9ef 100644 --- a/src/third_party/wiredtiger/src/meta/meta_ckpt.c +++ b/src/third_party/wiredtiger/src/meta/meta_ckpt.c @@ -598,6 +598,16 @@ __ckpt_load(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *k, WT_CONFIG_ITEM *v, WT_C WT_RET_NOTFOUND_OK(ret); if (ret != WT_NOTFOUND && a.len != 0) ckpt->ta.newest_start_durable_ts = (uint64_t)a.val; + else { + /* + * Backward compatibility changes, as the parameter name is different in older versions of + * WT, make sure that we read older format in case if we didn't find the newer format name. + */ + ret = __wt_config_subgets(session, v, "start_durable_ts", &a); + WT_RET_NOTFOUND_OK(ret); + if (ret != WT_NOTFOUND && a.len != 0) + ckpt->ta.newest_start_durable_ts = (uint64_t)a.val; + } ret = __wt_config_subgets(session, v, "newest_stop_ts", &a); WT_RET_NOTFOUND_OK(ret); @@ -613,6 +623,16 @@ __ckpt_load(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *k, WT_CONFIG_ITEM *v, WT_C WT_RET_NOTFOUND_OK(ret); if (ret != WT_NOTFOUND && a.len != 0) ckpt->ta.newest_stop_durable_ts = (uint64_t)a.val; + else { + /* + * Backward compatibility changes, as the parameter name is different in older versions of + * WT, make sure that we read older format in case if we didn't find the newer format name. + */ + ret = __wt_config_subgets(session, v, "stop_durable_ts", &a); + WT_RET_NOTFOUND_OK(ret); + if (ret != WT_NOTFOUND && a.len != 0) + ckpt->ta.newest_stop_durable_ts = (uint64_t)a.val; + } ret = __wt_config_subgets(session, v, "prepare", &a); WT_RET_NOTFOUND_OK(ret); diff --git a/src/third_party/wiredtiger/src/reconcile/rec_visibility.c b/src/third_party/wiredtiger/src/reconcile/rec_visibility.c index aa44301a21d..93991af5983 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_visibility.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_visibility.c @@ -378,7 +378,7 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, v tombstone = upd; /* Find the update this tombstone applies to. */ - if (!__wt_txn_visible_all(session, upd->txnid, upd->start_ts)) { + if (!__wt_txn_upd_visible_all(session, upd)) { while (upd->next != NULL && upd->next->txnid == WT_TXN_ABORTED) upd = upd->next; WT_ASSERT(session, upd->next == NULL || upd->next->txnid != WT_TXN_ABORTED); @@ -387,12 +387,12 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, v upd_select->upd = upd = upd->next; } } - if (upd != NULL) { + if (upd != NULL) /* The beginning of the validity window is the selected update's time pair. */ __wt_time_window_set_start(select_tw, upd); - } else if (select_tw->stop_ts != WT_TS_NONE || select_tw->stop_txn != WT_TXN_NONE) { + else if (select_tw->stop_ts != WT_TS_NONE || select_tw->stop_txn != WT_TXN_NONE) { /* If we only have a tombstone in the update list, we must have an ondisk value. */ - WT_ASSERT(session, vpack != NULL && tombstone != NULL); + WT_ASSERT(session, vpack != NULL && tombstone != NULL && last_upd->next == NULL); /* * It's possible to have a tombstone as the only update in the update list. If we * reconciled before with only a single update and then read the page back into cache, @@ -404,12 +404,22 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, v * window ends when this tombstone started. */ WT_ERR(__rec_append_orig_value(session, page, tombstone, vpack)); - WT_ASSERT(session, last_upd->next != NULL && - last_upd->next->txnid == vpack->tw.start_txn && - last_upd->next->start_ts == vpack->tw.start_ts && - last_upd->next->type == WT_UPDATE_STANDARD && last_upd->next->next == NULL); - upd_select->upd = last_upd->next; - __wt_time_window_set_start(select_tw, last_upd->next); + + /* + * We may have updated the global transaction concurrently and the tombstone is now + * globally visible. In this case, the on page value is not appended. Check that. + */ + if (last_upd->next != NULL) { + WT_ASSERT(session, last_upd->next->txnid == vpack->tw.start_txn && + last_upd->next->start_ts == vpack->tw.start_ts && + last_upd->next->type == WT_UPDATE_STANDARD && last_upd->next->next == NULL); + upd_select->upd = last_upd->next; + __wt_time_window_set_start(select_tw, last_upd->next); + } else { + WT_ASSERT( + session, __wt_txn_upd_visible_all(session, tombstone) && upd_select->upd == NULL); + upd_select->upd = tombstone; + } } } diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index 0fdf022bd51..bcde2932ab2 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -718,8 +718,7 @@ __txn_fixup_prepared_update(WT_SESSION_IMPL *session, WT_TXN_OP *op, WT_CURSOR * upd->next = cbt->ins->upd; else if (cbt->ref->page->modify != NULL && cbt->ref->page->modify->mod_row_update != NULL) upd->next = cbt->ref->page->modify->mod_row_update[cbt->slot]; - WT_ASSERT(session, - upd->next != NULL && upd->next->next == NULL && upd->next->txnid == WT_TXN_ABORTED); + WT_ASSERT(session, upd->next != NULL && upd->next->txnid == WT_TXN_ABORTED); /* Append a tombstone if the stop timestamp exists. */ if (hs_stop_ts != WT_TS_MAX) { diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c index 577281a2a47..2297f957e5f 100644 --- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c +++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c @@ -1259,12 +1259,19 @@ __checkpoint_lock_dirty_tree_int(WT_SESSION_IMPL *session, bool is_checkpoint, b continue; is_wt_ckpt = WT_PREFIX_MATCH(ckpt->name, WT_CHECKPOINT); +/* + * If there is a hot backup, don't delete any WiredTiger checkpoint that could possibly have been + * created before the backup started. Fail if trying to delete any other named checkpoint. + */ +#ifdef DISABLED_CODE + if (conn->hot_backup_start != 0 && ckpt->sec <= conn->hot_backup_start) { +#else /* - * If there is a hot backup, don't delete any WiredTiger checkpoint that could possibly have - * been created before the backup started. Fail if trying to delete any other named - * checkpoint. + * N.B. Despite the comment above, dropping checkpoints during backup can corrupt the + * backup. For now we retain all WiredTiger checkpoints. */ - if (conn->hot_backup_start != 0 && ckpt->sec <= conn->hot_backup_start) { + if (conn->hot_backup_start != 0) { +#endif if (is_wt_ckpt) { F_CLR(ckpt, WT_CKPT_DELETE); continue; diff --git a/src/third_party/wiredtiger/test/csuite/random_abort/smoke.sh b/src/third_party/wiredtiger/test/csuite/random_abort/smoke.sh index aea7fedf4b9..713b000b4f1 100755 --- a/src/third_party/wiredtiger/test/csuite/random_abort/smoke.sh +++ b/src/third_party/wiredtiger/test/csuite/random_abort/smoke.sh @@ -9,10 +9,6 @@ set -e top_builddir=${top_builddir:-../../build_posix} top_srcdir=${top_srcdir:-../..} -#FIXME-WT-6093: reenable calls to test_random_abort -echo "Warning: test_random_abort temporarily disabled" -exit 0 - $TEST_WRAPPER $top_builddir/test/csuite/test_random_abort -t 10 -T 5 $TEST_WRAPPER $top_builddir/test/csuite/test_random_abort -m -t 10 -T 5 $TEST_WRAPPER $top_builddir/test/csuite/test_random_abort -C -t 10 -T 5 diff --git a/src/third_party/wiredtiger/test/csuite/schema_abort/smoke.sh b/src/third_party/wiredtiger/test/csuite/schema_abort/smoke.sh index 2e52b6f438c..78effe1acab 100755 --- a/src/third_party/wiredtiger/test/csuite/schema_abort/smoke.sh +++ b/src/third_party/wiredtiger/test/csuite/schema_abort/smoke.sh @@ -13,5 +13,6 @@ $TEST_WRAPPER $top_builddir/test/csuite/test_schema_abort -t 10 -T 5 $TEST_WRAPPER $top_builddir/test/csuite/test_schema_abort -m -t 10 -T 5 $TEST_WRAPPER $top_builddir/test/csuite/test_schema_abort -C -t 10 -T 5 $TEST_WRAPPER $top_builddir/test/csuite/test_schema_abort -C -m -t 10 -T 5 -$TEST_WRAPPER $top_builddir/test/csuite/test_schema_abort -m -t 10 -T 5 -z +# FIXME: In WT-6116 the test is failing if timestamps are turned off. +#$TEST_WRAPPER $top_builddir/test/csuite/test_schema_abort -m -t 10 -T 5 -z $TEST_WRAPPER $top_builddir/test/csuite/test_schema_abort -m -t 10 -T 5 -x diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index 62b5c205035..1e306fe4b2e 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -198,9 +198,7 @@ functions: script: | set -o errexit set -o verbose - #FIXME-WT-6093: reenable all calls to test_random_abort - echo "Warning: test_random_abort temporarily disabled" - ##${test_env_vars|} ./test_random_abort ${random_abort_args|} 2>&1 + ${test_env_vars|} ./test_random_abort ${random_abort_args|} 2>&1 "timestamp abort test": command: shell.exec params: @@ -228,20 +226,19 @@ functions: for i in $(seq ${times|1}); do # Run the various combinations of args. Let time and threads be random. # Run current version with write-no-sync txns. - echo "Warning: test_random_abort temporarily disabled" - ##${test_env_vars|} ./test_random_abort 2>&1 + ${test_env_vars|} ./test_random_abort 2>&1 ${test_env_vars|} ./test_timestamp_abort 2>&1 # Current version with memory-based txns (MongoDB usage). - ##${test_env_vars|} ./test_random_abort -m 2>&1 + ${test_env_vars|} ./test_random_abort -m 2>&1 ${test_env_vars|} ./test_timestamp_abort -m 2>&1 # V1 log compatibility mode with write-no-sync txns. - ##${test_env_vars|} ./test_random_abort -C 2>&1 + ${test_env_vars|} ./test_random_abort -C 2>&1 ${test_env_vars|} ./test_timestamp_abort -C 2>&1 # V1 log compatibility mode with memory-based txns. - ##${test_env_vars|} ./test_random_abort -C -m 2>&1 + ${test_env_vars|} ./test_random_abort -C -m 2>&1 ${test_env_vars|} ./test_timestamp_abort -C -m 2>&1 ${test_env_vars|} ./test_truncated_log ${truncated_log_args|} 2>&1 @@ -1249,12 +1246,8 @@ tasks: dependent_task: compile-linux-no-ftruncate - func: "unit test" - # Break out Python unit tests into multiple buckets/tasks based on test name and runtime - # The test/suite/run.py script can work out test names by casting each command argument - # with "test_" prefix and "*.py" postfix. - # - # One example: - # "test/suite/run.py [ab]" will be translated to testing "test_a*.py" and "test_b*.py" + # Break out Python unit tests into multiple buckets/tasks. We have a fixed number of buckets, + # and we use the -b option of the test/suite/run.py script to split up the tests. - name: unit-test-bucket00 tags: ["pull_request", "unit_test"] @@ -1269,7 +1262,7 @@ tasks: set -o errexit set -o verbose - ${test_env_vars|} ${python_binary|python} ../test/suite/run.py [ab] ${unit_test_args|-v 2} ${smp_command|} 2>&1 + ${test_env_vars|} ${python_binary|python} ../test/suite/run.py -b 0/11 ${unit_test_args|-v 2} ${smp_command|} 2>&1 - name: unit-test-bucket01 tags: ["pull_request", "unit_test"] @@ -1284,8 +1277,7 @@ tasks: set -o errexit set -o verbose - # Reserve this bucket only for compat tests, which take a long time to run - ${test_env_vars|} ${python_binary|python} ../test/suite/run.py compat ${unit_test_args|-v 2} ${smp_command|} 2>&1 + ${test_env_vars|} ${python_binary|python} ../test/suite/run.py -b 1/11 ${unit_test_args|-v 2} ${smp_command|} 2>&1 - name: unit-test-bucket02 tags: ["pull_request", "unit_test"] @@ -1300,9 +1292,7 @@ tasks: set -o errexit set -o verbose - # Non-compat tests in the 'c' family - non_compat_tests=$(ls ../test/suite/test_c*.py | xargs -n1 basename | grep -v compat) - ${test_env_vars|} ${python_binary|python} ../test/suite/run.py $non_compat_tests ${unit_test_args|-v 2} ${smp_command|} 2>&1 + ${test_env_vars|} ${python_binary|python} ../test/suite/run.py -b 2/11 ${unit_test_args|-v 2} ${smp_command|} 2>&1 - name: unit-test-bucket03 tags: ["pull_request", "unit_test"] @@ -1317,7 +1307,8 @@ tasks: set -o errexit set -o verbose - ${test_env_vars|} ${python_binary|python} ../test/suite/run.py [defg] ${unit_test_args|-v 2} ${smp_command|} 2>&1 + ${test_env_vars|} ${python_binary|python} ../test/suite/run.py -b 3/11 ${unit_test_args|-v 2} ${smp_command|} 2>&1 + - name: unit-test-bucket04 tags: ["pull_request", "unit_test"] depends_on: @@ -1331,9 +1322,7 @@ tasks: set -o errexit set -o verbose - # Reserve this bucket only for history store tests, which take a long time to run - ${test_env_vars|} ${python_binary|python} ../test/suite/run.py hs ${unit_test_args|-v 2} ${smp_command|} 2>&1 - + ${test_env_vars|} ${python_binary|python} ../test/suite/run.py -b 4/11 ${unit_test_args|-v 2} ${smp_command|} 2>&1 - name: unit-test-bucket05 tags: ["pull_request", "unit_test"] @@ -1348,10 +1337,7 @@ tasks: set -o errexit set -o verbose - # Non-history store tests in the 'h' family - non_ts_tests=$(ls ../test/suite/test_h*.py | xargs -n1 basename | grep -v _hs) - ${test_env_vars|} ${python_binary|python} ../test/suite/run.py $non_ts_tests ${unit_test_args|-v 2} ${smp_command|} 2>&1 - ${test_env_vars|} ${python_binary|python} ../test/suite/run.py [ijk] ${unit_test_args|-v 2} ${smp_command|} 2>&1 + ${test_env_vars|} ${python_binary|python} ../test/suite/run.py -b 5/11 ${unit_test_args|-v 2} ${smp_command|} 2>&1 - name: unit-test-bucket06 tags: ["pull_request", "unit_test"] @@ -1366,7 +1352,7 @@ tasks: set -o errexit set -o verbose - ${test_env_vars|} ${python_binary|python} ../test/suite/run.py [lmnopq] ${unit_test_args|-v 2} ${smp_command|} 2>&1 + ${test_env_vars|} ${python_binary|python} ../test/suite/run.py -b 6/11 ${unit_test_args|-v 2} ${smp_command|} 2>&1 - name: unit-test-bucket07 tags: ["pull_request", "unit_test"] @@ -1381,7 +1367,7 @@ tasks: set -o errexit set -o verbose - ${test_env_vars|} ${python_binary|python} ../test/suite/run.py [rs] ${unit_test_args|-v 2} ${smp_command|} 2>&1 + ${test_env_vars|} ${python_binary|python} ../test/suite/run.py -b 7/11 ${unit_test_args|-v 2} ${smp_command|} 2>&1 - name: unit-test-bucket08 tags: ["pull_request", "unit_test"] @@ -1396,8 +1382,7 @@ tasks: set -o errexit set -o verbose - # Reserve this bucket only for timestamp tests, which take a long time to run - ${test_env_vars|} ${python_binary|python} ../test/suite/run.py timestamp ${unit_test_args|-v 2} ${smp_command|} 2>&1 + ${test_env_vars|} ${python_binary|python} ../test/suite/run.py -b 8/11 ${unit_test_args|-v 2} ${smp_command|} 2>&1 - name: unit-test-bucket09 tags: ["pull_request", "unit_test"] @@ -1412,9 +1397,7 @@ tasks: set -o errexit set -o verbose - # Non-timestamp tests in the 't' family - non_ts_tests=$(ls ../test/suite/test_t*.py | xargs -n1 basename | grep -v timestamp) - ${test_env_vars|} ${python_binary|python} ../test/suite/run.py $non_ts_tests ${unit_test_args|-v 2} ${smp_command|} 2>&1 + ${test_env_vars|} ${python_binary|python} ../test/suite/run.py -b 9/11 ${unit_test_args|-v 2} ${smp_command|} 2>&1 - name: unit-test-bucket10 tags: ["pull_request", "unit_test"] @@ -1429,7 +1412,7 @@ tasks: set -o errexit set -o verbose - ${test_env_vars|} ${python_binary|python} ../test/suite/run.py [uvwxyz] ${unit_test_args|-v 2} ${smp_command|} 2>&1 + ${test_env_vars|} ${python_binary|python} ../test/suite/run.py -b 10/11 ${unit_test_args|-v 2} ${smp_command|} 2>&1 # End of Python unit test tasks diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c index 54a09229ce4..eb86c0860e7 100644 --- a/src/third_party/wiredtiger/test/format/config.c +++ b/src/third_party/wiredtiger/test/format/config.c @@ -264,8 +264,13 @@ static void config_backup_incr(void) { /* Incremental backup requires backup. */ - if (g.c_backups == 0) + if (g.c_backups == 0) { + if (!config_is_perm("backup.incremental")) + config_single("backup.incremental=off", false); + if (g.c_backup_incr_flag != INCREMENTAL_OFF) + testutil_die(EINVAL, "backup.incremental requires backups be configured"); return; + } /* * Incremental backup using log files is incompatible with logging archival. Testing log file @@ -298,6 +303,7 @@ config_backup_incr(void) if (g.c_logging_archive) config_single("logging.archive=0", false); config_single("backup.incremental=log", false); + break; } /* FALLTHROUGH */ case 7: /* 40% block based incremental */ diff --git a/src/third_party/wiredtiger/test/format/wts.c b/src/third_party/wiredtiger/test/format/wts.c index ea72c4c7f0f..66a319a9982 100644 --- a/src/third_party/wiredtiger/test/format/wts.c +++ b/src/third_party/wiredtiger/test/format/wts.c @@ -281,8 +281,10 @@ wts_open(const char *home, bool set_api, WT_CONNECTION **connp) CONFIG_APPEND(p, ",split_8"); CONFIG_APPEND(p, "]"); +#if WIREDTIGER_VERSION_MAJOR >= 10 if (g.c_verify) CONFIG_APPEND(p, ",verify_metadata=true"); +#endif /* Extensions. */ CONFIG_APPEND(p, diff --git a/src/third_party/wiredtiger/test/suite/run.py b/src/third_party/wiredtiger/test/suite/run.py index 5fb4e0728a5..5cf39777fca 100755 --- a/src/third_party/wiredtiger/test/suite/run.py +++ b/src/third_party/wiredtiger/test/suite/run.py @@ -105,6 +105,8 @@ def usage(): \n\ Options:\n\ --asan run with an ASAN enabled shared library\n\ + -b K/N | --batch K/N run batch K of N, 0 <= K < N. The tests\n\ + are split into N batches and the Kth is run.\n\ -C file | --configcreate file create a config file for controlling tests\n\ -c file | --config file use a config file for controlling tests\n\ -D dir | --dir dir use dir rather than WT_TEST.\n\ @@ -301,6 +303,7 @@ if __name__ == '__main__': asan = False parallel = 0 random_sample = 0 + batchtotal = batchnum = 0 configfile = None configwrite = False dirarg = None @@ -318,6 +321,24 @@ if __name__ == '__main__': if option == '-asan': asan = True continue + if option == '-batch' or option == 'b': + if batchtotal != 0 or len(args) == 0: + usage() + sys.exit(2) + # Batch expects an argument that has int slash int. + # For example "-b 4/12" + try: + left, right = args.pop(0).split('/') + batchnum = int(left) + batchtotal = int(right) + except: + print('batch argument should be nnn/nnn') + usage() + sys.exit(2) + if batchtotal <= 0 or batchnum < 0 or batchnum >= batchtotal: + usage() + sys.exit(2) + continue if option == '-dir' or option == 'D': if dirarg != None or len(args) == 0: usage() @@ -511,12 +532,34 @@ if __name__ == '__main__': if debug: import pdb pdb.set_trace() + if batchtotal != 0: + # For test batching, we want to split up all the tests evenly, and + # spread out the tests, so each batch contains tests of all kinds. We'd + # like to prioritize the lowest scenario numbers first, so if there's a + # failure, we won't have to do all X thousand of some test's scenarios + # before we see a failure in the next test. To that end, we define a + # sort function that sorts by scenario first, and test name second. + hugetests = set() + def get_sort_keys(test): + s = 0 + name = test.simpleName() + if hasattr(test, 'scenario_number'): + s = test.scenario_number + if s > 1000: + hugetests.add(name) # warn for too many scenarios + return (s, test.simpleName()) # sort by scenerio number first + all_tests = sorted(tests, key = get_sort_keys) + if not longtest: + for name in hugetests: + print("WARNING: huge test " + name + " has > 1000 scenarios.\n" + + "That is only appropriate when using the --long option.\n" + + "The number of scenerios for the test should be pruned") + + # At this point we have an ordered list of all the tests. + # Break it into just our batch. + tests = unittest.TestSuite(all_tests[batchnum::batchtotal]) if dryRun: - # We have to de-dupe here as some scenarios overlap in the same suite - dryOutput = set() - for test in tests: - dryOutput.add(test.shortDesc()) - for line in dryOutput: + for line in tests: print(line) else: result = wttest.runsuite(tests, parallel) diff --git a/src/third_party/wiredtiger/test/suite/test_backup01.py b/src/third_party/wiredtiger/test/suite/test_backup01.py index a39da2e3d18..2494945dba2 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup01.py +++ b/src/third_party/wiredtiger/test/suite/test_backup01.py @@ -192,12 +192,15 @@ class test_backup(wttest.WiredTigerTestCase, suite_subprocess): # Confirm that a named checkpoint created after a backup cursor can be dropped. # Need to pause a couple seconds; checkpoints that are assigned the same timestamp as # the backup will be pinned, even if they occur after the backup starts. + time.sleep(2) - self.session.checkpoint("name=four") - self.session.checkpoint("drop=(four)") - self.assertRaises(wiredtiger.WiredTigerError, - lambda: self.session.open_cursor( - self.objs[0][0], None, "checkpoint=four")) + # N.B. This test is temporarily disabled because deleting checkpoints during backup + # can corrupt the backup. + #self.session.checkpoint("name=four") + #self.session.checkpoint("drop=(four)") + #self.assertRaises(wiredtiger.WiredTigerError, + # lambda: self.session.open_cursor( + # self.objs[0][0], None, "checkpoint=four")) # Confirm that after closing the backup cursor the original named checkpoint can # be deleted. diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint04.py b/src/third_party/wiredtiger/test/suite/test_checkpoint04.py old mode 100644 new mode 100755 index 5ba0612c680..eb6f0313d61 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint04.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint04.py @@ -66,8 +66,8 @@ class test_checkpoint04(wttest.WiredTigerTestCase): return val def test_checkpoint_stats(self): - nrows = 1000 - ntables = 50 + nrows = 100 + ntables = 5 self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + ',stable_timestamp=' + timestamp_str(10)) diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint05.py b/src/third_party/wiredtiger/test/suite/test_checkpoint05.py index 58af3003a60..041f1e53718 100644 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint05.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint05.py @@ -76,7 +76,10 @@ class test_checkpoint05(wttest.WiredTigerTestCase): # is generous. But if WT isn't deleting checkpoints there would # be about 30x more checkpoints here. final_count = self.count_checkpoints() - self.assertTrue (final_count < initial_count * 3) + + # N.B. This test is temporarily disabled because deleting checkpoints during backup + # can corrupt the backup. + # self.assertTrue (final_count < initial_count * 3) self.session.close() diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint06.py b/src/third_party/wiredtiger/test/suite/test_checkpoint06.py new file mode 100644 index 00000000000..2ec5f0e472c --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint06.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2020 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import time +import wiredtiger, wttest + +def timestamp_str(t): + return '%x' % t + +# test_checkpoint06.py +# Verify that we rollback the truncation that is committed after stable +# timestamp in the checkpoint. +class test_checkpoint06(wttest.WiredTigerTestCase): + conn_config = 'create,cache_size=50MB' + session_config = 'isolation=snapshot' + + def test_rollback_truncation_in_checkpoint(self): + self.uri = 'table:ckpt06' + self.session.create(self.uri, 'key_format=i,value_format=S') + + value = "abcdefghijklmnopqrstuvwxyz" * 3 + self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + cursor = self.session.open_cursor(self.uri) + self.session.begin_transaction() + # Setup: Insert some data + for i in range(10000): + cursor[i] = value + self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.conn.set_timestamp('stable_timestamp=' + timestamp_str(2)) + + # Flush everything to disk + self.reopen_conn() + + # Truncate large portion of the data in the table + self.session.begin_transaction() + start = self.session.open_cursor(self.uri) + start.set_key(5) + end = self.session.open_cursor(self.uri) + end.set_key(9995) + self.session.truncate(None, start, None, None) + self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + + # Do a checkpoint + self.session.checkpoint() + + # rollback to stable + self.reopen_conn() + + # Verify the truncation is rolled back. + cursor = self.session.open_cursor(self.uri) + for i in range(1000): + self.assertEqual(cursor[i], value) diff --git a/src/third_party/wiredtiger/test/suite/test_hs12.py b/src/third_party/wiredtiger/test/suite/test_hs12.py index 7403126fd5e..ed332dc3349 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs12.py +++ b/src/third_party/wiredtiger/test/suite/test_hs12.py @@ -102,4 +102,4 @@ class test_hs12(wttest.WiredTigerTestCase): self.assertEquals(cursor2.get_value(), 'AB' + value1) if __name__ == '__main__': - wttest.run() \ No newline at end of file + wttest.run() -- cgit v1.2.1