diff options
| author | Luke Chen <luke.chen@mongodb.com> | 2021-01-07 15:09:05 +1100 |
|---|---|---|
| committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-01-07 04:22:09 +0000 |
| commit | 8ea201889b4a4fafdd3acb5d651da398b0147d63 (patch) | |
| tree | 23de5bfd4624717c30400416d9d786be15c8a49b /src/third_party | |
| parent | 1034e166dd7cbe72b2a9b772b24f582fd701a14e (diff) | |
| download | mongo-8ea201889b4a4fafdd3acb5d651da398b0147d63.tar.gz | |
Import wiredtiger: 357d565eee34b54c7725a0c6f7fb65c1dffa26d1 from branch mongodb-4.4
ref: 1fcb0f1e1d..357d565eee
for: 4.4.4
WT-4625 Add a check for camelCase names
WT-5101 Make Clang Format recognise remaining loop macros
WT-6313 Added verify_metadata configuration to the exclusion list when writing to basecfg
WT-6568 Fix split generation use
WT-6677 Map read committed/uncommitted isolation to read-only transaction
WT-6710 Change the default transaction isolation to snapshot
WT-6711 Add new API WT_SESSION.reset_snapshot to update the snapshot
WT-6802 Don't set operation timer for internal and reentry api calls
WT-6848 Split the program into test and framework components
WT-6888 WTPERF reports incorrect upper bound item count during populate
WT-6969 Do not update the existing snapshot for autocommit transactions
WT-6990 New cursor debug configuration isn't considered in __wt_cursor_cache_get
WT-6996 Fix suite_random.rand32() in python test suite
WT-7027 Run the metadata checkpoint for force_stop at read-committed isolation for incremental backup
Diffstat (limited to 'src/third_party')
51 files changed, 654 insertions, 227 deletions
diff --git a/src/third_party/wiredtiger/.clang-format b/src/third_party/wiredtiger/.clang-format index 51cc1257cbb..db3bac132a4 100644 --- a/src/third_party/wiredtiger/.clang-format +++ b/src/third_party/wiredtiger/.clang-format @@ -66,7 +66,8 @@ ForEachMacros: - Q_FOREACH - BOOST_FOREACH - TAILQ_FOREACH - - WT_CELL_FOREACH_ADDR + - WT_CELL_FOREACH + - WT_CELL_FOREACH_ADDR - WT_CELL_FOREACH_KV - WT_CELL_FOREACH_VRFY - WT_CKPT_FOREACH @@ -75,9 +76,11 @@ ForEachMacros: - WT_EXT_FOREACH_OFF - WT_FIX_FOREACH - WT_INTL_FOREACH_BEGIN + - WT_INTL_FOREACH_REVERSE_BEGIN - WT_MODIFY_FOREACH_BEGIN - WT_MODIFY_FOREACH_REVERSE - WT_ROW_FOREACH + - WT_ROW_FOREACH_REVERSE - WT_SKIP_FOREACH IncludeBlocks: Preserve IncludeCategories: diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf.c b/src/third_party/wiredtiger/bench/wtperf/wtperf.c index ce08af6e407..21ecbe9afb7 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf.c +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf.c @@ -1425,9 +1425,9 @@ execute_populate(WTPERF *wtperf) wtperf->totalsec += opts->report_interval; wtperf->insert_ops = sum_pop_ops(wtperf); lprintf(wtperf, 0, 1, - "%" PRIu64 " populate inserts (%" PRIu64 " of %" PRIu32 ") in %" PRIu32 " secs (%" PRIu32 + "%" PRIu64 " populate inserts (%" PRIu64 " of %" PRIu64 ") in %" PRIu32 " secs (%" PRIu32 " total secs)", - wtperf->insert_ops - last_ops, wtperf->insert_ops, opts->icount, opts->report_interval, + wtperf->insert_ops - last_ops, wtperf->insert_ops, max_key, opts->report_interval, wtperf->totalsec); last_ops = wtperf->insert_ops; } diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 957ef40ff09..247a4777d29 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -910,7 +910,7 @@ session_config = [ option for operations that create cache pressure can starve ordinary sessions that obey the cache size.''', type='boolean'), - Config('isolation', 'read-committed', r''' + Config('isolation', 'snapshot', r''' the default isolation level for operations in this session''', choices=['read-uncommitted', 'read-committed', 'snapshot']), ] @@ -1404,6 +1404,7 @@ methods = { choices=['commit', 'first_commit', 'prepare', 'read']), ]), +'WT_SESSION.reset_snapshot' : Method([]), 'WT_SESSION.rename' : Method([]), 'WT_SESSION.reset' : Method([]), 'WT_SESSION.salvage' : Method([ diff --git a/src/third_party/wiredtiger/dist/api_err.py b/src/third_party/wiredtiger/dist/api_err.py index a1ea1974284..b2b18e1bac6 100644 --- a/src/third_party/wiredtiger/dist/api_err.py +++ b/src/third_party/wiredtiger/dist/api_err.py @@ -55,9 +55,10 @@ errors = [ 'operation would overflow cache', ''' This error is only generated when wiredtiger_open is configured to run in-memory, and an insert or update operation requires - more than the configured cache size to complete. The operation - may be retried; if a transaction is in progress, it should be - rolled back and the operation retried in a new transaction.'''), + more than the configured cache size to complete, or when an + application thread fails to do eviction within cache_max_wait_ms. + The operation may be retried; if a transaction is in progress, it + should be rolled back and the operation retried in a new transaction.'''), Error('WT_PREPARE_CONFLICT', -31808, 'conflict with a prepared update', ''' This error is generated when the application attempts to update diff --git a/src/third_party/wiredtiger/dist/s_style b/src/third_party/wiredtiger/dist/s_style index d19d180880a..e8cac692300 100755 --- a/src/third_party/wiredtiger/dist/s_style +++ b/src/third_party/wiredtiger/dist/s_style @@ -34,6 +34,20 @@ else exit 1; fi + # For files in the src directory, camel case style is not allowed. + # Ignore styling errors of external libraries. + if expr "$f" : 'src/' > /dev/null && + ! expr "$f" : 'src/os_win/' > /dev/null && + ! expr "$f" : 'src/docs/' > /dev/null && + ! expr "$f" : 'src/tags' > /dev/null && + ! expr "$f" : 'src/.*/hash_city.*' > /dev/null && + ! expr "$f" : 'src/.*/huffman.*' > /dev/null && + ! expr "$f" : 'src/checksum.*' > /dev/null; then + if egrep -r -n '\b[a-z]+[A-Z]' $f | egrep -v ':[ ]+\*|"|UNCHECKED_STRING'; then + echo "$f: Styling requires variables that use underscores to separate parts of a name instead of camel casing."; + fi + fi + egrep -w 'a a|an an|and and|are are|be be|by by|for for|from from|if if|in in[^-]|is is|it it|of of|the the|this this|to to|was was|were were|when when|with with|a an|an a|a the|the a' $f > $t test -s $t && { echo "paired typo" diff --git a/src/third_party/wiredtiger/examples/c/ex_all.c b/src/third_party/wiredtiger/examples/c/ex_all.c index e63ca868d64..94343ea58a0 100644 --- a/src/third_party/wiredtiger/examples/c/ex_all.c +++ b/src/third_party/wiredtiger/examples/c/ex_all.c @@ -846,6 +846,22 @@ transaction_ops(WT_SESSION *session_arg) /*! [transaction prepare] */ } + { + /*! [reset snapshot] */ + /* + * Resets snapshots for snapshot isolation transactions to update their existing snapshot. + * It raises an error when this API is used for isolation other than snapshot isolation + * mode. + */ + error_check(session->open_cursor(session, "table:mytable", NULL, NULL, &cursor)); + error_check(session->begin_transaction(session, "isolation=snapshot")); + cursor->set_key(cursor, "some-key"); + error_check(cursor->search(cursor)); + error_check(session->reset_snapshot(session)); + error_check(session->commit_transaction(session, NULL)); + /*! [reset snapshot] */ + } + /*! [session isolation configuration] */ /* Open a session configured for read-uncommitted isolation. */ error_check(conn->open_session(conn, NULL, "isolation=read-uncommitted", &session)); diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index f7578bb1ff4..4e65115a615 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-4.4", - "commit": "1fcb0f1e1d84c105a07588ca4be3f2e6fd25b2d4" + "commit": "357d565eee34b54c7725a0c6f7fb65c1dffa26d1" } diff --git a/src/third_party/wiredtiger/src/btree/bt_discard.c b/src/third_party/wiredtiger/src/btree/bt_discard.c index 6f3494f687c..caa179de7a1 100644 --- a/src/third_party/wiredtiger/src/btree/bt_discard.c +++ b/src/third_party/wiredtiger/src/btree/bt_discard.c @@ -38,6 +38,10 @@ __wt_ref_out(WT_SESSION_IMPL *session, WT_REF *ref) */ WT_ASSERT(session, __wt_hazard_check_assert(session, ref, true)); + WT_ASSERT(session, + !F_ISSET(ref, WT_REF_FLAG_INTERNAL) || F_ISSET(session->dhandle, WT_DHANDLE_EXCLUSIVE) || + !__wt_gen_active(session, WT_GEN_SPLIT, ref->page->pg_intl_split_gen)); + __wt_page_out(session, &ref->page); } diff --git a/src/third_party/wiredtiger/src/btree/bt_slvg.c b/src/third_party/wiredtiger/src/btree/bt_slvg.c index 8800f2ae6e6..c3d6c25b18c 100644 --- a/src/third_party/wiredtiger/src/btree/bt_slvg.c +++ b/src/third_party/wiredtiger/src/btree/bt_slvg.c @@ -1877,8 +1877,7 @@ __slvg_row_build_leaf(WT_SESSION_IMPL *session, WT_TRACK *trk, WT_REF *ref, WT_S ++skip_start; } if (F_ISSET(trk, WT_TRACK_CHECK_STOP)) - WT_ROW_FOREACH_REVERSE(page, rip, i) - { + WT_ROW_FOREACH_REVERSE (page, rip, i) { WT_ERR(__wt_row_leaf_key(session, page, rip, key, false)); /* diff --git a/src/third_party/wiredtiger/src/btree/bt_split.c b/src/third_party/wiredtiger/src/btree/bt_split.c index 291f1a1be1c..a3d22efbd7b 100644 --- a/src/third_party/wiredtiger/src/btree/bt_split.c +++ b/src/third_party/wiredtiger/src/btree/bt_split.c @@ -37,14 +37,10 @@ typedef enum { static int __split_safe_free(WT_SESSION_IMPL *session, uint64_t split_gen, bool exclusive, void *p, size_t s) { - /* We should only call safe free if we aren't pinning the memory. */ - WT_ASSERT(session, __wt_session_gen(session, WT_GEN_SPLIT) != split_gen); - /* - * We have swapped something in a page: if we don't have exclusive access, check whether there - * are other threads in the same tree. + * We have swapped something in a page: it's only safe to free it if we have exclusive access. */ - if (exclusive || !__wt_gen_active(session, WT_GEN_SPLIT, split_gen)) { + if (exclusive) { __wt_overwrite_and_free_len(session, p, s); return (0); } @@ -265,6 +261,7 @@ __split_ref_move(WT_SESSION_IMPL *session, WT_PAGE *from_home, WT_REF **from_ref default: WT_ERR(__wt_illegal_value(session, unpack.raw)); } + /* If the compare-and-swap is successful, clear addr to skip the free at the end. */ if (__wt_atomic_cas_ptr(&ref->addr, ref_addr, addr)) addr = NULL; } @@ -286,7 +283,7 @@ err: * Finalize the WT_REF move. */ static void -__split_ref_final(WT_SESSION_IMPL *session, WT_PAGE ***lockedp) +__split_ref_final(WT_SESSION_IMPL *session, uint64_t split_gen, WT_PAGE ***lockedp) { WT_PAGE **locked; size_t i; @@ -301,9 +298,18 @@ __split_ref_final(WT_SESSION_IMPL *session, WT_PAGE ***lockedp) /* * The moved child pages are locked to prevent them from splitting before the parent move * completes, unlock them as the final step. + * + * Once the split is live, newly created internal pages might be evicted and their WT_REF + * structures freed. If that happens before all threads exit the index of the page that + * previously "owned" the WT_REF, a thread might see a freed WT_REF. To ensure that doesn't + * happen, the created pages are set to the current split generation and so can't be evicted + * until all readers have left the old generation. */ - for (i = 0; locked[i] != NULL; ++i) + for (i = 0; locked[i] != NULL; ++i) { + if (split_gen != 0 && WT_PAGE_IS_INTERNAL(locked[i])) + locked[i]->pg_intl_split_gen = split_gen; WT_PAGE_UNLOCK(session, locked[i]); + } __wt_free(session, locked); } @@ -367,7 +373,7 @@ __split_ref_prepare( return (0); err: - __split_ref_final(session, &locked); + __split_ref_final(session, 0, &locked); return (ret); } @@ -443,16 +449,6 @@ __split_root(WT_SESSION_IMPL *session, WT_PAGE *root) WT_ERR(__wt_calloc_one(session, alloc_refp)); root_incr += children * sizeof(WT_REF); - /* - * Once the split is live, newly created internal pages might be evicted and their WT_REF - * structures freed. If that happens before all threads exit the index of the page that - * previously "owned" the WT_REF, a thread might see a freed WT_REF. To ensure that doesn't - * happen, the created pages are set to the current split generation and so can't be evicted - * until all readers have left the old generation. - */ - split_gen = __wt_gen_next(session, WT_GEN_SPLIT); - WT_ASSERT(session, root->pg_intl_split_gen < split_gen); - /* Allocate child pages, and connect them into the new page index. */ for (root_refp = pindex->index, alloc_refp = alloc_index->index, i = 0; i < children; ++i) { slots = i == children - 1 ? remain : chunk; @@ -479,7 +475,6 @@ __split_root(WT_SESSION_IMPL *session, WT_PAGE *root) * Initialize the child page. Block eviction in newly created pages and mark them dirty. */ child->pg_intl_parent_ref = ref; - child->pg_intl_split_gen = split_gen; WT_ERR(__wt_page_modify_init(session, child)); __wt_page_modify_set(session, child); @@ -523,19 +518,19 @@ __split_root(WT_SESSION_IMPL *session, WT_PAGE *root) __wt_timing_stress(session, WT_TIMING_STRESS_SPLIT_2); /* - * Get a generation for this split, mark the root page. This must be after the new index is - * swapped into place in order to know that no readers are looking at the old index. + * Mark the root page with the split generation. * * Note: as the root page cannot currently be evicted, the root split generation isn't ever * used. That said, it future proofs eviction and isn't expensive enough to special-case. * * Getting a new split generation implies a full barrier, no additional barrier is needed. */ - split_gen = __wt_gen_next(session, WT_GEN_SPLIT); + WT_FULL_BARRIER(); + split_gen = __wt_gen(session, WT_GEN_SPLIT); root->pg_intl_split_gen = split_gen; /* Finalize the WT_REF move. */ - __split_ref_final(session, &locked); + __split_ref_final(session, split_gen, &locked); #ifdef HAVE_DIAGNOSTIC WT_WITH_PAGE_INDEX(session, ret = __split_verify_root(session, root)); @@ -561,8 +556,9 @@ __split_root(WT_SESSION_IMPL *session, WT_PAGE *root) __wt_cache_page_inmem_incr(session, root, root_incr); __wt_cache_page_inmem_decr(session, root, root_decr); + __wt_gen_next(session, WT_GEN_SPLIT, NULL); err: - __split_ref_final(session, &locked); + __split_ref_final(session, 0, &locked); switch (complete) { case WT_ERR_RETURN: @@ -782,11 +778,13 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new, uint32_t /* * Get a generation for this split, mark the page. This must be after the new index is swapped - * into place in order to know that no readers are looking at the old index. + * into place in order to know that no readers with the new generation will look at the old + * index. * * Getting a new split generation implies a full barrier, no additional barrier is needed. */ - split_gen = __wt_gen_next(session, WT_GEN_SPLIT); + WT_FULL_BARRIER(); + split_gen = __wt_gen(session, WT_GEN_SPLIT); parent->pg_intl_split_gen = split_gen; #ifdef HAVE_DIAGNOSTIC @@ -848,6 +846,7 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new, uint32_t "%p: split into parent, %" PRIu32 "->%" PRIu32 ", %" PRIu32 " deleted", (void *)ref, parent_entries, result_entries, deleted_entries); + __wt_gen_next(session, WT_GEN_SPLIT, NULL); err: /* * A note on error handling: if we completed the split, return success, nothing really bad can @@ -978,16 +977,6 @@ __split_internal(WT_SESSION_IMPL *session, WT_PAGE *parent, WT_PAGE *page) WT_ERR(__wt_calloc_one(session, alloc_refp)); parent_incr += children * sizeof(WT_REF); - /* - * Once the split is live, newly created internal pages might be evicted and their WT_REF - * structures freed. If that happens before all threads exit the index of the page that - * previously "owned" the WT_REF, a thread might see a freed WT_REF. To ensure that doesn't - * happen, the created pages are set to the current split generation and so can't be evicted - * until all readers have left the old generation. - */ - split_gen = __wt_gen_next(session, WT_GEN_SPLIT); - WT_ASSERT(session, page->pg_intl_split_gen < split_gen); - /* Allocate child pages, and connect them into the new page index. */ WT_ASSERT(session, page_refp == pindex->index + chunk); for (alloc_refp = alloc_index->index + 1, i = 1; i < children; ++i) { @@ -1015,7 +1004,6 @@ __split_internal(WT_SESSION_IMPL *session, WT_PAGE *parent, WT_PAGE *page) * Initialize the child page. Block eviction in newly created pages and mark them dirty. */ child->pg_intl_parent_ref = ref; - child->pg_intl_split_gen = split_gen; WT_ERR(__wt_page_modify_init(session, child)); __wt_page_modify_set(session, child); @@ -1063,15 +1051,17 @@ __split_internal(WT_SESSION_IMPL *session, WT_PAGE *parent, WT_PAGE *page) /* * Get a generation for this split, mark the parent page. This must be after the new index is - * swapped into place in order to know that no readers are looking at the old index. + * swapped into place in order to know that no readers with the new generation will look at the + * old index. * * Getting a new split generation implies a full barrier, no additional barrier is needed. */ - split_gen = __wt_gen_next(session, WT_GEN_SPLIT); + WT_FULL_BARRIER(); + split_gen = __wt_gen(session, WT_GEN_SPLIT); page->pg_intl_split_gen = split_gen; /* Finalize the WT_REF move. */ - __split_ref_final(session, &locked); + __split_ref_final(session, split_gen, &locked); #ifdef HAVE_DIAGNOSTIC WT_WITH_PAGE_INDEX(session, __split_verify_intl_key_order(session, parent)); @@ -1103,8 +1093,9 @@ __split_internal(WT_SESSION_IMPL *session, WT_PAGE *parent, WT_PAGE *page) __wt_cache_page_inmem_incr(session, page, page_incr); __wt_cache_page_inmem_decr(session, page, page_decr); + __wt_gen_next(session, WT_GEN_SPLIT, NULL); err: - __split_ref_final(session, &locked); + __split_ref_final(session, 0, &locked); switch (complete) { case WT_ERR_RETURN: diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index ef434fa3e5d..dde2b81aafa 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -881,9 +881,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "config=,early_load=false,entry=wiredtiger_extension_init," "terminate=wiredtiger_extension_terminate", confchk_WT_CONNECTION_load_extension, 4}, - {"WT_CONNECTION.open_session", - "cache_cursors=true,ignore_cache_size=false," - "isolation=read-committed", + {"WT_CONNECTION.open_session", "cache_cursors=true,ignore_cache_size=false,isolation=snapshot", confchk_WT_CONNECTION_open_session, 3}, {"WT_CONNECTION.query_timestamp", "get=all_durable", confchk_WT_CONNECTION_query_timestamp, 1}, {"WT_CONNECTION.reconfigure", @@ -978,11 +976,10 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", {"WT_SESSION.prepare_transaction", "prepare_timestamp=", confchk_WT_SESSION_prepare_transaction, 1}, {"WT_SESSION.query_timestamp", "get=read", confchk_WT_SESSION_query_timestamp, 1}, - {"WT_SESSION.reconfigure", - "cache_cursors=true,ignore_cache_size=false," - "isolation=read-committed", + {"WT_SESSION.reconfigure", "cache_cursors=true,ignore_cache_size=false,isolation=snapshot", confchk_WT_SESSION_reconfigure, 3}, {"WT_SESSION.rename", "", NULL, 0}, {"WT_SESSION.reset", "", NULL, 0}, + {"WT_SESSION.reset_snapshot", "", NULL, 0}, {"WT_SESSION.rollback_transaction", "operation_timeout_ms=0", confchk_WT_SESSION_rollback_transaction, 1}, {"WT_SESSION.salvage", "force=false", confchk_WT_SESSION_salvage, 1}, diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index 1d792d74163..613c71a8d05 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -2101,7 +2101,8 @@ __conn_write_base_config(WT_SESSION_IMPL *session, const char *cfg[]) "readonly=," "timing_stress_for_test=," "use_environment_priv=," - "verbose=,", + "verbose=," + "verify_metadata=,", &base_config)); __wt_config_init(session, &parser, base_config); while ((ret = __wt_config_next(&parser, &k, &v)) == 0) { diff --git a/src/third_party/wiredtiger/src/cursor/cur_backup.c b/src/third_party/wiredtiger/src/cursor/cur_backup.c index a32b5ddf974..6a03b7ea5b3 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_backup.c +++ b/src/third_party/wiredtiger/src/cursor/cur_backup.c @@ -211,8 +211,14 @@ err: */ cfg[0] = WT_CONFIG_BASE(session, WT_SESSION_checkpoint); cfg[1] = "force=true"; + /* + * Metadata checkpoints rely on read-committed isolation. Use that here no matter what + * isolation the caller's session sets for isolation. + */ WT_WITH_DHANDLE(session, WT_SESSION_META_DHANDLE(session), - WT_WITH_METADATA_LOCK(session, ret = __wt_checkpoint(session, cfg))); + WT_WITH_METADATA_LOCK(session, + WT_WITH_TXN_ISOLATION( + session, WT_ISO_READ_COMMITTED, ret = __wt_checkpoint(session, cfg)))); } /* diff --git a/src/third_party/wiredtiger/src/cursor/cur_std.c b/src/third_party/wiredtiger/src/cursor/cur_std.c index 92b8e5c5b3e..c34d736b594 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_std.c +++ b/src/third_party/wiredtiger/src/cursor/cur_std.c @@ -798,6 +798,10 @@ __wt_cursor_cache_get(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *to_d if (cval.val) return (WT_NOTFOUND); + WT_RET(__wt_config_gets_def(session, cfg, "debug", 0, &cval)); + if (cval.len != 0) + return (WT_NOTFOUND); + WT_RET(__wt_config_gets_def(session, cfg, "dump", 0, &cval)); if (cval.len != 0) return (WT_NOTFOUND); diff --git a/src/third_party/wiredtiger/src/docs/cursors.dox b/src/third_party/wiredtiger/src/docs/cursors.dox index 5b067b83b05..ba7d603ad13 100644 --- a/src/third_party/wiredtiger/src/docs/cursors.dox +++ b/src/third_party/wiredtiger/src/docs/cursors.dox @@ -59,7 +59,7 @@ Any operation that consists of multiple related updates should be enclosed in an explicit transaction to ensure that the updates are applied atomically. -At \c read-committed (the default) or \c snapshot isolation levels, +At \c snapshot (the default) or \c read-committed isolation levels, committed changes from concurrent transactions become visible when no cursor is positioned. In other words, at these isolation levels, all cursors in a session read from a stable snapshot while any cursor in the diff --git a/src/third_party/wiredtiger/src/docs/error-handling.dox b/src/third_party/wiredtiger/src/docs/error-handling.dox index a83363f026b..cdf7524a0bf 100644 --- a/src/third_party/wiredtiger/src/docs/error-handling.dox +++ b/src/third_party/wiredtiger/src/docs/error-handling.dox @@ -51,9 +51,9 @@ required to use the database. @par <code>WT_CACHE_FULL</code> This error is only generated when wiredtiger_open is configured to run in-memory, and an insert or -update operation requires more than the configured cache size to complete. The operation may be -retried; if a transaction is in progress, it should be rolled back and the operation retried in a -new transaction. +update operation requires more than the configured cache size to complete, or when an application +thread fails to do eviction within cache_max_wait_ms. The operation may be retried; if a transaction +is in progress, it should be rolled back and the operation retried in a new transaction. @par <code>WT_PREPARE_CONFLICT</code> This error is generated when the application attempts to update an already updated record which is diff --git a/src/third_party/wiredtiger/src/docs/transactions.dox b/src/third_party/wiredtiger/src/docs/transactions.dox index 2411c65beca..3bfae988747 100644 --- a/src/third_party/wiredtiger/src/docs/transactions.dox +++ b/src/third_party/wiredtiger/src/docs/transactions.dox @@ -54,6 +54,17 @@ discarding any cursor position as well as any key and value. @snippet ex_all.c transaction commit/rollback +Applications can call WT_SESSION::reset_snapshot to reset snapshots for +snapshot isolation transactions to update their existing snapshot. It raises +an error when this API is used for isolation other than snapshot isolation mode +or when the session has performed any write operations. +This API internally releases the current snapshot and gets the new running +transactions snapshot to avoid pinning the content in the database that is no +longer needed. Applications that don't use read_timestamp for the search may +see different results compared to earlier with the updated snapshot. + +@snippet ex_all.c reset snapshot + @section transactions_implicit Implicit transactions If a cursor is used when no explicit transaction is active in a session, @@ -91,7 +102,7 @@ transactional readers, an operation may fail and return ::WT_ROLLBACK. WiredTiger supports <code>read-uncommitted</code>, <code>read-committed</code> and <code>snapshot</code> isolation levels; -the default isolation level is <code>read-committed</code>. +the default isolation level is <code>snapshot</code>. - <code>read-uncommitted</code>: Transactions can see changes made by other transactions before those diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 35b60b2fe4e..bccd79c8674 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -1133,6 +1133,9 @@ __evict_lru_pages(WT_SESSION_IMPL *session, bool is_server) if ((ret = __evict_page(session, is_server)) == EBUSY) ret = 0; + /* If any resources are pinned, release them now. */ + WT_TRET(__wt_session_release_resources(session)); + /* If a worker thread found the queue empty, pause. */ if (ret == WT_NOTFOUND && !is_server && F_ISSET(conn, WT_CONN_EVICTION_RUN)) __wt_cond_wait(session, conn->evict_threads.wait_cond, 10000, NULL); diff --git a/src/third_party/wiredtiger/src/evict/evict_page.c b/src/third_party/wiredtiger/src/evict/evict_page.c index cf6bc39bdb7..49a33870db0 100644 --- a/src/third_party/wiredtiger/src/evict/evict_page.c +++ b/src/third_party/wiredtiger/src/evict/evict_page.c @@ -166,6 +166,10 @@ __wt_evict(WT_SESSION_IMPL *session, WT_REF *ref, uint8_t previous_state, uint32 if (inmem_split) goto done; + /* Check that we are not about to evict an internal page with an active split generation. */ + if (F_ISSET(ref, WT_REF_FLAG_INTERNAL) && !closing) + WT_ASSERT(session, !__wt_gen_active(session, WT_GEN_SPLIT, page->pg_intl_split_gen)); + /* Count evictions of internal pages during normal operation. */ if (!closing && F_ISSET(ref, WT_REF_FLAG_INTERNAL)) { WT_STAT_CONN_INCR(session, cache_eviction_internal); @@ -447,8 +451,7 @@ __evict_child_check(WT_SESSION_IMPL *session, WT_REF *parent) } } WT_INTL_FOREACH_END; - WT_INTL_FOREACH_REVERSE_BEGIN(session, parent->page, child) - { + WT_INTL_FOREACH_REVERSE_BEGIN (session, parent->page, child) { switch (child->state) { case WT_REF_DISK: /* On-disk */ case WT_REF_DELETED: /* On-disk, deleted */ @@ -714,17 +717,6 @@ __evict_review(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t evict_flags, bool WT_RET(ret); /* - * Give up on eviction during a checkpoint if the page splits. - * - * We get here if checkpoint reads a page with history store entries: if more of those entries - * are visible now than when the original eviction happened, the page could split. In most - * workloads, this is very unlikely. However, since checkpoint is partway through reconciling - * the parent page, a split can corrupt the checkpoint. - */ - if (WT_SESSION_BTREE_SYNC(session) && page->modify->rec_result == WT_PM_REC_MULTIBLOCK) - return (__wt_set_return(session, EBUSY)); - - /* * Success: assert that the page is clean or reconciliation was configured to save updates. */ WT_ASSERT(session, diff --git a/src/third_party/wiredtiger/src/include/api.h b/src/third_party/wiredtiger/src/include/api.h index ed27d2854d0..4449b6bb3ea 100644 --- a/src/third_party/wiredtiger/src/include/api.h +++ b/src/third_party/wiredtiger/src/include/api.h @@ -32,30 +32,36 @@ #define WT_SINGLE_THREAD_CHECK_STOP(s) #endif -#define API_SESSION_PUSH(s, h, n, dh) \ - WT_DATA_HANDLE *__olddh = (s)->dhandle; \ - const char *__oldname = (s)->name; \ - (s)->dhandle = (dh); \ +#define API_SESSION_PUSH(s, h, n, dh) \ + WT_DATA_HANDLE *__olddh = (s)->dhandle; \ + const char *__oldname; \ + /* If this isn't an API reentry, the name should be NULL and the counter should be 0. */ \ + WT_ASSERT(session, (s)->name != NULL || s->api_call_counter == 0); \ + __oldname = (s)->name; \ + ++s->api_call_counter; \ + (s)->dhandle = (dh); \ (s)->name = (s)->lastop = #h "." #n #define API_SESSION_POP(s) \ (s)->dhandle = __olddh; \ - (s)->name = __oldname + (s)->name = __oldname; \ + --s->api_call_counter /* Standard entry points to the API: declares/initializes local variables. */ -#define API_SESSION_INIT(s, h, n, dh) \ - WT_TRACK_OP_DECL; \ - API_SESSION_PUSH(s, h, n, dh); \ - /* \ - * No code before this line, otherwise error handling won't be \ - * correct. \ - */ \ - WT_ERR(WT_SESSION_CHECK_PANIC(s)); \ - WT_SINGLE_THREAD_CHECK_START(s); \ - WT_TRACK_OP_INIT(s); \ - __wt_op_timer_start(s); \ - /* Reset wait time if this isn't an API reentry. */ \ - if (__oldname == NULL) \ - (s)->cache_wait_us = 0; \ +#define API_SESSION_INIT(s, h, n, dh) \ + WT_TRACK_OP_DECL; \ + API_SESSION_PUSH(s, h, n, dh); \ + /* \ + * No code before this line, otherwise error handling won't be \ + * correct. \ + */ \ + WT_ERR(WT_SESSION_CHECK_PANIC(s)); \ + WT_SINGLE_THREAD_CHECK_START(s); \ + WT_TRACK_OP_INIT(s); \ + if (s->api_call_counter == 1 && !F_ISSET(s, WT_SESSION_INTERNAL)) \ + __wt_op_timer_start(s); \ + /* Reset wait time if this isn't an API reentry. */ \ + if (s->api_call_counter == 1) \ + (s)->cache_wait_us = 0; \ __wt_verbose((s), WT_VERB_API, "%s", "CALL: " #h ":" #n) #define API_CALL_NOCONF(s, h, n, dh) \ @@ -69,20 +75,21 @@ if ((config) != NULL) \ WT_ERR(__wt_config_check((s), WT_CONFIG_REF(session, h##_##n), (config), 0)) -#define API_END(s, ret) \ - if ((s) != NULL) { \ - WT_TRACK_OP_END(s); \ - WT_SINGLE_THREAD_CHECK_STOP(s); \ - if ((ret) != 0) \ - __wt_txn_err_set(s, ret); \ - __wt_op_timer_stop(s); \ - /* \ - * No code after this line, otherwise error handling \ - * won't be correct. \ - */ \ - API_SESSION_POP(s); \ - } \ - } \ +#define API_END(s, ret) \ + if ((s) != NULL) { \ + WT_TRACK_OP_END(s); \ + WT_SINGLE_THREAD_CHECK_STOP(s); \ + if ((ret) != 0) \ + __wt_txn_err_set(s, ret); \ + if (s->api_call_counter == 1 && !F_ISSET(session, WT_SESSION_INTERNAL)) \ + __wt_op_timer_stop(s); \ + /* \ + * No code after this line, otherwise error handling \ + * won't be correct. \ + */ \ + API_SESSION_POP(s); \ + } \ + } \ while (0) /* An API call wrapped in a transaction if necessary. */ diff --git a/src/third_party/wiredtiger/src/include/config.h b/src/third_party/wiredtiger/src/include/config.h index 92ae7dc4a67..70f4fdef9a6 100644 --- a/src/third_party/wiredtiger/src/include/config.h +++ b/src/third_party/wiredtiger/src/include/config.h @@ -84,24 +84,25 @@ struct __wt_config_parser_impl { #define WT_CONFIG_ENTRY_WT_SESSION_reconfigure 30 #define WT_CONFIG_ENTRY_WT_SESSION_rename 31 #define WT_CONFIG_ENTRY_WT_SESSION_reset 32 -#define WT_CONFIG_ENTRY_WT_SESSION_rollback_transaction 33 -#define WT_CONFIG_ENTRY_WT_SESSION_salvage 34 -#define WT_CONFIG_ENTRY_WT_SESSION_strerror 35 -#define WT_CONFIG_ENTRY_WT_SESSION_timestamp_transaction 36 -#define WT_CONFIG_ENTRY_WT_SESSION_transaction_sync 37 -#define WT_CONFIG_ENTRY_WT_SESSION_truncate 38 -#define WT_CONFIG_ENTRY_WT_SESSION_upgrade 39 -#define WT_CONFIG_ENTRY_WT_SESSION_verify 40 -#define WT_CONFIG_ENTRY_colgroup_meta 41 -#define WT_CONFIG_ENTRY_file_config 42 -#define WT_CONFIG_ENTRY_file_meta 43 -#define WT_CONFIG_ENTRY_index_meta 44 -#define WT_CONFIG_ENTRY_lsm_meta 45 -#define WT_CONFIG_ENTRY_table_meta 46 -#define WT_CONFIG_ENTRY_wiredtiger_open 47 -#define WT_CONFIG_ENTRY_wiredtiger_open_all 48 -#define WT_CONFIG_ENTRY_wiredtiger_open_basecfg 49 -#define WT_CONFIG_ENTRY_wiredtiger_open_usercfg 50 +#define WT_CONFIG_ENTRY_WT_SESSION_reset_snapshot 33 +#define WT_CONFIG_ENTRY_WT_SESSION_rollback_transaction 34 +#define WT_CONFIG_ENTRY_WT_SESSION_salvage 35 +#define WT_CONFIG_ENTRY_WT_SESSION_strerror 36 +#define WT_CONFIG_ENTRY_WT_SESSION_timestamp_transaction 37 +#define WT_CONFIG_ENTRY_WT_SESSION_transaction_sync 38 +#define WT_CONFIG_ENTRY_WT_SESSION_truncate 39 +#define WT_CONFIG_ENTRY_WT_SESSION_upgrade 40 +#define WT_CONFIG_ENTRY_WT_SESSION_verify 41 +#define WT_CONFIG_ENTRY_colgroup_meta 42 +#define WT_CONFIG_ENTRY_file_config 43 +#define WT_CONFIG_ENTRY_file_meta 44 +#define WT_CONFIG_ENTRY_index_meta 45 +#define WT_CONFIG_ENTRY_lsm_meta 46 +#define WT_CONFIG_ENTRY_table_meta 47 +#define WT_CONFIG_ENTRY_wiredtiger_open 48 +#define WT_CONFIG_ENTRY_wiredtiger_open_all 49 +#define WT_CONFIG_ENTRY_wiredtiger_open_basecfg 50 +#define WT_CONFIG_ENTRY_wiredtiger_open_usercfg 51 /* * configuration section: END * DO NOT EDIT: automatically built by dist/flags.py. diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 1c646d6622c..83327b021ee 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -1590,8 +1590,6 @@ extern uint64_t __wt_ext_transaction_oldest(WT_EXTENSION_API *wt_api) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern uint64_t __wt_gen(WT_SESSION_IMPL *session, int which) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern uint64_t __wt_gen_next(WT_SESSION_IMPL *session, int which) - WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern uint64_t __wt_hash_city64(const void *s, size_t len) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern uint64_t __wt_hash_fnv64(const void *string, size_t len) @@ -1678,6 +1676,7 @@ extern void __wt_free_ref_index( extern void __wt_free_update_list(WT_SESSION_IMPL *session, WT_UPDATE **updp); extern void __wt_gen_drain(WT_SESSION_IMPL *session, int which, uint64_t generation); extern void __wt_gen_init(WT_SESSION_IMPL *session); +extern void __wt_gen_next(WT_SESSION_IMPL *session, int which, uint64_t *genp); extern void __wt_gen_next_drain(WT_SESSION_IMPL *session, int which); extern void __wt_hazard_close(WT_SESSION_IMPL *session); extern void __wt_hs_close(WT_SESSION_IMPL *session); diff --git a/src/third_party/wiredtiger/src/include/reconcile.h b/src/third_party/wiredtiger/src/include/reconcile.h index 1f702af3a26..99986450f21 100644 --- a/src/third_party/wiredtiger/src/include/reconcile.h +++ b/src/third_party/wiredtiger/src/include/reconcile.h @@ -135,7 +135,7 @@ struct __wt_reconcile { size_t min_offset; /* byte offset */ WT_ITEM image; /* disk-image */ - } chunkA, chunkB, *cur_ptr, *prev_ptr; + } chunk_A, chunk_B, *cur_ptr, *prev_ptr; size_t disk_img_buf_size; /* Base size needed for a chunk memory image */ diff --git a/src/third_party/wiredtiger/src/include/session.h b/src/third_party/wiredtiger/src/include/session.h index 9f11361aaf2..1350c6114b4 100644 --- a/src/third_party/wiredtiger/src/include/session.h +++ b/src/third_party/wiredtiger/src/include/session.h @@ -66,6 +66,7 @@ struct __wt_session_impl { uint64_t cache_wait_us; /* Wait time for cache for current operation */ uint64_t operation_start_us; /* Operation start */ uint64_t operation_timeout_us; /* Maximum operation period before rollback */ + u_int api_call_counter; /* Depth of api calls */ WT_DATA_HANDLE *dhandle; /* Current data handle */ diff --git a/src/third_party/wiredtiger/src/include/time_inline.h b/src/third_party/wiredtiger/src/include/time_inline.h index 80ae3048a8a..cff5e0850ea 100644 --- a/src/third_party/wiredtiger/src/include/time_inline.h +++ b/src/third_party/wiredtiger/src/include/time_inline.h @@ -162,9 +162,6 @@ __wt_op_timer_start(WT_SESSION_IMPL *session) { uint64_t timeout_us; - if (session->hs_cursor != NULL) - return; - /* Timer can be configured per-transaction, and defaults to per-connection. */ if (session->txn == NULL || (timeout_us = session->txn->operation_timeout_us) == 0) timeout_us = S2C(session)->operation_timeout_us; @@ -183,9 +180,6 @@ __wt_op_timer_start(WT_SESSION_IMPL *session) static inline void __wt_op_timer_stop(WT_SESSION_IMPL *session) { - if (session->hs_cursor != NULL) - return; - session->operation_start_us = session->operation_timeout_us = 0; } @@ -198,8 +192,6 @@ __wt_op_timer_fired(WT_SESSION_IMPL *session) { uint64_t diff, now; - if (session->hs_cursor != NULL) - return (false); if (session->operation_start_us == 0 || session->operation_timeout_us == 0) return (false); diff --git a/src/third_party/wiredtiger/src/include/txn_inline.h b/src/third_party/wiredtiger/src/include/txn_inline.h index 0433a252d63..c35183e0ef9 100644 --- a/src/third_party/wiredtiger/src/include/txn_inline.h +++ b/src/third_party/wiredtiger/src/include/txn_inline.h @@ -1061,8 +1061,12 @@ __wt_txn_begin(WT_SESSION_IMPL *session, const char *cfg[]) WT_RET(__wt_txn_config(session, cfg)); - /* Allocate a snapshot if required. */ - if (txn->isolation == WT_ISO_SNAPSHOT) { + /* + * Allocate a snapshot if required or update the existing snapshot. Do not update the existing + * snapshot of autocommit transactions because they are committed at the end of the operation. + */ + if (txn->isolation == WT_ISO_SNAPSHOT && + !(F_ISSET(txn, WT_TXN_AUTOCOMMIT) && F_ISSET(txn, WT_TXN_HAS_SNAPSHOT))) { if (session->ncursors > 0) WT_RET(__wt_session_copy_values(session)); @@ -1090,14 +1094,15 @@ __wt_txn_begin(WT_SESSION_IMPL *session, const char *cfg[]) static inline int __wt_txn_autocommit_check(WT_SESSION_IMPL *session) { + WT_DECL_RET; WT_TXN *txn; txn = session->txn; if (F_ISSET(txn, WT_TXN_AUTOCOMMIT)) { + ret = __wt_txn_begin(session, NULL); F_CLR(txn, WT_TXN_AUTOCOMMIT); - return (__wt_txn_begin(session, NULL)); } - return (0); + return (ret); } /* @@ -1189,6 +1194,19 @@ __wt_txn_id_check(WT_SESSION_IMPL *session) if (F_ISSET(txn, WT_TXN_HAS_ID)) return (0); + /* + * Return error when the transactions with read committed or uncommitted isolation tries to + * perform any write operation. Don't return an error for any update on metadata because it uses + * special transaction visibility rules, search and updates on metadata happens in + * read-uncommitted and read-committed isolation. + */ + if (session->dhandle != NULL && !WT_IS_METADATA(session->dhandle) && + (txn->isolation == WT_ISO_READ_COMMITTED || txn->isolation == WT_ISO_READ_UNCOMMITTED)) { + WT_ASSERT(session, !F_ISSET(session, WT_SESSION_INTERNAL)); + WT_RET_MSG(session, ENOTSUP, + "write operations are not supported in read-committed or read-uncommitted transactions."); + } + /* If the transaction is idle, check that the cache isn't full. */ WT_RET(__wt_txn_idle_cache_check(session)); @@ -1250,7 +1268,8 @@ __wt_txn_update_check( txn = session->txn; txn_global = &S2C(session)->txn_global; - if (txn->isolation != WT_ISO_SNAPSHOT) + /* Don't check if transaction isolation is not snapshot or the table is metadata. */ + if (txn->isolation != WT_ISO_SNAPSHOT || (cbt != NULL && WT_IS_METADATA(cbt->dhandle))) return (0); if (txn_global->debug_rollback != 0 && diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 249e6629680..ae1f3057624 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -793,7 +793,7 @@ struct __wt_session { * size., a boolean flag; default \c false.} * @config{isolation, the default isolation level for operations in this session., a * string\, chosen from the following options: \c "read-uncommitted"\, \c "read-committed"\, - * \c "snapshot"; default \c read-committed.} + * \c "snapshot"; default \c snapshot.} * @configend * @errors */ @@ -1696,6 +1696,27 @@ struct __wt_session { int __F(prepare_transaction)(WT_SESSION *session, const char *config); /*! + * Reset the snapshot. + * + * This method resets snapshots for snapshot isolation transactions to + * update their existing snapshot. It raises an error when this API + * is used for isolation other than snapshot isolation mode or when the session + * has performed any write operations. + * This API internally releases the current snapshot and gets the new running + * transactions snapshot to avoid pinning the content in the database that is no + * longer needed. Applications that don't use read_timestamp for the search may + * see different results compared to earlier with the updated snapshot. + * + * @requires_transaction + * + * @snippet ex_all.c reset snapshot + * + * @param session the session handle + * @errors + */ + int __F(reset_snapshot)(WT_SESSION *session); + + /*! * Roll back the current transaction. * * A transaction must be in progress when this method is called. @@ -2248,7 +2269,7 @@ struct __wt_connection { * size., a boolean flag; default \c false.} * @config{isolation, the default isolation level for operations in this session., a * string\, chosen from the following options: \c "read-uncommitted"\, \c "read-committed"\, - * \c "snapshot"; default \c read-committed.} + * \c "snapshot"; default \c snapshot.} * @configend * @param[out] sessionp the new session handle * @errors @@ -3527,8 +3548,9 @@ const char *wiredtiger_version(int *majorp, int *minorp, int *patchp) * Operation would overflow cache. * This error is only generated when wiredtiger_open is configured to run in- * memory, and an insert or update operation requires more than the configured - * cache size to complete. The operation may be retried; if a transaction is in - * progress, it should be rolled back and the operation retried in a new + * cache size to complete, or when an application thread fails to do eviction + * within cache_max_wait_ms. The operation may be retried; if a transaction is + * in progress, it should be rolled back and the operation retried in a new * transaction. */ #define WT_CACHE_FULL (-31807) diff --git a/src/third_party/wiredtiger/src/meta/meta_track.c b/src/third_party/wiredtiger/src/meta/meta_track.c index 10d2904dbf7..c99709f90e4 100644 --- a/src/third_party/wiredtiger/src/meta/meta_track.c +++ b/src/third_party/wiredtiger/src/meta/meta_track.c @@ -518,10 +518,11 @@ __wt_meta_track_init(WT_SESSION_IMPL *session) conn, "metadata-ckpt", false, WT_SESSION_NO_DATA_HANDLES, &conn->meta_ckpt_session)); /* - * Sessions default to read-committed isolation, we rely on that for the correctness of - * metadata checkpoints. + * Set session transaction isolation to read-committed isolation, we rely on that for the + * correctness of metadata checkpoints. */ - WT_ASSERT(session, conn->meta_ckpt_session->txn->isolation == WT_ISO_READ_COMMITTED); + conn->meta_ckpt_session->isolation = conn->meta_ckpt_session->txn->isolation = + WT_ISO_READ_COMMITTED; } return (0); diff --git a/src/third_party/wiredtiger/src/meta/meta_turtle.c b/src/third_party/wiredtiger/src/meta/meta_turtle.c index 0ed6257060a..a70efef3025 100644 --- a/src/third_party/wiredtiger/src/meta/meta_turtle.c +++ b/src/third_party/wiredtiger/src/meta/meta_turtle.c @@ -224,9 +224,9 @@ __wt_turtle_init(WT_SESSION_IMPL *session) WT_DECL_RET; char *metaconf, *unused_value; bool exist_backup, exist_incr, exist_isrc, exist_turtle; - bool load, loadTurtle, validate_turtle; + bool load, load_turtle, validate_turtle; - load = loadTurtle = validate_turtle = false; + load = load_turtle = validate_turtle = false; /* * Discard any turtle setup file left-over from previous runs. This doesn't matter for @@ -269,7 +269,7 @@ __wt_turtle_init(WT_SESSION_IMPL *session) if (ret != 0) { WT_RET(__wt_remove_if_exists(session, WT_METADATA_TURTLE, false)); - loadTurtle = true; + load_turtle = true; } else /* * Set a flag to specify that we should validate whether we can start up on the turtle @@ -312,7 +312,7 @@ __wt_turtle_init(WT_SESSION_IMPL *session) WT_RET(__metadata_load_bulk(session)); } - if (load || loadTurtle) { + if (load || load_turtle) { /* Create the turtle file. */ WT_RET(__metadata_config(session, &metaconf)); WT_WITH_TURTLE_LOCK(session, ret = __wt_turtle_update(session, WT_METAFILE_URI, metaconf)); diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index 8da98678e02..cc42a66be3b 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -493,8 +493,8 @@ __rec_init(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags, WT_SALVAGE_COO r->last = &r->_last; /* Disk buffers need to be aligned for writing. */ - F_SET(&r->chunkA.image, WT_ITEM_ALIGNED); - F_SET(&r->chunkB.image, WT_ITEM_ALIGNED); + F_SET(&r->chunk_A.image, WT_ITEM_ALIGNED); + F_SET(&r->chunk_B.image, WT_ITEM_ALIGNED); } /* Remember the configuration. */ @@ -706,12 +706,12 @@ __rec_destroy(WT_SESSION_IMPL *session, void *reconcilep) return; *(WT_RECONCILE **)reconcilep = NULL; - __wt_buf_free(session, &r->chunkA.key); - __wt_buf_free(session, &r->chunkA.min_key); - __wt_buf_free(session, &r->chunkA.image); - __wt_buf_free(session, &r->chunkB.key); - __wt_buf_free(session, &r->chunkB.min_key); - __wt_buf_free(session, &r->chunkB.image); + __wt_buf_free(session, &r->chunk_A.key); + __wt_buf_free(session, &r->chunk_A.min_key); + __wt_buf_free(session, &r->chunk_A.image); + __wt_buf_free(session, &r->chunk_B.key); + __wt_buf_free(session, &r->chunk_B.min_key); + __wt_buf_free(session, &r->chunk_B.image); __wt_free(session, r->supd); @@ -955,8 +955,8 @@ __wt_rec_split_init( r->disk_img_buf_size = WT_ALIGN(WT_MAX(corrected_page_size, r->split_size), btree->allocsize); /* Initialize the first split chunk. */ - WT_RET(__rec_split_chunk_init(session, r, &r->chunkA)); - r->cur_ptr = &r->chunkA; + WT_RET(__rec_split_chunk_init(session, r, &r->chunk_A)); + r->cur_ptr = &r->chunk_A; r->prev_ptr = NULL; /* Starting record number, entries, first free byte. */ @@ -1194,11 +1194,11 @@ __wt_rec_split(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len, bool if (forced) { WT_RET(__rec_split_write(session, r, r->cur_ptr, NULL, false)); r->prev_ptr = NULL; - r->cur_ptr = &r->chunkA; + r->cur_ptr = &r->chunk_A; } else { if (r->prev_ptr == NULL) { - WT_RET(__rec_split_chunk_init(session, r, &r->chunkB)); - r->prev_ptr = &r->chunkB; + WT_RET(__rec_split_chunk_init(session, r, &r->chunk_B)); + r->prev_ptr = &r->chunk_B; } tmp = r->prev_ptr; r->prev_ptr = r->cur_ptr; diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c index 1816942217f..b0b2fefb1fa 100644 --- a/src/third_party/wiredtiger/src/session/session_api.c +++ b/src/third_party/wiredtiger/src/session/session_api.c @@ -1683,6 +1683,31 @@ err: } /* + * __session_reset_snapshot -- + * WT_SESSION->reset_snapshot method. + */ +static int +__session_reset_snapshot(WT_SESSION *wt_session) +{ + WT_SESSION_IMPL *session; + + session = (WT_SESSION_IMPL *)wt_session; + /* Return error if the isolation mode is read committed. */ + if (session->txn->isolation != WT_ISO_SNAPSHOT) + WT_RET_MSG( + session, ENOTSUP, "not supported in read-committed or read-uncommitted transactions."); + + /* Return error if the session has performed any write operations. */ + if (F_ISSET(session->txn, WT_TXN_HAS_ID)) + WT_RET_MSG(session, ENOTSUP, "not supported in write transactions."); + + __wt_txn_release_snapshot(session); + __wt_txn_get_snapshot(session); + + return (0); +} + +/* * __session_transaction_pinned_range -- * WT_SESSION->transaction_pinned_range method. */ @@ -1922,9 +1947,9 @@ __open_session(WT_CONNECTION_IMPL *conn, WT_EVENT_HANDLER *event_handler, const __session_drop, __session_join, __session_log_flush, __session_log_printf, __session_rename, __session_reset, __session_salvage, __session_truncate, __session_upgrade, __session_verify, __session_begin_transaction, __session_commit_transaction, __session_prepare_transaction, - __session_rollback_transaction, __session_timestamp_transaction, __session_query_timestamp, - __session_checkpoint, __session_transaction_pinned_range, __session_transaction_sync, - __wt_session_breakpoint}, + __session_reset_snapshot, __session_rollback_transaction, __session_timestamp_transaction, + __session_query_timestamp, __session_checkpoint, __session_transaction_pinned_range, + __session_transaction_sync, __wt_session_breakpoint}, stds_readonly = {NULL, NULL, __session_close, __session_reconfigure, __wt_session_strerror, __session_open_cursor, __session_alter_readonly, __session_create_readonly, __wt_session_compact_readonly, __session_drop_readonly, __session_join, @@ -1932,9 +1957,10 @@ __open_session(WT_CONNECTION_IMPL *conn, WT_EVENT_HANDLER *event_handler, const __session_reset, __session_salvage_readonly, __session_truncate_readonly, __session_upgrade_readonly, __session_verify, __session_begin_transaction, __session_commit_transaction, __session_prepare_transaction_readonly, - __session_rollback_transaction, __session_timestamp_transaction, __session_query_timestamp, - __session_checkpoint_readonly, __session_transaction_pinned_range, - __session_transaction_sync_readonly, __wt_session_breakpoint}; + __session_reset_snapshot, __session_rollback_transaction, __session_timestamp_transaction, + __session_query_timestamp, __session_checkpoint_readonly, + __session_transaction_pinned_range, __session_transaction_sync_readonly, + __wt_session_breakpoint}; WT_DECL_RET; WT_SESSION_IMPL *session, *session_ret; uint32_t i; @@ -2002,8 +2028,8 @@ __open_session(WT_CONNECTION_IMPL *conn, WT_EVENT_HANDLER *event_handler, const TAILQ_INIT(&session_ret->cursor_cache[i]); session_ret->cursor_sweep_countdown = WT_SESSION_CURSOR_SWEEP_COUNTDOWN; - /* Initialize transaction support: default to read-committed. */ - session_ret->isolation = WT_ISO_READ_COMMITTED; + /* Initialize transaction support: default to snapshot. */ + session_ret->isolation = WT_ISO_SNAPSHOT; WT_ERR(__wt_txn_init(session, session_ret)); /* diff --git a/src/third_party/wiredtiger/src/support/generation.c b/src/third_party/wiredtiger/src/support/generation.c index e3a60621b5c..0c35bd96433 100644 --- a/src/third_party/wiredtiger/src/support/generation.c +++ b/src/third_party/wiredtiger/src/support/generation.c @@ -73,10 +73,14 @@ __wt_gen(WT_SESSION_IMPL *session, int which) * __wt_gen_next -- * Switch the resource to its next generation. */ -uint64_t -__wt_gen_next(WT_SESSION_IMPL *session, int which) +void +__wt_gen_next(WT_SESSION_IMPL *session, int which, uint64_t *genp) { - return (__wt_atomic_addv64(&S2C(session)->generations[which], 1)); + uint64_t gen; + + gen = __wt_atomic_addv64(&S2C(session)->generations[which], 1); + if (genp != NULL) + *genp = gen; } /* @@ -162,8 +166,8 @@ __wt_gen_drain(WT_SESSION_IMPL *session, int which, uint64_t generation) WT_IGNORE_RET(__wt_msg(session, "%s generation drain waited %u minutes", __gen_name(which), minutes)); ++minutes; + WT_ASSERT(session, minutes < 4); } - WT_ASSERT(session, stop - start < 3 * WT_MINUTE); } } } @@ -190,8 +194,7 @@ __gen_oldest(WT_SESSION_IMPL *session, int which) * the sessions that could have been active when we started our check. */ WT_ORDERED_READ(session_cnt, conn->session_cnt); - for (oldest = conn->generations[which] + 1, s = conn->sessions, i = 0; i < session_cnt; - ++s, ++i) { + for (oldest = conn->generations[which], s = conn->sessions, i = 0; i < session_cnt; ++s, ++i) { if (!s->active) continue; @@ -237,6 +240,12 @@ __wt_gen_active(WT_SESSION_IMPL *session, int which, uint64_t generation) return (true); } +#ifdef HAVE_DIAGNOSTIC + { + uint64_t oldest = __gen_oldest(session, which); + WT_ASSERT(session, generation < oldest); + } +#endif return (false); } @@ -262,6 +271,8 @@ __wt_session_gen_enter(WT_SESSION_IMPL *session, int which) * protected by a generation running outside one. */ WT_ASSERT(session, session->generations[which] == 0); + WT_ASSERT(session, session->active); + WT_ASSERT(session, session->id < S2C(session)->session_cnt); /* * Assign the thread's resource generation and publish it, ensuring threads waiting on a @@ -282,6 +293,9 @@ __wt_session_gen_enter(WT_SESSION_IMPL *session, int which) void __wt_session_gen_leave(WT_SESSION_IMPL *session, int which) { + WT_ASSERT(session, session->active); + WT_ASSERT(session, session->id < S2C(session)->session_cnt); + /* Ensure writes made by this thread are visible. */ WT_PUBLISH(session->generations[which], 0); diff --git a/src/third_party/wiredtiger/src/support/hazard.c b/src/third_party/wiredtiger/src/support/hazard.c index f1e552b8165..084bd54f1cd 100644 --- a/src/third_party/wiredtiger/src/support/hazard.c +++ b/src/third_party/wiredtiger/src/support/hazard.c @@ -50,7 +50,7 @@ hazard_grow(WT_SESSION_IMPL *session) * pointer generation number, and schedule a future free of the old memory. Ignore any failure, * leak the memory. */ - hazard_gen = __wt_gen_next(session, WT_GEN_HAZARD); + __wt_gen_next(session, WT_GEN_HAZARD, &hazard_gen); WT_IGNORE_RET(__wt_stash_add(session, WT_GEN_HAZARD, hazard_gen, ohazard, 0)); return (0); diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index f8c529ca74e..59e197c5c78 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -1621,7 +1621,7 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) * have to be refreshed. */ if (!readonly) - WT_IGNORE_RET(__wt_gen_next(session, WT_GEN_COMMIT)); + __wt_gen_next(session, WT_GEN_COMMIT, NULL); /* First check if we've made something durable in the future. */ update_durable_ts = false; diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c index 8178e593f6b..92066dd6e42 100644 --- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c +++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c @@ -834,7 +834,7 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) * We do need to update it before clearing the checkpoint's entry out of the transaction table, * or a thread evicting in a tree could ignore the checkpoint's transaction. */ - generation = __wt_gen_next(session, WT_GEN_CHECKPOINT); + __wt_gen_next(session, WT_GEN_CHECKPOINT, &generation); WT_STAT_CONN_SET(session, txn_checkpoint_generation, generation); /* diff --git a/src/third_party/wiredtiger/test/cppsuite/Makefile.am b/src/third_party/wiredtiger/test/cppsuite/Makefile.am index 4e819c3baea..e9855304fd2 100644 --- a/src/third_party/wiredtiger/test/cppsuite/Makefile.am +++ b/src/third_party/wiredtiger/test/cppsuite/Makefile.am @@ -1,5 +1,5 @@ AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include \ - -I$(top_srcdir)/test/utility + -I$(top_srcdir)/test/utility -I$(top_srcdir)/test/cppsuite LDADD = $(top_builddir)/test/utility/libtest_util.la \ $(top_builddir)/libwiredtiger.la AM_LDFLAGS = -static diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/test_harness.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/test_harness.h new file mode 100644 index 00000000000..a654b21e2ab --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/test_harness.h @@ -0,0 +1,20 @@ +/* Include guard. */ +#ifndef TEST_HARNESS_H +#define TEST_HARNESS_H + +extern "C" { +#include "wiredtiger.h" +} + +namespace test_harness { +class test { + public: + /* + * All tests will implement this initially, the return value from it will indicate whether the + * test was successful or not. + */ + virtual int run() = 0; +}; +} // namespace test_harness + +#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/poc.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/poc.cxx index a087f44c514..0bf50387344 100644 --- a/src/third_party/wiredtiger/test/cppsuite/tests/poc.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/tests/poc.cxx @@ -1,26 +1,30 @@ #include <iostream> -#include <stdlib.h> +#include <cstdlib> +#include "test_harness/test_harness.h" -extern "C" { - #include "wiredtiger.h" -} +class poc_test : public test_harness::test { + public: + int run() { + WT_CONNECTION *conn; + int ret = 0; + /* Setup basic test directory. */ + const std::string default_dir = "WT_TEST"; -int main(int argc, char *argv[]) { - WT_CONNECTION *conn; - int ret = 0; - /* Setup basic test directory. */ - const std::string default_dir = "WT_TEST"; + /* + * Csuite tests utilise a test_util.h command to make their directory, currently that doesn't + * compile under c++ and some extra work will be needed to make it work. Its unclear if the + * test framework will use test_util.h yet. + */ + const std::string mkdir_cmd = "mkdir " + default_dir; + ret = system(mkdir_cmd.c_str()); + if (ret != 0) + return (ret); - /* - * Csuite tests utilise a test_util.h command to make their directory, currently that doesn't - * compile under c++ and some extra work will be needed to make it work. Its unclear if the - * test framework will use test_util.h yet. - */ - const std::string mkdir_cmd = "mkdir " + default_dir; - ret = system(mkdir_cmd.c_str()); - if (ret != 0) + ret = wiredtiger_open(default_dir.c_str(), NULL, "create,cache_size=1G", &conn); return (ret); + } +}; - ret = wiredtiger_open(default_dir.c_str(), NULL, "create,cache_size=1G", &conn); - return (ret); +int main(int argc, char *argv[]) { + return poc_test().run(); } diff --git a/src/third_party/wiredtiger/test/format/compact.c b/src/third_party/wiredtiger/test/format/compact.c index fc4a993d1da..bd2a5df2155 100644 --- a/src/third_party/wiredtiger/test/format/compact.c +++ b/src/third_party/wiredtiger/test/format/compact.c @@ -67,7 +67,8 @@ compact(void *arg) * We don't configure a timeout and occasionally exceed the default of 1200 seconds. */ ret = session->compact(session, g.uri, NULL); - if (ret != 0 && ret != EBUSY && ret != ETIMEDOUT && ret != WT_ROLLBACK) + if (ret != 0 && ret != EBUSY && ret != ETIMEDOUT && ret != WT_ROLLBACK && + ret != WT_CACHE_FULL) testutil_die(ret, "session.compact"); } diff --git a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-6568 b/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-6568 new file mode 100644 index 00000000000..4e695a8d87e --- /dev/null +++ b/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-6568 @@ -0,0 +1,101 @@ +############################################ +# RUN PARAMETERS: V2 +############################################ +assert.commit_timestamp=0 +assert.read_timestamp=0 +backup=0 +backup.incremental=off +backup.incr_granularity=11956 +btree.bitcnt=4 +btree.compression=snappy +btree.dictionary=0 +btree.huffman_value=0 +btree.internal_key_truncation=1 +btree.internal_page_max=16 +btree.key_gap=2 +btree.key_max=54 +btree.key_min=20 +btree.leaf_page_max=9 +btree.memory_page_max=1 +btree.prefix_compression=1 +btree.prefix_compression_min=5 +btree.repeat_data_pct=79 +btree.reverse=0 +btree.split_pct=100 +btree.value_max=2589 +btree.value_min=13 +cache=64 +cache.evict_max=2 +cache.minimum=20 +checkpoint=wiredtiger +checkpoint.log_size=73 +checkpoint.wait=12 +disk.checksum=uncompressed +disk.data_extend=0 +disk.direct_io=0 +disk.encryption=none +disk.firstfit=0 +disk.mmap=0 +disk.mmap_all=0 +format.abort=0 +format.independent_thread_rng=1 +format.major_timeout=0 +logging=1 +logging.archive=0 +logging.compression=none +logging.file_max=461066 +logging.prealloc=0 +lsm.auto_throttle=1 +lsm.bloom=1 +lsm.bloom_bit_count=57 +lsm.bloom_hash_count=20 +lsm.bloom_oldest=0 +lsm.chunk_size=7 +lsm.merge_max=14 +lsm.worker_threads=4 +ops.alter=0 +ops.compaction=0 +ops.hs_cursor=0 +ops.pct.delete=33 +ops.pct.insert=2 +ops.pct.modify=27 +ops.pct.read=34 +ops.pct.write=4 +ops.prepare=0 +ops.random_cursor=0 +ops.salvage=0 +ops.truncate=1 +ops.verify=1 +quiet=1 +runs=1 +runs.in_memory=0 +runs.ops=0 +runs.rows=2600000 +runs.source=table +runs.threads=32 +runs.timer=6 +runs.type=row-store +runs.verify_failure_dump=0 +statistics=0 +statistics.server=0 +stress.aggressive_sweep=0 +stress.checkpoint=0 +stress.checkpoint_prepare=0 +stress.hs_checkpoint_delay=0 +stress.hs_sweep=0 +stress.split_1=0 +stress.split_2=0 +stress.split_3=0 +stress.split_4=0 +stress.split_5=0 +stress.split_6=1 +stress.split_7=0 +stress.split_8=0 +transaction.frequency=100 +transaction.isolation=snapshot +transaction.rollback_to_stable=0 +transaction.timestamps=1 +wiredtiger.config= +wiredtiger.rwlock=1 +wiredtiger.leak_memory=0 +############################################ diff --git a/src/third_party/wiredtiger/test/format/format.sh b/src/third_party/wiredtiger/test/format/format.sh index 2b678aa755d..9d462aed0df 100755 --- a/src/third_party/wiredtiger/test/format/format.sh +++ b/src/third_party/wiredtiger/test/format/format.sh @@ -364,7 +364,7 @@ resolve() grep 'data_source=file' $dir/CONFIG > /dev/null && uri="file:wt" # Use the wt utility to recover & verify the object. - if $($wt_binary -R -h $dir verify $uri >> $log 2>&1); then + if $($wt_binary -m -R -h $dir verify $uri >> $log 2>&1); then rm -rf $dir $dir.RECOVER $log success=$(($success + 1)) verbose "$name: job in $dir successfully completed" diff --git a/src/third_party/wiredtiger/test/format/hs.c b/src/third_party/wiredtiger/test/format/hs.c index 4bb29325faa..de895da17d6 100644 --- a/src/third_party/wiredtiger/test/format/hs.c +++ b/src/third_party/wiredtiger/test/format/hs.c @@ -107,7 +107,7 @@ hs_cursor(void *arg) cursor, &hs_stop_durable_ts, &hs_durable_timestamp, &hs_upd_type, &hs_value)); continue; } - testutil_assert(ret == WT_NOTFOUND || ret == WT_ROLLBACK); + testutil_assert(ret == WT_NOTFOUND || ret == WT_ROLLBACK || ret == WT_CACHE_FULL); break; } diff --git a/src/third_party/wiredtiger/test/format/ops.c b/src/third_party/wiredtiger/test/format/ops.c index 7b37f91573b..c4ca4fff542 100644 --- a/src/third_party/wiredtiger/test/format/ops.c +++ b/src/third_party/wiredtiger/test/format/ops.c @@ -636,7 +636,7 @@ prepare_transaction(TINFO *tinfo) #define OP_FAILED(notfound_ok) \ do { \ positioned = false; \ - if (intxn && (ret == WT_CACHE_FULL || ret == WT_ROLLBACK)) \ + if (intxn && (ret == WT_CACHE_FULL || ret == WT_ROLLBACK || ret == WT_CACHE_FULL)) \ goto rollback; \ testutil_assert( \ (notfound_ok && ret == WT_NOTFOUND) || ret == WT_CACHE_FULL || ret == WT_ROLLBACK); \ @@ -1059,8 +1059,8 @@ update_instead_of_chosen_op: __wt_yield(); /* Encourage races */ ret = snap_repeat_txn(cursor, tinfo); - testutil_assert(ret == 0 || ret == WT_ROLLBACK); - if (ret == WT_ROLLBACK) + testutil_assert(ret == 0 || ret == WT_ROLLBACK || ret == WT_CACHE_FULL); + if (ret == WT_ROLLBACK || ret == WT_CACHE_FULL) goto rollback; } @@ -1154,6 +1154,7 @@ wts_read_scan(void) case 0: case WT_NOTFOUND: case WT_ROLLBACK: + case WT_CACHE_FULL: case WT_PREPARE_CONFLICT: break; default: diff --git a/src/third_party/wiredtiger/test/format/random.c b/src/third_party/wiredtiger/test/format/random.c index f78697539a1..dcd5187917a 100644 --- a/src/third_party/wiredtiger/test/format/random.c +++ b/src/third_party/wiredtiger/test/format/random.c @@ -75,6 +75,7 @@ random_kv(void *arg) break; case WT_NOTFOUND: case WT_ROLLBACK: + case WT_CACHE_FULL: case WT_PREPARE_CONFLICT: continue; default: diff --git a/src/third_party/wiredtiger/test/format/recover.sh b/src/third_party/wiredtiger/test/format/recover.sh index 1fb94b6d27f..6f65db73eb5 100644 --- a/src/third_party/wiredtiger/test/format/recover.sh +++ b/src/third_party/wiredtiger/test/format/recover.sh @@ -48,5 +48,5 @@ while true; do fi # We know we aborted, so force recovery to run. - $wtcmd -R -h RUNDIR verify $uri || exit 1 + $wtcmd -m -R -h RUNDIR verify $uri || exit 1 done diff --git a/src/third_party/wiredtiger/test/format/snap.c b/src/third_party/wiredtiger/test/format/snap.c index 4fe784e87db..a0e1964f194 100644 --- a/src/third_party/wiredtiger/test/format/snap.c +++ b/src/third_party/wiredtiger/test/format/snap.c @@ -554,7 +554,7 @@ snap_repeat(WT_CURSOR *cursor, TINFO *tinfo, SNAP_OPS *snap, bool rollback_allow /* The only expected error is rollback. */ ret = snap_verify(cursor, tinfo, snap); - if (ret != 0 && (!rollback_allowed || ret != WT_ROLLBACK)) + if (ret != 0 && (!rollback_allowed || (ret != WT_ROLLBACK && ret != WT_CACHE_FULL))) testutil_check(ret); } else if (ret == EINVAL) snap_ts_clear(tinfo, snap->ts); diff --git a/src/third_party/wiredtiger/test/suite/suite_random.py b/src/third_party/wiredtiger/test/suite/suite_random.py index 4ae8691def8..f329dd8cabc 100644 --- a/src/third_party/wiredtiger/test/suite/suite_random.py +++ b/src/third_party/wiredtiger/test/suite/suite_random.py @@ -59,7 +59,7 @@ class suite_random: self.seedz = (36969 * (z & 65535) + (z >> 16)) & 0xffffffff self.seedw = (18000 * (w & 65535) + (w >> 16)) & 0xffffffff - return ((z << 16) + w & 65535) & 0xffffffff + return ((z << 16) + (w & 65535)) & 0xffffffff def rand_range(self, n, m): """ diff --git a/src/third_party/wiredtiger/test/suite/test_backup13.py b/src/third_party/wiredtiger/test/suite/test_backup13.py index 10faed087ca..e77df8f5fcb 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup13.py +++ b/src/third_party/wiredtiger/test/suite/test_backup13.py @@ -39,9 +39,16 @@ class test_backup13(wttest.WiredTigerTestCase, suite_subprocess): conn_config='cache_size=1G,log=(enabled,file_max=100K)' dir='backup.dir' # Backup directory name logmax="100K" - uri="table:test" - nops=1000 mult=0 + nops=1000 + uri="table:test" + + scenarios = make_scenarios([ + ('default', dict(sess_cfg='')), + ('read-committed', dict(sess_cfg='isolation=read-committed')), + ('read-uncommitted', dict(sess_cfg='isolation=read-uncommitted')), + ('snapshot', dict(sess_cfg='isolation=snapshot')), + ]) pfx = 'test_backup' # Set the key and value big enough that we modify a few blocks. @@ -72,14 +79,24 @@ class test_backup13(wttest.WiredTigerTestCase, suite_subprocess): num = i + (self.mult * self.nops) key = self.bigkey + str(num) val = self.bigval + str(num) - c[key] = val + c.set_key(key) + c.set_value(val) + # read committed and read uncommitted transactions are readonly, any write operations with + # these isolation levels should throw an error. + if self.sess_cfg == 'isolation=read-committed' or self.sess_cfg == 'isolation=read-uncommitted': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: c.insert(), "/not supported in read-committed or read-uncommitted transactions/") + else: + c.insert() self.session.checkpoint() c.close() # Increase the multiplier so that later calls insert unique items. self.mult += 1 - def test_backup13(self): + def session_config(self): + return self.sess_cfg + def test_backup13(self): self.session.create(self.uri, "key_format=S,value_format=S") self.add_data(self.uri) diff --git a/src/third_party/wiredtiger/test/suite/test_backup20.py b/src/third_party/wiredtiger/test/suite/test_backup20.py new file mode 100644 index 00000000000..81deb6026bb --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_backup20.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2020 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +import os, shutil +from helper import compare_files +from suite_subprocess import suite_subprocess +from wtdataset import simple_key +from wtscenario import make_scenarios + +# test_backup20.py +# Test cursor backup force stop without a checkpoint. +# This reproduces the issue from WT-7027 where we hit an assertion +# because the session was created with snapshot isolation. +class test_backup20(wttest.WiredTigerTestCase, suite_subprocess): + conn_config='cache_size=1G,log=(enabled,file_max=100K)' + dir='backup.dir' # Backup directory name + logmax="100K" + uri="table:test" + nops=1000 + mult=0 + + pfx = 'test_backup' + + scenarios = make_scenarios([ + ('default', dict(sess_cfg='')), + ('read-committed', dict(sess_cfg='isolation=read-committed')), + ('read-uncommitted', dict(sess_cfg='isolation=read-uncommitted')), + ('snapshot', dict(sess_cfg='isolation=snapshot')), + ]) + + def session_config(self): + return self.sess_cfg + + def test_backup20(self): + self.session.create(self.uri, "key_format=S,value_format=S") + + # Open up the backup cursor. This causes a new log file to be created. + # That log file is not part of the list returned. This is a full backup + # primary cursor with incremental configured. + config = 'incremental=(enabled,granularity=1M,this_id="ID1")' + bkup_c = self.session.open_cursor('backup:', None, config) + bkup_c.close() + + # Do a force stop to release resources and reset the system. + config = 'incremental=(force_stop=true)' + bkup_c = self.session.open_cursor('backup:', None, config) + bkup_c.close() + + self.session.close() + self.conn.close() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_cursor12.py b/src/third_party/wiredtiger/test/suite/test_cursor12.py index 79114828d2d..6d2d1a051c4 100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor12.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor12.py @@ -263,8 +263,7 @@ class test_cursor12(wttest.WiredTigerTestCase): row = row + 1 c.close() - # Smoke-test the modify API, anything other than an explicit transaction - # in snapshot isolation fails. + # Smoke-test the modify API, anything other than an snapshot isolation fails. def test_modify_txn_api(self): ds = SimpleDataSet(self, self.uri, 100, key_format=self.keyfmt, value_format=self.valuefmt) ds.populate() @@ -273,12 +272,6 @@ class test_cursor12(wttest.WiredTigerTestCase): c.set_key(ds.key(10)) msg = '/not supported/' - self.session.begin_transaction() - mods = [] - mods.append(wiredtiger.Modify('-', 1, 1)) - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: c.modify(mods), msg) - self.session.rollback_transaction() - self.session.begin_transaction("isolation=read-uncommitted") mods = [] mods.append(wiredtiger.Modify('-', 1, 1)) diff --git a/src/third_party/wiredtiger/test/suite/test_isolation01.py b/src/third_party/wiredtiger/test/suite/test_isolation01.py new file mode 100644 index 00000000000..90f73c7c008 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_isolation01.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2020 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_isolation01.py +# Transactions isolation mode: This test is to test for different isolation modes. +# The API reset_snapshot should return error when called withread committed isolation mode +# or when the session has performed any write operations. +# + +import wiredtiger, wttest +from wtscenario import make_scenarios + +class test_isolation01(wttest.WiredTigerTestCase): + + uri = 'table:test_isolation01' + iso_types = [ + ('isolation_read_uncommitted', dict(isolation='read-uncommitted')), + ('isolation_read_committed', dict(isolation='read-committed')), + ('isolation_snapshot', dict(isolation='snapshot')) + ] + scenarios = make_scenarios(iso_types) + key = 'key' + value = 'value' + + def test_isolation_level(self): + self.session.create(self.uri, 'key_format=S,value_format=S') + cursor = self.session.open_cursor(self.uri, None) + + # Begin a transaction with different isolation levels. + self.session.begin_transaction('isolation=' + self.isolation) + cursor.set_key(self.key) + cursor.set_value(self.value) + # read committed and read uncommitted transactions are readonly, any write operations with + # these isolation levels should throw an error. + if self.isolation != 'snapshot': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: cursor.insert(), "/not supported in read-committed or read-uncommitted transactions/") + else: + self.assertEqual(cursor.insert(), 0) + + if self.isolation == 'snapshot': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.reset_snapshot(), "/not supported in write transactions/") + else: + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.reset_snapshot(), "/not supported in read-committed or read-uncommitted transactions/") + self.session.commit_transaction() + + cursor2 = self.session.open_cursor(self.uri, None) + self.session.begin_transaction('isolation=' + self.isolation) + cursor2.set_key(self.key) + cursor2.search() + if self.isolation == 'snapshot': + self.session.reset_snapshot() + else: + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.reset_snapshot(), "/not supported in read-committed or read-uncommitted transactions/") + +if __name__ == '__main__': + wttest.run() |