diff options
47 files changed, 1465 insertions, 953 deletions
diff --git a/src/third_party/wiredtiger/.clang-format b/src/third_party/wiredtiger/.clang-format index 5cd39db2e69..573572df901 100644 --- a/src/third_party/wiredtiger/.clang-format +++ b/src/third_party/wiredtiger/.clang-format @@ -49,14 +49,16 @@ ForEachMacros: - Q_FOREACH - BOOST_FOREACH - TAILQ_FOREACH + - WT_CELL_FOREACH_BEGIN + - WT_CELL_FOREACH_VRFY - WT_CKPT_FOREACH + - WT_COL_FOREACH - WT_EXT_FOREACH - WT_EXT_FOREACH_OFF - WT_FIX_FOREACH - - WT_CELL_FOREACH_BEGIN - - WT_CELL_FOREACH_VRFY - - WT_COL_FOREACH - WT_INTL_FOREACH_BEGIN + - WT_MODIFY_FOREACH_BEGIN + - WT_MODIFY_FOREACH_REVERSE - WT_ROW_FOREACH - WT_SKIP_FOREACH IncludeCategories: diff --git a/src/third_party/wiredtiger/build_posix/Make.subdirs b/src/third_party/wiredtiger/build_posix/Make.subdirs index 307084aa7e5..f98a88e0daa 100644 --- a/src/third_party/wiredtiger/build_posix/Make.subdirs +++ b/src/third_party/wiredtiger/build_posix/Make.subdirs @@ -47,3 +47,6 @@ test/thread # Benchmark programs. bench/workgen PYTHON HAVE_CXX bench/wtperf + +# Utility programs. +tools/xray_to_optrack LLVM HAVE_CXX diff --git a/src/third_party/wiredtiger/build_posix/aclocal/options.m4 b/src/third_party/wiredtiger/build_posix/aclocal/options.m4 index 5520a94ac6a..132a0cb609f 100644 --- a/src/third_party/wiredtiger/build_posix/aclocal/options.m4 +++ b/src/third_party/wiredtiger/build_posix/aclocal/options.m4 @@ -275,4 +275,23 @@ no) wt_cv_crc32_hardware=no AC_MSG_RESULT(no);; esac +AC_MSG_CHECKING(if --enable-llvm option specified) +AC_ARG_ENABLE(llvm, + [AS_HELP_STRING([--enable-llvm], + [Configure with LLVM.])], r=$enableval, r=no) +case "$r" in +no) wt_cv_enable_llvm=no;; +*) wt_cv_enable_llvm=yes;; +esac +AC_MSG_RESULT($wt_cv_enable_llvm) +if test "$wt_cv_enable_llvm" = "yes"; then + AC_CHECK_PROG(wt_cv_llvm_config, llvm-config, yes) + if test "$wt_cv_llvm_config" != "yes"; then + AC_MSG_ERROR([--enable-llvm requires llvm-config]) + fi + if ! test $(llvm-config --version | grep "^8"); then + AC_MSG_ERROR([llvm-config must be version 8]) + fi +fi +AM_CONDITIONAL([LLVM], [test x$wt_cv_enable_llvm = xyes]) ]) diff --git a/src/third_party/wiredtiger/dist/s_funcs.list b/src/third_party/wiredtiger/dist/s_funcs.list index bcedc19a459..0218937fffc 100644 --- a/src/third_party/wiredtiger/dist/s_funcs.list +++ b/src/third_party/wiredtiger/dist/s_funcs.list @@ -16,6 +16,7 @@ __wt_config_getone __wt_cursor_get_raw_value __wt_debug_addr __wt_debug_addr_print +__wt_debug_cursor_las __wt_debug_cursor_page __wt_debug_offset __wt_debug_set_verbose diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index e770694e17c..66480a55eec 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -210,6 +210,7 @@ Kanowski's Kounavis LANGID LAS +LASdump LF LLLLLL LLLLLLL diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index 80c5d3e54c3..aa9ed2cd3fa 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -220,6 +220,10 @@ connection_stats = [ CacheStat('cache_eviction_dirty', 'modified pages evicted'), CacheStat('cache_eviction_empty_score', 'eviction empty score', 'no_clear,no_scale'), CacheStat('cache_eviction_fail', 'pages selected for eviction unable to be evicted'), + CacheStat('cache_eviction_fail_active_children_on_an_internal_page', 'pages selected for eviction unable to be evicted because of active children on an internal page'), + CacheStat('cache_eviction_fail_in_reconciliation', 'pages selected for eviction unable to be evicted because of failure in reconciliation'), + CacheStat('cache_eviction_fail_parent_has_overflow_items', 'pages selected for eviction unable to be evicted as the parent page has overflow items'), + CacheStat('cache_eviction_fail_with_newer_modifications_on_a_clean_page', 'pages selected for eviction unable to be evicted due to newer modifications on a clean page'), CacheStat('cache_eviction_force', 'forced eviction - pages selected count'), CacheStat('cache_eviction_force_clean', 'forced eviction - pages evicted that were clean count'), CacheStat('cache_eviction_force_clean_time', 'forced eviction - pages evicted that were clean time (usecs)'), @@ -234,12 +238,16 @@ connection_stats = [ CacheStat('cache_eviction_get_ref_empty2', 'eviction calls to get a page found queue empty after locking'), CacheStat('cache_eviction_hazard', 'hazard pointer blocked page eviction'), CacheStat('cache_eviction_internal', 'internal pages evicted'), + CacheStat('cache_eviction_internal_pages_seen', 'internal pages seen by eviction walk'), + CacheStat('cache_eviction_internal_pages_already_queued', 'internal pages seen by eviction walk that are already queued'), + CacheStat('cache_eviction_internal_pages_queued', 'internal pages queued for eviction'), CacheStat('cache_eviction_maximum_page_size', 'maximum page size at eviction', 'no_clear,no_scale,size'), CacheStat('cache_eviction_pages_queued', 'pages queued for eviction'), CacheStat('cache_eviction_pages_queued_oldest', 'pages queued for urgent eviction during walk'), CacheStat('cache_eviction_pages_queued_post_lru', 'pages queued for eviction post lru sorting'), CacheStat('cache_eviction_pages_queued_urgent', 'pages queued for urgent eviction'), CacheStat('cache_eviction_pages_seen', 'pages seen by eviction walk'), + CacheStat('cache_eviction_pages_already_queued', 'pages seen by eviction walk that are already queued'), CacheStat('cache_eviction_queue_empty', 'eviction server candidate queue empty when topping up'), CacheStat('cache_eviction_queue_not_empty', 'eviction server candidate queue not empty when topping up'), CacheStat('cache_eviction_server_evicting', 'eviction server evicting pages'), @@ -254,10 +262,11 @@ connection_stats = [ CacheStat('cache_eviction_target_page_lt128', 'eviction walk target pages histogram - 64-128'), CacheStat('cache_eviction_target_page_lt32', 'eviction walk target pages histogram - 10-31'), CacheStat('cache_eviction_target_page_lt64', 'eviction walk target pages histogram - 32-63'), + CacheStat('cache_eviction_target_strategy_clean', 'eviction walk target strategy only clean pages'), + CacheStat('cache_eviction_target_strategy_dirty', 'eviction walk target strategy only dirty pages'), + CacheStat('cache_eviction_target_strategy_both_clean_and_dirty', 'eviction walk target strategy both clean and dirty pages'), CacheStat('cache_eviction_walk', 'pages walked for eviction'), CacheStat('cache_eviction_walk_from_root', 'eviction walks started from root of tree'), - CacheStat('cache_eviction_walk_internal_yield', 'eviction server waiting for an internal page yields'), - CacheStat('cache_eviction_walk_internal_wait', 'eviction server waiting for an internal page sleep (usec)'), CacheStat('cache_eviction_walk_leaf_notfound', 'eviction server waiting for a leaf page'), CacheStat('cache_eviction_walk_passes', 'eviction passes of a file'), CacheStat('cache_eviction_walk_saved_pos', 'eviction walks started from saved location in tree'), @@ -576,7 +585,6 @@ connection_stats = [ TxnStat('txn_prepare_rollback', 'prepared transactions rolled back'), TxnStat('txn_prepared_updates_count', 'Number of prepared updates'), TxnStat('txn_prepared_updates_lookaside_inserts', 'Number of prepared updates added to cache overflow'), - TxnStat('txn_prepared_updates_resolved', 'Number of prepared updates resolved'), TxnStat('txn_query_ts', 'query timestamp calls'), TxnStat('txn_read_queue_empty', 'read timestamp queue insert to empty'), TxnStat('txn_read_queue_head', 'read timestamp queue inserts to head'), diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 988f14028e8..8a69d9b9ae6 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -1,5 +1,5 @@ { - "commit": "aad5d17b00b9921f206b5255a4d79f404db79321", + "commit": "1311372125d14fe97b07d0f4c165b1ad9efb4e9a", "github": "wiredtiger/wiredtiger.git", "vendor": "wiredtiger", "branch": "mongodb-4.2" diff --git a/src/third_party/wiredtiger/src/block/block_open.c b/src/third_party/wiredtiger/src/block/block_open.c index ba32bce74bc..868d7c5d149 100644 --- a/src/third_party/wiredtiger/src/block/block_open.c +++ b/src/third_party/wiredtiger/src/block/block_open.c @@ -345,8 +345,11 @@ __desc_read(WT_SESSION_IMPL *session, uint32_t allocsize, WT_BLOCK *block) * without some reason to believe they are WiredTiger files. The user may have entered the wrong * file name, and is now frantically pounding their interrupt key. */ - if (desc->magic != WT_BLOCK_MAGIC || !checksum_matched) + if (desc->magic != WT_BLOCK_MAGIC || !checksum_matched) { + if (strcmp(block->name, WT_METAFILE) == 0) + WT_ERR_MSG(session, WT_TRY_SALVAGE, "%s is corrupted", block->name); WT_ERR_MSG(session, WT_ERROR, "%s does not appear to be a WiredTiger file", block->name); + } if (desc->majorv > WT_BLOCK_MAJOR_VERSION || (desc->majorv == WT_BLOCK_MAJOR_VERSION && desc->minorv > WT_BLOCK_MINOR_VERSION)) diff --git a/src/third_party/wiredtiger/src/btree/bt_cursor.c b/src/third_party/wiredtiger/src/btree/bt_cursor.c index 4e85f1528a8..11201846a38 100644 --- a/src/third_party/wiredtiger/src/btree/bt_cursor.c +++ b/src/third_party/wiredtiger/src/btree/bt_cursor.c @@ -101,17 +101,16 @@ __cursor_page_pinned(WT_CURSOR_BTREE *cbt, bool search_operation) return (false); /* - * If we are doing an update, we need a page with history, release the page so we get it again - * with history if required. Eviction may be locking the page, wait until we see a "normal" - * state and then test against that state (eviction may have already locked the page again). + * We need a page with history: updates need complete update lists and a read might be based on + * a different timestamp than the one that brought the page into memory. Release the page and + * read it again with history if required. Eviction may be locking the page, wait until we see a + * "normal" state and then test against that state (eviction may have already locked the page + * again). */ - if (F_ISSET(&session->txn, WT_TXN_UPDATE)) { - while ((current_state = cbt->ref->state) == WT_REF_LOCKED) - __wt_yield(); - return (current_state == WT_REF_MEM); - } - - return (true); + while ((current_state = cbt->ref->state) == WT_REF_LOCKED) + __wt_yield(); + WT_ASSERT(session, current_state == WT_REF_LIMBO || current_state == WT_REF_MEM); + return (current_state == WT_REF_MEM); } /* diff --git a/src/third_party/wiredtiger/src/btree/bt_debug.c b/src/third_party/wiredtiger/src/btree/bt_debug.c index f971de0e4f0..d4babb1df7c 100644 --- a/src/third_party/wiredtiger/src/btree/bt_debug.c +++ b/src/third_party/wiredtiger/src/btree/bt_debug.c @@ -698,6 +698,28 @@ __wt_debug_cursor_page(void *cursor_arg, const char *ofile) } /* + * __wt_debug_cursor_las -- + * Dump the LAS tree given a user cursor. + */ +int +__wt_debug_cursor_las(void *cursor_arg, const char *ofile) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) +{ + WT_CONNECTION_IMPL *conn; + WT_CURSOR *cursor; + WT_CURSOR_BTREE *cbt; + WT_SESSION_IMPL *las_session; + + cursor = cursor_arg; + conn = S2C((WT_SESSION_IMPL *)cursor->session); + las_session = conn->cache->las_session[0]; + if (las_session == NULL) + return (0); + cbt = (WT_CURSOR_BTREE *)las_session->las_cursor; + return (__wt_debug_tree_all(las_session, cbt->btree, NULL, ofile)); +} + +/* * __debug_tree -- * Dump the in-memory information for a tree. */ diff --git a/src/third_party/wiredtiger/src/btree/bt_delete.c b/src/third_party/wiredtiger/src/btree/bt_delete.c index f7b63524d42..4686c206c0e 100644 --- a/src/third_party/wiredtiger/src/btree/bt_delete.c +++ b/src/third_party/wiredtiger/src/btree/bt_delete.c @@ -161,25 +161,12 @@ __wt_delete_page_rollback(WT_SESSION_IMPL *session, WT_REF *ref) uint32_t current_state; bool locked; - /* - * If the page is still "deleted", it's as we left it, reset the state to on-disk and we're - * done. Otherwise, we expect the page is either instantiated or being instantiated. Loop - * because it's possible for the page to return to the deleted state if instantiation fails. - */ + /* Lock the reference. We cannot access ref->page_del except when locked. */ for (locked = false, sleep_usecs = yield_count = 0;;) { switch (current_state = ref->state) { - case WT_REF_DELETED: - /* - * If the page is still "deleted", it's as we left it, reset the state. - */ - if (WT_REF_CAS_STATE(session, ref, WT_REF_DELETED, ref->page_del->previous_state)) - goto done; - break; case WT_REF_LOCKED: - /* - * A possible state, the page is being instantiated. - */ break; + case WT_REF_DELETED: case WT_REF_MEM: case WT_REF_SPLIT: if (WT_REF_CAS_STATE(session, ref, current_state, WT_REF_LOCKED)) @@ -205,6 +192,8 @@ __wt_delete_page_rollback(WT_SESSION_IMPL *session, WT_REF *ref) } /* + * If the page is still "deleted", it's as we left it, all we have to do is reset the state. + * * We can't use the normal read path to get a copy of the page because the session may have * closed the cursor, we no longer have the reference to the tree required for a hazard pointer. * We're safe because with unresolved transactions, the page isn't going anywhere. @@ -212,19 +201,16 @@ __wt_delete_page_rollback(WT_SESSION_IMPL *session, WT_REF *ref) * The page is in an in-memory state, which means it was instantiated at some point. Walk any * list of update structures and abort them. */ - WT_ASSERT(session, locked); - if ((updp = ref->page_del->update_list) != NULL) + if (current_state == WT_REF_DELETED) + current_state = ref->page_del->previous_state; + else if ((updp = ref->page_del->update_list) != NULL) for (; *updp != NULL; ++updp) (*updp)->txnid = WT_TXN_ABORTED; - WT_REF_SET_STATE(ref, current_state); + /* Finally mark the truncate aborted */ + ref->page_del->txnid = WT_TXN_ABORTED; -done: - /* - * Now mark the truncate aborted: this must come last because after this point there is nothing - * preventing the page from being evicted. - */ - WT_PUBLISH(ref->page_del->txnid, WT_TXN_ABORTED); + WT_REF_SET_STATE(ref, current_state); return (0); } diff --git a/src/third_party/wiredtiger/src/btree/bt_walk.c b/src/third_party/wiredtiger/src/btree/bt_walk.c index d1efbb2533d..22b3b717fe0 100644 --- a/src/third_party/wiredtiger/src/btree/bt_walk.c +++ b/src/third_party/wiredtiger/src/btree/bt_walk.c @@ -263,13 +263,13 @@ __tree_walk_internal(WT_SESSION_IMPL *session, WT_REF **refp, uint64_t *walkcntp WT_DECL_RET; WT_PAGE_INDEX *pindex; WT_REF *couple, *ref, *ref_orig; - uint64_t restart_sleep, restart_yield, swap_sleep, swap_yield; + uint64_t restart_sleep, restart_yield; uint32_t current_state, slot; bool empty_internal, prev, skip; btree = S2BT(session); pindex = NULL; - restart_sleep = restart_yield = swap_sleep = swap_yield = 0; + restart_sleep = restart_yield = 0; empty_internal = false; /* @@ -398,34 +398,25 @@ restart: if (LF_ISSET(WT_READ_SKIP_INTL)) continue; - for (;;) { - /* - * Swap our previous hazard pointer for the page we'll return. - * - * Not-found is an expected return, as eviction might have been attempted. The page - * can't be evicted, we're holding a hazard pointer on a child, spin until we're - * successful. - * - * Restart is not expected, our parent WT_REF should not have split. - */ - ret = __wt_page_swap(session, couple, ref, WT_READ_NOTFOUND_OK | flags); - if (ret == 0) { - /* Success, "couple" released. */ - couple = NULL; - *refp = ref; - goto done; - } + /* + * Swap our previous hazard pointer for the page we'll return. + * + * Not-found is an expected return, as eviction might have been attempted. Restart is + * not expected, our parent WT_REF should not have split. + */ + ret = __wt_page_swap(session, couple, ref, WT_READ_NOTFOUND_OK | flags); + if (ret == 0) { + /* Success, "couple" released. */ + couple = NULL; + *refp = ref; + goto done; + } - WT_ASSERT(session, ret == WT_NOTFOUND); - WT_ERR_NOTFOUND_OK(ret); + /* An expected error, so "couple" is unchanged. */ + if (ret == WT_NOTFOUND) + continue; - __wt_spin_backoff(&swap_yield, &swap_sleep); - if (swap_yield < 1000) - WT_STAT_CONN_INCR(session, cache_eviction_walk_internal_yield); - if (swap_sleep != 0) - WT_STAT_CONN_INCRV(session, cache_eviction_walk_internal_wait, swap_sleep); - } - /* NOTREACHED */ + goto err; } if (prev) diff --git a/src/third_party/wiredtiger/src/btree/row_modify.c b/src/third_party/wiredtiger/src/btree/row_modify.c index c34d2f1de70..30f27b19048 100644 --- a/src/third_party/wiredtiger/src/btree/row_modify.c +++ b/src/third_party/wiredtiger/src/btree/row_modify.c @@ -292,6 +292,7 @@ __wt_update_obsolete_check( size_t size; uint64_t oldest, stable; u_int count, upd_seen, upd_unstable; + bool upd_visible_all_seen; txn_global = &S2C(session)->txn_global; @@ -307,14 +308,18 @@ __wt_update_obsolete_check( * * Only updates with globally visible, self-contained data can terminate update chains. * - * Birthmarks are a special case: once a birthmark becomes obsolete, it can be discarded and - * subsequent reads will see the on-page value (as expected). Inserting updates into the - * lookaside table relies on this behavior to avoid creating update chains with multiple - * birthmarks. + * Birthmarks are a special case: once a birthmark becomes obsolete, it can be discarded if + * there is a globally visible update before it and subsequent reads will see the on-page value + * (as expected). Inserting updates into the lookaside table relies on this behavior to avoid + * creating update chains with multiple birthmarks. We cannot discard the birthmark if it's the + * first globally visible update as the previous updates can be aborted and be freed causing the + * entire update chain being removed. */ - for (first = prev = NULL, count = 0; upd != NULL; prev = upd, upd = upd->next, count++) { + for (first = prev = NULL, upd_visible_all_seen = false, count = 0; upd != NULL; + prev = upd, upd = upd->next, count++) { if (upd->txnid == WT_TXN_ABORTED) continue; + ++upd_seen; if (!__wt_txn_upd_visible_all(session, upd)) { first = NULL; @@ -324,10 +329,24 @@ __wt_update_obsolete_check( */ if (upd->start_ts != WT_TS_NONE && upd->start_ts >= oldest && upd->start_ts < stable) ++upd_unstable; - } else if (first == NULL && upd->type == WT_UPDATE_BIRTHMARK) - first = prev; - else if (first == NULL && WT_UPDATE_DATA_VALUE(upd)) - first = upd; + } else { + if (first == NULL) { + /* + * If we have seen a globally visible update before the birthmark, the birthmark can + * be discarded. + */ + if (upd_visible_all_seen && upd->type == WT_UPDATE_BIRTHMARK) + first = prev; + /* + * We cannot discard the birthmark if it is the first globally visible update as the + * previous updates can be aborted resulting the entire update chain being removed. + */ + else if (upd->type == WT_UPDATE_BIRTHMARK || WT_UPDATE_DATA_VALUE(upd)) + first = upd; + } + + upd_visible_all_seen = true; + } } __wt_cache_update_lookaside_score(session, upd_seen, upd_unstable); diff --git a/src/third_party/wiredtiger/src/cache/cache_las.c b/src/third_party/wiredtiger/src/cache/cache_las.c index f23e452236b..2cacc57e6fb 100644 --- a/src/third_party/wiredtiger/src/cache/cache_las.c +++ b/src/third_party/wiredtiger/src/cache/cache_las.c @@ -967,11 +967,8 @@ __las_sweep_init(WT_SESSION_IMPL *session) /* * If no files have been dropped and the lookaside file is empty, there's nothing to do. */ - if (cache->las_dropped_next == 0) { - if (__wt_las_empty(session)) - ret = WT_NOTFOUND; - goto err; - } + if (cache->las_dropped_next == 0 && __wt_las_empty(session)) + WT_ERR(WT_NOTFOUND); /* * Record the current page ID: sweep will stop after this point. @@ -1020,7 +1017,6 @@ __wt_las_sweep(WT_SESSION_IMPL *session) WT_DECL_RET; WT_ITEM las_key, las_value; WT_ITEM *sweep_key; - WT_TXN_ISOLATION saved_isolation; wt_timestamp_t durable_timestamp, las_timestamp; uint64_t cnt, remove_cnt, las_pageid, saved_pageid, visit_cnt; uint64_t las_counter, las_txnid; @@ -1032,7 +1028,6 @@ __wt_las_sweep(WT_SESSION_IMPL *session) cache = S2C(session)->cache; cursor = NULL; sweep_key = &cache->las_sweep_key; - saved_isolation = 0; /*[-Wconditional-uninitialized] */ remove_cnt = 0; session_flags = 0; /* [-Werror=maybe-uninitialized] */ local_txn = locked = removing_key_block = false; @@ -1053,7 +1048,6 @@ __wt_las_sweep(WT_SESSION_IMPL *session) __wt_las_cursor(session, &cursor, &session_flags); WT_ASSERT(session, cursor->session == &session->iface); WT_ERR(__wt_txn_begin(session, NULL)); - __las_set_isolation(session, &saved_isolation); local_txn = true; /* Encourage a race */ @@ -1230,7 +1224,6 @@ err: ret = __wt_txn_commit(session, NULL); else WT_TRET(__wt_txn_rollback(session, NULL)); - __las_restore_isolation(session, saved_isolation); if (ret == 0) (void)__wt_atomic_add64(&cache->las_remove_count, remove_cnt); } diff --git a/src/third_party/wiredtiger/src/config/config_collapse.c b/src/third_party/wiredtiger/src/config/config_collapse.c index e1bdc202b2b..aeda22d803e 100644 --- a/src/third_party/wiredtiger/src/config/config_collapse.c +++ b/src/third_party/wiredtiger/src/config/config_collapse.c @@ -71,94 +71,6 @@ err: } /* - * __wt_config_discard_defaults -- - * Copy non-default configuration strings into newly allocated memory. This function strips out - * entries from a configuration string that aren't the default values. It takes a - * NULL-terminated list of configuration strings, the defaults, and a configuration string, and - * copies into allocated memory the strings from the configuration string that aren't the same - * as the defaults. - */ -int -__wt_config_discard_defaults( - WT_SESSION_IMPL *session, const char **cfg, const char *config, char **config_ret) -{ - WT_CONFIG cparser; - WT_CONFIG_ITEM k, v, vtmp; - WT_DECL_ITEM(tmp); - WT_DECL_RET; - - *config_ret = NULL; - - WT_RET(__wt_scr_alloc(session, 0, &tmp)); - - /* - * Walk the configuration string, search the default configuration for each entry, and discard - * any matches. - */ - __wt_config_init(session, &cparser, config); - while ((ret = __wt_config_next(&cparser, &k, &v)) == 0) { - if (k.type != WT_CONFIG_ITEM_STRING && k.type != WT_CONFIG_ITEM_ID) - WT_ERR_MSG(session, EINVAL, "Invalid configuration key found: '%s'", k.str); - - /* - * Get the default value. There isn't a default value in some cases, so not finding one - * isn't an error. - */ - if ((ret = __wt_config_get(session, cfg, &k, &vtmp)) == WT_NOTFOUND) - goto keep; - WT_ERR(ret); - - /* - * Skip exact matches and simple things we can catch like "none" and an empty string, "true" - * and 1, "false" and 0. - */ - if (v.type == WT_CONFIG_ITEM_STRUCT && vtmp.type == WT_CONFIG_ITEM_STRUCT && - v.len == vtmp.len && memcmp(v.str, vtmp.str, v.len) == 0) - continue; - if ((v.type == WT_CONFIG_ITEM_BOOL || v.type == WT_CONFIG_ITEM_NUM) && - (vtmp.type == WT_CONFIG_ITEM_BOOL || vtmp.type == WT_CONFIG_ITEM_NUM) && - v.val == vtmp.val) - continue; - if ((v.type == WT_CONFIG_ITEM_ID || v.type == WT_CONFIG_ITEM_STRING) && - (vtmp.type == WT_CONFIG_ITEM_ID || vtmp.type == WT_CONFIG_ITEM_STRING) && - v.len == vtmp.len && memcmp(v.str, vtmp.str, v.len) == 0) - continue; - if (vtmp.len == 0 && v.len == strlen("none") && WT_STRING_MATCH("none", v.str, v.len)) - continue; - - /* Include the quotes around string keys/values. */ -keep: - if (k.type == WT_CONFIG_ITEM_STRING) { - --k.str; - k.len += 2; - } - if (v.type == WT_CONFIG_ITEM_STRING) { - --v.str; - v.len += 2; - } - WT_ERR(__wt_buf_catfmt(session, tmp, "%.*s=%.*s,", (int)k.len, k.str, (int)v.len, v.str)); - } - - /* We loop until error, and the expected error is WT_NOTFOUND. */ - if (ret != WT_NOTFOUND) - goto err; - - /* - * If the caller passes us only default configuration strings, we get here with no bytes to copy - * -- that's OK, the underlying string copy can handle empty strings. - * - * Strip any trailing comma. - */ - if (tmp->size != 0) - --tmp->size; - ret = __wt_strndup(session, tmp->data, tmp->size, config_ret); - -err: - __wt_scr_free(session, &tmp); - return (ret); -} - -/* * We need a character that can't appear in a key as a separator. */ #undef SEP /* separator key, character */ diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index 56b3febfeb1..462e096263e 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -1441,6 +1441,16 @@ __conn_config_file( err: WT_TRET(__wt_close(session, &fh)); + + /** + * Encountering an invalid configuration string from the base configuration file suggests + * that there is corruption present in the file. + */ + if (!is_user && ret == EINVAL) { + F_SET(S2C(session), WT_CONN_DATA_CORRUPTION); + return (WT_ERROR); + } + return (ret); } @@ -1641,6 +1651,20 @@ __conn_single(WT_SESSION_IMPL *session, const char *cfg[]) bytelock = false; ret = 0; } + + /** + * The WiredTiger lock file will not be created if the WiredTiger file does not exist in the + * directory, suggesting possible corruption if the WiredTiger file was deleted. Suggest running + * salvage. + */ + if (ret == ENOENT) { + WT_ERR(__wt_fs_exist(session, WT_WIREDTIGER, &exist)); + if (!exist) { + F_SET(conn, WT_CONN_DATA_CORRUPTION); + WT_ERR(WT_ERROR); + } + } + WT_ERR(ret); if (bytelock) { /* @@ -1683,6 +1707,10 @@ __conn_single(WT_SESSION_IMPL *session, const char *cfg[]) ret = 0; WT_ERR(ret); } else { + if (ret == ENOENT) { + F_SET(conn, WT_CONN_DATA_CORRUPTION); + WT_ERR(WT_ERROR); + } WT_ERR(ret); /* * Lock the WiredTiger file (for backward compatibility reasons as described above). diff --git a/src/third_party/wiredtiger/src/conn/conn_dhandle.c b/src/third_party/wiredtiger/src/conn/conn_dhandle.c index 7c60e0d8239..c947954e75c 100644 --- a/src/third_party/wiredtiger/src/conn/conn_dhandle.c +++ b/src/third_party/wiredtiger/src/conn/conn_dhandle.c @@ -25,6 +25,7 @@ __conn_dhandle_config_clear(WT_SESSION_IMPL *session) for (a = dhandle->cfg; *a != NULL; ++a) __wt_free(session, *a); __wt_free(session, dhandle->cfg); + __wt_free(session, dhandle->meta_base); } /* @@ -36,9 +37,12 @@ __conn_dhandle_config_set(WT_SESSION_IMPL *session) { WT_DATA_HANDLE *dhandle; WT_DECL_RET; - char *metaconf; + char *metaconf, *tmp; + const char *base, *cfg[3]; dhandle = session->dhandle; + base = NULL; + tmp = NULL; /* * Read the object's entry from the metadata file, we're done if we don't find one. @@ -63,17 +67,45 @@ __conn_dhandle_config_set(WT_SESSION_IMPL *session) WT_ERR(__wt_calloc_def(session, 3, &dhandle->cfg)); switch (dhandle->type) { case WT_DHANDLE_TYPE_BTREE: + /* + * We are stripping out the checkpoint and checkpoint_lsn information from the config + * string. We save the rest of the metadata string, that is essentially static and + * unchanging and then concatenate the new checkpoint and LSN information on each + * checkpoint. The reason is performance and avoiding a lot of calls to the config parsing + * functions during a checkpoint for information that changes in a very well known way. + */ + cfg[0] = metaconf; + cfg[1] = "checkpoint=()"; + cfg[2] = NULL; WT_ERR(__wt_strdup(session, WT_CONFIG_BASE(session, file_meta), &dhandle->cfg[0])); + WT_ASSERT(session, dhandle->meta_base == NULL); + /* + * First collapse and overwrite any checkpoint information because we do not know the name + * or how many checkpoints may be in this metadata. So first we have to set the string to + * the empty checkpoint string and call collapse to overwrite anything existing. + */ + WT_ERR(__wt_config_collapse(session, cfg, &tmp)); + /* + * Now strip out the checkpoint and checkpoint LSN items from the configuration string and + * that is now our base metadata string. + */ + cfg[0] = tmp; + cfg[1] = NULL; + WT_ERR(__wt_config_merge(session, cfg, "checkpoint=,checkpoint_lsn=", &base)); + __wt_free(session, tmp); break; case WT_DHANDLE_TYPE_TABLE: WT_ERR(__wt_strdup(session, WT_CONFIG_BASE(session, table_meta), &dhandle->cfg[0])); break; } dhandle->cfg[1] = metaconf; + dhandle->meta_base = base; return (0); err: + __wt_free(session, base); __wt_free(session, metaconf); + __wt_free(session, tmp); return (ret); } @@ -462,6 +494,11 @@ err: if (dhandle->type == WT_DHANDLE_TYPE_BTREE) __wt_evict_file_exclusive_off(session); + if (ret == ENOENT && F_ISSET(dhandle, WT_DHANDLE_IS_METADATA)) { + F_SET(S2C(session), WT_CONN_DATA_CORRUPTION); + return (WT_ERROR); + } + return (ret); } diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 8548ad810ba..7f729c2e661 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -1649,7 +1649,8 @@ __evict_walk_tree(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, u_int max_ent WT_EVICT_ENTRY *end, *evict, *start; WT_PAGE *last_parent, *page; WT_REF *ref; - uint64_t min_pages, pages_seen, pages_queued, refs_walked; + uint64_t internal_pages_already_queued, internal_pages_queued, internal_pages_seen; + uint64_t min_pages, pages_already_queued, pages_seen, pages_queued, refs_walked; uint32_t read_flags, remaining_slots, target_pages, walk_flags; int restarts; bool give_up, modified, urgent_queued; @@ -1710,8 +1711,13 @@ __evict_walk_tree(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, u_int max_ent * only looking for dirty pages, search the tree for longer. */ min_pages = 10 * (uint64_t)target_pages; - if (F_ISSET(cache, WT_CACHE_EVICT_DIRTY) && !F_ISSET(cache, WT_CACHE_EVICT_CLEAN)) + if (!F_ISSET(cache, WT_CACHE_EVICT_DIRTY) && F_ISSET(cache, WT_CACHE_EVICT_CLEAN)) + WT_STAT_CONN_INCR(session, cache_eviction_target_strategy_clean); + else if (F_ISSET(cache, WT_CACHE_EVICT_DIRTY) && !F_ISSET(cache, WT_CACHE_EVICT_CLEAN)) { min_pages *= 10; + WT_STAT_CONN_INCR(session, cache_eviction_target_strategy_dirty); + } else + WT_STAT_CONN_INCR(session, cache_eviction_target_strategy_both_clean_and_dirty); if (btree->evict_ref == NULL) { WT_STAT_CONN_INCR(session, cache_eviction_walk_from_root); @@ -1767,7 +1773,8 @@ __evict_walk_tree(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, u_int max_ent * Once we hit the page limit, do one more step through the walk in * case we are appending and only the last page in the file is live. */ - for (evict = start, pages_queued = pages_seen = refs_walked = 0; + internal_pages_already_queued = internal_pages_queued = internal_pages_seen = 0; + for (evict = start, pages_already_queued = pages_queued = pages_seen = refs_walked = 0; evict < end && (ret == 0 || ret == WT_NOTFOUND); last_parent = ref == NULL ? NULL : ref->home, ret = __wt_tree_walk_count(session, &ref, &refs_walked, walk_flags)) { @@ -1835,11 +1842,19 @@ __evict_walk_tree(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, u_int max_ent modified = __wt_page_is_modified(page); page->evict_pass_gen = cache->evict_pass_gen; + /* count internal pages seen. */ + if (WT_PAGE_IS_INTERNAL(page)) + internal_pages_seen++; + /* * Use the EVICT_LRU flag to avoid putting pages onto the list multiple times. */ - if (F_ISSET_ATOMIC(page, WT_PAGE_EVICT_LRU)) + if (F_ISSET_ATOMIC(page, WT_PAGE_EVICT_LRU)) { + pages_already_queued++; + if (WT_PAGE_IS_INTERNAL(page)) + internal_pages_already_queued++; continue; + } /* Don't queue dirty pages in trees during checkpoints. */ if (modified && WT_BTREE_SYNCING(btree)) @@ -1936,6 +1951,10 @@ fast: ++pages_queued; ++btree->evict_walk_progress; + /* count internal pages queued. */ + if (WT_PAGE_IS_INTERNAL(page)) + internal_pages_queued++; + __wt_verbose(session, WT_VERB_EVICTSERVER, "select: %p, size %" WT_SIZET_FMT, (void *)page, page->memory_footprint); } @@ -1987,6 +2006,11 @@ fast: WT_STAT_CONN_INCRV(session, cache_eviction_walk, refs_walked); WT_STAT_CONN_INCRV(session, cache_eviction_pages_seen, pages_seen); WT_STAT_DATA_INCRV(session, cache_eviction_pages_seen, pages_seen); + WT_STAT_CONN_INCRV(session, cache_eviction_pages_already_queued, pages_already_queued); + WT_STAT_CONN_INCRV(session, cache_eviction_internal_pages_seen, internal_pages_seen); + WT_STAT_CONN_INCRV( + session, cache_eviction_internal_pages_already_queued, internal_pages_already_queued); + WT_STAT_CONN_INCRV(session, cache_eviction_internal_pages_queued, internal_pages_queued); WT_STAT_CONN_INCRV(session, cache_eviction_walk_passes, 1); WT_STAT_DATA_INCRV(session, cache_eviction_walk_passes, 1); diff --git a/src/third_party/wiredtiger/src/evict/evict_page.c b/src/third_party/wiredtiger/src/evict/evict_page.c index a13526302a2..aa0c248390d 100644 --- a/src/third_party/wiredtiger/src/evict/evict_page.c +++ b/src/third_party/wiredtiger/src/evict/evict_page.c @@ -549,6 +549,8 @@ __evict_review(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t evict_flags, bool */ if (WT_PAGE_IS_INTERNAL(page)) { WT_WITH_PAGE_INDEX(session, ret = __evict_child_check(session, ref)); + if (ret != 0) + WT_STAT_CONN_INCR(session, cache_eviction_fail_active_children_on_an_internal_page); WT_RET(ret); } @@ -683,6 +685,9 @@ __evict_review(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t evict_flags, bool ret = __wt_reconcile(session, ref, NULL, flags, NULL); } + if (ret != 0) + WT_STAT_CONN_INCR(session, cache_eviction_fail_in_reconciliation); + WT_RET(ret); /* diff --git a/src/third_party/wiredtiger/src/include/btree.i b/src/third_party/wiredtiger/src/include/btree.i index 550ec45fbf0..18199997a19 100644 --- a/src/third_party/wiredtiger/src/include/btree.i +++ b/src/third_party/wiredtiger/src/include/btree.i @@ -1351,8 +1351,10 @@ __wt_page_can_evict(WT_SESSION_IMPL *session, WT_REF *ref, bool *inmem_splitp) * overflow item, because the split into the parent frees the backing blocks for any * no-longer-used overflow keys, which will corrupt the checkpoint's block management. */ - if (!__wt_btree_can_evict_dirty(session) && F_ISSET_ATOMIC(ref->home, WT_PAGE_OVERFLOW_KEYS)) + if (!__wt_btree_can_evict_dirty(session) && F_ISSET_ATOMIC(ref->home, WT_PAGE_OVERFLOW_KEYS)) { + WT_STAT_CONN_INCR(session, cache_eviction_fail_parent_has_overflow_items); return (false); + } /* * Check for in-memory splits before other eviction tests. If the page should split in-memory, @@ -1394,8 +1396,10 @@ __wt_page_can_evict(WT_SESSION_IMPL *session, WT_REF *ref, bool *inmem_splitp) /* * If the page is clean but has modifications that appear too new to evict, skip it. */ - if (!modified && !__wt_txn_visible_all(session, mod->rec_max_txn, mod->rec_max_timestamp)) + if (!modified && !__wt_txn_visible_all(session, mod->rec_max_txn, mod->rec_max_timestamp)) { + WT_STAT_CONN_INCR(session, cache_eviction_fail_with_newer_modifications_on_a_clean_page); return (false); + } return (true); } diff --git a/src/third_party/wiredtiger/src/include/dhandle.h b/src/third_party/wiredtiger/src/include/dhandle.h index 76bc3987024..01674140065 100644 --- a/src/third_party/wiredtiger/src/include/dhandle.h +++ b/src/third_party/wiredtiger/src/include/dhandle.h @@ -69,6 +69,7 @@ struct __wt_data_handle { uint64_t name_hash; /* Hash of name */ const char *checkpoint; /* Checkpoint name (or NULL) */ const char **cfg; /* Configuration information */ + const char *meta_base; /* Base metadata configuration */ /* * Sessions holding a connection's data handle will have a non-zero reference count; sessions diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index a41c3b62e9a..d42e0d43d9d 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -34,6 +34,8 @@ extern bool __wt_las_page_skip_locked(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern bool __wt_lsm_chunk_visible_all(WT_SESSION_IMPL *session, WT_LSM_CHUNK *chunk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern bool __wt_modify_idempotent(const void *modify) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern bool __wt_page_evict_urgent(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern bool __wt_rwlock_islocked(WT_SESSION_IMPL *session, WT_RWLOCK *l) @@ -383,8 +385,6 @@ extern int __wt_config_check(WT_SESSION_IMPL *session, const WT_CONFIG_ENTRY *en const char *config, size_t config_len) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_config_collapse(WT_SESSION_IMPL *session, const char **cfg, char **config_ret) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_config_discard_defaults(WT_SESSION_IMPL *session, const char **cfg, - const char *config, char **config_ret) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_config_get(WT_SESSION_IMPL *session, const char **cfg_arg, WT_CONFIG_ITEM *key, WT_CONFIG_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_config_getone(WT_SESSION_IMPL *session, const char *config, WT_CONFIG_ITEM *key, @@ -574,6 +574,8 @@ extern int __wt_debug_addr(WT_SESSION_IMPL *session, const uint8_t *addr, size_t const char *ofile) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_debug_addr_print(WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_debug_cursor_las(void *cursor_arg, const char *ofile) WT_GCC_FUNC_DECL_ATTRIBUTE( + (visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_debug_cursor_page(void *cursor_arg, const char *ofile) WT_GCC_FUNC_DECL_ATTRIBUTE( (visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_debug_disk(WT_SESSION_IMPL *session, const WT_PAGE_HEADER *dsk, const char *ofile) @@ -1161,10 +1163,10 @@ extern int __wt_rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE * WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_rec_row_leaf(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_REF *pageref, WT_SALVAGE_COOKIE *salvage) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_rec_split(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len) - WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_rec_split_crossing_bnd(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len) +extern int __wt_rec_split(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len, bool forced) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_rec_split_crossing_bnd(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len, + bool forced) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_rec_split_finish(WT_SESSION_IMPL *session, WT_RECONCILE *r) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_rec_split_init(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page, @@ -1968,8 +1970,6 @@ static inline int __wt_txn_op_set_key(WT_SESSION_IMPL *session, const WT_ITEM *k WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_txn_read(WT_SESSION_IMPL *session, WT_UPDATE *upd, WT_UPDATE **updp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -static inline int __wt_txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit, - int64_t *resolved_update_countp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_txn_search_check(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_txn_update_check(WT_SESSION_IMPL *session, WT_UPDATE *upd) diff --git a/src/third_party/wiredtiger/src/include/reconcile.h b/src/third_party/wiredtiger/src/include/reconcile.h index 1c02f8353c6..c9277f0736a 100644 --- a/src/third_party/wiredtiger/src/include/reconcile.h +++ b/src/third_party/wiredtiger/src/include/reconcile.h @@ -141,18 +141,19 @@ struct __wt_reconcile { WT_ITEM image; /* disk-image */ } chunkA, chunkB, *cur_ptr, *prev_ptr; + size_t disk_img_buf_size; /* Base size needed for a chunk memory image */ + /* * We track current information about the current record number, the number of entries copied * into the disk image buffer, where we are in the buffer, how much memory remains, and the * current min/max of the timestamps. Those values are packaged here rather than passing * pointers to stack locations around the code. */ - uint64_t recno; /* Current record number */ - uint32_t entries; /* Current number of entries */ - uint8_t *first_free; /* Current first free byte */ - size_t space_avail; /* Remaining space in this chunk */ - /* Remaining space in this chunk to put a minimum size boundary */ - size_t min_space_avail; + uint64_t recno; /* Current record number */ + uint32_t entries; /* Current number of entries */ + uint8_t *first_free; /* Current first free byte */ + size_t space_avail; /* Remaining space in this chunk */ + size_t min_space_avail; /* Remaining space in this chunk to put a minimum size boundary */ /* * Saved update list, supporting the WT_REC_UPDATE_RESTORE and WT_REC_LOOKASIDE configurations. diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index 14665c4df75..85361a819fd 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -358,14 +358,15 @@ struct __wt_connection_stats { int64_t cache_eviction_server_slept; int64_t cache_eviction_slow; int64_t cache_eviction_walk_leaf_notfound; - int64_t cache_eviction_walk_internal_wait; - int64_t cache_eviction_walk_internal_yield; int64_t cache_eviction_state; int64_t cache_eviction_target_page_lt10; int64_t cache_eviction_target_page_lt32; int64_t cache_eviction_target_page_ge128; int64_t cache_eviction_target_page_lt64; int64_t cache_eviction_target_page_lt128; + int64_t cache_eviction_target_strategy_both_clean_and_dirty; + int64_t cache_eviction_target_strategy_clean; + int64_t cache_eviction_target_strategy_dirty; int64_t cache_eviction_walks_abandoned; int64_t cache_eviction_walks_stopped; int64_t cache_eviction_walks_gave_up_no_targets; @@ -396,6 +397,9 @@ struct __wt_connection_stats { int64_t cache_inmem_splittable; int64_t cache_inmem_split; int64_t cache_eviction_internal; + int64_t cache_eviction_internal_pages_queued; + int64_t cache_eviction_internal_pages_seen; + int64_t cache_eviction_internal_pages_already_queued; int64_t cache_eviction_split_internal; int64_t cache_eviction_split_leaf; int64_t cache_bytes_max; @@ -422,7 +426,12 @@ struct __wt_connection_stats { int64_t cache_read_lookaside_delay_checkpoint; int64_t cache_pages_requested; int64_t cache_eviction_pages_seen; + int64_t cache_eviction_pages_already_queued; int64_t cache_eviction_fail; + int64_t cache_eviction_fail_parent_has_overflow_items; + int64_t cache_eviction_fail_active_children_on_an_internal_page; + int64_t cache_eviction_fail_in_reconciliation; + int64_t cache_eviction_fail_with_newer_modifications_on_a_clean_page; int64_t cache_eviction_walk; int64_t cache_write; int64_t cache_write_restore; @@ -642,7 +651,6 @@ struct __wt_connection_stats { int64_t child_modify_blocked_page; int64_t txn_prepared_updates_count; int64_t txn_prepared_updates_lookaside_inserts; - int64_t txn_prepared_updates_resolved; int64_t txn_durable_queue_walked; int64_t txn_durable_queue_empty; int64_t txn_durable_queue_head; diff --git a/src/third_party/wiredtiger/src/include/txn.h b/src/third_party/wiredtiger/src/include/txn.h index 7636cf42dd9..bdda7a4eae9 100644 --- a/src/third_party/wiredtiger/src/include/txn.h +++ b/src/third_party/wiredtiger/src/include/txn.h @@ -244,7 +244,6 @@ struct __wt_txn_op { /* AUTOMATIC FLAG VALUE GENERATION START */ #define WT_TXN_OP_KEY_REPEATED 0x1u -#define WT_TXN_OP_KEY_RESERVED 0x2u /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; }; diff --git a/src/third_party/wiredtiger/src/include/txn.i b/src/third_party/wiredtiger/src/include/txn.i index 3bef4c8b905..757627dff97 100644 --- a/src/third_party/wiredtiger/src/include/txn.i +++ b/src/third_party/wiredtiger/src/include/txn.i @@ -150,108 +150,6 @@ __txn_resolve_prepared_update(WT_SESSION_IMPL *session, WT_UPDATE *upd) } /* - * __wt_txn_resolve_prepared_op -- - * Resolve a transaction's operations indirect references. In case of prepared transactions, the - * prepared updates could be evicted using cache overflow mechanism. Transaction operations - * referring to these prepared updates would be referring to them using indirect references (i.e - * keys/recnos), which need to be resolved as part of that transaction commit/rollback. If no - * updates are resolved throw an error. Increment resolved update count for each resolved update - * count we locate. - */ -static inline int -__wt_txn_resolve_prepared_op( - WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit, int64_t *resolved_update_countp) -{ - WT_CURSOR *cursor; - WT_DECL_RET; - WT_TXN *txn; - WT_UPDATE *upd; - const char *open_cursor_cfg[] = {WT_CONFIG_BASE(session, WT_SESSION_open_cursor), NULL}; - - txn = &session->txn; - - if (op->type == WT_TXN_OP_NONE || op->type == WT_TXN_OP_REF_DELETE || - op->type == WT_TXN_OP_TRUNCATE_COL || op->type == WT_TXN_OP_TRUNCATE_ROW) - return (0); - - WT_RET(__wt_open_cursor(session, op->btree->dhandle->name, NULL, open_cursor_cfg, &cursor)); - - /* - * Transaction prepare is cleared temporarily as cursor functions are not allowed for prepared - * transactions. - */ - F_CLR(txn, WT_TXN_PREPARE); - if (op->type == WT_TXN_OP_BASIC_ROW || op->type == WT_TXN_OP_INMEM_ROW) - __wt_cursor_set_raw_key(cursor, &op->u.op_row.key); - else - ((WT_CURSOR_BTREE *)cursor)->iface.recno = op->u.op_col.recno; - F_SET(txn, WT_TXN_PREPARE); - - WT_WITH_BTREE( - session, op->btree, ret = __wt_btcur_search_uncommitted((WT_CURSOR_BTREE *)cursor, &upd)); - WT_ERR(ret); - - /* If we haven't found anything then there's an error. */ - if (upd == NULL) { - WT_ERR_ASSERT(session, upd != NULL, WT_NOTFOUND, - "Unable to locate update associated with a prepared operation."); - } - - for (; upd != NULL; upd = upd->next) { - /* - * Aborted updates can exist in the update chain of our txn. Generally this will occur due - * to a reserved update. As such we should skip over these updates. If the txn id is then - * different and not aborted we know we've reached the end of our update chain and can exit. - */ - if (upd->txnid == WT_TXN_ABORTED) - continue; - if (upd->txnid != txn->id) - break; - - ++(*resolved_update_countp); - - if (!commit) { - upd->txnid = WT_TXN_ABORTED; - continue; - } - - /* - * Newer updates are inserted at head of update chain, and - * transaction operations are added at the tail of the - * transaction modify chain. - * - * For example, a transaction has modified [k,v] as - * [k, v] -> [k, u1] (txn_op : txn_op1) - * [k, u1] -> [k, u2] (txn_op : txn_op2) - * update chain : u2->u1 - * txn_mod : txn_op1->txn_op2. - * - * Only the key is saved in the transaction operation - * structure, hence we cannot identify whether "txn_op1" - * corresponds to "u2" or "u1" during commit/rollback. - * - * To make things simpler we will handle all the updates - * that match the key saved in a transaction operation in a - * single go. As a result, multiple updates of a key, if any - * will be resolved as part of the first transaction operation - * resolution of that key, and subsequent transaction operation - * resolution of the same key will be effectively - * a no-op. - * - * In the above example, we will resolve "u2" and "u1" as part - * of resolving "txn_op1" and will not do any significant - * thing as part of "txn_op2". - */ - - /* Resolve the prepared update to be committed update. */ - __txn_resolve_prepared_update(session, upd); - } -err: - WT_TRET(cursor->close(cursor)); - return (ret); -} - -/* * __txn_next_op -- * Mark a WT_UPDATE object modified by the current transaction. */ diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index df0f582f58b..aec3014e788 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -4984,739 +4984,767 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_CACHE_EVICTION_SLOW 1061 /*! cache: eviction server waiting for a leaf page */ #define WT_STAT_CONN_CACHE_EVICTION_WALK_LEAF_NOTFOUND 1062 -/*! cache: eviction server waiting for an internal page sleep (usec) */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK_INTERNAL_WAIT 1063 -/*! cache: eviction server waiting for an internal page yields */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK_INTERNAL_YIELD 1064 /*! cache: eviction state */ -#define WT_STAT_CONN_CACHE_EVICTION_STATE 1065 +#define WT_STAT_CONN_CACHE_EVICTION_STATE 1063 /*! cache: eviction walk target pages histogram - 0-9 */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT10 1066 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT10 1064 /*! cache: eviction walk target pages histogram - 10-31 */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT32 1067 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT32 1065 /*! cache: eviction walk target pages histogram - 128 and higher */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_GE128 1068 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_GE128 1066 /*! cache: eviction walk target pages histogram - 32-63 */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT64 1069 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT64 1067 /*! cache: eviction walk target pages histogram - 64-128 */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT128 1070 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT128 1068 +/*! cache: eviction walk target strategy both clean and dirty pages */ +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_STRATEGY_BOTH_CLEAN_AND_DIRTY 1069 +/*! cache: eviction walk target strategy only clean pages */ +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_STRATEGY_CLEAN 1070 +/*! cache: eviction walk target strategy only dirty pages */ +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_STRATEGY_DIRTY 1071 /*! cache: eviction walks abandoned */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ABANDONED 1071 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ABANDONED 1072 /*! cache: eviction walks gave up because they restarted their walk twice */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_STOPPED 1072 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_STOPPED 1073 /*! * cache: eviction walks gave up because they saw too many pages and * found no candidates */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_GAVE_UP_NO_TARGETS 1073 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_GAVE_UP_NO_TARGETS 1074 /*! * cache: eviction walks gave up because they saw too many pages and * found too few candidates */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_GAVE_UP_RATIO 1074 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_GAVE_UP_RATIO 1075 /*! cache: eviction walks reached end of tree */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ENDED 1075 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ENDED 1076 /*! cache: eviction walks started from root of tree */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK_FROM_ROOT 1076 +#define WT_STAT_CONN_CACHE_EVICTION_WALK_FROM_ROOT 1077 /*! cache: eviction walks started from saved location in tree */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK_SAVED_POS 1077 +#define WT_STAT_CONN_CACHE_EVICTION_WALK_SAVED_POS 1078 /*! cache: eviction worker thread active */ -#define WT_STAT_CONN_CACHE_EVICTION_ACTIVE_WORKERS 1078 +#define WT_STAT_CONN_CACHE_EVICTION_ACTIVE_WORKERS 1079 /*! cache: eviction worker thread created */ -#define WT_STAT_CONN_CACHE_EVICTION_WORKER_CREATED 1079 +#define WT_STAT_CONN_CACHE_EVICTION_WORKER_CREATED 1080 /*! cache: eviction worker thread evicting pages */ -#define WT_STAT_CONN_CACHE_EVICTION_WORKER_EVICTING 1080 +#define WT_STAT_CONN_CACHE_EVICTION_WORKER_EVICTING 1081 /*! cache: eviction worker thread removed */ -#define WT_STAT_CONN_CACHE_EVICTION_WORKER_REMOVED 1081 +#define WT_STAT_CONN_CACHE_EVICTION_WORKER_REMOVED 1082 /*! cache: eviction worker thread stable number */ -#define WT_STAT_CONN_CACHE_EVICTION_STABLE_STATE_WORKERS 1082 +#define WT_STAT_CONN_CACHE_EVICTION_STABLE_STATE_WORKERS 1083 /*! cache: files with active eviction walks */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ACTIVE 1083 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ACTIVE 1084 /*! cache: files with new eviction walks started */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_STARTED 1084 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_STARTED 1085 /*! cache: force re-tuning of eviction workers once in a while */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_RETUNE 1085 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_RETUNE 1086 /*! cache: forced eviction - pages evicted that were clean count */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_CLEAN 1086 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_CLEAN 1087 /*! cache: forced eviction - pages evicted that were clean time (usecs) */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_CLEAN_TIME 1087 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_CLEAN_TIME 1088 /*! cache: forced eviction - pages evicted that were dirty count */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DIRTY 1088 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DIRTY 1089 /*! cache: forced eviction - pages evicted that were dirty time (usecs) */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DIRTY_TIME 1089 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DIRTY_TIME 1090 /*! * cache: forced eviction - pages selected because of too many deleted * items count */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE 1090 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE 1091 /*! cache: forced eviction - pages selected count */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE 1091 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE 1092 /*! cache: forced eviction - pages selected unable to be evicted count */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL 1092 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL 1093 /*! cache: forced eviction - pages selected unable to be evicted time */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL_TIME 1093 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL_TIME 1094 /*! cache: hazard pointer blocked page eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_HAZARD 1094 +#define WT_STAT_CONN_CACHE_EVICTION_HAZARD 1095 /*! cache: hazard pointer check calls */ -#define WT_STAT_CONN_CACHE_HAZARD_CHECKS 1095 +#define WT_STAT_CONN_CACHE_HAZARD_CHECKS 1096 /*! cache: hazard pointer check entries walked */ -#define WT_STAT_CONN_CACHE_HAZARD_WALKS 1096 +#define WT_STAT_CONN_CACHE_HAZARD_WALKS 1097 /*! cache: hazard pointer maximum array length */ -#define WT_STAT_CONN_CACHE_HAZARD_MAX 1097 +#define WT_STAT_CONN_CACHE_HAZARD_MAX 1098 /*! cache: in-memory page passed criteria to be split */ -#define WT_STAT_CONN_CACHE_INMEM_SPLITTABLE 1098 +#define WT_STAT_CONN_CACHE_INMEM_SPLITTABLE 1099 /*! cache: in-memory page splits */ -#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1099 +#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1100 /*! cache: internal pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1100 +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1101 +/*! cache: internal pages queued for eviction */ +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL_PAGES_QUEUED 1102 +/*! cache: internal pages seen by eviction walk */ +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL_PAGES_SEEN 1103 +/*! cache: internal pages seen by eviction walk that are already queued */ +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL_PAGES_ALREADY_QUEUED 1104 /*! cache: internal pages split during eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_INTERNAL 1101 +#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_INTERNAL 1105 /*! cache: leaf pages split during eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_LEAF 1102 +#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_LEAF 1106 /*! cache: maximum bytes configured */ -#define WT_STAT_CONN_CACHE_BYTES_MAX 1103 +#define WT_STAT_CONN_CACHE_BYTES_MAX 1107 /*! cache: maximum page size at eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_MAXIMUM_PAGE_SIZE 1104 +#define WT_STAT_CONN_CACHE_EVICTION_MAXIMUM_PAGE_SIZE 1108 /*! cache: modified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1105 +#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1109 /*! cache: modified pages evicted by application threads */ -#define WT_STAT_CONN_CACHE_EVICTION_APP_DIRTY 1106 +#define WT_STAT_CONN_CACHE_EVICTION_APP_DIRTY 1110 /*! cache: operations timed out waiting for space in cache */ -#define WT_STAT_CONN_CACHE_TIMED_OUT_OPS 1107 +#define WT_STAT_CONN_CACHE_TIMED_OUT_OPS 1111 /*! cache: overflow pages read into cache */ -#define WT_STAT_CONN_CACHE_READ_OVERFLOW 1108 +#define WT_STAT_CONN_CACHE_READ_OVERFLOW 1112 /*! cache: page split during eviction deepened the tree */ -#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1109 +#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1113 /*! cache: page written requiring cache overflow records */ -#define WT_STAT_CONN_CACHE_WRITE_LOOKASIDE 1110 +#define WT_STAT_CONN_CACHE_WRITE_LOOKASIDE 1114 /*! cache: pages currently held in the cache */ -#define WT_STAT_CONN_CACHE_PAGES_INUSE 1111 +#define WT_STAT_CONN_CACHE_PAGES_INUSE 1115 /*! cache: pages evicted by application threads */ -#define WT_STAT_CONN_CACHE_EVICTION_APP 1112 +#define WT_STAT_CONN_CACHE_EVICTION_APP 1116 /*! cache: pages queued for eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED 1113 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED 1117 /*! cache: pages queued for eviction post lru sorting */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_POST_LRU 1114 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_POST_LRU 1118 /*! cache: pages queued for urgent eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_URGENT 1115 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_URGENT 1119 /*! cache: pages queued for urgent eviction during walk */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_OLDEST 1116 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_OLDEST 1120 /*! cache: pages read into cache */ -#define WT_STAT_CONN_CACHE_READ 1117 +#define WT_STAT_CONN_CACHE_READ 1121 /*! cache: pages read into cache after truncate */ -#define WT_STAT_CONN_CACHE_READ_DELETED 1118 +#define WT_STAT_CONN_CACHE_READ_DELETED 1122 /*! cache: pages read into cache after truncate in prepare state */ -#define WT_STAT_CONN_CACHE_READ_DELETED_PREPARED 1119 +#define WT_STAT_CONN_CACHE_READ_DELETED_PREPARED 1123 /*! cache: pages read into cache requiring cache overflow entries */ -#define WT_STAT_CONN_CACHE_READ_LOOKASIDE 1120 +#define WT_STAT_CONN_CACHE_READ_LOOKASIDE 1124 /*! cache: pages read into cache requiring cache overflow for checkpoint */ -#define WT_STAT_CONN_CACHE_READ_LOOKASIDE_CHECKPOINT 1121 +#define WT_STAT_CONN_CACHE_READ_LOOKASIDE_CHECKPOINT 1125 /*! cache: pages read into cache skipping older cache overflow entries */ -#define WT_STAT_CONN_CACHE_READ_LOOKASIDE_SKIPPED 1122 +#define WT_STAT_CONN_CACHE_READ_LOOKASIDE_SKIPPED 1126 /*! * cache: pages read into cache with skipped cache overflow entries * needed later */ -#define WT_STAT_CONN_CACHE_READ_LOOKASIDE_DELAY 1123 +#define WT_STAT_CONN_CACHE_READ_LOOKASIDE_DELAY 1127 /*! * cache: pages read into cache with skipped cache overflow entries * needed later by checkpoint */ -#define WT_STAT_CONN_CACHE_READ_LOOKASIDE_DELAY_CHECKPOINT 1124 +#define WT_STAT_CONN_CACHE_READ_LOOKASIDE_DELAY_CHECKPOINT 1128 /*! cache: pages requested from the cache */ -#define WT_STAT_CONN_CACHE_PAGES_REQUESTED 1125 +#define WT_STAT_CONN_CACHE_PAGES_REQUESTED 1129 /*! cache: pages seen by eviction walk */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_SEEN 1126 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_SEEN 1130 +/*! cache: pages seen by eviction walk that are already queued */ +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_ALREADY_QUEUED 1131 /*! cache: pages selected for eviction unable to be evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL 1127 +#define WT_STAT_CONN_CACHE_EVICTION_FAIL 1132 +/*! + * cache: pages selected for eviction unable to be evicted as the parent + * page has overflow items + */ +#define WT_STAT_CONN_CACHE_EVICTION_FAIL_PARENT_HAS_OVERFLOW_ITEMS 1133 +/*! + * cache: pages selected for eviction unable to be evicted because of + * active children on an internal page + */ +#define WT_STAT_CONN_CACHE_EVICTION_FAIL_ACTIVE_CHILDREN_ON_AN_INTERNAL_PAGE 1134 +/*! + * cache: pages selected for eviction unable to be evicted because of + * failure in reconciliation + */ +#define WT_STAT_CONN_CACHE_EVICTION_FAIL_IN_RECONCILIATION 1135 +/*! + * cache: pages selected for eviction unable to be evicted due to newer + * modifications on a clean page + */ +#define WT_STAT_CONN_CACHE_EVICTION_FAIL_WITH_NEWER_MODIFICATIONS_ON_A_CLEAN_PAGE 1136 /*! cache: pages walked for eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK 1128 +#define WT_STAT_CONN_CACHE_EVICTION_WALK 1137 /*! cache: pages written from cache */ -#define WT_STAT_CONN_CACHE_WRITE 1129 +#define WT_STAT_CONN_CACHE_WRITE 1138 /*! cache: pages written requiring in-memory restoration */ -#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1130 +#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1139 /*! cache: percentage overhead */ -#define WT_STAT_CONN_CACHE_OVERHEAD 1131 +#define WT_STAT_CONN_CACHE_OVERHEAD 1140 /*! cache: tracked bytes belonging to internal pages in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_INTERNAL 1132 +#define WT_STAT_CONN_CACHE_BYTES_INTERNAL 1141 /*! cache: tracked bytes belonging to leaf pages in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_LEAF 1133 +#define WT_STAT_CONN_CACHE_BYTES_LEAF 1142 /*! cache: tracked dirty bytes in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_DIRTY 1134 +#define WT_STAT_CONN_CACHE_BYTES_DIRTY 1143 /*! cache: tracked dirty pages in the cache */ -#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1135 +#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1144 /*! cache: unmodified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1136 +#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1145 /*! capacity: background fsync file handles considered */ -#define WT_STAT_CONN_FSYNC_ALL_FH_TOTAL 1137 +#define WT_STAT_CONN_FSYNC_ALL_FH_TOTAL 1146 /*! capacity: background fsync file handles synced */ -#define WT_STAT_CONN_FSYNC_ALL_FH 1138 +#define WT_STAT_CONN_FSYNC_ALL_FH 1147 /*! capacity: background fsync time (msecs) */ -#define WT_STAT_CONN_FSYNC_ALL_TIME 1139 +#define WT_STAT_CONN_FSYNC_ALL_TIME 1148 /*! capacity: bytes read */ -#define WT_STAT_CONN_CAPACITY_BYTES_READ 1140 +#define WT_STAT_CONN_CAPACITY_BYTES_READ 1149 /*! capacity: bytes written for checkpoint */ -#define WT_STAT_CONN_CAPACITY_BYTES_CKPT 1141 +#define WT_STAT_CONN_CAPACITY_BYTES_CKPT 1150 /*! capacity: bytes written for eviction */ -#define WT_STAT_CONN_CAPACITY_BYTES_EVICT 1142 +#define WT_STAT_CONN_CAPACITY_BYTES_EVICT 1151 /*! capacity: bytes written for log */ -#define WT_STAT_CONN_CAPACITY_BYTES_LOG 1143 +#define WT_STAT_CONN_CAPACITY_BYTES_LOG 1152 /*! capacity: bytes written total */ -#define WT_STAT_CONN_CAPACITY_BYTES_WRITTEN 1144 +#define WT_STAT_CONN_CAPACITY_BYTES_WRITTEN 1153 /*! capacity: threshold to call fsync */ -#define WT_STAT_CONN_CAPACITY_THRESHOLD 1145 +#define WT_STAT_CONN_CAPACITY_THRESHOLD 1154 /*! capacity: time waiting due to total capacity (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_TOTAL 1146 +#define WT_STAT_CONN_CAPACITY_TIME_TOTAL 1155 /*! capacity: time waiting during checkpoint (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_CKPT 1147 +#define WT_STAT_CONN_CAPACITY_TIME_CKPT 1156 /*! capacity: time waiting during eviction (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_EVICT 1148 +#define WT_STAT_CONN_CAPACITY_TIME_EVICT 1157 /*! capacity: time waiting during logging (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_LOG 1149 +#define WT_STAT_CONN_CAPACITY_TIME_LOG 1158 /*! capacity: time waiting during read (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_READ 1150 +#define WT_STAT_CONN_CAPACITY_TIME_READ 1159 /*! connection: auto adjusting condition resets */ -#define WT_STAT_CONN_COND_AUTO_WAIT_RESET 1151 +#define WT_STAT_CONN_COND_AUTO_WAIT_RESET 1160 /*! connection: auto adjusting condition wait calls */ -#define WT_STAT_CONN_COND_AUTO_WAIT 1152 +#define WT_STAT_CONN_COND_AUTO_WAIT 1161 /*! connection: detected system time went backwards */ -#define WT_STAT_CONN_TIME_TRAVEL 1153 +#define WT_STAT_CONN_TIME_TRAVEL 1162 /*! connection: files currently open */ -#define WT_STAT_CONN_FILE_OPEN 1154 +#define WT_STAT_CONN_FILE_OPEN 1163 /*! connection: memory allocations */ -#define WT_STAT_CONN_MEMORY_ALLOCATION 1155 +#define WT_STAT_CONN_MEMORY_ALLOCATION 1164 /*! connection: memory frees */ -#define WT_STAT_CONN_MEMORY_FREE 1156 +#define WT_STAT_CONN_MEMORY_FREE 1165 /*! connection: memory re-allocations */ -#define WT_STAT_CONN_MEMORY_GROW 1157 +#define WT_STAT_CONN_MEMORY_GROW 1166 /*! connection: pthread mutex condition wait calls */ -#define WT_STAT_CONN_COND_WAIT 1158 +#define WT_STAT_CONN_COND_WAIT 1167 /*! connection: pthread mutex shared lock read-lock calls */ -#define WT_STAT_CONN_RWLOCK_READ 1159 +#define WT_STAT_CONN_RWLOCK_READ 1168 /*! connection: pthread mutex shared lock write-lock calls */ -#define WT_STAT_CONN_RWLOCK_WRITE 1160 +#define WT_STAT_CONN_RWLOCK_WRITE 1169 /*! connection: total fsync I/Os */ -#define WT_STAT_CONN_FSYNC_IO 1161 +#define WT_STAT_CONN_FSYNC_IO 1170 /*! connection: total read I/Os */ -#define WT_STAT_CONN_READ_IO 1162 +#define WT_STAT_CONN_READ_IO 1171 /*! connection: total write I/Os */ -#define WT_STAT_CONN_WRITE_IO 1163 +#define WT_STAT_CONN_WRITE_IO 1172 /*! cursor: cached cursor count */ -#define WT_STAT_CONN_CURSOR_CACHED_COUNT 1164 +#define WT_STAT_CONN_CURSOR_CACHED_COUNT 1173 /*! cursor: cursor bulk loaded cursor insert calls */ -#define WT_STAT_CONN_CURSOR_INSERT_BULK 1165 +#define WT_STAT_CONN_CURSOR_INSERT_BULK 1174 /*! cursor: cursor close calls that result in cache */ -#define WT_STAT_CONN_CURSOR_CACHE 1166 +#define WT_STAT_CONN_CURSOR_CACHE 1175 /*! cursor: cursor create calls */ -#define WT_STAT_CONN_CURSOR_CREATE 1167 +#define WT_STAT_CONN_CURSOR_CREATE 1176 /*! cursor: cursor insert calls */ -#define WT_STAT_CONN_CURSOR_INSERT 1168 +#define WT_STAT_CONN_CURSOR_INSERT 1177 /*! cursor: cursor insert key and value bytes */ -#define WT_STAT_CONN_CURSOR_INSERT_BYTES 1169 +#define WT_STAT_CONN_CURSOR_INSERT_BYTES 1178 /*! cursor: cursor modify calls */ -#define WT_STAT_CONN_CURSOR_MODIFY 1170 +#define WT_STAT_CONN_CURSOR_MODIFY 1179 /*! cursor: cursor modify key and value bytes affected */ -#define WT_STAT_CONN_CURSOR_MODIFY_BYTES 1171 +#define WT_STAT_CONN_CURSOR_MODIFY_BYTES 1180 /*! cursor: cursor modify value bytes modified */ -#define WT_STAT_CONN_CURSOR_MODIFY_BYTES_TOUCH 1172 +#define WT_STAT_CONN_CURSOR_MODIFY_BYTES_TOUCH 1181 /*! cursor: cursor next calls */ -#define WT_STAT_CONN_CURSOR_NEXT 1173 +#define WT_STAT_CONN_CURSOR_NEXT 1182 /*! cursor: cursor operation restarted */ -#define WT_STAT_CONN_CURSOR_RESTART 1174 +#define WT_STAT_CONN_CURSOR_RESTART 1183 /*! cursor: cursor prev calls */ -#define WT_STAT_CONN_CURSOR_PREV 1175 +#define WT_STAT_CONN_CURSOR_PREV 1184 /*! cursor: cursor remove calls */ -#define WT_STAT_CONN_CURSOR_REMOVE 1176 +#define WT_STAT_CONN_CURSOR_REMOVE 1185 /*! cursor: cursor remove key bytes removed */ -#define WT_STAT_CONN_CURSOR_REMOVE_BYTES 1177 +#define WT_STAT_CONN_CURSOR_REMOVE_BYTES 1186 /*! cursor: cursor reserve calls */ -#define WT_STAT_CONN_CURSOR_RESERVE 1178 +#define WT_STAT_CONN_CURSOR_RESERVE 1187 /*! cursor: cursor reset calls */ -#define WT_STAT_CONN_CURSOR_RESET 1179 +#define WT_STAT_CONN_CURSOR_RESET 1188 /*! cursor: cursor search calls */ -#define WT_STAT_CONN_CURSOR_SEARCH 1180 +#define WT_STAT_CONN_CURSOR_SEARCH 1189 /*! cursor: cursor search near calls */ -#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1181 +#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1190 /*! cursor: cursor sweep buckets */ -#define WT_STAT_CONN_CURSOR_SWEEP_BUCKETS 1182 +#define WT_STAT_CONN_CURSOR_SWEEP_BUCKETS 1191 /*! cursor: cursor sweep cursors closed */ -#define WT_STAT_CONN_CURSOR_SWEEP_CLOSED 1183 +#define WT_STAT_CONN_CURSOR_SWEEP_CLOSED 1192 /*! cursor: cursor sweep cursors examined */ -#define WT_STAT_CONN_CURSOR_SWEEP_EXAMINED 1184 +#define WT_STAT_CONN_CURSOR_SWEEP_EXAMINED 1193 /*! cursor: cursor sweeps */ -#define WT_STAT_CONN_CURSOR_SWEEP 1185 +#define WT_STAT_CONN_CURSOR_SWEEP 1194 /*! cursor: cursor truncate calls */ -#define WT_STAT_CONN_CURSOR_TRUNCATE 1186 +#define WT_STAT_CONN_CURSOR_TRUNCATE 1195 /*! cursor: cursor update calls */ -#define WT_STAT_CONN_CURSOR_UPDATE 1187 +#define WT_STAT_CONN_CURSOR_UPDATE 1196 /*! cursor: cursor update key and value bytes */ -#define WT_STAT_CONN_CURSOR_UPDATE_BYTES 1188 +#define WT_STAT_CONN_CURSOR_UPDATE_BYTES 1197 /*! cursor: cursor update value size change */ -#define WT_STAT_CONN_CURSOR_UPDATE_BYTES_CHANGED 1189 +#define WT_STAT_CONN_CURSOR_UPDATE_BYTES_CHANGED 1198 /*! cursor: cursors reused from cache */ -#define WT_STAT_CONN_CURSOR_REOPEN 1190 +#define WT_STAT_CONN_CURSOR_REOPEN 1199 /*! cursor: open cursor count */ -#define WT_STAT_CONN_CURSOR_OPEN_COUNT 1191 +#define WT_STAT_CONN_CURSOR_OPEN_COUNT 1200 /*! data-handle: connection data handle size */ -#define WT_STAT_CONN_DH_CONN_HANDLE_SIZE 1192 +#define WT_STAT_CONN_DH_CONN_HANDLE_SIZE 1201 /*! data-handle: connection data handles currently active */ -#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1193 +#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1202 /*! data-handle: connection sweep candidate became referenced */ -#define WT_STAT_CONN_DH_SWEEP_REF 1194 +#define WT_STAT_CONN_DH_SWEEP_REF 1203 /*! data-handle: connection sweep dhandles closed */ -#define WT_STAT_CONN_DH_SWEEP_CLOSE 1195 +#define WT_STAT_CONN_DH_SWEEP_CLOSE 1204 /*! data-handle: connection sweep dhandles removed from hash list */ -#define WT_STAT_CONN_DH_SWEEP_REMOVE 1196 +#define WT_STAT_CONN_DH_SWEEP_REMOVE 1205 /*! data-handle: connection sweep time-of-death sets */ -#define WT_STAT_CONN_DH_SWEEP_TOD 1197 +#define WT_STAT_CONN_DH_SWEEP_TOD 1206 /*! data-handle: connection sweeps */ -#define WT_STAT_CONN_DH_SWEEPS 1198 +#define WT_STAT_CONN_DH_SWEEPS 1207 /*! data-handle: session dhandles swept */ -#define WT_STAT_CONN_DH_SESSION_HANDLES 1199 +#define WT_STAT_CONN_DH_SESSION_HANDLES 1208 /*! data-handle: session sweep attempts */ -#define WT_STAT_CONN_DH_SESSION_SWEEPS 1200 +#define WT_STAT_CONN_DH_SESSION_SWEEPS 1209 /*! lock: checkpoint lock acquisitions */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1201 +#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1210 /*! lock: checkpoint lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1202 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1211 /*! lock: checkpoint lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1203 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1212 /*! lock: dhandle lock application thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_APPLICATION 1204 +#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_APPLICATION 1213 /*! lock: dhandle lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_INTERNAL 1205 +#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_INTERNAL 1214 /*! lock: dhandle read lock acquisitions */ -#define WT_STAT_CONN_LOCK_DHANDLE_READ_COUNT 1206 +#define WT_STAT_CONN_LOCK_DHANDLE_READ_COUNT 1215 /*! lock: dhandle write lock acquisitions */ -#define WT_STAT_CONN_LOCK_DHANDLE_WRITE_COUNT 1207 +#define WT_STAT_CONN_LOCK_DHANDLE_WRITE_COUNT 1216 /*! * lock: durable timestamp queue lock application thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_APPLICATION 1208 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_APPLICATION 1217 /*! * lock: durable timestamp queue lock internal thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_INTERNAL 1209 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_INTERNAL 1218 /*! lock: durable timestamp queue read lock acquisitions */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_READ_COUNT 1210 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_READ_COUNT 1219 /*! lock: durable timestamp queue write lock acquisitions */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WRITE_COUNT 1211 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WRITE_COUNT 1220 /*! lock: metadata lock acquisitions */ -#define WT_STAT_CONN_LOCK_METADATA_COUNT 1212 +#define WT_STAT_CONN_LOCK_METADATA_COUNT 1221 /*! lock: metadata lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1213 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1222 /*! lock: metadata lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1214 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1223 /*! * lock: read timestamp queue lock application thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_APPLICATION 1215 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_APPLICATION 1224 /*! lock: read timestamp queue lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_INTERNAL 1216 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_INTERNAL 1225 /*! lock: read timestamp queue read lock acquisitions */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_READ_COUNT 1217 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_READ_COUNT 1226 /*! lock: read timestamp queue write lock acquisitions */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WRITE_COUNT 1218 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WRITE_COUNT 1227 /*! lock: schema lock acquisitions */ -#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1219 +#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1228 /*! lock: schema lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1220 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1229 /*! lock: schema lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1221 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1230 /*! * lock: table lock application thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1222 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1231 /*! * lock: table lock internal thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1223 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1232 /*! lock: table read lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_READ_COUNT 1224 +#define WT_STAT_CONN_LOCK_TABLE_READ_COUNT 1233 /*! lock: table write lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_WRITE_COUNT 1225 +#define WT_STAT_CONN_LOCK_TABLE_WRITE_COUNT 1234 /*! lock: txn global lock application thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_APPLICATION 1226 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_APPLICATION 1235 /*! lock: txn global lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_INTERNAL 1227 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_INTERNAL 1236 /*! lock: txn global read lock acquisitions */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_READ_COUNT 1228 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_READ_COUNT 1237 /*! lock: txn global write lock acquisitions */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WRITE_COUNT 1229 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WRITE_COUNT 1238 /*! log: busy returns attempting to switch slots */ -#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1230 +#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1239 /*! log: force archive time sleeping (usecs) */ -#define WT_STAT_CONN_LOG_FORCE_ARCHIVE_SLEEP 1231 +#define WT_STAT_CONN_LOG_FORCE_ARCHIVE_SLEEP 1240 /*! log: log bytes of payload data */ -#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1232 +#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1241 /*! log: log bytes written */ -#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1233 +#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1242 /*! log: log files manually zero-filled */ -#define WT_STAT_CONN_LOG_ZERO_FILLS 1234 +#define WT_STAT_CONN_LOG_ZERO_FILLS 1243 /*! log: log flush operations */ -#define WT_STAT_CONN_LOG_FLUSH 1235 +#define WT_STAT_CONN_LOG_FLUSH 1244 /*! log: log force write operations */ -#define WT_STAT_CONN_LOG_FORCE_WRITE 1236 +#define WT_STAT_CONN_LOG_FORCE_WRITE 1245 /*! log: log force write operations skipped */ -#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1237 +#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1246 /*! log: log records compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1238 +#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1247 /*! log: log records not compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1239 +#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1248 /*! log: log records too small to compress */ -#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1240 +#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1249 /*! log: log release advances write LSN */ -#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1241 +#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1250 /*! log: log scan operations */ -#define WT_STAT_CONN_LOG_SCANS 1242 +#define WT_STAT_CONN_LOG_SCANS 1251 /*! log: log scan records requiring two reads */ -#define WT_STAT_CONN_LOG_SCAN_REREADS 1243 +#define WT_STAT_CONN_LOG_SCAN_REREADS 1252 /*! log: log server thread advances write LSN */ -#define WT_STAT_CONN_LOG_WRITE_LSN 1244 +#define WT_STAT_CONN_LOG_WRITE_LSN 1253 /*! log: log server thread write LSN walk skipped */ -#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1245 +#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1254 /*! log: log sync operations */ -#define WT_STAT_CONN_LOG_SYNC 1246 +#define WT_STAT_CONN_LOG_SYNC 1255 /*! log: log sync time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DURATION 1247 +#define WT_STAT_CONN_LOG_SYNC_DURATION 1256 /*! log: log sync_dir operations */ -#define WT_STAT_CONN_LOG_SYNC_DIR 1248 +#define WT_STAT_CONN_LOG_SYNC_DIR 1257 /*! log: log sync_dir time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1249 +#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1258 /*! log: log write operations */ -#define WT_STAT_CONN_LOG_WRITES 1250 +#define WT_STAT_CONN_LOG_WRITES 1259 /*! log: logging bytes consolidated */ -#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1251 +#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1260 /*! log: maximum log file size */ -#define WT_STAT_CONN_LOG_MAX_FILESIZE 1252 +#define WT_STAT_CONN_LOG_MAX_FILESIZE 1261 /*! log: number of pre-allocated log files to create */ -#define WT_STAT_CONN_LOG_PREALLOC_MAX 1253 +#define WT_STAT_CONN_LOG_PREALLOC_MAX 1262 /*! log: pre-allocated log files not ready and missed */ -#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1254 +#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1263 /*! log: pre-allocated log files prepared */ -#define WT_STAT_CONN_LOG_PREALLOC_FILES 1255 +#define WT_STAT_CONN_LOG_PREALLOC_FILES 1264 /*! log: pre-allocated log files used */ -#define WT_STAT_CONN_LOG_PREALLOC_USED 1256 +#define WT_STAT_CONN_LOG_PREALLOC_USED 1265 /*! log: records processed by log scan */ -#define WT_STAT_CONN_LOG_SCAN_RECORDS 1257 +#define WT_STAT_CONN_LOG_SCAN_RECORDS 1266 /*! log: slot close lost race */ -#define WT_STAT_CONN_LOG_SLOT_CLOSE_RACE 1258 +#define WT_STAT_CONN_LOG_SLOT_CLOSE_RACE 1267 /*! log: slot close unbuffered waits */ -#define WT_STAT_CONN_LOG_SLOT_CLOSE_UNBUF 1259 +#define WT_STAT_CONN_LOG_SLOT_CLOSE_UNBUF 1268 /*! log: slot closures */ -#define WT_STAT_CONN_LOG_SLOT_CLOSES 1260 +#define WT_STAT_CONN_LOG_SLOT_CLOSES 1269 /*! log: slot join atomic update races */ -#define WT_STAT_CONN_LOG_SLOT_RACES 1261 +#define WT_STAT_CONN_LOG_SLOT_RACES 1270 /*! log: slot join calls atomic updates raced */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_RACE 1262 +#define WT_STAT_CONN_LOG_SLOT_YIELD_RACE 1271 /*! log: slot join calls did not yield */ -#define WT_STAT_CONN_LOG_SLOT_IMMEDIATE 1263 +#define WT_STAT_CONN_LOG_SLOT_IMMEDIATE 1272 /*! log: slot join calls found active slot closed */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_CLOSE 1264 +#define WT_STAT_CONN_LOG_SLOT_YIELD_CLOSE 1273 /*! log: slot join calls slept */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_SLEEP 1265 +#define WT_STAT_CONN_LOG_SLOT_YIELD_SLEEP 1274 /*! log: slot join calls yielded */ -#define WT_STAT_CONN_LOG_SLOT_YIELD 1266 +#define WT_STAT_CONN_LOG_SLOT_YIELD 1275 /*! log: slot join found active slot closed */ -#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1267 +#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1276 /*! log: slot joins yield time (usecs) */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_DURATION 1268 +#define WT_STAT_CONN_LOG_SLOT_YIELD_DURATION 1277 /*! log: slot transitions unable to find free slot */ -#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1269 +#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1278 /*! log: slot unbuffered writes */ -#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1270 +#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1279 /*! log: total in-memory size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_MEM 1271 +#define WT_STAT_CONN_LOG_COMPRESS_MEM 1280 /*! log: total log buffer size */ -#define WT_STAT_CONN_LOG_BUFFER_SIZE 1272 +#define WT_STAT_CONN_LOG_BUFFER_SIZE 1281 /*! log: total size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_LEN 1273 +#define WT_STAT_CONN_LOG_COMPRESS_LEN 1282 /*! log: written slots coalesced */ -#define WT_STAT_CONN_LOG_SLOT_COALESCED 1274 +#define WT_STAT_CONN_LOG_SLOT_COALESCED 1283 /*! log: yields waiting for previous log file close */ -#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1275 +#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1284 /*! perf: file system read latency histogram (bucket 1) - 10-49ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT50 1276 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT50 1285 /*! perf: file system read latency histogram (bucket 2) - 50-99ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT100 1277 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT100 1286 /*! perf: file system read latency histogram (bucket 3) - 100-249ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT250 1278 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT250 1287 /*! perf: file system read latency histogram (bucket 4) - 250-499ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT500 1279 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT500 1288 /*! perf: file system read latency histogram (bucket 5) - 500-999ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT1000 1280 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT1000 1289 /*! perf: file system read latency histogram (bucket 6) - 1000ms+ */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_GT1000 1281 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_GT1000 1290 /*! perf: file system write latency histogram (bucket 1) - 10-49ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT50 1282 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT50 1291 /*! perf: file system write latency histogram (bucket 2) - 50-99ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT100 1283 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT100 1292 /*! perf: file system write latency histogram (bucket 3) - 100-249ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT250 1284 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT250 1293 /*! perf: file system write latency histogram (bucket 4) - 250-499ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT500 1285 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT500 1294 /*! perf: file system write latency histogram (bucket 5) - 500-999ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT1000 1286 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT1000 1295 /*! perf: file system write latency histogram (bucket 6) - 1000ms+ */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_GT1000 1287 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_GT1000 1296 /*! perf: operation read latency histogram (bucket 1) - 100-249us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT250 1288 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT250 1297 /*! perf: operation read latency histogram (bucket 2) - 250-499us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT500 1289 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT500 1298 /*! perf: operation read latency histogram (bucket 3) - 500-999us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT1000 1290 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT1000 1299 /*! perf: operation read latency histogram (bucket 4) - 1000-9999us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT10000 1291 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT10000 1300 /*! perf: operation read latency histogram (bucket 5) - 10000us+ */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_GT10000 1292 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_GT10000 1301 /*! perf: operation write latency histogram (bucket 1) - 100-249us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT250 1293 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT250 1302 /*! perf: operation write latency histogram (bucket 2) - 250-499us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT500 1294 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT500 1303 /*! perf: operation write latency histogram (bucket 3) - 500-999us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT1000 1295 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT1000 1304 /*! perf: operation write latency histogram (bucket 4) - 1000-9999us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT10000 1296 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT10000 1305 /*! perf: operation write latency histogram (bucket 5) - 10000us+ */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_GT10000 1297 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_GT10000 1306 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1298 +#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1307 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_CONN_REC_PAGES 1299 +#define WT_STAT_CONN_REC_PAGES 1308 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_CONN_REC_PAGES_EVICTION 1300 +#define WT_STAT_CONN_REC_PAGES_EVICTION 1309 /*! reconciliation: pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE 1301 +#define WT_STAT_CONN_REC_PAGE_DELETE 1310 /*! reconciliation: split bytes currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1302 +#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1311 /*! reconciliation: split objects currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1303 +#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1312 /*! session: open session count */ -#define WT_STAT_CONN_SESSION_OPEN 1304 +#define WT_STAT_CONN_SESSION_OPEN 1313 /*! session: session query timestamp calls */ -#define WT_STAT_CONN_SESSION_QUERY_TS 1305 +#define WT_STAT_CONN_SESSION_QUERY_TS 1314 /*! session: table alter failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1306 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1315 /*! session: table alter successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1307 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1316 /*! session: table alter unchanged and skipped */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1308 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1317 /*! session: table compact failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1309 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1318 /*! session: table compact successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1310 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1319 /*! session: table create failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1311 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1320 /*! session: table create successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1312 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1321 /*! session: table drop failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1313 +#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1322 /*! session: table drop successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1314 +#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1323 /*! session: table import failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_IMPORT_FAIL 1315 +#define WT_STAT_CONN_SESSION_TABLE_IMPORT_FAIL 1324 /*! session: table import successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_IMPORT_SUCCESS 1316 +#define WT_STAT_CONN_SESSION_TABLE_IMPORT_SUCCESS 1325 /*! session: table rebalance failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1317 +#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1326 /*! session: table rebalance successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1318 +#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1327 /*! session: table rename failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1319 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1328 /*! session: table rename successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1320 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1329 /*! session: table salvage failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1321 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1330 /*! session: table salvage successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1322 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1331 /*! session: table truncate failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1323 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1332 /*! session: table truncate successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1324 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1333 /*! session: table verify failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1325 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1334 /*! session: table verify successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1326 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1335 /*! thread-state: active filesystem fsync calls */ -#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1327 +#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1336 /*! thread-state: active filesystem read calls */ -#define WT_STAT_CONN_THREAD_READ_ACTIVE 1328 +#define WT_STAT_CONN_THREAD_READ_ACTIVE 1337 /*! thread-state: active filesystem write calls */ -#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1329 +#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1338 /*! thread-yield: application thread time evicting (usecs) */ -#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1330 +#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1339 /*! thread-yield: application thread time waiting for cache (usecs) */ -#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1331 +#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1340 /*! * thread-yield: connection close blocked waiting for transaction state * stabilization */ -#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1332 +#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1341 /*! thread-yield: connection close yielded for lsm manager shutdown */ -#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1333 +#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1342 /*! thread-yield: data handle lock yielded */ -#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1334 +#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1343 /*! * thread-yield: get reference for page index and slot time sleeping * (usecs) */ -#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1335 +#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1344 /*! thread-yield: log server sync yielded for log write */ -#define WT_STAT_CONN_LOG_SERVER_SYNC_BLOCKED 1336 +#define WT_STAT_CONN_LOG_SERVER_SYNC_BLOCKED 1345 /*! thread-yield: page access yielded due to prepare state change */ -#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1337 +#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1346 /*! thread-yield: page acquire busy blocked */ -#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1338 +#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1347 /*! thread-yield: page acquire eviction blocked */ -#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1339 +#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1348 /*! thread-yield: page acquire locked blocked */ -#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1340 +#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1349 /*! thread-yield: page acquire read blocked */ -#define WT_STAT_CONN_PAGE_READ_BLOCKED 1341 +#define WT_STAT_CONN_PAGE_READ_BLOCKED 1350 /*! thread-yield: page acquire time sleeping (usecs) */ -#define WT_STAT_CONN_PAGE_SLEEP 1342 +#define WT_STAT_CONN_PAGE_SLEEP 1351 /*! * thread-yield: page delete rollback time sleeping for state change * (usecs) */ -#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1343 +#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1352 /*! thread-yield: page reconciliation yielded due to child modification */ -#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1344 +#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1353 /*! transaction: Number of prepared updates */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COUNT 1345 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COUNT 1354 /*! transaction: Number of prepared updates added to cache overflow */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES_LOOKASIDE_INSERTS 1346 -/*! transaction: Number of prepared updates resolved */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES_RESOLVED 1347 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_LOOKASIDE_INSERTS 1355 /*! transaction: durable timestamp queue entries walked */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_WALKED 1348 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_WALKED 1356 /*! transaction: durable timestamp queue insert to empty */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_EMPTY 1349 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_EMPTY 1357 /*! transaction: durable timestamp queue inserts to head */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_HEAD 1350 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_HEAD 1358 /*! transaction: durable timestamp queue inserts total */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_INSERTS 1351 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_INSERTS 1359 /*! transaction: durable timestamp queue length */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_LEN 1352 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_LEN 1360 /*! transaction: number of named snapshots created */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1353 +#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1361 /*! transaction: number of named snapshots dropped */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1354 +#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1362 /*! transaction: prepared transactions */ -#define WT_STAT_CONN_TXN_PREPARE 1355 +#define WT_STAT_CONN_TXN_PREPARE 1363 /*! transaction: prepared transactions committed */ -#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1356 +#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1364 /*! transaction: prepared transactions currently active */ -#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1357 +#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1365 /*! transaction: prepared transactions rolled back */ -#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1358 +#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1366 /*! transaction: query timestamp calls */ -#define WT_STAT_CONN_TXN_QUERY_TS 1359 +#define WT_STAT_CONN_TXN_QUERY_TS 1367 /*! transaction: read timestamp queue entries walked */ -#define WT_STAT_CONN_TXN_READ_QUEUE_WALKED 1360 +#define WT_STAT_CONN_TXN_READ_QUEUE_WALKED 1368 /*! transaction: read timestamp queue insert to empty */ -#define WT_STAT_CONN_TXN_READ_QUEUE_EMPTY 1361 +#define WT_STAT_CONN_TXN_READ_QUEUE_EMPTY 1369 /*! transaction: read timestamp queue inserts to head */ -#define WT_STAT_CONN_TXN_READ_QUEUE_HEAD 1362 +#define WT_STAT_CONN_TXN_READ_QUEUE_HEAD 1370 /*! transaction: read timestamp queue inserts total */ -#define WT_STAT_CONN_TXN_READ_QUEUE_INSERTS 1363 +#define WT_STAT_CONN_TXN_READ_QUEUE_INSERTS 1371 /*! transaction: read timestamp queue length */ -#define WT_STAT_CONN_TXN_READ_QUEUE_LEN 1364 +#define WT_STAT_CONN_TXN_READ_QUEUE_LEN 1372 /*! transaction: rollback to stable calls */ -#define WT_STAT_CONN_TXN_ROLLBACK_TO_STABLE 1365 +#define WT_STAT_CONN_TXN_ROLLBACK_TO_STABLE 1373 /*! transaction: rollback to stable updates aborted */ -#define WT_STAT_CONN_TXN_ROLLBACK_UPD_ABORTED 1366 +#define WT_STAT_CONN_TXN_ROLLBACK_UPD_ABORTED 1374 /*! transaction: rollback to stable updates removed from cache overflow */ -#define WT_STAT_CONN_TXN_ROLLBACK_LAS_REMOVED 1367 +#define WT_STAT_CONN_TXN_ROLLBACK_LAS_REMOVED 1375 /*! transaction: set timestamp calls */ -#define WT_STAT_CONN_TXN_SET_TS 1368 +#define WT_STAT_CONN_TXN_SET_TS 1376 /*! transaction: set timestamp durable calls */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1369 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1377 /*! transaction: set timestamp durable updates */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1370 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1378 /*! transaction: set timestamp oldest calls */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1371 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1379 /*! transaction: set timestamp oldest updates */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1372 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1380 /*! transaction: set timestamp stable calls */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE 1373 +#define WT_STAT_CONN_TXN_SET_TS_STABLE 1381 /*! transaction: set timestamp stable updates */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1374 +#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1382 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1375 +#define WT_STAT_CONN_TXN_BEGIN 1383 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1376 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1384 /*! transaction: transaction checkpoint generation */ -#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1377 +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1385 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1378 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1386 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1379 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1387 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1380 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1388 /*! transaction: transaction checkpoint scrub dirty target */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1381 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1389 /*! transaction: transaction checkpoint scrub time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1382 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1390 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1383 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1391 /*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1384 +#define WT_STAT_CONN_TXN_CHECKPOINT 1392 /*! * transaction: transaction checkpoints skipped because database was * clean */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1385 +#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1393 /*! transaction: transaction failures due to cache overflow */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1386 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1394 /*! * transaction: transaction fsync calls for checkpoint after allocating * the transaction ID */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1387 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1395 /*! * transaction: transaction fsync duration for checkpoint after * allocating the transaction ID (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1388 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1396 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1389 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1397 /*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1390 +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1398 /*! * transaction: transaction range of IDs currently pinned by named * snapshots */ -#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1391 +#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1399 /*! transaction: transaction range of timestamps currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1392 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1400 /*! transaction: transaction range of timestamps pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1393 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1401 /*! * transaction: transaction range of timestamps pinned by the oldest * active read timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1394 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1402 /*! * transaction: transaction range of timestamps pinned by the oldest * timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1395 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1403 /*! transaction: transaction read timestamp of the oldest active reader */ -#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1396 +#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1404 /*! transaction: transaction sync calls */ -#define WT_STAT_CONN_TXN_SYNC 1397 +#define WT_STAT_CONN_TXN_SYNC 1405 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1398 +#define WT_STAT_CONN_TXN_COMMIT 1406 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1399 +#define WT_STAT_CONN_TXN_ROLLBACK 1407 /*! transaction: update conflicts */ -#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1400 +#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1408 /*! * @} diff --git a/src/third_party/wiredtiger/src/meta/meta_ckpt.c b/src/third_party/wiredtiger/src/meta/meta_ckpt.c index c8c4383f1a3..93a01fa6abb 100644 --- a/src/third_party/wiredtiger/src/meta/meta_ckpt.c +++ b/src/third_party/wiredtiger/src/meta/meta_ckpt.c @@ -12,7 +12,7 @@ static int __ckpt_last(WT_SESSION_IMPL *, const char *, WT_CKPT *); static int __ckpt_last_name(WT_SESSION_IMPL *, const char *, const char **); static int __ckpt_load(WT_SESSION_IMPL *, WT_CONFIG_ITEM *, WT_CONFIG_ITEM *, WT_CKPT *); static int __ckpt_named(WT_SESSION_IMPL *, const char *, const char *, WT_CKPT *); -static int __ckpt_set(WT_SESSION_IMPL *, const char *, const char *); +static int __ckpt_set(WT_SESSION_IMPL *, const char *, const char *, bool); static int __ckpt_version_chk(WT_SESSION_IMPL *, const char *, const char *); /* @@ -94,7 +94,7 @@ __wt_meta_checkpoint_clear(WT_SESSION_IMPL *session, const char *fname) * If we are unrolling a failed create, we may have already removed the metadata entry. If no * entry is found to update and we're trying to clear the checkpoint, just ignore it. */ - WT_RET_NOTFOUND_OK(__ckpt_set(session, fname, NULL)); + WT_RET_NOTFOUND_OK(__ckpt_set(session, fname, NULL, false)); return (0); } @@ -104,25 +104,40 @@ __wt_meta_checkpoint_clear(WT_SESSION_IMPL *session, const char *fname) * Set a file's checkpoint. */ static int -__ckpt_set(WT_SESSION_IMPL *session, const char *fname, const char *v) +__ckpt_set(WT_SESSION_IMPL *session, const char *fname, const char *v, bool use_base) { + WT_DECL_ITEM(tmp); WT_DECL_RET; char *config, *newcfg; - const char *cfg[3]; + const char *cfg[3], *str; + /* + * If the caller knows we're on a path like checkpoints where we have a valid checkpoint and + * checkpoint LSN and should use the base, then use that faster path. Some paths don't have a + * dhandle or want to have the older value retained from the existing metadata. In those cases, + * use the slower path through configuration parsing functions. + */ config = newcfg = NULL; - - /* Retrieve the metadata for this file. */ - WT_ERR(__wt_metadata_search(session, fname, &config)); - - /* Replace the checkpoint entry. */ - cfg[0] = config; - cfg[1] = v == NULL ? "checkpoint=()" : v; - cfg[2] = NULL; - WT_ERR(__wt_config_collapse(session, cfg, &newcfg)); - WT_ERR(__wt_metadata_update(session, fname, newcfg)); + str = v == NULL ? "checkpoint=(),checkpoint_lsn=" : v; + if (use_base && session->dhandle != NULL) { + WT_ERR(__wt_scr_alloc(session, 0, &tmp)); + WT_ASSERT(session, strcmp(session->dhandle->name, fname) == 0); + /* Concatenate the metadata base string with the checkpoint string. */ + WT_ERR(__wt_buf_fmt(session, tmp, "%s,%s", session->dhandle->meta_base, str)); + WT_ERR(__wt_metadata_update(session, fname, tmp->mem)); + } else { + /* Retrieve the metadata for this file. */ + WT_ERR(__wt_metadata_search(session, fname, &config)); + /* Replace the checkpoint entry. */ + cfg[0] = config; + cfg[1] = str; + cfg[2] = NULL; + WT_ERR(__wt_config_collapse(session, cfg, &newcfg)); + WT_ERR(__wt_metadata_update(session, fname, newcfg)); + } err: + __wt_scr_free(session, &tmp); __wt_free(session, config); __wt_free(session, newcfg); return (ret); @@ -236,37 +251,27 @@ __wt_meta_block_metadata(WT_SESSION_IMPL *session, const char *config, WT_CKPT * WT_DECL_RET; WT_KEYED_ENCRYPTOR *kencryptor; size_t encrypt_size, metadata_len; - char *min_config; const char *metadata, *filecfg[] = {WT_CONFIG_BASE(session, file_meta), NULL, NULL}; - min_config = NULL; WT_ERR(__wt_scr_alloc(session, 0, &a)); WT_ERR(__wt_scr_alloc(session, 0, &b)); + /* Fill out the configuration array for normal retrieval. */ + filecfg[1] = config; + /* + * Find out if this file is encrypted. If encrypting, encrypt and encode. * The metadata has to be encrypted because it contains private data * (for example, column names). We pass the block manager text that * describes the metadata (the encryption information), and the * possibly encrypted metadata encoded as a hexadecimal string. - * configuration string. - * - * Get a minimal configuration string, just the non-default entries. - */ - WT_ERR(__wt_config_discard_defaults(session, filecfg, config, &min_config)); - - /* Fill out the configuration array for normal retrieval. */ - filecfg[1] = config; - - /* - * Find out if this file is encrypted. If encrypting, encrypt and encode the minimal - * configuration. */ WT_ERR(__wt_btree_config_encryptor(session, filecfg, &kencryptor)); if (kencryptor == NULL) { - metadata = min_config; - metadata_len = strlen(min_config); + metadata = config; + metadata_len = strlen(config); } else { - WT_ERR(__wt_buf_set(session, a, min_config, strlen(min_config))); + WT_ERR(__wt_buf_set(session, a, config, strlen(config))); __wt_encrypt_size(session, kencryptor, a->size, &encrypt_size); WT_ERR(__wt_buf_grow(session, b, encrypt_size)); WT_ERR(__wt_encrypt(session, kencryptor, 0, a, b)); @@ -289,7 +294,6 @@ __wt_meta_block_metadata(WT_SESSION_IMPL *session, const char *config, WT_CKPT * WT_ERR(__wt_strndup(session, b->data, b->size, &ckpt->block_metadata)); err: - __wt_free(session, min_config); __wt_scr_free(session, &a); __wt_scr_free(session, &b); return (ret); @@ -584,16 +588,18 @@ __wt_meta_ckptlist_set( WT_CKPT *ckpt; WT_DECL_ITEM(buf); WT_DECL_RET; + bool has_lsn; WT_RET(__wt_scr_alloc(session, 1024, &buf)); WT_ERR(__wt_meta_ckptlist_to_meta(session, ckptbase, buf)); + has_lsn = ckptlsn != NULL; if (ckptlsn != NULL) WT_ERR(__wt_buf_catfmt(session, buf, ",checkpoint_lsn=(%" PRIu32 ",%" PRIuMAX ")", ckptlsn->l.file, (uintmax_t)ckptlsn->l.offset)); - WT_ERR(__ckpt_set(session, fname, buf->mem)); + WT_ERR(__ckpt_set(session, fname, buf->mem, has_lsn)); /* Review the checkpoint's write generation. */ WT_CKPT_FOREACH (ckptbase, ckpt) diff --git a/src/third_party/wiredtiger/src/meta/meta_turtle.c b/src/third_party/wiredtiger/src/meta/meta_turtle.c index 80e409f380f..db790537c79 100644 --- a/src/third_party/wiredtiger/src/meta/meta_turtle.c +++ b/src/third_party/wiredtiger/src/meta/meta_turtle.c @@ -335,7 +335,7 @@ err: */ if (ret == 0 || strcmp(key, WT_METADATA_COMPAT) == 0 || F_ISSET(S2C(session), WT_CONN_SALVAGE)) return (ret); - WT_PANIC_RET(session, ret, "%s: fatal turtle file read error", WT_METADATA_TURTLE); + WT_PANIC_RET(session, WT_TRY_SALVAGE, "%s: fatal turtle file read error", WT_METADATA_TURTLE); } /* diff --git a/src/third_party/wiredtiger/src/reconcile/rec_col.c b/src/third_party/wiredtiger/src/reconcile/rec_col.c index 0bc8a7f9009..ca6b0b4adaa 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_col.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_col.c @@ -35,7 +35,7 @@ __rec_col_fix_bulk_insert_split_check(WT_CURSOR_BULK *cbulk) */ __wt_rec_incr( session, r, cbulk->entry, __bitstr_size((size_t)cbulk->entry * btree->bitcnt)); - WT_RET(__wt_rec_split(session, r, 0)); + WT_RET(__wt_rec_split(session, r, 0, false)); } cbulk->entry = 0; cbulk->nrecs = WT_FIX_BYTES_TO_ENTRIES(btree, r->space_avail); @@ -131,7 +131,7 @@ __wt_bulk_insert_var(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk, bool delet /* Boundary: split or write the page. */ if (WT_CROSSING_SPLIT_BND(r, val->len)) - WT_RET(__wt_rec_split_crossing_bnd(session, r, val->len)); + WT_RET(__wt_rec_split_crossing_bnd(session, r, val->len, false)); /* Copy the value onto the page. */ if (btree->dictionary) @@ -174,7 +174,7 @@ __rec_col_merge(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) /* Boundary: split or write the page. */ if (__wt_rec_need_split(r, val->len)) - WT_RET(__wt_rec_split_crossing_bnd(session, r, val->len)); + WT_RET(__wt_rec_split_crossing_bnd(session, r, val->len, false)); /* Copy the value onto the page. */ __wt_rec_image_copy(session, r, val); @@ -298,7 +298,7 @@ __wt_rec_col_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_REF *pageref) /* Boundary: split or write the page. */ if (__wt_rec_need_split(r, val->len)) - WT_ERR(__wt_rec_split_crossing_bnd(session, r, val->len)); + WT_ERR(__wt_rec_split_crossing_bnd(session, r, val->len, false)); /* Copy the value onto the page. */ __wt_rec_image_copy(session, r, val); @@ -410,7 +410,7 @@ __wt_rec_col_fix(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_REF *pageref) * last, allowing it to grow in the future. */ __wt_rec_incr(session, r, entry, __bitstr_size((size_t)entry * btree->bitcnt)); - WT_RET(__wt_rec_split(session, r, 0)); + WT_RET(__wt_rec_split(session, r, 0, false)); /* Calculate the number of entries per page. */ entry = 0; @@ -554,7 +554,7 @@ __rec_col_var_helper(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_SALVAGE_COOKI /* Boundary: split or write the page. */ if (__wt_rec_need_split(r, val->len)) - WT_RET(__wt_rec_split_crossing_bnd(session, r, val->len)); + WT_RET(__wt_rec_split_crossing_bnd(session, r, val->len, false)); /* Copy the value onto the page. */ if (!deleted && !overflow_type && btree->dictionary) diff --git a/src/third_party/wiredtiger/src/reconcile/rec_row.c b/src/third_party/wiredtiger/src/reconcile/rec_row.c index f48ef1f0d8a..43360d702c5 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_row.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_row.c @@ -219,7 +219,7 @@ __wt_bulk_insert_row(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) if (!ovfl_key) WT_RET(__rec_cell_build_leaf_key(session, r, NULL, 0, &ovfl_key)); } - WT_RET(__wt_rec_split_crossing_bnd(session, r, key->len + val->len)); + WT_RET(__wt_rec_split_crossing_bnd(session, r, key->len + val->len, false)); } /* Copy the key/value pair onto the page. */ @@ -272,7 +272,7 @@ __rec_row_merge(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) /* Boundary: split or write the page. */ if (__wt_rec_need_split(r, key->len + val->len)) - WT_RET(__wt_rec_split_crossing_bnd(session, r, key->len + val->len)); + WT_RET(__wt_rec_split_crossing_bnd(session, r, key->len + val->len, false)); /* Copy the key and value onto the page. */ __wt_rec_image_copy(session, r, key); @@ -304,9 +304,9 @@ __wt_rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) WT_REC_KV *key, *val; WT_REF *ref; wt_timestamp_t newest_durable_ts, newest_stop_ts, oldest_start_ts; - size_t size; + size_t key_overflow_size, size; uint64_t newest_stop_txn, oldest_start_txn; - bool hazard, key_onpage_ovfl, ovfl_key; + bool force, hazard, key_onpage_ovfl, ovfl_key; const void *p; btree = S2BT(session); @@ -322,7 +322,6 @@ __wt_rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) ikey = NULL; /* -Wuninitialized */ cell = NULL; - key_onpage_ovfl = false; WT_RET(__wt_rec_split_init(session, r, page, 0, btree->maxintlpage_precomp)); @@ -340,6 +339,8 @@ __wt_rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) */ r->cell_zero = true; + key_overflow_size = 0; + /* For each entry in the in-memory page... */ WT_INTL_FOREACH_BEGIN (session, page, ref) { /* @@ -350,15 +351,13 @@ __wt_rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) * Note the cell reference and unpacked key cell are available only in the case of an * instantiated, off-page key, we don't bother setting them if that's not possible. */ - if (F_ISSET_ATOMIC(page, WT_PAGE_OVERFLOW_KEYS)) { - cell = NULL; - key_onpage_ovfl = false; - ikey = __wt_ref_key_instantiated(ref); - if (ikey != NULL && ikey->cell_offset != 0) { - cell = WT_PAGE_REF_OFFSET(page, ikey->cell_offset); - __wt_cell_unpack(session, page, cell, kpack); - key_onpage_ovfl = kpack->ovfl && kpack->raw != WT_CELL_KEY_OVFL_RM; - } + cell = NULL; + key_onpage_ovfl = false; + ikey = __wt_ref_key_instantiated(ref); + if (ikey != NULL && ikey->cell_offset != 0) { + cell = WT_PAGE_REF_OFFSET(page, ikey->cell_offset); + __wt_cell_unpack(session, page, cell, kpack); + key_onpage_ovfl = kpack->ovfl && kpack->raw != WT_CELL_KEY_OVFL_RM; } WT_ERR(__wt_rec_child_modify(session, r, ref, &hazard, &state)); @@ -467,25 +466,42 @@ __wt_rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) key->cell_len = 0; key->len = key->buf.size; ovfl_key = true; + key_overflow_size += ikey->size; } else { __wt_ref_key(page, ref, &p, &size); - WT_ERR(__rec_cell_build_int_key(session, r, p, r->cell_zero ? 1 : size, &ovfl_key)); + if (r->cell_zero) + size = 1; + WT_ERR(__rec_cell_build_int_key(session, r, p, size, &ovfl_key)); + if (ovfl_key) + key_overflow_size += size; } r->cell_zero = false; - /* Boundary: split or write the page. */ - if (__wt_rec_need_split(r, key->len + val->len)) { + /* + * Boundary: split or write the page. + * + * We always instantiate row-store internal page keys in order to avoid special casing the + * B+tree search code to handle keys that may not exist (and I/O in a search path). Because + * root pages are pinned, overflow keys on the root page can cause follow-on cache effects + * where huge root pages tie down lots of cache space and eviction frantically attempts to + * evict objects that can't be evicted. If the in-memory image is too large, force a split. + * Potentially, limiting ourselves to 10 items per page is going to result in deep trees + * which will impact search performance, but at some point, the application's configuration + * is too stupid to survive. + */ + force = r->entries > 20 && key_overflow_size > btree->maxmempage_image; + if (force || __wt_rec_need_split(r, key->len + val->len)) { + key_overflow_size = 0; + /* * In one path above, we copied address blocks from the page rather than building the * actual key. In that case, we have to build the key now because we are about to * promote it. */ - if (key_onpage_ovfl) { + if (key_onpage_ovfl) WT_ERR(__wt_buf_set(session, r->cur, WT_IKEY_DATA(ikey), ikey->size)); - key_onpage_ovfl = false; - } - WT_ERR(__wt_rec_split_crossing_bnd(session, r, key->len + val->len)); + WT_ERR(__wt_rec_split_crossing_bnd(session, r, key->len + val->len, force)); } /* Copy the key and value onto the page. */ @@ -572,7 +588,7 @@ __rec_row_leaf_insert(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins) /* Copy the current key into place and then split. */ WT_RET(__wt_buf_set(session, r->cur, WT_INSERT_KEY(ins), WT_INSERT_KEY_SIZE(ins))); - WT_RET(__wt_rec_split_crossing_bnd(session, r, WT_INSERT_KEY_SIZE(ins))); + WT_RET(__wt_rec_split_crossing_bnd(session, r, WT_INSERT_KEY_SIZE(ins), false)); /* * Turn off prefix and suffix compression until a full key is written into the new page. @@ -618,7 +634,7 @@ __rec_row_leaf_insert(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins) WT_RET(__rec_cell_build_leaf_key(session, r, NULL, 0, &ovfl_key)); } - WT_RET(__wt_rec_split_crossing_bnd(session, r, key->len + val->len)); + WT_RET(__wt_rec_split_crossing_bnd(session, r, key->len + val->len, false)); } /* Copy the key/value pair onto the page. */ @@ -954,7 +970,7 @@ build: WT_ERR(__rec_cell_build_leaf_key(session, r, NULL, 0, &ovfl_key)); } - WT_ERR(__wt_rec_split_crossing_bnd(session, r, key->len + val->len)); + WT_ERR(__wt_rec_split_crossing_bnd(session, r, key->len + val->len, false)); } /* Copy the key/value pair onto the page. */ diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index 5746e20273b..3b3d13770d2 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -912,8 +912,7 @@ __wt_split_page_size(int split_pct, uint32_t maxpagesize, uint32_t allocsize) * Initialize a single chunk structure. */ static int -__rec_split_chunk_init( - WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_REC_CHUNK *chunk, size_t memsize) +__rec_split_chunk_init(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_REC_CHUNK *chunk) { chunk->recno = WT_RECNO_OOB; /* Don't touch the key item memory, that memory is reused. */ @@ -940,8 +939,9 @@ __rec_split_chunk_init( * In the case of fixed-length column-store, clear the entire buffer: fixed-length column-store * sets bits in bytes, where the bytes are assumed to initially be 0. */ - WT_RET(__wt_buf_init(session, &chunk->image, memsize)); - memset(chunk->image.mem, 0, r->page->type == WT_PAGE_COL_FIX ? memsize : WT_PAGE_HEADER_SIZE); + WT_RET(__wt_buf_init(session, &chunk->image, r->disk_img_buf_size)); + memset(chunk->image.mem, 0, + r->page->type == WT_PAGE_COL_FIX ? r->disk_img_buf_size : WT_PAGE_HEADER_SIZE); return (0); } @@ -958,7 +958,7 @@ __wt_rec_split_init( WT_BTREE *btree; WT_REC_CHUNK *chunk; WT_REF *ref; - size_t corrected_page_size, disk_img_buf_size; + size_t corrected_page_size; btree = S2BT(session); bm = btree->bm; @@ -1030,10 +1030,10 @@ __wt_rec_split_init( */ corrected_page_size = r->page_size; WT_RET(bm->write_size(bm, session, &corrected_page_size)); - disk_img_buf_size = WT_ALIGN(WT_MAX(corrected_page_size, r->split_size), btree->allocsize); + r->disk_img_buf_size = WT_ALIGN(WT_MAX(corrected_page_size, r->split_size), btree->allocsize); /* Initialize the first split chunk. */ - WT_RET(__rec_split_chunk_init(session, r, &r->chunkA, disk_img_buf_size)); + WT_RET(__rec_split_chunk_init(session, r, &r->chunkA)); r->cur_ptr = &r->chunkA; r->prev_ptr = NULL; @@ -1226,7 +1226,7 @@ __rec_split_grow(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t add_len) * in a row? Sweet-tooth does, too.) */ int -__wt_rec_split(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len) +__wt_rec_split(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len, bool forced) { WT_BTREE *btree; WT_REC_CHUNK *tmp; @@ -1234,9 +1234,6 @@ __wt_rec_split(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len) btree = S2BT(session); - /* Fixed length col store can call with next_len 0 */ - WT_ASSERT(session, next_len == 0 || __wt_rec_need_split(r, next_len)); - /* * We should never split during salvage, and we're about to drop core because there's no parent * page. @@ -1245,14 +1242,12 @@ __wt_rec_split(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len) WT_PANIC_RET(session, WT_PANIC, "%s page too large, attempted split during salvage", __wt_page_type_string(r->page->type)); - inuse = WT_PTRDIFF(r->first_free, r->cur_ptr->image.mem); - /* - * We can get here if the first key/value pair won't fit. Additionally, grow the buffer to - * contain the current item if we haven't already consumed a reasonable portion of a split - * chunk. + * We can get here if the first key/value pair won't fit. Grow the buffer to contain the current + * item if we haven't already consumed a reasonable portion of a split chunk. */ - if (inuse < r->split_size / 2 && !__wt_rec_need_split(r, 0)) + inuse = WT_PTRDIFF(r->first_free, r->cur_ptr->image.mem); + if (!forced && inuse < r->split_size / 2 && !__wt_rec_need_split(r, 0)) goto done; /* All page boundaries reset the dictionary. */ @@ -1263,27 +1258,33 @@ __wt_rec_split(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len) r->cur_ptr->image.size = inuse; /* - * In case of bulk load, write out chunks as we get them. Otherwise we keep two chunks in memory - * at a given time. So, if there is a previous chunk, write it out, making space in the buffer - * for the next chunk to be written. + * Normally we keep two chunks in memory at a given time, and we write the previous chunk at + * each boundary, switching the previous and current check references. The exception is when + * doing a bulk load or a forced split, where we write out the chunks as we get them. */ if (r->is_bulk_load) WT_RET(__rec_split_write(session, r, r->cur_ptr, NULL, false)); else { - if (r->prev_ptr == NULL) { - WT_RET(__rec_split_chunk_init(session, r, &r->chunkB, r->cur_ptr->image.memsize)); - r->prev_ptr = &r->chunkB; - } else + if (r->prev_ptr != NULL) WT_RET(__rec_split_write(session, r, r->prev_ptr, NULL, false)); - /* Switch chunks. */ - tmp = r->prev_ptr; - r->prev_ptr = r->cur_ptr; - r->cur_ptr = tmp; + if (forced) { + WT_RET(__rec_split_write(session, r, r->cur_ptr, NULL, false)); + r->prev_ptr = NULL; + r->cur_ptr = &r->chunkA; + } else { + if (r->prev_ptr == NULL) { + WT_RET(__rec_split_chunk_init(session, r, &r->chunkB)); + r->prev_ptr = &r->chunkB; + } + tmp = r->prev_ptr; + r->prev_ptr = r->cur_ptr; + r->cur_ptr = tmp; + } } /* Initialize the next chunk, including the key. */ - WT_RET(__rec_split_chunk_init(session, r, r->cur_ptr, 0)); + WT_RET(__rec_split_chunk_init(session, r, r->cur_ptr)); r->cur_ptr->recno = r->recno; if (btree->type == BTREE_ROW) WT_RET(__rec_split_row_promote(session, r, &r->cur_ptr->key, r->page->type)); @@ -1321,16 +1322,14 @@ done: * Save the details for the minimum split size boundary or call for a split. */ int -__wt_rec_split_crossing_bnd(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len) +__wt_rec_split_crossing_bnd(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len, bool forced) { - WT_ASSERT(session, __wt_rec_need_split(r, next_len)); - /* * If crossing the minimum split size boundary, store the boundary details at the current * location in the buffer. If we are crossing the split boundary at the same time, possible when * the next record is large enough, just split at this point. */ - if (WT_CROSSING_MIN_BND(r, next_len) && !WT_CROSSING_SPLIT_BND(r, next_len) && + if (!forced && WT_CROSSING_MIN_BND(r, next_len) && !WT_CROSSING_SPLIT_BND(r, next_len) && !__wt_rec_need_split(r, 0)) { /* * If the first record doesn't fit into the minimum split size, we end up here. Write the @@ -1361,7 +1360,7 @@ __wt_rec_split_crossing_bnd(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t ne } /* We are crossing a split boundary */ - return (__wt_rec_split(session, r, next_len)); + return (__wt_rec_split(session, r, next_len, forced)); } /* @@ -1417,7 +1416,7 @@ __rec_split_finish_process_prev(WT_SESSION_IMPL *session, WT_RECONCILE *r) tmp = r->prev_ptr; r->prev_ptr = r->cur_ptr; r->cur_ptr = tmp; - return (__rec_split_chunk_init(session, r, r->prev_ptr, 0)); + return (__rec_split_chunk_init(session, r, r->prev_ptr)); } if (prev_ptr->min_offset != 0 && cur_ptr->image.size < r->min_split_size) { diff --git a/src/third_party/wiredtiger/src/session/session_dhandle.c b/src/third_party/wiredtiger/src/session/session_dhandle.c index 9bf35dca909..e8f81b52f89 100644 --- a/src/third_party/wiredtiger/src/session/session_dhandle.c +++ b/src/third_party/wiredtiger/src/session/session_dhandle.c @@ -244,14 +244,16 @@ __wt_session_release_dhandle(WT_SESSION_IMPL *session) } /* - * Close the handle if we are finishing a bulk load or if the handle is set to discard on - * release. + * Close the handle if we are finishing a bulk load or rebalance or if the handle is set to + * discard on release. Bulk loads and rebalanced trees are special because they may have huge + * root pages in memory, and we need to push those pages out of the cache. The only way to do + * that is to close the handle. */ - if (btree != NULL && F_ISSET(btree, WT_BTREE_BULK)) { + if (btree != NULL && F_ISSET(btree, WT_BTREE_BULK | WT_BTREE_REBALANCE)) { WT_ASSERT( session, F_ISSET(dhandle, WT_DHANDLE_EXCLUSIVE) && !F_ISSET(dhandle, WT_DHANDLE_DISCARD)); /* - * Acquire the schema lock while completing a bulk load. This avoids racing with a + * Acquire the schema lock while closing out the handles. This avoids racing with a * checkpoint while it gathers a set of handles. */ WT_WITH_SCHEMA_LOCK(session, ret = __wt_conn_dhandle_close(session, false, false)); diff --git a/src/third_party/wiredtiger/src/support/modify.c b/src/third_party/wiredtiger/src/support/modify.c index 086fb4b3920..56d44af77ef 100644 --- a/src/third_party/wiredtiger/src/support/modify.c +++ b/src/third_party/wiredtiger/src/support/modify.c @@ -39,6 +39,36 @@ while (0) /* + * __wt_modify_idempotent -- + * Check if a modify operation is idempotent. + */ +bool +__wt_modify_idempotent(const void *modify) +{ + WT_MODIFY mod; + size_t tmp; + const size_t *p; + int nentries; + + /* Get the number of modify entries. */ + p = modify; + memcpy(&tmp, p++, sizeof(size_t)); + nentries = (int)tmp; + + WT_MODIFY_FOREACH_BEGIN (mod, p, nentries, 0) { + /* + * If the number of bytes being replaced doesn't match the number of bytes being written, + * we're resizing and the operation isn't idempotent. + */ + if (mod.size != mod.data.size) + return (false); + } + WT_MODIFY_FOREACH_END; + + return (true); +} + +/* * __wt_modify_pack -- * Pack a modify structure into a buffer. */ @@ -224,8 +254,7 @@ __modify_fast_path(WT_ITEM *value, const size_t *p, int nentries, int *nappliedp */ fastpath = first = true; *nappliedp = 0; - WT_MODIFY_FOREACH_BEGIN(current, p, nentries, 0) - { + WT_MODIFY_FOREACH_BEGIN (current, p, nentries, 0) { datasz += current.data.size; if (fastpath && current.data.size == current.size && @@ -295,8 +324,7 @@ __modify_apply_no_overlap(WT_SESSION_IMPL *session, WT_ITEM *value, const size_t from = (const uint8_t *)value->data + value->size; to = (uint8_t *)value->data + destsz; - WT_MODIFY_FOREACH_REVERSE(current, p, nentries, napplied, datasz) - { + WT_MODIFY_FOREACH_REVERSE (current, p, nentries, napplied, datasz) { /* Move the current unmodified block into place if necessary. */ sz = WT_PTRDIFF(to, value->data) - (current.offset + current.data.size); from -= sz; @@ -374,8 +402,7 @@ __wt_modify_apply(WT_CURSOR *cursor, const void *modify) goto done; } - WT_MODIFY_FOREACH_BEGIN(mod, p, nentries, napplied) - { + WT_MODIFY_FOREACH_BEGIN (mod, p, nentries, napplied) { WT_RET(__modify_apply_one(session, value, &mod, sformat)); } WT_MODIFY_FOREACH_END; diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index a8b77dddc8e..0232ea1d2c6 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -661,14 +661,15 @@ static const char *const __stats_connection_desc[] = { "cache: eviction server evicting pages", "cache: eviction server slept, because we did not make progress with eviction", "cache: eviction server unable to reach eviction goal", - "cache: eviction server waiting for a leaf page", - "cache: eviction server waiting for an internal page sleep (usec)", - "cache: eviction server waiting for an internal page yields", "cache: eviction state", + "cache: eviction server waiting for a leaf page", "cache: eviction state", "cache: eviction walk target pages histogram - 0-9", "cache: eviction walk target pages histogram - 10-31", "cache: eviction walk target pages histogram - 128 and higher", "cache: eviction walk target pages histogram - 32-63", - "cache: eviction walk target pages histogram - 64-128", "cache: eviction walks abandoned", + "cache: eviction walk target pages histogram - 64-128", + "cache: eviction walk target strategy both clean and dirty pages", + "cache: eviction walk target strategy only clean pages", + "cache: eviction walk target strategy only dirty pages", "cache: eviction walks abandoned", "cache: eviction walks gave up because they restarted their walk twice", "cache: eviction walks gave up because they saw too many pages and found no candidates", "cache: eviction walks gave up because they saw too many pages and found too few candidates", @@ -690,10 +691,12 @@ static const char *const __stats_connection_desc[] = { "cache: hazard pointer blocked page eviction", "cache: hazard pointer check calls", "cache: hazard pointer check entries walked", "cache: hazard pointer maximum array length", "cache: in-memory page passed criteria to be split", "cache: in-memory page splits", - "cache: internal pages evicted", "cache: internal pages split during eviction", - "cache: leaf pages split during eviction", "cache: maximum bytes configured", - "cache: maximum page size at eviction", "cache: modified pages evicted", - "cache: modified pages evicted by application threads", + "cache: internal pages evicted", "cache: internal pages queued for eviction", + "cache: internal pages seen by eviction walk", + "cache: internal pages seen by eviction walk that are already queued", + "cache: internal pages split during eviction", "cache: leaf pages split during eviction", + "cache: maximum bytes configured", "cache: maximum page size at eviction", + "cache: modified pages evicted", "cache: modified pages evicted by application threads", "cache: operations timed out waiting for space in cache", "cache: overflow pages read into cache", "cache: page split during eviction deepened the tree", "cache: page written requiring cache overflow records", @@ -708,9 +711,17 @@ static const char *const __stats_connection_desc[] = { "cache: pages read into cache with skipped cache overflow entries needed later", "cache: pages read into cache with skipped cache overflow entries needed later by checkpoint", "cache: pages requested from the cache", "cache: pages seen by eviction walk", - "cache: pages selected for eviction unable to be evicted", "cache: pages walked for eviction", - "cache: pages written from cache", "cache: pages written requiring in-memory restoration", - "cache: percentage overhead", "cache: tracked bytes belonging to internal pages in the cache", + "cache: pages seen by eviction walk that are already queued", + "cache: pages selected for eviction unable to be evicted", + "cache: pages selected for eviction unable to be evicted as the parent page has overflow items", + "cache: pages selected for eviction unable to be evicted because of active children on an " + "internal page", + "cache: pages selected for eviction unable to be evicted because of failure in reconciliation", + "cache: pages selected for eviction unable to be evicted due to newer modifications on a clean " + "page", + "cache: pages walked for eviction", "cache: pages written from cache", + "cache: pages written requiring in-memory restoration", "cache: percentage overhead", + "cache: tracked bytes belonging to internal pages in the cache", "cache: tracked bytes belonging to leaf pages in the cache", "cache: tracked dirty bytes in the cache", "cache: tracked dirty pages in the cache", "cache: unmodified pages evicted", "capacity: background fsync file handles considered", @@ -849,7 +860,6 @@ static const char *const __stats_connection_desc[] = { "thread-yield: page reconciliation yielded due to child modification", "transaction: Number of prepared updates", "transaction: Number of prepared updates added to cache overflow", - "transaction: Number of prepared updates resolved", "transaction: durable timestamp queue entries walked", "transaction: durable timestamp queue insert to empty", "transaction: durable timestamp queue inserts to head", @@ -995,14 +1005,15 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->cache_eviction_server_slept = 0; stats->cache_eviction_slow = 0; stats->cache_eviction_walk_leaf_notfound = 0; - stats->cache_eviction_walk_internal_wait = 0; - stats->cache_eviction_walk_internal_yield = 0; /* not clearing cache_eviction_state */ stats->cache_eviction_target_page_lt10 = 0; stats->cache_eviction_target_page_lt32 = 0; stats->cache_eviction_target_page_ge128 = 0; stats->cache_eviction_target_page_lt64 = 0; stats->cache_eviction_target_page_lt128 = 0; + stats->cache_eviction_target_strategy_both_clean_and_dirty = 0; + stats->cache_eviction_target_strategy_clean = 0; + stats->cache_eviction_target_strategy_dirty = 0; stats->cache_eviction_walks_abandoned = 0; stats->cache_eviction_walks_stopped = 0; stats->cache_eviction_walks_gave_up_no_targets = 0; @@ -1033,6 +1044,9 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->cache_inmem_splittable = 0; stats->cache_inmem_split = 0; stats->cache_eviction_internal = 0; + stats->cache_eviction_internal_pages_queued = 0; + stats->cache_eviction_internal_pages_seen = 0; + stats->cache_eviction_internal_pages_already_queued = 0; stats->cache_eviction_split_internal = 0; stats->cache_eviction_split_leaf = 0; /* not clearing cache_bytes_max */ @@ -1059,7 +1073,12 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->cache_read_lookaside_delay_checkpoint = 0; stats->cache_pages_requested = 0; stats->cache_eviction_pages_seen = 0; + stats->cache_eviction_pages_already_queued = 0; stats->cache_eviction_fail = 0; + stats->cache_eviction_fail_parent_has_overflow_items = 0; + stats->cache_eviction_fail_active_children_on_an_internal_page = 0; + stats->cache_eviction_fail_in_reconciliation = 0; + stats->cache_eviction_fail_with_newer_modifications_on_a_clean_page = 0; stats->cache_eviction_walk = 0; stats->cache_write = 0; stats->cache_write_restore = 0; @@ -1279,7 +1298,6 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->child_modify_blocked_page = 0; stats->txn_prepared_updates_count = 0; stats->txn_prepared_updates_lookaside_inserts = 0; - stats->txn_prepared_updates_resolved = 0; stats->txn_durable_queue_walked = 0; stats->txn_durable_queue_empty = 0; stats->txn_durable_queue_head = 0; @@ -1414,15 +1432,18 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * to->cache_eviction_server_slept += WT_STAT_READ(from, cache_eviction_server_slept); to->cache_eviction_slow += WT_STAT_READ(from, cache_eviction_slow); to->cache_eviction_walk_leaf_notfound += WT_STAT_READ(from, cache_eviction_walk_leaf_notfound); - to->cache_eviction_walk_internal_wait += WT_STAT_READ(from, cache_eviction_walk_internal_wait); - to->cache_eviction_walk_internal_yield += - WT_STAT_READ(from, cache_eviction_walk_internal_yield); to->cache_eviction_state += WT_STAT_READ(from, cache_eviction_state); to->cache_eviction_target_page_lt10 += WT_STAT_READ(from, cache_eviction_target_page_lt10); to->cache_eviction_target_page_lt32 += WT_STAT_READ(from, cache_eviction_target_page_lt32); to->cache_eviction_target_page_ge128 += WT_STAT_READ(from, cache_eviction_target_page_ge128); to->cache_eviction_target_page_lt64 += WT_STAT_READ(from, cache_eviction_target_page_lt64); to->cache_eviction_target_page_lt128 += WT_STAT_READ(from, cache_eviction_target_page_lt128); + to->cache_eviction_target_strategy_both_clean_and_dirty += + WT_STAT_READ(from, cache_eviction_target_strategy_both_clean_and_dirty); + to->cache_eviction_target_strategy_clean += + WT_STAT_READ(from, cache_eviction_target_strategy_clean); + to->cache_eviction_target_strategy_dirty += + WT_STAT_READ(from, cache_eviction_target_strategy_dirty); to->cache_eviction_walks_abandoned += WT_STAT_READ(from, cache_eviction_walks_abandoned); to->cache_eviction_walks_stopped += WT_STAT_READ(from, cache_eviction_walks_stopped); to->cache_eviction_walks_gave_up_no_targets += @@ -1457,6 +1478,12 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * to->cache_inmem_splittable += WT_STAT_READ(from, cache_inmem_splittable); to->cache_inmem_split += WT_STAT_READ(from, cache_inmem_split); to->cache_eviction_internal += WT_STAT_READ(from, cache_eviction_internal); + to->cache_eviction_internal_pages_queued += + WT_STAT_READ(from, cache_eviction_internal_pages_queued); + to->cache_eviction_internal_pages_seen += + WT_STAT_READ(from, cache_eviction_internal_pages_seen); + to->cache_eviction_internal_pages_already_queued += + WT_STAT_READ(from, cache_eviction_internal_pages_already_queued); to->cache_eviction_split_internal += WT_STAT_READ(from, cache_eviction_split_internal); to->cache_eviction_split_leaf += WT_STAT_READ(from, cache_eviction_split_leaf); to->cache_bytes_max += WT_STAT_READ(from, cache_bytes_max); @@ -1487,7 +1514,17 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * WT_STAT_READ(from, cache_read_lookaside_delay_checkpoint); to->cache_pages_requested += WT_STAT_READ(from, cache_pages_requested); to->cache_eviction_pages_seen += WT_STAT_READ(from, cache_eviction_pages_seen); + to->cache_eviction_pages_already_queued += + WT_STAT_READ(from, cache_eviction_pages_already_queued); to->cache_eviction_fail += WT_STAT_READ(from, cache_eviction_fail); + to->cache_eviction_fail_parent_has_overflow_items += + WT_STAT_READ(from, cache_eviction_fail_parent_has_overflow_items); + to->cache_eviction_fail_active_children_on_an_internal_page += + WT_STAT_READ(from, cache_eviction_fail_active_children_on_an_internal_page); + to->cache_eviction_fail_in_reconciliation += + WT_STAT_READ(from, cache_eviction_fail_in_reconciliation); + to->cache_eviction_fail_with_newer_modifications_on_a_clean_page += + WT_STAT_READ(from, cache_eviction_fail_with_newer_modifications_on_a_clean_page); to->cache_eviction_walk += WT_STAT_READ(from, cache_eviction_walk); to->cache_write += WT_STAT_READ(from, cache_write); to->cache_write_restore += WT_STAT_READ(from, cache_write_restore); @@ -1712,7 +1749,6 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * to->txn_prepared_updates_count += WT_STAT_READ(from, txn_prepared_updates_count); to->txn_prepared_updates_lookaside_inserts += WT_STAT_READ(from, txn_prepared_updates_lookaside_inserts); - to->txn_prepared_updates_resolved += WT_STAT_READ(from, txn_prepared_updates_resolved); to->txn_durable_queue_walked += WT_STAT_READ(from, txn_durable_queue_walked); to->txn_durable_queue_empty += WT_STAT_READ(from, txn_durable_queue_empty); to->txn_durable_queue_head += WT_STAT_READ(from, txn_durable_queue_head); diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index 68e0c0fa920..5d4f4f8495d 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -625,6 +625,107 @@ __wt_txn_release(WT_SESSION_IMPL *session) } /* + * __txn_resolve_prepared_op -- + * Resolve a transaction's operations indirect references. In case of prepared transactions, the + * prepared updates could be evicted using cache overflow mechanism. Transaction operations + * referring to these prepared updates would be referring to them using indirect references (i.e + * keys/recnos), which need to be resolved as part of that transaction commit/rollback. If no + * updates are resolved throw an error. Increment resolved update count for each resolved update + * count we locate. + */ +static int +__txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit) +{ + WT_CURSOR *cursor; + WT_DECL_RET; + WT_TXN *txn; + WT_UPDATE *upd; + const char *open_cursor_cfg[] = {WT_CONFIG_BASE(session, WT_SESSION_open_cursor), NULL}; + + txn = &session->txn; + + WT_RET(__wt_open_cursor(session, op->btree->dhandle->name, NULL, open_cursor_cfg, &cursor)); + + switch (op->type) { + case WT_TXN_OP_BASIC_COL: + case WT_TXN_OP_INMEM_COL: + ((WT_CURSOR_BTREE *)cursor)->iface.recno = op->u.op_col.recno; + break; + case WT_TXN_OP_BASIC_ROW: + case WT_TXN_OP_INMEM_ROW: + /* + * Transaction prepare is cleared temporarily as cursor functions are not allowed for + * prepared transactions. + */ + F_CLR(txn, WT_TXN_PREPARE); + __wt_cursor_set_raw_key(cursor, &op->u.op_row.key); + F_SET(txn, WT_TXN_PREPARE); + break; + case WT_TXN_OP_NONE: + case WT_TXN_OP_REF_DELETE: + case WT_TXN_OP_TRUNCATE_COL: + case WT_TXN_OP_TRUNCATE_ROW: + WT_ERR_ASSERT(session, false, WT_PANIC, "invalid prepared operation update type"); + break; + } + + WT_WITH_BTREE( + session, op->btree, ret = __wt_btcur_search_uncommitted((WT_CURSOR_BTREE *)cursor, &upd)); + WT_ERR(ret); + + /* If we haven't found anything then there's an error. */ + WT_ERR_ASSERT(session, upd != NULL, WT_NOTFOUND, + "unable to locate update associated with a prepared operation"); + + for (; upd != NULL; upd = upd->next) { + /* + * Aborted updates can exist in the update chain of our txn. Generally this will occur due + * to a reserved update. As such we should skip over these updates. If the txn id is then + * different and not aborted we know we've reached the end of our update chain and can exit. + */ + if (upd->txnid == WT_TXN_ABORTED) + continue; + if (upd->txnid != txn->id) + break; + + if (!commit) { + upd->txnid = WT_TXN_ABORTED; + continue; + } + + /* + * Newer updates are inserted at head of update chain, and transaction operations are added + * at the tail of the transaction modify chain. + * + * For example, a transaction has modified [k,v] as + * [k, v] -> [k, u1] (txn_op : txn_op1) + * [k, u1] -> [k, u2] (txn_op : txn_op2) + * update chain : u2->u1 + * txn_mod : txn_op1->txn_op2. + * + * Only the key is saved in the transaction operation structure, hence we cannot identify + * whether "txn_op1" corresponds to "u2" or "u1" during commit/rollback. + * + * To make things simpler we will handle all the updates that match the key saved in a + * transaction operation in a single go. As a result, multiple updates of a key, if any + * will be resolved as part of the first transaction operation resolution of that key, + * and subsequent transaction operation resolution of the same key will be effectively a + * no-op. + * + * In the above example, we will resolve "u2" and "u1" as part of resolving "txn_op1" and + * will not do any significant thing as part of "txn_op2". + * + * Resolve the prepared update to be committed update. + */ + __txn_resolve_prepared_update(session, upd); + } + +err: + WT_TRET(cursor->close(cursor)); + return (ret); +} + +/* * __txn_commit_timestamps_assert -- * Validate that timestamps provided to commit are legal. */ @@ -779,24 +880,20 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) WT_TXN_OP *op; WT_UPDATE *upd; wt_timestamp_t candidate_durable_timestamp, prev_durable_timestamp; - int64_t resolved_update_count, visited_update_count; uint32_t fileid; u_int i; - bool locked, prepare, readonly, skip_update_assert, update_durable_ts; + bool locked, prepare, readonly, update_durable_ts; txn = &session->txn; conn = S2C(session); txn_global = &conn->txn_global; - locked = skip_update_assert = false; - resolved_update_count = visited_update_count = 0; + locked = false; + prepare = F_ISSET(txn, WT_TXN_PREPARE); + readonly = txn->mod_count == 0; WT_ASSERT(session, F_ISSET(txn, WT_TXN_RUNNING)); WT_ASSERT(session, !F_ISSET(txn, WT_TXN_ERROR) || txn->mod_count == 0); - readonly = txn->mod_count == 0; - - prepare = F_ISSET(txn, WT_TXN_PREPARE); - /* * Clear the prepared round up flag if the transaction is not prepared. There is no rounding up * to do in that case. @@ -809,26 +906,19 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) if (prepare) { if (!F_ISSET(txn, WT_TXN_HAS_TS_COMMIT)) - WT_ERR_MSG(session, EINVAL, - "commit_timestamp is required for a prepared " - "transaction"); + WT_ERR_MSG(session, EINVAL, "commit_timestamp is required for a prepared transaction"); if (!F_ISSET(txn, WT_TXN_HAS_TS_DURABLE)) - WT_ERR_MSG(session, EINVAL, - "durable_timestamp is required for a prepared " - "transaction"); + WT_ERR_MSG(session, EINVAL, "durable_timestamp is required for a prepared transaction"); WT_ASSERT(session, txn->prepare_timestamp <= txn->commit_timestamp); } else { if (F_ISSET(txn, WT_TXN_HAS_TS_PREPARE)) - WT_ERR_MSG(session, EINVAL, - "prepare timestamp is set for non-prepared " - "transaction"); + WT_ERR_MSG(session, EINVAL, "prepare timestamp is set for non-prepared transaction"); if (F_ISSET(txn, WT_TXN_HAS_TS_DURABLE)) WT_ERR_MSG(session, EINVAL, - "durable_timestamp should not be specified for " - "non-prepared transaction"); + "durable_timestamp should not be specified for non-prepared transaction"); } if (F_ISSET(txn, WT_TXN_HAS_TS_COMMIT)) @@ -919,10 +1009,6 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) case WT_TXN_OP_INMEM_ROW: upd = op->u.op_upd; - /* - * Need to resolve indirect references of transaction operation, in case of prepared - * transaction. - */ if (!prepare) { /* * Switch reserved operations to abort to simplify obsolete update list truncation. @@ -942,26 +1028,13 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) __wt_txn_op_set_timestamp(session, op); } else { - visited_update_count++; - /* - * If we have set the key repeated flag we can skip resolving prepared updates as it - * would have happened on a previous modification in this txn. - */ - if (!F_ISSET(op, WT_TXN_OP_KEY_REPEATED)) { - skip_update_assert = skip_update_assert || F_ISSET(op, WT_TXN_OP_KEY_RESERVED); - WT_ERR(__wt_txn_resolve_prepared_op(session, op, true, &resolved_update_count)); - } - /* - * We should resolve at least one or more - * updates each time we call - * __wt_txn_resolve_prepared_op, as such - * resolved update count should never be less - * than visited update count. + * If an operation has the key repeated flag set, skip resolving prepared updates as + * the work will happen on a different modification in this txn. */ - WT_ASSERT(session, resolved_update_count >= visited_update_count); + if (!F_ISSET(op, WT_TXN_OP_KEY_REPEATED)) + WT_ERR(__txn_resolve_prepared_op(session, op, true)); } - break; case WT_TXN_OP_REF_DELETE: __wt_txn_op_set_timestamp(session, op); @@ -974,12 +1047,6 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) __wt_txn_op_free(session, op); } - WT_ERR_ASSERT(session, skip_update_assert || resolved_update_count == visited_update_count, - EINVAL, - "Number of resolved prepared updates: %" PRId64 " does not match number visited: %" PRId64, - resolved_update_count, visited_update_count); - WT_STAT_CONN_INCRV(session, txn_prepared_updates_resolved, resolved_update_count); - txn->mod_count = 0; /* @@ -1055,10 +1122,12 @@ __wt_txn_prepare(WT_SESSION_IMPL *session, const char *cfg[]) { WT_TXN *txn; WT_TXN_OP *op; - WT_UPDATE *upd; + WT_UPDATE *upd, *tmp; + int64_t txn_prepared_updates_count; u_int i; txn = &session->txn; + txn_prepared_updates_count = 0; WT_ASSERT(session, F_ISSET(txn, WT_TXN_RUNNING)); WT_ASSERT(session, !F_ISSET(txn, WT_TXN_ERROR) || txn->mod_count == 0); @@ -1086,13 +1155,7 @@ __wt_txn_prepare(WT_SESSION_IMPL *session, const char *cfg[]) WT_RET(__wt_session_copy_values(session)); } - /* - * Prepare updates, traverse the modification array in reverse order so that we visit the update - * chain in newest to oldest order allowing us to set the key repeated flag with reserved - * updates in the chain. - */ - for (i = txn->mod_count; i > 0; i--) { - op = &txn->mod[i - 1]; + for (i = 0, op = txn->mod; i < txn->mod_count; i++, op++) { /* Assert it's not an update to the lookaside file. */ WT_ASSERT( session, S2C(session)->cache->las_fileid == 0 || !F_ISSET(op->btree, WT_BTREE_LOOKASIDE)); @@ -1102,8 +1165,6 @@ __wt_txn_prepare(WT_SESSION_IMPL *session, const char *cfg[]) if (WT_IS_METADATA(op->btree->dhandle)) continue; - upd = op->u.op_upd; - switch (op->type) { case WT_TXN_OP_NONE: break; @@ -1111,10 +1172,12 @@ __wt_txn_prepare(WT_SESSION_IMPL *session, const char *cfg[]) case WT_TXN_OP_BASIC_ROW: case WT_TXN_OP_INMEM_COL: case WT_TXN_OP_INMEM_ROW: + upd = op->u.op_upd; + /* * Switch reserved operation to abort to simplify obsolete update list truncation. The * object free function clears the operation type so we don't try to visit this update - * again: it can be evicted. + * again: it can be discarded. */ if (upd->type == WT_UPDATE_RESERVE) { upd->txnid = WT_TXN_ABORTED; @@ -1122,28 +1185,25 @@ __wt_txn_prepare(WT_SESSION_IMPL *session, const char *cfg[]) break; } + ++txn_prepared_updates_count; + /* Set prepare timestamp. */ upd->start_ts = txn->prepare_timestamp; WT_PUBLISH(upd->prepare_state, WT_PREPARE_INPROGRESS); op->u.op_upd = NULL; - WT_STAT_CONN_INCR(session, txn_prepared_updates_count); + /* - * Set the key repeated flag which tells us that we've got multiple updates to the same - * key by the same txn. This is later used in txn commit. - * - * When we see a reserved update we set the WT_UPDATE_RESERVED flag instead. We do this - * as we cannot know if our current update should specify the key repeated flag as we - * don't want to traverse the entire update chain to find out. i.e. if there is an - * update with our txnid after the reserved update we should set key repeated, but if - * there isn't we shouldn't. + * If there are older updates to this key by the same transaction, set the repeated key + * flag on this operation. This is later used in txn commit/rollback so we only resolve + * each set of prepared updates once. Skip reserved updates, they're ignored as they're + * simply discarded when we find them. */ - if (upd->next != NULL && upd->txnid == upd->next->txnid) { - if (upd->next->type == WT_UPDATE_RESERVE) - F_SET(op, WT_TXN_OP_KEY_RESERVED); - else + for (tmp = upd->next; tmp != NULL && tmp->txnid == upd->txnid; tmp = tmp->next) + if (tmp->type != WT_UPDATE_RESERVE) { F_SET(op, WT_TXN_OP_KEY_REPEATED); - } + break; + } break; case WT_TXN_OP_REF_DELETE: __wt_txn_op_apply_prepare_state(session, op->u.ref, false); @@ -1154,6 +1214,7 @@ __wt_txn_prepare(WT_SESSION_IMPL *session, const char *cfg[]) break; } } + WT_STAT_CONN_INCR(session, txn_prepared_updates_count); /* Set transaction state to prepare. */ F_SET(&session->txn, WT_TXN_PREPARE); @@ -1182,22 +1243,22 @@ __wt_txn_rollback(WT_SESSION_IMPL *session, const char *cfg[]) WT_TXN *txn; WT_TXN_OP *op; WT_UPDATE *upd; - int64_t resolved_update_count, visited_update_count; u_int i; - bool readonly, skip_update_assert; + bool prepare, readonly; WT_UNUSED(cfg); - resolved_update_count = visited_update_count = 0; + txn = &session->txn; + prepare = F_ISSET(txn, WT_TXN_PREPARE); readonly = txn->mod_count == 0; - skip_update_assert = false; + WT_ASSERT(session, F_ISSET(txn, WT_TXN_RUNNING)); /* Rollback notification. */ if (txn->notify != NULL) WT_TRET(txn->notify->notify(txn->notify, (WT_SESSION *)session, txn->id, 0)); - /* Rollback updates. */ + /* Rollback and free updates. */ for (i = 0, op = txn->mod; i < txn->mod_count; i++, op++) { /* Assert it's not an update to the lookaside file. */ WT_ASSERT( @@ -1208,8 +1269,6 @@ __wt_txn_rollback(WT_SESSION_IMPL *session, const char *cfg[]) if (WT_IS_METADATA(op->btree->dhandle)) continue; - upd = op->u.op_upd; - switch (op->type) { case WT_TXN_OP_NONE: break; @@ -1217,32 +1276,18 @@ __wt_txn_rollback(WT_SESSION_IMPL *session, const char *cfg[]) case WT_TXN_OP_BASIC_ROW: case WT_TXN_OP_INMEM_COL: case WT_TXN_OP_INMEM_ROW: - /* - * Need to resolve indirect references of transaction operation, in case of prepared - * transaction. - */ - if (F_ISSET(txn, WT_TXN_PREPARE)) { - visited_update_count++; - /* - * If we have set the key repeated flag we can skip resolving prepared updates as it - * would have happened on a previous modification in this txn. - */ - if (!F_ISSET(op, WT_TXN_OP_KEY_REPEATED)) { - skip_update_assert = skip_update_assert || F_ISSET(op, WT_TXN_OP_KEY_RESERVED); - WT_RET( - __wt_txn_resolve_prepared_op(session, op, false, &resolved_update_count)); - } - /* - * We should resolve at least one or more - * updates each time we call - * __wt_txn_resolve_prepared_op, as such - * resolved update count should never be less - * than visited update count. - */ - WT_ASSERT(session, resolved_update_count >= visited_update_count); - } else { + upd = op->u.op_upd; + + if (!prepare) { WT_ASSERT(session, upd->txnid == txn->id || upd->txnid == WT_TXN_ABORTED); upd->txnid = WT_TXN_ABORTED; + } else { + /* + * If an operation has the key repeated flag set, skip resolving prepared updates as + * the work will happen on a different modification in this txn. + */ + if (!F_ISSET(op, WT_TXN_OP_KEY_REPEATED)) + WT_RET(__txn_resolve_prepared_op(session, op, false)); } break; case WT_TXN_OP_REF_DELETE: @@ -1260,13 +1305,6 @@ __wt_txn_rollback(WT_SESSION_IMPL *session, const char *cfg[]) __wt_txn_op_free(session, op); } - WT_RET_ASSERT(session, skip_update_assert || resolved_update_count == visited_update_count, - EINVAL, "Number of resolved prepared updates: %" PRId64 - " does not match" - " number visited: %" PRId64, - resolved_update_count, visited_update_count); - WT_STAT_CONN_INCRV(session, txn_prepared_updates_resolved, resolved_update_count); - txn->mod_count = 0; __wt_txn_release(session); diff --git a/src/third_party/wiredtiger/src/txn/txn_log.c b/src/third_party/wiredtiger/src/txn/txn_log.c index 124465cc529..9dfb65306ca 100644 --- a/src/third_party/wiredtiger/src/txn/txn_log.c +++ b/src/third_party/wiredtiger/src/txn/txn_log.c @@ -80,7 +80,16 @@ __txn_op_log( #endif switch (upd->type) { case WT_UPDATE_MODIFY: - WT_RET(__wt_logop_row_modify_pack(session, logrec, fileid, &cursor->key, &value)); + /* + * Write full updates to the log for size-changing modify operations: they aren't + * idempotent and recovery cannot guarantee that they will be applied exactly once. We + * rely on the cursor value already having the modify applied. + */ + if (__wt_modify_idempotent(upd->data)) + WT_RET(__wt_logop_row_modify_pack(session, logrec, fileid, &cursor->key, &value)); + else + WT_RET( + __wt_logop_row_put_pack(session, logrec, fileid, &cursor->key, &cursor->value)); break; case WT_UPDATE_STANDARD: WT_RET(__wt_logop_row_put_pack(session, logrec, fileid, &cursor->key, &value)); @@ -97,7 +106,10 @@ __txn_op_log( switch (upd->type) { case WT_UPDATE_MODIFY: - WT_RET(__wt_logop_col_modify_pack(session, logrec, fileid, recno, &value)); + if (__wt_modify_idempotent(upd->data)) + WT_RET(__wt_logop_col_modify_pack(session, logrec, fileid, recno, &value)); + else + WT_RET(__wt_logop_col_put_pack(session, logrec, fileid, recno, &cursor->value)); break; case WT_UPDATE_STANDARD: WT_RET(__wt_logop_col_put_pack(session, logrec, fileid, recno, &value)); diff --git a/src/third_party/wiredtiger/test/csuite/time_shift_test.sh b/src/third_party/wiredtiger/test/csuite/time_shift_test.sh index 9c14ae86ae7..247a411e995 100755 --- a/src/third_party/wiredtiger/test/csuite/time_shift_test.sh +++ b/src/third_party/wiredtiger/test/csuite/time_shift_test.sh @@ -42,10 +42,13 @@ then exit $EXIT_FAILURE fi +# Locate Wiredtiger home directory. +RW_LOCK_FILE=$(git rev-parse --show-toplevel)/build_posix/test/csuite/test_rwlock + SEC1=`date +%s` if [ "$RUN_OS" = "Darwin" ] then - ./test_rwlock + $RW_LOCK_FILE elif [ "$RUN_OS" = "Linux" ] then if [ -z $2 ] @@ -54,7 +57,7 @@ then else CPU_SET=$2 fi - taskset -c $CPU_SET ./test_rwlock + taskset -c $CPU_SET $RW_LOCK_FILE else echo "not able to decide running OS, so exiting" exit $EXIT_FAILURE @@ -68,9 +71,9 @@ if [ "$RUN_OS" = "Darwin" ] then export DYLD_FORCE_FLAT_NAMESPACE=y export DYLD_INSERT_LIBRARIES=$1 - ./test_rwlock & + $RW_LOCK_FILE & else - LD_PRELOAD=$1 taskset -c $CPU_SET ./test_rwlock & + LD_PRELOAD=$1 taskset -c $CPU_SET $RW_LOCK_FILE & fi # get pid of test run in background diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index fd5b9bf4b3e..b4677e3293d 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -55,7 +55,7 @@ functions: set -o verbose if [ "$OS" != "Windows_NT" ]; then sh reconf - ${configure_env_vars|CC=/opt/mongodbtoolchain/v3/bin/gcc CXX=/opt/mongodbtoolchain/v3/bin/g++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-ggdb"} \ + ${configure_env_vars|CC=/opt/mongodbtoolchain/v3/bin/gcc CXX=/opt/mongodbtoolchain/v3/bin/g++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-ggdb -fPIC"} \ ../configure ${configure_python_setting|} \ ${posix_configure_flags|--enable-silent-rules --enable-diagnostic --enable-python --enable-zlib --enable-strict --enable-static --prefix=$(pwd)/LOCAL_INSTALL} fi @@ -123,7 +123,7 @@ functions: set -o errexit set -o verbose - ${test_env_vars|} ${python_binary|python} ../test/suite/run.py -v 2 ${smp_command|} 2>&1 + ${test_env_vars|} ${python_binary|python} ../test/suite/run.py ${unit_test_args|-v 2} ${smp_command|} 2>&1 "test format": command: shell.exec params: @@ -132,7 +132,7 @@ functions: set -o errexit set -o verbose for i in $(seq ${times|1}); do - ./t ${test_format_args|-1 -c ../../../test/format/CONFIG.stress} + ./t -1 -c ${config|../../../test/format/CONFIG.stress} ${extra_args|} done "upload artifact": - command: archive.targz_pack @@ -187,7 +187,7 @@ tasks: - func: "get project" - func: "compile wiredtiger" vars: - configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/clang CXX=/opt/mongodbtoolchain/v3/bin/clang++ CFLAGS="-fsanitize=address -fno-omit-frame-pointer -ggdb" + configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/clang CXX=/opt/mongodbtoolchain/v3/bin/clang++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-fsanitize=address -fno-omit-frame-pointer -ggdb" posix_configure_flags: --enable-silent-rules --enable-strict --enable-diagnostic --disable-static - name: compile-msan @@ -195,7 +195,7 @@ tasks: - func: "get project" - func: "compile wiredtiger" vars: - configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/clang CXX=/opt/mongodbtoolchain/v3/bin/clang++ CFLAGS="-fsanitize=memory -ggdb" + configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/clang CXX=/opt/mongodbtoolchain/v3/bin/clang++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-fsanitize=memory -ggdb" posix_configure_flags: --enable-silent-rules --enable-strict --enable-diagnostic --disable-static - name: compile-ubsan @@ -203,7 +203,7 @@ tasks: - func: "get project" - func: "compile wiredtiger" vars: - configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/gcc CXX=/opt/mongodbtoolchain/v3/bin/g++ CFLAGS="-fsanitize=undefined -ggdb" + configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/gcc CXX=/opt/mongodbtoolchain/v3/bin/g++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-fsanitize=undefined -ggdb" posix_configure_flags: --enable-silent-rules --enable-strict --enable-diagnostic # production build with --disable-shared @@ -231,6 +231,13 @@ tasks: vars: posix_configure_flags: --enable-silent-rules --enable-diagnostic --enable-strict --enable-python + - name: compile-wtperf + commands: + - func: "get project" + - func: "compile wiredtiger" + vars: + posix_configure_flags: --enable-strict --enable-diagnostic + - name: make-check-test depends_on: - name: compile @@ -248,7 +255,7 @@ tasks: dependent_task: compile-msan - func: "compile wiredtiger" vars: - configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/clang CXX=/opt/mongodbtoolchain/v3/bin/clang++ CFLAGS="-fsanitize=memory -ggdb" + configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/clang CXX=/opt/mongodbtoolchain/v3/bin/clang++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-fsanitize=memory -ggdb" posix_configure_flags: --enable-silent-rules --enable-strict --enable-diagnostic --disable-static - func: "make check all" vars: @@ -263,7 +270,7 @@ tasks: dependent_task: compile-asan - func: "compile wiredtiger" vars: - configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/clang CXX=/opt/mongodbtoolchain/v3/bin/clang++ CFLAGS="-fsanitize=address -fno-omit-frame-pointer -ggdb" + configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/clang CXX=/opt/mongodbtoolchain/v3/bin/clang++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-fsanitize=address -fno-omit-frame-pointer -ggdb" posix_configure_flags: --enable-silent-rules --enable-strict --enable-diagnostic --disable-static - func: "make check all" vars: @@ -315,7 +322,7 @@ tasks: dependent_task: compile-asan - func: "compile wiredtiger" vars: - configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/clang CXX=/opt/mongodbtoolchain/v3/bin/clang++ CFLAGS="-fsanitize=address -ggdb" + configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/clang CXX=/opt/mongodbtoolchain/v3/bin/clang++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-fsanitize=address -ggdb" posix_configure_flags: --enable-silent-rules --enable-strict --enable-diagnostic --disable-static - func: "make check directory" vars: @@ -493,7 +500,7 @@ tasks: - func: "fetch artifacts" - func: "compile wiredtiger" vars: - configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/gcc CXX=/opt/mongodbtoolchain/v3/bin/g++ CFLAGS="-fsanitize=undefined -ggdb" + configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/gcc CXX=/opt/mongodbtoolchain/v3/bin/g++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-fsanitize=undefined -ggdb" posix_configure_flags: --enable-silent-rules --enable-strict --enable-diagnostic - command: shell.exec params: @@ -1026,6 +1033,15 @@ tasks: - func: "fetch artifacts" - func: "unit test" + - name: unit-test-long + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "unit test" + vars: + unit_test_args: -v 2 --long + - name: unit-linux-no-ftruncate-test depends_on: - name: compile-linux-no-ftruncate @@ -1055,7 +1071,7 @@ tasks: set -o errexit set -o verbose - ${test_env_vars|} ${python_binary|python} ../test/suite/run.py [ab] -v 2 ${smp_command|} 2>&1 + ${test_env_vars|} ${python_binary|python} ../test/suite/run.py [ab] ${unit_test_args|-v 2} ${smp_command|} 2>&1 - name: unit-test-bucket01 tags: ["pull_request", "unit_test"] @@ -1071,7 +1087,7 @@ tasks: set -o verbose # Reserve this bucket only for compat tests, which take a long time to run - ${test_env_vars|} ${python_binary|python} ../test/suite/run.py compat -v 2 ${smp_command|} 2>&1 + ${test_env_vars|} ${python_binary|python} ../test/suite/run.py compat ${unit_test_args|-v 2} ${smp_command|} 2>&1 - name: unit-test-bucket02 tags: ["pull_request", "unit_test"] @@ -1088,7 +1104,7 @@ tasks: # Non-compat tests in the 'c' family non_compat_tests=$(ls ../test/suite/test_c*.py | xargs -n1 basename | grep -v compat) - ${test_env_vars|} ${python_binary|python} ../test/suite/run.py $non_compat_tests -v 2 ${smp_command|} 2>&1 + ${test_env_vars|} ${python_binary|python} ../test/suite/run.py $non_compat_tests ${unit_test_args|-v 2} ${smp_command|} 2>&1 - name: unit-test-bucket03 tags: ["pull_request", "unit_test"] @@ -1103,7 +1119,7 @@ tasks: set -o errexit set -o verbose - ${test_env_vars|} ${python_binary|python} ../test/suite/run.py [defghijk] -v 2 ${smp_command|} 2>&1 + ${test_env_vars|} ${python_binary|python} ../test/suite/run.py [defghijk] ${unit_test_args|-v 2} ${smp_command|} 2>&1 - name: unit-test-bucket04 tags: ["pull_request", "unit_test"] @@ -1118,7 +1134,7 @@ tasks: set -o errexit set -o verbose - ${test_env_vars|} ${python_binary|python} ../test/suite/run.py [lmnopq] -v 2 ${smp_command|} 2>&1 + ${test_env_vars|} ${python_binary|python} ../test/suite/run.py [lmnopq] ${unit_test_args|-v 2} ${smp_command|} 2>&1 - name: unit-test-bucket05 tags: ["pull_request", "unit_test"] @@ -1133,7 +1149,7 @@ tasks: set -o errexit set -o verbose - ${test_env_vars|} ${python_binary|python} ../test/suite/run.py [rs] -v 2 ${smp_command|} 2>&1 + ${test_env_vars|} ${python_binary|python} ../test/suite/run.py [rs] ${unit_test_args|-v 2} ${smp_command|} 2>&1 - name: unit-test-bucket06 tags: ["pull_request", "unit_test"] @@ -1149,7 +1165,7 @@ tasks: set -o verbose # Reserve this bucket only for timestamp tests, which take a long time to run - ${test_env_vars|} ${python_binary|python} ../test/suite/run.py timestamp -v 2 ${smp_command|} 2>&1 + ${test_env_vars|} ${python_binary|python} ../test/suite/run.py timestamp ${unit_test_args|-v 2} ${smp_command|} 2>&1 - name: unit-test-bucket07 tags: ["pull_request", "unit_test"] @@ -1166,8 +1182,8 @@ tasks: # Non-timestamp tests in the 't' family non_ts_tests=$(ls ../test/suite/test_t*.py | xargs -n1 basename | grep -v timestamp) - ${test_env_vars|} ${python_binary|python} ../test/suite/run.py $non_ts_tests -v 2 ${smp_command|} 2>&1 - ${test_env_vars|} ${python_binary|python} ../test/suite/run.py [uvwxyz] -v 2 ${smp_command|} 2>&1 + ${test_env_vars|} ${python_binary|python} ../test/suite/run.py $non_ts_tests ${unit_test_args|-v 2} ${smp_command|} 2>&1 + ${test_env_vars|} ${python_binary|python} ../test/suite/run.py [uvwxyz] ${unit_test_args|-v 2} ${smp_command|} 2>&1 # End of Python unit test tasks @@ -1300,7 +1316,8 @@ tasks: - func: "test format" vars: times: 10 - test_format_args: -1 -h "WT_TEST.$i" -c ../../../test/format/CONFIG.endian + config: ../../../test/format/CONFIG.endian + extra_args: -h "WT_TEST.$i" - command: shell.exec params: working_dir: "wiredtiger/build_posix/test/format" @@ -1361,7 +1378,8 @@ tasks: - func: "test format" vars: times: 10 - test_format_args: -1 -h "WT_TEST.$i" -c ../../../test/format/CONFIG.endian + config: ../../../test/format/CONFIG.endian + extra_args: -h "WT_TEST.$i" - command: shell.exec params: working_dir: "wiredtiger/build_posix/test/format" @@ -1435,6 +1453,15 @@ tasks: set -o errexit set -o verbose ./configure_combinations.sh ${smp_command|} 2>&1 + # Handle special build combination for running all the diagnostic tests. + - func: "configure wiredtiger" + vars: + configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/gcc CXX=/opt/mongodbtoolchain/v3/bin/g++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-g -Werror" + posix_configure_flags: --enable-silent-rules --enable-diagnostic --disable-static + - func: "make wiredtiger" + vars: + make_command: make all + - func: "make check all" - name: linux-directio depends_on: @@ -1445,7 +1472,8 @@ tasks: - func: "test format" vars: times: 3 - test_format_args: -1 -C "direct_io=[data]" -c ../../../test/format/CONFIG.stress + config: ../../../test/format/CONFIG.stress + extra_args: -C "direct_io=[data]" - name: format-linux-no-ftruncate depends_on: @@ -1482,7 +1510,7 @@ tasks: script: | set -o errexit set -o verbose - ${python_binary|python} syscall.py --verbose + ${python_binary|python} syscall.py --verbose --preserve - name: checkpoint-filetypes-test commands: @@ -1501,13 +1529,140 @@ tasks: vars: checkpoint_args: -t c -n 1000000 -k 5000000 -C cache_size=100MB + - name: coverage-report + commands: + - func: "get project" + - func: "compile wiredtiger" + vars: + configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/gcc CXX=/opt/mongodbtoolchain/v3/bin/g++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="--coverage -fPIC -ggdb" LDFLAGS=--coverage + posix_configure_flags: --enable-silent-rules --enable-diagnostic --enable-strict --enable-python --with-builtins=lz4,snappy,zlib + - func: "make check all" + - func: "unit test" + vars: + unit_test_args: -v 2 --long + - func: "test format" + vars: + extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=row compression=snappy logging=1 logging_compression=snappy logging_prealloc=1 + - func: "test format" + vars: + extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=row alter=1 backups=1 compaction=1 data_extend=1 prepare=1 rebalance=1 salvage=1 statistics=1 statistics_server=1 verify=1 + - func: "test format" + vars: + extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=row direct_io=1 firstfit=1 internal_key_truncation=1 + - func: "test format" + vars: + extra_args: leak_memory=0 mmap=1 file_type=row checkpoints=0 in_memory=1 reverse=1 truncate=1 + - func: "test format" + vars: + extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=row compression=zlib huffman_key=1 huffman_value=1 + - func: "test format" + vars: + extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=row isolation=random transaction_timestamps=0 + - func: "test format" + vars: + extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=row data_source=lsm bloom=1 + - func: "test format" + vars: + extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=var compression=snappy checksum=uncompressed dictionary=1 repeat_data_pct=10 + - func: "test format" + vars: + extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=row compression=lz4 prefix_compression=1 leaf_page_max=9 internal_page_max=9 key_min=256 value_min=256 + - func: "test format" + vars: + extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=var leaf_page_max=9 internal_page_max=9 value_min=256 + - func: "test format" + vars: + extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=fix + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix" + script: | + set -o errexit + set -o verbose + # FIX ME Remove once BUILD-5025 is done + pip install gcovr --user + GCOV=/opt/mongodbtoolchain/v3/bin/gcov /home/ubuntu/.local/bin/gcovr -r .. -e '.*/bt_(debug|dump|misc|salvage|vrfy).*' -e '.*/(log|progress|verify_build|strerror|env_msg|err_file|cur_config|os_abort)\..*' -e '.*_stat\..*' --html -o ../coverage_report.html + - command: s3.put + params: + aws_secret: ${aws_secret} + aws_key: ${aws_key} + local_file: wiredtiger/coverage_report.html + bucket: build_external + permissions: public-read + content_type: text/html + display_name: Coverage report + remote_file: wiredtiger/${build_variant}/${revision}/coverage_report/coverage_report_${build_id}.html + + - name: spinlock-gcc-test + commands: + - func: "get project" + - func: "compile wiredtiger" + vars: + posix_configure_flags: --enable-python --with-spinlock=gcc --enable-strict + - func: "make check all" + - func: "test format" + vars: + times: 3 + - func: "unit test" + + - name: spinlock-pthread-adaptive-test + commands: + - func: "get project" + - func: "compile wiredtiger" + vars: + posix_configure_flags: --enable-python --with-spinlock=pthread_adaptive --enable-strict + - func: "make check all" + - func: "test format" + vars: + times: 3 + - func: "unit test" + + - name: wtperf-test + depends_on: + - name: compile-wtperf + commands: + - func: "fetch artifacts" + vars: + dependent_task: compile-wtperf + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix" + script: | + set -o errexit + set -o verbose + # The test will generate WT_TEST directory automatically + dir=../bench/wtperf/stress + for file in `ls $dir` + do + ./bench/wtperf/wtperf -O $dir/$file -o verbose=2 + cp -rf WT_TEST WT_TEST_$file + done + + - name: time-shift-sensitivity-test + depends_on: + - name: compile + vars: + posix_configure_flags: --enable-strict + commands: + - func: "fetch artifacts" + vars: + posix_configure_flags: --enable-strict + - command: shell.exec + params: + working_dir: "wiredtiger/test/csuite" + script: | + set -o errexit + set -o verbose + + ./time_shift_test.sh /usr/local/lib/faketime/libfaketimeMT.so.1 0-1 2>&1 + buildvariants: - name: ubuntu1804 display_name: Ubuntu 18.04 run_on: - ubuntu1804-test expansions: - test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs top_srcdir=$(pwd)/.. top_builddir=$(pwd) + test_env_vars: LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libeatmydata.so PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs top_srcdir=$(pwd)/.. top_builddir=$(pwd) smp_command: -j $(grep -c ^processor /proc/cpuinfo) posix_configure_flags: --enable-silent-rules --enable-diagnostic --enable-python --enable-zlib --enable-snappy --enable-strict --enable-static --prefix=$(pwd)/LOCAL_INSTALL make_command: PATH=/opt/mongodbtoolchain/v3/bin:$PATH make @@ -1518,16 +1673,23 @@ buildvariants: - name: compile-ubsan - name: ubsan-test - name: linux-directio + - name: syscall-linux - name: make-check-asan-test - name: configure-combinations - name: checkpoint-filetypes-test + - name: coverage-report + - name: unit-test-long + - name: spinlock-gcc-test + - name: spinlock-pthread-adaptive-test + - name: compile-wtperf + - name: wtperf-test - name: ubuntu1804-python3 display_name: Ubuntu 18.04 (Python3) run_on: - ubuntu1804-test expansions: - test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs top_srcdir=$(pwd)/.. top_builddir=$(pwd) + test_env_vars: LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libeatmydata.so PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs top_srcdir=$(pwd)/.. top_builddir=$(pwd) smp_command: -j $(grep -c ^processor /proc/cpuinfo) configure_python_setting: PYTHON=python3 posix_configure_flags: --enable-silent-rules --enable-diagnostic --enable-python --enable-zlib --enable-snappy --enable-strict --enable-static --prefix=$(pwd)/LOCAL_INSTALL @@ -1546,26 +1708,13 @@ buildvariants: tasks: - name: package -- name: syscall-linux - display_name: Syscall Linux - batchtime: 10080 # a week - run_on: - - ubuntu1804-test - expansions: - test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs top_srcdir=$(pwd)/.. top_builddir=$(pwd) - smp_command: -j $(grep -c ^processor /proc/cpuinfo) - make_command: PATH=/opt/mongodbtoolchain/v3/bin:$PATH make - tasks: - - name: compile - - name: syscall-linux - - name: linux-no-ftruncate display_name: Linux no ftruncate batchtime: 10080 # 1 day run_on: - ubuntu1804-test expansions: - test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs top_srcdir=$(pwd)/.. top_builddir=$(pwd) + test_env_vars: LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libeatmydata.so PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs top_srcdir=$(pwd)/.. top_builddir=$(pwd) smp_command: -j $(grep -c ^processor /proc/cpuinfo) make_command: PATH=/opt/mongodbtoolchain/v3/bin:$PATH make tasks: @@ -1579,7 +1728,7 @@ buildvariants: run_on: - rhel80-test expansions: - test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs top_srcdir=$(pwd)/.. top_builddir=$(pwd) + test_env_vars: LD_PRELOAD=/usr/local/lib/libeatmydata.so PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs top_srcdir=$(pwd)/.. top_builddir=$(pwd) smp_command: -j $(grep -c ^processor /proc/cpuinfo) make_command: PATH=/opt/mongodbtoolchain/v3/bin:$PATH make tasks: @@ -1587,6 +1736,7 @@ buildvariants: - name: make-check-test - name: unit-test - name: fops + - name: time-shift-sensitivity-test - name: large-scale-test display_name: Large scale testing @@ -1619,8 +1769,9 @@ buildvariants: run_on: - macos-1012 expansions: + configure_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-ggdb -fPIC" + posix_configure_flags: --enable-silent-rules --enable-diagnostic --enable-python --enable-zlib --enable-strict --enable-static --prefix=$(pwd)/LOCAL_INSTALL smp_command: -j $(sysctl -n hw.logicalcpu) - configure_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-ggdb" make_command: PATH=/opt/mongodbtoolchain/v3/bin:$PATH ARCHFLAGS=-Wno-error=unused-command-line-argument-hard-error-in-future make test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH DYLD_LIBRARY_PATH=$(pwd)/.libs top_srcdir=$(pwd)/.. top_builddir=$(pwd) tasks: @@ -1658,3 +1809,29 @@ buildvariants: - name: generate-datafile-big-endian - name: verify-datafile-big-endian - name: verify-datafile-from-little-endian + +- name: ubuntu1804-ppc + display_name: Ubuntu 18.04 PPC + run_on: + - ubuntu1804-power8-test + expansions: + smp_command: -j $(grep -c ^processor /proc/cpuinfo) + posix_configure_flags: --enable-silent-rules --enable-diagnostic --enable-python --enable-zlib --enable-snappy --enable-strict --enable-static --prefix=$(pwd)/LOCAL_INSTALL + make_command: PATH=/opt/mongodbtoolchain/v3/bin:$PATH make + test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs top_srcdir=$(pwd)/.. top_builddir=$(pwd) + tasks: + - name: compile + - name: unit-test + +- name: ubuntu1804-zseries + display_name: Ubuntu 18.04 zSeries + run_on: + - ubuntu1804-zseries-test + expansions: + smp_command: -j $(grep -c ^processor /proc/cpuinfo) + posix_configure_flags: --enable-silent-rules --enable-diagnostic --enable-python --enable-zlib --enable-snappy --enable-strict --enable-static --prefix=$(pwd)/LOCAL_INSTALL + make_command: PATH=/opt/mongodbtoolchain/v3/bin:$PATH make + test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs:$(pwd)/lang/python top_srcdir=$(pwd)/.. top_builddir=$(pwd) + tasks: + - name: compile + - name: unit-test diff --git a/src/third_party/wiredtiger/test/evergreen/configure_combinations.sh b/src/third_party/wiredtiger/test/evergreen/configure_combinations.sh index 441c7d3b12a..f335cb4c0e6 100755 --- a/src/third_party/wiredtiger/test/evergreen/configure_combinations.sh +++ b/src/third_party/wiredtiger/test/evergreen/configure_combinations.sh @@ -11,11 +11,12 @@ CC=clang CFLAGS=\"-Wall -Werror -Qunused-arguments -Wno-self-assign -Wno-parenth options="--enable-diagnostic --disable-shared ---disable-static +--disable-static --enable-python --enable-java --enable-python --enable-snappy --enable-zlib --enable-lz4 --with-builtins=lz4,snappy,zlib ---enable-diagnostic --enable-java --enable-python" +--enable-diagnostic --enable-java --enable-python +--enable-strict --disable-shared" saved_IFS=$IFS cr_IFS=" @@ -64,7 +65,6 @@ for flag in $flags ; do echo "*** ERROR: $flag, $option" fi IFS="$cr_IFS" - echo hello done done IFS=$saved_IFS diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h index d8cfea5730a..bae89f7e2f6 100644 --- a/src/third_party/wiredtiger/test/format/format.h +++ b/src/third_party/wiredtiger/test/format/format.h @@ -63,6 +63,7 @@ typedef struct { char *home_backup_init; /* Initialize backup command */ char *home_config; /* Run CONFIG file path */ char *home_init; /* Initialize home command */ + char *home_lasdump; /* LAS dump filename */ char *home_log; /* Operation log file path */ char *home_pagedump; /* Page dump filename */ char *home_rand; /* RNG log file path */ diff --git a/src/third_party/wiredtiger/test/format/snap.c b/src/third_party/wiredtiger/test/format/snap.c index e68309c0149..eed296e212f 100644 --- a/src/third_party/wiredtiger/test/format/snap.c +++ b/src/third_party/wiredtiger/test/format/snap.c @@ -191,19 +191,14 @@ snap_verify(WT_CURSOR *cursor, TINFO *tinfo, SNAP_OPS *snap) return (0); } -/* Things went pear-shaped. */ -#ifdef HAVE_DIAGNOSTIC - fprintf(stderr, "snapshot-isolation error: Dumping page to %s\n", g.home_pagedump); - testutil_check(__wt_debug_cursor_page(cursor, g.home_pagedump)); -#endif + /* Things went pear-shaped. */ switch (g.type) { case FIX: - testutil_die(ret, "snapshot-isolation: %" PRIu64 - " search: " - "expected {0x%02x}, found {0x%02x}", - keyno, snap->op == REMOVE ? 0 : *(uint8_t *)snap->vdata, - ret == WT_NOTFOUND ? 0 : *(uint8_t *)value->data); - /* NOTREACHED */ + fprintf(stderr, + "snapshot-isolation: %" PRIu64 " search: expected {0x%02x}, found {0x%02x}\n", keyno, + snap->op == REMOVE ? 0U : *(uint8_t *)snap->vdata, + ret == WT_NOTFOUND ? 0U : *(uint8_t *)value->data); + break; case ROW: fprintf( stderr, "snapshot-isolation %.*s search mismatch\n", (int)key->size, (char *)key->data); @@ -216,10 +211,7 @@ snap_verify(WT_CURSOR *cursor, TINFO *tinfo, SNAP_OPS *snap) fprintf(stderr, " found {deleted}\n"); else print_item_data(" found", value->data, value->size); - - testutil_die( - ret, "snapshot-isolation: %.*s search mismatch", (int)key->size, (char *)key->data); - /* NOTREACHED */ + break; case VAR: fprintf(stderr, "snapshot-isolation %" PRIu64 " search mismatch\n", keyno); @@ -231,8 +223,35 @@ snap_verify(WT_CURSOR *cursor, TINFO *tinfo, SNAP_OPS *snap) fprintf(stderr, " found {deleted}\n"); else print_item_data(" found", value->data, value->size); + break; + } +#ifdef HAVE_DIAGNOSTIC + /* + * We have a mismatch. Try to print out as much information as we can. In doing so, we are + * calling into the debug code directly and that does not take locks. So it is possible that the + * calls may crash in some way. + * + * The most important information is the key/value mismatch information. Then try to dump out + * the other information. Right now we dump the entire lookaside table including what is on + * disk. That can potentially be very large. If it becomes a problem, this can be modified to + * just dump out the page this key is on. + */ + fprintf(stderr, "snapshot-isolation error: Dumping page to %s\n", g.home_pagedump); + testutil_check(__wt_debug_cursor_page(cursor, g.home_pagedump)); + fprintf(stderr, "snapshot-isolation error: Dumping LAS to %s\n", g.home_lasdump); + testutil_check(__wt_debug_cursor_las(cursor, g.home_lasdump)); + if (g.logging) + testutil_check(cursor->session->log_flush(cursor->session, "sync=off")); +#endif + switch (g.type) { + case FIX: + case VAR: testutil_die(ret, "snapshot-isolation: %" PRIu64 " search mismatch", keyno); + /* NOTREACHED */ + case ROW: + testutil_die( + ret, "snapshot-isolation: %.*s search mismatch", (int)key->size, (char *)key->data); /* NOTREACHED */ } diff --git a/src/third_party/wiredtiger/test/format/util.c b/src/third_party/wiredtiger/test/format/util.c index bdc98d25b46..95ded261097 100644 --- a/src/third_party/wiredtiger/test/format/util.c +++ b/src/third_party/wiredtiger/test/format/util.c @@ -329,6 +329,11 @@ path_setup(const char *home) g.home_log = dmalloc(len); testutil_check(__wt_snprintf(g.home_log, len, "%s/%s", g.home, "log")); + /* LAS dump file. */ + len = strlen(g.home) + strlen("LASdump") + 2; + g.home_lasdump = dmalloc(len); + testutil_check(__wt_snprintf(g.home_lasdump, len, "%s/%s", g.home, "LASdump")); + /* Page dump file. */ len = strlen(g.home) + strlen("pagedump") + 2; g.home_pagedump = dmalloc(len); diff --git a/src/third_party/wiredtiger/test/suite/test_txn19.py b/src/third_party/wiredtiger/test/suite/test_txn19.py index c63e9f9c6e9..87cbf608a7c 100755 --- a/src/third_party/wiredtiger/test/suite/test_txn19.py +++ b/src/third_party/wiredtiger/test/suite/test_txn19.py @@ -520,11 +520,7 @@ class test_txn19_meta(wttest.WiredTigerTestCase, suite_subprocess): if expect_fail: self.check_file_contains_one_of(errfile, - ['/unknown configuration key/', - '/handle-open:/', - '/turtle file read error: WT_NOTFOUND: item not found/', - 'WT_ERROR: non-specific WiredTiger error', - 'WT_TRY_SALVAGE: database corruption detected']) + ['WT_TRY_SALVAGE: database corruption detected']) for salvagedir in [ newdir, newdir2 ]: # Removing the 'WiredTiger.turtle' file has weird behavior: diff --git a/src/third_party/wiredtiger/test/syscall/wt2336_base/base.run b/src/third_party/wiredtiger/test/syscall/wt2336_base/base.run index ed501f50ce4..a68a171d62b 100644 --- a/src/third_party/wiredtiger/test/syscall/wt2336_base/base.run +++ b/src/third_party/wiredtiger/test/syscall/wt2336_base/base.run @@ -138,9 +138,6 @@ pwrite64(wt, ""..., 0x1000, 0x4000); #ifdef __linux__ fdatasync(wt); #endif /* __linux__ */ -fd = OPEN_EXISTING("./WiredTiger.turtle", O_RDWR|O_CLOEXEC); - -close(fd); fd = OPEN("./WiredTiger.turtle.set", O_RDWR|O_CREAT|O_EXCL|O_CLOEXEC, 0666); pwrite64(fd, "WiredTiger version string\nWiredTiger"..., ...); #ifdef __linux__ diff --git a/src/third_party/wiredtiger/tools/xray_to_optrack/Makefile.am b/src/third_party/wiredtiger/tools/xray_to_optrack/Makefile.am new file mode 100644 index 00000000000..7d6889efc32 --- /dev/null +++ b/src/third_party/wiredtiger/tools/xray_to_optrack/Makefile.am @@ -0,0 +1,5 @@ +AM_CPPFLAGS = `llvm-config --cxxflags --libs core` +AM_LDFLAGS = `llvm-config --ldflags --libs core` + +noinst_PROGRAMS = xray_to_optrack +xray_to_optrack_SOURCES = xray_to_optrack.cxx diff --git a/src/third_party/wiredtiger/tools/xray_to_optrack/xray_to_optrack.cxx b/src/third_party/wiredtiger/tools/xray_to_optrack/xray_to_optrack.cxx new file mode 100644 index 00000000000..347c5d067bb --- /dev/null +++ b/src/third_party/wiredtiger/tools/xray_to_optrack/xray_to_optrack.cxx @@ -0,0 +1,181 @@ +/*- + * Public Domain 2014-2019 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include <llvm/ADT/DenseMap.h> +#include <llvm/DebugInfo/Symbolize/Symbolize.h> +#include <llvm/XRay/InstrumentationMap.h> +#include <llvm/XRay/Trace.h> + +#include <fstream> +#include <sstream> + +namespace xray_to_optrack { + +/* + * make_error -- + * Helper for creating LLVM StringErrors. + */ +static llvm::Error +make_error(const std::string &msg) +{ + return llvm::make_error<llvm::StringError>(msg, llvm::inconvertibleErrorCode()); +} + +/* + * xray_to_optrack_record_type -- + * Convert from an XRay log record type to an OpTrack log record type. + */ +static llvm::Expected<int> +xray_to_optrack_record_type(const llvm::xray::RecordTypes record_type) +{ + switch (record_type) { + case llvm::xray::RecordTypes::ENTER: + return 0; + case llvm::xray::RecordTypes::EXIT: + case llvm::xray::RecordTypes::TAIL_EXIT: + return 1; + default: { + std::stringstream ss; + ss << "Unexpected record type \"" << static_cast<unsigned>(record_type) << "\""; + return make_error(ss.str()); + } + } +} + +/* + * write_optrack_record -- + * Write an OpTrack record to a log file. + */ +static void +write_optrack_record(std::ofstream &os, int record_type, const std::string &func_name, uint64_t tsc) +{ + os << record_type << " " << func_name << " " << tsc << "\n"; +} + +/* + * symbolize_func_id -- + * Symbolize the full function name for a given XRay function id. + */ +static llvm::Expected<std::string> +symbolize_func_id(uint32_t func_id, const std::string &instr_map, + llvm::symbolize::LLVMSymbolizer &symbolizer, + const std::unordered_map<int32_t, uint64_t> &address_map, + llvm::DenseMap<uint32_t, std::string> &cache) +{ + const auto cache_iter = cache.find(func_id); + if (cache_iter != cache.end()) + return cache_iter->second; + const auto iter = address_map.find(func_id); + if (iter == address_map.end()) { + std::stringstream ss; + ss << "Found function id \"" << func_id << "\" without a corresponding address"; + return make_error(ss.str()); + } + auto res = symbolizer.symbolizeCode(instr_map, iter->second); + if (!res) + return res.takeError(); + if (res->FunctionName == "<invalid>") { + std::stringstream ss; + ss << "Could not symbolize id \"" << func_id << "\", address \"" << iter->second << "\""; + return make_error(ss.str()); + } + cache.try_emplace(func_id, res->FunctionName); + return res->FunctionName; +} + +/* + * generate_optrack_log_name -- + * Create a filename for an OpTrack log given its process id and thread id. The filename is in + * the format "optrack_<pid>_<tid>". + */ +static std::string +generate_optrack_log_name(uint32_t process_id, uint32_t thread_id) +{ + std::stringstream ss; + ss << "optrack_" << process_id << "_" << thread_id; + return ss.str(); +} + +/* + * xray_to_optrack -- + * Write an OpTrack log for each thread given an XRay log and instrumentation map. + */ +static llvm::Error +xray_to_optrack(const std::string &instr_map, const std::string &input) +{ + auto map = llvm::xray::loadInstrumentationMap(instr_map); + if (!map) + return map.takeError(); + auto trace = llvm::xray::loadTraceFile(input); + if (!trace) + return trace.takeError(); + llvm::symbolize::LLVMSymbolizer symbolizer; + llvm::DenseMap<uint32_t, std::string> cache; + llvm::DenseMap<uint32_t, std::ofstream> files; + for (const llvm::xray::XRayRecord &record : *trace) { + auto file_iter = files.find(record.TId); + if (file_iter == files.end()) { + std::ofstream file(generate_optrack_log_name(record.PId, record.TId)); + if (!file) { + std::stringstream ss; + ss << "Failed to write OpTrack log for process id \"" << record.PId + << "\", thread id \"" << record.TId << "\""; + return make_error(ss.str()); + } + auto res = files.try_emplace(record.TId, std::move(file)); + file_iter = res.first; + } + assert(file_iter != files.end()); + auto record_type = xray_to_optrack_record_type(record.Type); + if (!record_type) + return record_type.takeError(); + auto func_name = symbolize_func_id( + record.FuncId, instr_map, symbolizer, map->getFunctionAddresses(), cache); + if (!func_name) + return func_name.takeError(); + write_optrack_record(file_iter->second, *record_type, *func_name, record.TSC); + } + return llvm::Error::success(); +} + +} // namespace xray_to_optrack + +int +main(int argc, char **argv) +{ + if (argc != 3) { + llvm::errs() << "Usage: xray_to_optrack <instr_map> <xray_log>\n"; + return EXIT_FAILURE; + } + auto error = xray_to_optrack::xray_to_optrack(argv[1], argv[2]); + if (error) { + llvm::errs() << "Error: " << error << ".\n"; + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} |