diff options
| author | Alex Gorrod <alexander.gorrod@mongodb.com> | 2017-07-20 12:20:46 +1000 |
|---|---|---|
| committer | Alex Gorrod <alexander.gorrod@mongodb.com> | 2017-07-20 12:24:11 +1000 |
| commit | 5a2533ebc3606973fbe237228b9bacdcb21a532b (patch) | |
| tree | 66805f578f761446f1cdd6637dcfaa90c60e1388 /src/third_party/wiredtiger/src | |
| parent | 634435949c4b855b9cc5bfbf5cf481d8158fd996 (diff) | |
| download | mongo-5a2533ebc3606973fbe237228b9bacdcb21a532b.tar.gz | |
Import wiredtiger: 6173a98979715ed727c432c1a31da64ea8a37048 from branch mongodb-3.6
ref: ff10db8811..6173a98979
for: 3.5.11
WT-3039 Enhance logging so new log files no longer need to wait for the previous log file to be synced
WT-3138 Enhance eviction statistics
WT-3140 Enhance json mode statistics to better support per-dhandle statistics
WT-3310 Add support to WT_SESSION::alter to change table log setting
WT-3329 With a uniform workload and a number of small collections, eviction does a poor job of selecting candidates for eviction.
WT-3381 Improve concurrency in the transaction subsystem
WT-3394 Build WiredTiger with gcc7
WT-3396 test/csuite/wt3363_checkpoint_op_races, test_utility cleanups
WT-3397 Coverity lint
WT-3398 in-memory configurations don't support schema_worker operations
WT-3401 Lint and minor cleanup
WT-3402 Move cached overflow records to the update list.
WT-3403 Automated tests timeout running csuite
WT-3409 WiredTiger generations can silently self-deadlock.
WT-3413 Add more aggressive compile warning flags to Jenkins Windows job
WT-3415 setting timestamp in txn_state requires the global write lock
WT-3421 Compilation error unreachable code in test/csuite/wt3363_checkpoint_op_races
WT-3424 additional gcc 7.1 compile warnings.
WT-3425 workgen: add ability to reopen cursors
WT-3426 Add new wtperf configuration to automated performance testing
WT-3438 Don't tune eviction thread count when the count is fixed
Also explicitly disable v2 log records via a configuration setting in MongoDB -
this will later be plumbed into the feature compatability version mechanism.
Diffstat (limited to 'src/third_party/wiredtiger/src')
64 files changed, 2562 insertions, 1556 deletions
diff --git a/src/third_party/wiredtiger/src/async/async_op.c b/src/third_party/wiredtiger/src/async/async_op.c index d4ca754b95f..dc8a3cdf348 100644 --- a/src/third_party/wiredtiger/src/async/async_op.c +++ b/src/third_party/wiredtiger/src/async/async_op.c @@ -7,6 +7,7 @@ */ #include "wt_internal.h" + /* * __async_get_key -- * WT_ASYNC_OP->get_key implementation for op handles. diff --git a/src/third_party/wiredtiger/src/async/async_worker.c b/src/third_party/wiredtiger/src/async/async_worker.c index 57ebe5d8bb1..2626a21435f 100644 --- a/src/third_party/wiredtiger/src/async/async_worker.c +++ b/src/third_party/wiredtiger/src/async/async_worker.c @@ -215,7 +215,7 @@ __async_worker_execop(WT_SESSION_IMPL *session, WT_ASYNC_OP_IMPL *op, break; case WT_AOP_NONE: WT_RET_MSG(session, EINVAL, - "Unknown async optype %d", op->optype); + "Unknown async optype %d", (int)op->optype); } return (0); } diff --git a/src/third_party/wiredtiger/src/block/block_compact.c b/src/third_party/wiredtiger/src/block/block_compact.c index e7b9beafb01..76c4b1de3ec 100644 --- a/src/third_party/wiredtiger/src/block/block_compact.c +++ b/src/third_party/wiredtiger/src/block/block_compact.c @@ -8,7 +8,9 @@ #include "wt_internal.h" +#ifdef HAVE_VERBOSE static void __block_dump_avail(WT_SESSION_IMPL *, WT_BLOCK *, bool); +#endif /* * __wt_block_compact_start -- @@ -78,9 +80,11 @@ __wt_block_compact_skip(WT_SESSION_IMPL *session, WT_BLOCK *block, bool *skipp) __wt_spin_lock(session, &block->live_lock); +#ifdef HAVE_VERBOSE /* Dump the current state of the file. */ if (WT_VERBOSE_ISSET(session, WT_VERB_COMPACT)) __block_dump_avail(session, block, true); +#endif /* Sum the available bytes in the initial 80% and 90% of the file. */ avail_eighty = avail_ninety = 0; @@ -195,6 +199,7 @@ __wt_block_compact_page_skip(WT_SESSION_IMPL *session, return (0); } +#ifdef HAVE_VERBOSE /* * __block_dump_avail -- * Dump out the avail list so we can see what compaction will look like. @@ -271,3 +276,4 @@ __block_dump_avail(WT_SESSION_IMPL *session, WT_BLOCK *block, bool start) (uintmax_t)((v * 100) / (wt_off_t)el->bytes)); } } +#endif diff --git a/src/third_party/wiredtiger/src/block/block_vrfy.c b/src/third_party/wiredtiger/src/block/block_vrfy.c index 1058f16bde6..d2ca561fe82 100644 --- a/src/third_party/wiredtiger/src/block/block_vrfy.c +++ b/src/third_party/wiredtiger/src/block/block_vrfy.c @@ -33,7 +33,7 @@ int __wt_block_verify_start(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckptbase, const char *cfg[]) { - WT_CKPT *ckpt; + WT_CKPT *ckpt, *t; WT_CONFIG_ITEM cval; wt_off_t size; @@ -50,14 +50,12 @@ __wt_block_verify_start(WT_SESSION_IMPL *session, * checkpoint we have is fake, there's no work to do. Don't complain, * that's not our problem to solve. */ - WT_CKPT_FOREACH(ckptbase, ckpt) - ; - for (;; --ckpt) { - if (ckpt->name != NULL && !F_ISSET(ckpt, WT_CKPT_FAKE)) - break; - if (ckpt == ckptbase) - return (0); - } + ckpt = NULL; + WT_CKPT_FOREACH(ckptbase, t) + if (t->name != NULL && !F_ISSET(t, WT_CKPT_FAKE)) + ckpt = t; + if (ckpt == NULL) + return (0); /* Set the size of the file to the size of the last checkpoint. */ WT_RET(__verify_set_file_size(session, block, ckpt)); @@ -387,6 +385,8 @@ __verify_filefrag_add(WT_SESSION_IMPL *session, WT_BLOCK *block, { uint64_t f, frag, frags, i; + WT_UNUSED(type); /* !HAVE_VERBOSE */ + __wt_verbose(session, WT_VERB_VERIFY, "add file block%s%s%s at %" PRIuMAX "-%" PRIuMAX " (%" PRIuMAX ")", type == NULL ? "" : " (", diff --git a/src/third_party/wiredtiger/src/btree/bt_debug.c b/src/third_party/wiredtiger/src/btree/bt_debug.c index 394ac6c7b84..c0aaf3f42d9 100644 --- a/src/third_party/wiredtiger/src/btree/bt_debug.c +++ b/src/third_party/wiredtiger/src/btree/bt_debug.c @@ -445,18 +445,28 @@ static char * __debug_tree_shape_info(WT_PAGE *page) { uint64_t v; - static char buf[32]; + static char buf[128]; + const char *unit; v = page->memory_footprint; - if (v >= WT_GIGABYTE) - (void)__wt_snprintf(buf, sizeof(buf), - "(%p %" PRIu64 "G)", (void *)page, v / WT_GIGABYTE); - else if (v >= WT_MEGABYTE) - (void)__wt_snprintf(buf, sizeof(buf), - "(%p %" PRIu64 "M)", (void *)page, v / WT_MEGABYTE); - else - (void)__wt_snprintf(buf, sizeof(buf), - "(%p %" PRIu64 ")", (void *)page, v); + + if (v > WT_GIGABYTE) { + v /= WT_GIGABYTE; + unit = "G"; + } else if (v > WT_MEGABYTE) { + v /= WT_MEGABYTE; + unit = "M"; + } else if (v > WT_KILOBYTE) { + v /= WT_KILOBYTE; + unit = "K"; + } else { + unit = "B"; + } + + (void)__wt_snprintf(buf, sizeof(buf), "(%p, %" PRIu64 + "%s, evict gen %" PRIu64 ", create gen %" PRIu64 ")", + (void *)page, v, unit, + page->evict_pass_gen, page->cache_create_gen); return (buf); } @@ -982,7 +992,7 @@ __debug_row_skip(WT_DBG *ds, WT_INSERT_HEAD *head) static int __debug_update(WT_DBG *ds, WT_UPDATE *upd, bool hexbyte) { - for (; upd != NULL; upd = upd->next) + for (; upd != NULL; upd = upd->next) { if (upd->type == WT_UPDATE_DELETED) WT_RET(ds->f(ds, "\tvalue {deleted}\n")); else if (upd->type == WT_UPDATE_RESERVED) @@ -995,6 +1005,30 @@ __debug_update(WT_DBG *ds, WT_UPDATE *upd, bool hexbyte) } else WT_RET(__debug_item(ds, "value", WT_UPDATE_DATA(upd), upd->size)); + WT_RET(ds->f(ds, "\t" "txn id %" PRIu64, upd->txnid)); + +#ifdef HAVE_TIMESTAMPS + if (!__wt_timestamp_iszero(upd->timestamp)) { +#if WT_TIMESTAMP_SIZE == 8 + { + uint64_t ts; + __wt_timestamp_set( + (uint8_t *)&ts, (uint8_t *)&upd->timestamp[0]); + ts = __wt_bswap64(ts); + WT_RET(ds->f(ds, ", stamp %" PRIu64, ts)); + } +#else + { + int i; + WT_RET(ds->f(ds, ", stamp 0x")); + for (i = 0; i < WT_TIMESTAMP_SIZE; ++i) + WT_RET(ds->f(ds, "%" PRIx8, upd->timestamp[i])); + } +#endif + } +#endif + WT_RET(ds->f(ds, "\n")); + } return (0); } diff --git a/src/third_party/wiredtiger/src/btree/bt_discard.c b/src/third_party/wiredtiger/src/btree/bt_discard.c index bfa8eb25aac..bc9356e2669 100644 --- a/src/third_party/wiredtiger/src/btree/bt_discard.c +++ b/src/third_party/wiredtiger/src/btree/bt_discard.c @@ -249,7 +249,8 @@ __free_page_modify(WT_SESSION_IMPL *session, WT_PAGE *page) /* Free the overflow on-page, reuse and transaction-cache skiplists. */ __wt_ovfl_reuse_free(session, page); - __wt_ovfl_txnc_free(session, page); + if (mod->ovfl_track != NULL) + __wt_free(session, mod->ovfl_track->remove); __wt_ovfl_discard_free(session, page); __wt_free(session, page->modify->ovfl_track); diff --git a/src/third_party/wiredtiger/src/btree/bt_handle.c b/src/third_party/wiredtiger/src/btree/bt_handle.c index 06fbd6b74c7..5fad76849dc 100644 --- a/src/third_party/wiredtiger/src/btree/bt_handle.c +++ b/src/third_party/wiredtiger/src/btree/bt_handle.c @@ -279,6 +279,9 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt) bool fixed; const char **cfg, *enc_cfg[] = { NULL, NULL }; + WT_UNUSED(maj_version); /* !HAVE_VERBOSE */ + WT_UNUSED(min_version); /* !HAVE_VERBOSE */ + btree = S2BT(session); cfg = btree->dhandle->cfg; conn = S2C(session); @@ -290,7 +293,7 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt) WT_RET(__wt_config_gets(session, cfg, "version.minor", &cval)); min_version = cval.val; __wt_verbose(session, WT_VERB_VERSION, - "%" PRIu64 ".%" PRIu64, maj_version, min_version); + "%" PRId64 ".%" PRId64, maj_version, min_version); } /* Get the file ID. */ diff --git a/src/third_party/wiredtiger/src/btree/bt_ovfl.c b/src/third_party/wiredtiger/src/btree/bt_ovfl.c index 76d1bfd121c..a0b1ff65006 100644 --- a/src/third_party/wiredtiger/src/btree/bt_ovfl.c +++ b/src/third_party/wiredtiger/src/btree/bt_ovfl.c @@ -48,6 +48,9 @@ __wt_ovfl_read(WT_SESSION_IMPL *session, WT_PAGE *page, WT_CELL_UNPACK *unpack, WT_ITEM *store) { WT_DECL_RET; + WT_OVFL_TRACK *track; + WT_UPDATE *upd; + size_t i; /* * If no page specified, there's no need to lock and there's no cache @@ -61,16 +64,24 @@ __wt_ovfl_read(WT_SESSION_IMPL *session, * WT_CELL_VALUE_OVFL_RM cells: If reconciliation deleted an overflow * value, but there was still a reader in the system that might need it, * the on-page cell type will have been reset to WT_CELL_VALUE_OVFL_RM - * and we will be passed a page so we can look-aside into the cache of - * such values. + * and we will be passed a page so we can check the on-page cell. * * Acquire the overflow lock, and retest the on-page cell's value inside * the lock. */ __wt_readlock(session, &S2BT(session)->ovfl_lock); - ret = __wt_cell_type_raw(unpack->cell) == WT_CELL_VALUE_OVFL_RM ? - __wt_ovfl_txnc_search(page, unpack->data, unpack->size, store) : - __ovfl_read(session, unpack->data, unpack->size, store); + if (__wt_cell_type_raw(unpack->cell) == WT_CELL_VALUE_OVFL_RM) { + track = page->modify->ovfl_track; + for (upd = NULL, i = 0; i < track->remove_next; ++i) + if (track->remove[i].cell == unpack->cell) { + upd = track->remove[i].upd; + break; + } + WT_ASSERT(session, i < track->remove_next); + store->data = WT_UPDATE_DATA(upd); + store->size = upd->size; + } else + ret = __ovfl_read(session, unpack->data, unpack->size, store); __wt_readunlock(session, &S2BT(session)->ovfl_lock); return (ret); @@ -98,8 +109,7 @@ __ovfl_cache_col_visible( * the value. */ if (__wt_cell_rle(unpack) == 1 && - upd != NULL && /* Sanity: upd should always be set. */ - __wt_txn_upd_visible_all(session, upd)) + WT_UPDATE_DATA_VALUE(upd) && __wt_txn_upd_visible_all(session, upd)) return (true); return (false); } @@ -109,51 +119,99 @@ __ovfl_cache_col_visible( * row-store: check for a globally visible update. */ static bool -__ovfl_cache_row_visible(WT_SESSION_IMPL *session, WT_PAGE *page, WT_ROW *rip) +__ovfl_cache_row_visible(WT_SESSION_IMPL *session, WT_UPDATE *upd) { - WT_UPDATE *upd; - /* Check to see if there's a globally visible update. */ - for (upd = WT_ROW_UPDATE(page, rip); upd != NULL; upd = upd->next) - if (__wt_txn_upd_visible_all(session, upd)) + for (; upd != NULL; upd = upd->next) + if (WT_UPDATE_DATA_VALUE(upd) && + __wt_txn_upd_visible_all(session, upd)) return (true); return (false); } /* - * __ovfl_cache -- - * Cache a deleted overflow value. + * __ovfl_cache_append_update -- + * Append an overflow value to the update list. */ static int -__ovfl_cache(WT_SESSION_IMPL *session, WT_PAGE *page, WT_CELL_UNPACK *unpack) +__ovfl_cache_append_update(WT_SESSION_IMPL *session, WT_PAGE *page, + WT_UPDATE *upd_list, WT_CELL_UNPACK *unpack, WT_UPDATE **updp) { WT_DECL_ITEM(tmp); WT_DECL_RET; - size_t addr_size; - const uint8_t *addr; + WT_UPDATE *append, *upd; + size_t size; - addr = unpack->data; - addr_size = unpack->size; + *updp = NULL; + /* Read the overflow value. */ WT_RET(__wt_scr_alloc(session, 1024, &tmp)); + WT_ERR(__ovfl_read(session, unpack->data, unpack->size, tmp)); + + /* + * Create an update entry with no transaction ID to ensure global + * visibility, append it to the update list. + * + * We don't need locks or barriers in this function: any thread reading + * the update list will see our newly appended record or not, it doesn't + * matter until the on-page cell is set to WT_CELL_VALUE_OVFL_RM. That + * involves atomic operations which will act as our barrier. Regardless, + * we update the page footprint as part of this operation, which acts as + * a barrier as well. + */ + WT_ERR(__wt_update_alloc( + session, tmp, &append, &size, WT_UPDATE_STANDARD)); + append->txnid = WT_TXN_NONE; + for (upd = upd_list; upd->next != NULL; upd = upd->next) + ; + WT_PUBLISH(upd->next, append); + + __wt_cache_page_inmem_incr(session, page, size); - /* Enter the value into the overflow cache. */ - WT_ERR(__ovfl_read(session, addr, addr_size, tmp)); - WT_ERR(__wt_ovfl_txnc_add( - session, page, addr, addr_size, tmp->data, tmp->size)); + *updp = append; err: __wt_scr_free(session, &tmp); return (ret); } /* - * __wt_ovfl_cache -- - * Handle deletion of an overflow value. + * __ovfl_cache -- + * Cache an overflow value. + */ +static int +__ovfl_cache(WT_SESSION_IMPL *session, + WT_PAGE *page, WT_UPDATE *upd_list, WT_CELL_UNPACK *unpack) +{ + WT_OVFL_TRACK *track; + WT_UPDATE *upd; + + /* Append a copy of the overflow value to the update list. */ + WT_RET(__ovfl_cache_append_update( + session, page, upd_list, unpack, &upd)); + + /* Allocating tracking structures as necessary. */ + if (page->modify->ovfl_track == NULL) + WT_RET(__wt_ovfl_track_init(session, page)); + track = page->modify->ovfl_track; + + /* Add the value's information to the update list. */ + WT_RET(__wt_realloc_def(session, + &track->remove_allocated, track->remove_next + 1, &track->remove)); + track->remove[track->remove_next].cell = unpack->cell; + track->remove[track->remove_next].upd = upd; + ++track->remove_next; + + return (0); +} + +/* + * __wt_ovfl_remove -- + * Remove an overflow value. */ int -__wt_ovfl_cache(WT_SESSION_IMPL *session, - WT_PAGE *page, void *cookie, WT_CELL_UNPACK *vpack) +__wt_ovfl_remove(WT_SESSION_IMPL *session, + WT_PAGE *page, WT_UPDATE *upd_list, WT_CELL_UNPACK *unpack) { bool visible; @@ -195,10 +253,10 @@ __wt_ovfl_cache(WT_SESSION_IMPL *session, */ switch (page->type) { case WT_PAGE_COL_VAR: - visible = __ovfl_cache_col_visible(session, cookie, vpack); + visible = __ovfl_cache_col_visible(session, upd_list, unpack); break; case WT_PAGE_ROW_LEAF: - visible = __ovfl_cache_row_visible(session, page, cookie); + visible = __ovfl_cache_row_visible(session, upd_list); break; WT_ILLEGAL_VALUE(session); } @@ -207,18 +265,15 @@ __wt_ovfl_cache(WT_SESSION_IMPL *session, * If there's no globally visible update, there's a reader in the system * that might try and read the old value, cache it. */ - if (!visible) { - WT_RET(__ovfl_cache(session, page, vpack)); - WT_STAT_CONN_INCR(session, cache_overflow_value); - WT_STAT_DATA_INCR(session, cache_overflow_value); - } + if (!visible) + WT_RET(__ovfl_cache(session, page, upd_list, unpack)); /* * Queue the on-page cell to be set to WT_CELL_VALUE_OVFL_RM and the * underlying overflow value's blocks to be freed when reconciliation * completes. */ - return (__wt_ovfl_discard_add(session, page, vpack->cell)); + return (__wt_ovfl_discard_add(session, page, unpack->cell)); } /* diff --git a/src/third_party/wiredtiger/src/btree/bt_page.c b/src/third_party/wiredtiger/src/btree/bt_page.c index ca5f05fe3dc..d7643e9ad26 100644 --- a/src/third_party/wiredtiger/src/btree/bt_page.c +++ b/src/third_party/wiredtiger/src/btree/bt_page.c @@ -116,6 +116,7 @@ err: if ((pindex = WT_INTL_INDEX_GET_SAFE(page)) != NULL) { __wt_cache_page_inmem_incr(session, page, size); (void)__wt_atomic_add64(&cache->bytes_read, size); (void)__wt_atomic_add64(&cache->pages_inmem, 1); + page->cache_create_gen = cache->evict_pass_gen; *pagep = page; return (0); diff --git a/src/third_party/wiredtiger/src/btree/bt_read.c b/src/third_party/wiredtiger/src/btree/bt_read.c index 5f8cf3a45e1..6a89f505c31 100644 --- a/src/third_party/wiredtiger/src/btree/bt_read.c +++ b/src/third_party/wiredtiger/src/btree/bt_read.c @@ -710,7 +710,7 @@ __btree_verbose_lookaside_read(WT_SESSION_IMPL *session) if (__wt_atomic_casv64(&conn->las_verb_gen_read, ckpt_gen_last, ckpt_gen_current)) { __wt_verbose(session, WT_VERB_LOOKASIDE, - "Read from lookaside file triggered."); + "%s", "Read from lookaside file triggered."); } } #else diff --git a/src/third_party/wiredtiger/src/btree/bt_slvg.c b/src/third_party/wiredtiger/src/btree/bt_slvg.c index eb39301abc7..1f686d904ec 100644 --- a/src/third_party/wiredtiger/src/btree/bt_slvg.c +++ b/src/third_party/wiredtiger/src/btree/bt_slvg.c @@ -1885,6 +1885,8 @@ __slvg_row_build_leaf( uint32_t i, skip_start, skip_stop; int cmp; + WT_UNUSED(ss); /* !HAVE_VERBOSE */ + btree = S2BT(session); page = NULL; diff --git a/src/third_party/wiredtiger/src/btree/row_srch.c b/src/third_party/wiredtiger/src/btree/row_srch.c index 76bebde7de7..3a9a6eb0f9b 100644 --- a/src/third_party/wiredtiger/src/btree/row_srch.c +++ b/src/third_party/wiredtiger/src/btree/row_srch.c @@ -591,13 +591,13 @@ leaf_match: cbt->compare = 0; */ if (base == 0) { cbt->compare = 1; - cbt->slot = WT_ROW_SLOT(page, page->pg_row); + cbt->slot = 0; F_SET(cbt, WT_CBT_SEARCH_SMALLEST); ins_head = WT_ROW_INSERT_SMALLEST(page); } else { cbt->compare = -1; - cbt->slot = WT_ROW_SLOT(page, page->pg_row + (base - 1)); + cbt->slot = base - 1; ins_head = WT_ROW_INSERT_SLOT(page, cbt->slot); } diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 30b2b3e0e4d..dda27fec57c 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -55,6 +55,12 @@ static const WT_CONFIG_CHECK }; static const WT_CONFIG_CHECK + confchk_wiredtiger_open_compatibility_subconfigs[] = { + { "release", "string", NULL, NULL, NULL, 0 }, + { NULL, NULL, NULL, NULL, NULL, 0 } +}; + +static const WT_CONFIG_CHECK confchk_wiredtiger_open_eviction_subconfigs[] = { { "threads_max", "int", NULL, "min=1,max=20", NULL, 0 }, { "threads_min", "int", NULL, "min=1,max=20", NULL, 0 }, @@ -117,6 +123,9 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { { "checkpoint", "category", NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2 }, + { "compatibility", "category", + NULL, NULL, + confchk_wiredtiger_open_compatibility_subconfigs, 1 }, { "diagnostic_timing_stress", "list", NULL, "choices=[\"checkpoint_slow\"]", NULL, 0 }, @@ -179,11 +188,20 @@ static const WT_CONFIG_CHECK confchk_WT_CURSOR_reconfigure[] = { { NULL, NULL, NULL, NULL, NULL, 0 } }; +static const WT_CONFIG_CHECK + confchk_WT_SESSION_create_log_subconfigs[] = { + { "enabled", "boolean", NULL, NULL, NULL, 0 }, + { NULL, NULL, NULL, NULL, NULL, 0 } +}; + static const WT_CONFIG_CHECK confchk_WT_SESSION_alter[] = { { "access_pattern_hint", "string", NULL, "choices=[\"none\",\"random\",\"sequential\"]", NULL, 0 }, { "cache_resident", "boolean", NULL, NULL, NULL, 0 }, + { "log", "category", + NULL, NULL, + confchk_WT_SESSION_create_log_subconfigs, 1 }, { NULL, NULL, NULL, NULL, NULL, 0 } }; @@ -230,12 +248,6 @@ static const WT_CONFIG_CHECK }; static const WT_CONFIG_CHECK - confchk_WT_SESSION_create_log_subconfigs[] = { - { "enabled", "boolean", NULL, NULL, NULL, 0 }, - { NULL, NULL, NULL, NULL, NULL, 0 } -}; - -static const WT_CONFIG_CHECK confchk_WT_SESSION_create_lsm_subconfigs[] = { { "auto_throttle", "boolean", NULL, NULL, NULL, 0 }, { "bloom", "boolean", NULL, NULL, NULL, 0 }, @@ -712,6 +724,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2 }, { "checkpoint_sync", "boolean", NULL, NULL, NULL, 0 }, + { "compatibility", "category", + NULL, NULL, + confchk_wiredtiger_open_compatibility_subconfigs, 1 }, { "config_base", "boolean", NULL, NULL, NULL, 0 }, { "create", "boolean", NULL, NULL, NULL, 0 }, { "diagnostic_timing_stress", "list", @@ -803,6 +818,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2 }, { "checkpoint_sync", "boolean", NULL, NULL, NULL, 0 }, + { "compatibility", "category", + NULL, NULL, + confchk_wiredtiger_open_compatibility_subconfigs, 1 }, { "config_base", "boolean", NULL, NULL, NULL, 0 }, { "create", "boolean", NULL, NULL, NULL, 0 }, { "diagnostic_timing_stress", "list", @@ -895,6 +913,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2 }, { "checkpoint_sync", "boolean", NULL, NULL, NULL, 0 }, + { "compatibility", "category", + NULL, NULL, + confchk_wiredtiger_open_compatibility_subconfigs, 1 }, { "diagnostic_timing_stress", "list", NULL, "choices=[\"checkpoint_slow\"]", NULL, 0 }, @@ -981,6 +1002,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2 }, { "checkpoint_sync", "boolean", NULL, NULL, NULL, 0 }, + { "compatibility", "category", + NULL, NULL, + confchk_wiredtiger_open_compatibility_subconfigs, 1 }, { "diagnostic_timing_stress", "list", NULL, "choices=[\"checkpoint_slow\"]", NULL, 0 }, @@ -1099,18 +1123,18 @@ static const WT_CONFIG_ENTRY config_entries[] = { { "WT_CONNECTION.reconfigure", "async=(enabled=false,ops_max=1024,threads=2),cache_overhead=8," "cache_size=100MB,checkpoint=(log_size=0,wait=0)," - "diagnostic_timing_stress=,error_prefix=,eviction=(threads_max=8," - "threads_min=1),eviction_checkpoint_target=5," - "eviction_dirty_target=5,eviction_dirty_trigger=20," - "eviction_target=80,eviction_trigger=95," - "file_manager=(close_handle_minimum=250,close_idle_time=30," + "compatibility=(release=),diagnostic_timing_stress=,error_prefix=" + ",eviction=(threads_max=8,threads_min=1)," + "eviction_checkpoint_target=5,eviction_dirty_target=5," + "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" + ",file_manager=(close_handle_minimum=250,close_idle_time=30," "close_scan_interval=10),log=(archive=true,prealloc=true," "zero_fill=false),lsm_manager=(merge=true,worker_thread_max=4)," "lsm_merge=true,shared_cache=(chunk=10MB,name=,quota=0,reserve=0," "size=500MB),statistics=none,statistics_log=(json=false," "on_close=false,sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "verbose=", - confchk_WT_CONNECTION_reconfigure, 20 + confchk_WT_CONNECTION_reconfigure, 21 }, { "WT_CONNECTION.set_file_system", "", @@ -1129,8 +1153,8 @@ static const WT_CONFIG_ENTRY config_entries[] = { confchk_WT_CURSOR_reconfigure, 2 }, { "WT_SESSION.alter", - "access_pattern_hint=none,cache_resident=false", - confchk_WT_SESSION_alter, 2 + "access_pattern_hint=none,cache_resident=false,log=(enabled=true)", + confchk_WT_SESSION_alter, 3 }, { "WT_SESSION.begin_transaction", "isolation=,name=,priority=0,read_timestamp=,snapshot=,sync=", @@ -1318,10 +1342,10 @@ static const WT_CONFIG_ENTRY config_entries[] = { "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" ",builtin_extension_config=,cache_overhead=8,cache_size=100MB," "checkpoint=(log_size=0,wait=0),checkpoint_sync=true," - "config_base=true,create=false,diagnostic_timing_stress=," - "direct_io=,encryption=(keyid=,name=,secretkey=),error_prefix=," - "eviction=(threads_max=8,threads_min=1)," - "eviction_checkpoint_target=5,eviction_dirty_target=5," + "compatibility=(release=),config_base=true,create=false," + "diagnostic_timing_stress=,direct_io=,encryption=(keyid=,name=," + "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" + ",eviction_checkpoint_target=5,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",exclusive=false,extensions=,file_extend=," "file_manager=(close_handle_minimum=250,close_idle_time=30," @@ -1336,16 +1360,16 @@ static const WT_CONFIG_ENTRY config_entries[] = { ",wait=0),transaction_sync=(enabled=false,method=fsync)," "use_environment=true,use_environment_priv=false,verbose=," "write_through=", - confchk_wiredtiger_open, 41 + confchk_wiredtiger_open, 42 }, { "wiredtiger_open_all", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" ",builtin_extension_config=,cache_overhead=8,cache_size=100MB," "checkpoint=(log_size=0,wait=0),checkpoint_sync=true," - "config_base=true,create=false,diagnostic_timing_stress=," - "direct_io=,encryption=(keyid=,name=,secretkey=),error_prefix=," - "eviction=(threads_max=8,threads_min=1)," - "eviction_checkpoint_target=5,eviction_dirty_target=5," + "compatibility=(release=),config_base=true,create=false," + "diagnostic_timing_stress=,direct_io=,encryption=(keyid=,name=," + "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" + ",eviction_checkpoint_target=5,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",exclusive=false,extensions=,file_extend=," "file_manager=(close_handle_minimum=250,close_idle_time=30," @@ -1360,15 +1384,16 @@ static const WT_CONFIG_ENTRY config_entries[] = { ",wait=0),transaction_sync=(enabled=false,method=fsync)," "use_environment=true,use_environment_priv=false,verbose=," "version=(major=0,minor=0),write_through=", - confchk_wiredtiger_open_all, 42 + confchk_wiredtiger_open_all, 43 }, { "wiredtiger_open_basecfg", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" ",builtin_extension_config=,cache_overhead=8,cache_size=100MB," "checkpoint=(log_size=0,wait=0),checkpoint_sync=true," - "diagnostic_timing_stress=,direct_io=,encryption=(keyid=,name=," - "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" - ",eviction_checkpoint_target=5,eviction_dirty_target=5," + "compatibility=(release=),diagnostic_timing_stress=,direct_io=," + "encryption=(keyid=,name=,secretkey=),error_prefix=," + "eviction=(threads_max=8,threads_min=1)," + "eviction_checkpoint_target=5,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",extensions=,file_extend=,file_manager=(close_handle_minimum=250" ",close_idle_time=30,close_scan_interval=10),hazard_max=1000," @@ -1381,15 +1406,16 @@ static const WT_CONFIG_ENTRY config_entries[] = { ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" ",wait=0),transaction_sync=(enabled=false,method=fsync),verbose=," "version=(major=0,minor=0),write_through=", - confchk_wiredtiger_open_basecfg, 36 + confchk_wiredtiger_open_basecfg, 37 }, { "wiredtiger_open_usercfg", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" ",builtin_extension_config=,cache_overhead=8,cache_size=100MB," "checkpoint=(log_size=0,wait=0),checkpoint_sync=true," - "diagnostic_timing_stress=,direct_io=,encryption=(keyid=,name=," - "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" - ",eviction_checkpoint_target=5,eviction_dirty_target=5," + "compatibility=(release=),diagnostic_timing_stress=,direct_io=," + "encryption=(keyid=,name=,secretkey=),error_prefix=," + "eviction=(threads_max=8,threads_min=1)," + "eviction_checkpoint_target=5,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",extensions=,file_extend=,file_manager=(close_handle_minimum=250" ",close_idle_time=30,close_scan_interval=10),hazard_max=1000," @@ -1402,7 +1428,7 @@ static const WT_CONFIG_ENTRY config_entries[] = { ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" ",wait=0),transaction_sync=(enabled=false,method=fsync),verbose=," "write_through=", - confchk_wiredtiger_open_usercfg, 35 + confchk_wiredtiger_open_usercfg, 36 }, { NULL, NULL, NULL, 0 } }; diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index cb0aa17791e..09eceac0a3b 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -190,6 +190,45 @@ __wt_conn_remove_collator(WT_SESSION_IMPL *session) } /* + * __conn_compat_config -- + * Configure compatibility version. + */ +static int +__conn_compat_config(WT_SESSION_IMPL *session, const char **cfg) +{ + WT_CONFIG_ITEM cval; + WT_CONNECTION_IMPL *conn; + uint16_t patch; + + conn = S2C(session); + WT_RET(__wt_config_gets(session, cfg, + "compatibility.release", &cval)); + if (cval.len != 0) { + /* + * Accept either a major.minor release string or a + * major.minor.patch release string. We ignore the patch + * value, but allow it in the string. + */ + if (sscanf(cval.str, "%" SCNu16 ".%" SCNu16, + &conn->compat_major, &conn->compat_minor) != 2 && + sscanf(cval.str, "%" SCNu16 ".%" SCNu16 ".%" SCNu16, + &conn->compat_major, &conn->compat_minor, &patch) != 3) + WT_RET_MSG(session, + EINVAL, "illegal compatibility release"); + if (conn->compat_major > WIREDTIGER_VERSION_MAJOR) + WT_RET_MSG(session, EINVAL, "unknown major version"); + if (conn->compat_major == WIREDTIGER_VERSION_MAJOR && + conn->compat_minor > WIREDTIGER_VERSION_MINOR) + WT_RET_MSG(session, + EINVAL, "illegal compatibility version"); + } else { + conn->compat_major = WIREDTIGER_VERSION_MAJOR; + conn->compat_minor = WIREDTIGER_VERSION_MINOR; + } + return (0); +} + +/* * __compressor_confchk -- * Validate the compressor. */ @@ -1134,6 +1173,7 @@ __conn_reconfigure(WT_CONNECTION *wt_conn, const char *config) cfg[1] = config; /* Second, reconfigure the system. */ + WT_ERR(__conn_compat_config(session, cfg)); WT_ERR(__conn_statistics_config(session, cfg)); WT_ERR(__wt_async_reconfig(session, cfg)); WT_ERR(__wt_cache_config(session, true, cfg)); @@ -2018,6 +2058,7 @@ __conn_write_base_config(WT_SESSION_IMPL *session, const char *cfg[]) * merge the rest to be written. */ WT_ERR(__wt_config_merge(session, cfg + 1, + "compatibility=(release=)," "config_base=," "create=," "encryption=(secretkey=)," @@ -2287,6 +2328,11 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, WT_ERR(__conn_load_extensions(session, cfg, true)); /* + * Set compatibility versions early so that any subsystem sees it. + */ + WT_ERR(__conn_compat_config(session, cfg)); + + /* * If the application didn't configure its own file system, configure * one of ours. Check to ensure we have a valid file system. */ diff --git a/src/third_party/wiredtiger/src/conn/conn_cache_pool.c b/src/third_party/wiredtiger/src/conn/conn_cache_pool.c index adc2e2bffc3..55393a86ba1 100644 --- a/src/third_party/wiredtiger/src/conn/conn_cache_pool.c +++ b/src/third_party/wiredtiger/src/conn/conn_cache_pool.c @@ -372,8 +372,8 @@ __wt_conn_cache_pool_destroy(WT_SESSION_IMPL *session) } if (!F_ISSET(cp, WT_CACHE_POOL_ACTIVE)) { - __wt_verbose( - session, WT_VERB_SHARED_CACHE, "Destroying cache pool"); + __wt_verbose(session, + WT_VERB_SHARED_CACHE, "%s", "Destroying cache pool"); __wt_spin_lock(session, &__wt_process.spinlock); /* * We have been holding the pool lock - no connections could @@ -401,7 +401,7 @@ __wt_conn_cache_pool_destroy(WT_SESSION_IMPL *session) /* Notify other participants if we were managing */ if (FLD_ISSET(cache->pool_flags, WT_CACHE_POOL_MANAGER)) { cp->pool_managed = 0; - __wt_verbose(session, WT_VERB_SHARED_CACHE, + __wt_verbose(session, WT_VERB_SHARED_CACHE, "%s", "Shutting down shared cache manager connection"); } } @@ -585,8 +585,8 @@ __cache_pool_adjust(WT_SESSION_IMPL *session, if (WT_VERBOSE_ISSET(session, WT_VERB_SHARED_CACHE)) { __wt_verbose(session, - WT_VERB_SHARED_CACHE, "Cache pool distribution: "); - __wt_verbose(session, WT_VERB_SHARED_CACHE, + WT_VERB_SHARED_CACHE, "%s", "Cache pool distribution: "); + __wt_verbose(session, WT_VERB_SHARED_CACHE, "%s", "\t" "cache (MB), pressure, skips, busy, %% full:"); } @@ -731,7 +731,7 @@ __cache_pool_adjust(WT_SESSION_IMPL *session, cp->currently_used -= adjustment; } __wt_verbose(session, WT_VERB_SHARED_CACHE, - "Allocated %s%" PRId64 " to %s", + "Allocated %s%" PRIu64 " to %s", grow ? "" : "-", adjustment, entry->home); /* @@ -778,7 +778,7 @@ __wt_cache_pool_server(void *arg) if (__wt_atomic_cas8(&cp->pool_managed, 0, 1)) { FLD_SET(cache->pool_flags, WT_CACHE_POOL_MANAGER); __wt_verbose(session, WT_VERB_SHARED_CACHE, - "Cache pool switched manager thread"); + "%s", "Cache pool switched manager thread"); } /* diff --git a/src/third_party/wiredtiger/src/conn/conn_log.c b/src/third_party/wiredtiger/src/conn/conn_log.c index 37acbe4a1a4..a6290e4ed92 100644 --- a/src/third_party/wiredtiger/src/conn/conn_log.c +++ b/src/third_party/wiredtiger/src/conn/conn_log.c @@ -40,6 +40,114 @@ __logmgr_sync_cfg(WT_SESSION_IMPL *session, const char **cfg) } /* + * __logmgr_force_ckpt -- + * Force a checkpoint out, waiting for the checkpoint LSN in the log + * is up to the given log number. + */ +static int +__logmgr_force_ckpt(WT_SESSION_IMPL *session, uint32_t lognum) +{ + WT_CONNECTION_IMPL *conn; + WT_LOG *log; + WT_SESSION_IMPL *tmp_session; + int yield; + + conn = S2C(session); + log = conn->log; + yield = 0; + WT_RET(__wt_open_internal_session(conn, + "compatibility-reconfig", true, 0, &tmp_session)); + while (log->ckpt_lsn.l.file < lognum) { + /* + * Force a checkpoint to be written in the new log file and + * force the archiving of all previous log files. We do the + * checkpoint in the loop because the checkpoint LSN in the + * log record could still reflect the previous log file in + * cases such as the write LSN has not yet advanced into the + * new log file due to another group of threads still in + * progress with their slot copies or writes. + */ + WT_RET(tmp_session->iface.checkpoint( + &tmp_session->iface, "force=1")); + WT_RET(WT_SESSION_CHECK_PANIC(tmp_session)); + /* + * Only sleep in the rare case that we had to come through + * this loop more than once. + */ + if (yield++) { + WT_STAT_CONN_INCR(session, log_force_ckpt_sleep); + __wt_sleep(0, WT_THOUSAND); + } + } + WT_RET(tmp_session->iface.close(&tmp_session->iface, NULL)); + return (0); +} + +/* + * __logmgr_version -- + * Set up the versions in the log manager. + */ +static int +__logmgr_version(WT_SESSION_IMPL *session, bool reconfig) +{ + WT_CONNECTION_IMPL *conn; + WT_LOG *log; + bool downgrade; + uint32_t first_record, lognum; + uint16_t new_version; + + conn = S2C(session); + log = conn->log; + if (log == NULL) + return (0); + + /* + * Set the log file format versions based on compatibility versions + * set in the connection. We must set this before we call log_open + * to open or create a log file. + * + * Since the log version changed at a major release number we only need + * to check the major number, not the minor number in the compatibility + * setting. + */ + if (conn->compat_major < WT_LOG_V2) { + new_version = 1; + first_record = WT_LOG_END_HEADER; + downgrade = true; + } else { + new_version = WT_LOG_VERSION; + first_record = WT_LOG_END_HEADER + log->allocsize; + downgrade = false; + } + + /* + * If the version is the same, there is nothing to do. + */ + if (log->log_version == new_version) + return (0); + /* + * If we are reconfiguring and at a new version we need to force + * the log file to advance so that we write out a log file at the + * correct version. When we are downgrading we must force a checkpoint + * and finally archive, even if disabled, so that all new version log + * files are gone. + * + * All of the version changes must be handled with locks on reconfigure + * because other threads may be changing log files, using pre-allocated + * files. + */ + /* + * Set the version. If it is a live change the logging subsystem will + * do other work as well to move to a new log file. + */ + WT_RET(__wt_log_set_version(session, new_version, + first_record, downgrade, reconfig, &lognum)); + if (reconfig && FLD_ISSET(conn->log_flags, WT_CONN_LOG_DOWNGRADED)) + WT_RET(__logmgr_force_ckpt(session, lognum)); + return (0); +} + +/* * __logmgr_config -- * Parse and setup the logging server options. */ @@ -187,7 +295,8 @@ __wt_logmgr_reconfig(WT_SESSION_IMPL *session, const char **cfg) { bool dummy; - return (__logmgr_config(session, cfg, &dummy, true)); + WT_RET(__logmgr_config(session, cfg, &dummy, true)); + return (__logmgr_version(session, true)); } /* @@ -329,19 +438,17 @@ err: __wt_err(session, ret, "log pre-alloc server error"); * currently running. */ int -__wt_log_truncate_files( - WT_SESSION_IMPL *session, WT_CURSOR *cursor, const char *cfg[]) +__wt_log_truncate_files(WT_SESSION_IMPL *session, WT_CURSOR *cursor, bool force) { WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_LOG *log; uint32_t backup_file; - WT_UNUSED(cfg); conn = S2C(session); if (!FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)) return (0); - if (F_ISSET(conn, WT_CONN_SERVER_LOG) && + if (!force && F_ISSET(conn, WT_CONN_SERVER_LOG) && FLD_ISSET(conn->log_flags, WT_CONN_LOG_ARCHIVE)) WT_RET_MSG(session, EINVAL, "Attempt to archive manually while a server is running"); @@ -349,8 +456,10 @@ __wt_log_truncate_files( log = conn->log; backup_file = 0; - if (cursor != NULL) + if (cursor != NULL) { + WT_ASSERT(session, force == false); backup_file = WT_CURSOR_BACKUP_ID(cursor); + } WT_ASSERT(session, backup_file <= log->alloc_lsn.l.file); __wt_verbose(session, WT_VERB_LOG, "log_truncate_files: Archive once up to %" PRIu32, backup_file); @@ -828,7 +937,7 @@ __log_server(void *arg) session, &log->log_archive_lock); WT_ERR(ret); } else - __wt_verbose(session, WT_VERB_LOG, + __wt_verbose(session, WT_VERB_LOG, "%s", "log_archive: Blocked due to open " "log cursor holding archive lock"); } @@ -875,6 +984,7 @@ __wt_logmgr_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_RET(__wt_calloc_one(session, &conn->log)); log = conn->log; WT_RET(__wt_spin_init(session, &log->log_lock, "log")); + WT_RET(__wt_spin_init(session, &log->log_fs_lock, "log files")); WT_RET(__wt_spin_init(session, &log->log_slot_lock, "log slot")); WT_RET(__wt_spin_init(session, &log->log_sync_lock, "log sync")); WT_RET(__wt_spin_init(session, &log->log_writelsn_lock, @@ -898,6 +1008,8 @@ __wt_logmgr_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_INIT_LSN(&log->write_lsn); WT_INIT_LSN(&log->write_start_lsn); log->fileid = 0; + WT_RET(__logmgr_version(session, false)); + WT_RET(__wt_cond_alloc(session, "log sync", &log->log_sync_cond)); WT_RET(__wt_cond_alloc(session, "log write", &log->log_write_cond)); WT_RET(__wt_log_open(session)); @@ -1050,6 +1162,7 @@ __wt_logmgr_destroy(WT_SESSION_IMPL *session) __wt_cond_destroy(session, &conn->log->log_write_cond); __wt_rwlock_destroy(session, &conn->log->log_archive_lock); __wt_spin_destroy(session, &conn->log->log_lock); + __wt_spin_destroy(session, &conn->log->log_fs_lock); __wt_spin_destroy(session, &conn->log->log_slot_lock); __wt_spin_destroy(session, &conn->log->log_sync_lock); __wt_spin_destroy(session, &conn->log->log_writelsn_lock); diff --git a/src/third_party/wiredtiger/src/conn/conn_stat.c b/src/third_party/wiredtiger/src/conn/conn_stat.c index f38d81a7f7a..4559a72a46e 100644 --- a/src/third_party/wiredtiger/src/conn/conn_stat.c +++ b/src/third_party/wiredtiger/src/conn/conn_stat.c @@ -221,6 +221,101 @@ err: __stat_sources_free(session, &sources); } /* + * __statlog_print_header -- + * Write the header for statistics when running in JSON mode. + */ +static int +__statlog_print_header(WT_SESSION_IMPL *session) +{ + WT_CONNECTION_IMPL *conn; + + conn = S2C(session); + + if (!FLD_ISSET(conn->stat_flags, WT_STAT_JSON)) + return (0); + + /* + * This flag is required in order to generate correct JSON when printing + * out stats for individual tables. When we are about to print the first + * table's stats we must print out the wiredTigerTables header once + * only and add a correct closing brace when we finish the tables + * section. To do this we maintain a flag variable to note when we have + * printed the first table. Unfortunately, the mechanism which we use + * to print stats for each table does not allow passing of variables + * by reference, this necessitates the use of a variable on the + * connection. The variable is safe as the JSON printing logic is only + * performed by the single threaded stat server. + */ + conn->stat_json_tables = false; + WT_RET(__wt_fprintf(session, conn->stat_fs, + "{\"version\":\"%s\",\"localTime\":\"%s\"", + WIREDTIGER_VERSION_STRING, conn->stat_stamp)); + return (0); +} + +/* + * __statlog_print_table_name -- + * Write the header for the wiredTigerTables section of statistics if + * running in JSON mode and the header has not been written this round, + * then print the name of the table. + */ +static int +__statlog_print_table_name( + WT_SESSION_IMPL *session, const char *name, bool conn_stats) +{ + WT_CONNECTION_IMPL *conn; + + conn = S2C(session); + + /* + * If printing the connection stats, write that header and we are done. + */ + if (conn_stats) { + WT_RET(__wt_fprintf( + session, conn->stat_fs, ",\"wiredTiger\":{")); + return (0); + } + + /* + * If this is the first table we are printing stats for print the header + * for the wiredTigerTables section. Otherwise print a comma as this + * is a subsequent table. + */ + if (conn->stat_json_tables) + WT_RET(__wt_fprintf(session, conn->stat_fs,",")); + else { + conn->stat_json_tables = true; + WT_RET(__wt_fprintf(session, + conn->stat_fs,",\"wiredTigerTables\":{")); + } + WT_RET(__wt_fprintf(session, conn->stat_fs, "\"%s\":{", name)); + return (0); +} + +/* + * __statlog_print_footer -- + * Write the footer for statistics when running in JSON mode. + */ +static int +__statlog_print_footer(WT_SESSION_IMPL *session) +{ + WT_CONNECTION_IMPL *conn; + + conn = S2C(session); + + if (!FLD_ISSET(conn->stat_flags, WT_STAT_JSON)) + return (0); + + /* If we have printed a tables stats, then close that section. */ + if (conn->stat_json_tables) { + WT_RET(__wt_fprintf(session, conn->stat_fs, "}")); + conn->stat_json_tables = false; + } + WT_RET(__wt_fprintf(session, conn->stat_fs, "}\n")); + return (0); +} + +/* * __statlog_dump -- * Dump out handle/connection statistics. */ @@ -264,14 +359,10 @@ __statlog_dump(WT_SESSION_IMPL *session, const char *name, bool conn_stats) goto err; } - if (FLD_ISSET(conn->stat_flags, WT_STAT_JSON)) { - WT_ERR(__wt_fprintf(session, conn->stat_fs, - "{\"version\":\"%s\",\"localTime\":\"%s\"", - WIREDTIGER_VERSION_STRING, conn->stat_stamp)); - WT_ERR(__wt_fprintf( - session, conn->stat_fs, ",\"wiredTiger\":{")); - while ((ret = cursor->next(cursor)) == 0) { - WT_ERR(cursor->get_value(cursor, &desc, &valstr, &val)); + WT_ERR(__statlog_print_table_name(session, name, conn_stats)); + while ((ret = cursor->next(cursor)) == 0) { + WT_ERR(cursor->get_value(cursor, &desc, &valstr, &val)); + if (FLD_ISSET(conn->stat_flags, WT_STAT_JSON)) { /* Check if we are starting a new section. */ endprefix = strchr(desc, ':'); prefixlen = WT_PTRDIFF(endprefix, desc); @@ -291,18 +382,15 @@ __statlog_dump(WT_SESSION_IMPL *session, const char *name, bool conn_stats) "%s\"%s\":%" PRId64, groupfirst ? "" : ",", endprefix + 2, val)); groupfirst = false; - } - WT_ERR_NOTFOUND_OK(ret); - WT_ERR(__wt_fprintf(session, conn->stat_fs, "}}}\n")); - } else { - while ((ret = cursor->next(cursor)) == 0) { - WT_ERR(cursor->get_value(cursor, &desc, &valstr, &val)); + } else { WT_ERR(__wt_fprintf(session, conn->stat_fs, "%s %" PRId64 " %s %s\n", conn->stat_stamp, val, name, desc)); } - WT_ERR_NOTFOUND_OK(ret); } + WT_ERR_NOTFOUND_OK(ret); + if (FLD_ISSET(conn->stat_flags, WT_STAT_JSON)) + WT_ERR(__wt_fprintf(session, conn->stat_fs, "}}")); err: __wt_scr_free(session, &tmp); if (cursor != NULL) @@ -437,6 +525,7 @@ __statlog_log_one(WT_SESSION_IMPL *session, WT_ITEM *path, WT_ITEM *tmp) if (strftime(tmp->mem, tmp->memsize, conn->stat_format, tm) == 0) WT_RET_MSG(session, ENOMEM, "strftime timestamp conversion"); conn->stat_stamp = tmp->mem; + WT_RET(__statlog_print_header(session)); /* Dump the connection statistics. */ WT_RET(__statlog_dump(session, conn->home, true)); @@ -459,6 +548,7 @@ __statlog_log_one(WT_SESSION_IMPL *session, WT_ITEM *path, WT_ITEM *tmp) */ if (conn->stat_sources != NULL) WT_RET(__statlog_lsm_apply(session)); + WT_RET(__statlog_print_footer(session)); /* Flush. */ return (__wt_fflush(session, conn->stat_fs)); diff --git a/src/third_party/wiredtiger/src/cursor/cur_backup.c b/src/third_party/wiredtiger/src/cursor/cur_backup.c index 60750b88900..490c3813cad 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_backup.c +++ b/src/third_party/wiredtiger/src/cursor/cur_backup.c @@ -210,6 +210,8 @@ __backup_start( cb->list = NULL; cb->list_next = 0; + WT_RET(__wt_inmem_unsupported_op(session, "backup cursor")); + /* * Single thread hot backups: we're holding the schema lock, so we * know we'll serialize with other attempts to start a hot backup. diff --git a/src/third_party/wiredtiger/src/cursor/cur_log.c b/src/third_party/wiredtiger/src/cursor/cur_log.c index 38e9d4a1784..d1166c5d402 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_log.c +++ b/src/third_party/wiredtiger/src/cursor/cur_log.c @@ -303,7 +303,6 @@ __curlog_close(WT_CURSOR *cursor) cl = (WT_CURSOR_LOG *)cursor; conn = S2C(session); - WT_ASSERT(session, FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)); if (F_ISSET(cl, WT_CURLOG_ARCHIVE_LOCK)) __wt_readunlock(session, &conn->log->log_archive_lock); @@ -355,9 +354,6 @@ __wt_curlog_open(WT_SESSION_IMPL *session, WT_STATIC_ASSERT(offsetof(WT_CURSOR_LOG, iface) == 0); conn = S2C(session); - if (!FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)) - WT_RET_MSG(session, EINVAL, - "Cannot open a log cursor without logging enabled"); log = conn->log; cl = NULL; @@ -378,15 +374,17 @@ __wt_curlog_open(WT_SESSION_IMPL *session, WT_ERR(__wt_cursor_init(cursor, uri, NULL, cfg, cursorp)); - /* - * The user may be trying to read a log record they just wrote. - * Log records may be buffered, so force out any now. - */ - WT_ERR(__wt_log_force_write(session, 1, NULL)); + if (log != NULL) { + /* + * The user may be trying to read a log record they just wrote. + * Log records may be buffered, so force out any now. + */ + WT_ERR(__wt_log_force_write(session, 1, NULL)); - /* Log cursors block archiving. */ - __wt_readlock(session, &log->log_archive_lock); - F_SET(cl, WT_CURLOG_ARCHIVE_LOCK); + /* Log cursors block archiving. */ + __wt_readlock(session, &log->log_archive_lock); + F_SET(cl, WT_CURLOG_ARCHIVE_LOCK); + } if (0) { err: WT_TRET(__curlog_close(cursor)); diff --git a/src/third_party/wiredtiger/src/docs/upgrading.dox b/src/third_party/wiredtiger/src/docs/upgrading.dox index 8640991e7cd..fc0fc4e022a 100644 --- a/src/third_party/wiredtiger/src/docs/upgrading.dox +++ b/src/third_party/wiredtiger/src/docs/upgrading.dox @@ -1,6 +1,29 @@ /*! @page upgrading Upgrading WiredTiger applications -@section version_293 Upgrading to Version 2.9.3 +@section version_300 Upgrading to Version 3.0.0 +<dl> + +<dt>WiredTiger on-disk log file format change</dt> +<dd> +The WiredTiger on-disk file format for write-ahead log files has changed +to include a new internal system log record. +</dd> + +<dt>Addition of compatibility configuration setting</dt> +<dd> +There is now a compatibility setting that allows the user to upgrade or +downgrade the log files generated so that they can be run with an older +release. +</dd> + +<dt>Cached overflow record statistics</dt> +<dd> +The \c cache_overflow_value ("overflow values cached in memory") statistic +has been removed, it no longer has any meaning. +</dd> + +</dl><hr> +@section version_292 Upgrading to Version 2.9.2 <dl> <dt>Logging subsystem statistics</dt> @@ -12,11 +35,7 @@ are no longer present. They were duplicates of \c log_writes and been added. </dd> -</dl><hr> -@section version_292 Upgrading to Version 2.9.2 -<dl> - -<dt>WiredTiger utility now supports truncate</dt> +<dt>WiredTiger Utility now supports truncate</dt> <dd> The WiredTiger utility \c wt can now \c truncate objects, removing all contents from the specified object. diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 4f4ecdf286c..1f26949c94f 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -305,12 +305,14 @@ __wt_evict_thread_run(WT_SESSION_IMPL *session, WT_THREAD *thread) F_ISSET(thread, WT_THREAD_RUN)) __wt_yield(); else { - __wt_verbose(session, WT_VERB_EVICTSERVER, "sleeping"); + __wt_verbose(session, + WT_VERB_EVICTSERVER, "%s", "sleeping"); /* Don't rely on signals: check periodically. */ __wt_cond_auto_wait(session, cache->evict_cond, did_work, NULL); - __wt_verbose(session, WT_VERB_EVICTSERVER, "waking"); + __wt_verbose(session, + WT_VERB_EVICTSERVER, "%s", "waking"); } } else WT_ERR(__evict_lru_pages(session, false)); @@ -351,8 +353,8 @@ __wt_evict_thread_stop(WT_SESSION_IMPL *session, WT_THREAD *thread) */ WT_ASSERT(session, F_ISSET(conn, WT_CONN_CLOSING | WT_CONN_RECOVERING)); - __wt_verbose( - session, WT_VERB_EVICTSERVER, "cache eviction thread exiting"); + __wt_verbose(session, + WT_VERB_EVICTSERVER, "%s", "cache eviction thread exiting"); if (0) { err: WT_PANIC_MSG(session, ret, "cache eviction thread error"); @@ -515,7 +517,7 @@ __wt_evict_destroy(WT_SESSION_IMPL *session) __wt_evict_server_wake(session); __wt_verbose( - session, WT_VERB_EVICTSERVER, "waiting for helper threads"); + session, WT_VERB_EVICTSERVER, "%s", "waiting for helper threads"); /* * We call the destroy function still holding the write lock. @@ -740,7 +742,7 @@ __evict_pass(WT_SESSION_IMPL *session) WT_STAT_CONN_INCR(session, cache_eviction_slow); __wt_verbose(session, WT_VERB_EVICTSERVER, - "unable to reach eviction goal"); + "%s", "unable to reach eviction goal"); break; } else { if (cache->evict_aggressive_score > 0) { @@ -779,6 +781,7 @@ __evict_clear_walk(WT_SESSION_IMPL *session) return (0); WT_STAT_CONN_INCR(session, cache_eviction_walks_abandoned); + WT_STAT_DATA_INCR(session, cache_eviction_walks_abandoned); /* * Clear evict_ref before releasing it in case that forces eviction (we @@ -957,6 +960,13 @@ __evict_tune_workers(WT_SESSION_IMPL *session) conn = S2C(session); cache = conn->cache; + /* + * If we have a fixed number of eviction threads, there is no value in + * calculating if we should do any tuning. + */ + if (conn->evict_threads_max == conn->evict_threads_min) + return; + WT_ASSERT(session, conn->evict_threads.threads[0]->session == session); pgs_evicted_cur = 0; @@ -1106,8 +1116,8 @@ __evict_tune_workers(WT_SESSION_IMPL *session) &conn->evict_threads, false); WT_STAT_CONN_INCR(session, cache_eviction_worker_created); - __wt_verbose(session, WT_VERB_EVICTSERVER, - "added worker thread"); + __wt_verbose(session, + WT_VERB_EVICTSERVER, "%s", "added worker thread"); } conn->evict_tune_last_action_time = current_time; } @@ -1641,26 +1651,16 @@ __evict_walk_file(WT_SESSION_IMPL *session, QUEUE_FILLS_PER_PASS; /* - * Randomly walk trees with a small fraction of the cache in case there - * are so many trees that none of them use enough of the cache to be - * allocated slots. - * - * The chance of walking a tree is equal to the chance that a random - * byte in cache belongs to the tree, weighted by how many times we - * want to fill queues during a pass through all the trees in cache. + * Walk trees with a small fraction of the cache in case there are so + * many trees that none of them use enough of the cache to be allocated + * slots. Only skip a tree if it has no bytes of interest. */ if (target_pages == 0) { - if (F_ISSET(cache, WT_CACHE_EVICT_CLEAN)) { - btree_inuse = __wt_btree_bytes_evictable(session); - cache_inuse = __wt_cache_bytes_inuse(cache); - } else { - btree_inuse = __wt_btree_dirty_leaf_inuse(session); - cache_inuse = __wt_cache_dirty_leaf_inuse(cache); - } - if (btree_inuse == 0 || cache_inuse == 0) - return (0); - if (__wt_random64(&session->rnd) % cache_inuse > - btree_inuse * QUEUE_FILLS_PER_PASS) + btree_inuse = F_ISSET(cache, WT_CACHE_EVICT_CLEAN) ? + __wt_btree_bytes_evictable(session) : + __wt_btree_dirty_leaf_inuse(session); + + if (btree_inuse == 0) return (0); } @@ -1678,6 +1678,32 @@ __evict_walk_file(WT_SESSION_IMPL *session, if (F_ISSET(session->dhandle, WT_DHANDLE_DEAD) || target_pages > remaining_slots) target_pages = remaining_slots; + + /* + * These statistics generate a histogram of the number of pages targeted + * for eviction each round. The range of values here start at + * MIN_PAGES_PER_TREE as this is the smallest number of pages we can + * target, unless there are fewer slots available. The aim is to cover + * the likely ranges of target pages in as few statistics as possible to + * reduce the overall overhead. + */ + if (target_pages < MIN_PAGES_PER_TREE) { + WT_STAT_CONN_INCR(session, cache_eviction_target_page_lt10); + WT_STAT_DATA_INCR(session, cache_eviction_target_page_lt10); + } else if (target_pages < 32) { + WT_STAT_CONN_INCR(session, cache_eviction_target_page_lt32); + WT_STAT_DATA_INCR(session, cache_eviction_target_page_lt32); + } else if (target_pages < 64) { + WT_STAT_CONN_INCR(session, cache_eviction_target_page_lt64); + WT_STAT_DATA_INCR(session, cache_eviction_target_page_lt64); + } else if (target_pages < 128) { + WT_STAT_CONN_INCR(session, cache_eviction_target_page_lt128); + WT_STAT_DATA_INCR(session, cache_eviction_target_page_lt128); + } else { + WT_STAT_CONN_INCR(session, cache_eviction_target_page_ge128); + WT_STAT_DATA_INCR(session, cache_eviction_target_page_ge128); + } + end = start + target_pages; /* @@ -1690,6 +1716,14 @@ __evict_walk_file(WT_SESSION_IMPL *session, !F_ISSET(cache, WT_CACHE_EVICT_CLEAN)) min_pages *= 10; + if (btree->evict_ref == NULL) { + WT_STAT_CONN_INCR(session, cache_eviction_walk_from_root); + WT_STAT_DATA_INCR(session, cache_eviction_walk_from_root); + } else { + WT_STAT_CONN_INCR(session, cache_eviction_walk_saved_pos); + WT_STAT_DATA_INCR(session, cache_eviction_walk_saved_pos); + } + walk_flags = WT_READ_CACHE | WT_READ_NO_EVICT | WT_READ_NO_GEN | WT_READ_NO_WAIT; @@ -1760,12 +1794,37 @@ __evict_walk_file(WT_SESSION_IMPL *session, btree->evict_start_type = (btree->evict_start_type + 1) % WT_EVICT_WALK_START_NUM; + /* + * We differentiate the reasons we gave up on this walk + * and increment the stats accordingly. + */ + if (pages_queued == 0) { + WT_STAT_CONN_INCR(session, + cache_eviction_walks_gave_up_no_targets); + WT_STAT_DATA_INCR(session, + cache_eviction_walks_gave_up_no_targets); + } else { + WT_STAT_CONN_INCR(session, + cache_eviction_walks_gave_up_ratio); + WT_STAT_DATA_INCR(session, + cache_eviction_walks_gave_up_ratio); + } break; } if (ref == NULL) { - if (++restarts == 2) + WT_STAT_CONN_INCR( + session, cache_eviction_walks_ended); + WT_STAT_DATA_INCR( + session, cache_eviction_walks_ended); + + if (++restarts == 2) { + WT_STAT_CONN_INCR( + session, cache_eviction_walks_stopped); + WT_STAT_DATA_INCR( + session, cache_eviction_walks_stopped); break; + } WT_STAT_CONN_INCR( session, cache_eviction_walks_started); continue; @@ -1919,6 +1978,9 @@ fast: /* If the page can't be evicted, give up. */ WT_STAT_CONN_INCRV(session, cache_eviction_walk, refs_walked); WT_STAT_CONN_INCRV(session, cache_eviction_pages_seen, pages_seen); + WT_STAT_DATA_INCRV(session, cache_eviction_pages_seen, pages_seen); + WT_STAT_CONN_INCRV(session, cache_eviction_walk_passes, 1); + WT_STAT_DATA_INCRV(session, cache_eviction_walk_passes, 1); return (0); } diff --git a/src/third_party/wiredtiger/src/evict/evict_stat.c b/src/third_party/wiredtiger/src/evict/evict_stat.c index 276e737ebbb..63d0aff60f1 100644 --- a/src/third_party/wiredtiger/src/evict/evict_stat.c +++ b/src/third_party/wiredtiger/src/evict/evict_stat.c @@ -16,28 +16,30 @@ static void __evict_stat_walk(WT_SESSION_IMPL *session) { WT_BTREE *btree; + WT_CACHE *cache; WT_PAGE *page; WT_REF *next_walk; - uint64_t dsk_size, gen_gap, size; - uint64_t written_size_cnt, written_size_sum; - uint64_t gen_gap_cnt, gen_gap_max, gen_gap_sum; - uint64_t max_pagesize, min_written_size; - uint64_t num_memory, num_queued, num_not_queueable, num_smaller_allocsz; - uint64_t pages_clean, pages_dirty, pages_internal, pages_leaf; - uint64_t seen_count, walk_count; + uint64_t dsk_size, gen_gap, gen_gap_max, gen_gap_sum, max_pagesize; + uint64_t min_written_size, num_memory, num_not_queueable, num_queued; + uint64_t num_smaller_allocsz, pages_clean, pages_dirty, pages_internal; + uint64_t pages_leaf, seen_count, size, visited_count; + uint64_t visited_age_gap_sum, unvisited_count, unvisited_age_gap_sum; + uint64_t walk_count, written_size_cnt, written_size_sum; btree = S2BT(session); + cache = S2C(session)->cache; next_walk = NULL; - written_size_cnt = written_size_sum = 0; - gen_gap_cnt = gen_gap_max = gen_gap_sum = 0; - max_pagesize = 0; - num_memory = num_queued = num_not_queueable = num_smaller_allocsz = 0; - pages_clean = pages_dirty = pages_internal = pages_leaf = 0; - seen_count = walk_count = 0; + dsk_size = gen_gap = gen_gap_max = gen_gap_sum = max_pagesize = 0; + num_memory = num_not_queueable = num_queued = 0; + num_smaller_allocsz = pages_clean = pages_dirty = pages_internal = 0; + pages_leaf = seen_count = size = visited_count = 0; + visited_age_gap_sum = unvisited_count = unvisited_age_gap_sum = 0; + walk_count = written_size_cnt = written_size_sum = 0; min_written_size = UINT64_MAX; while (__wt_tree_walk_count(session, &next_walk, &walk_count, - WT_READ_CACHE | WT_READ_NO_EVICT | WT_READ_NO_WAIT) == 0 && + WT_READ_CACHE | WT_READ_NO_EVICT | + WT_READ_NO_GEN | WT_READ_NO_WAIT) == 0 && next_walk != NULL) { ++seen_count; page = next_walk->page; @@ -78,19 +80,29 @@ __evict_stat_walk(WT_SESSION_IMPL *session) if (__wt_ref_is_root(next_walk)) continue; - gen_gap = - S2C(session)->cache->evict_pass_gen - page->evict_pass_gen; - if (gen_gap > gen_gap_max) - gen_gap_max = gen_gap; - gen_gap_sum += gen_gap; - ++gen_gap_cnt; + if (page->evict_pass_gen == 0) { + unvisited_age_gap_sum += + (cache->evict_pass_gen - page->cache_create_gen); + ++unvisited_count; + } else { + visited_age_gap_sum += + (cache->evict_pass_gen - page->cache_create_gen); + gen_gap = cache->evict_pass_gen - page->evict_pass_gen; + if (gen_gap > gen_gap_max) + gen_gap_max = gen_gap; + gen_gap_sum += gen_gap; + ++visited_count; + } } + WT_STAT_DATA_SET(session, cache_state_gen_avg_gap, + visited_count == 0 ? 0 : gen_gap_sum / visited_count); + WT_STAT_DATA_SET(session, cache_state_avg_unvisited_age, + unvisited_count == 0 ? 0 : unvisited_age_gap_sum / unvisited_count); + WT_STAT_DATA_SET(session, cache_state_avg_visited_age, + visited_count == 0 ? 0 : visited_age_gap_sum / visited_count); WT_STAT_DATA_SET(session, cache_state_avg_written_size, written_size_cnt == 0 ? 0 : written_size_sum / written_size_cnt); - WT_STAT_DATA_SET(session, cache_state_gen_avg_gap, - gen_gap_cnt == 0 ? 0 : gen_gap_sum / gen_gap_cnt); - WT_STAT_DATA_SET(session, cache_state_gen_max_gap, gen_gap_max); WT_STAT_DATA_SET(session, cache_state_max_pagesize, max_pagesize); WT_STAT_DATA_SET(session, @@ -98,8 +110,6 @@ __evict_stat_walk(WT_SESSION_IMPL *session) WT_STAT_DATA_SET(session, cache_state_memory, num_memory); WT_STAT_DATA_SET(session, cache_state_queued, num_queued); WT_STAT_DATA_SET(session, cache_state_not_queueable, num_not_queueable); - WT_STAT_DATA_SET(session, - cache_state_smaller_alloc_size, num_smaller_allocsz); WT_STAT_DATA_SET(session, cache_state_pages, walk_count); WT_STAT_DATA_SET(session, cache_state_pages_clean, pages_clean); WT_STAT_DATA_SET(session, cache_state_pages_dirty, pages_dirty); @@ -107,6 +117,10 @@ __evict_stat_walk(WT_SESSION_IMPL *session) WT_STAT_DATA_SET(session, cache_state_pages_leaf, pages_leaf); WT_STAT_DATA_SET(session, cache_state_refs_skipped, walk_count - seen_count); + WT_STAT_DATA_SET(session, + cache_state_smaller_alloc_size, num_smaller_allocsz); + WT_STAT_DATA_SET(session, + cache_state_unvisited_count, unvisited_count); } /* diff --git a/src/third_party/wiredtiger/src/include/api.h b/src/third_party/wiredtiger/src/include/api.h index 372ba063cd3..60ed31b64e8 100644 --- a/src/third_party/wiredtiger/src/include/api.h +++ b/src/third_party/wiredtiger/src/include/api.h @@ -13,7 +13,7 @@ (s)->dhandle = (dh); \ (s)->name = (s)->lastop = #h "." #n; \ WT_ERR(WT_SESSION_CHECK_PANIC(s)); \ - __wt_verbose((s), WT_VERB_API, "CALL: " #h ":" #n) + __wt_verbose((s), WT_VERB_API, "%s", "CALL: " #h ":" #n) #define API_CALL_NOCONF(s, h, n, dh) do { \ API_SESSION_INIT(s, h, n, dh) diff --git a/src/third_party/wiredtiger/src/include/bitstring.i b/src/third_party/wiredtiger/src/include/bitstring.i index d3dc3bebd0f..a9ec91d49ff 100644 --- a/src/third_party/wiredtiger/src/include/bitstring.i +++ b/src/third_party/wiredtiger/src/include/bitstring.i @@ -171,16 +171,18 @@ __bit_ffc(uint8_t *bitf, uint64_t nbits, uint64_t *retp) if (nbits == 0) return (-1); - for (byte = 0, - stopbyte = __bit_byte(nbits - 1); byte <= stopbyte; ++byte) + for (byte = 0, stopbyte = __bit_byte(nbits - 1);; ++byte) { if (bitf[byte] != 0xff) { value = byte << 3; for (lb = bitf[byte]; lb & 0x01; ++value, lb >>= 1) ; break; } + if (byte == stopbyte) + return (-1); + } - if (byte > stopbyte || value >= nbits) + if (value >= nbits) return (-1); *retp = value; @@ -201,16 +203,18 @@ __bit_ffs(uint8_t *bitf, uint64_t nbits, uint64_t *retp) if (nbits == 0) return (-1); - for (byte = 0, - stopbyte = __bit_byte(nbits - 1); byte <= stopbyte; ++byte) + for (byte = 0, stopbyte = __bit_byte(nbits - 1);; ++byte) { if (bitf[byte] != 0) { value = byte << 3; for (lb = bitf[byte]; !(lb & 0x01); ++value, lb >>= 1) ; break; } + if (byte == stopbyte) + return (-1); + } - if (byte > stopbyte || value >= nbits) + if (value >= nbits) return (-1); *retp = value; @@ -227,13 +231,6 @@ __bit_getv(uint8_t *bitf, uint64_t entry, uint8_t width) uint8_t value; uint64_t bit; -#define __BIT_GET(len, mask) \ - case len: \ - if (__bit_test(bitf, bit)) \ - value |= (mask); \ - ++bit \ - /* FALLTHROUGH */ - value = 0; bit = entry * width; @@ -241,17 +238,50 @@ __bit_getv(uint8_t *bitf, uint64_t entry, uint8_t width) * Fast-path single bytes, do repeated tests for the rest: we could * slice-and-dice instead, but the compiler is probably going to do * a better job than I will. + * + * The Berkeley version of this file uses a #define to compress this + * case statement. This code expands the case statement because gcc7 + * complains about implicit fallthrough and doesn't support explicit + * fallthrough comments in macros. */ switch (width) { case 8: return (bitf[__bit_byte(bit)]); - __BIT_GET(7, 0x40); - __BIT_GET(6, 0x20); - __BIT_GET(5, 0x10); - __BIT_GET(4, 0x08); - __BIT_GET(3, 0x04); - __BIT_GET(2, 0x02); - __BIT_GET(1, 0x01); + case 7: + if (__bit_test(bitf, bit)) + value |= 0x40; + ++bit; + /* FALLTHROUGH */ + case 6: + if (__bit_test(bitf, bit)) + value |= 0x20; + ++bit; + /* FALLTHROUGH */ + case 5: + if (__bit_test(bitf, bit)) + value |= 0x10; + ++bit; + /* FALLTHROUGH */ + case 4: + if (__bit_test(bitf, bit)) + value |= 0x08; + ++bit; + /* FALLTHROUGH */ + case 3: + if (__bit_test(bitf, bit)) + value |= 0x04; + ++bit; + /* FALLTHROUGH */ + case 2: + if (__bit_test(bitf, bit)) + value |= 0x02; + ++bit; + /* FALLTHROUGH */ + case 1: + if (__bit_test(bitf, bit)) + value |= 0x01; + ++bit; + break; } return (value); } @@ -276,32 +306,70 @@ __bit_setv(uint8_t *bitf, uint64_t entry, uint8_t width, uint8_t value) { uint64_t bit; -#define __BIT_SET(len, mask) \ - case len: \ - if (value & (mask)) \ - __bit_set(bitf, bit); \ - else \ - __bit_clear(bitf, bit); \ - ++bit \ - /* FALLTHROUGH */ - bit = entry * width; /* * Fast-path single bytes, do repeated tests for the rest: we could * slice-and-dice instead, but the compiler is probably going to do * a better job than I will. + * + * The Berkeley version of this file uses a #define to compress this + * case statement. This code expands the case statement because gcc7 + * complains about implicit fallthrough and doesn't support explicit + * fallthrough comments in macros. */ switch (width) { case 8: bitf[__bit_byte(bit)] = value; return; - __BIT_SET(7, 0x40); - __BIT_SET(6, 0x20); - __BIT_SET(5, 0x10); - __BIT_SET(4, 0x08); - __BIT_SET(3, 0x04); - __BIT_SET(2, 0x02); - __BIT_SET(1, 0x01); + case 7: + if (value & 0x40) + __bit_set(bitf, bit); + else + __bit_clear(bitf, bit); + ++bit; + /* FALLTHROUGH */ + case 6: + if (value & 0x20) + __bit_set(bitf, bit); + else + __bit_clear(bitf, bit); + ++bit; + /* FALLTHROUGH */ + case 5: + if (value & 0x10) + __bit_set(bitf, bit); + else + __bit_clear(bitf, bit); + ++bit; + /* FALLTHROUGH */ + case 4: + if (value & 0x08) + __bit_set(bitf, bit); + else + __bit_clear(bitf, bit); + ++bit; + /* FALLTHROUGH */ + case 3: + if (value & 0x04) + __bit_set(bitf, bit); + else + __bit_clear(bitf, bit); + ++bit; + /* FALLTHROUGH */ + case 2: + if (value & 0x02) + __bit_set(bitf, bit); + else + __bit_clear(bitf, bit); + ++bit; + /* FALLTHROUGH */ + case 1: + if (value & 0x01) + __bit_set(bitf, bit); + else + __bit_clear(bitf, bit); + ++bit; + break; } } diff --git a/src/third_party/wiredtiger/src/include/btmem.h b/src/third_party/wiredtiger/src/include/btmem.h index 54a03e5762c..e8d3307b013 100644 --- a/src/third_party/wiredtiger/src/include/btmem.h +++ b/src/third_party/wiredtiger/src/include/btmem.h @@ -162,32 +162,6 @@ struct __wt_ovfl_reuse { }; /* - * Overflow tracking for cached values: When a page is reconciled, we write new - * K/V overflow items, and discard previous underlying blocks. If there's a - * transaction in the system that needs to read the previous value, we have to - * cache the old value until no running transaction needs it. - */ -struct __wt_ovfl_txnc { - uint64_t current; /* Maximum transaction ID at store */ - - uint32_t value_offset; /* Overflow value offset */ - uint32_t value_size; /* Overflow value size */ - uint8_t addr_offset; /* Overflow addr offset */ - uint8_t addr_size; /* Overflow addr size */ - - /* - * The untyped address immediately follows the WT_OVFL_TXNC - * structure, the untyped value immediately follows the address. - */ -#define WT_OVFL_TXNC_ADDR(p) \ - ((void *)((uint8_t *)(p) + (p)->addr_offset)) -#define WT_OVFL_TXNC_VALUE(p) \ - ((void *)((uint8_t *)(p) + (p)->value_offset)) - - WT_OVFL_TXNC *next[0]; /* Forward-linked skip list */ -}; - -/* * Lookaside table support: when a page is being reconciled for eviction and has * updates that might be required by earlier readers in the system, the updates * are written into a lookaside table, and restored as necessary if the page is @@ -403,18 +377,20 @@ struct __wt_page_modify { WT_OVFL_REUSE *ovfl_reuse[WT_SKIP_MAXDEPTH]; /* - * Overflow value address/byte-string pairs cached until no - * running transaction will possibly read them. - */ - WT_OVFL_TXNC *ovfl_txnc[WT_SKIP_MAXDEPTH]; - - /* * Overflow key/value addresses to be discarded from the block * manager after reconciliation completes successfully. */ WT_CELL **discard; size_t discard_entries; size_t discard_allocated; + + /* Cached overflow value cell/update address pairs. */ + struct { + WT_CELL *cell; + WT_UPDATE *upd; + } *remove; + size_t remove_allocated; + uint32_t remove_next; } *ovfl_track; #define WT_PAGE_LOCK(s, p) \ @@ -644,6 +620,7 @@ struct __wt_page { /* This is the 64 byte boundary, try to keep hot fields above here. */ + uint64_t cache_create_gen; /* Page create timestamp */ uint64_t evict_pass_gen; /* Eviction pass generation */ }; @@ -921,6 +898,10 @@ WT_PACKED_STRUCT_BEGIN(__wt_update) #define WT_UPDATE_RESERVED 2 uint8_t type; /* type (one byte to conserve memory) */ + /* The update includes a complete value. */ +#define WT_UPDATE_DATA_VALUE(upd) \ + ((upd)->type == WT_UPDATE_STANDARD || (upd)->type == WT_UPDATE_DELETED) + /* The untyped value immediately follows the WT_UPDATE structure. */ #define WT_UPDATE_DATA(upd) \ ((void *)((uint8_t *)(upd) + sizeof(WT_UPDATE))) diff --git a/src/third_party/wiredtiger/src/include/btree.h b/src/third_party/wiredtiger/src/include/btree.h index e0f7570b869..4d2a4f0c4d9 100644 --- a/src/third_party/wiredtiger/src/include/btree.h +++ b/src/third_party/wiredtiger/src/include/btree.h @@ -165,24 +165,25 @@ struct __wt_btree { WT_SPINLOCK flush_lock; /* Lock to flush the tree's pages */ /* Flags values up to 0xff are reserved for WT_DHANDLE_* */ -#define WT_BTREE_BULK 0x000100 /* Bulk-load handle */ -#define WT_BTREE_CLOSED 0x000200 /* Handle closed */ -#define WT_BTREE_IGNORE_CACHE 0x000400 /* Cache-resident object */ -#define WT_BTREE_IN_MEMORY 0x000800 /* Cache-resident object */ -#define WT_BTREE_LOOKASIDE 0x001000 /* Look-aside table */ -#define WT_BTREE_NO_CHECKPOINT 0x002000 /* Disable checkpoints */ -#define WT_BTREE_NO_LOGGING 0x004000 /* Disable logging */ -#define WT_BTREE_REBALANCE 0x008000 /* Handle is for rebalance */ -#define WT_BTREE_SALVAGE 0x010000 /* Handle is for salvage */ -#define WT_BTREE_SKIP_CKPT 0x020000 /* Handle skipped checkpoint */ -#define WT_BTREE_UPGRADE 0x040000 /* Handle is for upgrade */ -#define WT_BTREE_VERIFY 0x080000 /* Handle is for verify */ +#define WT_BTREE_ALTER 0x000100 /* Handle is for alter */ +#define WT_BTREE_BULK 0x000200 /* Bulk-load handle */ +#define WT_BTREE_CLOSED 0x000400 /* Handle closed */ +#define WT_BTREE_IGNORE_CACHE 0x000800 /* Cache-resident object */ +#define WT_BTREE_IN_MEMORY 0x001000 /* Cache-resident object */ +#define WT_BTREE_LOOKASIDE 0x002000 /* Look-aside table */ +#define WT_BTREE_NO_CHECKPOINT 0x004000 /* Disable checkpoints */ +#define WT_BTREE_NO_LOGGING 0x008000 /* Disable logging */ +#define WT_BTREE_REBALANCE 0x010000 /* Handle is for rebalance */ +#define WT_BTREE_SALVAGE 0x020000 /* Handle is for salvage */ +#define WT_BTREE_SKIP_CKPT 0x040000 /* Handle skipped checkpoint */ +#define WT_BTREE_UPGRADE 0x080000 /* Handle is for upgrade */ +#define WT_BTREE_VERIFY 0x100000 /* Handle is for verify */ uint32_t flags; }; /* Flags that make a btree handle special (not for normal use). */ #define WT_BTREE_SPECIAL_FLAGS \ - (WT_BTREE_BULK | WT_BTREE_REBALANCE | \ + (WT_BTREE_ALTER | WT_BTREE_BULK | WT_BTREE_REBALANCE | \ WT_BTREE_SALVAGE | WT_BTREE_UPGRADE | WT_BTREE_VERIFY) /* diff --git a/src/third_party/wiredtiger/src/include/btree_cmp.i b/src/third_party/wiredtiger/src/include/btree_cmp.i index c1354a7ea4b..db7af8daaed 100644 --- a/src/third_party/wiredtiger/src/include/btree_cmp.i +++ b/src/third_party/wiredtiger/src/include/btree_cmp.i @@ -217,23 +217,61 @@ __wt_lex_compare_short(const WT_ITEM *user_item, const WT_ITEM *tree_item) /* * The maximum packed uint64_t is 9B, catch row-store objects using * packed record numbers as keys. + * + * Don't use a #define to compress this case statement: gcc7 complains + * about implicit fallthrough and doesn't support explicit fallthrough + * comments in macros. */ #define WT_COMPARE_SHORT_MAXLEN 9 -#undef WT_COMPARE_SHORT -#define WT_COMPARE_SHORT(n) \ - case n: \ - if (*userp != *treep) \ - break; \ - ++userp, ++treep switch (len) { - WT_COMPARE_SHORT(9); - WT_COMPARE_SHORT(8); - WT_COMPARE_SHORT(7); - WT_COMPARE_SHORT(6); - WT_COMPARE_SHORT(5); - WT_COMPARE_SHORT(4); - WT_COMPARE_SHORT(3); - WT_COMPARE_SHORT(2); + case 9: + if (*userp != *treep) + break; + ++userp; + ++treep; + /* FALLTHROUGH */ + case 8: + if (*userp != *treep) + break; + ++userp; + ++treep; + /* FALLTHROUGH */ + case 7: + if (*userp != *treep) + break; + ++userp; + ++treep; + /* FALLTHROUGH */ + case 6: + if (*userp != *treep) + break; + ++userp; + ++treep; + /* FALLTHROUGH */ + case 5: + if (*userp != *treep) + break; + ++userp; + ++treep; + /* FALLTHROUGH */ + case 4: + if (*userp != *treep) + break; + ++userp; + ++treep; + /* FALLTHROUGH */ + case 3: + if (*userp != *treep) + break; + ++userp; + ++treep; + /* FALLTHROUGH */ + case 2: + if (*userp != *treep) + break; + ++userp; + ++treep; + /* FALLTHROUGH */ case 1: if (*userp != *treep) break; diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index 94abe48ffe5..7bd4a818495 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -184,6 +184,9 @@ struct __wt_connection_impl { const char *error_prefix; /* Database error prefix */ int is_new; /* Connection created database */ + uint16_t compat_major; /* Compatibility major version */ + uint16_t compat_minor; /* Compatibility minor version */ + WT_EXTENSION_API extension_api; /* Extension API */ /* Configuration */ @@ -302,18 +305,22 @@ struct __wt_connection_impl { WT_CONDVAR *stat_cond; /* Statistics log wait mutex */ const char *stat_format; /* Statistics log timestamp format */ WT_FSTREAM *stat_fs; /* Statistics log stream */ + /* Statistics log json table printing state flag */ + bool stat_json_tables; char *stat_path; /* Statistics log path format */ char **stat_sources; /* Statistics log list of objects */ const char *stat_stamp; /* Statistics log entry timestamp */ uint64_t stat_usecs; /* Statistics log period */ -#define WT_CONN_LOG_ARCHIVE 0x01 /* Archive is enabled */ -#define WT_CONN_LOG_ENABLED 0x02 /* Logging is enabled */ -#define WT_CONN_LOG_EXISTED 0x04 /* Log files found */ -#define WT_CONN_LOG_RECOVER_DIRTY 0x08 /* Recovering unclean */ -#define WT_CONN_LOG_RECOVER_DONE 0x10 /* Recovery completed */ -#define WT_CONN_LOG_RECOVER_ERR 0x20 /* Error if recovery required */ -#define WT_CONN_LOG_ZERO_FILL 0x40 /* Manually zero files */ +#define WT_CONN_LOG_ARCHIVE 0x001 /* Archive is enabled */ +#define WT_CONN_LOG_DOWNGRADED 0x002 /* Running older version */ +#define WT_CONN_LOG_ENABLED 0x004 /* Logging is enabled */ +#define WT_CONN_LOG_EXISTED 0x008 /* Log files found */ +#define WT_CONN_LOG_FORCE_DOWNGRADE 0x010 /* Force downgrade */ +#define WT_CONN_LOG_RECOVER_DIRTY 0x020 /* Recovering unclean */ +#define WT_CONN_LOG_RECOVER_DONE 0x040 /* Recovery completed */ +#define WT_CONN_LOG_RECOVER_ERR 0x080 /* Error if recovery required */ +#define WT_CONN_LOG_ZERO_FILL 0x100 /* Manually zero files */ uint32_t log_flags; /* Global logging configuration */ WT_CONDVAR *log_cond; /* Log server wait mutex */ WT_SESSION_IMPL *log_session; /* Log server session */ diff --git a/src/third_party/wiredtiger/src/include/error.h b/src/third_party/wiredtiger/src/include/error.h index 465ab4fa859..2d302fb6879 100644 --- a/src/third_party/wiredtiger/src/include/error.h +++ b/src/third_party/wiredtiger/src/include/error.h @@ -124,3 +124,28 @@ #define WT_ASSERT(session, exp) \ WT_UNUSED(session) #endif + +/* + * __wt_verbose -- + * Display a verbose message. + * + * Not an inlined function because you can't inline functions taking variadic + * arguments and we don't want to make a function call in production systems + * just to find out a verbose flag isn't set. + * + * The macro must take a format string and at least one additional argument, + * there's no portable way to remove the comma before an empty __VA_ARGS__ + * value. + */ +#ifdef HAVE_VERBOSE +#define __wt_verbose(session, flag, fmt, ...) do { \ + if (WT_VERBOSE_ISSET(session, flag)) \ + __wt_verbose_worker(session, fmt, __VA_ARGS__); \ +} while (0) +#else +#define __wt_verbose(session, flag, fmt, ...) do { \ + WT_UNUSED(session); \ + WT_UNUSED(flag); \ + WT_UNUSED(fmt); \ +} while (0) +#endif diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 59e853d75f4..8b48fd587bd 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -148,7 +148,7 @@ extern const char *__wt_cell_type_string(uint8_t type); extern const char *__wt_page_addr_string(WT_SESSION_IMPL *session, WT_REF *ref, WT_ITEM *buf); extern const char *__wt_addr_string(WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size, WT_ITEM *buf); extern int __wt_ovfl_read(WT_SESSION_IMPL *session, WT_PAGE *page, WT_CELL_UNPACK *unpack, WT_ITEM *store) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_ovfl_cache(WT_SESSION_IMPL *session, WT_PAGE *page, void *cookie, WT_CELL_UNPACK *vpack) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ovfl_remove(WT_SESSION_IMPL *session, WT_PAGE *page, WT_UPDATE *upd_list, WT_CELL_UNPACK *unpack) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ovfl_discard(WT_SESSION_IMPL *session, WT_CELL *cell) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_page_alloc(WT_SESSION_IMPL *session, uint8_t type, uint32_t alloc_entries, bool alloc_refs, WT_PAGE **pagep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_page_inmem(WT_SESSION_IMPL *session, WT_REF *ref, const void *image, size_t memsize, uint32_t flags, WT_PAGE **pagep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -267,7 +267,7 @@ extern int __wt_conn_dhandle_discard(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ extern int __wt_connection_init(WT_CONNECTION_IMPL *conn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_connection_destroy(WT_CONNECTION_IMPL *conn); extern int __wt_logmgr_reconfig(WT_SESSION_IMPL *session, const char **cfg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_log_truncate_files( WT_SESSION_IMPL *session, WT_CURSOR *cursor, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_log_truncate_files(WT_SESSION_IMPL *session, WT_CURSOR *cursor, bool force) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_log_wrlsn(WT_SESSION_IMPL *session, int *yield); extern int __wt_logmgr_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_logmgr_open(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -371,6 +371,8 @@ extern void __wt_log_written_reset(WT_SESSION_IMPL *session); extern int __wt_log_get_all_files(WT_SESSION_IMPL *session, char ***filesp, u_int *countp, uint32_t *maxid, bool active_only) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_log_extract_lognum( WT_SESSION_IMPL *session, const char *name, uint32_t *id) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_log_reset(WT_SESSION_IMPL *session, uint32_t lognum) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_log_fill(WT_SESSION_IMPL *session, WT_MYSLOT *myslot, bool force, WT_ITEM *record, WT_LSN *lsnp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_log_set_version(WT_SESSION_IMPL *session, uint16_t version, uint32_t first_rec, bool downgrade, bool live_chg, uint32_t *lognump) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_log_acquire(WT_SESSION_IMPL *session, uint64_t recsize, WT_LOGSLOT *slot) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_log_allocfile( WT_SESSION_IMPL *session, uint32_t lognum, const char *dest) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_log_remove(WT_SESSION_IMPL *session, const char *file_prefix, uint32_t lognum) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -412,6 +414,8 @@ extern int __wt_log_slot_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTR extern void __wt_log_slot_join(WT_SESSION_IMPL *session, uint64_t mysize, uint32_t flags, WT_MYSLOT *myslot); extern int64_t __wt_log_slot_release(WT_MYSLOT *myslot, int64_t size); extern void __wt_log_slot_free(WT_SESSION_IMPL *session, WT_LOGSLOT *slot); +extern int __wt_log_system_record( WT_SESSION_IMPL *session, WT_FH *log_fh, WT_LSN *lsn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_log_recover_system(WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, WT_LSN *lsnp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_clsm_request_switch(WT_CURSOR_LSM *clsm) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_clsm_await_switch(WT_CURSOR_LSM *clsm) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_clsm_init_merge( WT_CURSOR *cursor, u_int start_chunk, uint32_t start_id, u_int nchunks) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -443,7 +447,6 @@ extern void __wt_lsm_tree_release(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tre extern void __wt_lsm_tree_throttle( WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, bool decrease_only); extern int __wt_lsm_tree_switch(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_lsm_tree_retire_chunks(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, u_int start_chunk, u_int nchunks) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_lsm_tree_alter( WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_lsm_tree_drop( WT_SESSION_IMPL *session, const char *name, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_lsm_tree_rename(WT_SESSION_IMPL *session, const char *olduri, const char *newuri, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_lsm_tree_truncate( WT_SESSION_IMPL *session, const char *name, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -541,14 +544,12 @@ extern int __wt_ext_unpack_item(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, WT extern int __wt_ext_unpack_int(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, int64_t *ip) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ext_unpack_str(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, const char **sp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ext_unpack_uint(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, uint64_t *up) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ovfl_track_init(WT_SESSION_IMPL *session, WT_PAGE *page) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ovfl_discard_add(WT_SESSION_IMPL *session, WT_PAGE *page, WT_CELL *cell) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_ovfl_discard_free(WT_SESSION_IMPL *session, WT_PAGE *page); extern int __wt_ovfl_reuse_search(WT_SESSION_IMPL *session, WT_PAGE *page, uint8_t **addrp, size_t *addr_sizep, const void *value, size_t value_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ovfl_reuse_add(WT_SESSION_IMPL *session, WT_PAGE *page, const uint8_t *addr, size_t addr_size, const void *value, size_t value_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_ovfl_reuse_free(WT_SESSION_IMPL *session, WT_PAGE *page); -extern int __wt_ovfl_txnc_search( WT_PAGE *page, const uint8_t *addr, size_t addr_size, WT_ITEM *store) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_ovfl_txnc_add(WT_SESSION_IMPL *session, WT_PAGE *page, const uint8_t *addr, size_t addr_size, const void *value, size_t value_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_ovfl_txnc_free(WT_SESSION_IMPL *session, WT_PAGE *page); extern int __wt_ovfl_track_wrapup(WT_SESSION_IMPL *session, WT_PAGE *page) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ovfl_track_wrapup_err(WT_SESSION_IMPL *session, WT_PAGE *page) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref, WT_SALVAGE_COOKIE *salvage, uint32_t flags, bool *lookaside_retryp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -559,7 +560,7 @@ extern int __wt_bulk_insert_row(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) extern int __wt_bulk_insert_fix( WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk, bool deleted) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bulk_insert_fix_bitmap(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bulk_insert_var( WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk, bool deleted) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_schema_alter(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_alter(WT_SESSION_IMPL *session, const char *newcfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_direct_io_size_check(WT_SESSION_IMPL *session, const char **cfg, const char *config_name, uint32_t *allocsizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_colgroup_source(WT_SESSION_IMPL *session, WT_TABLE *table, const char *cgname, const char *config, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_index_source(WT_SESSION_IMPL *session, WT_TABLE *table, const char *idxname, const char *config, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -631,6 +632,7 @@ extern int __wt_eventv(WT_SESSION_IMPL *session, bool msg_event, int error, cons extern void __wt_err(WT_SESSION_IMPL *session, int error, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern void __wt_errx(WT_SESSION_IMPL *session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 2, 3))); extern int __wt_ext_err_printf( WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_verbose_worker(WT_SESSION_IMPL *session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 2, 3))) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)); extern int __wt_msg(WT_SESSION_IMPL *session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 2, 3))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ext_msg_printf( WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern const char *__wt_ext_strerror(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, int error); @@ -646,6 +648,7 @@ __wt_assert(WT_SESSION_IMPL *session, WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern int __wt_panic(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_illegal_value(WT_SESSION_IMPL *session, const char *name) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_inmem_unsupported_op(WT_SESSION_IMPL *session, const char *tag) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_object_unsupported(WT_SESSION_IMPL *session, const char *uri) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bad_object_type(WT_SESSION_IMPL *session, const char *uri) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_unexpected_object_type( WT_SESSION_IMPL *session, const char *uri, const char *expect) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -705,7 +708,6 @@ extern uint32_t __wt_rduppo2(uint32_t n, uint32_t po2); extern void __wt_random_init(WT_RAND_STATE volatile *rnd_state) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern void __wt_random_init_seed( WT_SESSION_IMPL *session, WT_RAND_STATE volatile *rnd_state) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern uint32_t __wt_random(WT_RAND_STATE volatile *rnd_state) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); -extern uint64_t __wt_random64(WT_RAND_STATE volatile *rnd_state) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern int __wt_buf_grow_worker(WT_SESSION_IMPL *session, WT_ITEM *buf, size_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_buf_fmt(WT_SESSION_IMPL *session, WT_ITEM *buf, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_buf_catfmt(WT_SESSION_IMPL *session, WT_ITEM *buf, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -789,3 +791,7 @@ extern int __wt_txn_global_query_timestamp( WT_SESSION_IMPL *session, char *hex_ extern int __wt_txn_update_pinned_timestamp(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_global_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_txn_set_commit_timestamp(WT_SESSION_IMPL *session); +extern void __wt_txn_clear_commit_timestamp(WT_SESSION_IMPL *session); +extern void __wt_txn_set_read_timestamp(WT_SESSION_IMPL *session); +extern void __wt_txn_clear_read_timestamp(WT_SESSION_IMPL *session); diff --git a/src/third_party/wiredtiger/src/include/gcc.h b/src/third_party/wiredtiger/src/include/gcc.h index 21eaaaef049..043d9b44356 100644 --- a/src/third_party/wiredtiger/src/include/gcc.h +++ b/src/third_party/wiredtiger/src/include/gcc.h @@ -21,7 +21,7 @@ * dist/s_prototypes to create extern.h. */ #define WT_GCC_FUNC_ATTRIBUTE(x) -#define WT_GCC_FUNC_DECL_ATTRIBUTE(x) __attribute__(x) +#define WT_GCC_FUNC_DECL_ATTRIBUTE(x) __attribute__(x) /* * Atomic writes: diff --git a/src/third_party/wiredtiger/src/include/log.h b/src/third_party/wiredtiger/src/include/log.h index e7bc28cd220..e6acdd1e834 100644 --- a/src/third_party/wiredtiger/src/include/log.h +++ b/src/third_party/wiredtiger/src/include/log.h @@ -42,7 +42,7 @@ union __wt_lsn { #define WT_ZERO_LSN(l) WT_SET_LSN((l), 0, 0) /* - * Initialize LSN is (1,0). We only need to shift the 1 for comparison. + * Test for initial LSN. We only need to shift the 1 for comparison. */ #define WT_IS_INIT_LSN(l) ((l)->file_offset == ((uint64_t)1 << 32)) /* @@ -52,6 +52,10 @@ union __wt_lsn { #define WT_IS_MAX_LSN(lsn) \ ((lsn)->l.file == UINT32_MAX && \ ((lsn)->l.offset == INT32_MAX || (lsn)->l.offset == UINT32_MAX)) +/* + * Test for zero LSN. + */ +#define WT_IS_ZERO_LSN(l) ((l)->file_offset == 0) /* * Both of the macros below need to change if the content of __wt_lsn @@ -198,10 +202,11 @@ struct __wt_myslot { uint32_t flags; /* Flags */ }; -#define WT_LOG_FIRST_RECORD log->allocsize +#define WT_LOG_END_HEADER log->allocsize struct __wt_log { uint32_t allocsize; /* Allocation alignment size */ + uint32_t first_record; /* Offset of first record in file */ wt_off_t log_written; /* Amount of log written this period */ /* * Log file information @@ -215,6 +220,8 @@ struct __wt_log { WT_FH *log_close_fh; /* Logging file handle to close */ WT_LSN log_close_lsn; /* LSN needed to close */ + uint16_t log_version; /* Version of log file */ + /* * System LSNs */ @@ -232,6 +239,7 @@ struct __wt_log { * Synchronization resources */ WT_SPINLOCK log_lock; /* Locked: Logging fields */ + WT_SPINLOCK log_fs_lock; /* Locked: tmp, prep and log files */ WT_SPINLOCK log_slot_lock; /* Locked: Consolidation array */ WT_SPINLOCK log_sync_lock; /* Locked: Single-thread fsync */ WT_SPINLOCK log_writelsn_lock; /* Locked: write LSN */ @@ -261,8 +269,9 @@ struct __wt_log { uint64_t write_calls; /* Calls to log_write */ #endif -#define WT_LOG_OPENED 0x01 /* Log subsystem successfully open */ -#define WT_LOG_TRUNCATE_NOTSUP 0x02 /* File system truncate not supported */ +#define WT_LOG_FORCE_NEWFILE 0x01 /* Force switch to new log file */ +#define WT_LOG_OPENED 0x02 /* Log subsystem successfully open */ +#define WT_LOG_TRUNCATE_NOTSUP 0x04 /* File system truncate not supported */ uint32_t flags; }; @@ -303,12 +312,22 @@ __wt_log_record_byteswap(WT_LOG_RECORD *record) struct __wt_log_desc { #define WT_LOG_MAGIC 0x101064 uint32_t log_magic; /* 00-03: Magic number */ -#define WT_LOG_MAJOR_VERSION 1 - uint16_t majorv; /* 04-05: Major version */ -#define WT_LOG_MINOR_VERSION 0 - uint16_t minorv; /* 06-07: Minor version */ +#define WT_LOG_VERSION 2 + uint16_t version; /* 04-05: Log version */ + uint16_t unused; /* 06-07: Unused */ uint64_t log_size; /* 08-15: Log file size */ }; +/* + * This is the log version that introduced the system record. + */ +#define WT_LOG_VERSION_SYSTEM 2 + +/* + * WiredTiger release version where log format version changed. + * We only have to check the major version for now. It is minor + * version 0 once release numbers move on. + */ +#define WT_LOG_V2 3 /* * __wt_log_desc_byteswap -- @@ -320,8 +339,8 @@ __wt_log_desc_byteswap(WT_LOG_DESC *desc) { #ifdef WORDS_BIGENDIAN desc->log_magic = __wt_bswap32(desc->log_magic); - desc->majorv = __wt_bswap16(desc->majorv); - desc->minorv = __wt_bswap16(desc->minorv); + desc->version = __wt_bswap16(desc->version); + desc->unused = __wt_bswap16(desc->unused); desc->log_size = __wt_bswap64(desc->log_size); #else WT_UNUSED(desc); diff --git a/src/third_party/wiredtiger/src/include/misc.i b/src/third_party/wiredtiger/src/include/misc.i index 36a1e1f18eb..bb785a63072 100644 --- a/src/third_party/wiredtiger/src/include/misc.i +++ b/src/third_party/wiredtiger/src/include/misc.i @@ -80,39 +80,6 @@ __wt_time_check_monotonic(WT_SESSION_IMPL *session, struct timespec *tsp) } /* - * __wt_verbose -- - * Verbose message. - * - * Inline functions are not parsed for external prototypes, so in cases where we - * want GCC attributes attached to the functions, we have to do so explicitly. - */ -static inline void -__wt_verbose(WT_SESSION_IMPL *session, int flag, const char *fmt, ...) -WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))); - -/* - * __wt_verbose -- - * Verbose message. - */ -static inline void -__wt_verbose(WT_SESSION_IMPL *session, int flag, const char *fmt, ...) -{ -#ifdef HAVE_VERBOSE - va_list ap; - - if (WT_VERBOSE_ISSET(session, flag)) { - va_start(ap, fmt); - WT_IGNORE_RET(__wt_eventv(session, true, 0, NULL, 0, fmt, ap)); - va_end(ap); - } -#else - WT_UNUSED(session); - WT_UNUSED(flag); - WT_UNUSED(fmt); -#endif -} - -/* * __wt_snprintf -- * snprintf convenience function, ignoring the returned size. */ diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index 7d7d701590a..b340b278684 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -303,13 +303,25 @@ struct __wt_connection_stats { int64_t cache_eviction_get_ref_empty2; int64_t cache_eviction_aggressive_set; int64_t cache_eviction_empty_score; + int64_t cache_eviction_walk_passes; int64_t cache_eviction_queue_empty; int64_t cache_eviction_queue_not_empty; int64_t cache_eviction_server_evicting; int64_t cache_eviction_server_slept; int64_t cache_eviction_slow; int64_t cache_eviction_state; + int64_t cache_eviction_target_page_lt10; + int64_t cache_eviction_target_page_lt32; + int64_t cache_eviction_target_page_ge128; + int64_t cache_eviction_target_page_lt64; + int64_t cache_eviction_target_page_lt128; int64_t cache_eviction_walks_abandoned; + int64_t cache_eviction_walks_stopped; + int64_t cache_eviction_walks_gave_up_no_targets; + int64_t cache_eviction_walks_gave_up_ratio; + int64_t cache_eviction_walks_ended; + int64_t cache_eviction_walk_from_root; + int64_t cache_eviction_walk_saved_pos; int64_t cache_eviction_active_workers; int64_t cache_eviction_worker_created; int64_t cache_eviction_worker_evicting; @@ -336,7 +348,6 @@ struct __wt_connection_stats { int64_t cache_eviction_dirty; int64_t cache_eviction_app_dirty; int64_t cache_read_overflow; - int64_t cache_overflow_value; int64_t cache_eviction_deepen; int64_t cache_write_lookaside; int64_t cache_pages_inuse; @@ -414,6 +425,7 @@ struct __wt_connection_stats { int64_t lock_table_read_count; int64_t lock_table_write_count; int64_t log_slot_switch_busy; + int64_t log_force_ckpt_sleep; int64_t log_bytes_payload; int64_t log_bytes_written; int64_t log_zero_fills; @@ -570,6 +582,19 @@ struct __wt_dsrc_stats { int64_t cache_bytes_write; int64_t cache_eviction_checkpoint; int64_t cache_eviction_fail; + int64_t cache_eviction_walk_passes; + int64_t cache_eviction_target_page_lt10; + int64_t cache_eviction_target_page_lt32; + int64_t cache_eviction_target_page_ge128; + int64_t cache_eviction_target_page_lt64; + int64_t cache_eviction_target_page_lt128; + int64_t cache_eviction_walks_abandoned; + int64_t cache_eviction_walks_stopped; + int64_t cache_eviction_walks_gave_up_no_targets; + int64_t cache_eviction_walks_gave_up_ratio; + int64_t cache_eviction_walks_ended; + int64_t cache_eviction_walk_from_root; + int64_t cache_eviction_walk_saved_pos; int64_t cache_eviction_hazard; int64_t cache_inmem_splittable; int64_t cache_inmem_split; @@ -578,18 +603,20 @@ struct __wt_dsrc_stats { int64_t cache_eviction_split_leaf; int64_t cache_eviction_dirty; int64_t cache_read_overflow; - int64_t cache_overflow_value; int64_t cache_eviction_deepen; int64_t cache_write_lookaside; int64_t cache_read; int64_t cache_read_lookaside; int64_t cache_pages_requested; + int64_t cache_eviction_pages_seen; int64_t cache_write; int64_t cache_write_restore; int64_t cache_bytes_dirty; int64_t cache_eviction_clean; int64_t cache_state_gen_avg_gap; int64_t cache_state_avg_written_size; + int64_t cache_state_avg_visited_age; + int64_t cache_state_avg_unvisited_age; int64_t cache_state_pages_clean; int64_t cache_state_gen_current; int64_t cache_state_pages_dirty; @@ -599,6 +626,7 @@ struct __wt_dsrc_stats { int64_t cache_state_gen_max_gap; int64_t cache_state_max_pagesize; int64_t cache_state_min_written_size; + int64_t cache_state_unvisited_count; int64_t cache_state_smaller_alloc_size; int64_t cache_state_memory; int64_t cache_state_queued; diff --git a/src/third_party/wiredtiger/src/include/txn.h b/src/third_party/wiredtiger/src/include/txn.h index bf2d9aa21ef..e4cc0b04046 100644 --- a/src/third_party/wiredtiger/src/include/txn.h +++ b/src/third_party/wiredtiger/src/include/txn.h @@ -74,9 +74,6 @@ struct __wt_txn_state { volatile uint64_t pinned_id; volatile uint64_t metadata_pinned; - WT_DECL_TIMESTAMP(commit_timestamp) - WT_DECL_TIMESTAMP(read_timestamp) - WT_CACHE_LINE_PAD_END }; @@ -103,6 +100,14 @@ struct __wt_txn_global { /* Protects the active transaction states. */ WT_RWLOCK rwlock; + /* List of transactions sorted by commit timestamp. */ + WT_RWLOCK commit_timestamp_rwlock; + TAILQ_HEAD(__wt_txn_cts_qh, __wt_txn) commit_timestamph; + + /* List of transactions sorted by read timestamp. */ + WT_RWLOCK read_timestamp_rwlock; + TAILQ_HEAD(__wt_txn_rts_qh, __wt_txn) read_timestamph; + /* * Track information about the running checkpoint. The transaction * snapshot used when checkpointing are special. Checkpoints can run @@ -117,6 +122,7 @@ struct __wt_txn_global { volatile bool checkpoint_running; /* Checkpoint running */ volatile uint32_t checkpoint_id; /* Checkpoint's session ID */ WT_TXN_STATE checkpoint_state; /* Checkpoint's txn state */ + WT_TXN *checkpoint_txn; /* Checkpoint's txn structure */ volatile uint64_t metadata_pinned; /* Oldest ID for metadata */ @@ -194,8 +200,11 @@ struct __wt_txn { uint32_t snapshot_count; uint32_t txn_logsync; /* Log sync configuration */ - WT_DECL_TIMESTAMP(read_timestamp) WT_DECL_TIMESTAMP(commit_timestamp) + WT_DECL_TIMESTAMP(read_timestamp) + + TAILQ_ENTRY(__wt_txn) commit_timestampq; + TAILQ_ENTRY(__wt_txn) read_timestampq; /* Array of modifications by this transaction. */ WT_TXN_OP *mod; diff --git a/src/third_party/wiredtiger/src/include/txn.i b/src/third_party/wiredtiger/src/include/txn.i index 85d70fbde89..6de86eb0aaf 100644 --- a/src/third_party/wiredtiger/src/include/txn.i +++ b/src/third_party/wiredtiger/src/include/txn.i @@ -10,8 +10,6 @@ static inline int __wt_txn_id_check(WT_SESSION_IMPL *session); static inline void __wt_txn_read_last(WT_SESSION_IMPL *session); #ifdef HAVE_TIMESTAMPS -static const wt_timestamp_t zero_timestamp; - /* * __wt_timestamp_cmp -- * Compare two timestamps. @@ -39,8 +37,30 @@ __wt_timestamp_set(uint8_t *dest, const uint8_t *src) static inline bool __wt_timestamp_iszero(const uint8_t *ts) { + static const wt_timestamp_t zero_timestamp; + return (memcmp(ts, zero_timestamp, WT_TIMESTAMP_SIZE) == 0); } + +/* + * __wt_timestamp_set_inf -- + * Set a timestamp to the maximum value. + */ +static inline void +__wt_timestamp_set_inf(uint8_t *ts) +{ + memset(ts, 0xff, WT_TIMESTAMP_SIZE); +} + +/* + * __wt_timestamp_set_zero -- + * Zero out a timestamp. + */ +static inline void +__wt_timestamp_set_zero(uint8_t *ts) +{ + memset(ts, 0x00, WT_TIMESTAMP_SIZE); +} #endif /* diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 60bacbcf51d..d8d8b864766 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -921,6 +921,12 @@ struct __wt_session { * cache. Not compatible with LSM tables; see @ref * tuning_cache_resident for more information., a boolean flag; default * \c false.} + * @config{log = (, the transaction log configuration for this object. + * Only valid if log is enabled in ::wiredtiger_open., a set of related + * configuration options defined below.} + * @config{ enabled, if false\, this object has + * checkpoint-level durability., a boolean flag; default \c true.} + * @config{ ),,} * @configend * @errors */ @@ -1971,6 +1977,11 @@ struct __wt_connection { * checkpoint; setting this value above 0 configures periodic * checkpoints., an integer between 0 and 100000; default \c 0.} * @config{ ),,} + * @config{compatibility = (, set compatibility version of database., a + * set of related configuration options defined below.} + * @config{ release, compatibility release + * version string., a string; default empty.} + * @config{ ),,} * @config{error_prefix, prefix string for error messages., a string; * default empty.} * @config{eviction = (, eviction configuration options., a set of @@ -2454,6 +2465,11 @@ struct __wt_connection { * @config{ ),,} * @config{checkpoint_sync, flush files to stable storage when closing or * writing checkpoints., a boolean flag; default \c true.} + * @config{compatibility = (, set compatibility version of database., a set of + * related configuration options defined below.} + * @config{ release, compatibility release version + * string., a string; default empty.} + * @config{ ),,} * @config{config_base, write the base configuration file if creating the * database. If \c false in the config passed directly to ::wiredtiger_open\, * will ignore any existing base configuration file in addition to not creating @@ -4471,6 +4487,8 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_LOGREC_FILE_SYNC 2 /*! message */ #define WT_LOGREC_MESSAGE 3 +/*! system/internal record */ +#define WT_LOGREC_SYSTEM 4 /*! invalid operation */ #define WT_LOGOP_INVALID 0 /*! column put */ @@ -4596,475 +4614,505 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_CACHE_EVICTION_AGGRESSIVE_SET 1044 /*! cache: eviction empty score */ #define WT_STAT_CONN_CACHE_EVICTION_EMPTY_SCORE 1045 +/*! cache: eviction passes of a file */ +#define WT_STAT_CONN_CACHE_EVICTION_WALK_PASSES 1046 /*! cache: eviction server candidate queue empty when topping up */ -#define WT_STAT_CONN_CACHE_EVICTION_QUEUE_EMPTY 1046 +#define WT_STAT_CONN_CACHE_EVICTION_QUEUE_EMPTY 1047 /*! cache: eviction server candidate queue not empty when topping up */ -#define WT_STAT_CONN_CACHE_EVICTION_QUEUE_NOT_EMPTY 1047 +#define WT_STAT_CONN_CACHE_EVICTION_QUEUE_NOT_EMPTY 1048 /*! cache: eviction server evicting pages */ -#define WT_STAT_CONN_CACHE_EVICTION_SERVER_EVICTING 1048 +#define WT_STAT_CONN_CACHE_EVICTION_SERVER_EVICTING 1049 /*! * cache: eviction server slept, because we did not make progress with * eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SERVER_SLEPT 1049 +#define WT_STAT_CONN_CACHE_EVICTION_SERVER_SLEPT 1050 /*! cache: eviction server unable to reach eviction goal */ -#define WT_STAT_CONN_CACHE_EVICTION_SLOW 1050 +#define WT_STAT_CONN_CACHE_EVICTION_SLOW 1051 /*! cache: eviction state */ -#define WT_STAT_CONN_CACHE_EVICTION_STATE 1051 +#define WT_STAT_CONN_CACHE_EVICTION_STATE 1052 +/*! cache: eviction walk target pages histogram - 0-9 */ +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT10 1053 +/*! cache: eviction walk target pages histogram - 10-31 */ +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT32 1054 +/*! cache: eviction walk target pages histogram - 128 and higher */ +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_GE128 1055 +/*! cache: eviction walk target pages histogram - 32-63 */ +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT64 1056 +/*! cache: eviction walk target pages histogram - 64-128 */ +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT128 1057 /*! cache: eviction walks abandoned */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ABANDONED 1052 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ABANDONED 1058 +/*! cache: eviction walks gave up because they restarted their walk twice */ +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_STOPPED 1059 +/*! + * cache: eviction walks gave up because they saw too many pages and + * found no candidates + */ +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_GAVE_UP_NO_TARGETS 1060 +/*! + * cache: eviction walks gave up because they saw too many pages and + * found too few candidates + */ +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_GAVE_UP_RATIO 1061 +/*! cache: eviction walks reached end of tree */ +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ENDED 1062 +/*! cache: eviction walks started from root of tree */ +#define WT_STAT_CONN_CACHE_EVICTION_WALK_FROM_ROOT 1063 +/*! cache: eviction walks started from saved location in tree */ +#define WT_STAT_CONN_CACHE_EVICTION_WALK_SAVED_POS 1064 /*! cache: eviction worker thread active */ -#define WT_STAT_CONN_CACHE_EVICTION_ACTIVE_WORKERS 1053 +#define WT_STAT_CONN_CACHE_EVICTION_ACTIVE_WORKERS 1065 /*! cache: eviction worker thread created */ -#define WT_STAT_CONN_CACHE_EVICTION_WORKER_CREATED 1054 +#define WT_STAT_CONN_CACHE_EVICTION_WORKER_CREATED 1066 /*! cache: eviction worker thread evicting pages */ -#define WT_STAT_CONN_CACHE_EVICTION_WORKER_EVICTING 1055 +#define WT_STAT_CONN_CACHE_EVICTION_WORKER_EVICTING 1067 /*! cache: eviction worker thread removed */ -#define WT_STAT_CONN_CACHE_EVICTION_WORKER_REMOVED 1056 +#define WT_STAT_CONN_CACHE_EVICTION_WORKER_REMOVED 1068 /*! cache: eviction worker thread stable number */ -#define WT_STAT_CONN_CACHE_EVICTION_STABLE_STATE_WORKERS 1057 +#define WT_STAT_CONN_CACHE_EVICTION_STABLE_STATE_WORKERS 1069 /*! * cache: failed eviction of pages that exceeded the in-memory maximum * count */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL 1058 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL 1070 /*! * cache: failed eviction of pages that exceeded the in-memory maximum * time (usecs) */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL_TIME 1059 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL_TIME 1071 /*! cache: files with active eviction walks */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ACTIVE 1060 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ACTIVE 1072 /*! cache: files with new eviction walks started */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_STARTED 1061 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_STARTED 1073 /*! cache: force re-tuning of eviction workers once in a while */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_RETUNE 1062 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_RETUNE 1074 /*! cache: hazard pointer blocked page eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_HAZARD 1063 +#define WT_STAT_CONN_CACHE_EVICTION_HAZARD 1075 /*! cache: hazard pointer check calls */ -#define WT_STAT_CONN_CACHE_HAZARD_CHECKS 1064 +#define WT_STAT_CONN_CACHE_HAZARD_CHECKS 1076 /*! cache: hazard pointer check entries walked */ -#define WT_STAT_CONN_CACHE_HAZARD_WALKS 1065 +#define WT_STAT_CONN_CACHE_HAZARD_WALKS 1077 /*! cache: hazard pointer maximum array length */ -#define WT_STAT_CONN_CACHE_HAZARD_MAX 1066 +#define WT_STAT_CONN_CACHE_HAZARD_MAX 1078 /*! cache: in-memory page passed criteria to be split */ -#define WT_STAT_CONN_CACHE_INMEM_SPLITTABLE 1067 +#define WT_STAT_CONN_CACHE_INMEM_SPLITTABLE 1079 /*! cache: in-memory page splits */ -#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1068 +#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1080 /*! cache: internal pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1069 +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1081 /*! cache: internal pages split during eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_INTERNAL 1070 +#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_INTERNAL 1082 /*! cache: leaf pages split during eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_LEAF 1071 +#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_LEAF 1083 /*! cache: lookaside table insert calls */ -#define WT_STAT_CONN_CACHE_LOOKASIDE_INSERT 1072 +#define WT_STAT_CONN_CACHE_LOOKASIDE_INSERT 1084 /*! cache: lookaside table remove calls */ -#define WT_STAT_CONN_CACHE_LOOKASIDE_REMOVE 1073 +#define WT_STAT_CONN_CACHE_LOOKASIDE_REMOVE 1085 /*! cache: maximum bytes configured */ -#define WT_STAT_CONN_CACHE_BYTES_MAX 1074 +#define WT_STAT_CONN_CACHE_BYTES_MAX 1086 /*! cache: maximum page size at eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_MAXIMUM_PAGE_SIZE 1075 +#define WT_STAT_CONN_CACHE_EVICTION_MAXIMUM_PAGE_SIZE 1087 /*! cache: modified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1076 +#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1088 /*! cache: modified pages evicted by application threads */ -#define WT_STAT_CONN_CACHE_EVICTION_APP_DIRTY 1077 +#define WT_STAT_CONN_CACHE_EVICTION_APP_DIRTY 1089 /*! cache: overflow pages read into cache */ -#define WT_STAT_CONN_CACHE_READ_OVERFLOW 1078 -/*! cache: overflow values cached in memory */ -#define WT_STAT_CONN_CACHE_OVERFLOW_VALUE 1079 +#define WT_STAT_CONN_CACHE_READ_OVERFLOW 1090 /*! cache: page split during eviction deepened the tree */ -#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1080 +#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1091 /*! cache: page written requiring lookaside records */ -#define WT_STAT_CONN_CACHE_WRITE_LOOKASIDE 1081 +#define WT_STAT_CONN_CACHE_WRITE_LOOKASIDE 1092 /*! cache: pages currently held in the cache */ -#define WT_STAT_CONN_CACHE_PAGES_INUSE 1082 +#define WT_STAT_CONN_CACHE_PAGES_INUSE 1093 /*! cache: pages evicted because they exceeded the in-memory maximum count */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE 1083 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE 1094 /*! * cache: pages evicted because they exceeded the in-memory maximum time * (usecs) */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_TIME 1084 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_TIME 1095 /*! cache: pages evicted because they had chains of deleted items count */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE 1085 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE 1096 /*! * cache: pages evicted because they had chains of deleted items time * (usecs) */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE_TIME 1086 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE_TIME 1097 /*! cache: pages evicted by application threads */ -#define WT_STAT_CONN_CACHE_EVICTION_APP 1087 +#define WT_STAT_CONN_CACHE_EVICTION_APP 1098 /*! cache: pages queued for eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED 1088 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED 1099 /*! cache: pages queued for urgent eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_URGENT 1089 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_URGENT 1100 /*! cache: pages queued for urgent eviction during walk */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_OLDEST 1090 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_OLDEST 1101 /*! cache: pages read into cache */ -#define WT_STAT_CONN_CACHE_READ 1091 +#define WT_STAT_CONN_CACHE_READ 1102 /*! cache: pages read into cache requiring lookaside entries */ -#define WT_STAT_CONN_CACHE_READ_LOOKASIDE 1092 +#define WT_STAT_CONN_CACHE_READ_LOOKASIDE 1103 /*! cache: pages requested from the cache */ -#define WT_STAT_CONN_CACHE_PAGES_REQUESTED 1093 +#define WT_STAT_CONN_CACHE_PAGES_REQUESTED 1104 /*! cache: pages seen by eviction walk */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_SEEN 1094 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_SEEN 1105 /*! cache: pages selected for eviction unable to be evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL 1095 +#define WT_STAT_CONN_CACHE_EVICTION_FAIL 1106 /*! cache: pages walked for eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK 1096 +#define WT_STAT_CONN_CACHE_EVICTION_WALK 1107 /*! cache: pages written from cache */ -#define WT_STAT_CONN_CACHE_WRITE 1097 +#define WT_STAT_CONN_CACHE_WRITE 1108 /*! cache: pages written requiring in-memory restoration */ -#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1098 +#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1109 /*! cache: percentage overhead */ -#define WT_STAT_CONN_CACHE_OVERHEAD 1099 +#define WT_STAT_CONN_CACHE_OVERHEAD 1110 /*! cache: tracked bytes belonging to internal pages in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_INTERNAL 1100 +#define WT_STAT_CONN_CACHE_BYTES_INTERNAL 1111 /*! cache: tracked bytes belonging to leaf pages in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_LEAF 1101 +#define WT_STAT_CONN_CACHE_BYTES_LEAF 1112 /*! cache: tracked dirty bytes in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_DIRTY 1102 +#define WT_STAT_CONN_CACHE_BYTES_DIRTY 1113 /*! cache: tracked dirty pages in the cache */ -#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1103 +#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1114 /*! cache: unmodified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1104 +#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1115 /*! connection: auto adjusting condition resets */ -#define WT_STAT_CONN_COND_AUTO_WAIT_RESET 1105 +#define WT_STAT_CONN_COND_AUTO_WAIT_RESET 1116 /*! connection: auto adjusting condition wait calls */ -#define WT_STAT_CONN_COND_AUTO_WAIT 1106 +#define WT_STAT_CONN_COND_AUTO_WAIT 1117 /*! connection: detected system time went backwards */ -#define WT_STAT_CONN_TIME_TRAVEL 1107 +#define WT_STAT_CONN_TIME_TRAVEL 1118 /*! connection: files currently open */ -#define WT_STAT_CONN_FILE_OPEN 1108 +#define WT_STAT_CONN_FILE_OPEN 1119 /*! connection: memory allocations */ -#define WT_STAT_CONN_MEMORY_ALLOCATION 1109 +#define WT_STAT_CONN_MEMORY_ALLOCATION 1120 /*! connection: memory frees */ -#define WT_STAT_CONN_MEMORY_FREE 1110 +#define WT_STAT_CONN_MEMORY_FREE 1121 /*! connection: memory re-allocations */ -#define WT_STAT_CONN_MEMORY_GROW 1111 +#define WT_STAT_CONN_MEMORY_GROW 1122 /*! connection: pthread mutex condition wait calls */ -#define WT_STAT_CONN_COND_WAIT 1112 +#define WT_STAT_CONN_COND_WAIT 1123 /*! connection: pthread mutex shared lock read-lock calls */ -#define WT_STAT_CONN_RWLOCK_READ 1113 +#define WT_STAT_CONN_RWLOCK_READ 1124 /*! connection: pthread mutex shared lock write-lock calls */ -#define WT_STAT_CONN_RWLOCK_WRITE 1114 +#define WT_STAT_CONN_RWLOCK_WRITE 1125 /*! connection: total fsync I/Os */ -#define WT_STAT_CONN_FSYNC_IO 1115 +#define WT_STAT_CONN_FSYNC_IO 1126 /*! connection: total read I/Os */ -#define WT_STAT_CONN_READ_IO 1116 +#define WT_STAT_CONN_READ_IO 1127 /*! connection: total write I/Os */ -#define WT_STAT_CONN_WRITE_IO 1117 +#define WT_STAT_CONN_WRITE_IO 1128 /*! cursor: cursor create calls */ -#define WT_STAT_CONN_CURSOR_CREATE 1118 +#define WT_STAT_CONN_CURSOR_CREATE 1129 /*! cursor: cursor insert calls */ -#define WT_STAT_CONN_CURSOR_INSERT 1119 +#define WT_STAT_CONN_CURSOR_INSERT 1130 /*! cursor: cursor modify calls */ -#define WT_STAT_CONN_CURSOR_MODIFY 1120 +#define WT_STAT_CONN_CURSOR_MODIFY 1131 /*! cursor: cursor next calls */ -#define WT_STAT_CONN_CURSOR_NEXT 1121 +#define WT_STAT_CONN_CURSOR_NEXT 1132 /*! cursor: cursor prev calls */ -#define WT_STAT_CONN_CURSOR_PREV 1122 +#define WT_STAT_CONN_CURSOR_PREV 1133 /*! cursor: cursor remove calls */ -#define WT_STAT_CONN_CURSOR_REMOVE 1123 +#define WT_STAT_CONN_CURSOR_REMOVE 1134 /*! cursor: cursor reserve calls */ -#define WT_STAT_CONN_CURSOR_RESERVE 1124 +#define WT_STAT_CONN_CURSOR_RESERVE 1135 /*! cursor: cursor reset calls */ -#define WT_STAT_CONN_CURSOR_RESET 1125 +#define WT_STAT_CONN_CURSOR_RESET 1136 /*! cursor: cursor restarted searches */ -#define WT_STAT_CONN_CURSOR_RESTART 1126 +#define WT_STAT_CONN_CURSOR_RESTART 1137 /*! cursor: cursor search calls */ -#define WT_STAT_CONN_CURSOR_SEARCH 1127 +#define WT_STAT_CONN_CURSOR_SEARCH 1138 /*! cursor: cursor search near calls */ -#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1128 +#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1139 /*! cursor: cursor update calls */ -#define WT_STAT_CONN_CURSOR_UPDATE 1129 +#define WT_STAT_CONN_CURSOR_UPDATE 1140 /*! cursor: truncate calls */ -#define WT_STAT_CONN_CURSOR_TRUNCATE 1130 +#define WT_STAT_CONN_CURSOR_TRUNCATE 1141 /*! data-handle: connection data handles currently active */ -#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1131 +#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1142 /*! data-handle: connection sweep candidate became referenced */ -#define WT_STAT_CONN_DH_SWEEP_REF 1132 +#define WT_STAT_CONN_DH_SWEEP_REF 1143 /*! data-handle: connection sweep dhandles closed */ -#define WT_STAT_CONN_DH_SWEEP_CLOSE 1133 +#define WT_STAT_CONN_DH_SWEEP_CLOSE 1144 /*! data-handle: connection sweep dhandles removed from hash list */ -#define WT_STAT_CONN_DH_SWEEP_REMOVE 1134 +#define WT_STAT_CONN_DH_SWEEP_REMOVE 1145 /*! data-handle: connection sweep time-of-death sets */ -#define WT_STAT_CONN_DH_SWEEP_TOD 1135 +#define WT_STAT_CONN_DH_SWEEP_TOD 1146 /*! data-handle: connection sweeps */ -#define WT_STAT_CONN_DH_SWEEPS 1136 +#define WT_STAT_CONN_DH_SWEEPS 1147 /*! data-handle: session dhandles swept */ -#define WT_STAT_CONN_DH_SESSION_HANDLES 1137 +#define WT_STAT_CONN_DH_SESSION_HANDLES 1148 /*! data-handle: session sweep attempts */ -#define WT_STAT_CONN_DH_SESSION_SWEEPS 1138 +#define WT_STAT_CONN_DH_SESSION_SWEEPS 1149 /*! lock: checkpoint lock acquisitions */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1139 +#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1150 /*! lock: checkpoint lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1140 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1151 /*! lock: checkpoint lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1141 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1152 /*! * lock: dhandle lock application thread time waiting for the dhandle * lock (usecs) */ -#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_APPLICATION 1142 +#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_APPLICATION 1153 /*! * lock: dhandle lock internal thread time waiting for the dhandle lock * (usecs) */ -#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_INTERNAL 1143 +#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_INTERNAL 1154 /*! lock: dhandle read lock acquisitions */ -#define WT_STAT_CONN_LOCK_DHANDLE_READ_COUNT 1144 +#define WT_STAT_CONN_LOCK_DHANDLE_READ_COUNT 1155 /*! lock: dhandle write lock acquisitions */ -#define WT_STAT_CONN_LOCK_DHANDLE_WRITE_COUNT 1145 +#define WT_STAT_CONN_LOCK_DHANDLE_WRITE_COUNT 1156 /*! lock: metadata lock acquisitions */ -#define WT_STAT_CONN_LOCK_METADATA_COUNT 1146 +#define WT_STAT_CONN_LOCK_METADATA_COUNT 1157 /*! lock: metadata lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1147 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1158 /*! lock: metadata lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1148 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1159 /*! lock: schema lock acquisitions */ -#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1149 +#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1160 /*! lock: schema lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1150 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1161 /*! lock: schema lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1151 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1162 /*! * lock: table lock application thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1152 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1163 /*! * lock: table lock internal thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1153 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1164 /*! lock: table read lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_READ_COUNT 1154 +#define WT_STAT_CONN_LOCK_TABLE_READ_COUNT 1165 /*! lock: table write lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_WRITE_COUNT 1155 +#define WT_STAT_CONN_LOCK_TABLE_WRITE_COUNT 1166 /*! log: busy returns attempting to switch slots */ -#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1156 +#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1167 +/*! log: force checkpoint calls slept */ +#define WT_STAT_CONN_LOG_FORCE_CKPT_SLEEP 1168 /*! log: log bytes of payload data */ -#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1157 +#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1169 /*! log: log bytes written */ -#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1158 +#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1170 /*! log: log files manually zero-filled */ -#define WT_STAT_CONN_LOG_ZERO_FILLS 1159 +#define WT_STAT_CONN_LOG_ZERO_FILLS 1171 /*! log: log flush operations */ -#define WT_STAT_CONN_LOG_FLUSH 1160 +#define WT_STAT_CONN_LOG_FLUSH 1172 /*! log: log force write operations */ -#define WT_STAT_CONN_LOG_FORCE_WRITE 1161 +#define WT_STAT_CONN_LOG_FORCE_WRITE 1173 /*! log: log force write operations skipped */ -#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1162 +#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1174 /*! log: log records compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1163 +#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1175 /*! log: log records not compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1164 +#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1176 /*! log: log records too small to compress */ -#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1165 +#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1177 /*! log: log release advances write LSN */ -#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1166 +#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1178 /*! log: log scan operations */ -#define WT_STAT_CONN_LOG_SCANS 1167 +#define WT_STAT_CONN_LOG_SCANS 1179 /*! log: log scan records requiring two reads */ -#define WT_STAT_CONN_LOG_SCAN_REREADS 1168 +#define WT_STAT_CONN_LOG_SCAN_REREADS 1180 /*! log: log server thread advances write LSN */ -#define WT_STAT_CONN_LOG_WRITE_LSN 1169 +#define WT_STAT_CONN_LOG_WRITE_LSN 1181 /*! log: log server thread write LSN walk skipped */ -#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1170 +#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1182 /*! log: log sync operations */ -#define WT_STAT_CONN_LOG_SYNC 1171 +#define WT_STAT_CONN_LOG_SYNC 1183 /*! log: log sync time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DURATION 1172 +#define WT_STAT_CONN_LOG_SYNC_DURATION 1184 /*! log: log sync_dir operations */ -#define WT_STAT_CONN_LOG_SYNC_DIR 1173 +#define WT_STAT_CONN_LOG_SYNC_DIR 1185 /*! log: log sync_dir time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1174 +#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1186 /*! log: log write operations */ -#define WT_STAT_CONN_LOG_WRITES 1175 +#define WT_STAT_CONN_LOG_WRITES 1187 /*! log: logging bytes consolidated */ -#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1176 +#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1188 /*! log: maximum log file size */ -#define WT_STAT_CONN_LOG_MAX_FILESIZE 1177 +#define WT_STAT_CONN_LOG_MAX_FILESIZE 1189 /*! log: number of pre-allocated log files to create */ -#define WT_STAT_CONN_LOG_PREALLOC_MAX 1178 +#define WT_STAT_CONN_LOG_PREALLOC_MAX 1190 /*! log: pre-allocated log files not ready and missed */ -#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1179 +#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1191 /*! log: pre-allocated log files prepared */ -#define WT_STAT_CONN_LOG_PREALLOC_FILES 1180 +#define WT_STAT_CONN_LOG_PREALLOC_FILES 1192 /*! log: pre-allocated log files used */ -#define WT_STAT_CONN_LOG_PREALLOC_USED 1181 +#define WT_STAT_CONN_LOG_PREALLOC_USED 1193 /*! log: records processed by log scan */ -#define WT_STAT_CONN_LOG_SCAN_RECORDS 1182 +#define WT_STAT_CONN_LOG_SCAN_RECORDS 1194 /*! log: slot close lost race */ -#define WT_STAT_CONN_LOG_SLOT_CLOSE_RACE 1183 +#define WT_STAT_CONN_LOG_SLOT_CLOSE_RACE 1195 /*! log: slot close unbuffered waits */ -#define WT_STAT_CONN_LOG_SLOT_CLOSE_UNBUF 1184 +#define WT_STAT_CONN_LOG_SLOT_CLOSE_UNBUF 1196 /*! log: slot closures */ -#define WT_STAT_CONN_LOG_SLOT_CLOSES 1185 +#define WT_STAT_CONN_LOG_SLOT_CLOSES 1197 /*! log: slot join atomic update races */ -#define WT_STAT_CONN_LOG_SLOT_RACES 1186 +#define WT_STAT_CONN_LOG_SLOT_RACES 1198 /*! log: slot join calls atomic updates raced */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_RACE 1187 +#define WT_STAT_CONN_LOG_SLOT_YIELD_RACE 1199 /*! log: slot join calls did not yield */ -#define WT_STAT_CONN_LOG_SLOT_IMMEDIATE 1188 +#define WT_STAT_CONN_LOG_SLOT_IMMEDIATE 1200 /*! log: slot join calls found active slot closed */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_CLOSE 1189 +#define WT_STAT_CONN_LOG_SLOT_YIELD_CLOSE 1201 /*! log: slot join calls slept */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_SLEEP 1190 +#define WT_STAT_CONN_LOG_SLOT_YIELD_SLEEP 1202 /*! log: slot join calls yielded */ -#define WT_STAT_CONN_LOG_SLOT_YIELD 1191 +#define WT_STAT_CONN_LOG_SLOT_YIELD 1203 /*! log: slot join found active slot closed */ -#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1192 +#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1204 /*! log: slot joins yield time (usecs) */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_DURATION 1193 +#define WT_STAT_CONN_LOG_SLOT_YIELD_DURATION 1205 /*! log: slot transitions unable to find free slot */ -#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1194 +#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1206 /*! log: slot unbuffered writes */ -#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1195 +#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1207 /*! log: total in-memory size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_MEM 1196 +#define WT_STAT_CONN_LOG_COMPRESS_MEM 1208 /*! log: total log buffer size */ -#define WT_STAT_CONN_LOG_BUFFER_SIZE 1197 +#define WT_STAT_CONN_LOG_BUFFER_SIZE 1209 /*! log: total size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_LEN 1198 +#define WT_STAT_CONN_LOG_COMPRESS_LEN 1210 /*! log: written slots coalesced */ -#define WT_STAT_CONN_LOG_SLOT_COALESCED 1199 +#define WT_STAT_CONN_LOG_SLOT_COALESCED 1211 /*! log: yields waiting for previous log file close */ -#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1200 +#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1212 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1201 +#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1213 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_CONN_REC_PAGES 1202 +#define WT_STAT_CONN_REC_PAGES 1214 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_CONN_REC_PAGES_EVICTION 1203 +#define WT_STAT_CONN_REC_PAGES_EVICTION 1215 /*! reconciliation: pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE 1204 +#define WT_STAT_CONN_REC_PAGE_DELETE 1216 /*! reconciliation: split bytes currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1205 +#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1217 /*! reconciliation: split objects currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1206 +#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1218 /*! session: open cursor count */ -#define WT_STAT_CONN_SESSION_CURSOR_OPEN 1207 +#define WT_STAT_CONN_SESSION_CURSOR_OPEN 1219 /*! session: open session count */ -#define WT_STAT_CONN_SESSION_OPEN 1208 +#define WT_STAT_CONN_SESSION_OPEN 1220 /*! session: table alter failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1209 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1221 /*! session: table alter successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1210 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1222 /*! session: table alter unchanged and skipped */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1211 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1223 /*! session: table compact failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1212 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1224 /*! session: table compact successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1213 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1225 /*! session: table create failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1214 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1226 /*! session: table create successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1215 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1227 /*! session: table drop failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1216 +#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1228 /*! session: table drop successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1217 +#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1229 /*! session: table rebalance failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1218 +#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1230 /*! session: table rebalance successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1219 +#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1231 /*! session: table rename failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1220 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1232 /*! session: table rename successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1221 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1233 /*! session: table salvage failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1222 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1234 /*! session: table salvage successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1223 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1235 /*! session: table truncate failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1224 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1236 /*! session: table truncate successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1225 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1237 /*! session: table verify failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1226 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1238 /*! session: table verify successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1227 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1239 /*! thread-state: active filesystem fsync calls */ -#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1228 +#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1240 /*! thread-state: active filesystem read calls */ -#define WT_STAT_CONN_THREAD_READ_ACTIVE 1229 +#define WT_STAT_CONN_THREAD_READ_ACTIVE 1241 /*! thread-state: active filesystem write calls */ -#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1230 +#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1242 /*! thread-yield: application thread time evicting (usecs) */ -#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1231 +#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1243 /*! thread-yield: application thread time waiting for cache (usecs) */ -#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1232 +#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1244 /*! thread-yield: page acquire busy blocked */ -#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1233 +#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1245 /*! thread-yield: page acquire eviction blocked */ -#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1234 +#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1246 /*! thread-yield: page acquire locked blocked */ -#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1235 +#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1247 /*! thread-yield: page acquire read blocked */ -#define WT_STAT_CONN_PAGE_READ_BLOCKED 1236 +#define WT_STAT_CONN_PAGE_READ_BLOCKED 1248 /*! thread-yield: page acquire time sleeping (usecs) */ -#define WT_STAT_CONN_PAGE_SLEEP 1237 +#define WT_STAT_CONN_PAGE_SLEEP 1249 /*! transaction: number of named snapshots created */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1238 +#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1250 /*! transaction: number of named snapshots dropped */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1239 +#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1251 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1240 +#define WT_STAT_CONN_TXN_BEGIN 1252 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1241 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1253 /*! transaction: transaction checkpoint generation */ -#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1242 +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1254 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1243 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1255 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1244 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1256 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1245 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1257 /*! transaction: transaction checkpoint scrub dirty target */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1246 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1258 /*! transaction: transaction checkpoint scrub time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1247 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1259 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1248 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1260 /*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1249 +#define WT_STAT_CONN_TXN_CHECKPOINT 1261 /*! * transaction: transaction checkpoints skipped because database was * clean */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1250 +#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1262 /*! transaction: transaction failures due to cache overflow */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1251 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1263 /*! * transaction: transaction fsync calls for checkpoint after allocating * the transaction ID */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1252 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1264 /*! * transaction: transaction fsync duration for checkpoint after * allocating the transaction ID (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1253 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1265 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1254 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1266 /*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1255 +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1267 /*! * transaction: transaction range of IDs currently pinned by named * snapshots */ -#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1256 +#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1268 /*! transaction: transaction sync calls */ -#define WT_STAT_CONN_TXN_SYNC 1257 +#define WT_STAT_CONN_TXN_SYNC 1269 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1258 +#define WT_STAT_CONN_TXN_COMMIT 1270 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1259 +#define WT_STAT_CONN_TXN_ROLLBACK 1271 /*! transaction: update conflicts */ -#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1260 +#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1272 /*! * @} @@ -5192,219 +5240,268 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_DSRC_CACHE_EVICTION_CHECKPOINT 2043 /*! cache: data source pages selected for eviction unable to be evicted */ #define WT_STAT_DSRC_CACHE_EVICTION_FAIL 2044 +/*! cache: eviction walk passes of a file */ +#define WT_STAT_DSRC_CACHE_EVICTION_WALK_PASSES 2045 +/*! cache: eviction walk target pages histogram - 0-9 */ +#define WT_STAT_DSRC_CACHE_EVICTION_TARGET_PAGE_LT10 2046 +/*! cache: eviction walk target pages histogram - 10-31 */ +#define WT_STAT_DSRC_CACHE_EVICTION_TARGET_PAGE_LT32 2047 +/*! cache: eviction walk target pages histogram - 128 and higher */ +#define WT_STAT_DSRC_CACHE_EVICTION_TARGET_PAGE_GE128 2048 +/*! cache: eviction walk target pages histogram - 32-63 */ +#define WT_STAT_DSRC_CACHE_EVICTION_TARGET_PAGE_LT64 2049 +/*! cache: eviction walk target pages histogram - 64-128 */ +#define WT_STAT_DSRC_CACHE_EVICTION_TARGET_PAGE_LT128 2050 +/*! cache: eviction walks abandoned */ +#define WT_STAT_DSRC_CACHE_EVICTION_WALKS_ABANDONED 2051 +/*! cache: eviction walks gave up because they restarted their walk twice */ +#define WT_STAT_DSRC_CACHE_EVICTION_WALKS_STOPPED 2052 +/*! + * cache: eviction walks gave up because they saw too many pages and + * found no candidates + */ +#define WT_STAT_DSRC_CACHE_EVICTION_WALKS_GAVE_UP_NO_TARGETS 2053 +/*! + * cache: eviction walks gave up because they saw too many pages and + * found too few candidates + */ +#define WT_STAT_DSRC_CACHE_EVICTION_WALKS_GAVE_UP_RATIO 2054 +/*! cache: eviction walks reached end of tree */ +#define WT_STAT_DSRC_CACHE_EVICTION_WALKS_ENDED 2055 +/*! cache: eviction walks started from root of tree */ +#define WT_STAT_DSRC_CACHE_EVICTION_WALK_FROM_ROOT 2056 +/*! cache: eviction walks started from saved location in tree */ +#define WT_STAT_DSRC_CACHE_EVICTION_WALK_SAVED_POS 2057 /*! cache: hazard pointer blocked page eviction */ -#define WT_STAT_DSRC_CACHE_EVICTION_HAZARD 2045 +#define WT_STAT_DSRC_CACHE_EVICTION_HAZARD 2058 /*! cache: in-memory page passed criteria to be split */ -#define WT_STAT_DSRC_CACHE_INMEM_SPLITTABLE 2046 +#define WT_STAT_DSRC_CACHE_INMEM_SPLITTABLE 2059 /*! cache: in-memory page splits */ -#define WT_STAT_DSRC_CACHE_INMEM_SPLIT 2047 +#define WT_STAT_DSRC_CACHE_INMEM_SPLIT 2060 /*! cache: internal pages evicted */ -#define WT_STAT_DSRC_CACHE_EVICTION_INTERNAL 2048 +#define WT_STAT_DSRC_CACHE_EVICTION_INTERNAL 2061 /*! cache: internal pages split during eviction */ -#define WT_STAT_DSRC_CACHE_EVICTION_SPLIT_INTERNAL 2049 +#define WT_STAT_DSRC_CACHE_EVICTION_SPLIT_INTERNAL 2062 /*! cache: leaf pages split during eviction */ -#define WT_STAT_DSRC_CACHE_EVICTION_SPLIT_LEAF 2050 +#define WT_STAT_DSRC_CACHE_EVICTION_SPLIT_LEAF 2063 /*! cache: modified pages evicted */ -#define WT_STAT_DSRC_CACHE_EVICTION_DIRTY 2051 +#define WT_STAT_DSRC_CACHE_EVICTION_DIRTY 2064 /*! cache: overflow pages read into cache */ -#define WT_STAT_DSRC_CACHE_READ_OVERFLOW 2052 -/*! cache: overflow values cached in memory */ -#define WT_STAT_DSRC_CACHE_OVERFLOW_VALUE 2053 +#define WT_STAT_DSRC_CACHE_READ_OVERFLOW 2065 /*! cache: page split during eviction deepened the tree */ -#define WT_STAT_DSRC_CACHE_EVICTION_DEEPEN 2054 +#define WT_STAT_DSRC_CACHE_EVICTION_DEEPEN 2066 /*! cache: page written requiring lookaside records */ -#define WT_STAT_DSRC_CACHE_WRITE_LOOKASIDE 2055 +#define WT_STAT_DSRC_CACHE_WRITE_LOOKASIDE 2067 /*! cache: pages read into cache */ -#define WT_STAT_DSRC_CACHE_READ 2056 +#define WT_STAT_DSRC_CACHE_READ 2068 /*! cache: pages read into cache requiring lookaside entries */ -#define WT_STAT_DSRC_CACHE_READ_LOOKASIDE 2057 +#define WT_STAT_DSRC_CACHE_READ_LOOKASIDE 2069 /*! cache: pages requested from the cache */ -#define WT_STAT_DSRC_CACHE_PAGES_REQUESTED 2058 +#define WT_STAT_DSRC_CACHE_PAGES_REQUESTED 2070 +/*! cache: pages seen by eviction walk */ +#define WT_STAT_DSRC_CACHE_EVICTION_PAGES_SEEN 2071 /*! cache: pages written from cache */ -#define WT_STAT_DSRC_CACHE_WRITE 2059 +#define WT_STAT_DSRC_CACHE_WRITE 2072 /*! cache: pages written requiring in-memory restoration */ -#define WT_STAT_DSRC_CACHE_WRITE_RESTORE 2060 +#define WT_STAT_DSRC_CACHE_WRITE_RESTORE 2073 /*! cache: tracked dirty bytes in the cache */ -#define WT_STAT_DSRC_CACHE_BYTES_DIRTY 2061 +#define WT_STAT_DSRC_CACHE_BYTES_DIRTY 2074 /*! cache: unmodified pages evicted */ -#define WT_STAT_DSRC_CACHE_EVICTION_CLEAN 2062 +#define WT_STAT_DSRC_CACHE_EVICTION_CLEAN 2075 /*! * cache_walk: Average difference between current eviction generation * when the page was last considered, only reported if cache_walk or all * statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_GEN_AVG_GAP 2063 +#define WT_STAT_DSRC_CACHE_STATE_GEN_AVG_GAP 2076 /*! * cache_walk: Average on-disk page image size seen, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_AVG_WRITTEN_SIZE 2064 +#define WT_STAT_DSRC_CACHE_STATE_AVG_WRITTEN_SIZE 2077 +/*! + * cache_walk: Average time in cache for pages that have been visited by + * the eviction server, only reported if cache_walk or all statistics are + * enabled + */ +#define WT_STAT_DSRC_CACHE_STATE_AVG_VISITED_AGE 2078 +/*! + * cache_walk: Average time in cache for pages that have not been visited + * by the eviction server, only reported if cache_walk or all statistics + * are enabled + */ +#define WT_STAT_DSRC_CACHE_STATE_AVG_UNVISITED_AGE 2079 /*! * cache_walk: Clean pages currently in cache, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_PAGES_CLEAN 2065 +#define WT_STAT_DSRC_CACHE_STATE_PAGES_CLEAN 2080 /*! * cache_walk: Current eviction generation, only reported if cache_walk * or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_GEN_CURRENT 2066 +#define WT_STAT_DSRC_CACHE_STATE_GEN_CURRENT 2081 /*! * cache_walk: Dirty pages currently in cache, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_PAGES_DIRTY 2067 +#define WT_STAT_DSRC_CACHE_STATE_PAGES_DIRTY 2082 /*! * cache_walk: Entries in the root page, only reported if cache_walk or * all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_ROOT_ENTRIES 2068 +#define WT_STAT_DSRC_CACHE_STATE_ROOT_ENTRIES 2083 /*! * cache_walk: Internal pages currently in cache, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_PAGES_INTERNAL 2069 +#define WT_STAT_DSRC_CACHE_STATE_PAGES_INTERNAL 2084 /*! * cache_walk: Leaf pages currently in cache, only reported if cache_walk * or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_PAGES_LEAF 2070 +#define WT_STAT_DSRC_CACHE_STATE_PAGES_LEAF 2085 /*! * cache_walk: Maximum difference between current eviction generation * when the page was last considered, only reported if cache_walk or all * statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_GEN_MAX_GAP 2071 +#define WT_STAT_DSRC_CACHE_STATE_GEN_MAX_GAP 2086 /*! * cache_walk: Maximum page size seen, only reported if cache_walk or all * statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_MAX_PAGESIZE 2072 +#define WT_STAT_DSRC_CACHE_STATE_MAX_PAGESIZE 2087 /*! * cache_walk: Minimum on-disk page image size seen, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_MIN_WRITTEN_SIZE 2073 +#define WT_STAT_DSRC_CACHE_STATE_MIN_WRITTEN_SIZE 2088 +/*! + * cache_walk: Number of pages never visited by eviction server, only + * reported if cache_walk or all statistics are enabled + */ +#define WT_STAT_DSRC_CACHE_STATE_UNVISITED_COUNT 2089 /*! * cache_walk: On-disk page image sizes smaller than a single allocation * unit, only reported if cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_SMALLER_ALLOC_SIZE 2074 +#define WT_STAT_DSRC_CACHE_STATE_SMALLER_ALLOC_SIZE 2090 /*! * cache_walk: Pages created in memory and never written, only reported * if cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_MEMORY 2075 +#define WT_STAT_DSRC_CACHE_STATE_MEMORY 2091 /*! * cache_walk: Pages currently queued for eviction, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_QUEUED 2076 +#define WT_STAT_DSRC_CACHE_STATE_QUEUED 2092 /*! * cache_walk: Pages that could not be queued for eviction, only reported * if cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_NOT_QUEUEABLE 2077 +#define WT_STAT_DSRC_CACHE_STATE_NOT_QUEUEABLE 2093 /*! * cache_walk: Refs skipped during cache traversal, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_REFS_SKIPPED 2078 +#define WT_STAT_DSRC_CACHE_STATE_REFS_SKIPPED 2094 /*! * cache_walk: Size of the root page, only reported if cache_walk or all * statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_ROOT_SIZE 2079 +#define WT_STAT_DSRC_CACHE_STATE_ROOT_SIZE 2095 /*! * cache_walk: Total number of pages currently in cache, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_PAGES 2080 +#define WT_STAT_DSRC_CACHE_STATE_PAGES 2096 /*! compression: compressed pages read */ -#define WT_STAT_DSRC_COMPRESS_READ 2081 +#define WT_STAT_DSRC_COMPRESS_READ 2097 /*! compression: compressed pages written */ -#define WT_STAT_DSRC_COMPRESS_WRITE 2082 +#define WT_STAT_DSRC_COMPRESS_WRITE 2098 /*! compression: page written failed to compress */ -#define WT_STAT_DSRC_COMPRESS_WRITE_FAIL 2083 +#define WT_STAT_DSRC_COMPRESS_WRITE_FAIL 2099 /*! compression: page written was too small to compress */ -#define WT_STAT_DSRC_COMPRESS_WRITE_TOO_SMALL 2084 +#define WT_STAT_DSRC_COMPRESS_WRITE_TOO_SMALL 2100 /*! compression: raw compression call failed, additional data available */ -#define WT_STAT_DSRC_COMPRESS_RAW_FAIL_TEMPORARY 2085 +#define WT_STAT_DSRC_COMPRESS_RAW_FAIL_TEMPORARY 2101 /*! compression: raw compression call failed, no additional data available */ -#define WT_STAT_DSRC_COMPRESS_RAW_FAIL 2086 +#define WT_STAT_DSRC_COMPRESS_RAW_FAIL 2102 /*! compression: raw compression call succeeded */ -#define WT_STAT_DSRC_COMPRESS_RAW_OK 2087 +#define WT_STAT_DSRC_COMPRESS_RAW_OK 2103 /*! cursor: bulk-loaded cursor-insert calls */ -#define WT_STAT_DSRC_CURSOR_INSERT_BULK 2088 +#define WT_STAT_DSRC_CURSOR_INSERT_BULK 2104 /*! cursor: create calls */ -#define WT_STAT_DSRC_CURSOR_CREATE 2089 +#define WT_STAT_DSRC_CURSOR_CREATE 2105 /*! cursor: cursor-insert key and value bytes inserted */ -#define WT_STAT_DSRC_CURSOR_INSERT_BYTES 2090 +#define WT_STAT_DSRC_CURSOR_INSERT_BYTES 2106 /*! cursor: cursor-remove key bytes removed */ -#define WT_STAT_DSRC_CURSOR_REMOVE_BYTES 2091 +#define WT_STAT_DSRC_CURSOR_REMOVE_BYTES 2107 /*! cursor: cursor-update value bytes updated */ -#define WT_STAT_DSRC_CURSOR_UPDATE_BYTES 2092 +#define WT_STAT_DSRC_CURSOR_UPDATE_BYTES 2108 /*! cursor: insert calls */ -#define WT_STAT_DSRC_CURSOR_INSERT 2093 +#define WT_STAT_DSRC_CURSOR_INSERT 2109 /*! cursor: modify calls */ -#define WT_STAT_DSRC_CURSOR_MODIFY 2094 +#define WT_STAT_DSRC_CURSOR_MODIFY 2110 /*! cursor: next calls */ -#define WT_STAT_DSRC_CURSOR_NEXT 2095 +#define WT_STAT_DSRC_CURSOR_NEXT 2111 /*! cursor: prev calls */ -#define WT_STAT_DSRC_CURSOR_PREV 2096 +#define WT_STAT_DSRC_CURSOR_PREV 2112 /*! cursor: remove calls */ -#define WT_STAT_DSRC_CURSOR_REMOVE 2097 +#define WT_STAT_DSRC_CURSOR_REMOVE 2113 /*! cursor: reserve calls */ -#define WT_STAT_DSRC_CURSOR_RESERVE 2098 +#define WT_STAT_DSRC_CURSOR_RESERVE 2114 /*! cursor: reset calls */ -#define WT_STAT_DSRC_CURSOR_RESET 2099 +#define WT_STAT_DSRC_CURSOR_RESET 2115 /*! cursor: restarted searches */ -#define WT_STAT_DSRC_CURSOR_RESTART 2100 +#define WT_STAT_DSRC_CURSOR_RESTART 2116 /*! cursor: search calls */ -#define WT_STAT_DSRC_CURSOR_SEARCH 2101 +#define WT_STAT_DSRC_CURSOR_SEARCH 2117 /*! cursor: search near calls */ -#define WT_STAT_DSRC_CURSOR_SEARCH_NEAR 2102 +#define WT_STAT_DSRC_CURSOR_SEARCH_NEAR 2118 /*! cursor: truncate calls */ -#define WT_STAT_DSRC_CURSOR_TRUNCATE 2103 +#define WT_STAT_DSRC_CURSOR_TRUNCATE 2119 /*! cursor: update calls */ -#define WT_STAT_DSRC_CURSOR_UPDATE 2104 +#define WT_STAT_DSRC_CURSOR_UPDATE 2120 /*! reconciliation: dictionary matches */ -#define WT_STAT_DSRC_REC_DICTIONARY 2105 +#define WT_STAT_DSRC_REC_DICTIONARY 2121 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_DSRC_REC_PAGE_DELETE_FAST 2106 +#define WT_STAT_DSRC_REC_PAGE_DELETE_FAST 2122 /*! * reconciliation: internal page key bytes discarded using suffix * compression */ -#define WT_STAT_DSRC_REC_SUFFIX_COMPRESSION 2107 +#define WT_STAT_DSRC_REC_SUFFIX_COMPRESSION 2123 /*! reconciliation: internal page multi-block writes */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL 2108 +#define WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL 2124 /*! reconciliation: internal-page overflow keys */ -#define WT_STAT_DSRC_REC_OVERFLOW_KEY_INTERNAL 2109 +#define WT_STAT_DSRC_REC_OVERFLOW_KEY_INTERNAL 2125 /*! reconciliation: leaf page key bytes discarded using prefix compression */ -#define WT_STAT_DSRC_REC_PREFIX_COMPRESSION 2110 +#define WT_STAT_DSRC_REC_PREFIX_COMPRESSION 2126 /*! reconciliation: leaf page multi-block writes */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_LEAF 2111 +#define WT_STAT_DSRC_REC_MULTIBLOCK_LEAF 2127 /*! reconciliation: leaf-page overflow keys */ -#define WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF 2112 +#define WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF 2128 /*! reconciliation: maximum blocks required for a page */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_MAX 2113 +#define WT_STAT_DSRC_REC_MULTIBLOCK_MAX 2129 /*! reconciliation: overflow values written */ -#define WT_STAT_DSRC_REC_OVERFLOW_VALUE 2114 +#define WT_STAT_DSRC_REC_OVERFLOW_VALUE 2130 /*! reconciliation: page checksum matches */ -#define WT_STAT_DSRC_REC_PAGE_MATCH 2115 +#define WT_STAT_DSRC_REC_PAGE_MATCH 2131 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_DSRC_REC_PAGES 2116 +#define WT_STAT_DSRC_REC_PAGES 2132 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_DSRC_REC_PAGES_EVICTION 2117 +#define WT_STAT_DSRC_REC_PAGES_EVICTION 2133 /*! reconciliation: pages deleted */ -#define WT_STAT_DSRC_REC_PAGE_DELETE 2118 +#define WT_STAT_DSRC_REC_PAGE_DELETE 2134 /*! session: object compaction */ -#define WT_STAT_DSRC_SESSION_COMPACT 2119 +#define WT_STAT_DSRC_SESSION_COMPACT 2135 /*! session: open cursor count */ -#define WT_STAT_DSRC_SESSION_CURSOR_OPEN 2120 +#define WT_STAT_DSRC_SESSION_CURSOR_OPEN 2136 /*! transaction: update conflicts */ -#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2121 +#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2137 /*! * @} diff --git a/src/third_party/wiredtiger/src/include/wt_internal.h b/src/third_party/wiredtiger/src/include/wt_internal.h index 1c9600dd27f..74fdc4c3925 100644 --- a/src/third_party/wiredtiger/src/include/wt_internal.h +++ b/src/third_party/wiredtiger/src/include/wt_internal.h @@ -250,8 +250,6 @@ struct __wt_ovfl_reuse; typedef struct __wt_ovfl_reuse WT_OVFL_REUSE; struct __wt_ovfl_track; typedef struct __wt_ovfl_track WT_OVFL_TRACK; -struct __wt_ovfl_txnc; - typedef struct __wt_ovfl_txnc WT_OVFL_TXNC; struct __wt_page; typedef struct __wt_page WT_PAGE; struct __wt_page_deleted; diff --git a/src/third_party/wiredtiger/src/log/log.c b/src/third_party/wiredtiger/src/log/log.c index 868f5d0eaf4..2a912e6568f 100644 --- a/src/third_party/wiredtiger/src/log/log.c +++ b/src/third_party/wiredtiger/src/log/log.c @@ -9,8 +9,7 @@ #include "wt_internal.h" static int __log_newfile(WT_SESSION_IMPL *, bool, bool *); -static int __log_openfile( - WT_SESSION_IMPL *, WT_FH **, const char *, uint32_t, uint32_t); +static int __log_openfile(WT_SESSION_IMPL *, uint32_t, uint32_t, WT_FH **); static int __log_write_internal( WT_SESSION_IMPL *, WT_ITEM *, WT_LSN *, uint32_t); @@ -19,7 +18,109 @@ static int __log_write_internal( /* Flags to __log_openfile */ #define WT_LOG_OPEN_CREATE_OK 0x01 -#define WT_LOG_OPEN_VERIFY 0x02 + +/* + * __log_printf_internal -- + * Internal call to write a log message. + */ +static int +__log_printf_internal(WT_SESSION_IMPL *session, const char *fmt, ...) +{ + WT_DECL_RET; + va_list ap; + + va_start(ap, fmt); + ret = __wt_log_vprintf(session, fmt, ap); + va_end(ap); + return (ret); +} + +/* + * __log_checksum_match -- + * Given a log record, return whether the checksum matches. + */ +static bool +__log_checksum_match(WT_SESSION_IMPL *session, WT_ITEM *buf, uint32_t reclen) +{ + WT_LOG_RECORD *logrec; + uint32_t checksum_calculate, checksum_tmp; + + WT_UNUSED(session); + logrec = (WT_LOG_RECORD *)buf->mem; + checksum_tmp = logrec->checksum; + logrec->checksum = 0; + checksum_calculate = __wt_checksum(logrec, reclen); +#ifdef WORDS_BIGENDIAN + checksum_calculate = __wt_bswap32(checksum_calculate); +#endif + logrec->checksum = checksum_tmp; + if (logrec->checksum != checksum_calculate) + return (false); + else + return (true); +} + +/* + * __log_get_files -- + * Retrieve the list of all log-related files of the given prefix type. + */ +static int +__log_get_files(WT_SESSION_IMPL *session, + const char *file_prefix, char ***filesp, u_int *countp) +{ + WT_CONNECTION_IMPL *conn; + const char *log_path; + + *countp = 0; + *filesp = NULL; + + conn = S2C(session); + log_path = conn->log_path; + if (log_path == NULL) + log_path = ""; + return (__wt_fs_directory_list( + session, log_path, file_prefix, filesp, countp)); +} + +/* + * __log_prealloc_remove -- + * Remove all previously created pre-allocated files. + */ +static int +__log_prealloc_remove(WT_SESSION_IMPL *session) +{ + WT_DECL_RET; + WT_LOG *log; + uint32_t lognum; + u_int i, logcount; + char **logfiles; + + logfiles = NULL; + logcount = 0; + log = S2C(session)->log; + __wt_spin_lock(session, &log->log_fs_lock); + /* + * Clean up any old interim pre-allocated files. We clean + * up these files because settings may have changed upon reboot + * and we want those settings to take effect right away. + */ + WT_ERR(__log_get_files(session, + WT_LOG_TMPNAME, &logfiles, &logcount)); + for (i = 0; i < logcount; i++) { + WT_ERR(__wt_log_extract_lognum(session, logfiles[i], &lognum)); + WT_ERR(__wt_log_remove(session, WT_LOG_TMPNAME, lognum)); + } + WT_ERR(__wt_fs_directory_list_free(session, &logfiles, logcount)); + WT_ERR(__log_get_files(session, + WT_LOG_PREPNAME, &logfiles, &logcount)); + for (i = 0; i < logcount; i++) { + WT_ERR(__wt_log_extract_lognum(session, logfiles[i], &lognum)); + WT_ERR(__wt_log_remove(session, WT_LOG_PREPNAME, lognum)); + } +err: WT_TRET(__wt_fs_directory_list_free(session, &logfiles, logcount)); + __wt_spin_unlock(session, &log->log_fs_lock); + return (ret); +} /* * __log_wait_for_earlier_slot -- @@ -66,11 +167,13 @@ __log_fs_write(WT_SESSION_IMPL *session, WT_DECL_RET; /* - * If we're writing into a new log file, we have to wait for all + * If we're writing into a new log file and we're running in + * compatibility mode to an older release, we have to wait for all * writes to the previous log file to complete otherwise there could * be a hole at the end of the previous log file that we cannot detect. */ - if (slot->slot_release_lsn.l.file < slot->slot_start_lsn.l.file) { + if (S2C(session)->log->log_version != WT_LOG_VERSION && + slot->slot_release_lsn.l.file < slot->slot_start_lsn.l.file) { __log_wait_for_earlier_slot(session, slot); WT_RET(__wt_log_force_sync(session, &slot->slot_release_lsn)); } @@ -208,8 +311,7 @@ __wt_log_force_sync(WT_SESSION_IMPL *session, WT_LSN *min_lsn) * from under us and either be NULL or point to a different * file than we want. */ - WT_ERR(__log_openfile(session, - &log_fh, WT_LOG_FILENAME, min_lsn->l.file, 0)); + WT_ERR(__log_openfile(session, min_lsn->l.file, 0, &log_fh)); __wt_verbose(session, WT_VERB_LOG, "log_force_sync: sync %s to LSN %" PRIu32 "/%" PRIu32, log_fh->name, min_lsn->l.file, min_lsn->l.offset); @@ -314,28 +416,6 @@ __wt_log_written_reset(WT_SESSION_IMPL *session) } /* - * __log_get_files -- - * Retrieve the list of all log-related files of the given prefix type. - */ -static int -__log_get_files(WT_SESSION_IMPL *session, - const char *file_prefix, char ***filesp, u_int *countp) -{ - WT_CONNECTION_IMPL *conn; - const char *log_path; - - *countp = 0; - *filesp = NULL; - - conn = S2C(session); - log_path = conn->log_path; - if (log_path == NULL) - log_path = ""; - return (__wt_fs_directory_list( - session, log_path, file_prefix, filesp, countp)); -} - -/* * __wt_log_get_all_files -- * Retrieve the list of log files, either all of them or only the active * ones (those that are not candidates for archiving). The caller is @@ -571,7 +651,7 @@ __log_prealloc(WT_SESSION_IMPL *session, WT_FH *fh) */ if (FLD_ISSET(conn->log_flags, WT_CONN_LOG_ZERO_FILL)) return (__log_zero(session, fh, - WT_LOG_FIRST_RECORD, conn->log_file_max)); + WT_LOG_END_HEADER, conn->log_file_max)); /* * We have exclusive access to the log file and there are no other @@ -593,7 +673,7 @@ __log_size_fit(WT_SESSION_IMPL *session, WT_LSN *lsn, uint64_t recsize) conn = S2C(session); log = conn->log; - return (lsn->l.offset == WT_LOG_FIRST_RECORD || + return (lsn->l.offset == log->first_record || lsn->l.offset + (wt_off_t)recsize < conn->log_file_max); } @@ -662,17 +742,17 @@ __log_decrypt(WT_SESSION_IMPL *session, WT_ITEM *in, WT_ITEM *out) } /* - * __log_fill -- + * __wt_log_fill -- * Copy a thread's log records into the assigned slot. */ -static int -__log_fill(WT_SESSION_IMPL *session, +int +__wt_log_fill(WT_SESSION_IMPL *session, WT_MYSLOT *myslot, bool force, WT_ITEM *record, WT_LSN *lsnp) { WT_DECL_RET; /* - * Call __wt_write or copy into the buffer. For now the offset is the + * Call write or copy into the buffer. For now the offset is the * real byte offset. If the offset becomes a unit of WT_LOG_ALIGN this * is where we would multiply by WT_LOG_ALIGN to get the real file byte * offset for write(). @@ -733,8 +813,7 @@ __log_file_header( logrec = (WT_LOG_RECORD *)buf->mem; desc = (WT_LOG_DESC *)logrec->record; desc->log_magic = WT_LOG_MAGIC; - desc->majorv = WT_LOG_MAJOR_VERSION; - desc->minorv = WT_LOG_MINOR_VERSION; + desc->version = log->log_version; desc->log_size = (uint64_t)conn->log_file_max; __wt_log_desc_byteswap(desc); @@ -759,7 +838,7 @@ __log_file_header( /* * We may recursively call __wt_log_acquire to allocate log space for - * the log descriptor record. Call __log_fill to write it, but we + * the log descriptor record. Call __wt_log_fill to write it, but we * do not need to call __wt_log_release because we're not waiting for * any earlier operations to complete. */ @@ -770,7 +849,7 @@ __log_file_header( WT_ASSERT(session, fh == NULL); WT_ERR(__wt_log_acquire(session, log->allocsize, &tmp)); } - WT_ERR(__log_fill(session, &myslot, true, buf, NULL)); + WT_ERR(__wt_log_fill(session, &myslot, true, buf, NULL)); /* * Make sure the header gets to disk. */ @@ -787,63 +866,143 @@ err: __wt_scr_free(session, &buf); * Open a log file with the given log file number and return the WT_FH. */ static int -__log_openfile(WT_SESSION_IMPL *session, - WT_FH **fhp, const char *file_prefix, uint32_t id, uint32_t flags) +__log_openfile( + WT_SESSION_IMPL *session, uint32_t id, uint32_t flags, WT_FH **fhp) { WT_CONNECTION_IMPL *conn; WT_DECL_ITEM(buf); WT_DECL_RET; - WT_LOG *log; - WT_LOG_DESC *desc; - WT_LOG_RECORD *logrec; - uint32_t allocsize; u_int wtopen_flags; conn = S2C(session); - log = conn->log; - if (log == NULL) - allocsize = WT_LOG_ALIGN; - else - allocsize = log->allocsize; WT_RET(__wt_scr_alloc(session, 0, &buf)); - WT_ERR(__log_filename(session, id, file_prefix, buf)); + /* + * If we are creating the file then we use a temporary file name. + * Otherwise it is a log file name. + */ + if (LF_ISSET(WT_LOG_OPEN_CREATE_OK)) { + wtopen_flags = WT_FS_OPEN_CREATE; + WT_ERR(__log_filename(session, id, WT_LOG_TMPNAME, buf)); + } else { + wtopen_flags = 0; + WT_ERR(__log_filename(session, id, WT_LOG_FILENAME, buf)); + } __wt_verbose(session, WT_VERB_LOG, "opening log %s", (const char *)buf->data); - wtopen_flags = 0; - if (LF_ISSET(WT_LOG_OPEN_CREATE_OK)) - FLD_SET(wtopen_flags, WT_FS_OPEN_CREATE); if (FLD_ISSET(conn->direct_io, WT_DIRECT_IO_LOG)) FLD_SET(wtopen_flags, WT_FS_OPEN_DIRECTIO); WT_ERR(__wt_open( session, buf->data, WT_FS_OPEN_FILE_TYPE_LOG, wtopen_flags, fhp)); +err: __wt_scr_free(session, &buf); + return (ret); +} + +/* + * __log_open_verify -- + * Open a log file with the given log file number, verify its + * header and return various pieces of system information about + * this log file. + */ +static int +__log_open_verify(WT_SESSION_IMPL *session, uint32_t id, WT_FH **fhp, + WT_LSN *lsnp, uint16_t *versionp) +{ + WT_CONNECTION_IMPL *conn; + WT_DECL_ITEM(buf); + WT_DECL_RET; + WT_FH *fh; + WT_LOG *log; + WT_LOG_DESC *desc; + WT_LOG_RECORD *logrec; + uint32_t allocsize, rectype; + const uint8_t *end, *p; + + conn = S2C(session); + log = conn->log; + WT_RET(__wt_scr_alloc(session, 0, &buf)); + WT_ERR(__log_openfile(session, id, 0, &fh)); + + if (log == NULL) + allocsize = WT_LOG_ALIGN; + else + allocsize = log->allocsize; + if (lsnp != NULL) + WT_ZERO_LSN(lsnp); + WT_ERR(__wt_buf_grow(session, buf, allocsize)); + memset(buf->mem, 0, allocsize); /* - * If we are not creating the log file but opening it for reading, - * check that the magic number and versions are correct. + * Read in the log file header and verify it. */ - if (LF_ISSET(WT_LOG_OPEN_VERIFY)) { - WT_ERR(__wt_buf_grow(session, buf, allocsize)); - memset(buf->mem, 0, allocsize); - WT_ERR(__wt_read(session, *fhp, 0, allocsize, buf->mem)); - logrec = (WT_LOG_RECORD *)buf->mem; - __wt_log_record_byteswap(logrec); - desc = (WT_LOG_DESC *)logrec->record; - __wt_log_desc_byteswap(desc); - if (desc->log_magic != WT_LOG_MAGIC) - WT_PANIC_RET(session, WT_ERROR, - "log file %s corrupted: Bad magic number %" PRIu32, - (*fhp)->name, desc->log_magic); - if (desc->majorv > WT_LOG_MAJOR_VERSION || - (desc->majorv == WT_LOG_MAJOR_VERSION && - desc->minorv > WT_LOG_MINOR_VERSION)) - WT_ERR_MSG(session, WT_ERROR, - "unsupported WiredTiger file version: this build " - " only supports major/minor versions up to %d/%d, " - " and the file is version %" PRIu16 "/%" PRIu16, - WT_LOG_MAJOR_VERSION, WT_LOG_MINOR_VERSION, - desc->majorv, desc->minorv); + WT_ERR(__wt_read(session, fh, 0, allocsize, buf->mem)); + logrec = (WT_LOG_RECORD *)buf->mem; + __wt_log_record_byteswap(logrec); + desc = (WT_LOG_DESC *)logrec->record; + __wt_log_desc_byteswap(desc); + if (desc->log_magic != WT_LOG_MAGIC) + WT_PANIC_RET(session, WT_ERROR, + "log file %s corrupted: Bad magic number %" PRIu32, + fh->name, desc->log_magic); + /* + * We cannot read future log file formats. + */ + if (desc->version > WT_LOG_VERSION) + WT_ERR_MSG(session, WT_ERROR, + "unsupported WiredTiger file version: this build " + " only supports versions up to %d," + " and the file is version %" PRIu16, + WT_LOG_VERSION, desc->version); + + /* + * Set up the return values if the magic number is valid. + */ + if (versionp != NULL) + *versionp = desc->version; + + /* + * Skip reading in the previous LSN if log file is an old version + * or if the caller doesn't care about the LSN. Otherwise read that + * record in and set up the LSN. We already have a buffer that is + * the correct size. Reuse it. + */ + if (lsnp == NULL || + (desc->version < WT_LOG_VERSION_SYSTEM)) + goto err; + + memset(buf->mem, 0, allocsize); + WT_ERR(__wt_read(session, fh, allocsize, allocsize, buf->mem)); + logrec = (WT_LOG_RECORD *)buf->mem; + /* + * We have a valid header but the system record is not there. + * The log ends here. Return without setting the LSN. + */ + if (logrec->len == 0) { + __wt_verbose(session, WT_VERB_LOG, + "Log %s found empty log after header", fh->name); + goto err; } + + if (!__log_checksum_match(session, buf, allocsize)) + WT_ERR_MSG(session, WT_ERROR, + "%s: System log record checksum mismatch", fh->name); + __wt_log_record_byteswap(logrec); + p = WT_LOG_SKIP_HEADER(buf->data); + end = (const uint8_t *)buf->data + allocsize; + WT_ERR(__wt_logrec_read(session, &p, end, &rectype)); + if (rectype != WT_LOGREC_SYSTEM) + WT_ERR_MSG(session, WT_ERROR, "System log record missing"); + WT_ERR(__wt_log_recover_system(session, &p, end, lsnp)); + err: __wt_scr_free(session, &buf); + + /* + * Return the file handle if needed, otherwise close it. + */ + if (fhp != NULL && ret == 0) + *fhp = fh; + else + WT_TRET(__wt_close(session, &fh)); + return (ret); } @@ -858,6 +1017,7 @@ __log_alloc_prealloc(WT_SESSION_IMPL *session, uint32_t to_num) WT_DECL_ITEM(from_path); WT_DECL_ITEM(to_path); WT_DECL_RET; + WT_LOG *log; uint32_t from_num; u_int logcount; char **logfiles; @@ -865,6 +1025,7 @@ __log_alloc_prealloc(WT_SESSION_IMPL *session, uint32_t to_num) /* * If there are no pre-allocated files, return WT_NOTFOUND. */ + log = S2C(session)->log; logfiles = NULL; WT_ERR(__log_get_files(session, WT_LOG_PREPNAME, &logfiles, &logcount)); if (logcount == 0) @@ -879,6 +1040,7 @@ __log_alloc_prealloc(WT_SESSION_IMPL *session, uint32_t to_num) WT_ERR(__wt_scr_alloc(session, 0, &to_path)); WT_ERR(__log_filename(session, from_num, WT_LOG_PREPNAME, from_path)); WT_ERR(__log_filename(session, to_num, WT_LOG_FILENAME, to_path)); + __wt_spin_lock(session, &log->log_fs_lock); __wt_verbose(session, WT_VERB_LOG, "log_alloc_prealloc: rename log %s to %s", (const char *)from_path->data, (const char *)to_path->data); @@ -891,6 +1053,7 @@ __log_alloc_prealloc(WT_SESSION_IMPL *session, uint32_t to_num) err: __wt_scr_free(session, &from_path); __wt_scr_free(session, &to_path); + __wt_spin_unlock(session, &log->log_fs_lock); WT_TRET(__wt_fs_directory_list_free(session, &logfiles, logcount)); return (ret); } @@ -906,7 +1069,7 @@ __log_newfile(WT_SESSION_IMPL *session, bool conn_open, bool *created) WT_DECL_RET; WT_FH *log_fh; WT_LOG *log; - WT_LSN end_lsn; + WT_LSN end_lsn, logrec_lsn; u_int yield_cnt; bool create_log; @@ -941,7 +1104,7 @@ __log_newfile(WT_SESSION_IMPL *session, bool conn_open, bool *created) /* * If pre-allocating log files look for one; otherwise, or if we don't - * find one create a log file. We can't use pre-allocated log files in + * find one, create a log file. We can't use pre-allocated log files * while a hot backup is in progress: applications can copy the files * in any way they choose, and a log file rename might confuse things. */ @@ -986,15 +1149,31 @@ __log_newfile(WT_SESSION_IMPL *session, bool conn_open, bool *created) * we must pass in a local file handle. Otherwise there is a wide * window where another thread could see a NULL log file handle. */ - WT_RET(__log_openfile(session, - &log_fh, WT_LOG_FILENAME, log->fileid, 0)); - WT_PUBLISH(log->log_fh, log_fh); + WT_RET(__log_open_verify(session, log->fileid, &log_fh, NULL, NULL)); + /* + * Write the LSN at the end of the last record in the previous log file + * as the first record in this log file. + */ + if (log->fileid == 1) + WT_INIT_LSN(&logrec_lsn); + else + logrec_lsn = log->alloc_lsn; /* * We need to setup the LSNs. Set the end LSN and alloc LSN to * the end of the header. */ - WT_SET_LSN(&log->alloc_lsn, log->fileid, WT_LOG_FIRST_RECORD); + WT_SET_LSN(&log->alloc_lsn, log->fileid, WT_LOG_END_HEADER); + /* + * If we're running the version where we write a system record + * do so now and update the alloc_lsn. + */ + if (log->log_version == WT_LOG_VERSION) { + WT_RET(__wt_log_system_record(session, + log_fh, &logrec_lsn)); + WT_SET_LSN(&log->alloc_lsn, log->fileid, log->first_record); + } end_lsn = log->alloc_lsn; + WT_PUBLISH(log->log_fh, log_fh); /* * If we're called from connection creation code, we need to update @@ -1012,6 +1191,83 @@ __log_newfile(WT_SESSION_IMPL *session, bool conn_open, bool *created) } /* + * __log_set_version -- + * Set version related information under lock. + */ +static int +__log_set_version(WT_SESSION_IMPL *session, uint16_t version, + uint32_t first_rec, bool live_chg, bool downgrade) +{ + WT_CONNECTION_IMPL *conn; + WT_LOG *log; + + conn = S2C(session); + log = conn->log; + + log->log_version = version; + log->first_record = first_rec; + if (downgrade) + FLD_SET(conn->log_flags, WT_CONN_LOG_DOWNGRADED); + else + FLD_CLR(conn->log_flags, WT_CONN_LOG_DOWNGRADED); + if (live_chg) + F_SET(log, WT_LOG_FORCE_NEWFILE); + if (!F_ISSET(conn, WT_CONN_READONLY)) + return (__log_prealloc_remove(session)); + else + return (0); +} + +/* + * __wt_log_set_version -- + * Change the version number in logging. Will be done with locking. + * We need to force the log file to advance and remove all old + * pre-allocated files. + */ +int +__wt_log_set_version(WT_SESSION_IMPL *session, uint16_t version, + uint32_t first_rec, bool downgrade, bool live_chg, uint32_t *lognump) +{ + WT_CONNECTION_IMPL *conn; + WT_DECL_RET; + WT_LOG *log; + + conn = S2C(session); + log = conn->log; + + /* + * The steps are: + * - Set up versions and remove files under lock. + * - Set a flag so that the next slot change forces a file change. + * - Force out the slot that is currently active in the current log. + * - Write a log record to force a record into the new log file. + */ + WT_WITH_SLOT_LOCK(session, log, + ret = __log_set_version(session, + version, first_rec, live_chg, downgrade)); + if (!live_chg) + return (ret); + WT_ERR(ret); + /* + * A new log file will be used when we force out the earlier slot. + */ + WT_ERR(__wt_log_force_write(session, 1, NULL)); + + /* + * We need to write a record to the new version log file so that + * a potential checkpoint finds LSNs in that new log file and + * an archive correctly removes all earlier logs. + * Write an internal printf record. + */ + WT_ERR(__log_printf_internal(session, + "COMPATIBILITY: Version now %" PRIu16, log->log_version)); + if (lognump != NULL) + *lognump = log->alloc_lsn.l.file; +err: + return (ret); +} + +/* * __wt_log_acquire -- * Called serially when switching slots. Can be called recursively * from __log_newfile when we change log files. @@ -1043,8 +1299,10 @@ __wt_log_acquire(WT_SESSION_IMPL *session, uint64_t recsize, WT_LOGSLOT *slot) * that exceed the maximum file size. We want to minimize the risk * of an error due to no space. */ - if (!__log_size_fit(session, &log->alloc_lsn, recsize)) { + if (F_ISSET(log, WT_LOG_FORCE_NEWFILE) || + !__log_size_fit(session, &log->alloc_lsn, recsize)) { WT_RET(__log_newfile(session, false, &created_log)); + F_CLR(log, WT_LOG_FORCE_NEWFILE); if (log->log_close_fh != NULL) F_SET(slot, WT_SLOT_CLOSEFH); } @@ -1053,7 +1311,7 @@ __wt_log_acquire(WT_SESSION_IMPL *session, uint64_t recsize, WT_LOGSLOT *slot) * Pre-allocate on the first real write into the log file, if it * was just created (i.e. not pre-allocated). */ - if (log->alloc_lsn.l.offset == WT_LOG_FIRST_RECORD && created_log) + if (log->alloc_lsn.l.offset == log->first_record && created_log) WT_RET(__log_prealloc(session, log->log_fh)); /* * Initialize the slot for activation. @@ -1107,8 +1365,7 @@ __log_truncate_file(WT_SESSION_IMPL *session, WT_FH *log_fh, wt_off_t offset) * we are in recovery or other dedicated time and not during live running. */ static int -__log_truncate(WT_SESSION_IMPL *session, - WT_LSN *lsn, const char *file_prefix, uint32_t this_log) +__log_truncate(WT_SESSION_IMPL *session, WT_LSN *lsn, bool this_log) { WT_CONNECTION_IMPL *conn; WT_DECL_RET; @@ -1133,7 +1390,7 @@ __log_truncate(WT_SESSION_IMPL *session, * before doing work, and if there's a not-supported error, turn off * future truncates. */ - WT_ERR(__log_openfile(session, &log_fh, file_prefix, lsn->l.file, 0)); + WT_ERR(__log_openfile(session, lsn->l.file, 0, &log_fh)); WT_ERR(__log_truncate_file(session, log_fh, lsn->l.offset)); WT_ERR(__wt_fsync(session, log_fh, true)); WT_ERR(__wt_close(session, &log_fh)); @@ -1148,14 +1405,13 @@ __log_truncate(WT_SESSION_IMPL *session, for (i = 0; i < logcount; i++) { WT_ERR(__wt_log_extract_lognum(session, logfiles[i], &lognum)); if (lognum > lsn->l.file && lognum < log->trunc_lsn.l.file) { - WT_ERR(__log_openfile(session, - &log_fh, file_prefix, lognum, 0)); + WT_ERR(__log_openfile(session, lognum, 0, &log_fh)); /* * If there are intervening files pre-allocated, * truncate them to the end of the log file header. */ WT_ERR(__log_truncate_file( - session, log_fh, WT_LOG_FIRST_RECORD)); + session, log_fh, log->first_record)); WT_ERR(__wt_fsync(session, log_fh, true)); WT_ERR(__wt_close(session, &log_fh)); } @@ -1198,17 +1454,17 @@ __wt_log_allocfile( tmp_id = __wt_atomic_add32(&log->tmp_fileid, 1); WT_ERR(__log_filename(session, tmp_id, WT_LOG_TMPNAME, from_path)); WT_ERR(__log_filename(session, lognum, dest, to_path)); + __wt_spin_lock(session, &log->log_fs_lock); /* * Set up the temporary file. */ - WT_ERR(__log_openfile(session, - &log_fh, WT_LOG_TMPNAME, tmp_id, WT_LOG_OPEN_CREATE_OK)); + WT_ERR(__log_openfile(session, tmp_id, WT_LOG_OPEN_CREATE_OK, &log_fh)); WT_ERR(__log_file_header(session, log_fh, NULL, true)); WT_ERR(__log_prealloc(session, log_fh)); WT_ERR(__wt_fsync(session, log_fh, true)); WT_ERR(__wt_close(session, &log_fh)); __wt_verbose(session, WT_VERB_LOG, - "log_prealloc: rename %s to %s", + "log_allocfile: rename %s to %s", (const char *)from_path->data, (const char *)to_path->data); /* * Rename it into place and make it available. @@ -1217,6 +1473,7 @@ __wt_log_allocfile( err: __wt_scr_free(session, &from_path); __wt_scr_free(session, &to_path); + __wt_spin_unlock(session, &log->log_fs_lock); WT_TRET(__wt_close(session, &log_fh)); return (ret); } @@ -1255,6 +1512,7 @@ __wt_log_open(WT_SESSION_IMPL *session) WT_DECL_RET; WT_LOG *log; uint32_t firstlog, lastlog, lognum; + uint16_t version; u_int i, logcount; char **logfiles; @@ -1275,33 +1533,8 @@ __wt_log_open(WT_SESSION_IMPL *session) WT_FS_OPEN_FILE_TYPE_DIRECTORY, 0, &log->log_dir_fh)); } - if (!F_ISSET(conn, WT_CONN_READONLY)) { - /* - * Clean up any old interim pre-allocated files. We clean - * up these files because settings have changed upon reboot - * and we want those settings to take effect right away. - */ - WT_ERR(__log_get_files(session, - WT_LOG_TMPNAME, &logfiles, &logcount)); - for (i = 0; i < logcount; i++) { - WT_ERR(__wt_log_extract_lognum( - session, logfiles[i], &lognum)); - WT_ERR(__wt_log_remove( - session, WT_LOG_TMPNAME, lognum)); - } - WT_ERR( - __wt_fs_directory_list_free(session, &logfiles, logcount)); - WT_ERR(__log_get_files(session, - WT_LOG_PREPNAME, &logfiles, &logcount)); - for (i = 0; i < logcount; i++) { - WT_ERR(__wt_log_extract_lognum( - session, logfiles[i], &lognum)); - WT_ERR(__wt_log_remove( - session, WT_LOG_PREPNAME, lognum)); - } - WT_ERR( - __wt_fs_directory_list_free(session, &logfiles, logcount)); - } + if (!F_ISSET(conn, WT_CONN_READONLY)) + WT_ERR(__log_prealloc_remove(session)); /* * Now look at the log files and set our LSNs. @@ -1334,6 +1567,33 @@ __wt_log_open(WT_SESSION_IMPL *session) /* If we found log files, save the new state. */ if (logcount > 0) { + /* + * If we're running in a downgraded mode and there are earlier + * logs detect if they're at a higher version. If so, we need + * to force recovery (to write a full checkpoint) and force + * archiving to remove all higher version logs. + */ + if (FLD_ISSET(conn->log_flags, WT_CONN_LOG_DOWNGRADED)) { + for (i = 0; i < logcount; ++i) { + WT_ERR(__wt_log_extract_lognum( + session, logfiles[i], &lognum)); + /* + * By sending in a NULL file handle, we don't + * have to close the file. + */ + WT_ERR(__log_open_verify(session, + lognum, NULL, NULL, &version)); + /* + * If we find any log file at the wrong version + * set the flag and we're done. + */ + if (log->log_version != version) { + FLD_SET(conn->log_flags, + WT_CONN_LOG_FORCE_DOWNGRADE); + break; + } + } + } log->trunc_lsn = log->alloc_lsn; FLD_SET(conn->log_flags, WT_CONN_LOG_EXISTED); } @@ -1631,12 +1891,12 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, WT_ITEM *cbbuf; WT_LOG *log; WT_LOG_RECORD *logrec; - WT_LSN end_lsn, next_lsn, rd_lsn, start_lsn; + WT_LSN end_lsn, next_lsn, prev_eof, prev_lsn, rd_lsn, start_lsn; wt_off_t log_size; uint32_t allocsize, firstlog, lastlog, lognum, rdup_len, reclen; - uint32_t checksum_calculate, checksum_tmp; u_int i, logcount; int firstrecord; + uint16_t version; bool eol, partial_record; char **logfiles; @@ -1660,66 +1920,26 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, "__wt_log_scan truncating to %" PRIu32 "/%" PRIu32, log->trunc_lsn.l.file, log->trunc_lsn.l.offset); + if (lsnp != NULL && + LF_ISSET(WT_LOGSCAN_FIRST|WT_LOGSCAN_FROM_CKP)) + WT_RET_MSG(session, WT_ERROR, + "choose either a start LSN or a start flag"); + /* + * Set up the allocation size, starting and ending LSNs. The values + * for those depend on whether logging is currently enabled or not. + */ + lastlog = 0; if (log != NULL) { allocsize = log->allocsize; - + end_lsn = log->alloc_lsn; + start_lsn = log->first_lsn; if (lsnp == NULL) { - if (LF_ISSET(WT_LOGSCAN_FIRST)) - start_lsn = log->first_lsn; - else if (LF_ISSET(WT_LOGSCAN_FROM_CKP)) + if (LF_ISSET(WT_LOGSCAN_FROM_CKP)) start_lsn = log->ckpt_lsn; - else + else if (!LF_ISSET(WT_LOGSCAN_FIRST)) return (WT_ERROR); /* Illegal usage */ - } else { - if (LF_ISSET(WT_LOGSCAN_FIRST|WT_LOGSCAN_FROM_CKP)) - WT_RET_MSG(session, WT_ERROR, - "choose either a start LSN or a start flag"); - - /* - * Offsets must be on allocation boundaries. - * An invalid LSN from a user should just return - * WT_NOTFOUND. It is not an error. But if it is - * from recovery, we expect valid LSNs so give more - * information about that. - */ - if (lsnp->l.offset % allocsize != 0) { - if (LF_ISSET(WT_LOGSCAN_RECOVER)) - WT_RET_MSG(session, WT_NOTFOUND, - "__wt_log_scan unaligned LSN %" - PRIu32 "/%" PRIu32, - lsnp->l.file, lsnp->l.offset); - else - return (WT_NOTFOUND); - } - /* - * If the file is in the future it doesn't exist. - * An invalid LSN from a user should just return - * WT_NOTFOUND. It is not an error. But if it is - * from recovery, we expect valid LSNs so give more - * information about that. - */ - if (lsnp->l.file > log->fileid) { - if (LF_ISSET(WT_LOGSCAN_RECOVER)) - WT_RET_MSG(session, WT_NOTFOUND, - "__wt_log_scan LSN %" PRIu32 "/%" - PRIu32 - " larger than biggest log file %" - PRIu32, lsnp->l.file, - lsnp->l.offset, log->fileid); - else - return (WT_NOTFOUND); - } - - /* - * Log cursors may not know the starting LSN. If an - * LSN is passed in that it is equal to the smallest - * LSN, start from the beginning of the log. - */ - start_lsn = *lsnp; - if (WT_IS_INIT_LSN(&start_lsn)) - start_lsn = log->first_lsn; } - end_lsn = log->alloc_lsn; + lastlog = log->fileid; } else { /* * If logging is not configured, we can still print out the log @@ -1732,7 +1952,6 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, * a multiple of this. */ allocsize = WT_LOG_ALIGN; - lastlog = 0; firstlog = UINT32_MAX; WT_RET(__log_get_files(session, WT_LOG_FILENAME, &logfiles, &logcount)); @@ -1749,8 +1968,49 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, WT_ERR( __wt_fs_directory_list_free(session, &logfiles, logcount)); } - WT_ERR(__log_openfile(session, - &log_fh, WT_LOG_FILENAME, start_lsn.l.file, WT_LOG_OPEN_VERIFY)); + if (lsnp != NULL) { + /* + * Offsets must be on allocation boundaries. + * An invalid LSN from a user should just return + * WT_NOTFOUND. It is not an error. But if it is + * from recovery, we expect valid LSNs so give more + * information about that. + */ + if (lsnp->l.offset % allocsize != 0) { + if (LF_ISSET(WT_LOGSCAN_RECOVER)) + WT_ERR_MSG(session, WT_NOTFOUND, + "__wt_log_scan unaligned LSN %" + PRIu32 "/%" PRIu32, + lsnp->l.file, lsnp->l.offset); + else + return (WT_NOTFOUND); + } + /* + * If the file is in the future it doesn't exist. + * An invalid LSN from a user should just return + * WT_NOTFOUND. It is not an error. But if it is + * from recovery, we expect valid LSNs so give more + * information about that. + */ + if (lsnp->l.file > lastlog) { + if (LF_ISSET(WT_LOGSCAN_RECOVER)) + WT_ERR_MSG(session, WT_NOTFOUND, + "__wt_log_scan LSN %" PRIu32 "/%" PRIu32 + " larger than biggest log file %" PRIu32, + lsnp->l.file, lsnp->l.offset, lastlog); + else + return (WT_NOTFOUND); + } + /* + * Log cursors may not know the starting LSN. If an + * LSN is passed in that it is equal to the smallest + * LSN, start from the beginning of the log. + */ + if (!WT_IS_INIT_LSN(lsnp)) + start_lsn = *lsnp; + } + WT_ERR(__log_open_verify(session, + start_lsn.l.file, &log_fh, &prev_lsn, NULL)); WT_ERR(__wt_filesize(session, log_fh, &log_size)); rd_lsn = start_lsn; if (LF_ISSET(WT_LOGSCAN_RECOVER)) @@ -1784,8 +2044,7 @@ advance: * Truncate this log file before we move to the next. */ if (LF_ISSET(WT_LOGSCAN_RECOVER)) - WT_ERR(__log_truncate(session, - &rd_lsn, WT_LOG_FILENAME, 1)); + WT_ERR(__log_truncate(session, &rd_lsn, true)); /* * If we had a partial record, we'll want to break * now after closing and truncating. Although for now @@ -1795,21 +2054,47 @@ advance: */ if (partial_record) break; - WT_SET_LSN(&rd_lsn, rd_lsn.l.file + 1, 0); /* * Avoid an error message when we reach end of log * by checking here. */ + prev_eof = rd_lsn; + WT_SET_LSN(&rd_lsn, rd_lsn.l.file + 1, 0); if (rd_lsn.l.file > end_lsn.l.file) break; - WT_ERR(__log_openfile(session, - &log_fh, WT_LOG_FILENAME, - rd_lsn.l.file, WT_LOG_OPEN_VERIFY)); if (LF_ISSET(WT_LOGSCAN_RECOVER)) __wt_verbose(session, WT_VERB_RECOVERY_PROGRESS, "Recovering log %" PRIu32 " through %" PRIu32, rd_lsn.l.file, end_lsn.l.file); + WT_ERR(__log_open_verify(session, + rd_lsn.l.file, &log_fh, &prev_lsn, &version)); + /* + * Opening the log file reads with verify sets up the + * previous LSN from the first record. This detects + * a "hole" at the end of the previous log file. + */ + if (LF_ISSET(WT_LOGSCAN_RECOVER) && + !WT_IS_INIT_LSN(&prev_lsn) && + !WT_IS_ZERO_LSN(&prev_lsn) && + prev_lsn.l.offset != prev_eof.l.offset) { + WT_ASSERT(session, + prev_eof.l.file == prev_lsn.l.file); + break; + } + /* + * If we read a current version log file without a + * previous LSN record the log ended after writing + * that header. We're done. + */ + if (LF_ISSET(WT_LOGSCAN_RECOVER) && + version == WT_LOG_VERSION_SYSTEM && + WT_IS_ZERO_LSN(&prev_lsn)) { + __wt_verbose(session, WT_VERB_LOG, + "log_scan: Stopping, no system " + "record detected in %s.", log_fh->name); + break; + } WT_ERR(__wt_filesize(session, log_fh, &log_size)); eol = false; continue; @@ -1881,12 +2166,7 @@ advance: */ buf->size = reclen; logrec = (WT_LOG_RECORD *)buf->mem; - checksum_tmp = logrec->checksum; - logrec->checksum = 0; - checksum_calculate = __wt_checksum(logrec, reclen); - logrec->checksum = checksum_tmp; - __wt_log_record_byteswap(logrec); - if (logrec->checksum != checksum_calculate) { + if (!__log_checksum_match(session, buf, reclen)) { /* * A checksum mismatch means we have reached the end of * the useful part of the log. This should be found on @@ -1904,6 +2184,7 @@ advance: ret = WT_NOTFOUND; break; } + __wt_log_record_byteswap(logrec); /* * We have a valid log record. If it is not the log file @@ -1946,8 +2227,12 @@ advance: /* Truncate if we're in recovery. */ if (LF_ISSET(WT_LOGSCAN_RECOVER) && - __wt_log_cmp(&rd_lsn, &log->trunc_lsn) < 0) - WT_ERR(__log_truncate(session, &rd_lsn, WT_LOG_FILENAME, 0)); + __wt_log_cmp(&rd_lsn, &log->trunc_lsn) < 0) { + __wt_verbose(session, WT_VERB_LOG, + "__wt_log_scan truncating to %" PRIu32 "/%" PRIu32, + rd_lsn.l.file, rd_lsn.l.offset); + WT_ERR(__log_truncate(session, &rd_lsn, false)); + } err: WT_STAT_CONN_INCR(session, log_scans); /* @@ -2212,7 +2497,7 @@ __log_write_internal(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, F_ISSET(&myslot, WT_MYSLOT_UNBUFFERED) || force) ret = __wt_log_slot_switch(session, &myslot, true, false, NULL); if (ret == 0) - ret = __log_fill(session, &myslot, false, record, &lsn); + ret = __wt_log_fill(session, &myslot, false, record, &lsn); release_size = __wt_log_slot_release(&myslot, (int64_t)rdup_len); /* * If we get an error we still need to do proper accounting in @@ -2365,7 +2650,7 @@ __wt_log_flush(WT_SESSION_IMPL *session, uint32_t flags) * is single-threaded we could wait here forever because the write LSN * doesn't switch into the new file until it contains a record. */ - if (last_lsn.l.offset == WT_LOG_FIRST_RECORD) + if (last_lsn.l.offset == log->first_record) last_lsn = log->log_close_lsn; /* diff --git a/src/third_party/wiredtiger/src/log/log_slot.c b/src/third_party/wiredtiger/src/log/log_slot.c index 5bd3d53a973..e0b301398a3 100644 --- a/src/third_party/wiredtiger/src/log/log_slot.c +++ b/src/third_party/wiredtiger/src/log/log_slot.c @@ -31,7 +31,7 @@ __log_slot_dump(WT_SESSION_IMPL *session) earliest = i; __wt_errx(session, "Slot %d:", i); __wt_errx(session, " State: %" PRIx64 " Flags: %" PRIx32, - slot->slot_state, slot->flags); + (uint64_t)slot->slot_state, slot->flags); __wt_errx(session, " Start LSN: %" PRIu32 "/%" PRIu32, slot->slot_start_lsn.l.file, slot->slot_start_lsn.l.offset); __wt_errx(session, " End LSN: %" PRIu32 "/%" PRIu32, @@ -173,9 +173,9 @@ retry: if (WT_TIMEDIFF_SEC(now, begin) > 10) { __wt_errx(session, "SLOT_CLOSE: Slot %" PRIu32 " Timeout unbuffered, state 0x%" - PRIx64 " unbuffered %" PRIu64, + PRIx64 " unbuffered %" PRId64, (uint32_t)(slot - &log->slot_pool[0]), - slot->slot_state, + (uint64_t)slot->slot_state, slot->slot_unbuffered); __log_slot_dump(session); __wt_abort(session); @@ -238,6 +238,7 @@ __log_slot_new(WT_SESSION_IMPL *session) /* * Rotate among the slots to lessen collisions. */ + WT_RET(WT_SESSION_CHECK_PANIC(session)); for (i = 0, pool_i = log->pool_index; i < WT_SLOT_POOL; i++, pool_i++) { if (pool_i >= WT_SLOT_POOL) @@ -314,7 +315,7 @@ __log_slot_switch_internal( * because the slot could be part of an unbuffered operation. */ joined = WT_LOG_SLOT_JOINED(slot->slot_state); - if (joined == 0 && forced) { + if (joined == 0 && forced && !F_ISSET(log, WT_LOG_FORCE_NEWFILE)) { WT_STAT_CONN_INCR(session, log_force_write_skip); if (did_work != NULL) *did_work = false; @@ -393,6 +394,9 @@ __wt_log_slot_switch(WT_SESSION_IMPL *session, WT_STAT_CONN_INCR(session, log_slot_switch_busy); __wt_yield(); } + WT_RET(WT_SESSION_CHECK_PANIC(session)); + if (F_ISSET(S2C(session), WT_CONN_CLOSING)) + break; } while (F_ISSET(myslot, WT_MYSLOT_CLOSE) || (retry && ret == EBUSY)); return (ret); } diff --git a/src/third_party/wiredtiger/src/log/log_sys.c b/src/third_party/wiredtiger/src/log/log_sys.c new file mode 100644 index 00000000000..0f3cfdbc14b --- /dev/null +++ b/src/third_party/wiredtiger/src/log/log_sys.c @@ -0,0 +1,97 @@ +/*- + * Copyright (c) 2014-2017 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_log_system_record -- + * Write a system log record for the previous LSN. + */ +int +__wt_log_system_record( + WT_SESSION_IMPL *session, WT_FH *log_fh, WT_LSN *lsn) +{ + WT_DECL_ITEM(logrec_buf); + WT_DECL_RET; + WT_ITEM *dummy, empty; + WT_LOG *log; + WT_LOG_RECORD *logrec; + WT_LOGSLOT tmp; + WT_MYSLOT myslot; + const char *fmt = WT_UNCHECKED_STRING(IIIU); + uint32_t rectype = WT_LOGREC_SYSTEM; + size_t recsize; + + log = S2C(session)->log; + WT_RET(__wt_logrec_alloc(session, log->allocsize, &logrec_buf)); + memset((uint8_t *)logrec_buf->mem, 0, log->allocsize); + WT_CLEAR(empty); + dummy = ∅ + /* + * There is currently an unused portion of the system record for + * future use. Send in a NULL entry. + */ + WT_ERR(__wt_struct_size(session, &recsize, fmt, rectype, + lsn->l.file, lsn->l.offset, dummy)); + WT_ASSERT(session, recsize <= log->allocsize); + WT_ERR(__wt_struct_pack(session, + (uint8_t *)logrec_buf->data + logrec_buf->size, recsize, fmt, + rectype, lsn->l.file, lsn->l.offset, dummy)); + logrec = (WT_LOG_RECORD *)logrec_buf->mem; + /* + * We know system records are this size. And we have to adjust + * the size now because we're not going through the normal log + * write path and the packing functions needed the correct offset + * earlier. + */ + logrec_buf->size = logrec->len = log->allocsize; + /* + * We do not compress nor encrypt this record. + */ + logrec->checksum = 0; + logrec->flags = 0; + __wt_log_record_byteswap(logrec); + logrec->checksum = __wt_checksum(logrec, log->allocsize); +#ifdef WORDS_BIGENDIAN + logrec->checksum = __wt_bswap32(logrec->checksum); +#endif + WT_CLEAR(tmp); + memset(&myslot, 0, sizeof(myslot)); + myslot.slot = &tmp; + __wt_log_slot_activate(session, &tmp); + /* + * Override the file handle to the one we're using. + */ + tmp.slot_fh = log_fh; + WT_ERR(__wt_log_fill(session, &myslot, true, logrec_buf, NULL)); +err: __wt_logrec_free(session, &logrec_buf); + return (ret); +} + +/* + * __wt_log_recover_system -- + * Process a system log record for the previous LSN in recovery. + */ +int +__wt_log_recover_system(WT_SESSION_IMPL *session, + const uint8_t **pp, const uint8_t *end, WT_LSN *lsnp) +{ + WT_DECL_RET; + WT_ITEM unused; + uint32_t prev_file, prev_offset; + const char *fmt = WT_UNCHECKED_STRING(IIU); + + if ((ret = __wt_struct_unpack(session, *pp, WT_PTRDIFF(end, *pp), fmt, + &prev_file, &prev_offset, &unused)) != 0) + WT_RET_MSG(session, ret, + "log_recover_prevlsn: unpack failure"); + if (lsnp != NULL) + WT_SET_LSN(lsnp, prev_file, prev_offset); + *pp = end; + return (0); +} diff --git a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c index 06dfec725bb..6a1709b03f2 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c @@ -1775,9 +1775,7 @@ __wt_clsm_open(WT_SESSION_IMPL *session, if (!WT_PREFIX_MATCH(uri, "lsm:")) return (__wt_unexpected_object_type(session, uri, "lsm:")); - if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) - WT_RET_MSG(session, EINVAL, - "LSM trees not supported by in-memory configurations"); + WT_RET(__wt_inmem_unsupported_op(session, "LSM trees")); WT_RET(__wt_config_gets_def(session, cfg, "checkpoint", 0, &cval)); if (cval.len != 0) diff --git a/src/third_party/wiredtiger/src/lsm/lsm_merge.c b/src/third_party/wiredtiger/src/lsm/lsm_merge.c index 28e954c6f6a..f3990a72d95 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_merge.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_merge.c @@ -621,7 +621,7 @@ err: if (locked) if (ret == EINTR) __wt_verbose(session, WT_VERB_LSM, - "Merge aborted due to close"); + "%s", "Merge aborted due to close"); else __wt_verbose(session, WT_VERB_LSM, "Merge failed with %s", diff --git a/src/third_party/wiredtiger/src/lsm/lsm_tree.c b/src/third_party/wiredtiger/src/lsm/lsm_tree.c index ebdd1e492cc..18e1f6d3115 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_tree.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_tree.c @@ -15,6 +15,42 @@ static int __lsm_tree_open( static int __lsm_tree_set_name(WT_SESSION_IMPL *, WT_LSM_TREE *, const char *); /* + * __lsm_tree_discard_state -- + * Free the metadata configuration state related LSM tree pointers. + */ +static void +__lsm_tree_discard_state(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) +{ + WT_LSM_CHUNK *chunk; + u_int i; + + __wt_free(session, lsm_tree->config); + __wt_free(session, lsm_tree->key_format); + __wt_free(session, lsm_tree->value_format); + __wt_free(session, lsm_tree->collator_name); + __wt_free(session, lsm_tree->bloom_config); + __wt_free(session, lsm_tree->file_config); + + for (i = 0; i < lsm_tree->nchunks; i++) { + if ((chunk = lsm_tree->chunk[i]) == NULL) + continue; + + __wt_free(session, chunk->bloom_uri); + __wt_free(session, chunk->uri); + __wt_free(session, chunk); + } + + for (i = 0; i < lsm_tree->nold_chunks; i++) { + chunk = lsm_tree->old_chunks[i]; + WT_ASSERT(session, chunk != NULL); + + __wt_free(session, chunk->bloom_uri); + __wt_free(session, chunk->uri); + __wt_free(session, chunk); + } +} + +/* * __lsm_tree_discard -- * Free an LSM tree structure. */ @@ -22,8 +58,6 @@ static int __lsm_tree_discard(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, bool final) { WT_DECL_RET; - WT_LSM_CHUNK *chunk; - u_int i; WT_UNUSED(final); /* Only used in diagnostic builds */ @@ -48,34 +82,12 @@ __lsm_tree_discard(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, bool final) lsm_tree->collator, &session->iface)); __wt_free(session, lsm_tree->name); - __wt_free(session, lsm_tree->config); - __wt_free(session, lsm_tree->key_format); - __wt_free(session, lsm_tree->value_format); - __wt_free(session, lsm_tree->collator_name); - __wt_free(session, lsm_tree->bloom_config); - __wt_free(session, lsm_tree->file_config); - - __wt_rwlock_destroy(session, &lsm_tree->rwlock); - - for (i = 0; i < lsm_tree->nchunks; i++) { - if ((chunk = lsm_tree->chunk[i]) == NULL) - continue; - - __wt_free(session, chunk->bloom_uri); - __wt_free(session, chunk->uri); - __wt_free(session, chunk); - } + __lsm_tree_discard_state(session, lsm_tree); __wt_free(session, lsm_tree->chunk); + __wt_free(session, lsm_tree->old_chunks); - for (i = 0; i < lsm_tree->nold_chunks; i++) { - chunk = lsm_tree->old_chunks[i]; - WT_ASSERT(session, chunk != NULL); + __wt_rwlock_destroy(session, &lsm_tree->rwlock); - __wt_free(session, chunk->bloom_uri); - __wt_free(session, chunk->uri); - __wt_free(session, chunk); - } - __wt_free(session, lsm_tree->old_chunks); __wt_free(session, lsm_tree); return (ret); @@ -851,45 +863,6 @@ __wt_lsm_tree_retire_chunks(WT_SESSION_IMPL *session, } /* - * __wt_lsm_tree_alter -- - * Alter an LSM tree. - */ -int -__wt_lsm_tree_alter( - WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) -{ - WT_DECL_RET; - WT_LSM_CHUNK *chunk; - WT_LSM_TREE *lsm_tree; - u_int i; - bool locked; - - locked = false; - - /* Get the LSM tree. */ - WT_RET(__wt_lsm_tree_get(session, uri, false, &lsm_tree)); - - /* Prevent any new opens. */ - __wt_lsm_tree_writelock(session, lsm_tree); - locked = true; - - /* Alter the chunks. */ - for (i = 0; i < lsm_tree->nchunks; i++) { - chunk = lsm_tree->chunk[i]; - WT_ERR(__wt_schema_alter(session, chunk->uri, cfg)); - if (F_ISSET(chunk, WT_LSM_CHUNK_BLOOM)) - WT_ERR( - __wt_schema_alter(session, chunk->bloom_uri, cfg)); - } - WT_ERR(__wt_lsm_meta_write(session, lsm_tree, cfg[0])); - -err: if (locked) - __wt_lsm_tree_writeunlock(session, lsm_tree); - __wt_lsm_tree_release(session, lsm_tree); - return (ret); -} - -/* * __wt_lsm_tree_drop -- * Drop an LSM tree. */ @@ -1385,6 +1358,24 @@ __wt_lsm_tree_worker(WT_SESSION_IMPL *session, WT_ERR(__wt_schema_worker(session, chunk->bloom_uri, file_func, name_func, cfg, open_flags)); } + /* + * If this was an alter operation, we need to alter the configuration + * for the overall tree and then reread it so it isn't out of date. + * Reread it here so that we update the configuration of the + * current tree's structure to any new, altered values. + */ + if (FLD_ISSET(open_flags, WT_BTREE_ALTER)) { + WT_ERR(__wt_lsm_meta_write(session, lsm_tree, cfg[0])); + /* + * We're about to read in the new configuration that + * we just wrote. Free the old ones. + */ + __lsm_tree_discard_state(session, lsm_tree); + if ((ret = __wt_lsm_meta_read(session, lsm_tree)) != 0) + WT_PANIC_ERR(session, ret, + "Failed to read updated LSM configuration"); + } + err: if (locked) { if (exclusive) __wt_lsm_tree_writeunlock(session, lsm_tree); diff --git a/src/third_party/wiredtiger/src/meta/meta_ckpt.c b/src/third_party/wiredtiger/src/meta/meta_ckpt.c index 0e96c4ee6ca..24f991cf549 100644 --- a/src/third_party/wiredtiger/src/meta/meta_ckpt.c +++ b/src/third_party/wiredtiger/src/meta/meta_ckpt.c @@ -429,7 +429,7 @@ __wt_meta_ckptlist_set(WT_SESSION_IMPL *session, } if (strcmp(ckpt->name, WT_CHECKPOINT) == 0) WT_ERR(__wt_buf_catfmt(session, buf, - "%s%s.%" PRId64 "=(addr=\"%.*s\",order=%" PRIu64 + "%s%s.%" PRId64 "=(addr=\"%.*s\",order=%" PRId64 ",time=%" PRIuMAX ",size=%" PRIu64 ",write_gen=%" PRIu64 ")", sep, ckpt->name, ckpt->order, @@ -438,7 +438,7 @@ __wt_meta_ckptlist_set(WT_SESSION_IMPL *session, ckpt->write_gen)); else WT_ERR(__wt_buf_catfmt(session, buf, - "%s%s=(addr=\"%.*s\",order=%" PRIu64 + "%s%s=(addr=\"%.*s\",order=%" PRId64 ",time=%" PRIuMAX ",size=%" PRIu64 ",write_gen=%" PRIu64 ")", sep, ckpt->name, diff --git a/src/third_party/wiredtiger/src/os_win/os_map.c b/src/third_party/wiredtiger/src/os_win/os_map.c index c0aa6dac28f..25835d2615d 100644 --- a/src/third_party/wiredtiger/src/os_win/os_map.c +++ b/src/third_party/wiredtiger/src/os_win/os_map.c @@ -82,6 +82,8 @@ __wt_win_unmap(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, WT_FILE_HANDLE_WIN *win_fh; WT_SESSION_IMPL *session; + WT_UNUSED(length); /* !HAVE_VERBOSE */ + win_fh = (WT_FILE_HANDLE_WIN *)file_handle; session = (WT_SESSION_IMPL *)wt_session; diff --git a/src/third_party/wiredtiger/src/reconcile/rec_track.c b/src/third_party/wiredtiger/src/reconcile/rec_track.c index a431465661f..ee97d684529 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_track.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_track.c @@ -16,11 +16,11 @@ (sizeof(s) + 2 * sizeof(void *) + (p)->addr_size + (p)->value_size) /* - * __ovfl_track_init -- + * __wt_ovfl_track_init -- * Initialize the overflow tracking structure. */ -static int -__ovfl_track_init(WT_SESSION_IMPL *session, WT_PAGE *page) +int +__wt_ovfl_track_init(WT_SESSION_IMPL *session, WT_PAGE *page) { return (__wt_calloc_one(session, &page->modify->ovfl_track)); } @@ -36,6 +36,9 @@ __ovfl_discard_verbose( WT_CELL_UNPACK *unpack, _unpack; WT_DECL_ITEM(tmp); + WT_UNUSED(page); /* !HAVE_VERBOSE */ + WT_UNUSED(tag); /* !HAVE_VERBOSE */ + WT_RET(__wt_scr_alloc(session, 512, &tmp)); unpack = &_unpack; @@ -128,7 +131,7 @@ __wt_ovfl_discard_add(WT_SESSION_IMPL *session, WT_PAGE *page, WT_CELL *cell) WT_OVFL_TRACK *track; if (page->modify->ovfl_track == NULL) - WT_RET(__ovfl_track_init(session, page)); + WT_RET(__wt_ovfl_track_init(session, page)); track = page->modify->ovfl_track; WT_RET(__wt_realloc_def(session, &track->discard_allocated, @@ -169,6 +172,10 @@ __ovfl_reuse_verbose(WT_SESSION_IMPL *session, { WT_DECL_ITEM(tmp); + WT_UNUSED(page); /* !HAVE_VERBOSE */ + WT_UNUSED(reuse); /* !HAVE_VERBOSE */ + WT_UNUSED(tag); /* !HAVE_VERBOSE */ + WT_RET(__wt_scr_alloc(session, 64, &tmp)); __wt_verbose(session, WT_VERB_OVERFLOW, @@ -182,7 +189,8 @@ __ovfl_reuse_verbose(WT_SESSION_IMPL *session, F_ISSET(reuse, WT_OVFL_REUSE_INUSE) && F_ISSET(reuse, WT_OVFL_REUSE_JUST_ADDED) ? ", " : "", F_ISSET(reuse, WT_OVFL_REUSE_JUST_ADDED) ? "just-added" : "", - WT_MIN(reuse->value_size, 40), (char *)WT_OVFL_REUSE_VALUE(reuse)); + (int)WT_MIN(reuse->value_size, 40), + (char *)WT_OVFL_REUSE_VALUE(reuse)); __wt_scr_free(session, &tmp); return (0); @@ -487,7 +495,7 @@ __wt_ovfl_reuse_add(WT_SESSION_IMPL *session, WT_PAGE *page, uint8_t *p; if (page->modify->ovfl_track == NULL) - WT_RET(__ovfl_track_init(session, page)); + WT_RET(__wt_ovfl_track_init(session, page)); head = page->modify->ovfl_track->ovfl_reuse; @@ -557,311 +565,12 @@ __wt_ovfl_reuse_free(WT_SESSION_IMPL *session, WT_PAGE *page) } /* - * __ovfl_txnc_verbose -- - * Dump information about a transaction-cached overflow record. - */ -static int -__ovfl_txnc_verbose(WT_SESSION_IMPL *session, - WT_PAGE *page, WT_OVFL_TXNC *txnc, const char *tag) -{ - WT_DECL_ITEM(tmp); - - WT_RET(__wt_scr_alloc(session, 64, &tmp)); - - __wt_verbose(session, WT_VERB_OVERFLOW, - "txn-cache: %s%s%p %s %" PRIu64 " {%.*s}", - tag == NULL ? "" : tag, - tag == NULL ? "" : ": ", - (void *)page, - __wt_addr_string( - session, WT_OVFL_TXNC_ADDR(txnc), txnc->addr_size, tmp), - txnc->current, - WT_MIN(txnc->value_size, 40), (char *)WT_OVFL_TXNC_VALUE(txnc)); - - __wt_scr_free(session, &tmp); - return (0); -} - -#if 0 -/* - * __ovfl_txnc_dump -- - * Debugging information. - */ -static void -__ovfl_txnc_dump(WT_SESSION_IMPL *session, WT_PAGE *page) -{ - WT_OVFL_TXNC **head, *txnc; - - if (page->modify == NULL || page->modify->ovfl_track == NULL) - return; - head = page->modify->ovfl_track->ovfl_txnc; - - for (txnc = head[0]; txnc != NULL; txnc = txnc->next[0]) - (void)__ovfl_txnc_verbose(session, page, txnc, "dump"); -} -#endif - -/* - * __ovfl_txnc_skip_search -- - * Return the first matching addr in the overflow transaction-cache list. - */ -static WT_OVFL_TXNC * -__ovfl_txnc_skip_search(WT_OVFL_TXNC **head, const void *addr, size_t addr_size) -{ - WT_OVFL_TXNC **e; - size_t len; - int cmp, i; - - /* - * Start at the highest skip level, then go as far as possible at each - * level before stepping down to the next. - */ - for (i = WT_SKIP_MAXDEPTH - 1, e = &head[i]; i >= 0;) { - if (*e == NULL) { /* Empty levels */ - --i; - --e; - continue; - } - - /* - * Return any exact matches: we don't care in what search level - * we found a match. - */ - len = WT_MIN((*e)->addr_size, addr_size); - cmp = memcmp(WT_OVFL_TXNC_ADDR(*e), addr, len); - if (cmp == 0 && (*e)->addr_size == addr_size) - return (*e); - - /* - * If the skiplist address is larger than the search address, or - * they compare equally and the skiplist address is longer than - * the search address, drop down a level, otherwise continue on - * this level. - */ - if (cmp > 0 || (cmp == 0 && (*e)->addr_size > addr_size)) { - --i; /* Drop down a level */ - --e; - } else /* Keep going at this level */ - e = &(*e)->next[i]; - } - return (NULL); -} - -/* - * __ovfl_txnc_skip_search_stack -- - * Search an overflow transaction-cache skiplist, returning an - * insert/remove stack. - */ -static void -__ovfl_txnc_skip_search_stack(WT_OVFL_TXNC **head, - WT_OVFL_TXNC ***stack, const void *addr, size_t addr_size) -{ - WT_OVFL_TXNC **e; - size_t len; - int cmp, i; - - /* - * Start at the highest skip level, then go as far as possible at each - * level before stepping down to the next. - */ - for (i = WT_SKIP_MAXDEPTH - 1, e = &head[i]; i >= 0;) { - if (*e == NULL) { /* Empty levels */ - stack[i--] = e--; - continue; - } - - /* - * If the skiplist addr is larger than the search addr, or - * they compare equally and the skiplist addr is longer than - * the search addr, drop down a level, otherwise continue on - * this level. - */ - len = WT_MIN((*e)->addr_size, addr_size); - cmp = memcmp(WT_OVFL_TXNC_ADDR(*e), addr, len); - if (cmp > 0 || (cmp == 0 && (*e)->addr_size > addr_size)) - stack[i--] = e--; /* Drop down a level */ - else - e = &(*e)->next[i]; /* Keep going at this level */ - } -} - -/* - * __ovfl_txnc_wrapup -- - * Resolve the page's transaction-cache list. - */ -static int -__ovfl_txnc_wrapup(WT_SESSION_IMPL *session, WT_PAGE *page) -{ - WT_OVFL_TXNC **e, **head, *txnc; - uint64_t oldest_txn; - size_t decr; - int i; - - head = page->modify->ovfl_track->ovfl_txnc; - - /* - * Take a snapshot of the oldest transaction ID we need to keep alive. - * Since we do two passes through entries in the structure, the normal - * visibility check could give different results as the global ID moves - * forward. - */ - oldest_txn = __wt_txn_oldest_id(session); - - /* - * Discard any transaction-cache records with transaction IDs earlier - * than any in the system. - * - * First, walk the overflow transaction-cache skip lists (except for - * the lowest level), fixing up links. - */ - for (i = WT_SKIP_MAXDEPTH - 1; i > 0; --i) - for (e = &head[i]; (txnc = *e) != NULL;) { - if (WT_TXNID_LE(oldest_txn, txnc->current)) { - e = &txnc->next[i]; - continue; - } - *e = txnc->next[i]; - } - - /* Second, discard any no longer needed transaction-cache records. */ - decr = 0; - for (e = &head[0]; (txnc = *e) != NULL;) { - if (WT_TXNID_LE(oldest_txn, txnc->current)) { - e = &txnc->next[0]; - continue; - } - *e = txnc->next[0]; - - if (WT_VERBOSE_ISSET(session, WT_VERB_OVERFLOW)) - WT_RET( - __ovfl_txnc_verbose(session, page, txnc, "free")); - - decr += WT_OVFL_SIZE(txnc, WT_OVFL_TXNC); - __wt_free(session, txnc); - } - - if (decr != 0) - __wt_cache_page_inmem_decr(session, page, decr); - return (0); -} - -/* - * __wt_ovfl_txnc_search -- - * Search the page's list of transaction-cache overflow records for a - * match. - */ -int -__wt_ovfl_txnc_search( - WT_PAGE *page, const uint8_t *addr, size_t addr_size, WT_ITEM *store) -{ - WT_OVFL_TXNC **head, *txnc; - - if (page->modify->ovfl_track == NULL) - return (WT_NOTFOUND); - - head = page->modify->ovfl_track->ovfl_txnc; - - if ((txnc = __ovfl_txnc_skip_search(head, addr, addr_size)) == NULL) - return (WT_NOTFOUND); - - store->data = WT_OVFL_TXNC_VALUE(txnc); - store->size = txnc->value_size; - return (0); -} - -/* - * __wt_ovfl_txnc_add -- - * Add a new entry to the page's list of transaction-cached overflow - * records. - */ -int -__wt_ovfl_txnc_add(WT_SESSION_IMPL *session, WT_PAGE *page, - const uint8_t *addr, size_t addr_size, - const void *value, size_t value_size) -{ - WT_OVFL_TXNC **head, **stack[WT_SKIP_MAXDEPTH], *txnc; - size_t size; - u_int i, skipdepth; - uint8_t *p; - - if (page->modify->ovfl_track == NULL) - WT_RET(__ovfl_track_init(session, page)); - - head = page->modify->ovfl_track->ovfl_txnc; - - /* Choose a skiplist depth for this insert. */ - skipdepth = __wt_skip_choose_depth(session); - - /* - * Allocate the WT_OVFL_TXNC structure, next pointers for the skip - * list, room for the address and value, then copy everything into - * place. - * - * To minimize the WT_OVFL_TXNC structure size, the address offset - * and size are single bytes: that's safe because the address follows - * the structure (which can't be more than about 100B), and address - * cookies are limited to 255B. - */ - size = sizeof(WT_OVFL_TXNC) + - skipdepth * sizeof(WT_OVFL_TXNC *) + addr_size + value_size; - WT_RET(__wt_calloc(session, 1, size, &txnc)); - p = (uint8_t *)txnc + - sizeof(WT_OVFL_TXNC) + skipdepth * sizeof(WT_OVFL_TXNC *); - txnc->addr_offset = (uint8_t)WT_PTRDIFF(p, txnc); - txnc->addr_size = (uint8_t)addr_size; - memcpy(p, addr, addr_size); - p += addr_size; - txnc->value_offset = WT_PTRDIFF32(p, txnc); - txnc->value_size = WT_STORE_SIZE(value_size); - memcpy(p, value, value_size); - txnc->current = __wt_txn_id_alloc(session, false); - - __wt_cache_page_inmem_incr( - session, page, WT_OVFL_SIZE(txnc, WT_OVFL_TXNC)); - - /* Insert the new entry into the skiplist. */ - __ovfl_txnc_skip_search_stack(head, stack, addr, addr_size); - for (i = 0; i < skipdepth; ++i) { - txnc->next[i] = *stack[i]; - *stack[i] = txnc; - } - - if (WT_VERBOSE_ISSET(session, WT_VERB_OVERFLOW)) - WT_RET(__ovfl_txnc_verbose(session, page, txnc, "add")); - - return (0); -} - -/* - * __wt_ovfl_txnc_free -- - * Free the page's list of transaction-cached overflow records. - */ -void -__wt_ovfl_txnc_free(WT_SESSION_IMPL *session, WT_PAGE *page) -{ - WT_OVFL_TXNC *txnc; - WT_PAGE_MODIFY *mod; - void *next; - - mod = page->modify; - if (mod == NULL || mod->ovfl_track == NULL) - return; - - for (txnc = mod->ovfl_track->ovfl_txnc[0]; - txnc != NULL; txnc = next) { - next = txnc->next[0]; - __wt_free(session, txnc); - } -} - -/* * __wt_ovfl_track_wrapup -- * Resolve the page's overflow tracking on reconciliation success. */ int __wt_ovfl_track_wrapup(WT_SESSION_IMPL *session, WT_PAGE *page) { - WT_DECL_RET; WT_OVFL_TRACK *track; if (page->modify == NULL || page->modify->ovfl_track == NULL) @@ -874,12 +583,7 @@ __wt_ovfl_track_wrapup(WT_SESSION_IMPL *session, WT_PAGE *page) if (track->ovfl_reuse[0] != NULL) WT_RET(__ovfl_reuse_wrapup(session, page)); - if (track->ovfl_txnc[0] != NULL) { - __wt_writelock(session, &S2BT(session)->ovfl_lock); - ret = __ovfl_txnc_wrapup(session, page); - __wt_writeunlock(session, &S2BT(session)->ovfl_lock); - } - return (ret); + return (0); } /* @@ -889,7 +593,6 @@ __wt_ovfl_track_wrapup(WT_SESSION_IMPL *session, WT_PAGE *page) int __wt_ovfl_track_wrapup_err(WT_SESSION_IMPL *session, WT_PAGE *page) { - WT_DECL_RET; WT_OVFL_TRACK *track; if (page->modify == NULL || page->modify->ovfl_track == NULL) @@ -902,10 +605,5 @@ __wt_ovfl_track_wrapup_err(WT_SESSION_IMPL *session, WT_PAGE *page) if (track->ovfl_reuse[0] != NULL) WT_RET(__ovfl_reuse_wrapup_err(session, page)); - if (track->ovfl_txnc[0] != NULL) { - __wt_writelock(session, &S2BT(session)->ovfl_lock); - ret = __ovfl_txnc_wrapup(session, page); - __wt_writeunlock(session, &S2BT(session)->ovfl_lock); - } - return (ret); + return (0); } diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index a304dc54340..970f760f4ca 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -1179,7 +1179,7 @@ __rec_txn_read(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_DECL_TIMESTAMP(max_timestamp) WT_PAGE *page; WT_UPDATE *append, *upd, *upd_list; - size_t notused, update_mem; + size_t size, update_mem; uint64_t max_txn, min_txn, txnid; bool append_origv, skipped; @@ -1204,8 +1204,8 @@ __rec_txn_read(WT_SESSION_IMPL *session, WT_RECONCILE *r, update_mem = 0; max_txn = WT_TXN_NONE; #ifdef HAVE_TIMESTAMPS - __wt_timestamp_set(max_timestamp, zero_timestamp); - memset(min_timestamp, 0xff, WT_TIMESTAMP_SIZE); + __wt_timestamp_set_zero(max_timestamp); + __wt_timestamp_set_inf(min_timestamp); #endif min_txn = UINT64_MAX; @@ -1419,45 +1419,6 @@ __rec_txn_read(WT_SESSION_IMPL *session, WT_RECONCILE *r, /* The page can't be marked clean. */ r->leave_dirty = true; - - /* - * A special-case for overflow values, where we can't write the - * original on-page value item to disk because it's been updated - * or removed. - * - * What happens is that an overflow value is updated or removed - * and its backing blocks freed. If any reader in the system - * might still want the value, a copy was cached in the page - * reconciliation tracking memory, and the page cell set to - * WT_CELL_VALUE_OVFL_RM. Eviction then chose the page and - * we're splitting it up in order to push parts of it out of - * memory. - * - * We could write the original on-page value item to disk... if - * we had a copy. The cache may not have a copy (a globally - * visible update would have kept a value from being cached), or - * an update that subsequently became globally visible could - * cause a cached value to be discarded. Either way, once there - * is a globally visible update, we may not have the original - * value. - * - * Fortunately, if there's a globally visible update we don't - * care about the original version, so we simply ignore it, no - * transaction can ever try and read it. If there isn't a - * globally visible update, there had better be a cached value. - * - * In the latter case, we could write the value out to disk, but - * (1) we are planning on re-instantiating this page in memory, - * it isn't going to disk, and (2) the value item is eventually - * going to be discarded, that seems like a waste of a write. - * Instead, find the cached value and append it to the update - * list we're saving for later restoration. - */ - if (vpack != NULL && - vpack->raw == WT_CELL_VALUE_OVFL_RM && - !__wt_txn_visible_all( - session, min_txn, WT_TIMESTAMP(min_timestamp))) - append_origv = true; } else { /* * The lookaside table eviction path. @@ -1482,22 +1443,23 @@ __rec_txn_read(WT_SESSION_IMPL *session, WT_RECONCILE *r, * key/value pair which simply doesn't exist for some reader; * place a deleted record at the end of the update list. */ + size = 0; /* -Wconditional-uninitialized */ if (vpack == NULL || vpack->type == WT_CELL_DEL) WT_RET(__wt_update_alloc(session, - NULL, &append, ¬used, WT_UPDATE_DELETED)); + NULL, &append, &size, WT_UPDATE_DELETED)); else { WT_RET(__wt_scr_alloc(session, 0, &tmp)); if ((ret = __wt_page_cell_data_ref( session, page, vpack, tmp)) == 0) ret = __wt_update_alloc(session, - tmp, &append, ¬used, WT_UPDATE_STANDARD); + tmp, &append, &size, WT_UPDATE_STANDARD); __wt_scr_free(session, &tmp); WT_RET(ret); } /* - * Give the entry an impossibly low transaction ID to ensure its - * global visibility, append it to the update list. + * Give the entry no transaction ID to ensure global visibility, + * append it to the update list. * * Note the change to the actual reader-accessible update list: * from now on, the original on-page value appears at the end @@ -1508,8 +1470,7 @@ __rec_txn_read(WT_SESSION_IMPL *session, WT_RECONCILE *r, for (upd = upd_list; upd->next != NULL; upd = upd->next) ; WT_PUBLISH(upd->next, append); - __wt_cache_page_inmem_incr( - session, page, WT_UPDATE_MEMSIZE(append)); + __wt_cache_page_inmem_incr(session, page, size); } /* @@ -4752,29 +4713,25 @@ record_loop: /* deleted = false; /* - * If doing update save and restore, there's an - * update that's not globally visible, and the + * If doing an update save and restore, and the * underlying value is a removed overflow value, * we end up here. * - * When the update save/restore code noticed the - * removed overflow value, it appended a copy of - * the cached, original overflow value to the - * update list being saved (ensuring the on-page - * item will never be accessed after the page is - * re-instantiated), then returned a NULL update - * to us. + * If necessary, when the overflow value was + * originally removed, reconciliation appended + * a globally visible copy of the value to the + * key's update list, meaning the on-page item + * isn't accessed after page re-instantiation. * - * Assert the case: if we remove an underlying - * overflow object, checkpoint reconciliation - * should never see it again, there should be a - * visible update in the way. - * - * Write a placeholder. + * Assert the case. */ WT_ASSERT(session, F_ISSET(r, WT_EVICT_UPDATE_RESTORE)); + /* + * The on-page value will never be accessed, + * write a placeholder record. + */ data = "@"; size = 1; } else { @@ -4905,16 +4862,12 @@ compare: /* } /* - * If we had a reference to an overflow record we never used, + * The first time we find an overflow record we never used, * discard the underlying blocks, they're no longer useful. - * - * One complication: we must cache a copy before discarding the - * on-disk version if there's a transaction in the system that - * might read the original value. */ if (ovfl_state == OVFL_UNUSED && vpack->raw != WT_CELL_VALUE_OVFL_RM) - WT_ERR(__wt_ovfl_cache(session, page, upd, vpack)); + WT_ERR(__wt_ovfl_remove(session, page, upd, vpack)); } /* Walk any append list. */ @@ -5079,7 +5032,7 @@ __rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) cell = NULL; key_onpage_ovfl = false; - WT_RET(__rec_split_init(session, r, page, 0ULL, btree->maxintlpage)); + WT_RET(__rec_split_init(session, r, page, 0, btree->maxintlpage)); /* * Ideally, we'd never store the 0th key on row-store internal pages @@ -5361,7 +5314,7 @@ __rec_row_leaf(WT_SESSION_IMPL *session, key = &r->k; val = &r->v; - WT_RET(__rec_split_init(session, r, page, 0ULL, btree->maxleafpage)); + WT_RET(__rec_split_init(session, r, page, 0, btree->maxleafpage)); /* * Write any K/V pairs inserted into the page before the first from-disk @@ -5451,18 +5404,15 @@ __rec_row_leaf(WT_SESSION_IMPL *session, dictionary = true; } else if (vpack->raw == WT_CELL_VALUE_OVFL_RM) { /* - * If doing update save and restore in service - * of eviction, there's an update that's not - * globally visible, and the underlying value - * is a removed overflow value, we end up here. + * If doing an update save and restore, and the + * underlying value is a removed overflow value, + * we end up here. * - * When the update save/restore code noticed the - * removed overflow value, it appended a copy of - * the cached, original overflow value to the - * update list being saved (ensuring any on-page - * item will never be accessed after the page is - * re-instantiated), then returned a NULL update - * to us. + * If necessary, when the overflow value was + * originally removed, reconciliation appended + * a globally visible copy of the value to the + * key's update list, meaning the on-page item + * isn't accessed after page re-instantiation. * * Assert the case. */ @@ -5508,16 +5458,13 @@ __rec_row_leaf(WT_SESSION_IMPL *session, } } else { /* - * If the original value was an overflow and we've not - * already done so, discard it. One complication: we - * must cache a copy before discarding the on-disk - * version if there's a transaction in the system that - * might read the original value. + * The first time we find an overflow record we're not + * going to use, discard the underlying blocks. */ if (vpack != NULL && vpack->ovfl && vpack->raw != WT_CELL_VALUE_OVFL_RM) - WT_ERR( - __wt_ovfl_cache(session, page, rip, vpack)); + WT_ERR(__wt_ovfl_remove( + session, page, upd, vpack)); /* If this key/value pair was deleted, we're done. */ if (upd->type == WT_UPDATE_DELETED) { diff --git a/src/third_party/wiredtiger/src/schema/schema_alter.c b/src/third_party/wiredtiger/src/schema/schema_alter.c index 346f09a1a64..7af4b86bc85 100644 --- a/src/third_party/wiredtiger/src/schema/schema_alter.c +++ b/src/third_party/wiredtiger/src/schema/schema_alter.c @@ -9,16 +9,24 @@ #include "wt_internal.h" /* - * __alter_file -- + * __wt_alter -- * Alter a file. */ -static int -__alter_file(WT_SESSION_IMPL *session, const char *uri, const char *newcfg[]) +int +__wt_alter(WT_SESSION_IMPL *session, const char *newcfg[]) { WT_DECL_RET; - const char *cfg[4], *filename; + const char *cfg[4], *filename, *uri; char *config, *newconfig; + uri = session->dhandle->name; + WT_RET(__wt_meta_track_on(session)); + + /* + * We know that we have exclusive access to the file. So it will be + * closed after we're done with it and the next open will see the + * updated metadata. + */ filename = uri; newconfig = NULL; if (!WT_PREFIX_SKIP(filename, "file:")) @@ -48,122 +56,6 @@ __alter_file(WT_SESSION_IMPL *session, const char *uri, const char *newcfg[]) err: __wt_free(session, config); __wt_free(session, newconfig); - return (ret); -} - -/* - * __alter_colgroup -- - * WT_SESSION::alter for a colgroup. - */ -static int -__alter_colgroup( - WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) -{ - WT_COLGROUP *colgroup; - WT_DECL_RET; - - WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_TABLE)); - - /* If we can get the colgroup, perform any potential alterations. */ - if ((ret = __wt_schema_get_colgroup( - session, uri, false, NULL, &colgroup)) == 0) - WT_TRET(__wt_schema_alter(session, colgroup->source, cfg)); - - return (ret); -} - -/* - * __alter_index -- - * WT_SESSION::alter for an index. - */ -static int -__alter_index( - WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) -{ - WT_INDEX *idx; - WT_DECL_RET; - - /* If we can get the index, perform any potential alterations. */ - if ((ret = __wt_schema_get_index( - session, uri, false, NULL, &idx)) == 0) - WT_TRET(__wt_schema_alter(session, idx->source, cfg)); - - return (ret); -} - -/* - * __alter_table -- - * WT_SESSION::alter for a table. - */ -static int -__alter_table(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) -{ - WT_COLGROUP *colgroup; - WT_DECL_RET; - WT_TABLE *table; - const char *name; - u_int i; - - name = uri; - (void)WT_PREFIX_SKIP(name, "table:"); - - WT_RET(__wt_schema_get_table( - session, name, strlen(name), true, &table)); - - /* - * Alter the column groups only if we are using the default - * column group. Otherwise the user should alter each - * index or column group explicitly. - */ - if (table->ncolgroups == 0) - for (i = 0; i < WT_COLGROUPS(table); i++) { - if ((colgroup = table->cgroups[i]) == NULL) - continue; - /* - * Alter the column group before updating the metadata - * to avoid the metadata for the table becoming - * inconsistent if we can't get exclusive access. - */ - WT_ERR(__wt_schema_alter( - session, colgroup->source, cfg)); - } -err: __wt_schema_release_table(session, table); - return (ret); -} - -/* - * __wt_schema_alter -- - * Process a WT_SESSION::alter operation for all supported types. - */ -int -__wt_schema_alter(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) -{ - WT_DATA_SOURCE *dsrc; - WT_DECL_RET; - - WT_RET(__wt_meta_track_on(session)); - - /* Paranoia: clear any handle from our caller. */ - session->dhandle = NULL; - - if (WT_PREFIX_MATCH(uri, "colgroup:")) - ret = __alter_colgroup(session, uri, cfg); - else if (WT_PREFIX_MATCH(uri, "file:")) - ret = __alter_file(session, uri, cfg); - else if (WT_PREFIX_MATCH(uri, "index:")) - ret = __alter_index(session, uri, cfg); - else if (WT_PREFIX_MATCH(uri, "lsm:")) - ret = __wt_lsm_tree_alter(session, uri, cfg); - else if (WT_PREFIX_MATCH(uri, "table:")) - ret = __alter_table(session, uri, cfg); - else if ((dsrc = __wt_schema_get_source(session, uri)) != NULL) - ret = dsrc->alter == NULL ? - __wt_object_unsupported(session, uri) : - dsrc->alter(dsrc, - &session->iface, uri, (WT_CONFIG_ARG *)cfg); - else - ret = __wt_bad_object_type(session, uri); - /* * Map WT_NOTFOUND to ENOENT, based on the assumption WT_NOTFOUND means * there was no metadata entry. diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c index 882de30f12b..2ad2380dc2c 100644 --- a/src/third_party/wiredtiger/src/session/session_api.c +++ b/src/third_party/wiredtiger/src/session/session_api.c @@ -149,6 +149,10 @@ __session_alter(WT_SESSION *wt_session, const char *uri, const char *config) SESSION_API_CALL(session, alter, config, cfg); + /* In-memory ignores alter operations. */ + if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) + goto err; + /* Disallow objects in the WiredTiger name space. */ WT_ERR(__wt_str_name_check(session, uri)); @@ -161,8 +165,9 @@ __session_alter(WT_SESSION *wt_session, const char *uri, const char *config) cfg[1] = NULL; WT_WITH_CHECKPOINT_LOCK(session, WT_WITH_SCHEMA_LOCK(session, - WT_WITH_TABLE_WRITE_LOCK(session, - ret = __wt_schema_alter(session, uri, cfg)))); + ret = __wt_schema_worker(session, uri, __wt_alter, NULL, cfg, + WT_BTREE_ALTER | + WT_DHANDLE_DISCARD_FORCE | WT_DHANDLE_EXCLUSIVE))); err: if (ret != 0) WT_STAT_CONN_INCR(session, session_table_alter_fail); @@ -700,6 +705,10 @@ __session_rebalance(WT_SESSION *wt_session, const char *uri, const char *config) SESSION_API_CALL(session, rebalance, config, cfg); + /* In-memory ignores rebalance operations. */ + if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) + goto err; + /* Block out checkpoints to avoid spurious EBUSY errors. */ WT_WITH_CHECKPOINT_LOCK(session, WT_WITH_SCHEMA_LOCK(session, @@ -709,8 +718,7 @@ __session_rebalance(WT_SESSION *wt_session, const char *uri, const char *config) err: if (ret != 0) WT_STAT_CONN_INCR(session, session_table_rebalance_fail); else - WT_STAT_CONN_INCR(session, - session_table_rebalance_success); + WT_STAT_CONN_INCR(session, session_table_rebalance_success); API_END_RET_NOTFOUND_MAP(session, ret); } @@ -1046,10 +1054,7 @@ __session_salvage(WT_SESSION *wt_session, const char *uri, const char *config) SESSION_API_CALL(session, salvage, config, cfg); - if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) - WT_ERR_MSG(session, ENOTSUP, - "WT_SESSION.salvage not supported for in-memory " - "configurations"); + WT_ERR(__wt_inmem_unsupported_op(session, NULL)); /* Block out checkpoints to avoid spurious EBUSY errors. */ WT_WITH_CHECKPOINT_LOCK(session, @@ -1258,7 +1263,7 @@ __session_truncate(WT_SESSION *wt_session, WT_ERR_MSG(session, EINVAL, "the truncate method should not specify any" "target after the log: URI prefix"); - WT_ERR(__wt_log_truncate_files(session, start, cfg)); + WT_ERR(__wt_log_truncate_files(session, start, false)); } else if (WT_PREFIX_MATCH(uri, "file:")) WT_ERR(__wt_session_range_truncate( session, uri, start, stop)); @@ -1319,6 +1324,9 @@ __session_upgrade(WT_SESSION *wt_session, const char *uri, const char *config) session = (WT_SESSION_IMPL *)wt_session; SESSION_API_CALL(session, upgrade, config, cfg); + + WT_ERR(__wt_inmem_unsupported_op(session, NULL)); + /* Block out checkpoints to avoid spurious EBUSY errors. */ WT_WITH_CHECKPOINT_LOCK(session, WT_WITH_SCHEMA_LOCK(session, @@ -1363,10 +1371,7 @@ __session_verify(WT_SESSION *wt_session, const char *uri, const char *config) SESSION_API_CALL(session, verify, config, cfg); - if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) - WT_ERR_MSG(session, ENOTSUP, - "WT_SESSION.verify not supported for in-memory " - "configurations"); + WT_ERR(__wt_inmem_unsupported_op(session, NULL)); /* Block out checkpoints to avoid spurious EBUSY errors. */ WT_WITH_CHECKPOINT_LOCK(session, @@ -1632,10 +1637,7 @@ __session_checkpoint(WT_SESSION *wt_session, const char *config) WT_STAT_CONN_INCR(session, txn_checkpoint); SESSION_API_CALL(session, checkpoint, config, cfg); - if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) - WT_ERR_MSG(session, ENOTSUP, - "WT_SESSION.checkpoint not supported for in-memory " - "configurations"); + WT_ERR(__wt_inmem_unsupported_op(session, NULL)); /* * Checkpoints require a snapshot to write a transactionally consistent @@ -1826,7 +1828,7 @@ __open_session(WT_CONNECTION_IMPL *conn, if (i == conn->session_size) WT_ERR_MSG(session, ENOMEM, "only configured to support %" PRIu32 " sessions" - " (including %" PRIu32 " additional internal sessions)", + " (including %d additional internal sessions)", conn->session_size, WT_EXTRA_INTERNAL_SESSIONS); /* diff --git a/src/third_party/wiredtiger/src/session/session_compact.c b/src/third_party/wiredtiger/src/session/session_compact.c index c4710dbb1a5..10328e0db5b 100644 --- a/src/third_party/wiredtiger/src/session/session_compact.c +++ b/src/third_party/wiredtiger/src/session/session_compact.c @@ -324,7 +324,7 @@ __wt_session_compact( session = (WT_SESSION_IMPL *)wt_session; SESSION_API_CALL(session, compact, config, cfg); - /* In-memory is already as compact as it's going to get. */ + /* In-memory ignores compaction operations. */ if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) goto err; diff --git a/src/third_party/wiredtiger/src/support/err.c b/src/third_party/wiredtiger/src/support/err.c index 5ec995d8f65..94ae27628c2 100644 --- a/src/third_party/wiredtiger/src/support/err.c +++ b/src/third_party/wiredtiger/src/support/err.c @@ -361,6 +361,22 @@ __wt_ext_err_printf( } /* + * __wt_verbose_worker -- + * Verbose message. + */ +void +__wt_verbose_worker(WT_SESSION_IMPL *session, const char *fmt, ...) + WT_GCC_FUNC_ATTRIBUTE((format (printf, 2, 3))) + WT_GCC_FUNC_ATTRIBUTE((cold)) +{ + va_list ap; + + va_start(ap, fmt); + WT_IGNORE_RET(__wt_eventv(session, true, 0, NULL, 0, fmt, ap)); + va_end(ap); +} + +/* * info_msg -- * Informational message. */ @@ -533,6 +549,22 @@ __wt_illegal_value(WT_SESSION_IMPL *session, const char *name) } /* + * __wt_inmem_unsupported_op -- + * Print a standard error message for an operation that's not supported + * for in-memory configurations. + */ +int +__wt_inmem_unsupported_op(WT_SESSION_IMPL *session, const char *tag) + WT_GCC_FUNC_ATTRIBUTE((cold)) +{ + if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) + WT_RET_MSG(session, ENOTSUP, + "%s%snot supported for in-memory configurations", + tag == NULL ? "" : tag, tag == NULL ? "" : ": "); + return (0); +} + +/* * __wt_object_unsupported -- * Print a standard error message for an object that doesn't support a * particular operation. diff --git a/src/third_party/wiredtiger/src/support/generation.c b/src/third_party/wiredtiger/src/support/generation.c index 6e16d7e57fe..182fb2de095 100644 --- a/src/third_party/wiredtiger/src/support/generation.c +++ b/src/third_party/wiredtiger/src/support/generation.c @@ -116,6 +116,12 @@ __wt_gen_drain(WT_SESSION_IMPL *session, int which, uint64_t generation) if (v == 0 || v >= generation) break; + /* If we're waiting on ourselves, we're deadlocked. */ + if (session == s) { + WT_ASSERT(session, session != s); + WT_IGNORE_RET(__wt_panic(session)); + } + /* * The pause count is cumulative, quit spinning if it's * not doing us any good, that can happen in generations diff --git a/src/third_party/wiredtiger/src/support/rand.c b/src/third_party/wiredtiger/src/support/rand.c index 8083b8801c1..64ed2341eeb 100644 --- a/src/third_party/wiredtiger/src/support/rand.c +++ b/src/third_party/wiredtiger/src/support/rand.c @@ -120,15 +120,3 @@ __wt_random(WT_RAND_STATE volatile * rnd_state) return ((z << 16) + (w & 65535)); } - -/* - * __wt_random64 -- - * Return a 64-bit pseudo-random number. - */ -uint64_t -__wt_random64(WT_RAND_STATE volatile * rnd_state) - WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) -{ - return (((uint64_t)__wt_random(rnd_state) << 32) + - __wt_random(rnd_state)); -} diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index 2dc006da827..14a1d2a3b86 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -48,6 +48,19 @@ static const char * const __stats_dsrc_desc[] = { "cache: bytes written from cache", "cache: checkpoint blocked page eviction", "cache: data source pages selected for eviction unable to be evicted", + "cache: eviction walk passes of a file", + "cache: eviction walk target pages histogram - 0-9", + "cache: eviction walk target pages histogram - 10-31", + "cache: eviction walk target pages histogram - 128 and higher", + "cache: eviction walk target pages histogram - 32-63", + "cache: eviction walk target pages histogram - 64-128", + "cache: eviction walks abandoned", + "cache: eviction walks gave up because they restarted their walk twice", + "cache: eviction walks gave up because they saw too many pages and found no candidates", + "cache: eviction walks gave up because they saw too many pages and found too few candidates", + "cache: eviction walks reached end of tree", + "cache: eviction walks started from root of tree", + "cache: eviction walks started from saved location in tree", "cache: hazard pointer blocked page eviction", "cache: in-memory page passed criteria to be split", "cache: in-memory page splits", @@ -56,18 +69,20 @@ static const char * const __stats_dsrc_desc[] = { "cache: leaf pages split during eviction", "cache: modified pages evicted", "cache: overflow pages read into cache", - "cache: overflow values cached in memory", "cache: page split during eviction deepened the tree", "cache: page written requiring lookaside records", "cache: pages read into cache", "cache: pages read into cache requiring lookaside entries", "cache: pages requested from the cache", + "cache: pages seen by eviction walk", "cache: pages written from cache", "cache: pages written requiring in-memory restoration", "cache: tracked dirty bytes in the cache", "cache: unmodified pages evicted", "cache_walk: Average difference between current eviction generation when the page was last considered", "cache_walk: Average on-disk page image size seen", + "cache_walk: Average time in cache for pages that have been visited by the eviction server", + "cache_walk: Average time in cache for pages that have not been visited by the eviction server", "cache_walk: Clean pages currently in cache", "cache_walk: Current eviction generation", "cache_walk: Dirty pages currently in cache", @@ -77,6 +92,7 @@ static const char * const __stats_dsrc_desc[] = { "cache_walk: Maximum difference between current eviction generation when the page was last considered", "cache_walk: Maximum page size seen", "cache_walk: Minimum on-disk page image size seen", + "cache_walk: Number of pages never visited by eviction server", "cache_walk: On-disk page image sizes smaller than a single allocation unit", "cache_walk: Pages created in memory and never written", "cache_walk: Pages currently queued for eviction", @@ -212,6 +228,19 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats) stats->cache_bytes_write = 0; stats->cache_eviction_checkpoint = 0; stats->cache_eviction_fail = 0; + stats->cache_eviction_walk_passes = 0; + stats->cache_eviction_target_page_lt10 = 0; + stats->cache_eviction_target_page_lt32 = 0; + stats->cache_eviction_target_page_ge128 = 0; + stats->cache_eviction_target_page_lt64 = 0; + stats->cache_eviction_target_page_lt128 = 0; + stats->cache_eviction_walks_abandoned = 0; + stats->cache_eviction_walks_stopped = 0; + stats->cache_eviction_walks_gave_up_no_targets = 0; + stats->cache_eviction_walks_gave_up_ratio = 0; + stats->cache_eviction_walks_ended = 0; + stats->cache_eviction_walk_from_root = 0; + stats->cache_eviction_walk_saved_pos = 0; stats->cache_eviction_hazard = 0; stats->cache_inmem_splittable = 0; stats->cache_inmem_split = 0; @@ -220,18 +249,20 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats) stats->cache_eviction_split_leaf = 0; stats->cache_eviction_dirty = 0; stats->cache_read_overflow = 0; - stats->cache_overflow_value = 0; stats->cache_eviction_deepen = 0; stats->cache_write_lookaside = 0; stats->cache_read = 0; stats->cache_read_lookaside = 0; stats->cache_pages_requested = 0; + stats->cache_eviction_pages_seen = 0; stats->cache_write = 0; stats->cache_write_restore = 0; /* not clearing cache_bytes_dirty */ stats->cache_eviction_clean = 0; /* not clearing cache_state_gen_avg_gap */ /* not clearing cache_state_avg_written_size */ + /* not clearing cache_state_avg_visited_age */ + /* not clearing cache_state_avg_unvisited_age */ /* not clearing cache_state_pages_clean */ /* not clearing cache_state_gen_current */ /* not clearing cache_state_pages_dirty */ @@ -241,6 +272,7 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats) /* not clearing cache_state_gen_max_gap */ /* not clearing cache_state_max_pagesize */ /* not clearing cache_state_min_written_size */ + /* not clearing cache_state_unvisited_count */ /* not clearing cache_state_smaller_alloc_size */ /* not clearing cache_state_memory */ /* not clearing cache_state_queued */ @@ -361,6 +393,30 @@ __wt_stat_dsrc_aggregate_single( to->cache_bytes_write += from->cache_bytes_write; to->cache_eviction_checkpoint += from->cache_eviction_checkpoint; to->cache_eviction_fail += from->cache_eviction_fail; + to->cache_eviction_walk_passes += from->cache_eviction_walk_passes; + to->cache_eviction_target_page_lt10 += + from->cache_eviction_target_page_lt10; + to->cache_eviction_target_page_lt32 += + from->cache_eviction_target_page_lt32; + to->cache_eviction_target_page_ge128 += + from->cache_eviction_target_page_ge128; + to->cache_eviction_target_page_lt64 += + from->cache_eviction_target_page_lt64; + to->cache_eviction_target_page_lt128 += + from->cache_eviction_target_page_lt128; + to->cache_eviction_walks_abandoned += + from->cache_eviction_walks_abandoned; + to->cache_eviction_walks_stopped += + from->cache_eviction_walks_stopped; + to->cache_eviction_walks_gave_up_no_targets += + from->cache_eviction_walks_gave_up_no_targets; + to->cache_eviction_walks_gave_up_ratio += + from->cache_eviction_walks_gave_up_ratio; + to->cache_eviction_walks_ended += from->cache_eviction_walks_ended; + to->cache_eviction_walk_from_root += + from->cache_eviction_walk_from_root; + to->cache_eviction_walk_saved_pos += + from->cache_eviction_walk_saved_pos; to->cache_eviction_hazard += from->cache_eviction_hazard; to->cache_inmem_splittable += from->cache_inmem_splittable; to->cache_inmem_split += from->cache_inmem_split; @@ -370,12 +426,12 @@ __wt_stat_dsrc_aggregate_single( to->cache_eviction_split_leaf += from->cache_eviction_split_leaf; to->cache_eviction_dirty += from->cache_eviction_dirty; to->cache_read_overflow += from->cache_read_overflow; - to->cache_overflow_value += from->cache_overflow_value; to->cache_eviction_deepen += from->cache_eviction_deepen; to->cache_write_lookaside += from->cache_write_lookaside; to->cache_read += from->cache_read; to->cache_read_lookaside += from->cache_read_lookaside; to->cache_pages_requested += from->cache_pages_requested; + to->cache_eviction_pages_seen += from->cache_eviction_pages_seen; to->cache_write += from->cache_write; to->cache_write_restore += from->cache_write_restore; to->cache_bytes_dirty += from->cache_bytes_dirty; @@ -383,6 +439,9 @@ __wt_stat_dsrc_aggregate_single( to->cache_state_gen_avg_gap += from->cache_state_gen_avg_gap; to->cache_state_avg_written_size += from->cache_state_avg_written_size; + to->cache_state_avg_visited_age += from->cache_state_avg_visited_age; + to->cache_state_avg_unvisited_age += + from->cache_state_avg_unvisited_age; to->cache_state_pages_clean += from->cache_state_pages_clean; to->cache_state_gen_current += from->cache_state_gen_current; to->cache_state_pages_dirty += from->cache_state_pages_dirty; @@ -393,6 +452,7 @@ __wt_stat_dsrc_aggregate_single( to->cache_state_max_pagesize += from->cache_state_max_pagesize; to->cache_state_min_written_size += from->cache_state_min_written_size; + to->cache_state_unvisited_count += from->cache_state_unvisited_count; to->cache_state_smaller_alloc_size += from->cache_state_smaller_alloc_size; to->cache_state_memory += from->cache_state_memory; @@ -520,6 +580,32 @@ __wt_stat_dsrc_aggregate( to->cache_eviction_checkpoint += WT_STAT_READ(from, cache_eviction_checkpoint); to->cache_eviction_fail += WT_STAT_READ(from, cache_eviction_fail); + to->cache_eviction_walk_passes += + WT_STAT_READ(from, cache_eviction_walk_passes); + to->cache_eviction_target_page_lt10 += + WT_STAT_READ(from, cache_eviction_target_page_lt10); + to->cache_eviction_target_page_lt32 += + WT_STAT_READ(from, cache_eviction_target_page_lt32); + to->cache_eviction_target_page_ge128 += + WT_STAT_READ(from, cache_eviction_target_page_ge128); + to->cache_eviction_target_page_lt64 += + WT_STAT_READ(from, cache_eviction_target_page_lt64); + to->cache_eviction_target_page_lt128 += + WT_STAT_READ(from, cache_eviction_target_page_lt128); + to->cache_eviction_walks_abandoned += + WT_STAT_READ(from, cache_eviction_walks_abandoned); + to->cache_eviction_walks_stopped += + WT_STAT_READ(from, cache_eviction_walks_stopped); + to->cache_eviction_walks_gave_up_no_targets += + WT_STAT_READ(from, cache_eviction_walks_gave_up_no_targets); + to->cache_eviction_walks_gave_up_ratio += + WT_STAT_READ(from, cache_eviction_walks_gave_up_ratio); + to->cache_eviction_walks_ended += + WT_STAT_READ(from, cache_eviction_walks_ended); + to->cache_eviction_walk_from_root += + WT_STAT_READ(from, cache_eviction_walk_from_root); + to->cache_eviction_walk_saved_pos += + WT_STAT_READ(from, cache_eviction_walk_saved_pos); to->cache_eviction_hazard += WT_STAT_READ(from, cache_eviction_hazard); to->cache_inmem_splittable += @@ -533,7 +619,6 @@ __wt_stat_dsrc_aggregate( WT_STAT_READ(from, cache_eviction_split_leaf); to->cache_eviction_dirty += WT_STAT_READ(from, cache_eviction_dirty); to->cache_read_overflow += WT_STAT_READ(from, cache_read_overflow); - to->cache_overflow_value += WT_STAT_READ(from, cache_overflow_value); to->cache_eviction_deepen += WT_STAT_READ(from, cache_eviction_deepen); to->cache_write_lookaside += @@ -542,6 +627,8 @@ __wt_stat_dsrc_aggregate( to->cache_read_lookaside += WT_STAT_READ(from, cache_read_lookaside); to->cache_pages_requested += WT_STAT_READ(from, cache_pages_requested); + to->cache_eviction_pages_seen += + WT_STAT_READ(from, cache_eviction_pages_seen); to->cache_write += WT_STAT_READ(from, cache_write); to->cache_write_restore += WT_STAT_READ(from, cache_write_restore); to->cache_bytes_dirty += WT_STAT_READ(from, cache_bytes_dirty); @@ -550,6 +637,10 @@ __wt_stat_dsrc_aggregate( WT_STAT_READ(from, cache_state_gen_avg_gap); to->cache_state_avg_written_size += WT_STAT_READ(from, cache_state_avg_written_size); + to->cache_state_avg_visited_age += + WT_STAT_READ(from, cache_state_avg_visited_age); + to->cache_state_avg_unvisited_age += + WT_STAT_READ(from, cache_state_avg_unvisited_age); to->cache_state_pages_clean += WT_STAT_READ(from, cache_state_pages_clean); to->cache_state_gen_current += @@ -568,6 +659,8 @@ __wt_stat_dsrc_aggregate( WT_STAT_READ(from, cache_state_max_pagesize); to->cache_state_min_written_size += WT_STAT_READ(from, cache_state_min_written_size); + to->cache_state_unvisited_count += + WT_STAT_READ(from, cache_state_unvisited_count); to->cache_state_smaller_alloc_size += WT_STAT_READ(from, cache_state_smaller_alloc_size); to->cache_state_memory += WT_STAT_READ(from, cache_state_memory); @@ -678,13 +771,25 @@ static const char * const __stats_connection_desc[] = { "cache: eviction calls to get a page found queue empty after locking", "cache: eviction currently operating in aggressive mode", "cache: eviction empty score", + "cache: eviction passes of a file", "cache: eviction server candidate queue empty when topping up", "cache: eviction server candidate queue not empty when topping up", "cache: eviction server evicting pages", "cache: eviction server slept, because we did not make progress with eviction", "cache: eviction server unable to reach eviction goal", "cache: eviction state", + "cache: eviction walk target pages histogram - 0-9", + "cache: eviction walk target pages histogram - 10-31", + "cache: eviction walk target pages histogram - 128 and higher", + "cache: eviction walk target pages histogram - 32-63", + "cache: eviction walk target pages histogram - 64-128", "cache: eviction walks abandoned", + "cache: eviction walks gave up because they restarted their walk twice", + "cache: eviction walks gave up because they saw too many pages and found no candidates", + "cache: eviction walks gave up because they saw too many pages and found too few candidates", + "cache: eviction walks reached end of tree", + "cache: eviction walks started from root of tree", + "cache: eviction walks started from saved location in tree", "cache: eviction worker thread active", "cache: eviction worker thread created", "cache: eviction worker thread evicting pages", @@ -711,7 +816,6 @@ static const char * const __stats_connection_desc[] = { "cache: modified pages evicted", "cache: modified pages evicted by application threads", "cache: overflow pages read into cache", - "cache: overflow values cached in memory", "cache: page split during eviction deepened the tree", "cache: page written requiring lookaside records", "cache: pages currently held in the cache", @@ -789,6 +893,7 @@ static const char * const __stats_connection_desc[] = { "lock: table read lock acquisitions", "lock: table write lock acquisitions", "log: busy returns attempting to switch slots", + "log: force checkpoint calls slept", "log: log bytes of payload data", "log: log bytes written", "log: log files manually zero-filled", @@ -981,13 +1086,25 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->cache_eviction_get_ref_empty2 = 0; /* not clearing cache_eviction_aggressive_set */ /* not clearing cache_eviction_empty_score */ + stats->cache_eviction_walk_passes = 0; stats->cache_eviction_queue_empty = 0; stats->cache_eviction_queue_not_empty = 0; stats->cache_eviction_server_evicting = 0; stats->cache_eviction_server_slept = 0; stats->cache_eviction_slow = 0; /* not clearing cache_eviction_state */ + stats->cache_eviction_target_page_lt10 = 0; + stats->cache_eviction_target_page_lt32 = 0; + stats->cache_eviction_target_page_ge128 = 0; + stats->cache_eviction_target_page_lt64 = 0; + stats->cache_eviction_target_page_lt128 = 0; stats->cache_eviction_walks_abandoned = 0; + stats->cache_eviction_walks_stopped = 0; + stats->cache_eviction_walks_gave_up_no_targets = 0; + stats->cache_eviction_walks_gave_up_ratio = 0; + stats->cache_eviction_walks_ended = 0; + stats->cache_eviction_walk_from_root = 0; + stats->cache_eviction_walk_saved_pos = 0; /* not clearing cache_eviction_active_workers */ stats->cache_eviction_worker_created = 0; stats->cache_eviction_worker_evicting = 0; @@ -1014,7 +1131,6 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->cache_eviction_dirty = 0; stats->cache_eviction_app_dirty = 0; stats->cache_read_overflow = 0; - stats->cache_overflow_value = 0; stats->cache_eviction_deepen = 0; stats->cache_write_lookaside = 0; /* not clearing cache_pages_inuse */ @@ -1092,6 +1208,7 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->lock_table_read_count = 0; stats->lock_table_write_count = 0; stats->log_slot_switch_busy = 0; + stats->log_force_ckpt_sleep = 0; stats->log_bytes_payload = 0; stats->log_bytes_written = 0; stats->log_zero_fills = 0; @@ -1272,6 +1389,8 @@ __wt_stat_connection_aggregate( WT_STAT_READ(from, cache_eviction_aggressive_set); to->cache_eviction_empty_score += WT_STAT_READ(from, cache_eviction_empty_score); + to->cache_eviction_walk_passes += + WT_STAT_READ(from, cache_eviction_walk_passes); to->cache_eviction_queue_empty += WT_STAT_READ(from, cache_eviction_queue_empty); to->cache_eviction_queue_not_empty += @@ -1282,8 +1401,30 @@ __wt_stat_connection_aggregate( WT_STAT_READ(from, cache_eviction_server_slept); to->cache_eviction_slow += WT_STAT_READ(from, cache_eviction_slow); to->cache_eviction_state += WT_STAT_READ(from, cache_eviction_state); + to->cache_eviction_target_page_lt10 += + WT_STAT_READ(from, cache_eviction_target_page_lt10); + to->cache_eviction_target_page_lt32 += + WT_STAT_READ(from, cache_eviction_target_page_lt32); + to->cache_eviction_target_page_ge128 += + WT_STAT_READ(from, cache_eviction_target_page_ge128); + to->cache_eviction_target_page_lt64 += + WT_STAT_READ(from, cache_eviction_target_page_lt64); + to->cache_eviction_target_page_lt128 += + WT_STAT_READ(from, cache_eviction_target_page_lt128); to->cache_eviction_walks_abandoned += WT_STAT_READ(from, cache_eviction_walks_abandoned); + to->cache_eviction_walks_stopped += + WT_STAT_READ(from, cache_eviction_walks_stopped); + to->cache_eviction_walks_gave_up_no_targets += + WT_STAT_READ(from, cache_eviction_walks_gave_up_no_targets); + to->cache_eviction_walks_gave_up_ratio += + WT_STAT_READ(from, cache_eviction_walks_gave_up_ratio); + to->cache_eviction_walks_ended += + WT_STAT_READ(from, cache_eviction_walks_ended); + to->cache_eviction_walk_from_root += + WT_STAT_READ(from, cache_eviction_walk_from_root); + to->cache_eviction_walk_saved_pos += + WT_STAT_READ(from, cache_eviction_walk_saved_pos); to->cache_eviction_active_workers += WT_STAT_READ(from, cache_eviction_active_workers); to->cache_eviction_worker_created += @@ -1330,7 +1471,6 @@ __wt_stat_connection_aggregate( to->cache_eviction_app_dirty += WT_STAT_READ(from, cache_eviction_app_dirty); to->cache_read_overflow += WT_STAT_READ(from, cache_read_overflow); - to->cache_overflow_value += WT_STAT_READ(from, cache_overflow_value); to->cache_eviction_deepen += WT_STAT_READ(from, cache_eviction_deepen); to->cache_write_lookaside += @@ -1433,6 +1573,7 @@ __wt_stat_connection_aggregate( to->lock_table_write_count += WT_STAT_READ(from, lock_table_write_count); to->log_slot_switch_busy += WT_STAT_READ(from, log_slot_switch_busy); + to->log_force_ckpt_sleep += WT_STAT_READ(from, log_force_ckpt_sleep); to->log_bytes_payload += WT_STAT_READ(from, log_bytes_payload); to->log_bytes_written += WT_STAT_READ(from, log_bytes_written); to->log_zero_fills += WT_STAT_READ(from, log_zero_fills); diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index 02adad65ba5..18a3c0021f0 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -441,7 +441,6 @@ __wt_txn_config(WT_SESSION_IMPL *session, const char *cfg[]) if (cval.len > 0) { #ifdef HAVE_TIMESTAMPS WT_TXN_GLOBAL *txn_global = &S2C(session)->txn_global; - WT_TXN_STATE *txn_state = WT_SESSION_TXN_STATE(session); wt_timestamp_t oldest_timestamp; WT_RET(__wt_txn_parse_timestamp( @@ -455,9 +454,8 @@ __wt_txn_config(WT_SESSION_IMPL *session, const char *cfg[]) WT_RET_MSG(session, EINVAL, "read timestamp %.*s older than oldest timestamp", (int)cval.len, cval.str); - __wt_timestamp_set( - txn_state->read_timestamp, txn->read_timestamp); - F_SET(txn, WT_TXN_HAS_TS_READ); + + __wt_txn_set_read_timestamp(session); txn->isolation = WT_ISO_SNAPSHOT; #else WT_RET_MSG(session, EINVAL, "read_timestamp requires a " @@ -531,22 +529,15 @@ __wt_txn_release(WT_SESSION_IMPL *session) WT_ASSERT(session, txn_state->id != WT_TXN_NONE && txn->id != WT_TXN_NONE); WT_PUBLISH(txn_state->id, WT_TXN_NONE); -#ifdef HAVE_TIMESTAMPS - if (F_ISSET(txn, WT_TXN_HAS_TS_COMMIT)) { - /* - * We rely on a non-zero ID to protect our published - * commit timestamp. Otherwise we would need a lock - * here. - */ - WT_WRITE_BARRIER(); - __wt_timestamp_set( - txn_state->commit_timestamp, zero_timestamp); - } -#endif txn->id = WT_TXN_NONE; } +#ifdef HAVE_TIMESTAMPS + __wt_txn_clear_commit_timestamp(session); + __wt_txn_clear_read_timestamp(session); +#endif + /* Free the scratch buffer allocated for logging. */ __wt_logrec_free(session, &txn->logrec); @@ -578,7 +569,6 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) WT_TXN_OP *op; #ifdef HAVE_TIMESTAMPS WT_TXN_GLOBAL *txn_global = &S2C(session)->txn_global; - WT_TXN_STATE *txn_state = WT_SESSION_TXN_STATE(session); wt_timestamp_t prev_commit_timestamp; bool update_timestamp; #endif @@ -601,13 +591,7 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) #ifdef HAVE_TIMESTAMPS WT_ERR(__wt_txn_parse_timestamp( session, "commit", txn->commit_timestamp, &cval)); - if (!F_ISSET(txn, WT_TXN_HAS_TS_COMMIT)) { - __wt_writelock(session, &txn_global->rwlock); - __wt_timestamp_set(txn_state->commit_timestamp, - txn->commit_timestamp); - __wt_writeunlock(session, &txn_global->rwlock); - F_SET(txn, WT_TXN_HAS_TS_COMMIT); - } + __wt_txn_set_commit_timestamp(session); #else WT_ERR_MSG(session, EINVAL, "commit_timestamp requires a " "version of WiredTiger built with timestamp support"); @@ -944,8 +928,14 @@ __wt_txn_global_init(WT_SESSION_IMPL *session, const char *cfg[]) WT_RET(__wt_spin_init( session, &txn_global->id_lock, "transaction id lock")); WT_RET(__wt_rwlock_init(session, &txn_global->rwlock)); - WT_RET(__wt_rwlock_init(session, &txn_global->nsnap_rwlock)); + WT_RET(__wt_rwlock_init(session, &txn_global->commit_timestamp_rwlock)); + TAILQ_INIT(&txn_global->commit_timestamph); + + WT_RET(__wt_rwlock_init(session, &txn_global->read_timestamp_rwlock)); + TAILQ_INIT(&txn_global->read_timestamph); + + WT_RET(__wt_rwlock_init(session, &txn_global->nsnap_rwlock)); txn_global->nsnap_oldest_id = WT_TXN_NONE; TAILQ_INIT(&txn_global->nsnaph); @@ -976,6 +966,8 @@ __wt_txn_global_destroy(WT_SESSION_IMPL *session) __wt_spin_destroy(session, &txn_global->id_lock); __wt_rwlock_destroy(session, &txn_global->rwlock); + __wt_rwlock_destroy(session, &txn_global->commit_timestamp_rwlock); + __wt_rwlock_destroy(session, &txn_global->read_timestamp_rwlock); __wt_rwlock_destroy(session, &txn_global->nsnap_rwlock); __wt_free(session, txn_global->states); } diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c index 73ad96b5518..519d3469865 100644 --- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c +++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c @@ -625,6 +625,7 @@ __checkpoint_prepare(WT_SESSION_IMPL *session, const char *cfg[]) */ __wt_writelock(session, &txn_global->rwlock); txn_global->checkpoint_state = *txn_state; + txn_global->checkpoint_txn = txn; txn_global->checkpoint_state.pinned_id = WT_MIN(txn->id, txn->snap_min); /* @@ -646,6 +647,15 @@ __checkpoint_prepare(WT_SESSION_IMPL *session, const char *cfg[]) txn_state->metadata_pinned = WT_TXN_NONE; __wt_writeunlock(session, &txn_global->rwlock); +#ifdef HAVE_TIMESTAMPS + /* + * Now that the checkpoint transaction is published, clear it from the + * regular lists. + */ + __wt_txn_clear_commit_timestamp(session); + __wt_txn_clear_read_timestamp(session); +#endif + /* * Get a list of handles we want to flush; for named checkpoints this * may pull closed objects into the session cache. diff --git a/src/third_party/wiredtiger/src/txn/txn_log.c b/src/third_party/wiredtiger/src/txn/txn_log.c index 3d218aea94f..d291139284a 100644 --- a/src/third_party/wiredtiger/src/txn/txn_log.c +++ b/src/third_party/wiredtiger/src/txn/txn_log.c @@ -425,7 +425,8 @@ __wt_txn_checkpoint_log( * metadata LSN and we do not want to archive in that case. */ if (!conn->hot_backup && - !FLD_ISSET(conn->log_flags, WT_CONN_LOG_RECOVER_DIRTY) && + (!FLD_ISSET(conn->log_flags, WT_CONN_LOG_RECOVER_DIRTY) || + FLD_ISSET(conn->log_flags, WT_CONN_LOG_FORCE_DOWNGRADE)) && txn->full_ckpt) __wt_log_ckpt(session, ckpt_lsn); @@ -589,6 +590,16 @@ __txn_printlog(WT_SESSION_IMPL *session, WT_RET(__wt_fprintf(session, WT_STDOUT(session), " \"message\" : \"%s\"\n", msg)); break; + + case WT_LOGREC_SYSTEM: + WT_RET(__wt_struct_unpack(session, p, WT_PTRDIFF(end, p), + WT_UNCHECKED_STRING(II), &lsnfile, &lsnoffset)); + WT_RET(__wt_fprintf(session, WT_STDOUT(session), + " \"type\" : \"system\",\n")); + WT_RET(__wt_fprintf(session, WT_STDOUT(session), + " \"prev_lsn\" : [%" PRIu32 ",%" PRIu32 "]\n", + lsnfile, lsnoffset)); + break; } WT_RET(__wt_fprintf(session, WT_STDOUT(session), " }")); diff --git a/src/third_party/wiredtiger/src/txn/txn_recover.c b/src/third_party/wiredtiger/src/txn/txn_recover.c index 58f4f0750d7..d08c2717f7a 100644 --- a/src/third_party/wiredtiger/src/txn/txn_recover.c +++ b/src/third_party/wiredtiger/src/txn/txn_recover.c @@ -235,8 +235,8 @@ __txn_op_apply( return (0); err: __wt_err(session, ret, - "operation apply failed during recovery: operation type %d " - "at LSN %" PRIu32 "/%" PRIu32, + "operation apply failed during recovery: operation type %" + PRIu32 " at LSN %" PRIu32 "/%" PRIu32, optype, lsnp->l.file, lsnp->l.offset); return (ret); } @@ -588,7 +588,6 @@ __wt_txn_recover(WT_SESSION_IMPL *session) * LSN and archiving. */ ckpt: WT_ERR(session->iface.checkpoint(&session->iface, "force=1")); - done: FLD_SET(conn->log_flags, WT_CONN_LOG_RECOVER_DONE); err: WT_TRET(__recovery_free(&r)); __wt_free(session, config); diff --git a/src/third_party/wiredtiger/src/txn/txn_timestamp.c b/src/third_party/wiredtiger/src/txn/txn_timestamp.c index ebef0320d4f..a975341c189 100644 --- a/src/third_party/wiredtiger/src/txn/txn_timestamp.c +++ b/src/third_party/wiredtiger/src/txn/txn_timestamp.c @@ -24,7 +24,7 @@ __wt_txn_parse_timestamp(WT_SESSION_IMPL *session, const char *hexts; char padbuf[2 * WT_TIMESTAMP_SIZE + 1]; - __wt_timestamp_set(timestamp, zero_timestamp); + __wt_timestamp_set_zero(timestamp); if (cval->len == 0) return (0); @@ -80,9 +80,8 @@ __txn_global_query_timestamp( { WT_CONNECTION_IMPL *conn; WT_CONFIG_ITEM cval; + WT_TXN *txn; WT_TXN_GLOBAL *txn_global; - WT_TXN_STATE *s; - uint32_t i, session_cnt; conn = S2C(session); txn_global = &conn->txn_global; @@ -93,35 +92,36 @@ __txn_global_query_timestamp( return (WT_NOTFOUND); __wt_readlock(session, &txn_global->rwlock); __wt_timestamp_set(ts, txn_global->commit_timestamp); - WT_ORDERED_READ(session_cnt, conn->session_cnt); - for (i = 0, s = txn_global->states; i < session_cnt; i++, s++) { - if (s->id == WT_TXN_NONE || - __wt_timestamp_iszero(s->commit_timestamp)) - continue; - if (__wt_timestamp_cmp(s->commit_timestamp, ts) < 0) - __wt_timestamp_set(ts, s->commit_timestamp); - } __wt_readunlock(session, &txn_global->rwlock); + + /* Compare with the oldest running transaction. */ + __wt_readlock(session, &txn_global->commit_timestamp_rwlock); + txn = TAILQ_FIRST(&txn_global->commit_timestamph); + if (txn != NULL && + __wt_timestamp_cmp(txn->commit_timestamp, ts) < 0) + __wt_timestamp_set(ts, txn->commit_timestamp); + __wt_readunlock(session, &txn_global->commit_timestamp_rwlock); } else if (WT_STRING_MATCH("oldest_reader", cval.str, cval.len)) { if (!txn_global->has_oldest_timestamp) return (WT_NOTFOUND); __wt_readlock(session, &txn_global->rwlock); __wt_timestamp_set(ts, txn_global->oldest_timestamp); - /* Look at running checkpoints. */ - s = &txn_global->checkpoint_state; - if (s->pinned_id != WT_TXN_NONE && - !__wt_timestamp_iszero(s->read_timestamp) && - __wt_timestamp_cmp(s->read_timestamp, ts) < 0) - __wt_timestamp_set(ts, s->read_timestamp); - WT_ORDERED_READ(session_cnt, conn->session_cnt); - for (i = 0, s = txn_global->states; i < session_cnt; i++, s++) { - if (s->pinned_id == WT_TXN_NONE || - __wt_timestamp_iszero(s->read_timestamp)) - continue; - if (__wt_timestamp_cmp(s->read_timestamp, ts) < 0) - __wt_timestamp_set(ts, s->read_timestamp); - } + + /* Check for a running checkpoint */ + txn = txn_global->checkpoint_txn; + if (txn_global->checkpoint_state.pinned_id != WT_TXN_NONE && + !__wt_timestamp_iszero(txn->read_timestamp) && + __wt_timestamp_cmp(txn->read_timestamp, ts) < 0) + __wt_timestamp_set(ts, txn->read_timestamp); __wt_readunlock(session, &txn_global->rwlock); + + /* Look for the oldest ordinary reader. */ + __wt_readlock(session, &txn_global->read_timestamp_rwlock); + txn = TAILQ_FIRST(&txn_global->read_timestamph); + if (txn != NULL && + __wt_timestamp_cmp(txn->read_timestamp, ts) < 0) + __wt_timestamp_set(ts, txn->read_timestamp); + __wt_readunlock(session, &txn_global->read_timestamp_rwlock); } else return (__wt_illegal_value(session, NULL)); @@ -143,6 +143,11 @@ __wt_txn_global_query_timestamp( size_t len; uint8_t *tsp; + /* + * Keep clang-analyzer happy: it can't tell that ts will be set + * whenever the call below succeeds. + */ + WT_CLEAR(ts); WT_RET(__txn_global_query_timestamp(session, ts, cfg)); /* Avoid memory allocation: set up an item guaranteed large enough. */ @@ -279,18 +284,10 @@ __wt_txn_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) if (ret == 0 && cval.len != 0) { #ifdef HAVE_TIMESTAMPS WT_TXN *txn = &session->txn; - WT_TXN_GLOBAL *txn_global = &S2C(session)->txn_global; - WT_TXN_STATE *txn_state = WT_SESSION_TXN_STATE(session); WT_RET(__wt_txn_parse_timestamp( session, "commit", txn->commit_timestamp, &cval)); - if (!F_ISSET(txn, WT_TXN_HAS_TS_COMMIT)) { - __wt_writelock(session, &txn_global->rwlock); - __wt_timestamp_set(txn_state->commit_timestamp, - txn->commit_timestamp); - __wt_writeunlock(session, &txn_global->rwlock); - F_SET(txn, WT_TXN_HAS_TS_COMMIT); - } + __wt_txn_set_commit_timestamp(session); #else WT_RET_MSG(session, EINVAL, "commit_timestamp requires a " "version of WiredTiger built with timestamp support"); @@ -300,3 +297,110 @@ __wt_txn_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) return (0); } + +/* + * __wt_txn_set_commit_timestamp -- + * Publish a transaction's commit timestamp. + */ +void +__wt_txn_set_commit_timestamp(WT_SESSION_IMPL *session) +{ + WT_TXN *prev, *txn; + WT_TXN_GLOBAL *txn_global; + + txn = &session->txn; + txn_global = &S2C(session)->txn_global; + + if (F_ISSET(txn, WT_TXN_HAS_TS_COMMIT)) + return; + + __wt_writelock(session, &txn_global->commit_timestamp_rwlock); + for (prev = TAILQ_LAST(&txn_global->commit_timestamph, __wt_txn_cts_qh); + prev != NULL && __wt_timestamp_cmp( + prev->commit_timestamp, txn->commit_timestamp) > 0; + prev = TAILQ_PREV(prev, __wt_txn_cts_qh, commit_timestampq)) + ; + if (prev == NULL) + TAILQ_INSERT_HEAD( + &txn_global->commit_timestamph, txn, commit_timestampq); + else + TAILQ_INSERT_AFTER(&txn_global->commit_timestamph, + prev, txn, commit_timestampq); + __wt_writeunlock(session, &txn_global->commit_timestamp_rwlock); + F_SET(txn, WT_TXN_HAS_TS_COMMIT); +} + +/* + * __wt_txn_clear_commit_timestamp -- + * Clear a transaction's published commit timestamp. + */ +void +__wt_txn_clear_commit_timestamp(WT_SESSION_IMPL *session) +{ + WT_TXN *txn; + WT_TXN_GLOBAL *txn_global; + + txn = &session->txn; + txn_global = &S2C(session)->txn_global; + + if (!F_ISSET(txn, WT_TXN_HAS_TS_COMMIT)) + return; + + __wt_writelock(session, &txn_global->commit_timestamp_rwlock); + TAILQ_REMOVE(&txn_global->commit_timestamph, txn, commit_timestampq); + __wt_writeunlock(session, &txn_global->commit_timestamp_rwlock); +} + +/* + * __wt_txn_set_read_timestamp -- + * Publish a transaction's read timestamp. + */ +void +__wt_txn_set_read_timestamp(WT_SESSION_IMPL *session) +{ + WT_TXN *prev, *txn; + WT_TXN_GLOBAL *txn_global; + + txn = &session->txn; + txn_global = &S2C(session)->txn_global; + + if (F_ISSET(txn, WT_TXN_HAS_TS_READ)) + return; + + __wt_writelock(session, &txn_global->read_timestamp_rwlock); + for (prev = TAILQ_LAST(&txn_global->read_timestamph, __wt_txn_rts_qh); + prev != NULL && __wt_timestamp_cmp( + prev->read_timestamp, txn->read_timestamp) > 0; + prev = TAILQ_PREV(prev, __wt_txn_rts_qh, read_timestampq)) + ; + if (prev == NULL) + TAILQ_INSERT_HEAD( + &txn_global->read_timestamph, txn, read_timestampq); + else + TAILQ_INSERT_AFTER( + &txn_global->read_timestamph, prev, txn, read_timestampq); + __wt_writeunlock(session, &txn_global->read_timestamp_rwlock); + F_SET(txn, WT_TXN_HAS_TS_READ); +} + +/* + * __wt_txn_clear_read_timestamp -- + * Clear a transaction's published read timestamp. + */ +void +__wt_txn_clear_read_timestamp(WT_SESSION_IMPL *session) +{ + WT_TXN *txn; + WT_TXN_GLOBAL *txn_global; + + txn = &session->txn; + txn_global = &S2C(session)->txn_global; + + if (!F_ISSET(txn, WT_TXN_HAS_TS_READ)) + return; + + __wt_writelock(session, &txn_global->read_timestamp_rwlock); + TAILQ_REMOVE(&txn_global->read_timestamph, txn, read_timestampq); + __wt_writeunlock(session, &txn_global->read_timestamp_rwlock); + F_CLR(txn, WT_TXN_HAS_TS_READ); +} diff --git a/src/third_party/wiredtiger/src/utilities/util.h b/src/third_party/wiredtiger/src/utilities/util.h index 0238915df07..adf3f844295 100644 --- a/src/third_party/wiredtiger/src/utilities/util.h +++ b/src/third_party/wiredtiger/src/utilities/util.h @@ -32,6 +32,7 @@ int util_cerr(WT_CURSOR *, const char *, int); int util_compact(WT_SESSION *, int, char *[]); void util_copyright(void); int util_create(WT_SESSION *, int, char *[]); +int util_downgrade(WT_SESSION *, WT_CONNECTION *, int, char *[]); int util_drop(WT_SESSION *, int, char *[]); int util_dump(WT_SESSION *, int, char *[]); int util_err(WT_SESSION *, int, const char *, ...) diff --git a/src/third_party/wiredtiger/src/utilities/util_downgrade.c b/src/third_party/wiredtiger/src/utilities/util_downgrade.c new file mode 100644 index 00000000000..4263727242c --- /dev/null +++ b/src/third_party/wiredtiger/src/utilities/util_downgrade.c @@ -0,0 +1,57 @@ +/*- + * Copyright (c) 2014-2017 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "util.h" + +static int usage(void); + +int +util_downgrade(WT_SESSION *session, WT_CONNECTION *conn, int argc, char *argv[]) +{ + WT_DECL_RET; + int ch; + char config_str[128], *release; + + release = NULL; + while ((ch = __wt_getopt(progname, argc, argv, "V:")) != EOF) + switch (ch) { + case 'V': + release = __wt_optarg; + break; + case '?': + default: + return (usage()); + } + argc -= __wt_optind; + argv += __wt_optind; + + /* + * The release argument is required. + * There should not be any more arguments. + */ + if (argc != 0 || release == NULL) + return (usage()); + + if ((ret = __wt_snprintf(config_str, sizeof(config_str), + "compatibility=(release=%s)", release)) != 0) + return (util_err(session, ret, NULL)); + if ((ret = conn->reconfigure(conn, config_str)) != 0) + return (util_err(session, ret, "conn.downgrade")); + + return (0); +} + +static int +usage(void) +{ + (void)fprintf(stderr, + "usage: %s %s " + "downgrade -V release\n", + progname, usage_prefix); + return (1); +} diff --git a/src/third_party/wiredtiger/src/utilities/util_list.c b/src/third_party/wiredtiger/src/utilities/util_list.c index 72888e03183..dcb1c97b529 100644 --- a/src/third_party/wiredtiger/src/utilities/util_list.c +++ b/src/third_party/wiredtiger/src/utilities/util_list.c @@ -231,7 +231,8 @@ list_print_checkpoint(WT_SESSION *session, const char *key) if (ci.root_size != 0) { printf("\t\t" "root offset: %" PRIuMAX " (0x%" PRIxMAX ")\n", - (intmax_t)ci.root_offset, (intmax_t)ci.root_offset); + (uintmax_t)ci.root_offset, + (uintmax_t)ci.root_offset); printf("\t\t" "root size: %" PRIu32 " (0x%" PRIx32 ")\n", ci.root_size, ci.root_size); diff --git a/src/third_party/wiredtiger/src/utilities/util_main.c b/src/third_party/wiredtiger/src/utilities/util_main.c index 010af63ea30..a068bfb320d 100644 --- a/src/third_party/wiredtiger/src/utilities/util_main.c +++ b/src/third_party/wiredtiger/src/utilities/util_main.c @@ -41,6 +41,7 @@ usage(void) "\t" "compact\t compact an object\n" "\t" "copyright copyright information\n" "\t" "create\t create an object\n" + "\t" "downgrade\t downgrade an database\n" "\t" "drop\t drop an object\n" "\t" "dump\t dump an object\n" "\t" "list\t list database objects\n" @@ -66,7 +67,8 @@ main(int argc, char *argv[]) WT_SESSION *session; size_t len; int ch, major_v, minor_v, tret, (*func)(WT_SESSION *, int, char *[]); - bool logoff, recover; + int (*cfunc)(WT_SESSION *, WT_CONNECTION *, int, char *[]); + bool logoff, needconn, recover; char *p, *secretkey; const char *cmd_config, *config, *p1, *p2, *p3, *rec_config; @@ -80,6 +82,8 @@ main(int argc, char *argv[]) ++progname; command = ""; + needconn = false; + /* Check the version against the library build. */ (void)wiredtiger_version(&major_v, & minor_v, NULL); if (major_v != WIREDTIGER_VERSION_MAJOR || @@ -156,6 +160,7 @@ main(int argc, char *argv[]) __wt_optreset = __wt_optind = 1; func = NULL; + cfunc = NULL; switch (command[0]) { case 'a': if (strcmp(command, "alter") == 0) @@ -177,7 +182,10 @@ main(int argc, char *argv[]) } break; case 'd': - if (strcmp(command, "drop") == 0) + if (strcmp(command, "downgrade") == 0) { + cfunc = util_downgrade; + needconn = true; + } else if (strcmp(command, "drop") == 0) func = util_drop; else if (strcmp(command, "dump") == 0) func = util_dump; @@ -234,7 +242,7 @@ main(int argc, char *argv[]) default: break; } - if (func == NULL) { + if (func == NULL && cfunc == NULL) { usage(); goto err; } @@ -278,7 +286,10 @@ main(int argc, char *argv[]) } /* Call the function. */ - ret = func(session, argc, argv); + if (needconn) + ret = cfunc(session, conn, argc, argv); + else + ret = func(session, argc, argv); if (0) { err: ret = 1; |