diff options
| author | Luke Chen <luke.chen@mongodb.com> | 2021-01-27 13:28:13 +1100 |
|---|---|---|
| committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-01-27 03:07:52 +0000 |
| commit | 77d134c7a2bab846aaaa056b0883888a8219da2a (patch) | |
| tree | 8caac27a3573139d221fdf93ba9c99b3e042bc78 | |
| parent | 5fad1f69662696c5b789392622aa34d370fb4825 (diff) | |
| download | mongo-77d134c7a2bab846aaaa056b0883888a8219da2a.tar.gz | |
Import wiredtiger: a52cd5a47a7e9af9e2c341e66f0ffdd9bc977930 from branch mongodb-4.4
ref: ef1f2937c3..a52cd5a47a
for: 4.4.4
WT-6309 Add support for start/stop arguments to wt printlog command
WT-6866 Refactor python backup tests initial base class
WT-6924 Queue history store pages for urgent eviction when cache pressure is high
WT-6946 Adding test tags to an initial set of test programs
WT-7068 Add column store support to test_hs03
WT-7084 Fix assert in test code and a comment error
WT-7109 Retain no longer supported configuration options for backward compatibility
WT-7113 Integrate prototype tiered storage code into WT
WT-7114 Revert Makefile code to always run the prototype script
77 files changed, 3103 insertions, 1183 deletions
diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 247a4777d29..738d53d32a1 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -48,6 +48,14 @@ common_runtime_config = [ Config('assert', '', r''' enable enhanced checking. ''', type='category', subconfig= [ + Config('commit_timestamp', 'none', r''' + This option is no longer supported. Retained for backward + compatibility. Use \c write_timestamp option instead.''', + choices=['always', 'key_consistent', 'never', 'none']), + Config('durable_timestamp', 'none', r''' + This option is no longer supported. Retained for backward + compatibility. Use \c write_timestamp option instead.''', + choices=['always', 'key_consistent', 'never', 'none']), Config('write_timestamp', 'off', r''' verify that commit timestamps are used per the configured \c write_timestamp_usage option for this table''', @@ -197,6 +205,20 @@ lsm_config = [ ]), ] +tiered_config = common_runtime_config + [ + Config('tiered', '', r''' + options only relevant for tiered data sources''', + type='category', subconfig=[ + Config('chunk_size', '1GB', r''' + the maximum size of the hot chunk of tiered tree. This + limit is soft - it is possible for chunks to be temporarily + larger than this value''', + min='1M'), + Config('tiers', '', r''' + list of data sources to combine into a tiered storage structure''', type='list') + ]), +] + file_runtime_config = common_runtime_config + [ Config('access_pattern_hint', 'none', r''' It is recommended that workloads that consist primarily of @@ -229,16 +251,20 @@ file_runtime_config = common_runtime_config + [ system buffer cache after that many bytes from this object are written into the buffer cache''', min=0), + Config('readonly', 'false', r''' + the file is read-only. All methods that may modify a file are + disabled. See @ref readonly for more information''', + type='boolean'), ] # Per-file configuration file_config = format_meta + file_runtime_config + [ Config('block_allocation', 'best', r''' - configure block allocation. Permitted values are \c "first" or - \c "best"; the \c "first" configuration uses a first-available - algorithm during block allocation, the \c "best" configuration - uses a best-fit algorithm''', - choices=['first', 'best',]), + configure block allocation. Permitted values are \c "best" or \c "first"; + the \c "best" configuration uses a best-fit algorithm, + the \c "first" configuration uses a first-available algorithm during block allocation, + the \c "log-structure" configuration allocates a new file for each checkpoint''', + choices=['best', 'first', 'log-structured',]), Config('allocation_size', '4KB', r''' the file unit allocation size, in bytes, must a power-of-two; smaller values decrease the file space required by overflow @@ -284,6 +310,9 @@ file_config = format_meta + file_runtime_config + [ Config('format', 'btree', r''' the file format''', choices=['btree']), + Config('huffman_key', 'none', r''' + This option is no longer supported. Retained for backward + compatibility. See @ref huffman for more information'''), Config('huffman_value', 'none', r''' configure Huffman encoding for values. Permitted values are \c "none", \c "english", \c "utf8<file>" or \c "utf16<file>". @@ -409,6 +438,8 @@ lsm_meta = file_config + lsm_config + [ obsolete chunks in the LSM tree'''), ] +tiered_meta = tiered_config + table_only_config = [ Config('colgroups', '', r''' comma-separated list of names of column groups. Each column @@ -1141,11 +1172,15 @@ methods = { 'table.meta' : Method(table_meta), +'tiered.meta' : Method(tiered_meta), + 'WT_CURSOR.close' : Method([]), 'WT_CURSOR.reconfigure' : Method(cursor_runtime_config), 'WT_SESSION.alter' : Method(file_runtime_config + [ + Config('checkpoint', '', r''' + the file checkpoint entries''', undoc=True), Config('exclusive_refreshed', 'true', r''' refresh the in memory state and flush the metadata change to disk, disabling this flag is dangerous - it will only re-write the @@ -1165,8 +1200,8 @@ methods = { type='int'), ]), -'WT_SESSION.create' : Method(file_config + lsm_config + source_meta + - index_only_config + table_only_config + [ +'WT_SESSION.create' : Method(file_config + lsm_config + tiered_config + + source_meta + index_only_config + table_only_config + [ Config('exclusive', 'false', r''' fail if the object exists. When false (the default), if the object exists, check that its settings match the specified diff --git a/src/third_party/wiredtiger/dist/filelist b/src/third_party/wiredtiger/dist/filelist index 00ce38d68ef..7bbc5e5596a 100644 --- a/src/third_party/wiredtiger/dist/filelist +++ b/src/third_party/wiredtiger/dist/filelist @@ -206,6 +206,8 @@ src/support/scratch.c src/support/stat.c src/support/thread_group.c src/support/timestamp.c +src/tiered/tiered_cursor.c +src/tiered/tiered_schema.c src/txn/txn.c src/txn/txn_ckpt.c src/txn/txn_ext.c diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index 9fe47b19315..a60c3165259 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -645,6 +645,7 @@ cursoring cursorp curstat curtable +curtiered cust customp cv @@ -954,6 +955,7 @@ libs libtool libwiredtiger linkers +liveness llll llu llvm diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index d2c1ee31e96..997337c7896 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -231,6 +231,7 @@ connection_stats = [ CacheStat('cache_eviction_pages_queued_oldest', 'pages queued for urgent eviction during walk'), CacheStat('cache_eviction_pages_queued_post_lru', 'pages queued for eviction post lru sorting'), CacheStat('cache_eviction_pages_queued_urgent', 'pages queued for urgent eviction'), + CacheStat('cache_eviction_pages_queued_urgent_hs_dirty', 'pages queued for urgent eviction from history store due to high dirty content'), CacheStat('cache_eviction_pages_already_queued', 'pages seen by eviction walk that are already queued'), CacheStat('cache_eviction_pages_in_parallel_with_checkpoint', 'pages evicted in parallel with checkpoint'), CacheStat('cache_eviction_queue_empty', 'eviction server candidate queue empty when topping up'), @@ -764,6 +765,7 @@ conn_dsrc_stats = [ CacheStat('cache_eviction_target_page_lt128', 'eviction walk target pages histogram - 64-128'), CacheStat('cache_eviction_target_page_lt32', 'eviction walk target pages histogram - 10-31'), CacheStat('cache_eviction_target_page_lt64', 'eviction walk target pages histogram - 32-63'), + CacheStat('cache_eviction_target_page_reduced', 'eviction walk target pages reduced due to history store cache pressure'), CacheStat('cache_eviction_walk_from_root', 'eviction walks started from root of tree'), CacheStat('cache_eviction_walk_restart', 'eviction walks restarted'), CacheStat('cache_eviction_walk_saved_pos', 'eviction walks started from saved location in tree'), diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 67302efc1b0..eb0dde936e8 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-4.4", - "commit": "ef1f2937c3939dea0e251b8c7c499eb8c1506483" + "commit": "a52cd5a47a7e9af9e2c341e66f0ffdd9bc977930" } diff --git a/src/third_party/wiredtiger/src/block/block_addr.c b/src/third_party/wiredtiger/src/block/block_addr.c index f9bd4248642..312db74a24c 100644 --- a/src/third_party/wiredtiger/src/block/block_addr.c +++ b/src/third_party/wiredtiger/src/block/block_addr.c @@ -14,11 +14,15 @@ * reference so it can be called repeatedly to load a buffer. */ static int -__block_buffer_to_addr( - uint32_t allocsize, const uint8_t **pp, wt_off_t *offsetp, uint32_t *sizep, uint32_t *checksump) +__block_buffer_to_addr(WT_BLOCK *block, const uint8_t **pp, uint32_t *logidp, wt_off_t *offsetp, + uint32_t *sizep, uint32_t *checksump) { - uint64_t o, s, c; + uint64_t l, o, s, c; + if (block->log_structured) + WT_RET(__wt_vunpack_uint(pp, 0, &l)); + else + l = 0; WT_RET(__wt_vunpack_uint(pp, 0, &o)); WT_RET(__wt_vunpack_uint(pp, 0, &s)); WT_RET(__wt_vunpack_uint(pp, 0, &c)); @@ -37,10 +41,11 @@ __block_buffer_to_addr( */ if (s == 0) { *offsetp = 0; - *sizep = *checksump = 0; + *logidp = *sizep = *checksump = 0; } else { - *offsetp = (wt_off_t)(o + 1) * allocsize; - *sizep = (uint32_t)s * allocsize; + *logidp = (uint32_t)l; + *offsetp = (wt_off_t)(o + 1) * block->allocsize; + *sizep = (uint32_t)s * block->allocsize; *checksump = (uint32_t)c; } return (0); @@ -52,19 +57,22 @@ __block_buffer_to_addr( */ int __wt_block_addr_to_buffer( - WT_BLOCK *block, uint8_t **pp, wt_off_t offset, uint32_t size, uint32_t checksum) + WT_BLOCK *block, uint8_t **pp, uint32_t logid, wt_off_t offset, uint32_t size, uint32_t checksum) { - uint64_t o, s, c; + uint64_t l, o, s, c; /* See the comment above: this is the reverse operation. */ if (size == 0) { o = WT_BLOCK_INVALID_OFFSET; - s = c = 0; + l = s = c = 0; } else { + l = logid; o = (uint64_t)offset / block->allocsize - 1; s = size / block->allocsize; c = checksum; } + if (block->log_structured) + WT_RET(__wt_vpack_uint(pp, 0, l)); WT_RET(__wt_vpack_uint(pp, 0, o)); WT_RET(__wt_vpack_uint(pp, 0, s)); WT_RET(__wt_vpack_uint(pp, 0, c)); @@ -77,10 +85,10 @@ __wt_block_addr_to_buffer( * reference. */ int -__wt_block_buffer_to_addr( - WT_BLOCK *block, const uint8_t *p, wt_off_t *offsetp, uint32_t *sizep, uint32_t *checksump) +__wt_block_buffer_to_addr(WT_BLOCK *block, const uint8_t *p, uint32_t *logidp, wt_off_t *offsetp, + uint32_t *sizep, uint32_t *checksump) { - return (__block_buffer_to_addr(block->allocsize, &p, offsetp, sizep, checksump)); + return (__block_buffer_to_addr(block, &p, logidp, offsetp, sizep, checksump)); } /* @@ -92,14 +100,14 @@ __wt_block_addr_invalid( WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size, bool live) { wt_off_t offset; - uint32_t checksum, size; + uint32_t checksum, logid, size; WT_UNUSED(session); WT_UNUSED(addr_size); WT_UNUSED(live); /* Crack the cookie. */ - WT_RET(__wt_block_buffer_to_addr(block, addr, &offset, &size, &checksum)); + WT_RET(__wt_block_buffer_to_addr(block, addr, &logid, &offset, &size, &checksum)); #ifdef HAVE_DIAGNOSTIC /* @@ -111,7 +119,7 @@ __wt_block_addr_invalid( #endif /* Check if the address is past the end of the file. */ - return (offset + size > block->size ? EINVAL : 0); + return (logid == block->logid && offset + size > block->size ? EINVAL : 0); } /* @@ -123,15 +131,16 @@ __wt_block_addr_string( WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, const uint8_t *addr, size_t addr_size) { wt_off_t offset; - uint32_t checksum, size; + uint32_t checksum, logid, size; WT_UNUSED(addr_size); /* Crack the cookie. */ - WT_RET(__wt_block_buffer_to_addr(block, addr, &offset, &size, &checksum)); + WT_RET(__wt_block_buffer_to_addr(block, addr, &logid, &offset, &size, &checksum)); /* Printable representation. */ - WT_RET(__wt_buf_fmt(session, buf, "[%" PRIuMAX "-%" PRIuMAX ", %" PRIu32 ", %" PRIu32 "]", + WT_RET(__wt_buf_fmt(session, buf, + "[%" PRIu32 ": %" PRIuMAX "-%" PRIuMAX ", %" PRIu32 ", %" PRIu32 "]", logid, (uintmax_t)offset, (uintmax_t)offset + size, size, checksum)); return (0); @@ -143,7 +152,7 @@ __wt_block_addr_string( */ static int __block_buffer_to_ckpt( - WT_SESSION_IMPL *session, uint32_t allocsize, const uint8_t *p, WT_BLOCK_CKPT *ci) + WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *p, WT_BLOCK_CKPT *ci) { uint64_t a; const uint8_t **pp; @@ -153,14 +162,14 @@ __block_buffer_to_ckpt( WT_RET_MSG(session, WT_ERROR, "unsupported checkpoint version"); pp = &p; - WT_RET( - __block_buffer_to_addr(allocsize, pp, &ci->root_offset, &ci->root_size, &ci->root_checksum)); WT_RET(__block_buffer_to_addr( - allocsize, pp, &ci->alloc.offset, &ci->alloc.size, &ci->alloc.checksum)); + block, pp, &ci->root_logid, &ci->root_offset, &ci->root_size, &ci->root_checksum)); WT_RET(__block_buffer_to_addr( - allocsize, pp, &ci->avail.offset, &ci->avail.size, &ci->avail.checksum)); + block, pp, &ci->alloc.logid, &ci->alloc.offset, &ci->alloc.size, &ci->alloc.checksum)); WT_RET(__block_buffer_to_addr( - allocsize, pp, &ci->discard.offset, &ci->discard.size, &ci->discard.checksum)); + block, pp, &ci->avail.logid, &ci->avail.offset, &ci->avail.size, &ci->avail.checksum)); + WT_RET(__block_buffer_to_addr(block, pp, &ci->discard.logid, &ci->discard.offset, + &ci->discard.size, &ci->discard.checksum)); WT_RET(__wt_vunpack_uint(pp, 0, &a)); ci->file_size = (wt_off_t)a; WT_RET(__wt_vunpack_uint(pp, 0, &a)); @@ -177,7 +186,7 @@ int __wt_block_buffer_to_ckpt( WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *p, WT_BLOCK_CKPT *ci) { - return (__block_buffer_to_ckpt(session, block->allocsize, p, ci)); + return (__block_buffer_to_ckpt(session, block, p, ci)); } /* @@ -185,13 +194,13 @@ __wt_block_buffer_to_ckpt( * Convert a checkpoint cookie into its components, external utility version. */ int -__wt_block_ckpt_decode(WT_SESSION *wt_session, size_t allocsize, const uint8_t *p, - WT_BLOCK_CKPT *ci) WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) +__wt_block_ckpt_decode(WT_SESSION *wt_session, WT_BLOCK *block, const uint8_t *p, WT_BLOCK_CKPT *ci) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_SESSION_IMPL *session; session = (WT_SESSION_IMPL *)wt_session; - return (__block_buffer_to_ckpt(session, (uint32_t)allocsize, p, ci)); + return (__block_buffer_to_ckpt(session, block, p, ci)); } /* @@ -203,6 +212,9 @@ __wt_block_ckpt_to_buffer( WT_SESSION_IMPL *session, WT_BLOCK *block, uint8_t **pp, WT_BLOCK_CKPT *ci, bool skip_avail) { uint64_t a; + uint32_t logid; + + logid = block->logid; if (ci->version != WT_BM_CHECKPOINT_VERSION) WT_RET_MSG(session, WT_ERROR, "unsupported checkpoint version"); @@ -210,16 +222,17 @@ __wt_block_ckpt_to_buffer( (*pp)[0] = ci->version; (*pp)++; - WT_RET(__wt_block_addr_to_buffer(block, pp, ci->root_offset, ci->root_size, ci->root_checksum)); - WT_RET( - __wt_block_addr_to_buffer(block, pp, ci->alloc.offset, ci->alloc.size, ci->alloc.checksum)); + WT_RET(__wt_block_addr_to_buffer( + block, pp, logid, ci->root_offset, ci->root_size, ci->root_checksum)); + WT_RET(__wt_block_addr_to_buffer( + block, pp, logid, ci->alloc.offset, ci->alloc.size, ci->alloc.checksum)); if (skip_avail) - WT_RET(__wt_block_addr_to_buffer(block, pp, 0, 0, 0)); + WT_RET(__wt_block_addr_to_buffer(block, pp, 0, 0, 0, 0)); else WT_RET(__wt_block_addr_to_buffer( - block, pp, ci->avail.offset, ci->avail.size, ci->avail.checksum)); + block, pp, logid, ci->avail.offset, ci->avail.size, ci->avail.checksum)); WT_RET(__wt_block_addr_to_buffer( - block, pp, ci->discard.offset, ci->discard.size, ci->discard.checksum)); + block, pp, logid, ci->discard.offset, ci->discard.size, ci->discard.checksum)); a = (uint64_t)ci->file_size; WT_RET(__wt_vpack_uint(pp, 0, a)); a = ci->ckpt_size; diff --git a/src/third_party/wiredtiger/src/block/block_ckpt.c b/src/third_party/wiredtiger/src/block/block_ckpt.c index cdabd131e40..ceb9cf39262 100644 --- a/src/third_party/wiredtiger/src/block/block_ckpt.c +++ b/src/third_party/wiredtiger/src/block/block_ckpt.c @@ -95,8 +95,13 @@ __wt_block_checkpoint_load(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint if (ci->root_offset != WT_BLOCK_INVALID_OFFSET) { endp = root_addr; WT_ERR(__wt_block_addr_to_buffer( - block, &endp, ci->root_offset, ci->root_size, ci->root_checksum)); + block, &endp, ci->root_logid, ci->root_offset, ci->root_size, ci->root_checksum)); *root_addr_sizep = WT_PTRDIFF(endp, root_addr); + + if (block->log_structured) { + block->logid = ci->root_logid; + WT_ERR(__wt_block_newfile(session, block)); + } } /* @@ -113,7 +118,7 @@ __wt_block_checkpoint_load(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint * the end of the file, that was done when the checkpoint was first written (re-writing the * checkpoint might possibly make it relevant here, but it's unlikely enough I don't bother). */ - if (!checkpoint) + if (!checkpoint && !block->log_structured) WT_ERR(__wt_block_truncate(session, block, ci->file_size)); if (0) { @@ -237,10 +242,10 @@ __wt_block_checkpoint( */ if (buf == NULL) { ci->root_offset = WT_BLOCK_INVALID_OFFSET; - ci->root_size = ci->root_checksum = 0; + ci->root_logid = ci->root_size = ci->root_checksum = 0; } else - WT_ERR(__wt_block_write_off(session, block, buf, &ci->root_offset, &ci->root_size, - &ci->root_checksum, data_checksum, true, false)); + WT_ERR(__wt_block_write_off(session, block, buf, &ci->root_logid, &ci->root_offset, + &ci->root_size, &ci->root_checksum, data_checksum, true, false)); /* * Checkpoints are potentially reading/writing/merging lots of blocks, pre-allocate structures @@ -463,6 +468,37 @@ __ckpt_add_blk_mods_ext(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, WT_BLOCK_CK } /* + * __wt_block_newfile -- + * Switch a log-structured block object to a new file. + */ +int +__wt_block_newfile(WT_SESSION_IMPL *session, WT_BLOCK *block) +{ + WT_DECL_ITEM(tmp); + WT_DECL_RET; + const char *filename; + + /* Bump to a new file ID. */ + ++block->logid; + + WT_ERR(__wt_scr_alloc(session, 0, &tmp)); + WT_ERR(__wt_buf_fmt(session, tmp, "%s.%08" PRIu32, block->name, block->logid)); + filename = tmp->data; + WT_ERR(__wt_close(session, &block->fh)); + WT_ERR(__wt_open(session, filename, WT_FS_OPEN_FILE_TYPE_DATA, + WT_FS_OPEN_CREATE | block->file_flags, &block->fh)); + WT_ERR(__wt_desc_write(session, block->fh, block->allocsize)); + + block->size = block->allocsize; + __wt_block_ckpt_destroy(session, &block->live); + WT_ERR(__wt_block_ckpt_init(session, &block->live, "live")); + +err: + __wt_scr_free(session, &tmp); + return (ret); +} + +/* * __ckpt_process -- * Process the list of checkpoints. */ @@ -610,7 +646,7 @@ __ckpt_process(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckptbase) * lists, and the freed blocks will then be included when writing the live extent lists. */ WT_CKPT_FOREACH (ckptbase, ckpt) { - if (F_ISSET(ckpt, WT_CKPT_FAKE) || !F_ISSET(ckpt, WT_CKPT_DELETE)) + if (F_ISSET(ckpt, WT_CKPT_FAKE) || !F_ISSET(ckpt, WT_CKPT_DELETE) || block->log_structured) continue; if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT)) @@ -744,6 +780,9 @@ live_update: ci->ckpt_discard = ci->discard; WT_ERR(__wt_block_extlist_init(session, &ci->discard, "live", "discard", false)); + if (block->log_structured) + WT_ERR(__wt_block_newfile(session, block)); + #ifdef HAVE_DIAGNOSTIC /* * The first checkpoint in the system should always have an empty discard list. If we've read diff --git a/src/third_party/wiredtiger/src/block/block_ckpt_scan.c b/src/third_party/wiredtiger/src/block/block_ckpt_scan.c index 4d47c4301b2..2320076752a 100644 --- a/src/third_party/wiredtiger/src/block/block_ckpt_scan.c +++ b/src/third_party/wiredtiger/src/block/block_ckpt_scan.c @@ -220,13 +220,16 @@ __wt_block_checkpoint_last(WT_SESSION_IMPL *session, WT_BLOCK *block, char **met const WT_PAGE_HEADER *dsk; wt_off_t ext_off, ext_size, offset; uint64_t len, nblocks, write_gen; - uint32_t checksum, size; + uint32_t checksum, logid, size; const uint8_t *p, *t; bool found; *metadatap = *checkpoint_listp = NULL; WT_RET(__wt_buf_init(session, checkpoint, WT_BLOCK_CHECKPOINT_BUFFER)); + /* TODO: scan all log IDs. */ + logid = 0; + /* * Initialize a pair of structures that track the best and current checkpoints found so far. * This is a little trickier than normal because we don't want to start saving a checkpoint only @@ -279,7 +282,7 @@ __wt_block_checkpoint_last(WT_SESSION_IMPL *session, WT_BLOCK *block, char **met * block isn't valid, skip to the next possible block. */ if (__wt_block_offset_invalid(block, offset, size) || - __wt_block_read_off(session, block, tmp, offset, size, checksum) != 0) { + __wt_block_read_off(session, block, tmp, logid, offset, size, checksum) != 0) { size = WT_BTREE_MIN_ALLOC_SIZE; continue; } diff --git a/src/third_party/wiredtiger/src/block/block_compact.c b/src/third_party/wiredtiger/src/block/block_compact.c index 19ee0b664f6..6615d4e192d 100644 --- a/src/third_party/wiredtiger/src/block/block_compact.c +++ b/src/third_party/wiredtiger/src/block/block_compact.c @@ -136,13 +136,13 @@ __wt_block_compact_page_skip( WT_EXT *ext; WT_EXTLIST *el; wt_off_t limit, offset; - uint32_t size, checksum; + uint32_t checksum, logid, size; WT_UNUSED(addr_size); *skipp = true; /* Return a default skip. */ /* Crack the cookie. */ - WT_RET(__wt_block_buffer_to_addr(block, addr, &offset, &size, &checksum)); + WT_RET(__wt_block_buffer_to_addr(block, addr, &logid, &offset, &size, &checksum)); /* * If this block is in the chosen percentage of the file and there's a block on the available diff --git a/src/third_party/wiredtiger/src/block/block_ext.c b/src/third_party/wiredtiger/src/block/block_ext.c index 06c44680f1a..d0dbfa97646 100644 --- a/src/third_party/wiredtiger/src/block/block_ext.c +++ b/src/third_party/wiredtiger/src/block/block_ext.c @@ -380,7 +380,7 @@ corrupt: /* * __wt_block_off_remove_overlap -- - * Remove a range from an extent list, where the range may be part of a overlapping entry. + * Remove a range from an extent list, where the range may be part of an overlapping entry. */ int __wt_block_off_remove_overlap( @@ -565,24 +565,28 @@ __wt_block_free(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, { WT_DECL_RET; wt_off_t offset; - uint32_t checksum, size; + uint32_t checksum, logid, size; WT_UNUSED(addr_size); WT_STAT_DATA_INCR(session, block_free); /* Crack the cookie. */ - WT_RET(__wt_block_buffer_to_addr(block, addr, &offset, &size, &checksum)); + WT_RET(__wt_block_buffer_to_addr(block, addr, &logid, &offset, &size, &checksum)); - __wt_verbose( - session, WT_VERB_BLOCK, "free %" PRIdMAX "/%" PRIdMAX, (intmax_t)offset, (intmax_t)size); + __wt_verbose(session, WT_VERB_BLOCK, "free %" PRIu32 ": %" PRIdMAX "/%" PRIdMAX, logid, + (intmax_t)offset, (intmax_t)size); #ifdef HAVE_DIAGNOSTIC WT_RET(__wt_block_misplaced(session, block, "free", offset, size, true, __func__, __LINE__)); #endif - WT_RET(__wt_block_ext_prealloc(session, 5)); - __wt_spin_lock(session, &block->live_lock); - ret = __wt_block_off_free(session, block, offset, (wt_off_t)size); - __wt_spin_unlock(session, &block->live_lock); + if (logid == block->logid) { + WT_RET(__wt_block_ext_prealloc(session, 5)); + __wt_spin_lock(session, &block->live_lock); + ret = __wt_block_off_free(session, block, logid, offset, (wt_off_t)size); + __wt_spin_unlock(session, &block->live_lock); + } else { + /* TODO: update stats about older files to drive garbage collection. */ + } return (ret); } @@ -592,13 +596,18 @@ __wt_block_free(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, * Free a file range to the underlying file. */ int -__wt_block_off_free(WT_SESSION_IMPL *session, WT_BLOCK *block, wt_off_t offset, wt_off_t size) +__wt_block_off_free( + WT_SESSION_IMPL *session, WT_BLOCK *block, uint32_t logid, wt_off_t offset, wt_off_t size) { WT_DECL_RET; /* If a sync is running, no other sessions can free blocks. */ WT_ASSERT(session, WT_SESSION_BTREE_SYNC_SAFE(session, S2BT(session))); + /* TODO: track stats for old files to drive garbage collection. */ + if (logid != block->logid) + return (0); + /* * Callers of this function are expected to have already acquired any locks required to * manipulate the extent lists. @@ -1096,7 +1105,7 @@ __wt_block_extlist_read( return (0); WT_RET(__wt_scr_alloc(session, el->size, &tmp)); - WT_ERR(__wt_block_read_off(session, block, tmp, el->offset, el->size, el->checksum)); + WT_ERR(__wt_block_read_off(session, block, tmp, el->logid, el->offset, el->size, el->checksum)); p = WT_BLOCK_HEADER_BYTE(tmp->mem); WT_ERR(__wt_extlist_read_pair(&p, &off, &size)); @@ -1156,7 +1165,7 @@ __wt_block_extlist_write( WT_EXT *ext; WT_PAGE_HEADER *dsk; size_t size; - uint32_t entries; + uint32_t logid, entries; uint8_t *p; WT_RET(__block_extlist_dump(session, block, el, "write")); @@ -1214,7 +1223,8 @@ __wt_block_extlist_write( /* Write the extent list to disk. */ WT_ERR(__wt_block_write_off( - session, block, tmp, &el->offset, &el->size, &el->checksum, true, true, true)); + session, block, tmp, &logid, &el->offset, &el->size, &el->checksum, true, true, true)); + WT_UNUSED(logid); /* TODO check */ /* * Remove the allocated blocks from the system's allocation list, extent blocks never appear on diff --git a/src/third_party/wiredtiger/src/block/block_open.c b/src/third_party/wiredtiger/src/block/block_open.c index d97f3a86f7d..5eba665ea8d 100644 --- a/src/third_party/wiredtiger/src/block/block_open.c +++ b/src/third_party/wiredtiger/src/block/block_open.c @@ -33,6 +33,8 @@ __wt_block_manager_create(WT_SESSION_IMPL *session, const char *filename, uint32 int suffix; bool exists; + WT_ERR(__wt_scr_alloc(session, 0, &tmp)); + /* * Create the underlying file and open a handle. * @@ -46,8 +48,6 @@ __wt_block_manager_create(WT_SESSION_IMPL *session, const char *filename, uint32 break; WT_ERR_TEST(ret != EEXIST, ret, false); - if (tmp == NULL) - WT_ERR(__wt_scr_alloc(session, 0, &tmp)); for (suffix = 1;; ++suffix) { WT_ERR(__wt_buf_fmt(session, tmp, "%s.%d", filename, suffix)); WT_ERR(__wt_fs_exist(session, tmp->data, &exists)); @@ -91,6 +91,7 @@ __block_destroy(WT_SESSION_IMPL *session, WT_BLOCK *block) WT_CONNECTION_IMPL *conn; WT_DECL_RET; uint64_t bucket; + u_int i; conn = S2C(session); bucket = block->name_hash & (conn->hash_size - 1); @@ -98,6 +99,12 @@ __block_destroy(WT_SESSION_IMPL *session, WT_BLOCK *block) __wt_free(session, block->name); + if (block->log_structured && block->lfh != NULL) { + for (i = 0; i < block->max_logid; i++) + WT_TRET(__wt_close(session, &block->lfh[i])); + __wt_free(session, block->lfh); + } + if (block->fh != NULL) WT_TRET(__wt_close(session, &block->fh)); @@ -175,6 +182,7 @@ __wt_block_open(WT_SESSION_IMPL *session, const char *filename, const char *cfg[ WT_ERR(__wt_config_gets(session, cfg, "block_allocation", &cval)); block->allocfirst = WT_STRING_MATCH("first", cval.str, cval.len); + block->log_structured = WT_STRING_MATCH("log-structured", cval.str, cval.len); /* Configuration: optional OS buffer cache maximum size. */ WT_ERR(__wt_config_gets(session, cfg, "os_cache_max", &cval)); @@ -203,7 +211,8 @@ __wt_block_open(WT_SESSION_IMPL *session, const char *filename, const char *cfg[ LF_SET(WT_FS_OPEN_DIRECTIO); if (!readonly && FLD_ISSET(conn->direct_io, WT_DIRECT_IO_DATA)) LF_SET(WT_FS_OPEN_DIRECTIO); - WT_ERR(__wt_open(session, filename, WT_FS_OPEN_FILE_TYPE_DATA, flags, &block->fh)); + block->file_flags = flags; + WT_ERR(__wt_open(session, filename, WT_FS_OPEN_FILE_TYPE_DATA, block->file_flags, &block->fh)); /* Set the file's size. */ WT_ERR(__wt_filesize(session, block->fh, &block->size)); diff --git a/src/third_party/wiredtiger/src/block/block_read.c b/src/third_party/wiredtiger/src/block/block_read.c index 8680f1f90f8..c5e3a1c193c 100644 --- a/src/third_party/wiredtiger/src/block/block_read.c +++ b/src/third_party/wiredtiger/src/block/block_read.c @@ -18,9 +18,10 @@ __wt_bm_preload(WT_BM *bm, WT_SESSION_IMPL *session, const uint8_t *addr, size_t WT_BLOCK *block; WT_DECL_ITEM(tmp); WT_DECL_RET; + WT_FH *fh; WT_FILE_HANDLE *handle; wt_off_t offset; - uint32_t checksum, size; + uint32_t checksum, logid, size; bool mapped; block = bm->block; @@ -28,9 +29,10 @@ __wt_bm_preload(WT_BM *bm, WT_SESSION_IMPL *session, const uint8_t *addr, size_t WT_STAT_CONN_INCR(session, block_preload); /* Crack the cookie. */ - WT_RET(__wt_block_buffer_to_addr(block, addr, &offset, &size, &checksum)); + WT_RET(__wt_block_buffer_to_addr(block, addr, &logid, &offset, &size, &checksum)); - handle = block->fh->handle; + WT_RET(__wt_block_fh(session, block, logid, &fh)); + handle = fh->handle; mapped = bm->map != NULL && offset + size <= (wt_off_t)bm->maplen; if (mapped && handle->fh_map_preload != NULL) ret = handle->fh_map_preload( @@ -59,21 +61,23 @@ __wt_bm_read( { WT_BLOCK *block; WT_DECL_RET; + WT_FH *fh; WT_FILE_HANDLE *handle; wt_off_t offset; - uint32_t checksum, size; + uint32_t checksum, logid, size; bool mapped; WT_UNUSED(addr_size); block = bm->block; /* Crack the cookie. */ - WT_RET(__wt_block_buffer_to_addr(block, addr, &offset, &size, &checksum)); + WT_RET(__wt_block_buffer_to_addr(block, addr, &logid, &offset, &size, &checksum)); /* * Map the block if it's possible. */ - handle = block->fh->handle; + WT_RET(__wt_block_fh(session, block, logid, &fh)); + handle = fh->handle; mapped = bm->map != NULL && offset + size <= (wt_off_t)bm->maplen; if (mapped && handle->fh_map_preload != NULL) { buf->data = (uint8_t *)bm->map + offset; @@ -96,7 +100,7 @@ __wt_bm_read( #endif /* Read the block. */ __wt_capacity_throttle(session, size, WT_THROTTLE_READ); - WT_RET(__wt_block_read_off(session, block, buf, offset, size, checksum)); + WT_RET(__wt_block_read_off(session, block, buf, logid, offset, size, checksum)); /* Optionally discard blocks from the system's buffer cache. */ WT_RET(__wt_block_discard(session, block, (size_t)size)); @@ -109,17 +113,17 @@ __wt_bm_read( * Dump a block into the log in 1KB chunks. */ static int -__wt_bm_corrupt_dump(WT_SESSION_IMPL *session, WT_ITEM *buf, wt_off_t offset, uint32_t size, - uint32_t checksum) WT_GCC_FUNC_ATTRIBUTE((cold)) +__wt_bm_corrupt_dump(WT_SESSION_IMPL *session, WT_ITEM *buf, uint32_t logid, wt_off_t offset, + uint32_t size, uint32_t checksum) WT_GCC_FUNC_ATTRIBUTE((cold)) { WT_DECL_ITEM(tmp); WT_DECL_RET; size_t chunk, i, nchunks; -#define WT_CORRUPT_FMT "{%" PRIuMAX ", %" PRIu32 ", %#" PRIx32 "}" +#define WT_CORRUPT_FMT "{%" PRIu32 ": %" PRIuMAX ", %" PRIu32 ", %#" PRIx32 "}" if (buf->size == 0) { - __wt_errx(session, WT_CORRUPT_FMT ": empty buffer, no dump available", (uintmax_t)offset, - size, checksum); + __wt_errx(session, WT_CORRUPT_FMT ": empty buffer, no dump available", logid, + (uintmax_t)offset, size, checksum); return (0); } @@ -130,7 +134,7 @@ __wt_bm_corrupt_dump(WT_SESSION_IMPL *session, WT_ITEM *buf, wt_off_t offset, ui WT_ERR(__wt_buf_catfmt(session, tmp, "%02x ", ((uint8_t *)buf->data)[i])); if (++i == buf->size || i % 1024 == 0) { __wt_errx(session, - WT_CORRUPT_FMT ": (chunk %" WT_SIZET_FMT " of %" WT_SIZET_FMT "): %.*s", + WT_CORRUPT_FMT ": (chunk %" WT_SIZET_FMT " of %" WT_SIZET_FMT "): %.*s", logid, (uintmax_t)offset, size, checksum, ++chunk, nchunks, (int)tmp->size, (char *)tmp->data); if (i == buf->size) @@ -154,15 +158,15 @@ __wt_bm_corrupt(WT_BM *bm, WT_SESSION_IMPL *session, const uint8_t *addr, size_t WT_DECL_ITEM(tmp); WT_DECL_RET; wt_off_t offset; - uint32_t checksum, size; + uint32_t checksum, logid, size; /* Read the block. */ WT_RET(__wt_scr_alloc(session, 0, &tmp)); WT_ERR(__wt_bm_read(bm, session, tmp, addr, addr_size)); /* Crack the cookie, dump the block. */ - WT_ERR(__wt_block_buffer_to_addr(bm->block, addr, &offset, &size, &checksum)); - WT_ERR(__wt_bm_corrupt_dump(session, tmp, offset, size, checksum)); + WT_ERR(__wt_block_buffer_to_addr(bm->block, addr, &logid, &offset, &size, &checksum)); + WT_ERR(__wt_bm_corrupt_dump(session, tmp, logid, offset, size, checksum)); err: __wt_scr_free(session, &tmp); @@ -203,14 +207,60 @@ err: #endif /* + * __wt_block_fh -- + * Get a block file handle. + */ +int +__wt_block_fh(WT_SESSION_IMPL *session, WT_BLOCK *block, uint32_t logid, WT_FH **fhp) +{ + WT_DECL_ITEM(tmp); + WT_DECL_RET; + const char *filename; + + if (!block->log_structured || logid == block->logid) { + *fhp = block->fh; + return (0); + } + + /* TODO: fh readlock */ + if (logid * sizeof(WT_FILE_HANDLE *) < block->lfh_alloc && (*fhp = block->lfh[logid]) != NULL) + return (0); + + /* TODO: fh writelock */ + /* Ensure the array goes far enough. */ + WT_RET(__wt_realloc_def(session, &block->lfh_alloc, logid + 1, &block->lfh)); + if (logid >= block->max_logid) + block->max_logid = logid + 1; + if ((*fhp = block->lfh[logid]) != NULL) + return (0); + + WT_RET(__wt_scr_alloc(session, 0, &tmp)); + if (logid == 0) + filename = block->name; + else { + WT_ERR(__wt_buf_fmt(session, tmp, "%s.%08" PRIu32, block->name, logid)); + filename = tmp->data; + } + WT_ERR(__wt_open(session, filename, WT_FS_OPEN_FILE_TYPE_DATA, + WT_FS_OPEN_READONLY | block->file_flags, &block->lfh[logid])); + *fhp = block->lfh[logid]; + WT_ASSERT(session, *fhp != NULL); + +err: + __wt_scr_free(session, &tmp); + return (ret); +} + +/* * __wt_block_read_off -- * Read an addr/size pair referenced block into a buffer. */ int -__wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, wt_off_t offset, - uint32_t size, uint32_t checksum) +__wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint32_t logid, + wt_off_t offset, uint32_t size, uint32_t checksum) { WT_BLOCK_HEADER *blk, swap; + WT_FH *fh; size_t bufsize; __wt_verbose(session, WT_VERB_READ, "off %" PRIuMAX ", size %" PRIu32 ", checksum %#" PRIx32, @@ -243,7 +293,8 @@ __wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, wt_ block->name, size, block->allocsize); WT_RET(__wt_buf_init(session, buf, bufsize)); - WT_RET(__wt_read(session, block->fh, offset, size, buf->mem)); + WT_RET(__wt_block_fh(session, block, logid, &fh)); + WT_RET(__wt_read(session, fh, offset, size, buf->mem)); buf->size = size; /* @@ -276,7 +327,7 @@ __wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, wt_ block->name, size, (uintmax_t)offset, swap.checksum, checksum); if (!F_ISSET(session, WT_SESSION_QUIET_CORRUPT_FILE)) - WT_IGNORE_RET(__wt_bm_corrupt_dump(session, buf, offset, size, checksum)); + WT_IGNORE_RET(__wt_bm_corrupt_dump(session, buf, logid, offset, size, checksum)); /* Panic if a checksum fails during an ordinary read. */ F_SET(S2C(session), WT_CONN_DATA_CORRUPTION); diff --git a/src/third_party/wiredtiger/src/block/block_slvg.c b/src/third_party/wiredtiger/src/block/block_slvg.c index 8a632e892b4..857f15d7848 100644 --- a/src/third_party/wiredtiger/src/block/block_slvg.c +++ b/src/third_party/wiredtiger/src/block/block_slvg.c @@ -104,11 +104,14 @@ __wt_block_salvage_next( WT_DECL_RET; WT_FH *fh; wt_off_t max, offset; - uint32_t allocsize, checksum, size; + uint32_t allocsize, checksum, logid, size; uint8_t *endp; *eofp = 0; + /* FIXME: salvage across all chunks in a log-structured tree. */ + logid = 0; + fh = block->fh; allocsize = block->allocsize; WT_ERR(__wt_scr_alloc(session, allocsize, &tmp)); @@ -137,19 +140,19 @@ __wt_block_salvage_next( * otherwise, move past it. */ if (!__wt_block_offset_invalid(block, offset, size) && - __wt_block_read_off(session, block, tmp, offset, size, checksum) == 0) + __wt_block_read_off(session, block, tmp, logid, offset, size, checksum) == 0) break; /* Free the allocation-size block. */ __wt_verbose(session, WT_VERB_SALVAGE, "skipping %" PRIu32 "B at file offset %" PRIuMAX, allocsize, (uintmax_t)offset); - WT_ERR(__wt_block_off_free(session, block, offset, (wt_off_t)allocsize)); + WT_ERR(__wt_block_off_free(session, block, logid, offset, (wt_off_t)allocsize)); block->slvg_off += allocsize; } /* Re-create the address cookie that should reference this block. */ endp = addr; - WT_ERR(__wt_block_addr_to_buffer(block, &endp, offset, size, checksum)); + WT_ERR(__wt_block_addr_to_buffer(block, &endp, logid, offset, size, checksum)); *addr_sizep = WT_PTRDIFF(endp, addr); done: @@ -167,7 +170,7 @@ __wt_block_salvage_valid( WT_SESSION_IMPL *session, WT_BLOCK *block, uint8_t *addr, size_t addr_size, bool valid) { wt_off_t offset; - uint32_t size, checksum; + uint32_t size, logid, checksum; WT_UNUSED(addr_size); @@ -175,11 +178,11 @@ __wt_block_salvage_valid( * Crack the cookie. If the upper layer took the block, move past it; if the upper layer * rejected the block, move past an allocation size chunk and free it. */ - WT_RET(__wt_block_buffer_to_addr(block, addr, &offset, &size, &checksum)); + WT_RET(__wt_block_buffer_to_addr(block, addr, &logid, &offset, &size, &checksum)); if (valid) block->slvg_off = offset + size; else { - WT_RET(__wt_block_off_free(session, block, offset, (wt_off_t)block->allocsize)); + WT_RET(__wt_block_off_free(session, block, logid, offset, (wt_off_t)block->allocsize)); block->slvg_off = offset + block->allocsize; } diff --git a/src/third_party/wiredtiger/src/block/block_vrfy.c b/src/third_party/wiredtiger/src/block/block_vrfy.c index 57bba04e5da..0a434aebfce 100644 --- a/src/third_party/wiredtiger/src/block/block_vrfy.c +++ b/src/third_party/wiredtiger/src/block/block_vrfy.c @@ -316,12 +316,12 @@ __wt_block_verify_addr( WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size) { wt_off_t offset; - uint32_t checksum, size; + uint32_t checksum, logid, size; WT_UNUSED(addr_size); /* Crack the cookie. */ - WT_RET(__wt_block_buffer_to_addr(block, addr, &offset, &size, &checksum)); + WT_RET(__wt_block_buffer_to_addr(block, addr, &logid, &offset, &size, &checksum)); /* Add to the per-file list. */ WT_RET(__verify_filefrag_add(session, block, NULL, offset, size, false)); diff --git a/src/third_party/wiredtiger/src/block/block_write.c b/src/third_party/wiredtiger/src/block/block_write.c index c5890efd038..bafee786c5b 100644 --- a/src/third_party/wiredtiger/src/block/block_write.c +++ b/src/third_party/wiredtiger/src/block/block_write.c @@ -189,14 +189,14 @@ __wt_block_write(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint8_ size_t *addr_sizep, bool data_checksum, bool checkpoint_io) { wt_off_t offset; - uint32_t checksum, size; + uint32_t checksum, logid, size; uint8_t *endp; WT_RET(__wt_block_write_off( - session, block, buf, &offset, &size, &checksum, data_checksum, checkpoint_io, false)); + session, block, buf, &logid, &offset, &size, &checksum, data_checksum, checkpoint_io, false)); endp = addr; - WT_RET(__wt_block_addr_to_buffer(block, &endp, offset, size, checksum)); + WT_RET(__wt_block_addr_to_buffer(block, &endp, logid, offset, size, checksum)); *addr_sizep = WT_PTRDIFF(endp, addr); return (0); @@ -207,15 +207,16 @@ __wt_block_write(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint8_ * Write a buffer into a block, returning the block's offset, size and checksum. */ static int -__block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, wt_off_t *offsetp, - uint32_t *sizep, uint32_t *checksump, bool data_checksum, bool checkpoint_io, bool caller_locked) +__block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint32_t *logidp, + wt_off_t *offsetp, uint32_t *sizep, uint32_t *checksump, bool data_checksum, bool checkpoint_io, + bool caller_locked) { WT_BLOCK_HEADER *blk; WT_DECL_RET; WT_FH *fh; wt_off_t offset; size_t align_size; - uint32_t checksum; + uint32_t checksum, logid; uint8_t *file_sizep; bool local_locked; @@ -224,6 +225,7 @@ __block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, wt_of *checksump = 0; /* -Werror=maybe-uninitialized */ fh = block->fh; + logid = block->logid; /* Buffers should be aligned for writing. */ if (!F_ISSET(buf, WT_ITEM_ALIGNED)) { @@ -325,7 +327,7 @@ __block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, wt_of if ((ret = __wt_write(session, fh, offset, align_size, buf->mem)) != 0) { if (!caller_locked) __wt_spin_lock(session, &block->live_lock); - WT_TRET(__wt_block_off_free(session, block, offset, (wt_off_t)align_size)); + WT_TRET(__wt_block_off_free(session, block, logid, offset, (wt_off_t)align_size)); if (!caller_locked) __wt_spin_unlock(session, &block->live_lock); WT_RET(ret); @@ -359,6 +361,7 @@ __block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, wt_of __wt_verbose(session, WT_VERB_WRITE, "off %" PRIuMAX ", size %" PRIuMAX ", checksum %#" PRIx32, (uintmax_t)offset, (uintmax_t)align_size, checksum); + *logidp = logid; *offsetp = offset; *sizep = WT_STORE_SIZE(align_size); *checksump = checksum; @@ -371,8 +374,9 @@ __block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, wt_of * Write a buffer into a block, returning the block's offset, size and checksum. */ int -__wt_block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, wt_off_t *offsetp, - uint32_t *sizep, uint32_t *checksump, bool data_checksum, bool checkpoint_io, bool caller_locked) +__wt_block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint32_t *logidp, + wt_off_t *offsetp, uint32_t *sizep, uint32_t *checksump, bool data_checksum, bool checkpoint_io, + bool caller_locked) { WT_DECL_RET; @@ -382,8 +386,8 @@ __wt_block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, wt * never see anything other than their original content. */ __wt_page_header_byteswap(buf->mem); - ret = __block_write_off( - session, block, buf, offsetp, sizep, checksump, data_checksum, checkpoint_io, caller_locked); + ret = __block_write_off(session, block, buf, logidp, offsetp, sizep, checksump, data_checksum, + checkpoint_io, caller_locked); __wt_page_header_byteswap(buf->mem); return (ret); } diff --git a/src/third_party/wiredtiger/src/btree/bt_debug.c b/src/third_party/wiredtiger/src/btree/bt_debug.c index fc68db0e1bb..2f93982a461 100644 --- a/src/third_party/wiredtiger/src/btree/bt_debug.c +++ b/src/third_party/wiredtiger/src/btree/bt_debug.c @@ -377,6 +377,7 @@ int __wt_debug_offset( WT_SESSION_IMPL *session, wt_off_t offset, uint32_t size, uint32_t checksum, const char *ofile) { + WT_BLOCK *block; WT_DECL_ITEM(buf); WT_DECL_RET; uint8_t addr[WT_BTREE_MAX_ADDR_COOKIE], *endp; @@ -390,8 +391,9 @@ __wt_debug_offset( * * Convert the triplet into an address structure. */ + block = S2BT(session)->bm->block; endp = addr; - WT_RET(__wt_block_addr_to_buffer(S2BT(session)->bm->block, &endp, offset, size, checksum)); + WT_RET(__wt_block_addr_to_buffer(block, &endp, block->logid, offset, size, checksum)); /* * Read the address through the btree I/O functions (so the block is decompressed as necessary). diff --git a/src/third_party/wiredtiger/src/btree/bt_handle.c b/src/third_party/wiredtiger/src/btree/bt_handle.c index c23b2e05240..2d26a1bf573 100644 --- a/src/third_party/wiredtiger/src/btree/bt_handle.c +++ b/src/third_party/wiredtiger/src/btree/bt_handle.c @@ -484,6 +484,11 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt) /* Configure encryption. */ WT_RET(__wt_btree_config_encryptor(session, cfg, &btree->kencryptor)); + /* Configure read-only. */ + WT_RET(__wt_config_gets(session, cfg, "readonly", &cval)); + if (cval.val) + F_SET(btree, WT_BTREE_READONLY); + /* Initialize locks. */ WT_RET(__wt_rwlock_init(session, &btree->ovfl_lock)); WT_RET(__wt_spin_init(session, &btree->flush_lock, "btree flush")); diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index dde2b81aafa..bb87d9e3535 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -154,6 +154,14 @@ static const WT_CONFIG_CHECK confchk_WT_CURSOR_reconfigure[] = { {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_assert_subconfigs[] = { + {"commit_timestamp", "string", NULL, + "choices=[\"always\",\"key_consistent\",\"never\"," + "\"none\"]", + NULL, 0}, + {"durable_timestamp", "string", NULL, + "choices=[\"always\",\"key_consistent\",\"never\"," + "\"none\"]", + NULL, 0}, {"read_timestamp", "string", NULL, "choices=[\"always\",\"never\",\"none\"]", NULL, 0}, {"write_timestamp", "string", NULL, "choices=[\"off\",\"on\"]", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; @@ -164,12 +172,12 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_create_log_subconfigs[] = { static const WT_CONFIG_CHECK confchk_WT_SESSION_alter[] = { {"access_pattern_hint", "string", NULL, "choices=[\"none\",\"random\",\"sequential\"]", NULL, 0}, {"app_metadata", "string", NULL, NULL, NULL, 0}, - {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 2}, - {"cache_resident", "boolean", NULL, NULL, NULL, 0}, + {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 4}, + {"cache_resident", "boolean", NULL, NULL, NULL, 0}, {"checkpoint", "string", NULL, NULL, NULL, 0}, {"exclusive_refreshed", "boolean", NULL, NULL, NULL, 0}, {"log", "category", NULL, NULL, confchk_WT_SESSION_create_log_subconfigs, 1}, {"os_cache_dirty_max", "int", NULL, "min=0", NULL, 0}, - {"os_cache_max", "int", NULL, "min=0", NULL, 0}, + {"os_cache_max", "int", NULL, "min=0", NULL, 0}, {"readonly", "boolean", NULL, NULL, NULL, 0}, {"verbose", "list", NULL, "choices=[\"write_timestamp\"]", NULL, 0}, {"write_timestamp_usage", "string", NULL, "choices=[\"always\",\"key_consistent\",\"mixed_mode\"," @@ -236,12 +244,16 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_create_lsm_subconfigs[] = { {"merge_max", "int", NULL, "min=2,max=100", NULL, 0}, {"merge_min", "int", NULL, "max=100", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; +static const WT_CONFIG_CHECK confchk_WT_SESSION_create_tiered_subconfigs[] = { + {"chunk_size", "int", NULL, "min=1M", NULL, 0}, {"tiers", "list", NULL, NULL, NULL, 0}, + {NULL, NULL, NULL, NULL, NULL, 0}}; + static const WT_CONFIG_CHECK confchk_WT_SESSION_create[] = { {"access_pattern_hint", "string", NULL, "choices=[\"none\",\"random\",\"sequential\"]", NULL, 0}, {"allocation_size", "int", NULL, "min=512B,max=128MB", NULL, 0}, {"app_metadata", "string", NULL, NULL, NULL, 0}, - {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 2}, - {"block_allocation", "string", NULL, "choices=[\"first\",\"best\"]", NULL, 0}, + {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 4}, + {"block_allocation", "string", NULL, "choices=[\"best\",\"first\",\"log-structured\"]", NULL, 0}, {"block_compressor", "string", NULL, NULL, NULL, 0}, {"cache_resident", "boolean", NULL, NULL, NULL, 0}, {"checksum", "string", NULL, "choices=[\"on\",\"off\",\"uncompressed\"]", NULL, 0}, @@ -250,7 +262,7 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_create[] = { {"encryption", "category", NULL, NULL, confchk_WT_SESSION_create_encryption_subconfigs, 2}, {"exclusive", "boolean", NULL, NULL, NULL, 0}, {"extractor", "string", NULL, NULL, NULL, 0}, {"format", "string", NULL, "choices=[\"btree\"]", NULL, 0}, - {"huffman_value", "string", NULL, NULL, NULL, 0}, + {"huffman_key", "string", NULL, NULL, NULL, 0}, {"huffman_value", "string", NULL, NULL, NULL, 0}, {"ignore_in_memory_cache_size", "boolean", NULL, NULL, NULL, 0}, {"immutable", "boolean", NULL, NULL, NULL, 0}, {"import", "category", NULL, NULL, confchk_WT_SESSION_create_import_subconfigs, 3}, @@ -271,9 +283,12 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_create[] = { {"os_cache_max", "int", NULL, "min=0", NULL, 0}, {"prefix_compression", "boolean", NULL, NULL, NULL, 0}, {"prefix_compression_min", "int", NULL, "min=0", NULL, 0}, - {"source", "string", NULL, NULL, NULL, 0}, {"split_deepen_min_child", "int", NULL, NULL, NULL, 0}, + {"readonly", "boolean", NULL, NULL, NULL, 0}, {"source", "string", NULL, NULL, NULL, 0}, + {"split_deepen_min_child", "int", NULL, NULL, NULL, 0}, {"split_deepen_per_child", "int", NULL, NULL, NULL, 0}, - {"split_pct", "int", NULL, "min=50,max=100", NULL, 0}, {"type", "string", NULL, NULL, NULL, 0}, + {"split_pct", "int", NULL, "min=50,max=100", NULL, 0}, + {"tiered", "category", NULL, NULL, confchk_WT_SESSION_create_tiered_subconfigs, 2}, + {"type", "string", NULL, NULL, NULL, 0}, {"value_format", "format", __wt_struct_confchk, NULL, NULL, 0}, {"verbose", "list", NULL, "choices=[\"write_timestamp\"]", NULL, 0}, {"write_timestamp_usage", "string", NULL, @@ -372,7 +387,7 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_verify[] = { static const WT_CONFIG_CHECK confchk_colgroup_meta[] = { {"app_metadata", "string", NULL, NULL, NULL, 0}, - {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 2}, + {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 4}, {"collator", "string", NULL, NULL, NULL, 0}, {"columns", "list", NULL, NULL, NULL, 0}, {"source", "string", NULL, NULL, NULL, 0}, {"type", "string", NULL, NULL, NULL, 0}, {"verbose", "list", NULL, "choices=[\"write_timestamp\"]", NULL, 0}, @@ -386,8 +401,8 @@ static const WT_CONFIG_CHECK confchk_file_config[] = { {"access_pattern_hint", "string", NULL, "choices=[\"none\",\"random\",\"sequential\"]", NULL, 0}, {"allocation_size", "int", NULL, "min=512B,max=128MB", NULL, 0}, {"app_metadata", "string", NULL, NULL, NULL, 0}, - {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 2}, - {"block_allocation", "string", NULL, "choices=[\"first\",\"best\"]", NULL, 0}, + {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 4}, + {"block_allocation", "string", NULL, "choices=[\"best\",\"first\",\"log-structured\"]", NULL, 0}, {"block_compressor", "string", NULL, NULL, NULL, 0}, {"cache_resident", "boolean", NULL, NULL, NULL, 0}, {"checksum", "string", NULL, "choices=[\"on\",\"off\",\"uncompressed\"]", NULL, 0}, @@ -395,7 +410,7 @@ static const WT_CONFIG_CHECK confchk_file_config[] = { {"dictionary", "int", NULL, "min=0", NULL, 0}, {"encryption", "category", NULL, NULL, confchk_WT_SESSION_create_encryption_subconfigs, 2}, {"format", "string", NULL, "choices=[\"btree\"]", NULL, 0}, - {"huffman_value", "string", NULL, NULL, NULL, 0}, + {"huffman_key", "string", NULL, NULL, NULL, 0}, {"huffman_value", "string", NULL, NULL, NULL, 0}, {"ignore_in_memory_cache_size", "boolean", NULL, NULL, NULL, 0}, {"internal_item_max", "int", NULL, "min=0", NULL, 0}, {"internal_key_max", "int", NULL, "min=0", NULL, 0}, @@ -413,6 +428,7 @@ static const WT_CONFIG_CHECK confchk_file_config[] = { {"os_cache_max", "int", NULL, "min=0", NULL, 0}, {"prefix_compression", "boolean", NULL, NULL, NULL, 0}, {"prefix_compression_min", "int", NULL, "min=0", NULL, 0}, + {"readonly", "boolean", NULL, NULL, NULL, 0}, {"split_deepen_min_child", "int", NULL, NULL, NULL, 0}, {"split_deepen_per_child", "int", NULL, NULL, NULL, 0}, {"split_pct", "int", NULL, "min=50,max=100", NULL, 0}, @@ -428,8 +444,8 @@ static const WT_CONFIG_CHECK confchk_file_meta[] = { {"access_pattern_hint", "string", NULL, "choices=[\"none\",\"random\",\"sequential\"]", NULL, 0}, {"allocation_size", "int", NULL, "min=512B,max=128MB", NULL, 0}, {"app_metadata", "string", NULL, NULL, NULL, 0}, - {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 2}, - {"block_allocation", "string", NULL, "choices=[\"first\",\"best\"]", NULL, 0}, + {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 4}, + {"block_allocation", "string", NULL, "choices=[\"best\",\"first\",\"log-structured\"]", NULL, 0}, {"block_compressor", "string", NULL, NULL, NULL, 0}, {"cache_resident", "boolean", NULL, NULL, NULL, 0}, {"checkpoint", "string", NULL, NULL, NULL, 0}, {"checkpoint_backup_info", "string", NULL, NULL, NULL, 0}, @@ -439,7 +455,8 @@ static const WT_CONFIG_CHECK confchk_file_meta[] = { {"dictionary", "int", NULL, "min=0", NULL, 0}, {"encryption", "category", NULL, NULL, confchk_WT_SESSION_create_encryption_subconfigs, 2}, {"format", "string", NULL, "choices=[\"btree\"]", NULL, 0}, - {"huffman_value", "string", NULL, NULL, NULL, 0}, {"id", "string", NULL, NULL, NULL, 0}, + {"huffman_key", "string", NULL, NULL, NULL, 0}, {"huffman_value", "string", NULL, NULL, NULL, 0}, + {"id", "string", NULL, NULL, NULL, 0}, {"ignore_in_memory_cache_size", "boolean", NULL, NULL, NULL, 0}, {"internal_item_max", "int", NULL, "min=0", NULL, 0}, {"internal_key_max", "int", NULL, "min=0", NULL, 0}, @@ -457,6 +474,7 @@ static const WT_CONFIG_CHECK confchk_file_meta[] = { {"os_cache_max", "int", NULL, "min=0", NULL, 0}, {"prefix_compression", "boolean", NULL, NULL, NULL, 0}, {"prefix_compression_min", "int", NULL, "min=0", NULL, 0}, + {"readonly", "boolean", NULL, NULL, NULL, 0}, {"split_deepen_min_child", "int", NULL, NULL, NULL, 0}, {"split_deepen_per_child", "int", NULL, NULL, NULL, 0}, {"split_pct", "int", NULL, "min=50,max=100", NULL, 0}, @@ -471,7 +489,7 @@ static const WT_CONFIG_CHECK confchk_file_meta[] = { static const WT_CONFIG_CHECK confchk_index_meta[] = { {"app_metadata", "string", NULL, NULL, NULL, 0}, - {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 2}, + {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 4}, {"collator", "string", NULL, NULL, NULL, 0}, {"columns", "list", NULL, NULL, NULL, 0}, {"extractor", "string", NULL, NULL, NULL, 0}, {"immutable", "boolean", NULL, NULL, NULL, 0}, {"index_key_columns", "int", NULL, NULL, NULL, 0}, @@ -489,8 +507,8 @@ static const WT_CONFIG_CHECK confchk_lsm_meta[] = { {"access_pattern_hint", "string", NULL, "choices=[\"none\",\"random\",\"sequential\"]", NULL, 0}, {"allocation_size", "int", NULL, "min=512B,max=128MB", NULL, 0}, {"app_metadata", "string", NULL, NULL, NULL, 0}, - {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 2}, - {"block_allocation", "string", NULL, "choices=[\"first\",\"best\"]", NULL, 0}, + {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 4}, + {"block_allocation", "string", NULL, "choices=[\"best\",\"first\",\"log-structured\"]", NULL, 0}, {"block_compressor", "string", NULL, NULL, NULL, 0}, {"cache_resident", "boolean", NULL, NULL, NULL, 0}, {"checksum", "string", NULL, "choices=[\"on\",\"off\",\"uncompressed\"]", NULL, 0}, @@ -498,7 +516,7 @@ static const WT_CONFIG_CHECK confchk_lsm_meta[] = { {"columns", "list", NULL, NULL, NULL, 0}, {"dictionary", "int", NULL, "min=0", NULL, 0}, {"encryption", "category", NULL, NULL, confchk_WT_SESSION_create_encryption_subconfigs, 2}, {"format", "string", NULL, "choices=[\"btree\"]", NULL, 0}, - {"huffman_value", "string", NULL, NULL, NULL, 0}, + {"huffman_key", "string", NULL, NULL, NULL, 0}, {"huffman_value", "string", NULL, NULL, NULL, 0}, {"ignore_in_memory_cache_size", "boolean", NULL, NULL, NULL, 0}, {"internal_item_max", "int", NULL, "min=0", NULL, 0}, {"internal_key_max", "int", NULL, "min=0", NULL, 0}, @@ -518,6 +536,7 @@ static const WT_CONFIG_CHECK confchk_lsm_meta[] = { {"os_cache_max", "int", NULL, "min=0", NULL, 0}, {"prefix_compression", "boolean", NULL, NULL, NULL, 0}, {"prefix_compression_min", "int", NULL, "min=0", NULL, 0}, + {"readonly", "boolean", NULL, NULL, NULL, 0}, {"split_deepen_min_child", "int", NULL, NULL, NULL, 0}, {"split_deepen_per_child", "int", NULL, NULL, NULL, 0}, {"split_pct", "int", NULL, "min=50,max=100", NULL, 0}, @@ -531,7 +550,7 @@ static const WT_CONFIG_CHECK confchk_lsm_meta[] = { static const WT_CONFIG_CHECK confchk_table_meta[] = { {"app_metadata", "string", NULL, NULL, NULL, 0}, - {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 2}, + {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 4}, {"colgroups", "list", NULL, NULL, NULL, 0}, {"collator", "string", NULL, NULL, NULL, 0}, {"columns", "list", NULL, NULL, NULL, 0}, {"key_format", "format", __wt_struct_confchk, NULL, NULL, 0}, @@ -543,6 +562,17 @@ static const WT_CONFIG_CHECK confchk_table_meta[] = { NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; +static const WT_CONFIG_CHECK confchk_tiered_meta[] = { + {"app_metadata", "string", NULL, NULL, NULL, 0}, + {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 4}, + {"tiered", "category", NULL, NULL, confchk_WT_SESSION_create_tiered_subconfigs, 2}, + {"verbose", "list", NULL, "choices=[\"write_timestamp\"]", NULL, 0}, + {"write_timestamp_usage", "string", NULL, + "choices=[\"always\",\"key_consistent\",\"mixed_mode\"," + "\"never\",\"none\",\"ordered\"]", + NULL, 0}, + {NULL, NULL, NULL, NULL, NULL, 0}}; + static const WT_CONFIG_CHECK confchk_wiredtiger_open_compatibility_subconfigs[] = { {"release", "string", NULL, NULL, NULL, 0}, {"require_max", "string", NULL, NULL, NULL, 0}, {"require_min", "string", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; @@ -915,11 +945,12 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", {"WT_CURSOR.reconfigure", "append=false,overwrite=true", confchk_WT_CURSOR_reconfigure, 2}, {"WT_SESSION.alter", "access_pattern_hint=none,app_metadata=," - "assert=(read_timestamp=none,write_timestamp=off)," - "cache_resident=false,exclusive_refreshed=true,log=(enabled=true)" - ",os_cache_dirty_max=0,os_cache_max=0,verbose=[]," + "assert=(commit_timestamp=none,durable_timestamp=none," + "read_timestamp=none,write_timestamp=off),cache_resident=false," + "checkpoint=,exclusive_refreshed=true,log=(enabled=true)," + "os_cache_dirty_max=0,os_cache_max=0,readonly=false,verbose=[]," "write_timestamp_usage=none", - confchk_WT_SESSION_alter, 10}, + confchk_WT_SESSION_alter, 12}, {"WT_SESSION.begin_transaction", "ignore_prepare=false,isolation=,name=,operation_timeout_ms=0," "priority=0,read_before_oldest=false,read_timestamp=," @@ -935,13 +966,14 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", {"WT_SESSION.compact", "timeout=1200", confchk_WT_SESSION_compact, 1}, {"WT_SESSION.create", "access_pattern_hint=none,allocation_size=4KB,app_metadata=," - "assert=(read_timestamp=none,write_timestamp=off)," - "block_allocation=best,block_compressor=,cache_resident=false," - "checksum=uncompressed,colgroups=,collator=,columns=,dictionary=0" - ",encryption=(keyid=,name=),exclusive=false,extractor=," - "format=btree,huffman_value=,ignore_in_memory_cache_size=false," - "immutable=false,import=(enabled=false,file_metadata=," - "repair=false),internal_item_max=0,internal_key_max=0," + "assert=(commit_timestamp=none,durable_timestamp=none," + "read_timestamp=none,write_timestamp=off),block_allocation=best," + "block_compressor=,cache_resident=false,checksum=uncompressed," + "colgroups=,collator=,columns=,dictionary=0,encryption=(keyid=," + "name=),exclusive=false,extractor=,format=btree,huffman_key=," + "huffman_value=,ignore_in_memory_cache_size=false,immutable=false" + ",import=(enabled=false,file_metadata=,repair=false)," + "internal_item_max=0,internal_key_max=0," "internal_key_truncate=true,internal_page_max=4KB,key_format=u," "key_gap=10,leaf_item_max=0,leaf_key_max=0,leaf_page_max=32KB," "leaf_value_max=0,log=(enabled=true),lsm=(auto_throttle=true," @@ -950,10 +982,11 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "chunk_size=10MB,merge_custom=(prefix=,start_generation=0," "suffix=),merge_max=15,merge_min=0),memory_page_image_max=0," "memory_page_max=5MB,os_cache_dirty_max=0,os_cache_max=0," - "prefix_compression=false,prefix_compression_min=4,source=," - "split_deepen_min_child=0,split_deepen_per_child=0,split_pct=90," - "type=file,value_format=u,verbose=[],write_timestamp_usage=none", - confchk_WT_SESSION_create, 46}, + "prefix_compression=false,prefix_compression_min=4,readonly=false" + ",source=,split_deepen_min_child=0,split_deepen_per_child=0," + "split_pct=90,tiered=(chunk_size=1GB,tiers=),type=file," + "value_format=u,verbose=[],write_timestamp_usage=none", + confchk_WT_SESSION_create, 49}, {"WT_SESSION.drop", "checkpoint_wait=true,force=false,lock_wait=true," "remove_files=true", @@ -996,55 +1029,60 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "strict=false", confchk_WT_SESSION_verify, 7}, {"colgroup.meta", - "app_metadata=,assert=(read_timestamp=none,write_timestamp=off)," + "app_metadata=,assert=(commit_timestamp=none," + "durable_timestamp=none,read_timestamp=none,write_timestamp=off)," "collator=,columns=,source=,type=file,verbose=[]," "write_timestamp_usage=none", confchk_colgroup_meta, 8}, {"file.config", "access_pattern_hint=none,allocation_size=4KB,app_metadata=," - "assert=(read_timestamp=none,write_timestamp=off)," - "block_allocation=best,block_compressor=,cache_resident=false," - "checksum=uncompressed,collator=,columns=,dictionary=0," - "encryption=(keyid=,name=),format=btree,huffman_value=," + "assert=(commit_timestamp=none,durable_timestamp=none," + "read_timestamp=none,write_timestamp=off),block_allocation=best," + "block_compressor=,cache_resident=false,checksum=uncompressed," + "collator=,columns=,dictionary=0,encryption=(keyid=,name=)," + "format=btree,huffman_key=,huffman_value=," "ignore_in_memory_cache_size=false,internal_item_max=0," "internal_key_max=0,internal_key_truncate=true," "internal_page_max=4KB,key_format=u,key_gap=10,leaf_item_max=0," "leaf_key_max=0,leaf_page_max=32KB,leaf_value_max=0," "log=(enabled=true),memory_page_image_max=0,memory_page_max=5MB," "os_cache_dirty_max=0,os_cache_max=0,prefix_compression=false," - "prefix_compression_min=4,split_deepen_min_child=0," - "split_deepen_per_child=0,split_pct=90,value_format=u,verbose=[]," - "write_timestamp_usage=none", - confchk_file_config, 38}, + "prefix_compression_min=4,readonly=false,split_deepen_min_child=0" + ",split_deepen_per_child=0,split_pct=90,value_format=u,verbose=[]" + ",write_timestamp_usage=none", + confchk_file_config, 40}, {"file.meta", "access_pattern_hint=none,allocation_size=4KB,app_metadata=," - "assert=(read_timestamp=none,write_timestamp=off)," - "block_allocation=best,block_compressor=,cache_resident=false," - "checkpoint=,checkpoint_backup_info=,checkpoint_lsn=," - "checksum=uncompressed,collator=,columns=,dictionary=0," - "encryption=(keyid=,name=),format=btree,huffman_value=,id=," + "assert=(commit_timestamp=none,durable_timestamp=none," + "read_timestamp=none,write_timestamp=off),block_allocation=best," + "block_compressor=,cache_resident=false,checkpoint=," + "checkpoint_backup_info=,checkpoint_lsn=,checksum=uncompressed," + "collator=,columns=,dictionary=0,encryption=(keyid=,name=)," + "format=btree,huffman_key=,huffman_value=,id=," "ignore_in_memory_cache_size=false,internal_item_max=0," "internal_key_max=0,internal_key_truncate=true," "internal_page_max=4KB,key_format=u,key_gap=10,leaf_item_max=0," "leaf_key_max=0,leaf_page_max=32KB,leaf_value_max=0," "log=(enabled=true),memory_page_image_max=0,memory_page_max=5MB," "os_cache_dirty_max=0,os_cache_max=0,prefix_compression=false," - "prefix_compression_min=4,split_deepen_min_child=0," - "split_deepen_per_child=0,split_pct=90,value_format=u,verbose=[]," - "version=(major=0,minor=0),write_timestamp_usage=none", - confchk_file_meta, 43}, + "prefix_compression_min=4,readonly=false,split_deepen_min_child=0" + ",split_deepen_per_child=0,split_pct=90,value_format=u,verbose=[]" + ",version=(major=0,minor=0),write_timestamp_usage=none", + confchk_file_meta, 45}, {"index.meta", - "app_metadata=,assert=(read_timestamp=none,write_timestamp=off)," + "app_metadata=,assert=(commit_timestamp=none," + "durable_timestamp=none,read_timestamp=none,write_timestamp=off)," "collator=,columns=,extractor=,immutable=false,index_key_columns=" ",key_format=u,source=,type=file,value_format=u,verbose=[]," "write_timestamp_usage=none", confchk_index_meta, 13}, {"lsm.meta", "access_pattern_hint=none,allocation_size=4KB,app_metadata=," - "assert=(read_timestamp=none,write_timestamp=off)," - "block_allocation=best,block_compressor=,cache_resident=false," - "checksum=uncompressed,chunks=,collator=,columns=,dictionary=0," - "encryption=(keyid=,name=),format=btree,huffman_value=," + "assert=(commit_timestamp=none,durable_timestamp=none," + "read_timestamp=none,write_timestamp=off),block_allocation=best," + "block_compressor=,cache_resident=false,checksum=uncompressed," + "chunks=,collator=,columns=,dictionary=0,encryption=(keyid=," + "name=),format=btree,huffman_key=,huffman_value=," "ignore_in_memory_cache_size=false,internal_item_max=0," "internal_key_max=0,internal_key_truncate=true," "internal_page_max=4KB,key_format=u,key_gap=10,last=," @@ -1056,14 +1094,22 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "suffix=),merge_max=15,merge_min=0),memory_page_image_max=0," "memory_page_max=5MB,old_chunks=,os_cache_dirty_max=0," "os_cache_max=0,prefix_compression=false,prefix_compression_min=4" - ",split_deepen_min_child=0,split_deepen_per_child=0,split_pct=90," - "value_format=u,verbose=[],write_timestamp_usage=none", - confchk_lsm_meta, 42}, + ",readonly=false,split_deepen_min_child=0," + "split_deepen_per_child=0,split_pct=90,value_format=u,verbose=[]," + "write_timestamp_usage=none", + confchk_lsm_meta, 44}, {"table.meta", - "app_metadata=,assert=(read_timestamp=none,write_timestamp=off)," + "app_metadata=,assert=(commit_timestamp=none," + "durable_timestamp=none,read_timestamp=none,write_timestamp=off)," "colgroups=,collator=,columns=,key_format=u,value_format=u," "verbose=[],write_timestamp_usage=none", confchk_table_meta, 9}, + {"tiered.meta", + "app_metadata=,assert=(commit_timestamp=none," + "durable_timestamp=none,read_timestamp=none,write_timestamp=off)," + "tiered=(chunk_size=1GB,tiers=),verbose=[]," + "write_timestamp_usage=none", + confchk_tiered_meta, 5}, {"wiredtiger_open", "buffer_alignment=-1,builtin_extension_config=,cache_cursors=true" ",cache_max_wait_ms=0,cache_overhead=8,cache_size=100MB," diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index 613c71a8d05..420494d985c 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -1857,7 +1857,7 @@ __wt_verbose_config(WT_SESSION_IMPL *session, const char *cfg[]) {"salvage", WT_VERB_SALVAGE}, {"shared_cache", WT_VERB_SHARED_CACHE}, {"split", WT_VERB_SPLIT}, {"temporary", WT_VERB_TEMPORARY}, {"thread_group", WT_VERB_THREAD_GROUP}, {"timestamp", WT_VERB_TIMESTAMP}, - {"transaction", WT_VERB_TRANSACTION}, {"verify", WT_VERB_VERIFY}, + {"tiered", WT_VERB_TIERED}, {"transaction", WT_VERB_TRANSACTION}, {"verify", WT_VERB_VERIFY}, {"version", WT_VERB_VERSION}, {"write", WT_VERB_WRITE}, {NULL, 0}}; WT_CONFIG_ITEM cval, sval; WT_CONNECTION_IMPL *conn; diff --git a/src/third_party/wiredtiger/src/conn/conn_dhandle.c b/src/third_party/wiredtiger/src/conn/conn_dhandle.c index 44b2d86f7c5..b9302ccb602 100644 --- a/src/third_party/wiredtiger/src/conn/conn_dhandle.c +++ b/src/third_party/wiredtiger/src/conn/conn_dhandle.c @@ -99,6 +99,9 @@ __conn_dhandle_config_set(WT_SESSION_IMPL *session) case WT_DHANDLE_TYPE_TABLE: WT_ERR(__wt_strdup(session, WT_CONFIG_BASE(session, table_meta), &dhandle->cfg[0])); break; + case WT_DHANDLE_TYPE_TIERED: + WT_ERR(__wt_strdup(session, WT_CONFIG_BASE(session, tiered_meta), &dhandle->cfg[0])); + break; } dhandle->cfg[1] = metaconf; dhandle->meta_base = base; @@ -127,6 +130,9 @@ __conn_dhandle_destroy(WT_SESSION_IMPL *session, WT_DATA_HANDLE *dhandle) case WT_DHANDLE_TYPE_TABLE: ret = __wt_schema_close_table(session, (WT_TABLE *)dhandle); break; + case WT_DHANDLE_TYPE_TIERED: + ret = __wt_tiered_close(session, (WT_TIERED *)dhandle); + break; } __wt_rwlock_destroy(session, &dhandle->rwlock); @@ -150,6 +156,7 @@ __wt_conn_dhandle_alloc(WT_SESSION_IMPL *session, const char *uri, const char *c WT_DATA_HANDLE *dhandle; WT_DECL_RET; WT_TABLE *table; + WT_TIERED *tiered; uint64_t bucket; /* @@ -165,6 +172,10 @@ __wt_conn_dhandle_alloc(WT_SESSION_IMPL *session, const char *uri, const char *c WT_RET(__wt_calloc_one(session, &table)); dhandle = (WT_DATA_HANDLE *)table; dhandle->type = WT_DHANDLE_TYPE_TABLE; + } else if (WT_PREFIX_MATCH(uri, "tiered:")) { + WT_RET(__wt_calloc_one(session, &tiered)); + dhandle = (WT_DATA_HANDLE *)tiered; + dhandle->type = WT_DHANDLE_TYPE_TIERED; } else WT_RET_PANIC(session, EINVAL, "illegal handle allocation URI %s", uri); @@ -364,6 +375,9 @@ __wt_conn_dhandle_close(WT_SESSION_IMPL *session, bool final, bool mark_dead) case WT_DHANDLE_TYPE_TABLE: WT_TRET(__wt_schema_close_table(session, (WT_TABLE *)dhandle)); break; + case WT_DHANDLE_TYPE_TIERED: + WT_TRET(__wt_tiered_close(session, (WT_TIERED *)dhandle)); + break; } /* @@ -519,6 +533,9 @@ __wt_conn_dhandle_open(WT_SESSION_IMPL *session, const char *cfg[], uint32_t fla case WT_DHANDLE_TYPE_TABLE: WT_ERR(__wt_schema_open_table(session)); break; + case WT_DHANDLE_TYPE_TIERED: + WT_ERR(__wt_tiered_open(session, cfg)); + break; } /* diff --git a/src/third_party/wiredtiger/src/cursor/cur_backup.c b/src/third_party/wiredtiger/src/cursor/cur_backup.c index 274786c2866..f576a3ffa12 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_backup.c +++ b/src/third_party/wiredtiger/src/cursor/cur_backup.c @@ -883,7 +883,8 @@ __backup_list_uri_append(WT_SESSION_IMPL *session, const char *name, bool *skip) */ if (!WT_PREFIX_MATCH(name, "file:") && !WT_PREFIX_MATCH(name, "colgroup:") && !WT_PREFIX_MATCH(name, "index:") && !WT_PREFIX_MATCH(name, "lsm:") && - !WT_PREFIX_MATCH(name, WT_SYSTEM_PREFIX) && !WT_PREFIX_MATCH(name, "table:")) + !WT_PREFIX_MATCH(name, WT_SYSTEM_PREFIX) && !WT_PREFIX_MATCH(name, "table:") && + !WT_PREFIX_MATCH(name, "tiered:")) WT_RET_MSG(session, ENOTSUP, "hot backup is not supported for objects of type %s", name); /* Add the metadata entry to the backup file. */ diff --git a/src/third_party/wiredtiger/src/cursor/cur_log.c b/src/third_party/wiredtiger/src/cursor/cur_log.c index 84c7e15c30c..03956a1fc36 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_log.c +++ b/src/third_party/wiredtiger/src/cursor/cur_log.c @@ -208,7 +208,7 @@ __curlog_next(WT_CURSOR *cursor) */ if (cl->stepp == NULL || cl->stepp >= cl->stepp_end || !*cl->stepp) { cl->txnid = 0; - ret = __wt_log_scan(session, cl->next_lsn, WT_LOGSCAN_ONE, __curlog_logrec, cl); + ret = __wt_log_scan(session, cl->next_lsn, NULL, WT_LOGSCAN_ONE, __curlog_logrec, cl); if (ret == ENOENT) ret = WT_NOTFOUND; WT_ERR(ret); @@ -246,7 +246,7 @@ __curlog_search(WT_CURSOR *cursor) */ WT_ERR(__wt_cursor_get_key(cursor, &key_file, &key_offset, &counter)); WT_SET_LSN(&key, key_file, key_offset); - ret = __wt_log_scan(session, &key, WT_LOGSCAN_ONE, __curlog_logrec, cl); + ret = __wt_log_scan(session, &key, NULL, WT_LOGSCAN_ONE, __curlog_logrec, cl); if (ret == ENOENT) ret = WT_NOTFOUND; WT_ERR(ret); diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 99a560fbe19..2cf014137a7 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -584,8 +584,15 @@ __evict_update_work(WT_SESSION_IMPL *session) if (!__evict_queue_empty(cache->evict_urgent_queue, false)) LF_SET(WT_CACHE_EVICT_URGENT); - if (F_ISSET(conn, WT_CONN_HS_OPEN) && __wt_hs_get_btree(session, &hs_tree) == 0) + /* + * TODO: We are caching the cache usage values associated with the history store because the + * history store dhandle isn't always available to eviction. Keeping potentially out-of-date + * values could lead to surprising bugs in the future. + */ + if (F_ISSET(conn, WT_CONN_HS_OPEN) && __wt_hs_get_btree(session, &hs_tree) == 0) { cache->bytes_hs = hs_tree->bytes_inmem; + cache->bytes_hs_dirty = hs_tree->bytes_dirty_intl + hs_tree->bytes_dirty_leaf; + } /* * If we need space in the cache, try to find clean pages to evict. @@ -1726,6 +1733,19 @@ __evict_walk_tree(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, u_int max_ent if (target_pages > remaining_slots) target_pages = remaining_slots; + /* + * Reduce the number of pages to be selected from btrees other than the history store (HS) if + * the cache pressure is high and HS content dominates the cache. Evicting unclean non-HS pages + * can generate even more HS content and will not help with the cache pressure, and will + * probably just amplify it further. + */ + if (!WT_IS_HS(btree->dhandle) && __wt_cache_hs_dirty(session)) { + /* If target pages are less than 10, keep it like that. */ + target_pages = target_pages < 10 ? target_pages : target_pages / 10; + WT_STAT_CONN_INCR(session, cache_eviction_target_page_reduced); + WT_STAT_DATA_INCR(session, cache_eviction_target_page_reduced); + } + /* If we don't want any pages from this tree, move on. */ if (target_pages == 0) return (0); @@ -1930,6 +1950,19 @@ __evict_walk_tree(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, u_int max_ent continue; } + /* + * If history store dirty content is dominating the cache, we want to prioritize evicting + * history store pages over other btree pages. This helps in keeping cache contents below + * the configured cache size during checkpoints where reconciling non-HS pages can generate + * significant amount of HS dirty content very quickly. + */ + if (WT_IS_HS(btree->dhandle) && __wt_cache_hs_dirty(session)) { + WT_STAT_CONN_INCR(session, cache_eviction_pages_queued_urgent_hs_dirty); + if (__wt_page_evict_urgent(session, ref)) + urgent_queued = true; + continue; + } + /* Pages that are empty or from dead trees are fast-tracked. */ if (__wt_page_is_empty(page) || F_ISSET(session->dhandle, WT_DHANDLE_DEAD)) goto fast; diff --git a/src/third_party/wiredtiger/src/include/block.h b/src/third_party/wiredtiger/src/include/block.h index f35d09f05f3..74b677b6cd0 100644 --- a/src/third_party/wiredtiger/src/include/block.h +++ b/src/third_party/wiredtiger/src/include/block.h @@ -51,6 +51,7 @@ struct __wt_extlist { uint64_t bytes; /* Byte count */ uint32_t entries; /* Entry count */ + uint32_t logid; /* Written log ID */ wt_off_t offset; /* Written extent offset */ uint32_t checksum; /* Written extent checksum */ uint32_t size; /* Written extent size */ @@ -140,6 +141,7 @@ struct __wt_size { struct __wt_block_ckpt { uint8_t version; /* Version */ + uint32_t root_logid; wt_off_t root_offset; /* The root */ uint32_t root_checksum, root_size; @@ -236,12 +238,18 @@ struct __wt_block { /* Configuration information, set when the file is opened. */ uint32_t allocfirst; /* Allocation is first-fit */ uint32_t allocsize; /* Allocation size */ + bool log_structured; /* Write checkpoint as separate files */ size_t os_cache; /* System buffer cache flush max */ size_t os_cache_max; size_t os_cache_dirty_max; u_int block_header; /* Header length */ + /* Log-structured tracking. */ + uint32_t file_flags, logid, max_logid; + WT_FH **lfh; + size_t lfh_alloc; + /* * There is only a single checkpoint in a file that can be written. The information could * logically live in the WT_BM structure, but then we would be re-creating it every time we diff --git a/src/third_party/wiredtiger/src/include/cache.h b/src/third_party/wiredtiger/src/include/cache.h index 16bad36997b..85b3ab35f83 100644 --- a/src/third_party/wiredtiger/src/include/cache.h +++ b/src/third_party/wiredtiger/src/include/cache.h @@ -71,7 +71,6 @@ struct __wt_cache { uint64_t bytes_dirty_leaf; uint64_t bytes_dirty_total; uint64_t bytes_evict; /* Bytes/pages discarded by eviction */ - uint64_t bytes_hs; /* History store bytes inmem */ uint64_t bytes_image_intl; /* Bytes of disk images (internal) */ uint64_t bytes_image_leaf; /* Bytes of disk images (leaf) */ uint64_t bytes_inmem; /* Bytes/pages in memory */ @@ -80,6 +79,13 @@ struct __wt_cache { uint64_t bytes_updates; /* Bytes of updates to pages */ uint64_t bytes_written; + /* + * History store cache usage. TODO: The values for these variables are cached and potentially + * outdated. + */ + uint64_t bytes_hs; /* History store bytes inmem */ + uint64_t bytes_hs_dirty; /* History store bytes inmem dirty */ + uint64_t pages_dirty_intl; uint64_t pages_dirty_leaf; uint64_t pages_evicted; diff --git a/src/third_party/wiredtiger/src/include/cache_inline.h b/src/third_party/wiredtiger/src/include/cache_inline.h index 4061d5b52f6..4015196fd6d 100644 --- a/src/third_party/wiredtiger/src/include/cache_inline.h +++ b/src/third_party/wiredtiger/src/include/cache_inline.h @@ -446,6 +446,24 @@ __wt_cache_full(WT_SESSION_IMPL *session) } /* + * __wt_cache_hs_dirty -- + * Return if a major portion of the cache is dirty due to history store content. + */ +static inline bool +__wt_cache_hs_dirty(WT_SESSION_IMPL *session) +{ + WT_CACHE *cache; + WT_CONNECTION_IMPL *conn; + uint64_t bytes_max; + conn = S2C(session); + cache = conn->cache; + bytes_max = S2C(session)->cache_size; + + return (__wt_cache_bytes_plus_overhead(cache, cache->bytes_hs_dirty) >= + ((uint64_t)(cache->eviction_dirty_trigger * bytes_max) / 100)); +} + +/* * __wt_cache_eviction_check -- * Evict pages if the cache crosses its boundaries. */ diff --git a/src/third_party/wiredtiger/src/include/config.h b/src/third_party/wiredtiger/src/include/config.h index 70f4fdef9a6..8b405657f28 100644 --- a/src/third_party/wiredtiger/src/include/config.h +++ b/src/third_party/wiredtiger/src/include/config.h @@ -99,10 +99,11 @@ struct __wt_config_parser_impl { #define WT_CONFIG_ENTRY_index_meta 45 #define WT_CONFIG_ENTRY_lsm_meta 46 #define WT_CONFIG_ENTRY_table_meta 47 -#define WT_CONFIG_ENTRY_wiredtiger_open 48 -#define WT_CONFIG_ENTRY_wiredtiger_open_all 49 -#define WT_CONFIG_ENTRY_wiredtiger_open_basecfg 50 -#define WT_CONFIG_ENTRY_wiredtiger_open_usercfg 51 +#define WT_CONFIG_ENTRY_tiered_meta 48 +#define WT_CONFIG_ENTRY_wiredtiger_open 49 +#define WT_CONFIG_ENTRY_wiredtiger_open_all 50 +#define WT_CONFIG_ENTRY_wiredtiger_open_basecfg 51 +#define WT_CONFIG_ENTRY_wiredtiger_open_usercfg 52 /* * configuration section: END * DO NOT EDIT: automatically built by dist/flags.py. diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index d52fe680d81..f62570a8041 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -508,11 +508,12 @@ struct __wt_connection_impl { #define WT_VERB_SPLIT 0x0020000000u #define WT_VERB_TEMPORARY 0x0040000000u #define WT_VERB_THREAD_GROUP 0x0080000000u -#define WT_VERB_TIMESTAMP 0x0100000000u -#define WT_VERB_TRANSACTION 0x0200000000u -#define WT_VERB_VERIFY 0x0400000000u -#define WT_VERB_VERSION 0x0800000000u -#define WT_VERB_WRITE 0x1000000000u +#define WT_VERB_TIERED 0x0100000000u +#define WT_VERB_TIMESTAMP 0x0200000000u +#define WT_VERB_TRANSACTION 0x0400000000u +#define WT_VERB_VERIFY 0x0800000000u +#define WT_VERB_VERSION 0x1000000000u +#define WT_VERB_WRITE 0x2000000000u /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint64_t verbose; diff --git a/src/third_party/wiredtiger/src/include/dhandle.h b/src/third_party/wiredtiger/src/include/dhandle.h index 9f98cf87e3f..12f173d3802 100644 --- a/src/third_party/wiredtiger/src/include/dhandle.h +++ b/src/third_party/wiredtiger/src/include/dhandle.h @@ -85,7 +85,7 @@ struct __wt_data_handle { WT_DATA_SOURCE *dsrc; /* Data source for this handle */ void *handle; /* Generic handle */ - enum { WT_DHANDLE_TYPE_BTREE, WT_DHANDLE_TYPE_TABLE } type; + enum { WT_DHANDLE_TYPE_BTREE, WT_DHANDLE_TYPE_TABLE, WT_DHANDLE_TYPE_TIERED } type; bool compact_skip; /* If the handle failed to compact */ diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 83327b021ee..68e636cdebf 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -85,12 +85,13 @@ extern int __wt_block_addr_invalid(WT_SESSION_IMPL *session, WT_BLOCK *block, co size_t addr_size, bool live) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_addr_string(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_block_addr_to_buffer(WT_BLOCK *block, uint8_t **pp, wt_off_t offset, uint32_t size, - uint32_t checksum) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_addr_to_buffer(WT_BLOCK *block, uint8_t **pp, uint32_t logid, wt_off_t offset, + uint32_t size, uint32_t checksum) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_alloc(WT_SESSION_IMPL *session, WT_BLOCK *block, wt_off_t *offp, wt_off_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_block_buffer_to_addr(WT_BLOCK *block, const uint8_t *p, wt_off_t *offsetp, - uint32_t *sizep, uint32_t *checksump) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_buffer_to_addr(WT_BLOCK *block, const uint8_t *p, uint32_t *logidp, + wt_off_t *offsetp, uint32_t *sizep, uint32_t *checksump) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_buffer_to_ckpt(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *p, WT_BLOCK_CKPT *ci) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_checkpoint(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, @@ -108,7 +109,7 @@ extern int __wt_block_checkpoint_start(WT_SESSION_IMPL *session, WT_BLOCK *block WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_checkpoint_unload(WT_SESSION_IMPL *session, WT_BLOCK *block, bool checkpoint) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_block_ckpt_decode(WT_SESSION *wt_session, size_t allocsize, const uint8_t *p, +extern int __wt_block_ckpt_decode(WT_SESSION *wt_session, WT_BLOCK *block, const uint8_t *p, WT_BLOCK_CKPT *ci) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_ckpt_init(WT_SESSION_IMPL *session, WT_BLOCK_CKPT *ci, const char *name) @@ -150,6 +151,8 @@ extern int __wt_block_extlist_truncate(WT_SESSION_IMPL *session, WT_BLOCK *block WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_extlist_write(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_EXTLIST *el, WT_EXTLIST *additional) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_fh(WT_SESSION_IMPL *session, WT_BLOCK *block, uint32_t logid, WT_FH **fhp) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_free(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_insert_ext(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_EXTLIST *el, @@ -170,15 +173,17 @@ extern int __wt_block_map(WT_SESSION_IMPL *session, WT_BLOCK *block, void *mappe extern int __wt_block_misplaced(WT_SESSION_IMPL *session, WT_BLOCK *block, const char *list, wt_off_t offset, uint32_t size, bool live, const char *func, int line) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_block_off_free(WT_SESSION_IMPL *session, WT_BLOCK *block, wt_off_t offset, - wt_off_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_newfile(WT_SESSION_IMPL *session, WT_BLOCK *block) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_off_free(WT_SESSION_IMPL *session, WT_BLOCK *block, uint32_t logid, + wt_off_t offset, wt_off_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_off_remove_overlap(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_EXTLIST *el, wt_off_t off, wt_off_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_open(WT_SESSION_IMPL *session, const char *filename, const char *cfg[], bool forced_salvage, bool readonly, uint32_t allocsize, WT_BLOCK **blockp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, - wt_off_t offset, uint32_t size, uint32_t checksum) + uint32_t logid, wt_off_t offset, uint32_t size, uint32_t checksum) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_read_off_blind(WT_SESSION_IMPL *session, WT_BLOCK *block, wt_off_t offset, uint32_t *sizep, uint32_t *checksump) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -206,8 +211,8 @@ extern int __wt_block_write(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM * size_t *addr_sizep, bool data_checksum, bool checkpoint_io) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, - wt_off_t *offsetp, uint32_t *sizep, uint32_t *checksump, bool data_checksum, bool checkpoint_io, - bool caller_locked) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); + uint32_t *logidp, wt_off_t *offsetp, uint32_t *sizep, uint32_t *checksump, bool data_checksum, + bool checkpoint_io, bool caller_locked) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_write_size(WT_SESSION_IMPL *session, WT_BLOCK *block, size_t *sizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bloom_close(WT_BLOOM *bloom) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) @@ -578,6 +583,9 @@ extern int __wt_curtable_get_value(WT_CURSOR *cursor, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_curtable_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_curtiered_close(WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_curtiered_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, + const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_debug_addr(WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size, const char *ofile) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_debug_addr_print(WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size) @@ -834,7 +842,8 @@ extern int __wt_log_remove(WT_SESSION_IMPL *session, const char *file_prefix, ui WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_log_reset(WT_SESSION_IMPL *session, uint32_t lognum) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, +extern int __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *start_lsnp, WT_LSN *end_lsnp, + uint32_t flags, int (*func)(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, WT_LSN *next_lsnp, void *cookie, int firstrecord), void *cookie) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -1432,6 +1441,22 @@ extern int __wt_thread_group_destroy(WT_SESSION_IMPL *session, WT_THREAD_GROUP * extern int __wt_thread_group_resize(WT_SESSION_IMPL *session, WT_THREAD_GROUP *group, uint32_t new_min, uint32_t new_max, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_tiered_close(WT_SESSION_IMPL *session, WT_TIERED *tiered) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_tiered_create(WT_SESSION_IMPL *session, const char *uri, bool exclusive, + const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_tiered_drop(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_tiered_open(WT_SESSION_IMPL *session, const char *cfg[]) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_tiered_rename(WT_SESSION_IMPL *session, const char *olduri, const char *newuri, + const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_tiered_truncate(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_tiered_worker(WT_SESSION_IMPL *session, const char *uri, + int (*file_func)(WT_SESSION_IMPL *, const char *[]), + int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), const char *cfg[], uint32_t open_flags) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_time_aggregate_validate(WT_SESSION_IMPL *session, WT_TIME_AGGREGATE *ta, WT_TIME_AGGREGATE *parent, bool silent) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_time_value_validate(WT_SESSION_IMPL *session, WT_TIME_WINDOW *tw, @@ -1497,9 +1522,9 @@ extern int __wt_txn_parse_timestamp_raw(WT_SESSION_IMPL *session, const char *na wt_timestamp_t *timestamp, WT_CONFIG_ITEM *cval) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_prepare(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_txn_printlog(WT_SESSION *wt_session, const char *ofile, uint32_t flags) - WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) - WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_txn_printlog(WT_SESSION *wt_session, const char *ofile, uint32_t flags, + WT_LSN *start_lsn, WT_LSN *end_lsn) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_query_timestamp(WT_SESSION_IMPL *session, char *hex_timestamp, const char *cfg[], bool global_txn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_reconfigure(WT_SESSION_IMPL *session, const char *config) @@ -1800,6 +1825,8 @@ static inline bool __wt_cache_aggressive(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline bool __wt_cache_full(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +static inline bool __wt_cache_hs_dirty(WT_SESSION_IMPL *session) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline bool __wt_cache_stuck(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline bool __wt_checksum_match(const void *chunk, size_t len, uint32_t v) diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index dcac8f3e569..e1c6cea488a 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -400,6 +400,7 @@ struct __wt_connection_stats { int64_t cache_eviction_pages_queued_post_lru; int64_t cache_eviction_pages_queued_urgent; int64_t cache_eviction_pages_queued_oldest; + int64_t cache_eviction_pages_queued_urgent_hs_dirty; int64_t cache_eviction_pages_already_queued; int64_t cache_eviction_fail; int64_t cache_eviction_fail_parent_has_overflow_items; @@ -697,6 +698,7 @@ struct __wt_connection_stats { int64_t cache_eviction_target_page_ge128; int64_t cache_eviction_target_page_lt64; int64_t cache_eviction_target_page_lt128; + int64_t cache_eviction_target_page_reduced; int64_t cache_eviction_walks_abandoned; int64_t cache_eviction_walks_stopped; int64_t cache_eviction_walks_gave_up_no_targets; @@ -909,6 +911,7 @@ struct __wt_dsrc_stats { int64_t cache_eviction_target_page_ge128; int64_t cache_eviction_target_page_lt64; int64_t cache_eviction_target_page_lt128; + int64_t cache_eviction_target_page_reduced; int64_t cache_eviction_walks_abandoned; int64_t cache_eviction_walks_stopped; int64_t cache_eviction_walks_gave_up_no_targets; diff --git a/src/third_party/wiredtiger/src/include/tiered.h b/src/third_party/wiredtiger/src/include/tiered.h new file mode 100644 index 00000000000..ac4343661c5 --- /dev/null +++ b/src/third_party/wiredtiger/src/include/tiered.h @@ -0,0 +1,44 @@ +/*- + * Copyright (c) 2014-2020 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +/* + * WT_CURSOR_TIERED -- + * An tiered cursor. + */ +struct __wt_cursor_tiered { + WT_CURSOR iface; + + WT_TIERED *tiered; + + WT_CURSOR **cursors; + WT_CURSOR *current; /* The current cursor for iteration */ + WT_CURSOR *primary; /* The current primary */ + +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_CURTIERED_ACTIVE 0x1u /* Incremented the session count */ +#define WT_CURTIERED_ITERATE_NEXT 0x2u /* Forward iteration */ +#define WT_CURTIERED_ITERATE_PREV 0x4u /* Backward iteration */ + /* AUTOMATIC FLAG VALUE GENERATION STOP */ + uint32_t flags; +}; + +/* + * WT_TIERED -- + * Handle for a tiered data source. + */ +struct __wt_tiered { + WT_DATA_HANDLE iface; + + const char *name, *config, *filename; + const char *key_format, *value_format; + + WT_DATA_HANDLE **tiers; + u_int ntiers; + + WT_COLLATOR *collator; /* TODO: handle custom collation */ +}; diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 40d7f5ea6a5..d59138e1bb0 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -1002,6 +1002,8 @@ struct __wt_session { * object blocks from the system buffer cache after that many bytes from this object are * read or written into the buffer cache., an integer greater than or equal to 0; default \c * 0.} + * @config{readonly, the file is read-only. All methods that may modify a file are + * disabled. See @ref readonly for more information., a boolean flag; default \c false.} * @config{verbose, enable messages for various events. Options are given as a list\, such * as <code>"verbose=[write_timestamp]"</code>., a list\, with values chosen from the * following options: \c "write_timestamp"; default \c [].} @@ -1050,10 +1052,12 @@ struct __wt_session { * an integer between 512B and 128MB; default \c 4KB.} * @config{app_metadata, application-owned metadata for this object., a string; default * empty.} - * @config{block_allocation, configure block allocation. Permitted values are \c "first" or - * \c "best"; the \c "first" configuration uses a first-available algorithm during block - * allocation\, the \c "best" configuration uses a best-fit algorithm., a string\, chosen - * from the following options: \c "first"\, \c "best"; default \c best.} + * @config{block_allocation, configure block allocation. Permitted values are \c "best" or + * \c "first"; the \c "best" configuration uses a best-fit algorithm\, the \c "first" + * configuration uses a first-available algorithm during block allocation\, the \c + * "log-structure" configuration allocates a new file for each checkpoint., a string\, + * chosen from the following options: \c "best"\, \c "first"\, \c "log-structured"; default + * \c best.} * @config{block_compressor, configure a compressor for file blocks. Permitted values are * \c "none" or custom compression engine name created with WT_CONNECTION::add_compressor. * If WiredTiger has builtin support for \c "lz4"\, \c "snappy"\, \c "zlib" or \c "zstd" @@ -1104,6 +1108,8 @@ struct __wt_session { * \c none.} * @config{format, the file format., a string\, chosen from the following options: \c * "btree"; default \c btree.} + * @config{huffman_key, This option is no longer supported. Retained for backward + * compatibility. See @ref huffman for more information., a string; default \c none.} * @config{huffman_value, configure Huffman encoding for values. Permitted values are \c * "none"\, \c "english"\, \c "utf8<file>" or \c "utf16<file>". See @ref huffman for more * information., a string; default \c none.} @@ -1230,10 +1236,21 @@ struct __wt_session { * boolean flag; default \c false.} * @config{prefix_compression_min, minimum gain before prefix compression will be used on * row-store leaf pages., an integer greater than or equal to 0; default \c 4.} + * @config{readonly, the file is read-only. All methods that may modify a file are + * disabled. See @ref readonly for more information., a boolean flag; default \c false.} * @config{split_pct, the Btree page split size as a percentage of the maximum Btree page * size\, that is\, when a Btree page is split\, it will be split into smaller pages\, where * each page is the specified percentage of the maximum Btree page size., an integer between * 50 and 100; default \c 90.} + * @config{tiered = (, options only relevant for tiered data sources., a set of related + * configuration options defined below.} + * @config{ chunk_size, the + * maximum size of the hot chunk of tiered tree. This limit is soft - it is possible for + * chunks to be temporarily larger than this value., an integer greater than or equal to 1M; + * default \c 1GB.} + * @config{ tiers, list of data sources to combine + * into a tiered storage structure., a list of strings; default empty.} + * @config{ ),,} * @config{type, set the type of data source used to store a column group\, index or simple * table. By default\, a \c "file:" URI is derived from the object name. The \c type * configuration can be used to switch to a different data source\, such as LSM or an @@ -4930,930 +4947,940 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_URGENT 1085 /*! cache: pages queued for urgent eviction during walk */ #define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_OLDEST 1086 +/*! + * cache: pages queued for urgent eviction from history store due to high + * dirty content + */ +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_URGENT_HS_DIRTY 1087 /*! cache: pages seen by eviction walk that are already queued */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_ALREADY_QUEUED 1087 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_ALREADY_QUEUED 1088 /*! cache: pages selected for eviction unable to be evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL 1088 +#define WT_STAT_CONN_CACHE_EVICTION_FAIL 1089 /*! * cache: pages selected for eviction unable to be evicted as the parent * page has overflow items */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL_PARENT_HAS_OVERFLOW_ITEMS 1089 +#define WT_STAT_CONN_CACHE_EVICTION_FAIL_PARENT_HAS_OVERFLOW_ITEMS 1090 /*! * cache: pages selected for eviction unable to be evicted because of * active children on an internal page */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL_ACTIVE_CHILDREN_ON_AN_INTERNAL_PAGE 1090 +#define WT_STAT_CONN_CACHE_EVICTION_FAIL_ACTIVE_CHILDREN_ON_AN_INTERNAL_PAGE 1091 /*! * cache: pages selected for eviction unable to be evicted because of * failure in reconciliation */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL_IN_RECONCILIATION 1091 +#define WT_STAT_CONN_CACHE_EVICTION_FAIL_IN_RECONCILIATION 1092 /*! cache: pages walked for eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK 1092 +#define WT_STAT_CONN_CACHE_EVICTION_WALK 1093 /*! cache: percentage overhead */ -#define WT_STAT_CONN_CACHE_OVERHEAD 1093 +#define WT_STAT_CONN_CACHE_OVERHEAD 1094 /*! cache: tracked bytes belonging to internal pages in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_INTERNAL 1094 +#define WT_STAT_CONN_CACHE_BYTES_INTERNAL 1095 /*! cache: tracked bytes belonging to leaf pages in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_LEAF 1095 +#define WT_STAT_CONN_CACHE_BYTES_LEAF 1096 /*! cache: tracked dirty pages in the cache */ -#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1096 +#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1097 /*! capacity: background fsync file handles considered */ -#define WT_STAT_CONN_FSYNC_ALL_FH_TOTAL 1097 +#define WT_STAT_CONN_FSYNC_ALL_FH_TOTAL 1098 /*! capacity: background fsync file handles synced */ -#define WT_STAT_CONN_FSYNC_ALL_FH 1098 +#define WT_STAT_CONN_FSYNC_ALL_FH 1099 /*! capacity: background fsync time (msecs) */ -#define WT_STAT_CONN_FSYNC_ALL_TIME 1099 +#define WT_STAT_CONN_FSYNC_ALL_TIME 1100 /*! capacity: bytes read */ -#define WT_STAT_CONN_CAPACITY_BYTES_READ 1100 +#define WT_STAT_CONN_CAPACITY_BYTES_READ 1101 /*! capacity: bytes written for checkpoint */ -#define WT_STAT_CONN_CAPACITY_BYTES_CKPT 1101 +#define WT_STAT_CONN_CAPACITY_BYTES_CKPT 1102 /*! capacity: bytes written for eviction */ -#define WT_STAT_CONN_CAPACITY_BYTES_EVICT 1102 +#define WT_STAT_CONN_CAPACITY_BYTES_EVICT 1103 /*! capacity: bytes written for log */ -#define WT_STAT_CONN_CAPACITY_BYTES_LOG 1103 +#define WT_STAT_CONN_CAPACITY_BYTES_LOG 1104 /*! capacity: bytes written total */ -#define WT_STAT_CONN_CAPACITY_BYTES_WRITTEN 1104 +#define WT_STAT_CONN_CAPACITY_BYTES_WRITTEN 1105 /*! capacity: threshold to call fsync */ -#define WT_STAT_CONN_CAPACITY_THRESHOLD 1105 +#define WT_STAT_CONN_CAPACITY_THRESHOLD 1106 /*! capacity: time waiting due to total capacity (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_TOTAL 1106 +#define WT_STAT_CONN_CAPACITY_TIME_TOTAL 1107 /*! capacity: time waiting during checkpoint (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_CKPT 1107 +#define WT_STAT_CONN_CAPACITY_TIME_CKPT 1108 /*! capacity: time waiting during eviction (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_EVICT 1108 +#define WT_STAT_CONN_CAPACITY_TIME_EVICT 1109 /*! capacity: time waiting during logging (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_LOG 1109 +#define WT_STAT_CONN_CAPACITY_TIME_LOG 1110 /*! capacity: time waiting during read (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_READ 1110 +#define WT_STAT_CONN_CAPACITY_TIME_READ 1111 /*! connection: auto adjusting condition resets */ -#define WT_STAT_CONN_COND_AUTO_WAIT_RESET 1111 +#define WT_STAT_CONN_COND_AUTO_WAIT_RESET 1112 /*! connection: auto adjusting condition wait calls */ -#define WT_STAT_CONN_COND_AUTO_WAIT 1112 +#define WT_STAT_CONN_COND_AUTO_WAIT 1113 /*! * connection: auto adjusting condition wait raced to update timeout and * skipped updating */ -#define WT_STAT_CONN_COND_AUTO_WAIT_SKIPPED 1113 +#define WT_STAT_CONN_COND_AUTO_WAIT_SKIPPED 1114 /*! connection: detected system time went backwards */ -#define WT_STAT_CONN_TIME_TRAVEL 1114 +#define WT_STAT_CONN_TIME_TRAVEL 1115 /*! connection: files currently open */ -#define WT_STAT_CONN_FILE_OPEN 1115 +#define WT_STAT_CONN_FILE_OPEN 1116 /*! connection: hash bucket array size for data handles */ -#define WT_STAT_CONN_BUCKETS_DH 1116 +#define WT_STAT_CONN_BUCKETS_DH 1117 /*! connection: hash bucket array size general */ -#define WT_STAT_CONN_BUCKETS 1117 +#define WT_STAT_CONN_BUCKETS 1118 /*! connection: memory allocations */ -#define WT_STAT_CONN_MEMORY_ALLOCATION 1118 +#define WT_STAT_CONN_MEMORY_ALLOCATION 1119 /*! connection: memory frees */ -#define WT_STAT_CONN_MEMORY_FREE 1119 +#define WT_STAT_CONN_MEMORY_FREE 1120 /*! connection: memory re-allocations */ -#define WT_STAT_CONN_MEMORY_GROW 1120 +#define WT_STAT_CONN_MEMORY_GROW 1121 /*! connection: pthread mutex condition wait calls */ -#define WT_STAT_CONN_COND_WAIT 1121 +#define WT_STAT_CONN_COND_WAIT 1122 /*! connection: pthread mutex shared lock read-lock calls */ -#define WT_STAT_CONN_RWLOCK_READ 1122 +#define WT_STAT_CONN_RWLOCK_READ 1123 /*! connection: pthread mutex shared lock write-lock calls */ -#define WT_STAT_CONN_RWLOCK_WRITE 1123 +#define WT_STAT_CONN_RWLOCK_WRITE 1124 /*! connection: total fsync I/Os */ -#define WT_STAT_CONN_FSYNC_IO 1124 +#define WT_STAT_CONN_FSYNC_IO 1125 /*! connection: total read I/Os */ -#define WT_STAT_CONN_READ_IO 1125 +#define WT_STAT_CONN_READ_IO 1126 /*! connection: total write I/Os */ -#define WT_STAT_CONN_WRITE_IO 1126 +#define WT_STAT_CONN_WRITE_IO 1127 /*! cursor: cached cursor count */ -#define WT_STAT_CONN_CURSOR_CACHED_COUNT 1127 +#define WT_STAT_CONN_CURSOR_CACHED_COUNT 1128 /*! cursor: cursor bulk loaded cursor insert calls */ -#define WT_STAT_CONN_CURSOR_INSERT_BULK 1128 +#define WT_STAT_CONN_CURSOR_INSERT_BULK 1129 /*! cursor: cursor close calls that result in cache */ -#define WT_STAT_CONN_CURSOR_CACHE 1129 +#define WT_STAT_CONN_CURSOR_CACHE 1130 /*! cursor: cursor create calls */ -#define WT_STAT_CONN_CURSOR_CREATE 1130 +#define WT_STAT_CONN_CURSOR_CREATE 1131 /*! cursor: cursor insert calls */ -#define WT_STAT_CONN_CURSOR_INSERT 1131 +#define WT_STAT_CONN_CURSOR_INSERT 1132 /*! cursor: cursor insert key and value bytes */ -#define WT_STAT_CONN_CURSOR_INSERT_BYTES 1132 +#define WT_STAT_CONN_CURSOR_INSERT_BYTES 1133 /*! cursor: cursor modify calls */ -#define WT_STAT_CONN_CURSOR_MODIFY 1133 +#define WT_STAT_CONN_CURSOR_MODIFY 1134 /*! cursor: cursor modify key and value bytes affected */ -#define WT_STAT_CONN_CURSOR_MODIFY_BYTES 1134 +#define WT_STAT_CONN_CURSOR_MODIFY_BYTES 1135 /*! cursor: cursor modify value bytes modified */ -#define WT_STAT_CONN_CURSOR_MODIFY_BYTES_TOUCH 1135 +#define WT_STAT_CONN_CURSOR_MODIFY_BYTES_TOUCH 1136 /*! cursor: cursor next calls */ -#define WT_STAT_CONN_CURSOR_NEXT 1136 +#define WT_STAT_CONN_CURSOR_NEXT 1137 /*! * cursor: cursor next calls that skip due to a globally visible history * store tombstone in rollback to stable */ -#define WT_STAT_CONN_CURSOR_NEXT_HS_TOMBSTONE_RTS 1137 +#define WT_STAT_CONN_CURSOR_NEXT_HS_TOMBSTONE_RTS 1138 /*! cursor: cursor operation restarted */ -#define WT_STAT_CONN_CURSOR_RESTART 1138 +#define WT_STAT_CONN_CURSOR_RESTART 1139 /*! cursor: cursor prev calls */ -#define WT_STAT_CONN_CURSOR_PREV 1139 +#define WT_STAT_CONN_CURSOR_PREV 1140 /*! * cursor: cursor prev calls that skip due to a globally visible history * store tombstone in rollback to stable */ -#define WT_STAT_CONN_CURSOR_PREV_HS_TOMBSTONE_RTS 1140 +#define WT_STAT_CONN_CURSOR_PREV_HS_TOMBSTONE_RTS 1141 /*! cursor: cursor remove calls */ -#define WT_STAT_CONN_CURSOR_REMOVE 1141 +#define WT_STAT_CONN_CURSOR_REMOVE 1142 /*! cursor: cursor remove key bytes removed */ -#define WT_STAT_CONN_CURSOR_REMOVE_BYTES 1142 +#define WT_STAT_CONN_CURSOR_REMOVE_BYTES 1143 /*! cursor: cursor reserve calls */ -#define WT_STAT_CONN_CURSOR_RESERVE 1143 +#define WT_STAT_CONN_CURSOR_RESERVE 1144 /*! cursor: cursor reset calls */ -#define WT_STAT_CONN_CURSOR_RESET 1144 +#define WT_STAT_CONN_CURSOR_RESET 1145 /*! cursor: cursor search calls */ -#define WT_STAT_CONN_CURSOR_SEARCH 1145 +#define WT_STAT_CONN_CURSOR_SEARCH 1146 /*! cursor: cursor search history store calls */ -#define WT_STAT_CONN_CURSOR_SEARCH_HS 1146 +#define WT_STAT_CONN_CURSOR_SEARCH_HS 1147 /*! cursor: cursor search near calls */ -#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1147 +#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1148 /*! cursor: cursor sweep buckets */ -#define WT_STAT_CONN_CURSOR_SWEEP_BUCKETS 1148 +#define WT_STAT_CONN_CURSOR_SWEEP_BUCKETS 1149 /*! cursor: cursor sweep cursors closed */ -#define WT_STAT_CONN_CURSOR_SWEEP_CLOSED 1149 +#define WT_STAT_CONN_CURSOR_SWEEP_CLOSED 1150 /*! cursor: cursor sweep cursors examined */ -#define WT_STAT_CONN_CURSOR_SWEEP_EXAMINED 1150 +#define WT_STAT_CONN_CURSOR_SWEEP_EXAMINED 1151 /*! cursor: cursor sweeps */ -#define WT_STAT_CONN_CURSOR_SWEEP 1151 +#define WT_STAT_CONN_CURSOR_SWEEP 1152 /*! cursor: cursor truncate calls */ -#define WT_STAT_CONN_CURSOR_TRUNCATE 1152 +#define WT_STAT_CONN_CURSOR_TRUNCATE 1153 /*! cursor: cursor update calls */ -#define WT_STAT_CONN_CURSOR_UPDATE 1153 +#define WT_STAT_CONN_CURSOR_UPDATE 1154 /*! cursor: cursor update key and value bytes */ -#define WT_STAT_CONN_CURSOR_UPDATE_BYTES 1154 +#define WT_STAT_CONN_CURSOR_UPDATE_BYTES 1155 /*! cursor: cursor update value size change */ -#define WT_STAT_CONN_CURSOR_UPDATE_BYTES_CHANGED 1155 +#define WT_STAT_CONN_CURSOR_UPDATE_BYTES_CHANGED 1156 /*! cursor: cursors reused from cache */ -#define WT_STAT_CONN_CURSOR_REOPEN 1156 +#define WT_STAT_CONN_CURSOR_REOPEN 1157 /*! data-handle: connection data handle size */ -#define WT_STAT_CONN_DH_CONN_HANDLE_SIZE 1157 +#define WT_STAT_CONN_DH_CONN_HANDLE_SIZE 1158 /*! data-handle: connection data handles currently active */ -#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1158 +#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1159 /*! data-handle: connection sweep candidate became referenced */ -#define WT_STAT_CONN_DH_SWEEP_REF 1159 +#define WT_STAT_CONN_DH_SWEEP_REF 1160 /*! data-handle: connection sweep dhandles closed */ -#define WT_STAT_CONN_DH_SWEEP_CLOSE 1160 +#define WT_STAT_CONN_DH_SWEEP_CLOSE 1161 /*! data-handle: connection sweep dhandles removed from hash list */ -#define WT_STAT_CONN_DH_SWEEP_REMOVE 1161 +#define WT_STAT_CONN_DH_SWEEP_REMOVE 1162 /*! data-handle: connection sweep time-of-death sets */ -#define WT_STAT_CONN_DH_SWEEP_TOD 1162 +#define WT_STAT_CONN_DH_SWEEP_TOD 1163 /*! data-handle: connection sweeps */ -#define WT_STAT_CONN_DH_SWEEPS 1163 +#define WT_STAT_CONN_DH_SWEEPS 1164 /*! data-handle: session dhandles swept */ -#define WT_STAT_CONN_DH_SESSION_HANDLES 1164 +#define WT_STAT_CONN_DH_SESSION_HANDLES 1165 /*! data-handle: session sweep attempts */ -#define WT_STAT_CONN_DH_SESSION_SWEEPS 1165 +#define WT_STAT_CONN_DH_SESSION_SWEEPS 1166 /*! lock: checkpoint lock acquisitions */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1166 +#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1167 /*! lock: checkpoint lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1167 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1168 /*! lock: checkpoint lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1168 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1169 /*! lock: dhandle lock application thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_APPLICATION 1169 +#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_APPLICATION 1170 /*! lock: dhandle lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_INTERNAL 1170 +#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_INTERNAL 1171 /*! lock: dhandle read lock acquisitions */ -#define WT_STAT_CONN_LOCK_DHANDLE_READ_COUNT 1171 +#define WT_STAT_CONN_LOCK_DHANDLE_READ_COUNT 1172 /*! lock: dhandle write lock acquisitions */ -#define WT_STAT_CONN_LOCK_DHANDLE_WRITE_COUNT 1172 +#define WT_STAT_CONN_LOCK_DHANDLE_WRITE_COUNT 1173 /*! * lock: durable timestamp queue lock application thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_APPLICATION 1173 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_APPLICATION 1174 /*! * lock: durable timestamp queue lock internal thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_INTERNAL 1174 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_INTERNAL 1175 /*! lock: durable timestamp queue read lock acquisitions */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_READ_COUNT 1175 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_READ_COUNT 1176 /*! lock: durable timestamp queue write lock acquisitions */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WRITE_COUNT 1176 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WRITE_COUNT 1177 /*! lock: metadata lock acquisitions */ -#define WT_STAT_CONN_LOCK_METADATA_COUNT 1177 +#define WT_STAT_CONN_LOCK_METADATA_COUNT 1178 /*! lock: metadata lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1178 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1179 /*! lock: metadata lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1179 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1180 /*! * lock: read timestamp queue lock application thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_APPLICATION 1180 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_APPLICATION 1181 /*! lock: read timestamp queue lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_INTERNAL 1181 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_INTERNAL 1182 /*! lock: read timestamp queue read lock acquisitions */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_READ_COUNT 1182 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_READ_COUNT 1183 /*! lock: read timestamp queue write lock acquisitions */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WRITE_COUNT 1183 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WRITE_COUNT 1184 /*! lock: schema lock acquisitions */ -#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1184 +#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1185 /*! lock: schema lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1185 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1186 /*! lock: schema lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1186 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1187 /*! * lock: table lock application thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1187 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1188 /*! * lock: table lock internal thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1188 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1189 /*! lock: table read lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_READ_COUNT 1189 +#define WT_STAT_CONN_LOCK_TABLE_READ_COUNT 1190 /*! lock: table write lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_WRITE_COUNT 1190 +#define WT_STAT_CONN_LOCK_TABLE_WRITE_COUNT 1191 /*! lock: txn global lock application thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_APPLICATION 1191 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_APPLICATION 1192 /*! lock: txn global lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_INTERNAL 1192 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_INTERNAL 1193 /*! lock: txn global read lock acquisitions */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_READ_COUNT 1193 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_READ_COUNT 1194 /*! lock: txn global write lock acquisitions */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WRITE_COUNT 1194 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WRITE_COUNT 1195 /*! log: busy returns attempting to switch slots */ -#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1195 +#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1196 /*! log: force archive time sleeping (usecs) */ -#define WT_STAT_CONN_LOG_FORCE_ARCHIVE_SLEEP 1196 +#define WT_STAT_CONN_LOG_FORCE_ARCHIVE_SLEEP 1197 /*! log: log bytes of payload data */ -#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1197 +#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1198 /*! log: log bytes written */ -#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1198 +#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1199 /*! log: log files manually zero-filled */ -#define WT_STAT_CONN_LOG_ZERO_FILLS 1199 +#define WT_STAT_CONN_LOG_ZERO_FILLS 1200 /*! log: log flush operations */ -#define WT_STAT_CONN_LOG_FLUSH 1200 +#define WT_STAT_CONN_LOG_FLUSH 1201 /*! log: log force write operations */ -#define WT_STAT_CONN_LOG_FORCE_WRITE 1201 +#define WT_STAT_CONN_LOG_FORCE_WRITE 1202 /*! log: log force write operations skipped */ -#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1202 +#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1203 /*! log: log records compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1203 +#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1204 /*! log: log records not compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1204 +#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1205 /*! log: log records too small to compress */ -#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1205 +#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1206 /*! log: log release advances write LSN */ -#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1206 +#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1207 /*! log: log scan operations */ -#define WT_STAT_CONN_LOG_SCANS 1207 +#define WT_STAT_CONN_LOG_SCANS 1208 /*! log: log scan records requiring two reads */ -#define WT_STAT_CONN_LOG_SCAN_REREADS 1208 +#define WT_STAT_CONN_LOG_SCAN_REREADS 1209 /*! log: log server thread advances write LSN */ -#define WT_STAT_CONN_LOG_WRITE_LSN 1209 +#define WT_STAT_CONN_LOG_WRITE_LSN 1210 /*! log: log server thread write LSN walk skipped */ -#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1210 +#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1211 /*! log: log sync operations */ -#define WT_STAT_CONN_LOG_SYNC 1211 +#define WT_STAT_CONN_LOG_SYNC 1212 /*! log: log sync time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DURATION 1212 +#define WT_STAT_CONN_LOG_SYNC_DURATION 1213 /*! log: log sync_dir operations */ -#define WT_STAT_CONN_LOG_SYNC_DIR 1213 +#define WT_STAT_CONN_LOG_SYNC_DIR 1214 /*! log: log sync_dir time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1214 +#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1215 /*! log: log write operations */ -#define WT_STAT_CONN_LOG_WRITES 1215 +#define WT_STAT_CONN_LOG_WRITES 1216 /*! log: logging bytes consolidated */ -#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1216 +#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1217 /*! log: maximum log file size */ -#define WT_STAT_CONN_LOG_MAX_FILESIZE 1217 +#define WT_STAT_CONN_LOG_MAX_FILESIZE 1218 /*! log: number of pre-allocated log files to create */ -#define WT_STAT_CONN_LOG_PREALLOC_MAX 1218 +#define WT_STAT_CONN_LOG_PREALLOC_MAX 1219 /*! log: pre-allocated log files not ready and missed */ -#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1219 +#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1220 /*! log: pre-allocated log files prepared */ -#define WT_STAT_CONN_LOG_PREALLOC_FILES 1220 +#define WT_STAT_CONN_LOG_PREALLOC_FILES 1221 /*! log: pre-allocated log files used */ -#define WT_STAT_CONN_LOG_PREALLOC_USED 1221 +#define WT_STAT_CONN_LOG_PREALLOC_USED 1222 /*! log: records processed by log scan */ -#define WT_STAT_CONN_LOG_SCAN_RECORDS 1222 +#define WT_STAT_CONN_LOG_SCAN_RECORDS 1223 /*! log: slot close lost race */ -#define WT_STAT_CONN_LOG_SLOT_CLOSE_RACE 1223 +#define WT_STAT_CONN_LOG_SLOT_CLOSE_RACE 1224 /*! log: slot close unbuffered waits */ -#define WT_STAT_CONN_LOG_SLOT_CLOSE_UNBUF 1224 +#define WT_STAT_CONN_LOG_SLOT_CLOSE_UNBUF 1225 /*! log: slot closures */ -#define WT_STAT_CONN_LOG_SLOT_CLOSES 1225 +#define WT_STAT_CONN_LOG_SLOT_CLOSES 1226 /*! log: slot join atomic update races */ -#define WT_STAT_CONN_LOG_SLOT_RACES 1226 +#define WT_STAT_CONN_LOG_SLOT_RACES 1227 /*! log: slot join calls atomic updates raced */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_RACE 1227 +#define WT_STAT_CONN_LOG_SLOT_YIELD_RACE 1228 /*! log: slot join calls did not yield */ -#define WT_STAT_CONN_LOG_SLOT_IMMEDIATE 1228 +#define WT_STAT_CONN_LOG_SLOT_IMMEDIATE 1229 /*! log: slot join calls found active slot closed */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_CLOSE 1229 +#define WT_STAT_CONN_LOG_SLOT_YIELD_CLOSE 1230 /*! log: slot join calls slept */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_SLEEP 1230 +#define WT_STAT_CONN_LOG_SLOT_YIELD_SLEEP 1231 /*! log: slot join calls yielded */ -#define WT_STAT_CONN_LOG_SLOT_YIELD 1231 +#define WT_STAT_CONN_LOG_SLOT_YIELD 1232 /*! log: slot join found active slot closed */ -#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1232 +#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1233 /*! log: slot joins yield time (usecs) */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_DURATION 1233 +#define WT_STAT_CONN_LOG_SLOT_YIELD_DURATION 1234 /*! log: slot transitions unable to find free slot */ -#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1234 +#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1235 /*! log: slot unbuffered writes */ -#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1235 +#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1236 /*! log: total in-memory size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_MEM 1236 +#define WT_STAT_CONN_LOG_COMPRESS_MEM 1237 /*! log: total log buffer size */ -#define WT_STAT_CONN_LOG_BUFFER_SIZE 1237 +#define WT_STAT_CONN_LOG_BUFFER_SIZE 1238 /*! log: total size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_LEN 1238 +#define WT_STAT_CONN_LOG_COMPRESS_LEN 1239 /*! log: written slots coalesced */ -#define WT_STAT_CONN_LOG_SLOT_COALESCED 1239 +#define WT_STAT_CONN_LOG_SLOT_COALESCED 1240 /*! log: yields waiting for previous log file close */ -#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1240 +#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1241 /*! perf: file system read latency histogram (bucket 1) - 10-49ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT50 1241 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT50 1242 /*! perf: file system read latency histogram (bucket 2) - 50-99ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT100 1242 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT100 1243 /*! perf: file system read latency histogram (bucket 3) - 100-249ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT250 1243 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT250 1244 /*! perf: file system read latency histogram (bucket 4) - 250-499ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT500 1244 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT500 1245 /*! perf: file system read latency histogram (bucket 5) - 500-999ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT1000 1245 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT1000 1246 /*! perf: file system read latency histogram (bucket 6) - 1000ms+ */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_GT1000 1246 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_GT1000 1247 /*! perf: file system write latency histogram (bucket 1) - 10-49ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT50 1247 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT50 1248 /*! perf: file system write latency histogram (bucket 2) - 50-99ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT100 1248 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT100 1249 /*! perf: file system write latency histogram (bucket 3) - 100-249ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT250 1249 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT250 1250 /*! perf: file system write latency histogram (bucket 4) - 250-499ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT500 1250 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT500 1251 /*! perf: file system write latency histogram (bucket 5) - 500-999ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT1000 1251 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT1000 1252 /*! perf: file system write latency histogram (bucket 6) - 1000ms+ */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_GT1000 1252 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_GT1000 1253 /*! perf: operation read latency histogram (bucket 1) - 100-249us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT250 1253 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT250 1254 /*! perf: operation read latency histogram (bucket 2) - 250-499us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT500 1254 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT500 1255 /*! perf: operation read latency histogram (bucket 3) - 500-999us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT1000 1255 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT1000 1256 /*! perf: operation read latency histogram (bucket 4) - 1000-9999us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT10000 1256 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT10000 1257 /*! perf: operation read latency histogram (bucket 5) - 10000us+ */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_GT10000 1257 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_GT10000 1258 /*! perf: operation write latency histogram (bucket 1) - 100-249us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT250 1258 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT250 1259 /*! perf: operation write latency histogram (bucket 2) - 250-499us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT500 1259 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT500 1260 /*! perf: operation write latency histogram (bucket 3) - 500-999us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT1000 1260 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT1000 1261 /*! perf: operation write latency histogram (bucket 4) - 1000-9999us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT10000 1261 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT10000 1262 /*! perf: operation write latency histogram (bucket 5) - 10000us+ */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_GT10000 1262 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_GT10000 1263 /*! reconciliation: maximum seconds spent in a reconciliation call */ -#define WT_STAT_CONN_REC_MAXIMUM_SECONDS 1263 +#define WT_STAT_CONN_REC_MAXIMUM_SECONDS 1264 /*! * reconciliation: page reconciliation calls that resulted in values with * prepared transaction metadata */ -#define WT_STAT_CONN_REC_PAGES_WITH_PREPARE 1264 +#define WT_STAT_CONN_REC_PAGES_WITH_PREPARE 1265 /*! * reconciliation: page reconciliation calls that resulted in values with * timestamps */ -#define WT_STAT_CONN_REC_PAGES_WITH_TS 1265 +#define WT_STAT_CONN_REC_PAGES_WITH_TS 1266 /*! * reconciliation: page reconciliation calls that resulted in values with * transaction ids */ -#define WT_STAT_CONN_REC_PAGES_WITH_TXN 1266 +#define WT_STAT_CONN_REC_PAGES_WITH_TXN 1267 /*! reconciliation: pages written including at least one prepare state */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_PREPARED 1267 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_PREPARED 1268 /*! reconciliation: pages written including at least one start timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TS 1268 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TS 1269 /*! reconciliation: records written including a prepare state */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PREPARED 1269 +#define WT_STAT_CONN_REC_TIME_WINDOW_PREPARED 1270 /*! reconciliation: split bytes currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1270 +#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1271 /*! reconciliation: split objects currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1271 +#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1272 /*! session: open session count */ -#define WT_STAT_CONN_SESSION_OPEN 1272 +#define WT_STAT_CONN_SESSION_OPEN 1273 /*! session: session query timestamp calls */ -#define WT_STAT_CONN_SESSION_QUERY_TS 1273 +#define WT_STAT_CONN_SESSION_QUERY_TS 1274 /*! session: table alter failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1274 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1275 /*! session: table alter successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1275 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1276 /*! session: table alter unchanged and skipped */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1276 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1277 /*! session: table compact failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1277 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1278 /*! session: table compact successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1278 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1279 /*! session: table create failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1279 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1280 /*! session: table create successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1280 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1281 /*! session: table drop failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1281 +#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1282 /*! session: table drop successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1282 +#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1283 /*! session: table rename failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1283 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1284 /*! session: table rename successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1284 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1285 /*! session: table salvage failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1285 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1286 /*! session: table salvage successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1286 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1287 /*! session: table truncate failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1287 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1288 /*! session: table truncate successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1288 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1289 /*! session: table verify failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1289 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1290 /*! session: table verify successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1290 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1291 /*! thread-state: active filesystem fsync calls */ -#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1291 +#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1292 /*! thread-state: active filesystem read calls */ -#define WT_STAT_CONN_THREAD_READ_ACTIVE 1292 +#define WT_STAT_CONN_THREAD_READ_ACTIVE 1293 /*! thread-state: active filesystem write calls */ -#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1293 +#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1294 /*! thread-yield: application thread time evicting (usecs) */ -#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1294 +#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1295 /*! thread-yield: application thread time waiting for cache (usecs) */ -#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1295 +#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1296 /*! * thread-yield: connection close blocked waiting for transaction state * stabilization */ -#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1296 +#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1297 /*! thread-yield: connection close yielded for lsm manager shutdown */ -#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1297 +#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1298 /*! thread-yield: data handle lock yielded */ -#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1298 +#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1299 /*! * thread-yield: get reference for page index and slot time sleeping * (usecs) */ -#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1299 +#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1300 /*! thread-yield: log server sync yielded for log write */ -#define WT_STAT_CONN_LOG_SERVER_SYNC_BLOCKED 1300 +#define WT_STAT_CONN_LOG_SERVER_SYNC_BLOCKED 1301 /*! thread-yield: page access yielded due to prepare state change */ -#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1301 +#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1302 /*! thread-yield: page acquire busy blocked */ -#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1302 +#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1303 /*! thread-yield: page acquire eviction blocked */ -#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1303 +#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1304 /*! thread-yield: page acquire locked blocked */ -#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1304 +#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1305 /*! thread-yield: page acquire read blocked */ -#define WT_STAT_CONN_PAGE_READ_BLOCKED 1305 +#define WT_STAT_CONN_PAGE_READ_BLOCKED 1306 /*! thread-yield: page acquire time sleeping (usecs) */ -#define WT_STAT_CONN_PAGE_SLEEP 1306 +#define WT_STAT_CONN_PAGE_SLEEP 1307 /*! * thread-yield: page delete rollback time sleeping for state change * (usecs) */ -#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1307 +#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1308 /*! thread-yield: page reconciliation yielded due to child modification */ -#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1308 +#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1309 /*! transaction: Number of prepared updates */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COUNT 1309 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COUNT 1310 /*! transaction: durable timestamp queue entries walked */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_WALKED 1310 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_WALKED 1311 /*! transaction: durable timestamp queue insert to empty */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_EMPTY 1311 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_EMPTY 1312 /*! transaction: durable timestamp queue inserts to head */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_HEAD 1312 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_HEAD 1313 /*! transaction: durable timestamp queue inserts total */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_INSERTS 1313 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_INSERTS 1314 /*! transaction: durable timestamp queue length */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_LEN 1314 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_LEN 1315 /*! transaction: prepared transactions */ -#define WT_STAT_CONN_TXN_PREPARE 1315 +#define WT_STAT_CONN_TXN_PREPARE 1316 /*! transaction: prepared transactions committed */ -#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1316 +#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1317 /*! transaction: prepared transactions currently active */ -#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1317 +#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1318 /*! transaction: prepared transactions rolled back */ -#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1318 +#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1319 /*! transaction: query timestamp calls */ -#define WT_STAT_CONN_TXN_QUERY_TS 1319 +#define WT_STAT_CONN_TXN_QUERY_TS 1320 /*! transaction: read timestamp queue entries walked */ -#define WT_STAT_CONN_TXN_READ_QUEUE_WALKED 1320 +#define WT_STAT_CONN_TXN_READ_QUEUE_WALKED 1321 /*! transaction: read timestamp queue insert to empty */ -#define WT_STAT_CONN_TXN_READ_QUEUE_EMPTY 1321 +#define WT_STAT_CONN_TXN_READ_QUEUE_EMPTY 1322 /*! transaction: read timestamp queue inserts to head */ -#define WT_STAT_CONN_TXN_READ_QUEUE_HEAD 1322 +#define WT_STAT_CONN_TXN_READ_QUEUE_HEAD 1323 /*! transaction: read timestamp queue inserts total */ -#define WT_STAT_CONN_TXN_READ_QUEUE_INSERTS 1323 +#define WT_STAT_CONN_TXN_READ_QUEUE_INSERTS 1324 /*! transaction: read timestamp queue length */ -#define WT_STAT_CONN_TXN_READ_QUEUE_LEN 1324 +#define WT_STAT_CONN_TXN_READ_QUEUE_LEN 1325 /*! transaction: rollback to stable calls */ -#define WT_STAT_CONN_TXN_RTS 1325 +#define WT_STAT_CONN_TXN_RTS 1326 /*! transaction: rollback to stable pages visited */ -#define WT_STAT_CONN_TXN_RTS_PAGES_VISITED 1326 +#define WT_STAT_CONN_TXN_RTS_PAGES_VISITED 1327 /*! transaction: rollback to stable tree walk skipping pages */ -#define WT_STAT_CONN_TXN_RTS_TREE_WALK_SKIP_PAGES 1327 +#define WT_STAT_CONN_TXN_RTS_TREE_WALK_SKIP_PAGES 1328 /*! transaction: rollback to stable updates aborted */ -#define WT_STAT_CONN_TXN_RTS_UPD_ABORTED 1328 +#define WT_STAT_CONN_TXN_RTS_UPD_ABORTED 1329 /*! transaction: set timestamp calls */ -#define WT_STAT_CONN_TXN_SET_TS 1329 +#define WT_STAT_CONN_TXN_SET_TS 1330 /*! transaction: set timestamp durable calls */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1330 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1331 /*! transaction: set timestamp durable updates */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1331 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1332 /*! transaction: set timestamp oldest calls */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1332 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1333 /*! transaction: set timestamp oldest updates */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1333 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1334 /*! transaction: set timestamp stable calls */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE 1334 +#define WT_STAT_CONN_TXN_SET_TS_STABLE 1335 /*! transaction: set timestamp stable updates */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1335 +#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1336 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1336 +#define WT_STAT_CONN_TXN_BEGIN 1337 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1337 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1338 /*! transaction: transaction checkpoint generation */ -#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1338 +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1339 /*! * transaction: transaction checkpoint history store file duration * (usecs) */ -#define WT_STAT_CONN_TXN_HS_CKPT_DURATION 1339 +#define WT_STAT_CONN_TXN_HS_CKPT_DURATION 1340 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1340 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1341 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1341 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1342 /*! * transaction: transaction checkpoint most recent duration for gathering * all handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION 1342 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION 1343 /*! * transaction: transaction checkpoint most recent duration for gathering * applied handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_APPLY 1343 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_APPLY 1344 /*! * transaction: transaction checkpoint most recent duration for gathering * skipped handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_SKIP 1344 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_SKIP 1345 /*! transaction: transaction checkpoint most recent handles applied */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_APPLIED 1345 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_APPLIED 1346 /*! transaction: transaction checkpoint most recent handles skipped */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_SKIPPED 1346 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_SKIPPED 1347 /*! transaction: transaction checkpoint most recent handles walked */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_WALKED 1347 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_WALKED 1348 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1348 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1349 /*! transaction: transaction checkpoint prepare currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RUNNING 1349 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RUNNING 1350 /*! transaction: transaction checkpoint prepare max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MAX 1350 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MAX 1351 /*! transaction: transaction checkpoint prepare min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MIN 1351 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MIN 1352 /*! transaction: transaction checkpoint prepare most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RECENT 1352 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RECENT 1353 /*! transaction: transaction checkpoint prepare total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_TOTAL 1353 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_TOTAL 1354 /*! transaction: transaction checkpoint scrub dirty target */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1354 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1355 /*! transaction: transaction checkpoint scrub time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1355 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1356 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1356 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1357 /*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1357 +#define WT_STAT_CONN_TXN_CHECKPOINT 1358 /*! * transaction: transaction checkpoints skipped because database was * clean */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1358 +#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1359 /*! transaction: transaction failures due to history store */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1359 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1360 /*! * transaction: transaction fsync calls for checkpoint after allocating * the transaction ID */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1360 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1361 /*! * transaction: transaction fsync duration for checkpoint after * allocating the transaction ID (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1361 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1362 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1362 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1363 /*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1363 +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1364 /*! transaction: transaction range of timestamps currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1364 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1365 /*! transaction: transaction range of timestamps pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1365 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1366 /*! * transaction: transaction range of timestamps pinned by the oldest * active read timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1366 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1367 /*! * transaction: transaction range of timestamps pinned by the oldest * timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1367 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1368 /*! transaction: transaction read timestamp of the oldest active reader */ -#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1368 +#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1369 /*! transaction: transaction sync calls */ -#define WT_STAT_CONN_TXN_SYNC 1369 +#define WT_STAT_CONN_TXN_SYNC 1370 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1370 +#define WT_STAT_CONN_TXN_COMMIT 1371 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1371 +#define WT_STAT_CONN_TXN_ROLLBACK 1372 /*! LSM: sleep for LSM checkpoint throttle */ -#define WT_STAT_CONN_LSM_CHECKPOINT_THROTTLE 1372 +#define WT_STAT_CONN_LSM_CHECKPOINT_THROTTLE 1373 /*! LSM: sleep for LSM merge throttle */ -#define WT_STAT_CONN_LSM_MERGE_THROTTLE 1373 +#define WT_STAT_CONN_LSM_MERGE_THROTTLE 1374 /*! cache: bytes currently in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_INUSE 1374 +#define WT_STAT_CONN_CACHE_BYTES_INUSE 1375 /*! cache: bytes dirty in the cache cumulative */ -#define WT_STAT_CONN_CACHE_BYTES_DIRTY_TOTAL 1375 +#define WT_STAT_CONN_CACHE_BYTES_DIRTY_TOTAL 1376 /*! cache: bytes read into cache */ -#define WT_STAT_CONN_CACHE_BYTES_READ 1376 +#define WT_STAT_CONN_CACHE_BYTES_READ 1377 /*! cache: bytes written from cache */ -#define WT_STAT_CONN_CACHE_BYTES_WRITE 1377 +#define WT_STAT_CONN_CACHE_BYTES_WRITE 1378 /*! cache: checkpoint blocked page eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_CHECKPOINT 1378 +#define WT_STAT_CONN_CACHE_EVICTION_CHECKPOINT 1379 /*! cache: eviction walk target pages histogram - 0-9 */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT10 1379 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT10 1380 /*! cache: eviction walk target pages histogram - 10-31 */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT32 1380 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT32 1381 /*! cache: eviction walk target pages histogram - 128 and higher */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_GE128 1381 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_GE128 1382 /*! cache: eviction walk target pages histogram - 32-63 */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT64 1382 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT64 1383 /*! cache: eviction walk target pages histogram - 64-128 */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT128 1383 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT128 1384 +/*! + * cache: eviction walk target pages reduced due to history store cache + * pressure + */ +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_REDUCED 1385 /*! cache: eviction walks abandoned */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ABANDONED 1384 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ABANDONED 1386 /*! cache: eviction walks gave up because they restarted their walk twice */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_STOPPED 1385 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_STOPPED 1387 /*! * cache: eviction walks gave up because they saw too many pages and * found no candidates */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_GAVE_UP_NO_TARGETS 1386 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_GAVE_UP_NO_TARGETS 1388 /*! * cache: eviction walks gave up because they saw too many pages and * found too few candidates */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_GAVE_UP_RATIO 1387 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_GAVE_UP_RATIO 1389 /*! cache: eviction walks reached end of tree */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ENDED 1388 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ENDED 1390 /*! cache: eviction walks restarted */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK_RESTART 1389 +#define WT_STAT_CONN_CACHE_EVICTION_WALK_RESTART 1391 /*! cache: eviction walks started from root of tree */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK_FROM_ROOT 1390 +#define WT_STAT_CONN_CACHE_EVICTION_WALK_FROM_ROOT 1392 /*! cache: eviction walks started from saved location in tree */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK_SAVED_POS 1391 +#define WT_STAT_CONN_CACHE_EVICTION_WALK_SAVED_POS 1393 /*! cache: hazard pointer blocked page eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_HAZARD 1392 +#define WT_STAT_CONN_CACHE_EVICTION_HAZARD 1394 /*! cache: history store table insert calls */ -#define WT_STAT_CONN_CACHE_HS_INSERT 1393 +#define WT_STAT_CONN_CACHE_HS_INSERT 1395 /*! cache: history store table insert calls that returned restart */ -#define WT_STAT_CONN_CACHE_HS_INSERT_RESTART 1394 +#define WT_STAT_CONN_CACHE_HS_INSERT_RESTART 1396 /*! * cache: history store table out-of-order resolved updates that lose * their durable timestamp */ -#define WT_STAT_CONN_CACHE_HS_ORDER_LOSE_DURABLE_TIMESTAMP 1395 +#define WT_STAT_CONN_CACHE_HS_ORDER_LOSE_DURABLE_TIMESTAMP 1397 /*! * cache: history store table out-of-order updates that were fixed up by * moving existing records */ -#define WT_STAT_CONN_CACHE_HS_ORDER_FIXUP_MOVE 1396 +#define WT_STAT_CONN_CACHE_HS_ORDER_FIXUP_MOVE 1398 /*! * cache: history store table out-of-order updates that were fixed up * during insertion */ -#define WT_STAT_CONN_CACHE_HS_ORDER_FIXUP_INSERT 1397 +#define WT_STAT_CONN_CACHE_HS_ORDER_FIXUP_INSERT 1399 /*! cache: history store table reads */ -#define WT_STAT_CONN_CACHE_HS_READ 1398 +#define WT_STAT_CONN_CACHE_HS_READ 1400 /*! cache: history store table reads missed */ -#define WT_STAT_CONN_CACHE_HS_READ_MISS 1399 +#define WT_STAT_CONN_CACHE_HS_READ_MISS 1401 /*! cache: history store table reads requiring squashed modifies */ -#define WT_STAT_CONN_CACHE_HS_READ_SQUASH 1400 +#define WT_STAT_CONN_CACHE_HS_READ_SQUASH 1402 /*! * cache: history store table truncation by rollback to stable to remove * an unstable update */ -#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_RTS_UNSTABLE 1401 +#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_RTS_UNSTABLE 1403 /*! * cache: history store table truncation by rollback to stable to remove * an update */ -#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_RTS 1402 +#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_RTS 1404 /*! cache: history store table truncation to remove an update */ -#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE 1403 +#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE 1405 /*! * cache: history store table truncation to remove range of updates due * to key being removed from the data page during reconciliation */ -#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_ONPAGE_REMOVAL 1404 +#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_ONPAGE_REMOVAL 1406 /*! * cache: history store table truncation to remove range of updates due * to non timestamped update on data page */ -#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_NON_TS 1405 +#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_NON_TS 1407 /*! cache: history store table writes requiring squashed modifies */ -#define WT_STAT_CONN_CACHE_HS_WRITE_SQUASH 1406 +#define WT_STAT_CONN_CACHE_HS_WRITE_SQUASH 1408 /*! cache: in-memory page passed criteria to be split */ -#define WT_STAT_CONN_CACHE_INMEM_SPLITTABLE 1407 +#define WT_STAT_CONN_CACHE_INMEM_SPLITTABLE 1409 /*! cache: in-memory page splits */ -#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1408 +#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1410 /*! cache: internal pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1409 +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1411 /*! cache: internal pages split during eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_INTERNAL 1410 +#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_INTERNAL 1412 /*! cache: leaf pages split during eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_LEAF 1411 +#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_LEAF 1413 /*! cache: modified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1412 +#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1414 /*! cache: overflow pages read into cache */ -#define WT_STAT_CONN_CACHE_READ_OVERFLOW 1413 +#define WT_STAT_CONN_CACHE_READ_OVERFLOW 1415 /*! cache: page split during eviction deepened the tree */ -#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1414 +#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1416 /*! cache: page written requiring history store records */ -#define WT_STAT_CONN_CACHE_WRITE_HS 1415 +#define WT_STAT_CONN_CACHE_WRITE_HS 1417 /*! cache: pages read into cache */ -#define WT_STAT_CONN_CACHE_READ 1416 +#define WT_STAT_CONN_CACHE_READ 1418 /*! cache: pages read into cache after truncate */ -#define WT_STAT_CONN_CACHE_READ_DELETED 1417 +#define WT_STAT_CONN_CACHE_READ_DELETED 1419 /*! cache: pages read into cache after truncate in prepare state */ -#define WT_STAT_CONN_CACHE_READ_DELETED_PREPARED 1418 +#define WT_STAT_CONN_CACHE_READ_DELETED_PREPARED 1420 /*! cache: pages requested from the cache */ -#define WT_STAT_CONN_CACHE_PAGES_REQUESTED 1419 +#define WT_STAT_CONN_CACHE_PAGES_REQUESTED 1421 /*! cache: pages seen by eviction walk */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_SEEN 1420 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_SEEN 1422 /*! cache: pages written from cache */ -#define WT_STAT_CONN_CACHE_WRITE 1421 +#define WT_STAT_CONN_CACHE_WRITE 1423 /*! cache: pages written requiring in-memory restoration */ -#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1422 +#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1424 /*! cache: tracked dirty bytes in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_DIRTY 1423 +#define WT_STAT_CONN_CACHE_BYTES_DIRTY 1425 /*! cache: unmodified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1424 +#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1426 /*! checkpoint-cleanup: pages added for eviction */ -#define WT_STAT_CONN_CC_PAGES_EVICT 1425 +#define WT_STAT_CONN_CC_PAGES_EVICT 1427 /*! checkpoint-cleanup: pages removed */ -#define WT_STAT_CONN_CC_PAGES_REMOVED 1426 +#define WT_STAT_CONN_CC_PAGES_REMOVED 1428 /*! checkpoint-cleanup: pages skipped during tree walk */ -#define WT_STAT_CONN_CC_PAGES_WALK_SKIPPED 1427 +#define WT_STAT_CONN_CC_PAGES_WALK_SKIPPED 1429 /*! checkpoint-cleanup: pages visited */ -#define WT_STAT_CONN_CC_PAGES_VISITED 1428 +#define WT_STAT_CONN_CC_PAGES_VISITED 1430 /*! cursor: Total number of entries skipped by cursor next calls */ -#define WT_STAT_CONN_CURSOR_NEXT_SKIP_TOTAL 1429 +#define WT_STAT_CONN_CURSOR_NEXT_SKIP_TOTAL 1431 /*! cursor: Total number of entries skipped by cursor prev calls */ -#define WT_STAT_CONN_CURSOR_PREV_SKIP_TOTAL 1430 +#define WT_STAT_CONN_CURSOR_PREV_SKIP_TOTAL 1432 /*! * cursor: Total number of entries skipped to position the history store * cursor */ -#define WT_STAT_CONN_CURSOR_SKIP_HS_CUR_POSITION 1431 +#define WT_STAT_CONN_CURSOR_SKIP_HS_CUR_POSITION 1433 /*! * cursor: cursor next calls that skip due to a globally visible history * store tombstone */ -#define WT_STAT_CONN_CURSOR_NEXT_HS_TOMBSTONE 1432 +#define WT_STAT_CONN_CURSOR_NEXT_HS_TOMBSTONE 1434 /*! * cursor: cursor next calls that skip greater than or equal to 100 * entries */ -#define WT_STAT_CONN_CURSOR_NEXT_SKIP_GE_100 1433 +#define WT_STAT_CONN_CURSOR_NEXT_SKIP_GE_100 1435 /*! cursor: cursor next calls that skip less than 100 entries */ -#define WT_STAT_CONN_CURSOR_NEXT_SKIP_LT_100 1434 +#define WT_STAT_CONN_CURSOR_NEXT_SKIP_LT_100 1436 /*! * cursor: cursor prev calls that skip due to a globally visible history * store tombstone */ -#define WT_STAT_CONN_CURSOR_PREV_HS_TOMBSTONE 1435 +#define WT_STAT_CONN_CURSOR_PREV_HS_TOMBSTONE 1437 /*! * cursor: cursor prev calls that skip greater than or equal to 100 * entries */ -#define WT_STAT_CONN_CURSOR_PREV_SKIP_GE_100 1436 +#define WT_STAT_CONN_CURSOR_PREV_SKIP_GE_100 1438 /*! cursor: cursor prev calls that skip less than 100 entries */ -#define WT_STAT_CONN_CURSOR_PREV_SKIP_LT_100 1437 +#define WT_STAT_CONN_CURSOR_PREV_SKIP_LT_100 1439 /*! cursor: open cursor count */ -#define WT_STAT_CONN_CURSOR_OPEN_COUNT 1438 +#define WT_STAT_CONN_CURSOR_OPEN_COUNT 1440 /*! reconciliation: approximate byte size of timestamps in pages written */ -#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TS 1439 +#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TS 1441 /*! * reconciliation: approximate byte size of transaction IDs in pages * written */ -#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TXN 1440 +#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TXN 1442 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1441 +#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1443 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_CONN_REC_PAGES 1442 +#define WT_STAT_CONN_REC_PAGES 1444 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_CONN_REC_PAGES_EVICTION 1443 +#define WT_STAT_CONN_REC_PAGES_EVICTION 1445 /*! reconciliation: pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE 1444 +#define WT_STAT_CONN_REC_PAGE_DELETE 1446 /*! * reconciliation: pages written including an aggregated newest start * durable timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_START_DURABLE_TS 1445 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_START_DURABLE_TS 1447 /*! * reconciliation: pages written including an aggregated newest stop * durable timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_DURABLE_TS 1446 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_DURABLE_TS 1448 /*! * reconciliation: pages written including an aggregated newest stop * timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TS 1447 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TS 1449 /*! * reconciliation: pages written including an aggregated newest stop * transaction ID */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TXN 1448 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TXN 1450 /*! * reconciliation: pages written including an aggregated newest * transaction ID */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_TXN 1449 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_TXN 1451 /*! * reconciliation: pages written including an aggregated oldest start * timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_OLDEST_START_TS 1450 +#define WT_STAT_CONN_REC_TIME_AGGR_OLDEST_START_TS 1452 /*! reconciliation: pages written including an aggregated prepare */ -#define WT_STAT_CONN_REC_TIME_AGGR_PREPARED 1451 +#define WT_STAT_CONN_REC_TIME_AGGR_PREPARED 1453 /*! * reconciliation: pages written including at least one start durable * timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_START_TS 1452 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_START_TS 1454 /*! * reconciliation: pages written including at least one start transaction * ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TXN 1453 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TXN 1455 /*! * reconciliation: pages written including at least one stop durable * timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_STOP_TS 1454 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_STOP_TS 1456 /*! reconciliation: pages written including at least one stop timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TS 1455 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TS 1457 /*! * reconciliation: pages written including at least one stop transaction * ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TXN 1456 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TXN 1458 /*! reconciliation: records written including a start durable timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_START_TS 1457 +#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_START_TS 1459 /*! reconciliation: records written including a start timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_START_TS 1458 +#define WT_STAT_CONN_REC_TIME_WINDOW_START_TS 1460 /*! reconciliation: records written including a start transaction ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_START_TXN 1459 +#define WT_STAT_CONN_REC_TIME_WINDOW_START_TXN 1461 /*! reconciliation: records written including a stop durable timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_STOP_TS 1460 +#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_STOP_TS 1462 /*! reconciliation: records written including a stop timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TS 1461 +#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TS 1463 /*! reconciliation: records written including a stop transaction ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TXN 1462 +#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TXN 1464 /*! transaction: race to read prepared update retry */ -#define WT_STAT_CONN_TXN_READ_RACE_PREPARE_UPDATE 1463 +#define WT_STAT_CONN_TXN_READ_RACE_PREPARE_UPDATE 1465 /*! * transaction: rollback to stable hs records with stop timestamps older * than newer records */ -#define WT_STAT_CONN_TXN_RTS_HS_STOP_OLDER_THAN_NEWER_START 1464 +#define WT_STAT_CONN_TXN_RTS_HS_STOP_OLDER_THAN_NEWER_START 1466 /*! transaction: rollback to stable keys removed */ -#define WT_STAT_CONN_TXN_RTS_KEYS_REMOVED 1465 +#define WT_STAT_CONN_TXN_RTS_KEYS_REMOVED 1467 /*! transaction: rollback to stable keys restored */ -#define WT_STAT_CONN_TXN_RTS_KEYS_RESTORED 1466 +#define WT_STAT_CONN_TXN_RTS_KEYS_RESTORED 1468 /*! transaction: rollback to stable restored tombstones from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_TOMBSTONES 1467 +#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_TOMBSTONES 1469 /*! transaction: rollback to stable sweeping history store keys */ -#define WT_STAT_CONN_TXN_RTS_SWEEP_HS_KEYS 1468 +#define WT_STAT_CONN_TXN_RTS_SWEEP_HS_KEYS 1470 /*! transaction: rollback to stable updates removed from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_REMOVED 1469 +#define WT_STAT_CONN_TXN_RTS_HS_REMOVED 1471 /*! transaction: update conflicts */ -#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1470 +#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1472 /*! * @} @@ -6206,258 +6233,263 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_DSRC_CACHE_EVICTION_TARGET_PAGE_LT64 2116 /*! cache: eviction walk target pages histogram - 64-128 */ #define WT_STAT_DSRC_CACHE_EVICTION_TARGET_PAGE_LT128 2117 +/*! + * cache: eviction walk target pages reduced due to history store cache + * pressure + */ +#define WT_STAT_DSRC_CACHE_EVICTION_TARGET_PAGE_REDUCED 2118 /*! cache: eviction walks abandoned */ -#define WT_STAT_DSRC_CACHE_EVICTION_WALKS_ABANDONED 2118 +#define WT_STAT_DSRC_CACHE_EVICTION_WALKS_ABANDONED 2119 /*! cache: eviction walks gave up because they restarted their walk twice */ -#define WT_STAT_DSRC_CACHE_EVICTION_WALKS_STOPPED 2119 +#define WT_STAT_DSRC_CACHE_EVICTION_WALKS_STOPPED 2120 /*! * cache: eviction walks gave up because they saw too many pages and * found no candidates */ -#define WT_STAT_DSRC_CACHE_EVICTION_WALKS_GAVE_UP_NO_TARGETS 2120 +#define WT_STAT_DSRC_CACHE_EVICTION_WALKS_GAVE_UP_NO_TARGETS 2121 /*! * cache: eviction walks gave up because they saw too many pages and * found too few candidates */ -#define WT_STAT_DSRC_CACHE_EVICTION_WALKS_GAVE_UP_RATIO 2121 +#define WT_STAT_DSRC_CACHE_EVICTION_WALKS_GAVE_UP_RATIO 2122 /*! cache: eviction walks reached end of tree */ -#define WT_STAT_DSRC_CACHE_EVICTION_WALKS_ENDED 2122 +#define WT_STAT_DSRC_CACHE_EVICTION_WALKS_ENDED 2123 /*! cache: eviction walks restarted */ -#define WT_STAT_DSRC_CACHE_EVICTION_WALK_RESTART 2123 +#define WT_STAT_DSRC_CACHE_EVICTION_WALK_RESTART 2124 /*! cache: eviction walks started from root of tree */ -#define WT_STAT_DSRC_CACHE_EVICTION_WALK_FROM_ROOT 2124 +#define WT_STAT_DSRC_CACHE_EVICTION_WALK_FROM_ROOT 2125 /*! cache: eviction walks started from saved location in tree */ -#define WT_STAT_DSRC_CACHE_EVICTION_WALK_SAVED_POS 2125 +#define WT_STAT_DSRC_CACHE_EVICTION_WALK_SAVED_POS 2126 /*! cache: hazard pointer blocked page eviction */ -#define WT_STAT_DSRC_CACHE_EVICTION_HAZARD 2126 +#define WT_STAT_DSRC_CACHE_EVICTION_HAZARD 2127 /*! cache: history store table insert calls */ -#define WT_STAT_DSRC_CACHE_HS_INSERT 2127 +#define WT_STAT_DSRC_CACHE_HS_INSERT 2128 /*! cache: history store table insert calls that returned restart */ -#define WT_STAT_DSRC_CACHE_HS_INSERT_RESTART 2128 +#define WT_STAT_DSRC_CACHE_HS_INSERT_RESTART 2129 /*! * cache: history store table out-of-order resolved updates that lose * their durable timestamp */ -#define WT_STAT_DSRC_CACHE_HS_ORDER_LOSE_DURABLE_TIMESTAMP 2129 +#define WT_STAT_DSRC_CACHE_HS_ORDER_LOSE_DURABLE_TIMESTAMP 2130 /*! * cache: history store table out-of-order updates that were fixed up by * moving existing records */ -#define WT_STAT_DSRC_CACHE_HS_ORDER_FIXUP_MOVE 2130 +#define WT_STAT_DSRC_CACHE_HS_ORDER_FIXUP_MOVE 2131 /*! * cache: history store table out-of-order updates that were fixed up * during insertion */ -#define WT_STAT_DSRC_CACHE_HS_ORDER_FIXUP_INSERT 2131 +#define WT_STAT_DSRC_CACHE_HS_ORDER_FIXUP_INSERT 2132 /*! cache: history store table reads */ -#define WT_STAT_DSRC_CACHE_HS_READ 2132 +#define WT_STAT_DSRC_CACHE_HS_READ 2133 /*! cache: history store table reads missed */ -#define WT_STAT_DSRC_CACHE_HS_READ_MISS 2133 +#define WT_STAT_DSRC_CACHE_HS_READ_MISS 2134 /*! cache: history store table reads requiring squashed modifies */ -#define WT_STAT_DSRC_CACHE_HS_READ_SQUASH 2134 +#define WT_STAT_DSRC_CACHE_HS_READ_SQUASH 2135 /*! * cache: history store table truncation by rollback to stable to remove * an unstable update */ -#define WT_STAT_DSRC_CACHE_HS_KEY_TRUNCATE_RTS_UNSTABLE 2135 +#define WT_STAT_DSRC_CACHE_HS_KEY_TRUNCATE_RTS_UNSTABLE 2136 /*! * cache: history store table truncation by rollback to stable to remove * an update */ -#define WT_STAT_DSRC_CACHE_HS_KEY_TRUNCATE_RTS 2136 +#define WT_STAT_DSRC_CACHE_HS_KEY_TRUNCATE_RTS 2137 /*! cache: history store table truncation to remove an update */ -#define WT_STAT_DSRC_CACHE_HS_KEY_TRUNCATE 2137 +#define WT_STAT_DSRC_CACHE_HS_KEY_TRUNCATE 2138 /*! * cache: history store table truncation to remove range of updates due * to key being removed from the data page during reconciliation */ -#define WT_STAT_DSRC_CACHE_HS_KEY_TRUNCATE_ONPAGE_REMOVAL 2138 +#define WT_STAT_DSRC_CACHE_HS_KEY_TRUNCATE_ONPAGE_REMOVAL 2139 /*! * cache: history store table truncation to remove range of updates due * to non timestamped update on data page */ -#define WT_STAT_DSRC_CACHE_HS_KEY_TRUNCATE_NON_TS 2139 +#define WT_STAT_DSRC_CACHE_HS_KEY_TRUNCATE_NON_TS 2140 /*! cache: history store table writes requiring squashed modifies */ -#define WT_STAT_DSRC_CACHE_HS_WRITE_SQUASH 2140 +#define WT_STAT_DSRC_CACHE_HS_WRITE_SQUASH 2141 /*! cache: in-memory page passed criteria to be split */ -#define WT_STAT_DSRC_CACHE_INMEM_SPLITTABLE 2141 +#define WT_STAT_DSRC_CACHE_INMEM_SPLITTABLE 2142 /*! cache: in-memory page splits */ -#define WT_STAT_DSRC_CACHE_INMEM_SPLIT 2142 +#define WT_STAT_DSRC_CACHE_INMEM_SPLIT 2143 /*! cache: internal pages evicted */ -#define WT_STAT_DSRC_CACHE_EVICTION_INTERNAL 2143 +#define WT_STAT_DSRC_CACHE_EVICTION_INTERNAL 2144 /*! cache: internal pages split during eviction */ -#define WT_STAT_DSRC_CACHE_EVICTION_SPLIT_INTERNAL 2144 +#define WT_STAT_DSRC_CACHE_EVICTION_SPLIT_INTERNAL 2145 /*! cache: leaf pages split during eviction */ -#define WT_STAT_DSRC_CACHE_EVICTION_SPLIT_LEAF 2145 +#define WT_STAT_DSRC_CACHE_EVICTION_SPLIT_LEAF 2146 /*! cache: modified pages evicted */ -#define WT_STAT_DSRC_CACHE_EVICTION_DIRTY 2146 +#define WT_STAT_DSRC_CACHE_EVICTION_DIRTY 2147 /*! cache: overflow pages read into cache */ -#define WT_STAT_DSRC_CACHE_READ_OVERFLOW 2147 +#define WT_STAT_DSRC_CACHE_READ_OVERFLOW 2148 /*! cache: page split during eviction deepened the tree */ -#define WT_STAT_DSRC_CACHE_EVICTION_DEEPEN 2148 +#define WT_STAT_DSRC_CACHE_EVICTION_DEEPEN 2149 /*! cache: page written requiring history store records */ -#define WT_STAT_DSRC_CACHE_WRITE_HS 2149 +#define WT_STAT_DSRC_CACHE_WRITE_HS 2150 /*! cache: pages read into cache */ -#define WT_STAT_DSRC_CACHE_READ 2150 +#define WT_STAT_DSRC_CACHE_READ 2151 /*! cache: pages read into cache after truncate */ -#define WT_STAT_DSRC_CACHE_READ_DELETED 2151 +#define WT_STAT_DSRC_CACHE_READ_DELETED 2152 /*! cache: pages read into cache after truncate in prepare state */ -#define WT_STAT_DSRC_CACHE_READ_DELETED_PREPARED 2152 +#define WT_STAT_DSRC_CACHE_READ_DELETED_PREPARED 2153 /*! cache: pages requested from the cache */ -#define WT_STAT_DSRC_CACHE_PAGES_REQUESTED 2153 +#define WT_STAT_DSRC_CACHE_PAGES_REQUESTED 2154 /*! cache: pages seen by eviction walk */ -#define WT_STAT_DSRC_CACHE_EVICTION_PAGES_SEEN 2154 +#define WT_STAT_DSRC_CACHE_EVICTION_PAGES_SEEN 2155 /*! cache: pages written from cache */ -#define WT_STAT_DSRC_CACHE_WRITE 2155 +#define WT_STAT_DSRC_CACHE_WRITE 2156 /*! cache: pages written requiring in-memory restoration */ -#define WT_STAT_DSRC_CACHE_WRITE_RESTORE 2156 +#define WT_STAT_DSRC_CACHE_WRITE_RESTORE 2157 /*! cache: tracked dirty bytes in the cache */ -#define WT_STAT_DSRC_CACHE_BYTES_DIRTY 2157 +#define WT_STAT_DSRC_CACHE_BYTES_DIRTY 2158 /*! cache: unmodified pages evicted */ -#define WT_STAT_DSRC_CACHE_EVICTION_CLEAN 2158 +#define WT_STAT_DSRC_CACHE_EVICTION_CLEAN 2159 /*! checkpoint-cleanup: pages added for eviction */ -#define WT_STAT_DSRC_CC_PAGES_EVICT 2159 +#define WT_STAT_DSRC_CC_PAGES_EVICT 2160 /*! checkpoint-cleanup: pages removed */ -#define WT_STAT_DSRC_CC_PAGES_REMOVED 2160 +#define WT_STAT_DSRC_CC_PAGES_REMOVED 2161 /*! checkpoint-cleanup: pages skipped during tree walk */ -#define WT_STAT_DSRC_CC_PAGES_WALK_SKIPPED 2161 +#define WT_STAT_DSRC_CC_PAGES_WALK_SKIPPED 2162 /*! checkpoint-cleanup: pages visited */ -#define WT_STAT_DSRC_CC_PAGES_VISITED 2162 +#define WT_STAT_DSRC_CC_PAGES_VISITED 2163 /*! cursor: Total number of entries skipped by cursor next calls */ -#define WT_STAT_DSRC_CURSOR_NEXT_SKIP_TOTAL 2163 +#define WT_STAT_DSRC_CURSOR_NEXT_SKIP_TOTAL 2164 /*! cursor: Total number of entries skipped by cursor prev calls */ -#define WT_STAT_DSRC_CURSOR_PREV_SKIP_TOTAL 2164 +#define WT_STAT_DSRC_CURSOR_PREV_SKIP_TOTAL 2165 /*! * cursor: Total number of entries skipped to position the history store * cursor */ -#define WT_STAT_DSRC_CURSOR_SKIP_HS_CUR_POSITION 2165 +#define WT_STAT_DSRC_CURSOR_SKIP_HS_CUR_POSITION 2166 /*! * cursor: cursor next calls that skip due to a globally visible history * store tombstone */ -#define WT_STAT_DSRC_CURSOR_NEXT_HS_TOMBSTONE 2166 +#define WT_STAT_DSRC_CURSOR_NEXT_HS_TOMBSTONE 2167 /*! * cursor: cursor next calls that skip greater than or equal to 100 * entries */ -#define WT_STAT_DSRC_CURSOR_NEXT_SKIP_GE_100 2167 +#define WT_STAT_DSRC_CURSOR_NEXT_SKIP_GE_100 2168 /*! cursor: cursor next calls that skip less than 100 entries */ -#define WT_STAT_DSRC_CURSOR_NEXT_SKIP_LT_100 2168 +#define WT_STAT_DSRC_CURSOR_NEXT_SKIP_LT_100 2169 /*! * cursor: cursor prev calls that skip due to a globally visible history * store tombstone */ -#define WT_STAT_DSRC_CURSOR_PREV_HS_TOMBSTONE 2169 +#define WT_STAT_DSRC_CURSOR_PREV_HS_TOMBSTONE 2170 /*! * cursor: cursor prev calls that skip greater than or equal to 100 * entries */ -#define WT_STAT_DSRC_CURSOR_PREV_SKIP_GE_100 2170 +#define WT_STAT_DSRC_CURSOR_PREV_SKIP_GE_100 2171 /*! cursor: cursor prev calls that skip less than 100 entries */ -#define WT_STAT_DSRC_CURSOR_PREV_SKIP_LT_100 2171 +#define WT_STAT_DSRC_CURSOR_PREV_SKIP_LT_100 2172 /*! cursor: open cursor count */ -#define WT_STAT_DSRC_CURSOR_OPEN_COUNT 2172 +#define WT_STAT_DSRC_CURSOR_OPEN_COUNT 2173 /*! reconciliation: approximate byte size of timestamps in pages written */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_BYTES_TS 2173 +#define WT_STAT_DSRC_REC_TIME_WINDOW_BYTES_TS 2174 /*! * reconciliation: approximate byte size of transaction IDs in pages * written */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_BYTES_TXN 2174 +#define WT_STAT_DSRC_REC_TIME_WINDOW_BYTES_TXN 2175 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_DSRC_REC_PAGE_DELETE_FAST 2175 +#define WT_STAT_DSRC_REC_PAGE_DELETE_FAST 2176 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_DSRC_REC_PAGES 2176 +#define WT_STAT_DSRC_REC_PAGES 2177 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_DSRC_REC_PAGES_EVICTION 2177 +#define WT_STAT_DSRC_REC_PAGES_EVICTION 2178 /*! reconciliation: pages deleted */ -#define WT_STAT_DSRC_REC_PAGE_DELETE 2178 +#define WT_STAT_DSRC_REC_PAGE_DELETE 2179 /*! * reconciliation: pages written including an aggregated newest start * durable timestamp */ -#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_START_DURABLE_TS 2179 +#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_START_DURABLE_TS 2180 /*! * reconciliation: pages written including an aggregated newest stop * durable timestamp */ -#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_STOP_DURABLE_TS 2180 +#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_STOP_DURABLE_TS 2181 /*! * reconciliation: pages written including an aggregated newest stop * timestamp */ -#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_STOP_TS 2181 +#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_STOP_TS 2182 /*! * reconciliation: pages written including an aggregated newest stop * transaction ID */ -#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_STOP_TXN 2182 +#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_STOP_TXN 2183 /*! * reconciliation: pages written including an aggregated newest * transaction ID */ -#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_TXN 2183 +#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_TXN 2184 /*! * reconciliation: pages written including an aggregated oldest start * timestamp */ -#define WT_STAT_DSRC_REC_TIME_AGGR_OLDEST_START_TS 2184 +#define WT_STAT_DSRC_REC_TIME_AGGR_OLDEST_START_TS 2185 /*! reconciliation: pages written including an aggregated prepare */ -#define WT_STAT_DSRC_REC_TIME_AGGR_PREPARED 2185 +#define WT_STAT_DSRC_REC_TIME_AGGR_PREPARED 2186 /*! * reconciliation: pages written including at least one start durable * timestamp */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_DURABLE_START_TS 2186 +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_DURABLE_START_TS 2187 /*! * reconciliation: pages written including at least one start transaction * ID */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_START_TXN 2187 +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_START_TXN 2188 /*! * reconciliation: pages written including at least one stop durable * timestamp */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_DURABLE_STOP_TS 2188 +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_DURABLE_STOP_TS 2189 /*! reconciliation: pages written including at least one stop timestamp */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_STOP_TS 2189 +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_STOP_TS 2190 /*! * reconciliation: pages written including at least one stop transaction * ID */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_STOP_TXN 2190 +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_STOP_TXN 2191 /*! reconciliation: records written including a start durable timestamp */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_DURABLE_START_TS 2191 +#define WT_STAT_DSRC_REC_TIME_WINDOW_DURABLE_START_TS 2192 /*! reconciliation: records written including a start timestamp */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_START_TS 2192 +#define WT_STAT_DSRC_REC_TIME_WINDOW_START_TS 2193 /*! reconciliation: records written including a start transaction ID */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_START_TXN 2193 +#define WT_STAT_DSRC_REC_TIME_WINDOW_START_TXN 2194 /*! reconciliation: records written including a stop durable timestamp */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_DURABLE_STOP_TS 2194 +#define WT_STAT_DSRC_REC_TIME_WINDOW_DURABLE_STOP_TS 2195 /*! reconciliation: records written including a stop timestamp */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_STOP_TS 2195 +#define WT_STAT_DSRC_REC_TIME_WINDOW_STOP_TS 2196 /*! reconciliation: records written including a stop transaction ID */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_STOP_TXN 2196 +#define WT_STAT_DSRC_REC_TIME_WINDOW_STOP_TXN 2197 /*! transaction: race to read prepared update retry */ -#define WT_STAT_DSRC_TXN_READ_RACE_PREPARE_UPDATE 2197 +#define WT_STAT_DSRC_TXN_READ_RACE_PREPARE_UPDATE 2198 /*! * transaction: rollback to stable hs records with stop timestamps older * than newer records */ -#define WT_STAT_DSRC_TXN_RTS_HS_STOP_OLDER_THAN_NEWER_START 2198 +#define WT_STAT_DSRC_TXN_RTS_HS_STOP_OLDER_THAN_NEWER_START 2199 /*! transaction: rollback to stable keys removed */ -#define WT_STAT_DSRC_TXN_RTS_KEYS_REMOVED 2199 +#define WT_STAT_DSRC_TXN_RTS_KEYS_REMOVED 2200 /*! transaction: rollback to stable keys restored */ -#define WT_STAT_DSRC_TXN_RTS_KEYS_RESTORED 2200 +#define WT_STAT_DSRC_TXN_RTS_KEYS_RESTORED 2201 /*! transaction: rollback to stable restored tombstones from history store */ -#define WT_STAT_DSRC_TXN_RTS_HS_RESTORE_TOMBSTONES 2201 +#define WT_STAT_DSRC_TXN_RTS_HS_RESTORE_TOMBSTONES 2202 /*! transaction: rollback to stable sweeping history store keys */ -#define WT_STAT_DSRC_TXN_RTS_SWEEP_HS_KEYS 2202 +#define WT_STAT_DSRC_TXN_RTS_SWEEP_HS_KEYS 2203 /*! transaction: rollback to stable updates removed from history store */ -#define WT_STAT_DSRC_TXN_RTS_HS_REMOVED 2203 +#define WT_STAT_DSRC_TXN_RTS_HS_REMOVED 2204 /*! transaction: update conflicts */ -#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2204 +#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2205 /*! * @} diff --git a/src/third_party/wiredtiger/src/include/wt_internal.h b/src/third_party/wiredtiger/src/include/wt_internal.h index d3aa5905d4c..62b97d7e047 100644 --- a/src/third_party/wiredtiger/src/include/wt_internal.h +++ b/src/third_party/wiredtiger/src/include/wt_internal.h @@ -165,6 +165,8 @@ struct __wt_cursor_stat; typedef struct __wt_cursor_stat WT_CURSOR_STAT; struct __wt_cursor_table; typedef struct __wt_cursor_table WT_CURSOR_TABLE; +struct __wt_cursor_tiered; +typedef struct __wt_cursor_tiered WT_CURSOR_TIERED; struct __wt_data_handle; typedef struct __wt_data_handle WT_DATA_HANDLE; struct __wt_data_handle_cache; @@ -313,6 +315,8 @@ struct __wt_thread; typedef struct __wt_thread WT_THREAD; struct __wt_thread_group; typedef struct __wt_thread_group WT_THREAD_GROUP; +struct __wt_tiered; +typedef struct __wt_tiered WT_TIERED; struct __wt_time_aggregate; typedef struct __wt_time_aggregate WT_TIME_AGGREGATE; struct __wt_time_window; @@ -402,6 +406,7 @@ typedef uint64_t wt_timestamp_t; #include "reconcile.h" #include "schema.h" #include "thread_group.h" +#include "tiered.h" #include "txn.h" #include "session.h" /* required by connection.h */ diff --git a/src/third_party/wiredtiger/src/log/log.c b/src/third_party/wiredtiger/src/log/log.c index 46dce80113c..6ba60e60129 100644 --- a/src/third_party/wiredtiger/src/log/log.c +++ b/src/third_party/wiredtiger/src/log/log.c @@ -2042,7 +2042,7 @@ __log_salvage_message( * Scan the logs, calling a function on each record found. */ int -__wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, +__wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *start_lsnp, WT_LSN *end_lsnp, uint32_t flags, int (*func)(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, WT_LSN *next_lsnp, void *cookie, int firstrecord), void *cookie) @@ -2080,7 +2080,7 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, if (func == NULL) return (0); - if (lsnp != NULL && LF_ISSET(WT_LOGSCAN_FIRST | WT_LOGSCAN_FROM_CKP)) + if (start_lsnp != NULL && LF_ISSET(WT_LOGSCAN_FIRST | WT_LOGSCAN_FROM_CKP)) WT_RET_MSG(session, WT_ERROR, "choose either a start LSN or a start flag"); /* * Set up the allocation size, starting and ending LSNs. The values for those depend on whether @@ -2091,7 +2091,7 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, allocsize = log->allocsize; WT_ASSIGN_LSN(&end_lsn, &log->alloc_lsn); WT_ASSIGN_LSN(&start_lsn, &log->first_lsn); - if (lsnp == NULL) { + if (start_lsnp == NULL) { if (LF_ISSET(WT_LOGSCAN_FROM_CKP)) WT_ASSIGN_LSN(&start_lsn, &log->ckpt_lsn); else if (!LF_ISSET(WT_LOGSCAN_FIRST)) @@ -2121,16 +2121,17 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, WT_SET_LSN(&end_lsn, lastlog, 0); WT_ERR(__wt_fs_directory_list_free(session, &logfiles, logcount)); } - if (lsnp != NULL) { + + if (start_lsnp != NULL) { /* * Offsets must be on allocation boundaries. An invalid LSN from a user should just return * WT_NOTFOUND. It is not an error. But if it is from recovery, we expect valid LSNs so give * more information about that. */ - if (lsnp->l.offset % allocsize != 0) { + if (start_lsnp->l.offset % allocsize != 0) { if (LF_ISSET(WT_LOGSCAN_RECOVER | WT_LOGSCAN_RECOVER_METADATA)) WT_ERR_MSG(session, WT_NOTFOUND, "__wt_log_scan unaligned LSN %" PRIu32 "/%" PRIu32, - lsnp->l.file, lsnp->l.offset); + start_lsnp->l.file, start_lsnp->l.offset); else WT_ERR(WT_NOTFOUND); } @@ -2139,11 +2140,11 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, * return WT_NOTFOUND. It is not an error. But if it is from recovery, we expect valid LSNs * so give more information about that. */ - if (lsnp->l.file > lastlog) { + if (start_lsnp->l.file > lastlog) { if (LF_ISSET(WT_LOGSCAN_RECOVER | WT_LOGSCAN_RECOVER_METADATA)) WT_ERR_MSG(session, WT_NOTFOUND, "__wt_log_scan LSN %" PRIu32 "/%" PRIu32 " larger than biggest log file %" PRIu32, - lsnp->l.file, lsnp->l.offset, lastlog); + start_lsnp->l.file, start_lsnp->l.offset, lastlog); else WT_ERR(WT_NOTFOUND); } @@ -2151,8 +2152,8 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, * Log cursors may not know the starting LSN. If an LSN is passed in that it is equal to the * smallest LSN, start from the beginning of the log. */ - if (!WT_IS_INIT_LSN(lsnp)) - WT_ASSIGN_LSN(&start_lsn, lsnp); + if (!WT_IS_INIT_LSN(start_lsnp)) + WT_ASSIGN_LSN(&start_lsn, start_lsnp); } WT_ERR(__log_open_verify(session, start_lsn.l.file, &log_fh, &prev_lsn, NULL, &need_salvage)); if (need_salvage) @@ -2387,6 +2388,13 @@ advance: if (LF_ISSET(WT_LOGSCAN_ONE)) break; } + + /* + * Exit the scanning loop if the next LSN seen is greater than our user set end range LSN. + */ + if (end_lsnp != NULL && __wt_log_cmp(&next_lsn, end_lsnp) > 0) + break; + WT_ASSIGN_LSN(&rd_lsn, &next_lsn); } @@ -2687,7 +2695,7 @@ __log_write_internal(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, ui __wt_log_slot_free(session, myslot.slot); } else if (force) { /* - * If we are going to wait for this slot to get written, signal the wrlsn thread. + * If we are going to wait for this slot to get written, signal the log server thread. * * XXX I've seen times when conditions are NULL. */ diff --git a/src/third_party/wiredtiger/src/schema/schema_alter.c b/src/third_party/wiredtiger/src/schema/schema_alter.c index 49a78fa5784..667071df670 100644 --- a/src/third_party/wiredtiger/src/schema/schema_alter.c +++ b/src/third_party/wiredtiger/src/schema/schema_alter.c @@ -220,6 +220,8 @@ __schema_alter(WT_SESSION_IMPL *session, const char *uri, const char *newcfg[]) return (__wt_lsm_tree_worker(session, uri, __alter_file, NULL, newcfg, flags)); if (WT_PREFIX_MATCH(uri, "table:")) return (__alter_table(session, uri, newcfg, exclusive_refreshed)); + if (WT_PREFIX_MATCH(uri, "tiered:")) + return (__wt_tiered_worker(session, uri, __alter_file, NULL, newcfg, flags)); return (__wt_bad_object_type(session, uri)); } diff --git a/src/third_party/wiredtiger/src/schema/schema_create.c b/src/third_party/wiredtiger/src/schema/schema_create.c index f49bea7250e..35ae706ee1c 100644 --- a/src/third_party/wiredtiger/src/schema/schema_create.c +++ b/src/third_party/wiredtiger/src/schema/schema_create.c @@ -834,6 +834,8 @@ __schema_create(WT_SESSION_IMPL *session, const char *uri, const char *config) ret = __create_index(session, uri, exclusive, config); else if (WT_PREFIX_MATCH(uri, "table:")) ret = __create_table(session, uri, exclusive, import, config); + else if (WT_PREFIX_MATCH(uri, "tiered:")) + ret = __wt_tiered_create(session, uri, exclusive, config); else if ((dsrc = __wt_schema_get_source(session, uri)) != NULL) ret = dsrc->create == NULL ? __wt_object_unsupported(session, uri) : __create_data_source(session, uri, config, dsrc); diff --git a/src/third_party/wiredtiger/src/schema/schema_drop.c b/src/third_party/wiredtiger/src/schema/schema_drop.c index 21daf4b5b37..09ab32affcf 100644 --- a/src/third_party/wiredtiger/src/schema/schema_drop.c +++ b/src/third_party/wiredtiger/src/schema/schema_drop.c @@ -199,6 +199,8 @@ __schema_drop(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) ret = __wt_lsm_tree_drop(session, uri, cfg); else if (WT_PREFIX_MATCH(uri, "table:")) ret = __drop_table(session, uri, cfg); + else if (WT_PREFIX_MATCH(uri, "tiered:")) + ret = __wt_tiered_drop(session, uri, cfg); else if ((dsrc = __wt_schema_get_source(session, uri)) != NULL) ret = dsrc->drop == NULL ? __wt_object_unsupported(session, uri) : dsrc->drop(dsrc, &session->iface, uri, (WT_CONFIG_ARG *)cfg); diff --git a/src/third_party/wiredtiger/src/schema/schema_rename.c b/src/third_party/wiredtiger/src/schema/schema_rename.c index 53c006ed226..6b5eb98d008 100644 --- a/src/third_party/wiredtiger/src/schema/schema_rename.c +++ b/src/third_party/wiredtiger/src/schema/schema_rename.c @@ -305,6 +305,8 @@ __schema_rename(WT_SESSION_IMPL *session, const char *uri, const char *newuri, c ret = __wt_lsm_tree_rename(session, uri, newuri, cfg); else if (WT_PREFIX_MATCH(uri, "table:")) ret = __rename_table(session, uri, newuri, cfg); + else if (WT_PREFIX_MATCH(uri, "tiered:")) + ret = __wt_tiered_rename(session, uri, newuri, cfg); else if ((dsrc = __wt_schema_get_source(session, uri)) != NULL) ret = dsrc->rename == NULL ? __wt_object_unsupported(session, uri) : diff --git a/src/third_party/wiredtiger/src/schema/schema_truncate.c b/src/third_party/wiredtiger/src/schema/schema_truncate.c index 390ecb0b5a8..27d8c78050f 100644 --- a/src/third_party/wiredtiger/src/schema/schema_truncate.c +++ b/src/third_party/wiredtiger/src/schema/schema_truncate.c @@ -83,6 +83,8 @@ __wt_schema_truncate(WT_SESSION_IMPL *session, const char *uri, const char *cfg[ ret = __wt_lsm_tree_truncate(session, uri, cfg); else if (WT_PREFIX_SKIP(tablename, "table:")) ret = __truncate_table(session, tablename, cfg); + else if (WT_PREFIX_MATCH(uri, "tiered:")) + ret = __wt_tiered_truncate(session, uri, cfg); else if ((dsrc = __wt_schema_get_source(session, uri)) != NULL) ret = dsrc->truncate == NULL ? __truncate_dsrc(session, uri) : diff --git a/src/third_party/wiredtiger/src/schema/schema_worker.c b/src/third_party/wiredtiger/src/schema/schema_worker.c index e8beb10c670..37d069b8457 100644 --- a/src/third_party/wiredtiger/src/schema/schema_worker.c +++ b/src/third_party/wiredtiger/src/schema/schema_worker.c @@ -118,6 +118,8 @@ __wt_schema_worker(WT_SESSION_IMPL *session, const char *uri, WT_ERR( __wt_schema_worker(session, idx->source, file_func, name_func, cfg, open_flags)); } + } else if (WT_PREFIX_MATCH(uri, "tiered:")) { + WT_ERR(__wt_tiered_worker(session, uri, file_func, name_func, cfg, open_flags)); } else if ((dsrc = __wt_schema_get_source(session, uri)) != NULL) { wt_session = (WT_SESSION *)session; if (file_func == __wt_salvage && dsrc->salvage != NULL) diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c index b221c7eb1f6..7ebdf736c2f 100644 --- a/src/third_party/wiredtiger/src/session/session_api.c +++ b/src/third_party/wiredtiger/src/session/session_api.c @@ -452,6 +452,8 @@ __session_open_cursor_int(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR * case 't': if (WT_PREFIX_MATCH(uri, "table:")) WT_RET(__wt_curtable_open(session, uri, owner, cfg, cursorp)); + if (WT_PREFIX_MATCH(uri, "tiered:")) + WT_RET(__wt_curtiered_open(session, uri, owner, cfg, cursorp)); break; case 'c': if (WT_PREFIX_MATCH(uri, "colgroup:")) { @@ -590,7 +592,8 @@ __session_open_cursor(WT_SESSION *wt_session, const char *uri, WT_CURSOR *to_dup if (!WT_PREFIX_MATCH(uri, "backup:") && !WT_PREFIX_MATCH(uri, "colgroup:") && !WT_PREFIX_MATCH(uri, "index:") && !WT_PREFIX_MATCH(uri, "file:") && !WT_PREFIX_MATCH(uri, "lsm:") && !WT_PREFIX_MATCH(uri, WT_METADATA_URI) && - !WT_PREFIX_MATCH(uri, "table:") && __wt_schema_get_source(session, uri) == NULL) + !WT_PREFIX_MATCH(uri, "table:") && !WT_PREFIX_MATCH(uri, "tiered:") && + __wt_schema_get_source(session, uri) == NULL) WT_ERR(__wt_bad_object_type(session, uri)); } } diff --git a/src/third_party/wiredtiger/src/session/session_compact.c b/src/third_party/wiredtiger/src/session/session_compact.c index 3184cdd579d..28eac90138e 100644 --- a/src/third_party/wiredtiger/src/session/session_compact.c +++ b/src/third_party/wiredtiger/src/session/session_compact.c @@ -142,6 +142,8 @@ __compact_uri_analyze(WT_SESSION_IMPL *session, const char *uri, bool *skipp) *skipp = true; } else if (WT_PREFIX_MATCH(uri, "file:")) session->compact->file_count++; + if (WT_PREFIX_MATCH(uri, "tiered:")) + WT_RET(ENOTSUP); return (0); } @@ -362,7 +364,7 @@ __wt_session_compact(WT_SESSION *wt_session, const char *uri, const char *config if (!WT_PREFIX_MATCH(uri, "colgroup:") && !WT_PREFIX_MATCH(uri, "file:") && !WT_PREFIX_MATCH(uri, "index:") && !WT_PREFIX_MATCH(uri, "lsm:") && - !WT_PREFIX_MATCH(uri, "table:")) { + !WT_PREFIX_MATCH(uri, "table:") && !WT_PREFIX_MATCH(uri, "tiered:")) { if ((dsrc = __wt_schema_get_source(session, uri)) != NULL) ret = dsrc->compact == NULL ? __wt_object_unsupported(session, uri) : diff --git a/src/third_party/wiredtiger/src/support/err.c b/src/third_party/wiredtiger/src/support/err.c index 2655b698a10..1a2c5d40add 100644 --- a/src/third_party/wiredtiger/src/support/err.c +++ b/src/third_party/wiredtiger/src/support/err.c @@ -574,7 +574,7 @@ __wt_bad_object_type(WT_SESSION_IMPL *session, const char *uri) WT_GCC_FUNC_ATTR WT_PREFIX_MATCH(uri, "config:") || WT_PREFIX_MATCH(uri, "file:") || WT_PREFIX_MATCH(uri, "index:") || WT_PREFIX_MATCH(uri, "log:") || WT_PREFIX_MATCH(uri, "lsm:") || WT_PREFIX_MATCH(uri, "statistics:") || - WT_PREFIX_MATCH(uri, "table:")) + WT_PREFIX_MATCH(uri, "table:") || WT_PREFIX_MATCH(uri, "tiered:")) return (__wt_object_unsupported(session, uri)); WT_RET_MSG(session, ENOTSUP, "unknown object type: %s", uri); diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index 83ad2859c54..1ee255be706 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -123,6 +123,7 @@ static const char *const __stats_dsrc_desc[] = { "cache: eviction walk target pages histogram - 128 and higher", "cache: eviction walk target pages histogram - 32-63", "cache: eviction walk target pages histogram - 64-128", + "cache: eviction walk target pages reduced due to history store cache pressure", "cache: eviction walks abandoned", "cache: eviction walks gave up because they restarted their walk twice", "cache: eviction walks gave up because they saw too many pages and found no candidates", @@ -370,6 +371,7 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats) stats->cache_eviction_target_page_ge128 = 0; stats->cache_eviction_target_page_lt64 = 0; stats->cache_eviction_target_page_lt128 = 0; + stats->cache_eviction_target_page_reduced = 0; stats->cache_eviction_walks_abandoned = 0; stats->cache_eviction_walks_stopped = 0; stats->cache_eviction_walks_gave_up_no_targets = 0; @@ -602,6 +604,7 @@ __wt_stat_dsrc_aggregate_single(WT_DSRC_STATS *from, WT_DSRC_STATS *to) to->cache_eviction_target_page_ge128 += from->cache_eviction_target_page_ge128; to->cache_eviction_target_page_lt64 += from->cache_eviction_target_page_lt64; to->cache_eviction_target_page_lt128 += from->cache_eviction_target_page_lt128; + to->cache_eviction_target_page_reduced += from->cache_eviction_target_page_reduced; to->cache_eviction_walks_abandoned += from->cache_eviction_walks_abandoned; to->cache_eviction_walks_stopped += from->cache_eviction_walks_stopped; to->cache_eviction_walks_gave_up_no_targets += from->cache_eviction_walks_gave_up_no_targets; @@ -829,6 +832,8 @@ __wt_stat_dsrc_aggregate(WT_DSRC_STATS **from, WT_DSRC_STATS *to) to->cache_eviction_target_page_ge128 += WT_STAT_READ(from, cache_eviction_target_page_ge128); to->cache_eviction_target_page_lt64 += WT_STAT_READ(from, cache_eviction_target_page_lt64); to->cache_eviction_target_page_lt128 += WT_STAT_READ(from, cache_eviction_target_page_lt128); + to->cache_eviction_target_page_reduced += + WT_STAT_READ(from, cache_eviction_target_page_reduced); to->cache_eviction_walks_abandoned += WT_STAT_READ(from, cache_eviction_walks_abandoned); to->cache_eviction_walks_stopped += WT_STAT_READ(from, cache_eviction_walks_stopped); to->cache_eviction_walks_gave_up_no_targets += @@ -1021,6 +1026,7 @@ static const char *const __stats_connection_desc[] = { "cache: pages queued for eviction post lru sorting", "cache: pages queued for urgent eviction", "cache: pages queued for urgent eviction during walk", + "cache: pages queued for urgent eviction from history store due to high dirty content", "cache: pages seen by eviction walk that are already queued", "cache: pages selected for eviction unable to be evicted", "cache: pages selected for eviction unable to be evicted as the parent page has overflow items", @@ -1323,6 +1329,7 @@ static const char *const __stats_connection_desc[] = { "cache: eviction walk target pages histogram - 128 and higher", "cache: eviction walk target pages histogram - 32-63", "cache: eviction walk target pages histogram - 64-128", + "cache: eviction walk target pages reduced due to history store cache pressure", "cache: eviction walks abandoned", "cache: eviction walks gave up because they restarted their walk twice", "cache: eviction walks gave up because they saw too many pages and found no candidates", @@ -1539,6 +1546,7 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->cache_eviction_pages_queued_post_lru = 0; stats->cache_eviction_pages_queued_urgent = 0; stats->cache_eviction_pages_queued_oldest = 0; + stats->cache_eviction_pages_queued_urgent_hs_dirty = 0; stats->cache_eviction_pages_already_queued = 0; stats->cache_eviction_fail = 0; stats->cache_eviction_fail_parent_has_overflow_items = 0; @@ -1836,6 +1844,7 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->cache_eviction_target_page_ge128 = 0; stats->cache_eviction_target_page_lt64 = 0; stats->cache_eviction_target_page_lt128 = 0; + stats->cache_eviction_target_page_reduced = 0; stats->cache_eviction_walks_abandoned = 0; stats->cache_eviction_walks_stopped = 0; stats->cache_eviction_walks_gave_up_no_targets = 0; @@ -2038,6 +2047,8 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * WT_STAT_READ(from, cache_eviction_pages_queued_urgent); to->cache_eviction_pages_queued_oldest += WT_STAT_READ(from, cache_eviction_pages_queued_oldest); + to->cache_eviction_pages_queued_urgent_hs_dirty += + WT_STAT_READ(from, cache_eviction_pages_queued_urgent_hs_dirty); to->cache_eviction_pages_already_queued += WT_STAT_READ(from, cache_eviction_pages_already_queued); to->cache_eviction_fail += WT_STAT_READ(from, cache_eviction_fail); @@ -2346,6 +2357,8 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * to->cache_eviction_target_page_ge128 += WT_STAT_READ(from, cache_eviction_target_page_ge128); to->cache_eviction_target_page_lt64 += WT_STAT_READ(from, cache_eviction_target_page_lt64); to->cache_eviction_target_page_lt128 += WT_STAT_READ(from, cache_eviction_target_page_lt128); + to->cache_eviction_target_page_reduced += + WT_STAT_READ(from, cache_eviction_target_page_reduced); to->cache_eviction_walks_abandoned += WT_STAT_READ(from, cache_eviction_walks_abandoned); to->cache_eviction_walks_stopped += WT_STAT_READ(from, cache_eviction_walks_stopped); to->cache_eviction_walks_gave_up_no_targets += diff --git a/src/third_party/wiredtiger/src/tiered/tiered_cursor.c b/src/third_party/wiredtiger/src/tiered/tiered_cursor.c new file mode 100644 index 00000000000..26c750fb496 --- /dev/null +++ b/src/third_party/wiredtiger/src/tiered/tiered_cursor.c @@ -0,0 +1,1186 @@ +/*- + * Copyright (c) 2014-2020 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +#define WT_FORALL_CURSORS(curtiered, c, i) \ + for ((i) = (curtiered)->tiered->ntiers; (i) > 0;) \ + if (((c) = (curtiered)->cursors[--(i)]) != NULL) + +#define WT_TIERED_CURCMP(s, tiered, c1, c2, cmp) \ + __wt_compare(s, (tiered)->collator, &(c1)->key, &(c2)->key, &(cmp)) + +/* + * __curtiered_open_cursors -- + * Open cursors for the current set of files. + */ +static int +__curtiered_open_cursors(WT_CURSOR_TIERED *curtiered) +{ + WT_CURSOR *cursor; + WT_DATA_HANDLE *dhandle; + WT_DECL_RET; + WT_SESSION_IMPL *session; + WT_TIERED *tiered; + u_int i; + + cursor = &curtiered->iface; + session = CUR2S(curtiered); + dhandle = NULL; + tiered = curtiered->tiered; + + if (tiered->ntiers == 0) + return (0); + + /* + * If the key is pointing to memory that is pinned by a chunk cursor, take a copy before closing + * cursors. + */ + if (F_ISSET(cursor, WT_CURSTD_KEY_INT)) + WT_ERR(__cursor_needkey(cursor)); + + F_CLR(curtiered, WT_CURTIERED_ITERATE_NEXT | WT_CURTIERED_ITERATE_PREV); + + WT_ASSERT(session, curtiered->cursors == NULL); + WT_ERR(__wt_calloc_def(session, tiered->ntiers, &curtiered->cursors)); + + /* Open the cursors for chunks that have changed. */ + __wt_verbose(session, WT_VERB_TIERED, + "tiered opening cursor session(%p):tiered cursor(%p), tiers: %u", (void *)session, + (void *)curtiered, tiered->ntiers); + for (i = 0; i != tiered->ntiers; i++) { + dhandle = tiered->tiers[i]; + + /* + * Read from the checkpoint if the file has been written. Once all cursors switch, the + * in-memory tree can be evicted. + */ + WT_ASSERT(session, curtiered->cursors[i] == NULL); + WT_ERR(__wt_open_cursor(session, dhandle->name, cursor, NULL, &curtiered->cursors[i])); + + /* Child cursors always use overwrite and raw mode. */ + F_SET(curtiered->cursors[i], WT_CURSTD_OVERWRITE | WT_CURSTD_RAW); + } + +err: + return (ret); +} + +/* + * __curtiered_close_cursors -- + * Close any btree cursors that are not needed. + */ +static int +__curtiered_close_cursors(WT_SESSION_IMPL *session, WT_CURSOR_TIERED *curtiered) +{ + WT_CURSOR *c; + u_int i; + + __wt_verbose(session, WT_VERB_TIERED, "tiered close cursors session(%p):tiered cursor(%p)", + (void *)session, (void *)curtiered); + + if (curtiered->cursors == NULL) + return (0); + + /* Walk the cursors, closing them. */ + for (i = 0; i < curtiered->tiered->ntiers; i++) { + if ((c = (curtiered)->cursors[i]) != NULL) { + curtiered->cursors[i] = NULL; + WT_RET(c->close(c)); + } + } + + __wt_free(session, curtiered->cursors); + return (0); +} + +/* + * __curtiered_reset_cursors -- + * Reset any positioned chunk cursors. If the skip parameter is non-NULL, that cursor is about + * to be used, so there is no need to reset it. + */ +static int +__curtiered_reset_cursors(WT_CURSOR_TIERED *curtiered, WT_CURSOR *skip) +{ + WT_CURSOR *c; + WT_DECL_RET; + u_int i; + + /* Fast path if the cursor is not positioned. */ + if ((curtiered->current == NULL || curtiered->current == skip) && + !F_ISSET(curtiered, WT_CLSM_ITERATE_NEXT | WT_CLSM_ITERATE_PREV)) + return (0); + + WT_FORALL_CURSORS(curtiered, c, i) + { + if (c == skip) + continue; + if (F_ISSET(c, WT_CURSTD_KEY_INT)) + WT_TRET(c->reset(c)); + } + + curtiered->current = NULL; + F_CLR(curtiered, WT_CLSM_ITERATE_NEXT | WT_CLSM_ITERATE_PREV); + + return (ret); +} + +/* + * __curtiered_enter -- + * Start an operation on a tiered cursor. + */ +static inline int +__curtiered_enter(WT_CURSOR_TIERED *curtiered, bool reset) +{ + WT_SESSION_IMPL *session; + + session = CUR2S(curtiered); + + if (curtiered->cursors == NULL) + WT_RET(__curtiered_open_cursors(curtiered)); + + if (reset) { + WT_ASSERT(session, !F_ISSET(&curtiered->iface, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT)); + WT_RET(__curtiered_reset_cursors(curtiered, NULL)); + } + + if (!F_ISSET(curtiered, WT_CURTIERED_ACTIVE)) { + /* + * Opening this tiered cursor has opened a number of other cursors, ensure we don't mistake + * this as the first cursor in a session. + */ + ++session->ncursors; + WT_RET(__cursor_enter(session)); + F_SET(curtiered, WT_CURTIERED_ACTIVE); + } + + return (0); +} +/* + * __curtiered_leave -- + * Finish an operation on a tiered cursor. + */ +static void +__curtiered_leave(WT_CURSOR_TIERED *curtiered) +{ + WT_SESSION_IMPL *session; + + session = CUR2S(curtiered); + + if (F_ISSET(curtiered, WT_CURTIERED_ACTIVE)) { + --session->ncursors; + __cursor_leave(session); + F_CLR(curtiered, WT_CURTIERED_ACTIVE); + } +} + +/* + * We need a tombstone to mark deleted records, and we use the special value below for that purpose. + * We use two 0x14 (Device Control 4) bytes to minimize the likelihood of colliding with an + * application-chosen encoding byte, if the application uses two leading DC4 byte for some reason, + * we'll do a wasted data copy each time a new value is inserted into the object. + */ +static const WT_ITEM __tombstone = {"\x14\x14", 2, NULL, 0, 0}; + +/* + * __curtiered_deleted -- + * Check whether the current value is a tombstone. + */ +static inline bool +__curtiered_deleted(WT_CURSOR_TIERED *curtiered, const WT_ITEM *item) +{ + WT_UNUSED(curtiered); + return (item->size == __tombstone.size && + memcmp(item->data, __tombstone.data, __tombstone.size) == 0); +} + +/* + * __curtiered_deleted_encode -- + * Encode values that are in the encoded name space. + */ +static inline int +__curtiered_deleted_encode( + WT_SESSION_IMPL *session, const WT_ITEM *value, WT_ITEM *final_value, WT_ITEM **tmpp) +{ + WT_ITEM *tmp; + + /* + * If value requires encoding, get a scratch buffer of the right size and create a copy of the + * data with the first byte of the tombstone appended. + */ + if (value->size >= __tombstone.size && + memcmp(value->data, __tombstone.data, __tombstone.size) == 0) { + WT_RET(__wt_scr_alloc(session, value->size + 1, tmpp)); + tmp = *tmpp; + + memcpy(tmp->mem, value->data, value->size); + memcpy((uint8_t *)tmp->mem + value->size, __tombstone.data, 1); + final_value->data = tmp->mem; + final_value->size = value->size + 1; + } else { + final_value->data = value->data; + final_value->size = value->size; + } + + return (0); +} + +/* + * __curtiered_deleted_decode -- + * Decode values that start with the tombstone. + */ +static inline void +__curtiered_deleted_decode(WT_CURSOR_TIERED *curtiered, WT_ITEM *value) +{ + WT_UNUSED(curtiered); + /* + * Take care with this check: when a tiered cursor is used for a merge, it is valid to return + * the tombstone value. + */ + if (value->size > __tombstone.size && + memcmp(value->data, __tombstone.data, __tombstone.size) == 0) + --value->size; +} + +/* + * __wt_curtiered_close -- + * WT_CURSOR->close method for the tiered cursor type. + */ +int +__wt_curtiered_close(WT_CURSOR *cursor) +{ + WT_CURSOR_TIERED *curtiered; + WT_DECL_RET; + WT_SESSION_IMPL *session; + + /* + * Don't use the normal __curtiered_enter path: that is wasted work when closing, and the cursor + * may never have been used. + */ + curtiered = (WT_CURSOR_TIERED *)cursor; + CURSOR_API_CALL_PREPARE_ALLOWED(cursor, session, close, NULL); +err: + WT_TRET(__curtiered_close_cursors(session, curtiered)); + + /* In case we were somehow left positioned, clear that. */ + __curtiered_leave(curtiered); + + if (curtiered->tiered != NULL) + WT_WITH_DHANDLE(session, (WT_DATA_HANDLE *)curtiered->tiered, + WT_TRET(__wt_session_release_dhandle(session))); + __wt_cursor_close(cursor); + + API_END_RET(session, ret); +} + +/* + * __curtiered_get_current -- + * Find the smallest / largest of the cursors and copy its key/value. + */ +static int +__curtiered_get_current( + WT_SESSION_IMPL *session, WT_CURSOR_TIERED *curtiered, bool smallest, bool *deletedp) +{ + WT_CURSOR *c, *current; + u_int i; + int cmp; + bool multiple; + + current = NULL; + multiple = false; + + WT_FORALL_CURSORS(curtiered, c, i) + { + if (!F_ISSET(c, WT_CURSTD_KEY_INT)) + continue; + if (current == NULL) { + current = c; + continue; + } + WT_RET(WT_TIERED_CURCMP(session, curtiered->tiered, c, current, cmp)); + if (smallest ? cmp < 0 : cmp > 0) { + current = c; + multiple = false; + } else if (cmp == 0) + multiple = true; + } + + c = &curtiered->iface; + if ((curtiered->current = current) == NULL) { + F_CLR(c, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + return (WT_NOTFOUND); + } + + if (multiple) + F_SET(curtiered, WT_CLSM_MULTIPLE); + else + F_CLR(curtiered, WT_CLSM_MULTIPLE); + + WT_RET(current->get_key(current, &c->key)); + WT_RET(current->get_value(current, &c->value)); + + F_CLR(c, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + if ((*deletedp = __curtiered_deleted(curtiered, &c->value)) == false) + F_SET(c, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); + + return (0); +} + +/* + * __curtiered_compare -- + * WT_CURSOR->compare implementation for the LSM cursor type. + */ +static int +__curtiered_compare(WT_CURSOR *a, WT_CURSOR *b, int *cmpp) +{ + WT_CURSOR_TIERED *alsm; + WT_DECL_RET; + WT_SESSION_IMPL *session; + + /* There's no need to sync with the LSM tree, avoid WT_LSM_ENTER. */ + alsm = (WT_CURSOR_TIERED *)a; + CURSOR_API_CALL(a, session, compare, NULL); + + /* + * Confirm both cursors refer to the same source and have keys, then compare the keys. + */ + if (strcmp(a->uri, b->uri) != 0) + WT_ERR_MSG(session, EINVAL, "comparison method cursors must reference the same object"); + + WT_ERR(__cursor_needkey(a)); + WT_ERR(__cursor_needkey(b)); + + WT_ERR(__wt_compare(session, alsm->tiered->collator, &a->key, &b->key, cmpp)); + +err: + API_END_RET(session, ret); +} + +/* + * __curtiered_position_chunk -- + * Position a chunk cursor. + */ +static int +__curtiered_position_chunk(WT_CURSOR_TIERED *curtiered, WT_CURSOR *c, bool forward, int *cmpp) +{ + WT_CURSOR *cursor; + WT_SESSION_IMPL *session; + + cursor = &curtiered->iface; + session = CUR2S(cursor); + + c->set_key(c, &cursor->key); + WT_RET(c->search_near(c, cmpp)); + + while (forward ? *cmpp < 0 : *cmpp > 0) { + WT_RET(forward ? c->next(c) : c->prev(c)); + + /* + * With higher isolation levels, where we have stable reads, we're done: the cursor is now + * positioned as expected. + * + * With read-uncommitted isolation, a new record could have appeared in between the search + * and stepping forward / back. In that case, keep going until we see a key in the expected + * range. + */ + if (session->txn->isolation != WT_ISO_READ_UNCOMMITTED) + return (0); + + WT_RET(WT_TIERED_CURCMP(session, curtiered->tiered, c, cursor, *cmpp)); + } + + return (0); +} + +/* + * __curtiered_next -- + * WT_CURSOR->next method for the LSM cursor type. + */ +static int +__curtiered_next(WT_CURSOR *cursor) +{ + WT_CURSOR *c; + WT_CURSOR_TIERED *curtiered; + WT_DECL_RET; + WT_SESSION_IMPL *session; + u_int i; + int cmp; + bool deleted; + + curtiered = (WT_CURSOR_TIERED *)cursor; + + CURSOR_API_CALL(cursor, session, next, NULL); + __cursor_novalue(cursor); + WT_ERR(__curtiered_enter(curtiered, false)); + + /* If we aren't positioned for a forward scan, get started. */ + if (curtiered->current == NULL || !F_ISSET(curtiered, WT_CLSM_ITERATE_NEXT)) { + WT_FORALL_CURSORS(curtiered, c, i) + { + if (!F_ISSET(cursor, WT_CURSTD_KEY_SET)) { + WT_ERR(c->reset(c)); + ret = c->next(c); + } else if (c != curtiered->current && + (ret = __curtiered_position_chunk(curtiered, c, true, &cmp)) == 0 && cmp == 0 && + curtiered->current == NULL) + curtiered->current = c; + WT_ERR_NOTFOUND_OK(ret, false); + } + F_SET(curtiered, WT_CLSM_ITERATE_NEXT | WT_CLSM_MULTIPLE); + F_CLR(curtiered, WT_CLSM_ITERATE_PREV); + + /* We just positioned *at* the key, now move. */ + if (curtiered->current != NULL) + goto retry; + } else { +retry: + /* + * If there are multiple cursors on that key, move them forward. + */ + if (F_ISSET(curtiered, WT_CLSM_MULTIPLE)) { + WT_FORALL_CURSORS(curtiered, c, i) + { + if (!F_ISSET(c, WT_CURSTD_KEY_INT)) + continue; + if (c != curtiered->current) { + WT_ERR( + WT_TIERED_CURCMP(session, curtiered->tiered, c, curtiered->current, cmp)); + if (cmp == 0) + WT_ERR_NOTFOUND_OK(c->next(c), false); + } + } + } + + /* Move the smallest cursor forward. */ + c = curtiered->current; + WT_ERR_NOTFOUND_OK(c->next(c), false); + } + + /* Find the cursor(s) with the smallest key. */ + if ((ret = __curtiered_get_current(session, curtiered, true, &deleted)) == 0 && deleted) + goto retry; + +err: + __curtiered_leave(curtiered); + if (ret == 0) + __curtiered_deleted_decode(curtiered, &cursor->value); + API_END_RET(session, ret); +} + +/* + * __curtiered_prev -- + * WT_CURSOR->prev method for the LSM cursor type. + */ +static int +__curtiered_prev(WT_CURSOR *cursor) +{ + WT_CURSOR *c; + WT_CURSOR_TIERED *curtiered; + WT_DECL_RET; + WT_SESSION_IMPL *session; + u_int i; + int cmp; + bool deleted; + + curtiered = (WT_CURSOR_TIERED *)cursor; + + CURSOR_API_CALL(cursor, session, prev, NULL); + __cursor_novalue(cursor); + WT_ERR(__curtiered_enter(curtiered, false)); + + /* If we aren't positioned for a reverse scan, get started. */ + if (curtiered->current == NULL || !F_ISSET(curtiered, WT_CLSM_ITERATE_PREV)) { + WT_FORALL_CURSORS(curtiered, c, i) + { + if (!F_ISSET(cursor, WT_CURSTD_KEY_SET)) { + WT_ERR(c->reset(c)); + ret = c->prev(c); + } else if (c != curtiered->current && + (ret = __curtiered_position_chunk(curtiered, c, false, &cmp)) == 0 && cmp == 0 && + curtiered->current == NULL) + curtiered->current = c; + WT_ERR_NOTFOUND_OK(ret, false); + } + F_SET(curtiered, WT_CLSM_ITERATE_PREV | WT_CLSM_MULTIPLE); + F_CLR(curtiered, WT_CLSM_ITERATE_NEXT); + + /* We just positioned *at* the key, now move. */ + if (curtiered->current != NULL) + goto retry; + } else { +retry: + /* + * If there are multiple cursors on that key, move them backwards. + */ + if (F_ISSET(curtiered, WT_CLSM_MULTIPLE)) { + WT_FORALL_CURSORS(curtiered, c, i) + { + if (!F_ISSET(c, WT_CURSTD_KEY_INT)) + continue; + if (c != curtiered->current) { + WT_ERR( + WT_TIERED_CURCMP(session, curtiered->tiered, c, curtiered->current, cmp)); + if (cmp == 0) + WT_ERR_NOTFOUND_OK(c->prev(c), false); + } + } + } + + /* Move the largest cursor backwards. */ + c = curtiered->current; + WT_ERR_NOTFOUND_OK(c->prev(c), false); + } + + /* Find the cursor(s) with the largest key. */ + if ((ret = __curtiered_get_current(session, curtiered, false, &deleted)) == 0 && deleted) + goto retry; + +err: + __curtiered_leave(curtiered); + if (ret == 0) + __curtiered_deleted_decode(curtiered, &cursor->value); + API_END_RET(session, ret); +} + +/* + * __curtiered_reset -- + * WT_CURSOR->reset method for the LSM cursor type. + */ +static int +__curtiered_reset(WT_CURSOR *cursor) +{ + WT_CURSOR_TIERED *curtiered; + WT_DECL_RET; + WT_SESSION_IMPL *session; + + /* + * Don't use the normal __curtiered_enter path: that is wasted work when all we want to do is + * give up our position. + */ + curtiered = (WT_CURSOR_TIERED *)cursor; + CURSOR_API_CALL_PREPARE_ALLOWED(cursor, session, reset, NULL); + F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + + WT_TRET(__curtiered_reset_cursors(curtiered, NULL)); + + /* In case we were left positioned, clear that. */ + __curtiered_leave(curtiered); + +err: + API_END_RET(session, ret); +} + +/* + * __curtiered_lookup -- + * Position an LSM cursor. + */ +static int +__curtiered_lookup(WT_CURSOR_TIERED *curtiered, WT_ITEM *value) +{ + WT_CURSOR *c, *cursor; + WT_DECL_RET; + u_int i; + + c = NULL; + cursor = &curtiered->iface; + + WT_FORALL_CURSORS(curtiered, c, i) + { + c->set_key(c, &cursor->key); + if ((ret = c->search(c)) == 0) { + WT_ERR(c->get_key(c, &cursor->key)); + WT_ERR(c->get_value(c, value)); + if (__curtiered_deleted(curtiered, value)) + ret = WT_NOTFOUND; + goto done; + } + WT_ERR_NOTFOUND_OK(ret, false); + F_CLR(c, WT_CURSTD_KEY_SET); + } + WT_ERR(WT_NOTFOUND); + +done: +err: + if (ret == 0) { + F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + F_SET(cursor, WT_CURSTD_KEY_INT); + curtiered->current = c; + if (value == &cursor->value) + F_SET(cursor, WT_CURSTD_VALUE_INT); + } else if (c != NULL) + WT_TRET(c->reset(c)); + + return (ret); +} + +/* + * __curtiered_search -- + * WT_CURSOR->search method for the LSM cursor type. + */ +static int +__curtiered_search(WT_CURSOR *cursor) +{ + WT_CURSOR_TIERED *curtiered; + WT_DECL_RET; + WT_SESSION_IMPL *session; + + curtiered = (WT_CURSOR_TIERED *)cursor; + + CURSOR_API_CALL(cursor, session, search, NULL); + WT_ERR(__cursor_needkey(cursor)); + __cursor_novalue(cursor); + WT_ERR(__curtiered_enter(curtiered, true)); + F_CLR(curtiered, WT_CLSM_ITERATE_NEXT | WT_CLSM_ITERATE_PREV); + + ret = __curtiered_lookup(curtiered, &cursor->value); + +err: + __curtiered_leave(curtiered); + if (ret == 0) + __curtiered_deleted_decode(curtiered, &cursor->value); + API_END_RET(session, ret); +} + +/* + * __curtiered_search_near -- + * WT_CURSOR->search_near method for the LSM cursor type. + */ +static int +__curtiered_search_near(WT_CURSOR *cursor, int *exactp) +{ + WT_CURSOR *c, *closest; + WT_CURSOR_TIERED *curtiered; + WT_DECL_RET; + WT_SESSION_IMPL *session; + u_int i; + int cmp, exact; + bool deleted; + + closest = NULL; + curtiered = (WT_CURSOR_TIERED *)cursor; + exact = 0; + + CURSOR_API_CALL(cursor, session, search_near, NULL); + WT_ERR(__cursor_needkey(cursor)); + __cursor_novalue(cursor); + WT_ERR(__curtiered_enter(curtiered, true)); + F_CLR(curtiered, WT_CLSM_ITERATE_NEXT | WT_CLSM_ITERATE_PREV); + + /* + * search_near is somewhat fiddly: we can't just use a nearby key from the in-memory chunk + * because there could be a closer key on disk. + * + * As we search down the chunks, we stop as soon as we find an exact match. Otherwise, we + * maintain the smallest cursor larger than the search key and the largest cursor smaller than + * the search key. At the end, we prefer the larger cursor, but if no record is larger, position + * on the last record in the tree. + */ + WT_FORALL_CURSORS(curtiered, c, i) + { + c->set_key(c, &cursor->key); + if ((ret = c->search_near(c, &cmp)) == WT_NOTFOUND) { + ret = 0; + continue; + } + if (ret != 0) + goto err; + + /* Do we have an exact match? */ + if (cmp == 0) { + closest = c; + exact = 1; + break; + } + + /* + * Prefer larger cursors. There are two reasons: (1) we expect + * prefix searches to be a common case (as in our own indices); + * and (2) we need a way to unambiguously know we have the + * "closest" result. + */ + if (cmp < 0) { + if ((ret = c->next(c)) == WT_NOTFOUND) { + ret = 0; + continue; + } + if (ret != 0) + goto err; + } + + /* + * We are trying to find the smallest cursor greater than the search key. + */ + if (closest == NULL) + closest = c; + else { + WT_ERR(WT_TIERED_CURCMP(session, curtiered->tiered, c, closest, cmp)); + if (cmp < 0) + closest = c; + } + } + + /* + * At this point, we either have an exact match, or closest is the smallest cursor larger than + * the search key, or it is NULL if the search key is larger than any record in the tree. + */ + cmp = exact ? 0 : 1; + + /* + * If we land on a deleted item, try going forwards or backwards to find one that isn't deleted. + * If the whole tree is empty, we'll end up with WT_NOTFOUND, as expected. + */ + if (closest == NULL) + deleted = true; + else { + WT_ERR(closest->get_key(closest, &cursor->key)); + WT_ERR(closest->get_value(closest, &cursor->value)); + curtiered->current = closest; + closest = NULL; + deleted = __curtiered_deleted(curtiered, &cursor->value); + if (!deleted) + __curtiered_deleted_decode(curtiered, &cursor->value); + else { + /* + * We have a key pointing at memory that is pinned by the current chunk cursor. In the + * unlikely event that we have to reopen cursors to move to the next record, make sure + * the cursor flags are set so a copy is made before the current chunk cursor releases + * its position. + */ + F_CLR(cursor, WT_CURSTD_KEY_SET); + F_SET(cursor, WT_CURSTD_KEY_INT); + /* + * We call __curtiered_next here as we want to advance forward. If we are a random LSM + * cursor calling next on the cursor will not advance as we intend. + */ + if ((ret = __curtiered_next(cursor)) == 0) { + cmp = 1; + deleted = false; + } + } + WT_ERR_NOTFOUND_OK(ret, false); + } + if (deleted) { + curtiered->current = NULL; + /* + * We call prev directly here as cursor->prev may be "invalid" if this is a random cursor. + */ + WT_ERR(__curtiered_prev(cursor)); + cmp = -1; + } + *exactp = cmp; + +err: + __curtiered_leave(curtiered); + if (closest != NULL) + WT_TRET(closest->reset(closest)); + + F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + if (ret == 0) { + F_SET(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); + } else + curtiered->current = NULL; + + API_END_RET(session, ret); +} + +/* + * __curtiered_put -- + * Put an entry into the primary tree. + */ +static inline int +__curtiered_put(WT_CURSOR_TIERED *curtiered, const WT_ITEM *key, const WT_ITEM *value, + bool position, bool reserve) +{ + WT_CURSOR *primary; + WT_TIERED *tiered; + + tiered = curtiered->tiered; + + /* + * Clear the existing cursor position. Don't clear the primary cursor: we're about to use it + * anyway. + */ + primary = curtiered->cursors[tiered->ntiers - 1]; + WT_RET(__curtiered_reset_cursors(curtiered, primary)); + + /* If necessary, set the position for future scans. */ + if (position) + curtiered->current = primary; + + primary->set_key(primary, key); + if (reserve) { + WT_RET(primary->reserve(primary)); + } else { + primary->set_value(primary, value); + WT_RET(primary->insert(primary)); + } + + return (0); +} + +/* + * __curtiered_insert -- + * WT_CURSOR->insert method for the LSM cursor type. + */ +static int +__curtiered_insert(WT_CURSOR *cursor) +{ + WT_CURSOR_TIERED *curtiered; + WT_DECL_ITEM(buf); + WT_DECL_RET; + WT_ITEM value; + WT_SESSION_IMPL *session; + + curtiered = (WT_CURSOR_TIERED *)cursor; + + CURSOR_UPDATE_API_CALL(cursor, session, insert); + WT_ERR(__cursor_needkey(cursor)); + WT_ERR(__cursor_needvalue(cursor)); + WT_ERR(__curtiered_enter(curtiered, false)); + + /* + * It isn't necessary to copy the key out after the lookup in this case because any non-failed + * lookup results in an error, and a failed lookup leaves the original key intact. + */ + if (!F_ISSET(cursor, WT_CURSTD_OVERWRITE) && + (ret = __curtiered_lookup(curtiered, &value)) != WT_NOTFOUND) { + if (ret == 0) + ret = WT_DUPLICATE_KEY; + goto err; + } + + WT_ERR(__curtiered_deleted_encode(session, &cursor->value, &value, &buf)); + WT_ERR(__curtiered_put(curtiered, &cursor->key, &value, false, false)); + + /* + * WT_CURSOR.insert doesn't leave the cursor positioned, and the application may want to free + * the memory used to configure the insert; don't read that memory again (matching the + * underlying file object cursor insert semantics). + */ + F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + +err: + __wt_scr_free(session, &buf); + __curtiered_leave(curtiered); + CURSOR_UPDATE_API_END(session, ret); + return (ret); +} + +/* + * __curtiered_update -- + * WT_CURSOR->update method for the LSM cursor type. + */ +static int +__curtiered_update(WT_CURSOR *cursor) +{ + WT_CURSOR_TIERED *curtiered; + WT_DECL_ITEM(buf); + WT_DECL_RET; + WT_ITEM value; + WT_SESSION_IMPL *session; + + curtiered = (WT_CURSOR_TIERED *)cursor; + + CURSOR_UPDATE_API_CALL(cursor, session, update); + WT_ERR(__cursor_needkey(cursor)); + WT_ERR(__cursor_needvalue(cursor)); + WT_ERR(__curtiered_enter(curtiered, false)); + + if (!F_ISSET(cursor, WT_CURSTD_OVERWRITE)) { + WT_ERR(__curtiered_lookup(curtiered, &value)); + /* + * Copy the key out, since the insert resets non-primary chunk cursors which our lookup may + * have landed on. + */ + WT_ERR(__cursor_needkey(cursor)); + } + WT_ERR(__curtiered_deleted_encode(session, &cursor->value, &value, &buf)); + WT_ERR(__curtiered_put(curtiered, &cursor->key, &value, true, false)); + + /* + * Set the cursor to reference the internal key/value of the positioned cursor. + */ + F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + WT_ITEM_SET(cursor->key, curtiered->current->key); + WT_ITEM_SET(cursor->value, curtiered->current->value); + WT_ASSERT(session, F_MASK(curtiered->current, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT); + WT_ASSERT(session, F_MASK(curtiered->current, WT_CURSTD_VALUE_SET) == WT_CURSTD_VALUE_INT); + F_SET(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); + +err: + __wt_scr_free(session, &buf); + __curtiered_leave(curtiered); + CURSOR_UPDATE_API_END(session, ret); + return (ret); +} + +/* + * __curtiered_remove -- + * WT_CURSOR->remove method for the LSM cursor type. + */ +static int +__curtiered_remove(WT_CURSOR *cursor) +{ + WT_CURSOR_TIERED *curtiered; + WT_DECL_RET; + WT_ITEM value; + WT_SESSION_IMPL *session; + bool positioned; + + curtiered = (WT_CURSOR_TIERED *)cursor; + + /* Check if the cursor is positioned. */ + positioned = F_ISSET(cursor, WT_CURSTD_KEY_INT); + + CURSOR_REMOVE_API_CALL(cursor, session, NULL); + WT_ERR(__cursor_needkey(cursor)); + __cursor_novalue(cursor); + WT_ERR(__curtiered_enter(curtiered, false)); + + if (!F_ISSET(cursor, WT_CURSTD_OVERWRITE)) { + WT_ERR(__curtiered_lookup(curtiered, &value)); + /* + * Copy the key out, since the insert resets non-primary chunk cursors which our lookup may + * have landed on. + */ + WT_ERR(__cursor_needkey(cursor)); + } + WT_ERR(__curtiered_put(curtiered, &cursor->key, &__tombstone, positioned, false)); + + /* + * If the cursor was positioned, it stays positioned with a key but no no value, otherwise, + * there's no position, key or value. This isn't just cosmetic, without a reset, iteration on + * this cursor won't start at the beginning/end of the table. + */ + F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + if (positioned) + F_SET(cursor, WT_CURSTD_KEY_INT); + else + WT_TRET(cursor->reset(cursor)); + +err: + __curtiered_leave(curtiered); + CURSOR_UPDATE_API_END(session, ret); + return (ret); +} + +/* + * __curtiered_reserve -- + * WT_CURSOR->reserve method for the LSM cursor type. + */ +static int +__curtiered_reserve(WT_CURSOR *cursor) +{ + WT_CURSOR_TIERED *curtiered; + WT_DECL_RET; + WT_ITEM value; + WT_SESSION_IMPL *session; + + curtiered = (WT_CURSOR_TIERED *)cursor; + + CURSOR_UPDATE_API_CALL(cursor, session, reserve); + WT_ERR(__cursor_needkey(cursor)); + __cursor_novalue(cursor); + WT_ERR(__wt_txn_context_check(session, true)); + WT_ERR(__curtiered_enter(curtiered, false)); + + WT_ERR(__curtiered_lookup(curtiered, &value)); + /* + * Copy the key out, since the insert resets non-primary chunk cursors which our lookup may have + * landed on. + */ + WT_ERR(__cursor_needkey(cursor)); + ret = __curtiered_put(curtiered, &cursor->key, NULL, true, true); + +err: + __curtiered_leave(curtiered); + CURSOR_UPDATE_API_END(session, ret); + + /* + * The application might do a WT_CURSOR.get_value call when we return, so we need a value and + * the underlying functions didn't set one up. For various reasons, those functions may not have + * done a search and any previous value in the cursor might race with WT_CURSOR.reserve (and in + * cases like LSM, the reserve never encountered the original key). For simplicity, repeat the + * search here. + */ + return (ret == 0 ? cursor->search(cursor) : ret); +} + +/* + * __curtiered_random_chunk -- + * Pick a chunk at random, weighted by the size of all chunks. Weighting proportional to + * documents avoids biasing towards small chunks. Then return the cursor on the chunk we have + * picked. + */ +static int +__curtiered_random_chunk(WT_SESSION_IMPL *session, WT_CURSOR_TIERED *curtiered, WT_CURSOR **cursor) +{ + u_int i, ntiers; + + /* + * If the tree is empty we cannot do a random lookup, so return a WT_NOTFOUND. + */ + if ((ntiers = curtiered->tiered->ntiers) == 0) + return (WT_NOTFOUND); + + /* TODO: make randomness respect tree size. */ + i = __wt_random(&session->rnd) % ntiers; + *cursor = curtiered->cursors[i]; + return (0); +} + +/* + * __curtiered_next_random -- + * WT_CURSOR->next method for the LSM cursor type when configured with next_random. + */ +static int +__curtiered_next_random(WT_CURSOR *cursor) +{ + WT_CURSOR *c; + WT_CURSOR_TIERED *curtiered; + WT_DECL_RET; + WT_SESSION_IMPL *session; + int exact; + + c = NULL; + curtiered = (WT_CURSOR_TIERED *)cursor; + + CURSOR_API_CALL(cursor, session, next, NULL); + __cursor_novalue(cursor); + WT_ERR(__curtiered_enter(curtiered, false)); + + for (;;) { + WT_ERR(__curtiered_random_chunk(session, curtiered, &c)); + /* + * This call to next_random on the chunk can potentially end in WT_NOTFOUND if the chunk we + * picked is empty. We want to retry in that case. + */ + WT_ERR_NOTFOUND_OK(__wt_curfile_next_random(c), true); + if (ret == WT_NOTFOUND) + continue; + + F_SET(cursor, WT_CURSTD_KEY_INT); + WT_ERR(c->get_key(c, &cursor->key)); + /* + * Search near the current key to resolve any tombstones and position to a valid document. + * If we see a WT_NOTFOUND here that is valid, as the tree has no documents visible to us. + */ + WT_ERR(__curtiered_search_near(cursor, &exact)); + break; + } + + /* We have found a valid doc. Set that we are now positioned */ + if (0) { +err: + F_CLR(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); + } + __curtiered_leave(curtiered); + API_END_RET(session, ret); +} + +/* + * __wt_curtiered_open -- + * WT_SESSION->open_cursor method for LSM cursors. + */ +int +__wt_curtiered_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], + WT_CURSOR **cursorp) +{ + WT_CONFIG_ITEM cval; + WT_CURSOR_STATIC_INIT(iface, __wt_cursor_get_key, /* get-key */ + __wt_cursor_get_value, /* get-value */ + __wt_cursor_set_key, /* set-key */ + __wt_cursor_set_value, /* set-value */ + __curtiered_compare, /* compare */ + __wt_cursor_equals, /* equals */ + __curtiered_next, /* next */ + __curtiered_prev, /* prev */ + __curtiered_reset, /* reset */ + __curtiered_search, /* search */ + __curtiered_search_near, /* search-near */ + __curtiered_insert, /* insert */ + __wt_cursor_modify_value_format_notsup, /* modify */ + __curtiered_update, /* update */ + __curtiered_remove, /* remove */ + __curtiered_reserve, /* reserve */ + __wt_cursor_reconfigure, /* reconfigure */ + __wt_cursor_notsup, /* cache */ + __wt_cursor_reopen_notsup, /* reopen */ + __wt_curtiered_close); /* close */ + WT_CURSOR *cursor; + WT_CURSOR_TIERED *curtiered; + WT_DECL_RET; + WT_TIERED *tiered; + bool bulk; + + WT_STATIC_ASSERT(offsetof(WT_CURSOR_TIERED, iface) == 0); + + curtiered = NULL; + cursor = NULL; + tiered = NULL; + + if (!WT_PREFIX_MATCH(uri, "tiered:")) + return (__wt_unexpected_object_type(session, uri, "tiered:")); + + WT_RET(__wt_config_gets_def(session, cfg, "checkpoint", 0, &cval)); + if (cval.len != 0) + WT_RET_MSG(session, EINVAL, "LSM does not support opening by checkpoint"); + + WT_RET(__wt_config_gets_def(session, cfg, "bulk", 0, &cval)); + bulk = cval.val != 0; + + /* Get the tiered data handle. */ + ret = __wt_session_get_dhandle(session, uri, NULL, cfg, bulk ? WT_DHANDLE_EXCLUSIVE : 0); + + /* Check whether the exclusive open for a bulk load succeeded. */ + if (bulk && ret == EBUSY) + WT_ERR_MSG(session, EINVAL, "bulk-load is only supported on newly created trees"); + /* Flag any errors from the tree get. */ + WT_ERR(ret); + + tiered = (WT_TIERED *)session->dhandle; + + /* Make sure we have exclusive access if and only if we want it */ + WT_ASSERT(session, !bulk || tiered->iface.excl_session != NULL); + + WT_ERR(__wt_calloc_one(session, &curtiered)); + cursor = (WT_CURSOR *)curtiered; + *cursor = iface; + cursor->session = (WT_SESSION *)session; + WT_ERR(__wt_strdup(session, tiered->name, &cursor->uri)); + cursor->key_format = tiered->key_format; + cursor->value_format = tiered->value_format; + + curtiered->tiered = tiered; + tiered = NULL; + + /* If the next_random option is set, configure a random cursor */ + WT_ERR(__wt_config_gets_def(session, cfg, "next_random", 0, &cval)); + if (cval.val != 0) { + __wt_cursor_set_notsup(cursor); + cursor->next = __curtiered_next_random; + } + + WT_ERR(__wt_cursor_init(cursor, cursor->uri, owner, cfg, cursorp)); + + if (bulk) + WT_ERR(ENOTSUP); /* TODO */ + + if (0) { +err: + if (curtiered != NULL) + WT_TRET(__wt_curtiered_close(cursor)); + else if (tiered != NULL) + WT_WITH_DHANDLE( + session, (WT_DATA_HANDLE *)tiered, WT_TRET(__wt_session_release_dhandle(session))); + + *cursorp = NULL; + } + + return (ret); +} diff --git a/src/third_party/wiredtiger/src/tiered/tiered_schema.c b/src/third_party/wiredtiger/src/tiered/tiered_schema.c new file mode 100644 index 00000000000..dc153b31e43 --- /dev/null +++ b/src/third_party/wiredtiger/src/tiered/tiered_schema.c @@ -0,0 +1,255 @@ +/*- + * Copyright (c) 2014-2020 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_tiered_create -- + * Create a tiered tree structure for the given name. + */ +int +__wt_tiered_create(WT_SESSION_IMPL *session, const char *uri, bool exclusive, const char *config) +{ + WT_DECL_RET; + char *meta_value; + const char *cfg[] = {WT_CONFIG_BASE(session, tiered_meta), config, NULL}; + const char *metadata; + + metadata = NULL; + + /* If it can be opened, it already exists. */ + if ((ret = __wt_metadata_search(session, uri, &meta_value)) != WT_NOTFOUND) { + if (exclusive) + WT_TRET(EEXIST); + goto err; + } + WT_RET_NOTFOUND_OK(ret); + + if (!F_ISSET(S2C(session), WT_CONN_READONLY)) { + WT_ERR(__wt_config_merge(session, cfg, NULL, &metadata)); + WT_ERR(__wt_metadata_insert(session, uri, metadata)); + } + +err: + __wt_free(session, metadata); + return (ret); +} + +/* + * __wt_tiered_drop -- + * Drop a tiered store. + */ +int +__wt_tiered_drop(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) +{ + WT_DATA_HANDLE *tier; + WT_DECL_RET; + WT_TIERED *tiered; + u_int i; + + /* Get the tiered data handle. */ + WT_RET(__wt_session_get_dhandle(session, uri, NULL, NULL, WT_DHANDLE_EXCLUSIVE)); + tiered = (WT_TIERED *)session->dhandle; + + /* Drop the tiers. */ + for (i = 0; i < tiered->ntiers; i++) { + tier = tiered->tiers[i]; + WT_ERR(__wt_schema_drop(session, tier->name, cfg)); + } + + ret = __wt_metadata_remove(session, uri); + +err: + F_SET(session->dhandle, WT_DHANDLE_DISCARD); + WT_TRET(__wt_session_release_dhandle(session)); + return (ret); +} + +/* + * __wt_tiered_rename -- + * Rename a tiered data source. + */ +int +__wt_tiered_rename( + WT_SESSION_IMPL *session, const char *olduri, const char *newuri, const char *cfg[]) +{ + WT_DECL_RET; + WT_TIERED *tiered; + + /* Get the tiered data handle. */ + WT_RET(__wt_session_get_dhandle(session, olduri, NULL, NULL, WT_DHANDLE_EXCLUSIVE)); + tiered = (WT_TIERED *)session->dhandle; + + /* TODO */ + WT_UNUSED(olduri); + WT_UNUSED(newuri); + WT_UNUSED(cfg); + WT_UNUSED(tiered); + + F_SET(session->dhandle, WT_DHANDLE_DISCARD); + WT_TRET(__wt_session_release_dhandle(session)); + + return (ret); +} + +/* + * __wt_tiered_truncate -- + * Truncate for a tiered data source. + */ +int +__wt_tiered_truncate(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) +{ + WT_DECL_RET; + WT_TIERED *tiered; + u_int i; + + WT_RET(__wt_session_get_dhandle(session, uri, NULL, NULL, WT_DHANDLE_EXCLUSIVE)); + tiered = (WT_TIERED *)session->dhandle; + + WT_STAT_DATA_INCR(session, cursor_truncate); + + /* Truncate the column groups. */ + for (i = 0; i < tiered->ntiers; i++) + WT_ERR(__wt_schema_truncate(session, tiered->tiers[i]->name, cfg)); + +err: + WT_TRET(__wt_session_release_dhandle(session)); + return (ret); +} + +/* + * __wt_tiered_worker -- + * Run a schema worker operation on each tier of a tiered data source. + */ +int +__wt_tiered_worker(WT_SESSION_IMPL *session, const char *uri, + int (*file_func)(WT_SESSION_IMPL *, const char *[]), + int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), const char *cfg[], uint32_t open_flags) +{ + WT_DATA_HANDLE *dhandle; + WT_DECL_RET; + WT_TIERED *tiered; + u_int i; + + /* + * If this was an alter operation, we need to alter the configuration for the overall tree and + * then reread it so it isn't out of date. TODO not yet supported. + */ + if (FLD_ISSET(open_flags, WT_BTREE_ALTER)) + WT_RET(ENOTSUP); + + WT_RET(__wt_session_get_dhandle(session, uri, NULL, NULL, open_flags)); + tiered = (WT_TIERED *)session->dhandle; + + for (i = 0; i < tiered->ntiers; i++) { + dhandle = tiered->tiers[i]; + WT_SAVE_DHANDLE(session, + ret = __wt_schema_worker(session, dhandle->name, file_func, name_func, cfg, open_flags)); + WT_ERR(ret); + } + +err: + WT_TRET(__wt_session_release_dhandle(session)); + return (ret); +} + +/* + * __tiered_open -- + * Open a tiered data handle (internal version). + */ +static int +__tiered_open(WT_SESSION_IMPL *session, const char *cfg[]) +{ + WT_CONFIG cparser; + WT_CONFIG_ITEM ckey, cval, tierconf; + WT_DATA_HANDLE *dhandle; + WT_DECL_ITEM(buf); + WT_DECL_RET; + WT_TIERED *tiered; + u_int i; + const char **tiered_cfg; + + dhandle = session->dhandle; + tiered = (WT_TIERED *)dhandle; + tiered_cfg = dhandle->cfg; + + WT_UNUSED(cfg); + + WT_RET(__wt_config_gets(session, tiered_cfg, "key_format", &cval)); + WT_RET(__wt_strndup(session, cval.str, cval.len, &tiered->key_format)); + WT_RET(__wt_config_gets(session, tiered_cfg, "value_format", &cval)); + WT_RET(__wt_strndup(session, cval.str, cval.len, &tiered->value_format)); + + /* Point to some items in the copy to save re-parsing. */ + WT_RET(__wt_config_gets(session, tiered_cfg, "tiered.tiers", &tierconf)); + + /* + * Count the number of tiers. + */ + __wt_config_subinit(session, &cparser, &tierconf); + while ((ret = __wt_config_next(&cparser, &ckey, &cval)) == 0) + ++tiered->ntiers; + WT_RET_NOTFOUND_OK(ret); + + WT_RET(__wt_scr_alloc(session, 0, &buf)); + WT_ERR(__wt_calloc_def(session, tiered->ntiers, &tiered->tiers)); + + __wt_config_subinit(session, &cparser, &tierconf); + for (i = 0; i < tiered->ntiers; i++) { + WT_ERR(__wt_config_next(&cparser, &ckey, &cval)); + WT_ERR(__wt_buf_fmt(session, buf, "%.*s", (int)ckey.len, ckey.str)); + WT_ERR(__wt_session_get_dhandle(session, (const char *)buf->data, NULL, cfg, 0)); + __wt_atomic_addi32(&session->dhandle->session_inuse, 1); + /* Load in reverse order (based on LSM logic). */ + tiered->tiers[(tiered->ntiers - 1) - i] = session->dhandle; + WT_ERR(__wt_session_release_dhandle(session)); + } + + if (0) { +err: + __wt_free(session, tiered->tiers); + } + __wt_scr_free(session, &buf); + return (ret); +} + +/* + * __wt_tiered_open -- + * Open a tiered data handle. + */ +int +__wt_tiered_open(WT_SESSION_IMPL *session, const char *cfg[]) +{ + WT_DECL_RET; + + WT_WITH_TXN_ISOLATION(session, WT_ISO_READ_UNCOMMITTED, ret = __tiered_open(session, cfg)); + + return (ret); +} + +/* + * __wt_tiered_close -- + * Close a tiered data handle. + */ +int +__wt_tiered_close(WT_SESSION_IMPL *session, WT_TIERED *tiered) +{ + WT_DECL_RET; + u_int i; + + ret = 0; + __wt_free(session, tiered->key_format); + __wt_free(session, tiered->value_format); + if (tiered->tiers != NULL) { + for (i = 0; i < tiered->ntiers; i++) + __wt_atomic_subi32(&tiered->tiers[i]->session_inuse, 1); + __wt_free(session, tiered->tiers); + } + + return (ret); +} diff --git a/src/third_party/wiredtiger/src/txn/txn_log.c b/src/third_party/wiredtiger/src/txn/txn_log.c index b335d4ca3cb..818f03314aa 100644 --- a/src/third_party/wiredtiger/src/txn/txn_log.c +++ b/src/third_party/wiredtiger/src/txn/txn_log.c @@ -730,8 +730,8 @@ __txn_printlog(WT_SESSION_IMPL *session, WT_ITEM *rawrec, WT_LSN *lsnp, WT_LSN * * Print the log in a human-readable format. */ int -__wt_txn_printlog(WT_SESSION *wt_session, const char *ofile, uint32_t flags) - WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) +__wt_txn_printlog(WT_SESSION *wt_session, const char *ofile, uint32_t flags, WT_LSN *start_lsn, + WT_LSN *end_lsn) WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_DECL_RET; WT_FSTREAM *fs; @@ -749,7 +749,7 @@ __wt_txn_printlog(WT_SESSION *wt_session, const char *ofile, uint32_t flags) WT_ERR(__wt_fprintf(session, fs, "[\n")); args.fs = fs; args.flags = flags; - WT_ERR(__wt_log_scan(session, NULL, WT_LOGSCAN_FIRST, __txn_printlog, &args)); + WT_ERR(__wt_log_scan(session, start_lsn, end_lsn, 0x0, __txn_printlog, &args)); if (!LF_ISSET(WT_TXN_PRINTLOG_MSG)) ret = __wt_fprintf(session, fs, "\n]\n"); diff --git a/src/third_party/wiredtiger/src/txn/txn_recover.c b/src/third_party/wiredtiger/src/txn/txn_recover.c index 26154f2d776..e81451a96b3 100644 --- a/src/third_party/wiredtiger/src/txn/txn_recover.c +++ b/src/third_party/wiredtiger/src/txn/txn_recover.c @@ -817,15 +817,15 @@ __wt_txn_recover(WT_SESSION_IMPL *session, const char *cfg[]) WT_ERR_MSG(session, WT_RUN_RECOVERY, "Read-only database needs recovery"); } if (WT_IS_INIT_LSN(&metafile->ckpt_lsn)) - ret = __wt_log_scan(session, NULL, WT_LOGSCAN_FIRST, __txn_log_recover, &r); + ret = __wt_log_scan(session, NULL, NULL, WT_LOGSCAN_FIRST, __txn_log_recover, &r); else { /* * Start at the last checkpoint LSN referenced in the metadata. If we see the end of a * checkpoint while scanning, we will change the full scan to start from there. */ WT_ASSIGN_LSN(&r.ckpt_lsn, &metafile->ckpt_lsn); - ret = __wt_log_scan( - session, &metafile->ckpt_lsn, WT_LOGSCAN_RECOVER_METADATA, __txn_log_recover, &r); + ret = __wt_log_scan(session, &metafile->ckpt_lsn, NULL, WT_LOGSCAN_RECOVER_METADATA, + __txn_log_recover, &r); } if (F_ISSET(conn, WT_CONN_SALVAGE)) ret = 0; @@ -917,9 +917,9 @@ __wt_txn_recover(WT_SESSION_IMPL *session, const char *cfg[]) FLD_SET(conn->log_flags, WT_CONN_LOG_RECOVER_DIRTY); if (WT_IS_INIT_LSN(&r.ckpt_lsn)) ret = __wt_log_scan( - session, NULL, WT_LOGSCAN_FIRST | WT_LOGSCAN_RECOVER, __txn_log_recover, &r); + session, NULL, NULL, WT_LOGSCAN_FIRST | WT_LOGSCAN_RECOVER, __txn_log_recover, &r); else - ret = __wt_log_scan(session, &r.ckpt_lsn, WT_LOGSCAN_RECOVER, __txn_log_recover, &r); + ret = __wt_log_scan(session, &r.ckpt_lsn, NULL, WT_LOGSCAN_RECOVER, __txn_log_recover, &r); if (F_ISSET(conn, WT_CONN_SALVAGE)) ret = 0; WT_ERR(ret); diff --git a/src/third_party/wiredtiger/src/utilities/util_list.c b/src/third_party/wiredtiger/src/utilities/util_list.c index ab2e6ad1299..13a3577745f 100644 --- a/src/third_party/wiredtiger/src/utilities/util_list.c +++ b/src/third_party/wiredtiger/src/utilities/util_list.c @@ -8,7 +8,7 @@ #include "util.h" -static int list_get_allocsize(WT_SESSION *, const char *, size_t *); +static int list_init_block(WT_SESSION *, const char *, WT_BLOCK *); static int list_print(WT_SESSION *, const char *, bool, bool); static int list_print_checkpoint(WT_SESSION *, const char *); @@ -67,20 +67,20 @@ util_list(WT_SESSION *session, int argc, char *argv[]) } /* - * list_get_allocsize -- - * Get the allocation size for this file from the metadata. + * list_init_block -- + * Initialize a dummy block structure for a file. */ static int -list_get_allocsize(WT_SESSION *session, const char *key, size_t *allocsize) +list_init_block(WT_SESSION *session, const char *key, WT_BLOCK *block) { - WT_CONFIG_ITEM szvalue; + WT_CONFIG_ITEM cval; WT_CONFIG_PARSER *parser; WT_DECL_RET; WT_EXTENSION_API *wt_api; int tret; char *config; - *allocsize = 0; + WT_CLEAR(*block); parser = NULL; config = NULL; @@ -90,10 +90,14 @@ list_get_allocsize(WT_SESSION *session, const char *key, size_t *allocsize) WT_ERR(util_err(session, ret, "%s: WT_EXTENSION_API.metadata_search", key)); if ((ret = wt_api->config_parser_open(wt_api, session, config, strlen(config), &parser)) != 0) WT_ERR(util_err(session, ret, "WT_EXTENSION_API.config_parser_open")); - if ((ret = parser->get(parser, "allocation_size", &szvalue)) == 0) - *allocsize = (size_t)szvalue.val; - else - ret = ret == WT_NOTFOUND ? 0 : util_err(session, ret, "WT_CONFIG_PARSER.get"); + if ((ret = parser->get(parser, "allocation_size", &cval)) == 0) + block->allocsize = (uint32_t)cval.val; + else if (ret != WT_NOTFOUND) + WT_ERR(util_err(session, ret, "WT_CONFIG_PARSER.get")); + + if ((ret = parser->get(parser, "block_allocation", &cval)) == 0) + block->log_structured = WT_STRING_MATCH("log_structured", cval.str, cval.len); + err: if (parser != NULL && (tret = parser->close(parser)) != 0) { tret = util_err(session, tret, "WT_CONFIG_PARSER.close"); @@ -202,10 +206,11 @@ list_print_size(uint64_t v) static int list_print_checkpoint(WT_SESSION *session, const char *key) { + WT_BLOCK _block, *block; WT_BLOCK_CKPT ci; WT_CKPT *ckpt, *ckptbase; WT_DECL_RET; - size_t allocsize, len; + size_t len; time_t t; /* @@ -217,7 +222,9 @@ list_print_checkpoint(WT_SESSION *session, const char *key) return (ret == WT_NOTFOUND ? 0 : ret); /* We need the allocation size for decoding the checkpoint addr */ - if ((ret = list_get_allocsize(session, key, &allocsize)) != 0) + /* TODO this is a kludge: fix */ + block = &_block; + if ((ret = list_init_block(session, key, block)) != 0) return (ret); /* Find the longest name, so we can pretty-print. */ @@ -245,7 +252,7 @@ list_print_checkpoint(WT_SESSION *session, const char *key) /* Decode the checkpoint block. */ if (ckpt->raw.data == NULL) continue; - if ((ret = __wt_block_ckpt_decode(session, allocsize, ckpt->raw.data, &ci)) == 0) { + if ((ret = __wt_block_ckpt_decode(session, block, ckpt->raw.data, &ci)) == 0) { printf( "\t\t" "file-size: "); diff --git a/src/third_party/wiredtiger/src/utilities/util_load.c b/src/third_party/wiredtiger/src/utilities/util_load.c index 3aeb00c2e8b..0e9bd922eeb 100644 --- a/src/third_party/wiredtiger/src/utilities/util_load.c +++ b/src/third_party/wiredtiger/src/utilities/util_load.c @@ -341,7 +341,8 @@ config_reorder(WT_SESSION *session, char **list) * information. */ if ((list[0] == NULL || list[1] == NULL || list[2] != NULL) || - (WT_PREFIX_MATCH(list[0], "file:") && WT_PREFIX_MATCH(list[0], "lsm:"))) + (!WT_PREFIX_MATCH(list[0], "file:") && !WT_PREFIX_MATCH(list[0], "lsm:") && + !WT_PREFIX_MATCH(list[0], "tiered:"))) return (format(session)); entry = list; @@ -383,7 +384,7 @@ config_update(WT_SESSION *session, char **list) for (listp = list; *listp != NULL; listp += 2) if (WT_PREFIX_MATCH(*listp, "colgroup:") || WT_PREFIX_MATCH(*listp, "file:") || WT_PREFIX_MATCH(*listp, "index:") || WT_PREFIX_MATCH(*listp, "lsm:") || - WT_PREFIX_MATCH(*listp, "table:")) + WT_PREFIX_MATCH(*listp, "table:") || WT_PREFIX_MATCH(*listp, "tiered:")) if (config_rename(session, listp, cmdname)) return (1); diff --git a/src/third_party/wiredtiger/src/utilities/util_printlog.c b/src/third_party/wiredtiger/src/utilities/util_printlog.c index 9d09470bbd9..615f11768ab 100644 --- a/src/third_party/wiredtiger/src/utilities/util_printlog.c +++ b/src/third_party/wiredtiger/src/utilities/util_printlog.c @@ -12,9 +12,15 @@ static int usage(void) { static const char *options[] = {"-f", "output to the specified file", "-x", - "display key and value items in hexadecimal format", NULL, NULL}; + "display key and value items in hexadecimal format", "-l", + "the start LSN from which the log will be printed, optionally the end LSN can also be " + "specified", + NULL, NULL}; - util_usage("printlog [-x] [-f output-file]", "options:", options); + util_usage( + "printlog [-x] [-f output-file] [-l start-file,start-offset]|[-l " + "start-file,start-offset,end-file,end-offset]", + "options:", options); return (1); } @@ -22,17 +28,37 @@ int util_printlog(WT_SESSION *session, int argc, char *argv[]) { WT_DECL_RET; + WT_LSN end_lsn, start_lsn; + uint32_t end_lsnfile, end_lsnoffset, start_lsnfile, start_lsnoffset; uint32_t flags; int ch; - char *ofile; + int n_args; + char *ofile, *start_str; + bool end_set, start_set; + end_set = start_set = false; flags = 0; ofile = NULL; - while ((ch = __wt_getopt(progname, argc, argv, "f:mx")) != EOF) + + while ((ch = __wt_getopt(progname, argc, argv, "f:l:mx")) != EOF) switch (ch) { case 'f': /* output file */ ofile = __wt_optarg; break; + case 'l': + start_str = __wt_optarg; + n_args = sscanf(start_str, "%" SCNu32 ",%" SCNu32 ",%" SCNu32 ",%" SCNu32, + &start_lsnfile, &start_lsnoffset, &end_lsnfile, &end_lsnoffset); + if (n_args == 2) { + WT_SET_LSN(&start_lsn, start_lsnfile, start_lsnoffset); + start_set = true; + } else if (n_args == 4) { + WT_SET_LSN(&start_lsn, start_lsnfile, start_lsnoffset); + WT_SET_LSN(&end_lsn, end_lsnfile, end_lsnoffset); + end_set = start_set = true; + } else + return (usage()); + break; case 'm': /* messages only */ LF_SET(WT_TXN_PRINTLOG_MSG); break; @@ -49,7 +75,8 @@ util_printlog(WT_SESSION *session, int argc, char *argv[]) if (argc != 0) return (usage()); - if ((ret = __wt_txn_printlog(session, ofile, flags)) != 0) + if ((ret = __wt_txn_printlog(session, ofile, flags, start_set == true ? &start_lsn : NULL, + end_set == true ? &end_lsn : NULL)) != 0) (void)util_err(session, ret, "printlog"); return (ret); diff --git a/src/third_party/wiredtiger/test/csuite/wt3363_checkpoint_op_races/main.c b/src/third_party/wiredtiger/test/csuite/wt3363_checkpoint_op_races/main.c index baca92846bc..b45c573c25b 100644 --- a/src/third_party/wiredtiger/test/csuite/wt3363_checkpoint_op_races/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt3363_checkpoint_op_races/main.c @@ -25,6 +25,13 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ + +/* + * [TEST_TAGS] + * checkpoints:liveness:liveness + * [END_TAGS] + */ + #include "test_util.h" /* diff --git a/src/third_party/wiredtiger/test/suite/suite_subprocess.py b/src/third_party/wiredtiger/test/suite/suite_subprocess.py index d586f2a7997..a1bdeb24241 100755 --- a/src/third_party/wiredtiger/test/suite/suite_subprocess.py +++ b/src/third_party/wiredtiger/test/suite/suite_subprocess.py @@ -93,7 +93,9 @@ class suite_subprocess: got = f.read(len(expect) + 100) self.assertEqual(got, expect, filename + ': does not contain expected:\n\'' + expect + '\', but contains:\n\'' + got + '\'.') - def check_file_contains_one_of(self, filename, expectlist): + # Check contents of the file against a provided checklist. Expected is used as a bool to either + # ensure checklist is included or ensure the checklist is not included in the file. + def check_file_contains_one_of(self, filename, checklist, expected): """ Check that the file contains the expected string in the first 100K bytes """ @@ -101,21 +103,27 @@ class suite_subprocess: with open(filename, 'r') as f: got = f.read(maxbytes) found = False - for expect in expectlist: + for expect in checklist: pat = self.convert_to_pattern(expect) if pat == None: if expect in got: found = True - break + if expected: + break + else: + self.fail("Did not expect: " + got) else: if re.search(pat, got): found = True - break - if not found: - if len(expectlist) == 1: - expect = '\'' + expectlist[0] + '\'' + if expected: + break + else: + self.fail("Did not expect: " + got) + if not found and expected: + if len(checklist) == 1: + expect = '\'' + checklist[0] + '\'' else: - expect = str(expectlist) + expect = str(checklist) gotstr = '\'' + \ (got if len(got) < 1000 else (got[0:1000] + '...')) + '\'' if len(got) >= maxbytes: @@ -123,8 +131,11 @@ class suite_subprocess: else: self.fail(filename + ': does not contain expected ' + expect + ', got ' + gotstr) - def check_file_contains(self, filename, expect): - self.check_file_contains_one_of(filename, [expect]) + def check_file_contains(self, filename, content): + self.check_file_contains_one_of(filename, [content], True) + + def check_file_not_contains(self, filename, content): + self.check_file_contains_one_of(filename, [content], False) def check_empty_file(self, filename): """ diff --git a/src/third_party/wiredtiger/test/suite/test_backup01.py b/src/third_party/wiredtiger/test/suite/test_backup01.py index f2745710545..65b6b3735f3 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup01.py +++ b/src/third_party/wiredtiger/test/suite/test_backup01.py @@ -31,15 +31,15 @@ import os import shutil import string import time -from suite_subprocess import suite_subprocess -import wiredtiger, wttest +from wtbackup import backup_base +import wiredtiger from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet from helper import compare_files # test_backup.py # Utilities: wt backup # Test backup (both backup cursors and the wt backup command). -class test_backup(wttest.WiredTigerTestCase, suite_subprocess): +class test_backup(backup_base): dir='backup.dir' # Backup directory name pfx = 'test_backup' @@ -54,20 +54,6 @@ class test_backup(wttest.WiredTigerTestCase, suite_subprocess): ('table:' + pfx + '.8', ComplexLSMDataSet, 1), ] - # Populate a set of objects. - def populate(self, skiplsm): - for i in self.objs: - if i[2]: - if skiplsm: - continue - i[1](self, i[0], 100).populate() - - # Compare the original and backed-up files using the wt dump command. - def compare(self, uri): - self.runWt(['dump', uri], outfilename='orig') - self.runWt(['-h', self.dir, 'dump', uri], outfilename='backup') - self.assertEqual(True, compare_files(self, 'orig', 'backup')) - # Test simple backup cursor open/close. def test_cursor_simple(self): cursor = self.session.open_cursor('backup:', None, None) @@ -83,7 +69,7 @@ class test_backup(wttest.WiredTigerTestCase, suite_subprocess): # Test backup of a database using the wt backup command. def test_backup_database(self): - self.populate(0) + self.populate(self.objs) os.mkdir(self.dir) self.runWt(['backup', self.dir]) @@ -95,20 +81,7 @@ class test_backup(wttest.WiredTigerTestCase, suite_subprocess): # And that the contents are the same. for i in self.objs: - self.compare(i[0]) - - # Check that a URI doesn't exist, both the meta-data and the file names. - def confirmPathDoesNotExist(self, uri): - conn = self.wiredtiger_open(self.dir) - session = conn.open_session() - self.assertRaises(wiredtiger.WiredTigerError, - lambda: session.open_cursor(uri, None, None)) - conn.close() - - self.assertEqual( - glob.glob(self.dir + '*' + uri.split(":")[1] + '*'), [], - 'confirmPathDoesNotExist: URI exists, file name matching \"' + - uri.split(":")[1] + '\" found') + self.compare_backups(i[0], self.dir, './') # Backup a set of chosen tables/files using the wt backup command. def backup_table(self, l): @@ -127,16 +100,16 @@ class test_backup(wttest.WiredTigerTestCase, suite_subprocess): # Confirm the objects we backed up exist, with correct contents. for i in range(0, len(self.objs)): if i in l: - self.compare(self.objs[i][0]) + self.compare_backups(self.objs[i][0], self.dir, './') # Confirm the other objects don't exist. for i in range(0, len(self.objs)): if i not in l: - self.confirmPathDoesNotExist(self.objs[i][0]) + self.confirmPathDoesNotExist(self.objs[i][0], self.dir) # Test backup of database subsets. def test_backup_table(self): - self.populate(0) + self.populate(self.objs) self.backup_table([0,2,4,6]) self.backup_table([1,3,5,7]) self.backup_table([0,1,2]) @@ -145,7 +118,7 @@ class test_backup(wttest.WiredTigerTestCase, suite_subprocess): # Test cursor reset runs through the list twice. def test_cursor_reset(self): - self.populate(0) + self.populate(self.objs) cursor = self.session.open_cursor('backup:', None, None) i = 0 while True: @@ -167,7 +140,7 @@ class test_backup(wttest.WiredTigerTestCase, suite_subprocess): # Test interaction between checkpoints and a backup cursor. def test_checkpoint_delete(self): # You cannot name checkpoints including LSM tables, skip those. - self.populate(1) + self.populate(self.objs, False, True) # Confirm checkpoints are being deleted. self.session.checkpoint("name=one") diff --git a/src/third_party/wiredtiger/test/suite/test_backup03.py b/src/third_party/wiredtiger/test/suite/test_backup03.py index 9426e468264..6ea7a7f93c5 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup03.py +++ b/src/third_party/wiredtiger/test/suite/test_backup03.py @@ -27,16 +27,15 @@ # OTHER DEALINGS IN THE SOFTWARE. import glob, os, shutil, string -import wiredtiger, wttest -from helper import compare_files -from suite_subprocess import suite_subprocess +import wiredtiger +from wtbackup import backup_base from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet from wtscenario import make_scenarios # test_backup03.py # Utilities: wt backup # Test cursor backup with target URIs -class test_backup_target(wttest.WiredTigerTestCase, suite_subprocess): +class test_backup_target(backup_base): dir='backup.dir' # Backup directory name # This test is written to test LSM hot backups: we test a simple LSM object @@ -81,36 +80,6 @@ class test_backup_target(wttest.WiredTigerTestCase, suite_subprocess): # Create a large cache, otherwise this test runs quite slowly. conn_config = 'cache_size=1G' - # Populate a set of objects. - def populate(self): - for i in self.objs: - if self.big == i[2]: - rows = 50000 # Big object - else: - rows = 1000 # Small object - i[1](self, i[0], rows, cgconfig = i[3]).populate() - # Backup needs a checkpoint - self.session.checkpoint(None) - - # Compare the original and backed-up files using the wt dump command. - def compare(self, uri): - self.runWt(['dump', uri], outfilename='orig') - self.runWt(['-h', self.dir, 'dump', uri], outfilename='backup') - self.assertEqual(True, compare_files(self, 'orig', 'backup')) - - # Check that a URI doesn't exist, both the meta-data and the file names. - def confirmPathDoesNotExist(self, uri): - conn = self.wiredtiger_open(self.dir) - session = conn.open_session() - self.assertRaises(wiredtiger.WiredTigerError, - lambda: session.open_cursor(uri, None, None)) - conn.close() - - self.assertEqual( - glob.glob(self.dir + '*' + uri.split(":")[1] + '*'), [], - 'confirmPathDoesNotExist: URI exists, file name matching \"' + - uri.split(":")[1] + '\" found') - # Backup a set of target tables using a backup cursor. def backup_table_cursor(self, l): # Create the backup directory. @@ -139,17 +108,18 @@ class test_backup_target(wttest.WiredTigerTestCase, suite_subprocess): # Confirm the objects we backed up exist, with correct contents. for i in range(0, len(self.objs)): if not l or i in l: - self.compare(self.objs[i][0]) + self.compare_backups(self.objs[i][0], self.dir, './') # Confirm the other objects don't exist. if l: for i in range(0, len(self.objs)): if i not in l: - self.confirmPathDoesNotExist(self.objs[i][0]) + self.confirmPathDoesNotExist(self.objs[i][0], self.dir) # Test backup with targets. def test_backup_target(self): - self.populate() + self.populate_big = self.big + self.populate(self.objs, True) self.backup_table_cursor(self.list) if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_backup04.py b/src/third_party/wiredtiger/test/suite/test_backup04.py index 5de3b1647fc..a378e5dbd63 100755 --- a/src/third_party/wiredtiger/test/suite/test_backup04.py +++ b/src/third_party/wiredtiger/test/suite/test_backup04.py @@ -26,10 +26,9 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import threading, time, wiredtiger, wttest +import threading, time, wiredtiger import glob, os, shutil -from helper import compare_files -from suite_subprocess import suite_subprocess +from wtbackup import backup_base from wtdataset import SimpleDataSet, simple_key from wtscenario import make_scenarios from wtthread import op_thread @@ -37,7 +36,7 @@ from wtthread import op_thread # test_backup04.py # Utilities: wt backup # Test incremental cursor backup. -class test_backup_target(wttest.WiredTigerTestCase, suite_subprocess): +class test_backup_target(backup_base): dir='backup.dir' # Backup directory name logmax="100K" @@ -63,7 +62,7 @@ class test_backup_target(wttest.WiredTigerTestCase, suite_subprocess): return 'cache_size=1G,log=(archive=false,enabled,file_max=%s)' % \ self.logmax - def populate(self, uri, dsize, rows): + def populate_with_string(self, uri, dsize, rows): self.pr('populate: ' + uri + ' with ' + str(rows) + ' rows') cursor = self.session.open_cursor(uri, None) for i in range(1, rows + 1): @@ -77,30 +76,6 @@ class test_backup_target(wttest.WiredTigerTestCase, suite_subprocess): cursor[simple_key(cursor, i)] = str(i) + ':' + upd * dsize cursor.close() - # Compare the original and backed-up files using the wt dump command. - def compare(self, uri, dir_full, dir_incr): - # print "Compare: full URI: " + uri + " with incremental URI " - if dir_full == None: - full_name='original' - else: - full_name='backup_full' - incr_name='backup_incr' - if os.path.exists(full_name): - os.remove(full_name) - if os.path.exists(incr_name): - os.remove(incr_name) - # - # We have been copying the logs only, so we need to force 'wt' to - # run recovery in order to apply all the logs and check the data. - # - if dir_full == None: - self.runWt(['-R', 'dump', uri], outfilename=full_name) - else: - self.runWt(['-R', '-h', dir_full, 'dump', uri], outfilename=full_name) - self.runWt(['-R', '-h', dir_incr, 'dump', uri], outfilename=incr_name) - self.assertEqual(True, - compare_files(self, full_name, incr_name)) - def take_full_backup(self, dir): # Open up the backup cursor, and copy the files. Do a full backup. cursor = self.session.open_cursor('backup:', None, None) @@ -140,7 +115,7 @@ class test_backup_target(wttest.WiredTigerTestCase, suite_subprocess): # Create the backup directory. self.session.create(self.uri, "key_format=S,value_format=S") - self.populate(self.uri, self.dsize, self.nops) + self.populate_with_string(self.uri, self.dsize, self.nops) # We need to start the directory for the incremental backup with # a full backup. The full backup function creates the directory. @@ -157,7 +132,6 @@ class test_backup_target(wttest.WiredTigerTestCase, suite_subprocess): # Close the backup cursor updstr="bcdefghi" for increment in range(0, 5): - full_dir = self.dir + str(increment) # Add more work to move the logs forward. self.update(self.uri, self.dsize, updstr[increment], self.nops) self.session.checkpoint(None) @@ -167,9 +141,10 @@ class test_backup_target(wttest.WiredTigerTestCase, suite_subprocess): # After running, take a full backup. Compare the incremental # backup to the original database and the full backup database. + full_dir = self.dir + ".full" self.take_full_backup(full_dir) - self.compare(self.uri, full_dir, self.dir) - self.compare(self.uri, None, self.dir) + self.compare_backups(self.uri, self.dir, full_dir) + self.compare_backups(self.uri, self.dir, './') if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_backup06.py b/src/third_party/wiredtiger/test/suite/test_backup06.py index 8ff3d7127a4..5821e5e9af6 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup06.py +++ b/src/third_party/wiredtiger/test/suite/test_backup06.py @@ -26,11 +26,10 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import glob import os import shutil import string -from suite_subprocess import suite_subprocess +from wtbackup import backup_base import wiredtiger, wttest from wiredtiger import stat from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet @@ -43,7 +42,7 @@ except: # test_backup06.py # Test that opening a backup cursor does not open file handles. -class test_backup06(wttest.WiredTigerTestCase, suite_subprocess): +class test_backup06(backup_base): conn_config = 'statistics=(fast)' # This will create several hundred tables. num_table_sets = 10 @@ -78,12 +77,6 @@ class test_backup06(wttest.WiredTigerTestCase, suite_subprocess): uri = i[0] + "." + str(t) i[1](self, uri, 10).populate() - def populate(self): - for i in self.fobjs: - i[1](self, i[0], 100).populate() - for i in self.tobjs: - i[1](self, i[0], 100).populate() - # Test that the open handle count does not change. def test_cursor_open_handles(self): if os.name == "nt": @@ -130,7 +123,8 @@ class test_backup06(wttest.WiredTigerTestCase, suite_subprocess): # We also want to make sure we detect and get an error when set to # false. When set to true the open handles protect against schema # operations. - self.populate() + self.populate(self.fobjs) + self.populate(self.tobjs) cursor = self.session.open_cursor('backup:', None, None) # Check that we can create. self.session.create(schema_uri, None) @@ -148,7 +142,8 @@ class test_backup06(wttest.WiredTigerTestCase, suite_subprocess): # Test cursor reset runs through the list twice. def test_cursor_reset(self): - self.populate() + self.populate(self.fobjs) + self.populate(self.tobjs) cursor = self.session.open_cursor('backup:', None, None) i = 0 while True: diff --git a/src/third_party/wiredtiger/test/suite/test_backup07.py b/src/third_party/wiredtiger/test/suite/test_backup07.py index 5a90ddbd8cf..360432690eb 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup07.py +++ b/src/third_party/wiredtiger/test/suite/test_backup07.py @@ -29,21 +29,21 @@ import wiredtiger, wttest import os, shutil from helper import compare_files -from suite_subprocess import suite_subprocess +from wtbackup import backup_base from wtdataset import simple_key from wtscenario import make_scenarios # test_backup07.py # Test cursor backup with target URIs, logging and create during backup -class test_backup07(wttest.WiredTigerTestCase, suite_subprocess): +class test_backup07(backup_base): dir='backup.dir' # Backup directory name logmax="100K" newuri="table:newtable" pfx = 'test_backup' scenarios = make_scenarios([ - ('table', dict(uri='table:test',dsize=100,nops=100,nthreads=1)), + ('table', dict(uri='table:test',dsize=100,nthreads=1)) ]) # Create a large cache, otherwise this test runs quite slowly. @@ -59,15 +59,8 @@ class test_backup07(wttest.WiredTigerTestCase, suite_subprocess): # Insert small amounts of data at a time stopping just after we # cross into log file 2. - loop = 0 - c = self.session.open_cursor(self.uri) while not os.path.exists(log2): - for i in range(0, self.nops): - num = i + (loop * self.nops) - key = 'key' + str(num) - val = 'value' + str(num) - c[key] = val - loop += 1 + self.add_data(self.uri, 'key', 'value') # Test a potential bug in full backups and creates. # We allow creates during backup because the file doesn't exist @@ -82,12 +75,7 @@ class test_backup07(wttest.WiredTigerTestCase, suite_subprocess): # Now create and populate the new table. Make sure the log records # are on disk and will be copied to the backup. self.session.create(self.newuri, "key_format=S,value_format=S") - c = self.session.open_cursor(self.newuri) - for i in range(0, self.nops): - key = 'key' + str(i) - val = 'value' + str(i) - c[key] = val - c.close() + self.add_data(self.newuri, 'key', 'value') self.session.log_flush('sync=on') # Now copy the files returned by the backup cursor. This should not diff --git a/src/third_party/wiredtiger/test/suite/test_backup10.py b/src/third_party/wiredtiger/test/suite/test_backup10.py index 1d6304c1ba2..9a74e190b3f 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup10.py +++ b/src/third_party/wiredtiger/test/suite/test_backup10.py @@ -29,17 +29,16 @@ import wiredtiger, wttest import os, shutil from helper import compare_files -from suite_subprocess import suite_subprocess +from wtbackup import backup_base from wtdataset import simple_key from wtscenario import make_scenarios # test_backup10.py # Test cursor backup with a duplicate backup cursor. -class test_backup10(wttest.WiredTigerTestCase, suite_subprocess): +class test_backup10(backup_base): dir='backup.dir' # Backup directory name logmax="100K" uri="table:test" - nops=100 pfx = 'test_backup' @@ -62,15 +61,8 @@ class test_backup10(wttest.WiredTigerTestCase, suite_subprocess): # Insert small amounts of data at a time stopping after we # cross into log file 2. - loop = 0 - c = self.session.open_cursor(self.uri) while not os.path.exists(log2): - for i in range(0, self.nops): - num = i + (loop * self.nops) - key = 'key' + str(num) - val = 'value' + str(num) - c[key] = val - loop += 1 + self.add_data(self.uri, 'key', 'value') # Open up the backup cursor. This causes a new log file to be created. # That log file is not part of the list returned. @@ -78,13 +70,7 @@ class test_backup10(wttest.WiredTigerTestCase, suite_subprocess): bkup_c = self.session.open_cursor('backup:', None, None) # Add some data that will appear in log file 3. - for i in range(0, self.nops): - num = i + (loop * self.nops) - key = 'key' + str(num) - val = 'value' + str(num) - c[key] = val - loop += 1 - c.close() + self.add_data(self.uri, 'key', 'value') self.session.log_flush('sync=on') # Now copy the files returned by the backup cursor. diff --git a/src/third_party/wiredtiger/test/suite/test_backup11.py b/src/third_party/wiredtiger/test/suite/test_backup11.py index 3e6bd347b03..a974d505654 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup11.py +++ b/src/third_party/wiredtiger/test/suite/test_backup11.py @@ -29,35 +29,21 @@ import wiredtiger, wttest import os, shutil from helper import compare_files -from suite_subprocess import suite_subprocess +from wtbackup import backup_base from wtdataset import simple_key from wtscenario import make_scenarios # test_backup11.py # Test cursor backup with a duplicate backup cursor. -class test_backup11(wttest.WiredTigerTestCase, suite_subprocess): +class test_backup11(backup_base): conn_config= 'cache_size=1G,log=(enabled,file_max=100K)' dir='backup.dir' # Backup directory name - mult=0 - nops=100 pfx = 'test_backup' uri="table:test" - def add_data(self): - c = self.session.open_cursor(self.uri) - for i in range(0, self.nops): - num = i + (self.mult * self.nops) - key = 'key' + str(num) - val = 'value' + str(num) - c[key] = val - self.mult += 1 - self.session.checkpoint() - c.close() - def test_backup11(self): self.session.create(self.uri, "key_format=S,value_format=S") - self.add_data() - + self.add_data(self.uri, 'key', 'value', True) # Open up the backup cursor. This causes a new log file to be created. # That log file is not part of the list returned. This is a full backup # primary cursor with incremental configured. @@ -66,7 +52,7 @@ class test_backup11(wttest.WiredTigerTestCase, suite_subprocess): bkup_c = self.session.open_cursor('backup:', None, config) # Add data while the backup cursor is open. - self.add_data() + self.add_data(self.uri, 'key', 'value', True) # Now copy the files returned by the backup cursor. orig_logs = [] @@ -106,7 +92,7 @@ class test_backup11(wttest.WiredTigerTestCase, suite_subprocess): bkup_c.close() # Add more data - self.add_data() + self.add_data(self.uri, 'key', 'value', True) # Test error cases now. diff --git a/src/third_party/wiredtiger/test/suite/test_backup12.py b/src/third_party/wiredtiger/test/suite/test_backup12.py index 90ca31aa76e..53ad7845634 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup12.py +++ b/src/third_party/wiredtiger/test/suite/test_backup12.py @@ -29,47 +29,34 @@ import wiredtiger, wttest import os, shutil from helper import compare_files -from suite_subprocess import suite_subprocess +from wtbackup import backup_base from wtdataset import simple_key from wtscenario import make_scenarios # test_backup12.py # Test cursor backup with a block-based incremental cursor. -class test_backup12(wttest.WiredTigerTestCase, suite_subprocess): +class test_backup12(backup_base): conn_config='cache_size=1G,log=(enabled,file_max=100K)' dir='backup.dir' # Backup directory name logmax="100K" uri="table:test" uri2="table:test2" uri_rem="table:test_rem" - nops=1000 - mult=0 pfx = 'test_backup' # Set the key and value big enough that we modify a few blocks. bigkey = 'Key' * 100 bigval = 'Value' * 100 - def add_data(self, uri): - c = self.session.open_cursor(uri) - for i in range(0, self.nops): - num = i + (self.mult * self.nops) - key = self.bigkey + str(num) - val = self.bigval + str(num) - c[key] = val - self.session.checkpoint() - c.close() - # Increase the multiplier so that later calls insert unique items. - self.mult += 1 - + nops = 1000 def test_backup12(self): self.session.create(self.uri, "key_format=S,value_format=S") self.session.create(self.uri2, "key_format=S,value_format=S") self.session.create(self.uri_rem, "key_format=S,value_format=S") - self.add_data(self.uri) - self.add_data(self.uri2) - self.add_data(self.uri_rem) + self.add_data(self.uri, self.bigkey, self.bigval, True) + self.add_data(self.uri2, self.bigkey, self.bigval, True) + self.add_data(self.uri_rem, self.bigkey, self.bigval, True) # Open up the backup cursor. This causes a new log file to be created. # That log file is not part of the list returned. This is a full backup @@ -82,7 +69,7 @@ class test_backup12(wttest.WiredTigerTestCase, suite_subprocess): bkup_c = self.session.open_cursor('backup:', None, config) # Add more data while the backup cursor is open. - self.add_data(self.uri) + self.add_data(self.uri, self.bigkey, self.bigval, True) # Now copy the files returned by the backup cursor. all_files = [] @@ -122,8 +109,8 @@ class test_backup12(wttest.WiredTigerTestCase, suite_subprocess): bkup_c.close() # Add more data. - self.add_data(self.uri) - self.add_data(self.uri2) + self.add_data(self.uri, self.bigkey, self.bigval, True) + self.add_data(self.uri2, self.bigkey, self.bigval, True) # Drop a table. self.session.drop(self.uri_rem) diff --git a/src/third_party/wiredtiger/test/suite/test_backup13.py b/src/third_party/wiredtiger/test/suite/test_backup13.py index e77df8f5fcb..8992440c038 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup13.py +++ b/src/third_party/wiredtiger/test/suite/test_backup13.py @@ -29,18 +29,16 @@ import wiredtiger, wttest import os, shutil from helper import compare_files -from suite_subprocess import suite_subprocess +from wtbackup import backup_base from wtdataset import simple_key from wtscenario import make_scenarios # test_backup13.py # Test cursor backup with a block-based incremental cursor and force_stop. -class test_backup13(wttest.WiredTigerTestCase, suite_subprocess): +class test_backup13(backup_base): conn_config='cache_size=1G,log=(enabled,file_max=100K)' dir='backup.dir' # Backup directory name logmax="100K" - mult=0 - nops=1000 uri="table:test" scenarios = make_scenarios([ @@ -55,6 +53,8 @@ class test_backup13(wttest.WiredTigerTestCase, suite_subprocess): bigkey = 'Key' * 100 bigval = 'Value' * 100 + nops = 1000 + def simulate_crash_restart(self, olddir, newdir): ''' Simulate a crash from olddir and restart in newdir. ''' # with the connection still open, copy files to new directory @@ -73,33 +73,20 @@ class test_backup13(wttest.WiredTigerTestCase, suite_subprocess): self.conn = self.setUpConnectionOpen(newdir) self.session = self.setUpSessionOpen(self.conn) - def add_data(self, uri): - c = self.session.open_cursor(uri) - for i in range(0, self.nops): - num = i + (self.mult * self.nops) - key = self.bigkey + str(num) - val = self.bigval + str(num) - c.set_key(key) - c.set_value(val) - # read committed and read uncommitted transactions are readonly, any write operations with - # these isolation levels should throw an error. - if self.sess_cfg == 'isolation=read-committed' or self.sess_cfg == 'isolation=read-uncommitted': - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: c.insert(), "/not supported in read-committed or read-uncommitted transactions/") - else: - c.insert() - self.session.checkpoint() - c.close() - # Increase the multiplier so that later calls insert unique items. - self.mult += 1 - def session_config(self): return self.sess_cfg + def add_data_and_check(self): + if self.sess_cfg == 'isolation=read-committed' or self.sess_cfg == 'isolation=read-uncommitted': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.add_data(self.uri, self.bigkey, self.bigval, True), + "/not supported in read-committed or read-uncommitted transactions/") + else: + self.add_data(self.uri, self.bigkey, self.bigval, True) + def test_backup13(self): self.session.create(self.uri, "key_format=S,value_format=S") - self.add_data(self.uri) - + self.add_data_and_check() # Open up the backup cursor. This causes a new log file to be created. # That log file is not part of the list returned. This is a full backup # primary cursor with incremental configured. @@ -108,7 +95,7 @@ class test_backup13(wttest.WiredTigerTestCase, suite_subprocess): bkup_c = self.session.open_cursor('backup:', None, config) # Add more data while the backup cursor is open. - self.add_data(self.uri) + self.add_data_and_check() # Now copy the files returned by the backup cursor. all_files = [] @@ -129,7 +116,7 @@ class test_backup13(wttest.WiredTigerTestCase, suite_subprocess): bkup_c.close() # Add more data. - self.add_data(self.uri) + self.add_data_and_check() # Now do an incremental backup. config = 'incremental=(src_id="ID1",this_id="ID2")' diff --git a/src/third_party/wiredtiger/test/suite/test_backup14.py b/src/third_party/wiredtiger/test/suite/test_backup14.py index 67ebe68a8af..a4933140833 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup14.py +++ b/src/third_party/wiredtiger/test/suite/test_backup14.py @@ -26,17 +26,16 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import wiredtiger, wttest +import wiredtiger import os, shutil -from helper import compare_files -from suite_subprocess import suite_subprocess +from wtbackup import backup_base from wtdataset import simple_key from wtscenario import make_scenarios import glob # test_backup14.py # Test cursor backup with a block-based incremental cursor. -class test_backup14(wttest.WiredTigerTestCase, suite_subprocess): +class test_backup14(backup_base): conn_config='cache_size=1G,log=(enabled,file_max=100K)' dir='backup.dir' # Backup directory name logmax="100K" @@ -44,46 +43,20 @@ class test_backup14(wttest.WiredTigerTestCase, suite_subprocess): uri2="table:extra" uri_logged="table:logged_table" uri_not_logged="table:not_logged_table" - full_out = "./backup_block_full" - incr_out = "./backup_block_incr" + bkp_home = "WT_BLOCK" home_full = "WT_BLOCK_LOG_FULL" home_incr = "WT_BLOCK_LOG_INCR" logpath = "logpath" - nops=1000 - mult=0 + nops = 1000 max_iteration=7 - counter=0 new_table=False - initial_backup=False pfx = 'test_backup' # Set the key and value big enough that we modify a few blocks. bigkey = 'Key' * 100 bigval = 'Value' * 100 - # - # Set up all the directories needed for the test. We have a full backup directory for each - # iteration and an incremental backup for each iteration. That way we can compare the full and - # incremental each time through. - # - def setup_directories(self): - for i in range(0, self.max_iteration): - remove_dir = self.home_incr + '.' + str(i) - - create_dir = self.home_incr + '.' + str(i) + '/' + self.logpath - if os.path.exists(remove_dir): - os.remove(remove_dir) - os.makedirs(create_dir) - - if i == 0: - continue - remove_dir = self.home_full + '.' + str(i) - create_dir = self.home_full + '.' + str(i) + '/' + self.logpath - if os.path.exists(remove_dir): - os.remove(remove_dir) - os.makedirs(create_dir) - def take_full_backup(self): if self.counter != 0: hdir = self.home_full + '.' + str(self.counter) @@ -129,7 +102,7 @@ class test_backup14(wttest.WiredTigerTestCase, suite_subprocess): def take_incr_backup(self): # Open the backup data source for incremental backup. - buf = 'incremental=(src_id="ID' + str(self.counter-1) + '",this_id="ID' + str(self.counter) + '")' + buf = 'incremental=(src_id="ID' + str(self.counter - 1) + '",this_id="ID' + str(self.counter) + '")' bkup_c = self.session.open_cursor('backup:', None, buf) while True: ret = bkup_c.next() @@ -201,44 +174,6 @@ class test_backup14(wttest.WiredTigerTestCase, suite_subprocess): self.assertEqual(ret, wiredtiger.WT_NOTFOUND) bkup_c.close() - def compare_backups(self, t_uri): - # - # Run wt dump on full backup directory - # - full_backup_out = self.full_out + '.' + str(self.counter) - home_dir = self.home_full + '.' + str(self.counter) - if self.counter == 0: - home_dir = self.home - - self.runWt(['-R', '-h', home_dir, 'dump', t_uri], outfilename=full_backup_out) - # - # Run wt dump on incremental backup directory - # - incr_backup_out = self.incr_out + '.' + str(self.counter) - home_dir = self.home_incr + '.' + str(self.counter) - self.runWt(['-R', '-h', home_dir, 'dump', t_uri], outfilename=incr_backup_out) - - self.assertEqual(True, - compare_files(self, full_backup_out, incr_backup_out)) - - # - # Add data to the given uri. - # - def add_data(self, uri, bulk_option): - c = self.session.open_cursor(uri, None, bulk_option) - for i in range(0, self.nops): - num = i + (self.mult * self.nops) - key = self.bigkey + str(num) - val = self.bigval + str(num) - c[key] = val - c.close() - - # Increase the multiplier so that later calls insert unique items. - self.mult += 1 - # Increase the counter so that later backups have unique ids. - if self.initial_backup == False: - self.counter += 1 - # # Remove data from uri (table:main) # @@ -265,15 +200,17 @@ class test_backup14(wttest.WiredTigerTestCase, suite_subprocess): def add_data_validate_backups(self): self.pr('Adding initial data') self.initial_backup = True - self.add_data(self.uri, None) + self.add_data(self.uri, self.bigkey, self.bigval) + self.take_full_backup() self.initial_backup = False self.session.checkpoint() - self.add_data(self.uri, None) + self.add_data(self.uri, self.bigkey, self.bigval) + self.take_full_backup() self.take_incr_backup() - self.compare_backups(self.uri) + self.compare_backups(self.uri, self.home_full, self.home_incr, str(self.counter)) # # This function will remove all the records from table (table:main), take backup and validate the @@ -283,7 +220,7 @@ class test_backup14(wttest.WiredTigerTestCase, suite_subprocess): self.remove_data() self.take_full_backup() self.take_incr_backup() - self.compare_backups(self.uri) + self.compare_backups(self.uri, self.home_full, self.home_incr, str(self.counter)) # # This function will drop the existing table uri (table:main) that is part of the backups and @@ -298,7 +235,7 @@ class test_backup14(wttest.WiredTigerTestCase, suite_subprocess): self.session.create(self.uri2, "key_format=S,value_format=S") self.new_table = True - self.add_data(self.uri2, None) + self.add_data(self.uri2, self.bigkey, self.bigval) self.take_incr_backup() table_list = 'tablelist.txt' @@ -313,10 +250,10 @@ class test_backup14(wttest.WiredTigerTestCase, suite_subprocess): # def create_dropped_table_add_new_content(self): self.session.create(self.uri, "key_format=S,value_format=S") - self.add_data(self.uri, None) + self.add_data(self.uri, self.bigkey, self.bigval) self.take_full_backup() self.take_incr_backup() - self.compare_backups(self.uri) + self.compare_backups(self.uri, self.home_full, self.home_incr, str(self.counter)) # # This function will insert bulk data in logged and not-logged table, take backups and validate the @@ -327,26 +264,27 @@ class test_backup14(wttest.WiredTigerTestCase, suite_subprocess): # Insert bulk data into uri3 (table:logged_table). # self.session.create(self.uri_logged, "key_format=S,value_format=S") - self.add_data(self.uri_logged, 'bulk') + self.add_data(self.uri_logged, self.bigkey, self.bigval) + self.take_full_backup() self.take_incr_backup() - self.compare_backups(self.uri_logged) - + self.compare_backups(self.uri_logged, self.home_full, self.home_incr, str(self.counter)) # # Insert bulk data into uri4 (table:not_logged_table). # self.session.create(self.uri_not_logged, "key_format=S,value_format=S,log=(enabled=false)") - self.add_data(self.uri_not_logged, 'bulk') + self.add_data(self.uri_not_logged, self.bigkey, self.bigval) + self.take_full_backup() self.take_incr_backup() - self.compare_backups(self.uri_not_logged) + self.compare_backups(self.uri_not_logged, self.home_full, self.home_incr, str(self.counter)) def test_backup14(self): os.mkdir(self.bkp_home) self.home = self.bkp_home self.session.create(self.uri, "key_format=S,value_format=S") - self.setup_directories() + self.setup_directories(self.max_iteration, self.home_incr, self.home_full, self.logpath) self.pr('*** Add data, checkpoint, take backups and validate ***') self.add_data_validate_backups() @@ -361,6 +299,7 @@ class test_backup14(wttest.WiredTigerTestCase, suite_subprocess): self.create_dropped_table_add_new_content() self.pr('*** Insert data into Logged and Not-Logged tables ***') + self.cursor_config = 'bulk' self.insert_bulk_data() if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_backup15.py b/src/third_party/wiredtiger/test/suite/test_backup15.py index 509efcacb4f..669618c151c 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup15.py +++ b/src/third_party/wiredtiger/test/suite/test_backup15.py @@ -26,17 +26,16 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import wiredtiger, wttest +import wiredtiger import os, shutil -from helper import compare_files -from suite_subprocess import suite_subprocess +from wtbackup import backup_base from wtdataset import simple_key from wtscenario import make_scenarios import glob # test_backup15.py # Test cursor backup with a block-based incremental cursor. -class test_backup15(wttest.WiredTigerTestCase, suite_subprocess): +class test_backup15(backup_base): bkp_home = "WT_BLOCK" counter=0 conn_config='cache_size=1G,log=(enabled,file_max=100K)' @@ -52,8 +51,6 @@ class test_backup15(wttest.WiredTigerTestCase, suite_subprocess): home_full = "WT_BLOCK_LOG_FULL" home_incr = "WT_BLOCK_LOG_INCR" - full_out = "./backup_block_full" - incr_out = "./backup_block_incr" logpath = "logpath" new_table=False initial_backup=False @@ -63,21 +60,6 @@ class test_backup15(wttest.WiredTigerTestCase, suite_subprocess): bigkey = 'Key' * 100 bigval = 'Value' * 100 - # - # Set up all the directories needed for the test. We have a full backup directory for each - # iteration and an incremental backup for each iteration. That way we can compare the full and - # incremental each time through. - # - def setup_directories(self): - for i in range(0, self.max_iteration): - # The log directory is a subdirectory of the home directory, - # creating that will make the home directory also. - log_dir = self.home_incr + '.' + str(i) + '/' + self.logpath - os.makedirs(log_dir) - if i != 0: - log_dir = self.home_full + '.' + str(i) + '/' + self.logpath - os.makedirs(log_dir) - def range_copy(self, filename, offset, size): read_from = filename old_to = self.home_incr + '.' + str(self.counter - 1) + '/' + filename @@ -218,27 +200,10 @@ class test_backup15(wttest.WiredTigerTestCase, suite_subprocess): shutil.copy(copy_from, copy_to) self.assertEqual(ret, wiredtiger.WT_NOTFOUND) bkup_c.close() - - def compare_backups(self, t_uri): - # Run wt dump on full backup directory. - full_backup_out = self.full_out + '.' + str(self.counter) - home_dir = self.home_full + '.' + str(self.counter) - if self.counter == 0: - home_dir = self.home - self.runWt(['-R', '-h', home_dir, 'dump', t_uri], outfilename=full_backup_out) - - # Run wt dump on incremental backup directory. - incr_backup_out = self.incr_out + '.' + str(self.counter) - home_dir = self.home_incr + '.' + str(self.counter) - self.runWt(['-R', '-h', home_dir, 'dump', t_uri], outfilename=incr_backup_out) - - self.assertEqual(True, - compare_files(self, full_backup_out, incr_backup_out)) - # # Add data to the given uri. # - def add_data(self, uri): + def add_complex_data(self, uri): c = self.session.open_cursor(uri, None, None) # The first time we want to add in a lot of data. Then after that we want to # rapidly change a single key to create a hotspot in one block. @@ -270,12 +235,12 @@ class test_backup15(wttest.WiredTigerTestCase, suite_subprocess): self.home = self.bkp_home self.session.create(self.uri, "key_format=S,value_format=S") - self.setup_directories() + self.setup_directories(self.max_iteration, self.home_incr, self.home_full, self.logpath) self.pr('*** Add data, checkpoint, take backups and validate ***') self.pr('Adding initial data') self.initial_backup = True - self.add_data(self.uri) + self.add_complex_data(self.uri) self.take_full_backup() self.initial_backup = False self.session.checkpoint() @@ -283,7 +248,7 @@ class test_backup15(wttest.WiredTigerTestCase, suite_subprocess): # Each call now to take a full backup will make a copy into a full directory. Then # each incremental will take an incremental backup and we can compare them. for i in range(1, self.max_iteration): - self.add_data(self.uri) + self.add_complex_data(self.uri) self.session.checkpoint() # Swap the order of the full and incremental backups. It should not matter. They # should not interfere with each other. @@ -293,7 +258,7 @@ class test_backup15(wttest.WiredTigerTestCase, suite_subprocess): else: self.take_incr_backup() self.take_full_backup() - self.compare_backups(self.uri) + self.compare_backups(self.uri, self.home_full, self.home_incr, str(self.counter)) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_backup16.py b/src/third_party/wiredtiger/test/suite/test_backup16.py index 82df7c39307..a589cbd9691 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup16.py +++ b/src/third_party/wiredtiger/test/suite/test_backup16.py @@ -29,19 +29,17 @@ import wiredtiger, wttest import os, shutil from helper import compare_files -from suite_subprocess import suite_subprocess +from wtbackup import backup_base from wtdataset import simple_key from wtscenario import make_scenarios # test_backup16.py # Ensure incremental backup doesn't copy unnecessary files. -class test_backup16(wttest.WiredTigerTestCase, suite_subprocess): +class test_backup16(backup_base): conn_config='cache_size=1G,log=(enabled,file_max=100K)' counter=1 logmax='100K' - mult=1 - nops=10 # Define the table name and its on-disk file name together. file1='test1.wt' @@ -62,21 +60,12 @@ class test_backup16(wttest.WiredTigerTestCase, suite_subprocess): bigkey = 'Key' * 10 bigval = 'Value' * 10 - def add_data(self, uri): - c = self.session.open_cursor(uri) - for i in range(0, self.nops): - num = i + (self.mult * self.nops) - key = self.bigkey + str(num) - val = self.bigval + str(num) - c[key] = val - self.session.checkpoint() - c.close() - # Increase the multiplier so that later calls insert unique items. - self.mult += 1 + mult = 1 + counter = 1 + nops = 10 def verify_incr_backup(self, expected_file_list): - - bkup_config = ('incremental=(src_id="ID' + str(self.counter-1) + + bkup_config = ('incremental=(src_id="ID' + str(self.counter - 1) + '",this_id="ID' + str(self.counter) + '")') bkup_cur = self.session.open_cursor('backup:', None, bkup_config) self.counter += 1 @@ -129,8 +118,8 @@ class test_backup16(wttest.WiredTigerTestCase, suite_subprocess): self.session.create(self.uri2, 'key_format=S,value_format=S') self.session.create(self.uri3, 'key_format=S,value_format=S') self.session.create(self.uri6, 'key_format=S,value_format=S') - self.add_data(self.uri1) - self.add_data(self.uri2) + self.add_data(self.uri1, self.bigkey, self.bigval, True) + self.add_data(self.uri2, self.bigkey, self.bigval, True) # Checkpoint and simulate full backup. self.session.checkpoint() @@ -154,8 +143,8 @@ class test_backup16(wttest.WiredTigerTestCase, suite_subprocess): # self.session.create(self.uri4, "key_format=S,value_format=S") self.session.create(self.uri5, "key_format=S,value_format=S") - self.add_data(self.uri1) - self.add_data(self.uri5) + self.add_data(self.uri1, self.bigkey, self.bigval, True) + self.add_data(self.uri5, self.bigkey, self.bigval, True) self.session.checkpoint() # Validate these three files are included in the incremental. @@ -167,10 +156,10 @@ class test_backup16(wttest.WiredTigerTestCase, suite_subprocess): # Add more data and checkpoint. Earlier old tables without new data should not # appear in the list. The table with no data at all continues to appear in the # list. - self.add_data(self.uri3) - self.add_data(self.uri5) - self.session.checkpoint() + self.add_data(self.uri3, self.bigkey, self.bigval, True) + self.add_data(self.uri5, self.bigkey, self.bigval, True) + self.session.checkpoint() # Validate these three files are included in the incremental. files_to_backup = [self.file3, self.file4, self.file5] self.verify_incr_backup(files_to_backup) diff --git a/src/third_party/wiredtiger/test/suite/test_backup17.py b/src/third_party/wiredtiger/test/suite/test_backup17.py index 5fa250fd485..04bff3ea9d8 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup17.py +++ b/src/third_party/wiredtiger/test/suite/test_backup17.py @@ -29,21 +29,19 @@ import wiredtiger, wttest import os, shutil from helper import compare_files -from suite_subprocess import suite_subprocess +from wtbackup import backup_base from wtdataset import simple_key from wtscenario import make_scenarios # test_backup17.py # Test cursor backup with a block-based incremental cursor and consolidate. -class test_backup17(wttest.WiredTigerTestCase, suite_subprocess): +class test_backup17(backup_base): dir='backup.dir' # Backup directory name gran="100K" granval=100*1024 logmax="100K" uri="table:test" uri2="table:test2" - nops=1000 - mult=0 conn_config='cache_size=1G,log=(enabled,file_max=%s)' % logmax @@ -52,15 +50,7 @@ class test_backup17(wttest.WiredTigerTestCase, suite_subprocess): bigkey = 'Key' * 100 bigval = 'Value' * 100 - def add_data(self, uri): - c = self.session.open_cursor(uri) - for i in range(0, self.nops): - num = i + (self.mult * self.nops) - key = self.bigkey + str(num) - val = self.bigval + str(num) - c[key] = val - self.session.checkpoint() - c.close() + nops = 1000 def take_incr_backup(self, id, consolidate): # Open the backup data source for incremental backup. @@ -121,9 +111,9 @@ class test_backup17(wttest.WiredTigerTestCase, suite_subprocess): self.session.create(self.uri, "key_format=S,value_format=S") self.session.create(self.uri2, "key_format=S,value_format=S") - self.add_data(self.uri) - self.add_data(self.uri2) - self.mult += 1 + self.add_data(self.uri, self.bigkey, self.bigval, True) + self.mult = 0 + self.add_data(self.uri2, self.bigkey, self.bigval, True) # Open up the backup cursor. This causes a new log file to be created. # That log file is not part of the list returned. This is a full backup @@ -150,10 +140,14 @@ class test_backup17(wttest.WiredTigerTestCase, suite_subprocess): # Then perform the incremental backup with consolidate off (the default). Then add the # same data to the second table. Perform an incremental backup with consolidate on and # verify we get fewer, consolidated values. - self.add_data(self.uri) + self.mult = 1 + self.add_data(self.uri, self.bigkey, self.bigval, True) + uri1_lens = self.take_incr_backup(2, False) - self.add_data(self.uri2) + self.mult = 1 + self.add_data(self.uri2, self.bigkey, self.bigval, True) + uri2_lens = self.take_incr_backup(3, True) # Assert that we recorded fewer lengths on the consolidated backup. diff --git a/src/third_party/wiredtiger/test/suite/test_backup18.py b/src/third_party/wiredtiger/test/suite/test_backup18.py index 2034c8ac6d1..e7c9d65271e 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup18.py +++ b/src/third_party/wiredtiger/test/suite/test_backup18.py @@ -29,16 +29,14 @@ import wiredtiger, wttest import os, shutil from helper import compare_files -from suite_subprocess import suite_subprocess +from wtbackup import backup_base from wtdataset import simple_key from wtscenario import make_scenarios # test_backup18.py # Test backup:query_id API. -class test_backup18(wttest.WiredTigerTestCase, suite_subprocess): +class test_backup18(backup_base): conn_config= 'cache_size=1G,log=(enabled,file_max=100K)' - mult=0 - nops=100 pfx = 'test_backup' uri="table:test" @@ -58,22 +56,11 @@ class test_backup18(wttest.WiredTigerTestCase, suite_subprocess): expect.sort() self.assertEqual(got, expect) - def add_data(self): - c = self.session.open_cursor(self.uri) - for i in range(0, self.nops): - num = i + (self.mult * self.nops) - key = 'key' + str(num) - val = 'value' + str(num) - c[key] = val - self.mult += 1 - self.session.checkpoint() - c.close() - def test_backup18(self): # We're not taking actual backups in this test, but we do want a table to # exist for the backup cursor to generate something. self.session.create(self.uri, "key_format=S,value_format=S") - self.add_data() + self.add_data(self.uri, 'key', 'value', True) msg = "/is not configured/" self.pr("Query IDs before any backup") diff --git a/src/third_party/wiredtiger/test/suite/test_backup19.py b/src/third_party/wiredtiger/test/suite/test_backup19.py index 547445cafbe..c94bf381790 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup19.py +++ b/src/third_party/wiredtiger/test/suite/test_backup19.py @@ -26,17 +26,16 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import wiredtiger, wttest +import wiredtiger import os, shutil -from helper import compare_files -from suite_subprocess import suite_subprocess +from wtbackup import backup_base from wtdataset import simple_key from wtscenario import make_scenarios import glob # test_backup19.py # Test cursor backup with a block-based incremental cursor source id only. -class test_backup19(wttest.WiredTigerTestCase, suite_subprocess): +class test_backup19(backup_base): bkp_home = "WT_BLOCK" counter=0 conn_config='cache_size=1G,log=(enabled,file_max=100K)' @@ -51,8 +50,6 @@ class test_backup19(wttest.WiredTigerTestCase, suite_subprocess): home_full = "WT_BLOCK_LOG_FULL" home_incr = "WT_BLOCK_LOG_INCR" - full_out = "./backup_block_full" - incr_out = "./backup_block_incr" logpath = "logpath" new_table=False initial_backup=False @@ -62,22 +59,6 @@ class test_backup19(wttest.WiredTigerTestCase, suite_subprocess): bigkey = 'Key' * 100 bigval = 'Value' * 100 - # - # Set up all the directories needed for the test. We have a full backup directory for each - # iteration and an incremental backup for each iteration. That way we can compare the full and - # incremental each time through. - # - def setup_directories(self): - # We're only coming through once so just set up the 0 and 1 directories. - for i in range(0, 2): - # The log directory is a subdirectory of the home directory, - # creating that will make the home directory also. - log_dir = self.home_incr + '.' + str(i) + '/' + self.logpath - os.makedirs(log_dir) - if i != 0: - log_dir = self.home_full + '.' + str(i) + '/' + self.logpath - os.makedirs(log_dir) - def range_copy(self, filename, offset, size): read_from = filename old_to = self.home_incr + '.' + str(self.counter - 1) + '/' + filename @@ -219,26 +200,10 @@ class test_backup19(wttest.WiredTigerTestCase, suite_subprocess): self.assertEqual(ret, wiredtiger.WT_NOTFOUND) bkup_c.close() - def compare_backups(self, t_uri): - # Run wt dump on full backup directory. - full_backup_out = self.full_out + '.' + str(self.counter) - home_dir = self.home_full + '.' + str(self.counter) - if self.counter == 0: - home_dir = self.home - self.runWt(['-R', '-h', home_dir, 'dump', t_uri], outfilename=full_backup_out) - - # Run wt dump on incremental backup directory. - incr_backup_out = self.incr_out + '.' + str(self.counter) - home_dir = self.home_incr + '.' + str(self.counter) - self.runWt(['-R', '-h', home_dir, 'dump', t_uri], outfilename=incr_backup_out) - - self.assertEqual(True, - compare_files(self, full_backup_out, incr_backup_out)) - # # Add data to the given uri. # - def add_data(self, uri): + def add_complex_data(self, uri): c = self.session.open_cursor(uri, None, None) # The first time we want to add in a lot of data. Then after that we want to # rapidly change a single key to create a hotspot in one block. @@ -270,21 +235,20 @@ class test_backup19(wttest.WiredTigerTestCase, suite_subprocess): self.home = self.bkp_home self.session.create(self.uri, "key_format=S,value_format=S") - self.setup_directories() + self.setup_directories(2, self.home_incr, self.home_full, self.logpath) self.pr('*** Add data, checkpoint, take backups and validate ***') self.pr('Adding initial data') self.initial_backup = True - self.add_data(self.uri) + self.add_complex_data(self.uri) self.take_full_backup() self.initial_backup = False self.session.checkpoint() - self.add_data(self.uri) + self.add_complex_data(self.uri) self.session.checkpoint() self.take_full_backup() self.take_incr_backup() - self.compare_backups(self.uri) - + self.compare_backups(self.uri, self.home_full, self.home_incr, str(self.counter)) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_base01.py b/src/third_party/wiredtiger/test/suite/test_base01.py index cb4219a706a..4638aa7ef27 100644 --- a/src/third_party/wiredtiger/test/suite/test_base01.py +++ b/src/third_party/wiredtiger/test/suite/test_base01.py @@ -66,7 +66,7 @@ class test_base01(wttest.WiredTigerTestCase): gotException = True self.pr('got expected exception: ' + str(e)) self.assertTrue(str(e).find('nvalid argument') >= 0) - self.assertTrue(gotException, 'expected exception') + self.assertTrue(gotException, msg = 'expected exception') def test_empty(self): """ @@ -96,7 +96,7 @@ class test_base01(wttest.WiredTigerTestCase): getcursor = self.cursor_s(self.table_name2, 'key1') ret = getcursor.search() self.assertTrue(ret == 0) - self.assertTrue(getcursor.get_value(), 'value1') + self.assertEqual(getcursor.get_value(), 'value1') self.pr('closing cursor') getcursor.close() diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint02.py b/src/third_party/wiredtiger/test/suite/test_checkpoint02.py index 19fdb426eac..c64e1c32596 100755 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint02.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint02.py @@ -25,6 +25,11 @@ # OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. +# +# [TEST_TAGS] +# checkpoint:correctness:checkpoint_data +# [END_TAGS] +# import queue, threading, time, wiredtiger, wttest from wtthread import checkpoint_thread, op_thread diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint03.py b/src/third_party/wiredtiger/test/suite/test_checkpoint03.py index 9e2c299404f..2f4abdb532e 100644 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint03.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint03.py @@ -26,6 +26,10 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. # +# [TEST_TAGS] +# checkpoint:correctness:checkpoint_data +# [END_TAGS] +# # test_checkpoint03.py # Test that checkpoint writes out updates to the history store file. # diff --git a/src/third_party/wiredtiger/test/suite/test_hs03.py b/src/third_party/wiredtiger/test/suite/test_hs03.py index ea98944c3e1..a70a1bc35b6 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs03.py +++ b/src/third_party/wiredtiger/test/suite/test_hs03.py @@ -30,6 +30,7 @@ from helper import copy_wiredtiger_home import wiredtiger, wttest from wiredtiger import stat from wtdataset import SimpleDataSet +from wtscenario import make_scenarios def timestamp_str(t): return '%x' % t @@ -40,6 +41,12 @@ class test_hs03(wttest.WiredTigerTestCase): # Force a small cache. conn_config = 'cache_size=50MB,statistics=(fast)' session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer', dict(key_format='i')), + ('string', dict(key_format='S')) + ] + scenarios = make_scenarios(key_format_values) def get_stat(self, stat): stat_cursor = self.session.open_cursor('statistics:') @@ -61,7 +68,7 @@ class test_hs03(wttest.WiredTigerTestCase): # Create a small table. uri = "table:test_hs03" nrows = 100 - ds = SimpleDataSet(self, uri, nrows, key_format="S", value_format='u') + ds = SimpleDataSet(self, uri, nrows, key_format=self.key_format, value_format='u') ds.populate() bigvalue = b"aaaaa" * 100 diff --git a/src/third_party/wiredtiger/test/suite/test_lsm01.py b/src/third_party/wiredtiger/test/suite/test_lsm01.py index 6dae354e330..c5d84145277 100644 --- a/src/third_party/wiredtiger/test/suite/test_lsm01.py +++ b/src/third_party/wiredtiger/test/suite/test_lsm01.py @@ -77,7 +77,7 @@ class test_lsm01(wttest.WiredTigerTestCase): args += ')' # Close the LSM configuration option group self.verbose(3, 'Test LSM with config: ' + args + ' count: ' + str(self.nrecs)) - SimpleDataSet(self, self.uri, self.nrecs).populate() + SimpleDataSet(self, self.uri, self.nrecs, config=args).populate() # TODO: Adding an explicit drop here can cause deadlocks, if a merge # is still happening. See issue #349. diff --git a/src/third_party/wiredtiger/test/suite/test_tiered01.py b/src/third_party/wiredtiger/test/suite/test_tiered01.py new file mode 100644 index 00000000000..2a41c3ff7ef --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_tiered01.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2020 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wtscenario, wttest +from wtdataset import SimpleDataSet + +# test_tiered01.py +# Basic tiered tree test +class test_tiered01(wttest.WiredTigerTestCase): + K = 1024 + M = 1024 * K + G = 1024 * M + uri = "table:test_tiered01" + + chunk_size_scenarios = wtscenario.quick_scenarios('s_chunk_size', + [1*M,20*M,None], [0.6,0.6,0.6]) + # Occasionally add a lot of records, so that merges (and bloom) happen. + record_count_scenarios = wtscenario.quick_scenarios( + 'nrecs', [10, 10000], [0.9, 0.1]) + + config_vars = [ 'chunk_size', ] + + scenarios = wtscenario.make_scenarios( + chunk_size_scenarios, record_count_scenarios, + prune=100, prunelong=500) + + # Test create of an object. + def test_tiered(self): + self.session.create('file:first.wt', 'key_format=S') + self.session.create('file:second.wt', 'key_format=S') + args = 'type=tiered,key_format=S' + args += ',tiered=(' # Start the tiered configuration options. + args += 'tiers=("file:first.wt", "file:second.wt"),' + # add names to args, e.g. args += ',session_max=30' + for var in self.config_vars: + value = getattr(self, 's_' + var) + if value != None: + if var == 'verbose': + value = '[' + str(value) + ']' + value = {True : 'true', False : 'false'}.get(value, value) + args += ',' + var + '=' + str(value) + args += ')' # Close the tiered configuration option group + self.verbose(3, + 'Test tiered with config: ' + args + ' count: ' + str(self.nrecs)) + SimpleDataSet(self, self.uri, self.nrecs, config=args).populate() + + # self.session.drop(self.uri) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_txn08.py b/src/third_party/wiredtiger/test/suite/test_txn08.py index 7116aa4eb75..0f82f079867 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn08.py +++ b/src/third_party/wiredtiger/test/suite/test_txn08.py @@ -72,6 +72,47 @@ class test_txn08(wttest.WiredTigerTestCase, suite_subprocess): '\\u0001\\u0002abcd\\u0003\\u0004') self.check_file_contains('printlog-hex.out', '0102616263640304') + # Check the printlog start LSN and stop LSN feature. + self.runWt(['printlog', '-l 2,128'], outfilename='printlog-range01.out') + self.check_file_contains('printlog-range01.out', + '"lsn" : [2,128],') + self.check_file_contains('printlog-range01.out', + '"lsn" : [2,256],') + self.check_file_not_contains('printlog-range01.out', + '"lsn" : [1,128],') + self.runWt(['printlog', '-l 2,128,3,128'], outfilename='printlog-range02.out') + self.check_file_contains('printlog-range02.out', + '"lsn" : [2,128],') + self.check_file_not_contains('printlog-range02.out', + '"lsn" : [1,128],') + self.check_file_not_contains('printlog-range02.out', + '"lsn" : [3,256],') + # Test for invalid LSN, return WT_NOTFOUND + self.runWt(['printlog', '-l 2,300'], outfilename='printlog-range03.out', errfilename='printlog-range03.err', failure=True) + self.check_file_contains('printlog-range03.err','WT_NOTFOUND') + # Test for Start > end, print the start lsn and then stop + self.runWt(['printlog', '-l 3,128,2,128'], outfilename='printlog-range04.out') + self.check_file_contains('printlog-range04.out','"lsn" : [3,128],') + self.check_file_not_contains('printlog-range04.out','"lsn" : [3,256],') + # Test for usage error, print the usage message if arguments are invalid + self.runWt(['printlog', '-l'], outfilename='printlog-range05.out', errfilename='printlog-range05.err', failure=True) + self.check_file_contains('printlog-range05.err','wt: option requires an argument -- l') + # Test start and end offset of 0 + self.runWt(['printlog', '-l 2,0,3,0'], outfilename='printlog-range06.out') + self.check_file_contains('printlog-range06.out', + '"lsn" : [2,128],') + self.check_file_not_contains('printlog-range06.out', + '"lsn" : [1,128],') + self.check_file_not_contains('printlog-range06.out', + '"lsn" : [3,256],') + # Test for start == end + self.runWt(['printlog', '-l 1,256,1,256'], outfilename='printlog-range07.out') + self.check_file_contains('printlog-range07.out', + '"lsn" : [1,256],') + self.check_file_not_contains('printlog-range07.out', + '"lsn" : [1,128],') + self.check_file_not_contains('printlog-range07.out', + '"lsn" : [1,384],') if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/wtbackup.py b/src/third_party/wiredtiger/test/suite/wtbackup.py new file mode 100644 index 00000000000..f418d11ddff --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/wtbackup.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2021 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +import os, glob +import wttest, wiredtiger +from suite_subprocess import suite_subprocess +from helper import compare_files + +# Shared base class used by backup tests. +class backup_base(wttest.WiredTigerTestCase, suite_subprocess): + cursor_config = None # a config string for cursors + mult = 0 # counter to have variance in data + nops = 100 # number of operations added to uri + + # We use counter to produce unique backup names for multiple iterations + # of incremental backup tests. + counter = 0 + # To determine whether to increase/decrease counter, which determines + initial_backup = True + # Used for populate function + rows = 100 + populate_big = None + + # + # Add data to the given uri. + # Allows the option for doing a session checkpoint after adding data. + # + def add_data(self, uri, key, val, do_checkpoint=False): + assert(self.nops != 0) + c = self.session.open_cursor(uri, None, self.cursor_config) + for i in range(0, self.nops): + num = i + (self.mult * self.nops) + k = key + str(num) + v = val + str(num) + c[k] = v + c.close() + if do_checkpoint: + self.session.checkpoint() + # Increase the counter so that later backups have unique ids. + if not self.initial_backup: + self.counter += 1 + # Increase the multiplier so that later calls insert unique items. + self.mult += 1 + + # + # Populate a set of objects. + # + def populate(self, objs, do_checkpoint=False, skiplsm=False): + cg_config = '' + for i in objs: + if len(i) > 2: + if i[2] and skiplsm: + continue + if i[2] == self.populate_big: + self.rows = 50000 # Big Object + else: + self.rows = 1000 # Small Object + if len(i) > 3: + cg_config = i[3] + i[1](self, i[0], self.rows, cgconfig = cg_config).populate() + + # Backup needs a checkpoint + if do_checkpoint: + self.session.checkpoint() + + # + # Set up all the directories needed for the test. We have a full backup directory for each + # iteration and an incremental backup for each iteration. That way we can compare the full and + # incremental each time through. + # + def setup_directories(self, max_iteration, home_incr, home_full, logpath): + for i in range(0, max_iteration): + # The log directory is a subdirectory of the home directory, + # creating that will make the home directory also. + + home_incr_dir = home_incr + '.' + str(i) + if os.path.exists(home_incr_dir): + os.remove(home_incr_dir) + os.makedirs(home_incr_dir + '/' + logpath) + + if i == 0: + continue + home_full_dir = home_full + '.' + str(i) + if os.path.exists(home_full_dir): + os.remove(home_full_dir) + os.makedirs(home_full_dir + '/' + logpath) + + # + # Check that a URI doesn't exist, both the meta-data and the file names. + # + def confirmPathDoesNotExist(self, uri, dir): + conn = self.wiredtiger_open(dir) + session = conn.open_session() + self.assertRaises(wiredtiger.WiredTigerError, + lambda: session.open_cursor(uri, None, None)) + conn.close() + + self.assertEqual( + glob.glob(dir + '*' + uri.split(":")[1] + '*'), [], + 'confirmPathDoesNotExist: URI exists, file name matching \"' + + uri.split(":")[1] + '\" found') + + # + # Compare against two directory paths using the wt dump command. + # The suffix allows the option to add distinctive tests adding suffix to both the output files and directories + # + def compare_backups(self, uri, base_dir_home, other_dir_home, suffix = None): + sfx = "" + if suffix != None: + sfx = "." + suffix + base_out = "./backup_base" + sfx + base_dir = base_dir_home + sfx + + if os.path.exists(base_out): + os.remove(base_out) + + # Run wt dump on base backup directory + self.runWt(['-R', '-h', base_dir, 'dump', uri], outfilename=base_out) + other_out = "./backup_other" + sfx + if os.path.exists(other_out): + os.remove(other_out) + # Run wt dump on incremental backup + other_dir = other_dir_home + sfx + self.runWt(['-R', '-h', other_dir, 'dump', uri], outfilename=other_out) + self.assertEqual(True, compare_files(self, base_out, other_out)) |