diff options
author | Luke Chen <luke.chen@mongodb.com> | 2019-06-14 13:56:58 +1000 |
---|---|---|
committer | Luke Chen <luke.chen@mongodb.com> | 2019-06-14 13:56:58 +1000 |
commit | 518f382a78f904b2e7adf8ac36acc3a34a82fb4f (patch) | |
tree | 9b40350606570fdcea425ade967855ebb8d707de /src | |
parent | fab9fde6ba3e08b710b07a42c82b635b1b31c989 (diff) | |
download | mongo-518f382a78f904b2e7adf8ac36acc3a34a82fb4f.tar.gz |
Import wiredtiger: 40e3225e55ffbf0c9819af138fa4a014c0639c72 from branch mongodb-4.2
ref: 9b85ad8968..40e3225e55
for: 4.3.1
WT-4477 Add eviction debug mode and extra checks
WT-4494 Support import of of standalone WiredTiger files
WT-4580 Abort transactions that perform updates with ignore_prepare=true
WT-4719 Non-zero update count detected when committing prepared transaction
WT-4787 Coverity: Potential infinite loops
WT-4790 Coverity: Memory leak within __wt_curbulk_init
WT-4827 Apply commit timestamps when a truncated page is read
WT-4829 File names are generated incorrectly in operation tracking scripts
WT-4837 Disabled cache during verify
WT-4839 Add a feature to python testsuite to run a random subset of tests
WT-4844 Only log an informational message when a set read-timestamp is older than the oldest timestamp
WT-4848 Fix perf regression when calculating differences
WT-4850 Fix __slvg_checkpoint() metadata memory leak
WT-4851 heap-use-after-free when block manager grows buffer during final checkpoint
WT-4852 Evergreen runs of test/format require shared library extensions
WT-4853 Standalone recovery code cannot handle deleted checkpoints
WT-4854 wt utility list command has a memory leak and minor error handling bugs
WT-4855 WiredTiger recovery should detect files without unique IDs
WT-4859 Fix test/format program path issue for endian tests
Diffstat (limited to 'src')
84 files changed, 2589 insertions, 938 deletions
diff --git a/src/third_party/wiredtiger/SConstruct b/src/third_party/wiredtiger/SConstruct index 20916ac8da0..786dff93a6d 100644 --- a/src/third_party/wiredtiger/SConstruct +++ b/src/third_party/wiredtiger/SConstruct @@ -290,12 +290,13 @@ Default(wtlib, wtdll) wtbin = env.Program("wt", [ "src/utilities/util_alter.c", "src/utilities/util_backup.c", - "src/utilities/util_cpyright.c", "src/utilities/util_compact.c", + "src/utilities/util_cpyright.c", "src/utilities/util_create.c", "src/utilities/util_downgrade.c", "src/utilities/util_drop.c", "src/utilities/util_dump.c", + "src/utilities/util_import.c", "src/utilities/util_list.c", "src/utilities/util_load.c", "src/utilities/util_load_json.c", diff --git a/src/third_party/wiredtiger/build_posix/Make.base b/src/third_party/wiredtiger/build_posix/Make.base index 7543b346ea5..db8ae2fabf4 100644 --- a/src/third_party/wiredtiger/build_posix/Make.base +++ b/src/third_party/wiredtiger/build_posix/Make.base @@ -19,6 +19,7 @@ wt_SOURCES =\ src/utilities/util_downgrade.c \ src/utilities/util_drop.c \ src/utilities/util_dump.c \ + src/utilities/util_import.c \ src/utilities/util_list.c \ src/utilities/util_load.c \ src/utilities/util_load_json.c \ diff --git a/src/third_party/wiredtiger/build_posix/Make.subdirs b/src/third_party/wiredtiger/build_posix/Make.subdirs index d598c74c631..a7fc291fc61 100644 --- a/src/third_party/wiredtiger/build_posix/Make.subdirs +++ b/src/third_party/wiredtiger/build_posix/Make.subdirs @@ -37,6 +37,7 @@ test/cursor_order test/fops test/format test/huge +test/import test/manydbs test/packing test/readonly diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 37b84df00e1..146d6c06668 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -495,6 +495,12 @@ connection_runtime_config = [ adjust log archiving to retain the log records of this number of checkpoints. Zero or one means perform normal archiving.''', min='0', max='1024'), + Config('eviction', 'false', r''' + if true, modify internal algorithms to change skew to force + lookaside eviction to happen more aggressively. This includes but + is not limited to not skewing newest, not favoring leaf pages, + and modifying the eviction score mechanism.''', + type='boolean'), Config('rollback_error', '0', r''' return a WT_ROLLBACK error from a transaction operation about every Nth operation to simulate a collision''', @@ -1109,6 +1115,8 @@ methods = { type='boolean'), ]), +'WT_SESSION.import' : Method([]), + 'WT_SESSION.join' : Method([ Config('compare', '"eq"', r''' modifies the set of items to be returned so that the index key diff --git a/src/third_party/wiredtiger/dist/filelist b/src/third_party/wiredtiger/dist/filelist index 036b1a8b1a9..c529ebc0366 100644 --- a/src/third_party/wiredtiger/dist/filelist +++ b/src/third_party/wiredtiger/dist/filelist @@ -6,6 +6,7 @@ src/async/async_op.c src/async/async_worker.c src/block/block_addr.c src/block/block_ckpt.c +src/block/block_ckpt_scan.c src/block/block_compact.c src/block/block_ext.c src/block/block_map.c @@ -26,6 +27,7 @@ src/btree/bt_delete.c src/btree/bt_discard.c src/btree/bt_handle.c src/btree/bt_huffman.c +src/btree/bt_import.c src/btree/bt_io.c src/btree/bt_misc.c src/btree/bt_ovfl.c @@ -185,7 +187,6 @@ src/schema/schema_worker.c src/session/session_api.c src/session/session_compact.c src/session/session_dhandle.c -src/session/session_salvage.c src/support/cond_auto.c src/support/crypto.c src/support/err.c diff --git a/src/third_party/wiredtiger/dist/s_define.list b/src/third_party/wiredtiger/dist/s_define.list index e82ef2ad31f..1c9121da3f2 100644 --- a/src/third_party/wiredtiger/dist/s_define.list +++ b/src/third_party/wiredtiger/dist/s_define.list @@ -16,6 +16,7 @@ WT_ATOMIC_CAS WT_ATOMIC_CAS_FUNC WT_ATOMIC_FUNC WT_BLOCK_DESC_SIZE +WT_BLOCK_EXTLIST_VERSION_ORIG WT_BLOCK_HEADER_SIZE WT_CACHE_LINE_ALIGNMENT WT_CACHE_LINE_PAD_BEGIN diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index 06e58b46236..d8f89a9e6b9 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -105,6 +105,7 @@ DeadStores Decrement Decrypt DeleteFileW +Dh EACCES EAGAIN EB @@ -526,6 +527,7 @@ buf bufsz bugfix bugfixes +builddir builtin builtins bytelock diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index 478830108f3..92008c8f7e0 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -514,6 +514,8 @@ connection_stats = [ SessionOpStat('session_table_create_success', 'table create successful calls', 'no_clear,no_scale'), SessionOpStat('session_table_drop_fail', 'table drop failed calls', 'no_clear,no_scale'), SessionOpStat('session_table_drop_success', 'table drop successful calls', 'no_clear,no_scale'), + SessionOpStat('session_table_import_fail', 'table import failed calls', 'no_clear,no_scale'), + SessionOpStat('session_table_import_success', 'table import successful calls', 'no_clear,no_scale'), SessionOpStat('session_table_rebalance_fail', 'table rebalance failed calls', 'no_clear,no_scale'), SessionOpStat('session_table_rebalance_success', 'table rebalance successful calls', 'no_clear,no_scale'), SessionOpStat('session_table_rename_fail', 'table rename failed calls', 'no_clear,no_scale'), diff --git a/src/third_party/wiredtiger/examples/c/ex_all.c b/src/third_party/wiredtiger/examples/c/ex_all.c index 9f1e1df17f3..5e5dbee6c03 100644 --- a/src/third_party/wiredtiger/examples/c/ex_all.c +++ b/src/third_party/wiredtiger/examples/c/ex_all.c @@ -603,16 +603,8 @@ named_snapshot_ops(WT_SESSION *session) } static void -session_ops(WT_SESSION *session) +session_ops_create(WT_SESSION *session) { - WT_CONNECTION *conn; - - conn = session->connection; - - /*! [Reconfigure a session] */ - error_check(session->reconfigure(session, "isolation=snapshot")); - /*! [Reconfigure a session] */ - /*! [Create a table] */ error_check(session->create(session, "table:mytable", "key_format=S,value_format=S")); @@ -622,12 +614,6 @@ session_ops(WT_SESSION *session) /*! [Create a column-store table] */ error_check(session->create(session, "table:mytable", "key_format=r,value_format=S")); - - /*! [Alter a table] */ - error_check(session->alter(session, - "table:mytable", "access_pattern_hint=random")); - /*! [Alter a table] */ - /*! [Create a column-store table] */ error_check(session->drop(session, "table:mytable", NULL)); @@ -736,16 +722,41 @@ session_ops(WT_SESSION *session) "key_format=r,value_format=S,cache_resident=true")); /*! [Create a cache-resident object] */ error_check(session->drop(session, "table:mytable", NULL)); +} +static void +session_ops(WT_SESSION *session) +{ + WT_CONNECTION *conn; + + conn = session->connection; + + /* WT_SESSION.create operations. */ + session_ops_create(session); + + /*! [Reconfigure a session] */ + error_check(session->reconfigure(session, "isolation=snapshot")); + /*! [Reconfigure a session] */ { /* Create a table for the session operations. */ error_check(session->create( session, "table:mytable", "key_format=S,value_format=S")); + /*! [Alter a table] */ + error_check(session->alter(session, + "table:mytable", "access_pattern_hint=random")); + /*! [Alter a table] */ + /*! [Compact a table] */ error_check(session->compact(session, "table:mytable", NULL)); /*! [Compact a table] */ +#ifdef MIGHT_NOT_RUN + /*! [Import a file] */ + error_check(session->import(session, "file:import", NULL)); + /*! [Import a file] */ +#endif + /*! [Rebalance a table] */ error_check(session->rebalance(session, "table:mytable", NULL)); /*! [Rebalance a table] */ diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 22114815861..d5a8a1b8330 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -1,5 +1,5 @@ { - "commit": "9b85ad89688bd72b8a649d844a7e458832955764", + "commit": "40e3225e55ffbf0c9819af138fa4a014c0639c72", "github": "wiredtiger/wiredtiger.git", "vendor": "wiredtiger", "branch": "mongodb-4.2" diff --git a/src/third_party/wiredtiger/lang/java/java_doc.i b/src/third_party/wiredtiger/lang/java/java_doc.i index 8b363ee4cf7..fa6a4c883c4 100644 --- a/src/third_party/wiredtiger/lang/java/java_doc.i +++ b/src/third_party/wiredtiger/lang/java/java_doc.i @@ -36,6 +36,7 @@ COPYDOC(__wt_session, WT_SESSION, reconfigure) COPYDOC(__wt_session, WT_SESSION, open_cursor) COPYDOC(__wt_session, WT_SESSION, alter) COPYDOC(__wt_session, WT_SESSION, create) +COPYDOC(__wt_session, WT_SESSION, import) COPYDOC(__wt_session, WT_SESSION, compact) COPYDOC(__wt_session, WT_SESSION, drop) COPYDOC(__wt_session, WT_SESSION, join) diff --git a/src/third_party/wiredtiger/src/block/block_addr.c b/src/third_party/wiredtiger/src/block/block_addr.c index f08e4430a73..d7b0c56bd56 100644 --- a/src/third_party/wiredtiger/src/block/block_addr.c +++ b/src/third_party/wiredtiger/src/block/block_addr.c @@ -206,7 +206,7 @@ __wt_block_ckpt_decode(WT_SESSION *wt_session, */ int __wt_block_ckpt_to_buffer(WT_SESSION_IMPL *session, - WT_BLOCK *block, uint8_t **pp, WT_BLOCK_CKPT *ci) + WT_BLOCK *block, uint8_t **pp, WT_BLOCK_CKPT *ci, bool skip_avail) { uint64_t a; @@ -220,8 +220,11 @@ __wt_block_ckpt_to_buffer(WT_SESSION_IMPL *session, ci->root_offset, ci->root_size, ci->root_checksum)); WT_RET(__wt_block_addr_to_buffer(block, pp, ci->alloc.offset, ci->alloc.size, ci->alloc.checksum)); - WT_RET(__wt_block_addr_to_buffer(block, pp, - ci->avail.offset, ci->avail.size, ci->avail.checksum)); + if (skip_avail) + WT_RET(__wt_block_addr_to_buffer(block, pp, 0, 0, 0)); + else + WT_RET(__wt_block_addr_to_buffer(block, pp, + ci->avail.offset, ci->avail.size, ci->avail.checksum)); WT_RET(__wt_block_addr_to_buffer(block, pp, ci->discard.offset, ci->discard.size, ci->discard.checksum)); a = (uint64_t)ci->file_size; @@ -231,3 +234,81 @@ __wt_block_ckpt_to_buffer(WT_SESSION_IMPL *session, return (0); } + +/* + * __wt_ckpt_verbose -- + * Display a printable string representation of a checkpoint. + */ +void +__wt_ckpt_verbose(WT_SESSION_IMPL *session, WT_BLOCK *block, + const char *tag, const char *ckpt_name, const uint8_t *ckpt_string) +{ + WT_BLOCK_CKPT *ci, _ci; + WT_DECL_ITEM(tmp); + WT_DECL_RET; + + if (ckpt_string == NULL) { + __wt_verbose_worker(session, + "%s: %s: %s%s[Empty]", block->name, tag, + ckpt_name ? ckpt_name : "", + ckpt_name ? ": " : ""); + return; + } + + /* Initialize the checkpoint, crack the cookie. */ + ci = &_ci; + WT_ERR(__wt_block_ckpt_init(session, ci, "string")); + WT_ERR(__wt_block_buffer_to_ckpt(session, block, ckpt_string, ci)); + + WT_ERR(__wt_scr_alloc(session, 0, &tmp)); + WT_ERR(__wt_buf_fmt(session, tmp, "version=%" PRIu8, ci->version)); + if (ci->root_offset == WT_BLOCK_INVALID_OFFSET) + WT_ERR(__wt_buf_catfmt(session, tmp, ", root=[Empty]")); + else + WT_ERR(__wt_buf_catfmt(session, tmp, + ", root=[%" + PRIuMAX "-%" PRIuMAX ", %" PRIu32 ", %" PRIu32 "]", + (uintmax_t)ci->root_offset, + (uintmax_t)(ci->root_offset + ci->root_size), + ci->root_size, ci->root_checksum)); + if (ci->alloc.offset == WT_BLOCK_INVALID_OFFSET) + WT_ERR(__wt_buf_catfmt(session, tmp, ", alloc=[Empty]")); + else + WT_ERR(__wt_buf_catfmt(session, tmp, + ", alloc=[%" + PRIuMAX "-%" PRIuMAX ", %" PRIu32 ", %" PRIu32 "]", + (uintmax_t)ci->alloc.offset, + (uintmax_t)(ci->alloc.offset + ci->alloc.size), + ci->alloc.size, ci->alloc.checksum)); + if (ci->avail.offset == WT_BLOCK_INVALID_OFFSET) + WT_ERR(__wt_buf_catfmt(session, tmp, ", avail=[Empty]")); + else + WT_ERR(__wt_buf_catfmt(session, tmp, + ", avail=[%" + PRIuMAX "-%" PRIuMAX ", %" PRIu32 ", %" PRIu32 "]", + (uintmax_t)ci->avail.offset, + (uintmax_t)(ci->avail.offset + ci->avail.size), + ci->avail.size, ci->avail.checksum)); + if (ci->discard.offset == WT_BLOCK_INVALID_OFFSET) + WT_ERR(__wt_buf_catfmt(session, tmp, ", discard=[Empty]")); + else + WT_ERR(__wt_buf_catfmt(session, tmp, + ", discard=[%" + PRIuMAX "-%" PRIuMAX ", %" PRIu32 ", %" PRIu32 "]", + (uintmax_t)ci->discard.offset, + (uintmax_t)(ci->discard.offset + ci->discard.size), + ci->discard.size, ci->discard.checksum)); + WT_ERR(__wt_buf_catfmt(session, tmp, + ", file size=%" PRIuMAX, (uintmax_t)ci->file_size)); + WT_ERR(__wt_buf_catfmt(session, tmp, + ", checkpoint size=%" PRIu64, ci->ckpt_size)); + + __wt_verbose_worker(session, + "%s: %s: %s%s%s", + block->name, tag, + ckpt_name ? ckpt_name : "", + ckpt_name ? ": " : "", (const char *)tmp->data); + +err: __wt_scr_free(session, &tmp); + __wt_block_ckpt_destroy(session, ci); +} diff --git a/src/third_party/wiredtiger/src/block/block_ckpt.c b/src/third_party/wiredtiger/src/block/block_ckpt.c index 40b0fef5a29..db6b458d671 100644 --- a/src/third_party/wiredtiger/src/block/block_ckpt.c +++ b/src/third_party/wiredtiger/src/block/block_ckpt.c @@ -9,10 +9,8 @@ #include "wt_internal.h" static int __ckpt_process(WT_SESSION_IMPL *, WT_BLOCK *, WT_CKPT *); -static int __ckpt_string( - WT_SESSION_IMPL *, WT_BLOCK *, const uint8_t *, WT_ITEM *); -static int __ckpt_update( - WT_SESSION_IMPL *, WT_BLOCK *, WT_CKPT *, WT_BLOCK_CKPT *, bool); +static int __ckpt_update(WT_SESSION_IMPL *, + WT_BLOCK *, WT_CKPT *, WT_CKPT *, WT_BLOCK_CKPT *, bool); /* * __wt_block_ckpt_init -- @@ -49,7 +47,6 @@ __wt_block_checkpoint_load(WT_SESSION_IMPL *session, WT_BLOCK *block, uint8_t *root_addr, size_t *root_addr_sizep, bool checkpoint) { WT_BLOCK_CKPT *ci, _ci; - WT_DECL_ITEM(tmp); WT_DECL_RET; uint8_t *endp; @@ -62,15 +59,8 @@ __wt_block_checkpoint_load(WT_SESSION_IMPL *session, WT_BLOCK *block, ci = NULL; - if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT)) { - if (addr != NULL) { - WT_ERR(__wt_scr_alloc(session, 0, &tmp)); - WT_ERR(__ckpt_string(session, block, addr, tmp)); - } - __wt_verbose(session, WT_VERB_CHECKPOINT, - "%s: load-checkpoint: %s", block->name, - addr == NULL ? "[Empty]" : (const char *)tmp->data); - } + if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT)) + __wt_ckpt_verbose(session, block, "load", NULL, addr); /* * There's a single checkpoint in the file that can be written, all of @@ -153,7 +143,6 @@ err: /* if (checkpoint && ci != NULL) __wt_block_ckpt_destroy(session, ci); - __wt_scr_free(session, &tmp); return (ret); } @@ -388,7 +377,6 @@ __ckpt_process(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckptbase) { WT_BLOCK_CKPT *a, *b, *ci; WT_CKPT *ckpt, *next_ckpt; - WT_DECL_ITEM(tmp); WT_DECL_RET; uint64_t ckpt_size; bool deleting, fatal, locked; @@ -544,15 +532,10 @@ __ckpt_process(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckptbase) !F_ISSET(ckpt, WT_CKPT_DELETE)) continue; - if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT)) { - if (tmp == NULL) - WT_ERR(__wt_scr_alloc(session, 0, &tmp)); - WT_ERR(__ckpt_string( - session, block, ckpt->raw.data, tmp)); - __wt_verbose(session, WT_VERB_CHECKPOINT, - "%s: delete-checkpoint: %s: %s", - block->name, ckpt->name, (const char *)tmp->data); - } + if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT)) + __wt_ckpt_verbose(session, + block, "delete", ckpt->name, ckpt->raw.data); + /* * Find the checkpoint into which we'll roll this checkpoint's * blocks: it's the next real checkpoint in the list, and it @@ -642,8 +625,8 @@ __ckpt_process(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckptbase) /* Update checkpoints marked for update. */ WT_CKPT_FOREACH(ckptbase, ckpt) if (F_ISSET(ckpt, WT_CKPT_UPDATE)) - WT_ERR(__ckpt_update( - session, block, ckpt, ckpt->bpriv, false)); + WT_ERR(__ckpt_update(session, + block, ckptbase, ckpt, ckpt->bpriv, false)); live_update: /* Truncate the file if that's possible. */ @@ -686,7 +669,8 @@ live_update: ci->ckpt_size = WT_MIN(ckpt_size, (uint64_t)block->size); - WT_ERR(__ckpt_update(session, block, ckpt, ci, true)); + WT_ERR(__ckpt_update( + session, block, ckptbase, ckpt, ci, true)); } /* @@ -733,7 +717,6 @@ err: if (ret != 0 && fatal) { if ((ci = ckpt->bpriv) != NULL) __wt_block_ckpt_destroy(session, ci); - __wt_scr_free(session, &tmp); return (ret); } @@ -742,10 +725,10 @@ err: if (ret != 0 && fatal) { * Update a checkpoint. */ static int -__ckpt_update(WT_SESSION_IMPL *session, - WT_BLOCK *block, WT_CKPT *ckpt, WT_BLOCK_CKPT *ci, bool is_live) +__ckpt_update(WT_SESSION_IMPL *session, WT_BLOCK *block, + WT_CKPT *ckptbase, WT_CKPT *ckpt, WT_BLOCK_CKPT *ci, bool is_live) { - WT_DECL_ITEM(tmp); + WT_DECL_ITEM(a); WT_DECL_RET; uint8_t *endp; @@ -764,6 +747,37 @@ __ckpt_update(WT_SESSION_IMPL *session, WT_RET(__wt_block_extlist_write(session, block, &ci->discard, NULL)); /* + * If this is the final block, we append an incomplete copy of the + * checkpoint information to the avail list for standalone retrieval. + */ + if (is_live) { + /* + * Copy the INCOMPLETE checkpoint information into the + * checkpoint. + */ + WT_RET(__wt_buf_init( + session, &ckpt->raw, WT_BLOCK_CHECKPOINT_BUFFER)); + endp = ckpt->raw.mem; + WT_RET(__wt_block_ckpt_to_buffer( + session, block, &endp, ci, true)); + ckpt->raw.size = WT_PTRDIFF(endp, ckpt->raw.mem); + + /* + * Convert the INCOMPLETE checkpoint array into its metadata + * representation. This must match what is eventually written + * into the metadata file, in other words, everything must be + * initialized before the block manager does the checkpoint. + */ + WT_RET(__wt_scr_alloc(session, 8 * 1024, &a)); + ret = __wt_meta_ckptlist_to_meta(session, ckptbase, a); + if (ret == 0) + ret = __wt_strndup( + session, a->data, a->size, &ckpt->block_checkpoint); + __wt_scr_free(session, &a); + WT_RET(ret); + } + + /* * We only write an avail list for the live system, other checkpoint's * avail lists are static and never change. * @@ -775,9 +789,13 @@ __ckpt_update(WT_SESSION_IMPL *session, * it's not truly available until the new checkpoint locations have been * saved to the metadata. */ - if (is_live) - WT_RET(__wt_block_extlist_write( - session, block, &ci->avail, &ci->ckpt_avail)); + if (is_live) { + block->final_ckpt = ckpt; + ret = __wt_block_extlist_write( + session, block, &ci->avail, &ci->ckpt_avail); + block->final_ckpt = NULL; + WT_RET(ret); + } /* * Set the file size for the live system. @@ -800,25 +818,17 @@ __ckpt_update(WT_SESSION_IMPL *session, if (is_live) ci->file_size = block->size; - /* - * Copy the checkpoint information into the checkpoint array's address - * cookie. - */ - WT_RET(__wt_buf_init(session, &ckpt->raw, WT_BTREE_MAX_ADDR_COOKIE)); + /* Copy the COMPLETE checkpoint information into the checkpoint. */ + WT_RET(__wt_buf_init(session, &ckpt->raw, WT_BLOCK_CHECKPOINT_BUFFER)); endp = ckpt->raw.mem; - WT_RET(__wt_block_ckpt_to_buffer(session, block, &endp, ci)); + WT_RET(__wt_block_ckpt_to_buffer(session, block, &endp, ci, false)); ckpt->raw.size = WT_PTRDIFF(endp, ckpt->raw.mem); - if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT)) { - WT_RET(__wt_scr_alloc(session, 0, &tmp)); - WT_ERR(__ckpt_string(session, block, ckpt->raw.data, tmp)); - __wt_verbose(session, WT_VERB_CHECKPOINT, - "%s: create-checkpoint: %s: %s", - block->name, ckpt->name, (const char *)tmp->data); - } + if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT)) + __wt_ckpt_verbose( + session, block, "create", ckpt->name, ckpt->raw.data); -err: __wt_scr_free(session, &tmp); - return (ret); + return (0); } /* @@ -882,64 +892,3 @@ err: __wt_spin_unlock(session, &block->live_lock); return (ret); } - -/* - * __ckpt_string -- - * Return a printable string representation of a checkpoint address cookie. - */ -static int -__ckpt_string(WT_SESSION_IMPL *session, - WT_BLOCK *block, const uint8_t *addr, WT_ITEM *buf) -{ - WT_BLOCK_CKPT *ci, _ci; - - /* Initialize the checkpoint, crack the cookie. */ - ci = &_ci; - WT_RET(__wt_block_ckpt_init(session, ci, "string")); - WT_RET(__wt_block_buffer_to_ckpt(session, block, addr, ci)); - - WT_RET(__wt_buf_fmt(session, buf, - "version=%" PRIu8, ci->version)); - if (ci->root_offset == WT_BLOCK_INVALID_OFFSET) - WT_RET(__wt_buf_catfmt(session, buf, ", root=[Empty]")); - else - WT_RET(__wt_buf_catfmt(session, buf, - ", root=[%" - PRIuMAX "-%" PRIuMAX ", %" PRIu32 ", %" PRIu32 "]", - (uintmax_t)ci->root_offset, - (uintmax_t)(ci->root_offset + ci->root_size), - ci->root_size, ci->root_checksum)); - if (ci->alloc.offset == WT_BLOCK_INVALID_OFFSET) - WT_RET(__wt_buf_catfmt(session, buf, ", alloc=[Empty]")); - else - WT_RET(__wt_buf_catfmt(session, buf, - ", alloc=[%" - PRIuMAX "-%" PRIuMAX ", %" PRIu32 ", %" PRIu32 "]", - (uintmax_t)ci->alloc.offset, - (uintmax_t)(ci->alloc.offset + ci->alloc.size), - ci->alloc.size, ci->alloc.checksum)); - if (ci->avail.offset == WT_BLOCK_INVALID_OFFSET) - WT_RET(__wt_buf_catfmt(session, buf, ", avail=[Empty]")); - else - WT_RET(__wt_buf_catfmt(session, buf, - ", avail=[%" - PRIuMAX "-%" PRIuMAX ", %" PRIu32 ", %" PRIu32 "]", - (uintmax_t)ci->avail.offset, - (uintmax_t)(ci->avail.offset + ci->avail.size), - ci->avail.size, ci->avail.checksum)); - if (ci->discard.offset == WT_BLOCK_INVALID_OFFSET) - WT_RET(__wt_buf_catfmt(session, buf, ", discard=[Empty]")); - else - WT_RET(__wt_buf_catfmt(session, buf, - ", discard=[%" - PRIuMAX "-%" PRIuMAX ", %" PRIu32 ", %" PRIu32 "]", - (uintmax_t)ci->discard.offset, - (uintmax_t)(ci->discard.offset + ci->discard.size), - ci->discard.size, ci->discard.checksum)); - WT_RET(__wt_buf_catfmt(session, buf, - ", file size=%" PRIuMAX, (uintmax_t)ci->file_size)); - - __wt_block_ckpt_destroy(session, ci); - - return (0); -} diff --git a/src/third_party/wiredtiger/src/block/block_ckpt_scan.c b/src/third_party/wiredtiger/src/block/block_ckpt_scan.c new file mode 100644 index 00000000000..5db299c399e --- /dev/null +++ b/src/third_party/wiredtiger/src/block/block_ckpt_scan.c @@ -0,0 +1,417 @@ +/*- + * Copyright (c) 2014-2019 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * It wasn't possible to open standalone files in historic WiredTiger databases, + * you're done if you lose the file's associated metadata. That was a mistake + * and this code is the workaround. What we need to crack a file is database + * metadata plus a list of active checkpoints as of the file's clean shutdown + * (normally stored in the database metadata). The last write done in a block + * manager's checkpoint is the avail list. If current metadata and checkpoint + * information is included in that write, we're close. We can open the file, + * read the blocks, scan until we find the avail list, and read the metadata + * and checkpoint information from there. + * Two problems remain: first, the checkpoint information isn't correct + * until we write the avail list and the checkpoint information has to include + * the avail list address plus the final file size after the write. Fortunately, + * when scanning the file for the avail lists, we're figuring out exactly the + * information needed to fix up the checkpoint information we wrote, that is, + * the avail list's offset, size and checksum triplet. As for the final file + * size, we allocate all space in the file before we calculate block checksums, + * so we can do that space allocation, then fill in the final file size before + * calculating the checksum and writing the actual block. + * The second problem is we have to be able to find the avail lists that + * include checkpoint information (ignoring previous files created by previous + * releases, and, of course, making upgrade/downgrade work seamlessly). Extent + * lists are written to their own pages, and we could version this change using + * the page header version. Extent lists have WT_PAGE_BLOCK_MANAGER page types, + * we could version this change using the upcoming WT_PAGE_VERSION_TS upgrade. + * However, that requires waiting a release (we would have to first release a + * version that ignores those new page header versions so downgrade works), and + * we're not planning a release that writes WT_PAGE_VERSION_TS page headers for + * awhile. Happily, historic WiredTiger releases have a bug. Extent lists + * consist of a set of offset/size pairs, with magic offset/size pairs at the + * beginning and end of the list. Historic releases only verified the offset of + * the special pair at the end of the list, ignoring the size. To detect avail + * lists that include appended metadata and checkpoint information, this change + * adds a version to the extent list: if size is WT_BLOCK_EXTLIST_VERSION_CKPT, + * then metadata/checkpoint information follows. + */ + +/* + * __wt_block_checkpoint_final -- + * Append metadata and checkpoint information to a buffer. + */ +int +__wt_block_checkpoint_final(WT_SESSION_IMPL *session, + WT_BLOCK *block, WT_ITEM *buf, uint8_t **file_sizep) +{ + WT_CKPT *ckpt; + size_t align_size, file_size_offset, len, size; + uint8_t *p; + + *file_sizep = 0; + + ckpt = block->final_ckpt; + p = (uint8_t *)buf->mem + buf->size; + + /* + * First, add in a counter to uniquely order checkpoints at our level. + * There's order and time information in the checkpoint itself, but the + * order isn't written and the time is only at second granularity. + * I'm using the Btree write generation for this purpose. That's + * safe and guaranteed correct because everything is locked down for the + * checkpoint, we're the only writer. Plus, because we use the write + * generation as a database connection generation, it's guaranteed to + * move forward and never repeat. + * It's a layering violation though, this is the only place the + * block manager uses the write generation. The alternative would be to + * add our own write-generation scheme in the block manager, storing a + * value and recovering it when we open the file. We could do that, as + * reading the final avail list when a file is opened is unavoidable, + * so we can retrieve the value written here when we open the file, but + * this approach is simpler. + */ + size = buf->size + WT_INTPACK64_MAXSIZE; + WT_RET(__wt_buf_extend(session, buf, size)); + p = (uint8_t *)buf->mem + buf->size; + WT_RET(__wt_vpack_uint(&p, 0, ++S2BT(session)->write_gen)); + buf->size = WT_PTRDIFF(p, buf->mem); + + /* + * Second, add space for the final file size as a packed value. We don't + * know how large it will be so skip the maximum required space. + */ + size = buf->size + WT_INTPACK64_MAXSIZE; + WT_RET(__wt_buf_extend(session, buf, size)); + file_size_offset = buf->size; + buf->size = size; + + /* 3a, copy the metadata length into the buffer. */ + len = strlen(ckpt->block_metadata); + size = buf->size + WT_INTPACK64_MAXSIZE; + WT_RET(__wt_buf_extend(session, buf, size)); + p = (uint8_t *)buf->mem + buf->size; + WT_RET(__wt_vpack_uint(&p, 0, (uint64_t)len)); + buf->size = WT_PTRDIFF(p, buf->mem); + + /* 3b, copy the metadata into the buffer. */ + size = buf->size + len; + WT_RET(__wt_buf_extend(session, buf, size)); + p = (uint8_t *)buf->mem + buf->size; + memcpy(p, ckpt->block_metadata, len); + buf->size = size; + + /* 4a, copy the checkpoint list length into the buffer. */ + len = strlen(ckpt->block_checkpoint); + size = buf->size + WT_INTPACK64_MAXSIZE; + WT_RET(__wt_buf_extend(session, buf, size)); + p = (uint8_t *)buf->mem + buf->size; + WT_RET(__wt_vpack_uint(&p, 0, (uint64_t)len)); + buf->size = WT_PTRDIFF(p, buf->mem); + + /* 4b, copy the checkpoint list into the buffer. */ + size = buf->size + len; + WT_RET(__wt_buf_extend(session, buf, size)); + p = (uint8_t *)buf->mem + buf->size; + memcpy(p, ckpt->block_checkpoint, len); + buf->size = size; + + /* + * 5a, copy the not-quite-right checkpoint information length into the + * buffer. + */ + len = ckpt->raw.size; + size = buf->size + WT_INTPACK64_MAXSIZE; + WT_RET(__wt_buf_extend(session, buf, size)); + p = (uint8_t *)buf->mem + buf->size; + WT_RET(__wt_vpack_uint(&p, 0, (uint64_t)len)); + buf->size = WT_PTRDIFF(p, buf->mem); + + /* + * 5b, copy the not-quite-right checkpoint information into the buffer. + */ + size = buf->size + len; + WT_RET(__wt_buf_extend(session, buf, size)); + p = (uint8_t *)buf->mem + buf->size; + memcpy(p, ckpt->raw.data, len); + buf->size = size; + + /* + * We might have grown the buffer beyond the original allocation size, + * make sure that we're still in compliance. + */ + align_size = WT_ALIGN(buf->size, block->allocsize); + if (align_size > buf->memsize) + WT_RET(__wt_buf_extend(session, buf, align_size)); + + *file_sizep = (uint8_t *)buf->mem + file_size_offset; + + return (0); +} + +struct saved_block_info { + uint64_t write_gen; + wt_off_t offset; + uint32_t size; + uint32_t checksum; + uint64_t file_size; + + char *metadata; + char *checkpoint_list; + + WT_ITEM *checkpoint; +}; + +/* + * __block_checkpoint_update -- + * Update the checkpoint information for the file. + */ +static int +__block_checkpoint_update( + WT_SESSION_IMPL *session, WT_BLOCK *block, struct saved_block_info *info) +{ + WT_BLOCK_CKPT ci; + WT_ITEM *checkpoint; + uint8_t *endp; + + memset(&ci, 0, sizeof(ci)); + checkpoint = info->checkpoint; + + if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT)) + __wt_ckpt_verbose( + session, block, "import original", NULL, checkpoint->mem); + + /* + * Convert the final checkpoint data blob to a WT_BLOCK_CKPT structure, + * update it with the avail list information, and convert it back to a + * data blob. + */ + WT_RET(__wt_block_buffer_to_ckpt( + session, block, checkpoint->data, &ci)); + ci.avail.offset = info->offset; + ci.avail.size = info->size; + ci.avail.checksum = info->checksum; + ci.file_size = (wt_off_t)info->file_size; + WT_RET(__wt_buf_extend( + session, checkpoint, WT_BLOCK_CHECKPOINT_BUFFER)); + endp = checkpoint->mem; + WT_RET(__wt_block_ckpt_to_buffer(session, block, &endp, &ci, false)); + checkpoint->size = WT_PTRDIFF(endp, checkpoint->mem); + + if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT)) + __wt_ckpt_verbose( + session, block, "import replace", NULL, checkpoint->mem); + + return (0); +} + +#define WT_BLOCK_SKIP(a) do { \ + if ((a) != 0) \ + continue; \ +} while (0) + +/* + * __wt_block_checkpoint_last -- + * Scan a file for checkpoints, returning the last one we find. + */ +int +__wt_block_checkpoint_last(WT_SESSION_IMPL *session, WT_BLOCK *block, + char **metadatap, char **checkpoint_listp, WT_ITEM *checkpoint) +{ + struct saved_block_info *best, _best, *current, _current, *saved_tmp; + WT_BLOCK_HEADER *blk; + WT_DECL_ITEM(tmp); + WT_DECL_RET; + WT_FH *fh; + const WT_PAGE_HEADER *dsk; + wt_off_t ext_off, ext_size, offset; + uint64_t len, nblocks, write_gen; + uint32_t checksum, size; + const uint8_t *p, *t; + bool found; + + *metadatap = *checkpoint_listp = NULL; + __wt_buf_init(session, checkpoint, WT_BLOCK_CHECKPOINT_BUFFER); + + /* + * Initialize a pair of structures that track the best and current + * checkpoints found so far. This is a little trickier than normal + * because we don't want to start saving a checkpoint only to find + * out it's not one we can use. I doubt that can happen and it + * suggests corruption, but half-a-checkpoint isn't a good place to + * be. Only swap to a new "best" checkpoint if we read the whole + * thing successfully. + * + * Don't re-order these lines: it's done this way so the WT_ITEMs + * are always initialized and error handling works. + */ + memset((best = &_best), 0, sizeof(_best)); + memset((current = &_current), 0, sizeof(_current)); + WT_ERR(__wt_scr_alloc(session, 0, &best->checkpoint)); + WT_ERR(__wt_scr_alloc(session, 0, ¤t->checkpoint)); + + found = false; + ext_off = 0; /* [-Werror=maybe-uninitialized] */ + ext_size = 0; + len = write_gen = 0; + + WT_ERR(__wt_scr_alloc(session, 64 * 1024, &tmp)); + + F_SET(session, WT_SESSION_QUIET_CORRUPT_FILE); + + /* + * Scan the file for pages, using the minimum possible WiredTiger + * allocation size. + */ + fh = block->fh; + for (nblocks = 0, offset = 0; offset < block->size; offset += size) { + /* Report progress occasionally. */ +#define WT_CHECKPOINT_LIST_PROGRESS_INTERVAL 100 + if (++nblocks % WT_CHECKPOINT_LIST_PROGRESS_INTERVAL == 0) + WT_ERR(__wt_progress(session, NULL, nblocks)); + + /* + * Read the start of a possible page and get a block length from + * it. Move to the next allocation sized boundary, we'll never + * consider this one again. + */ + if ((ret = __wt_read(session, fh, + offset, (size_t)WT_BTREE_MIN_ALLOC_SIZE, tmp->mem)) != 0) + break; + blk = WT_BLOCK_HEADER_REF(tmp->mem); + __wt_block_header_byteswap(blk); + size = blk->disk_size; + checksum = blk->checksum; + + /* + * Check the block size: if it's not insane, read the block. + * Reading the block validates any checksum. The file might + * reasonably have garbage at the end, and we're not here to + * detect that. Ignore problems, subsequent file verification + * can deal with any corruption. If the block isn't valid, + * skip to the next possible block. + */ + if (__wt_block_offset_invalid(block, offset, size) || + __wt_block_read_off( + session, block, tmp, offset, size, checksum) != 0) { + size = WT_BTREE_MIN_ALLOC_SIZE; + continue; + } + + dsk = tmp->mem; + if (dsk->type != WT_PAGE_BLOCK_MANAGER) + continue; + + p = WT_BLOCK_HEADER_BYTE(tmp->mem); + WT_BLOCK_SKIP(__wt_extlist_read_pair(&p, &ext_off, &ext_size)); + if (ext_off != WT_BLOCK_EXTLIST_MAGIC || ext_size != 0) + continue; + for (;;) { + if ((ret = __wt_extlist_read_pair( + &p, &ext_off, &ext_size)) != 0) + break; + if (ext_off == WT_BLOCK_INVALID_OFFSET) + break; + } + if (ret != 0) { + ret = 0; + continue; + } + /* + * Note the less-than check of WT_BLOCK_EXTLIST_VERSION_CKPT, + * that way we can extend this with additional values in the + * future. + */ + if (ext_size < WT_BLOCK_EXTLIST_VERSION_CKPT) + continue; + + /* + * Skip any entries that aren't the most recent we've seen so + * far. + */ + WT_BLOCK_SKIP(__wt_vunpack_uint(&p, 0, &write_gen)); + if (write_gen < best->write_gen) + continue; + + __wt_verbose(session, WT_VERB_CHECKPOINT, + "scan: checkpoint block at offset %" PRIuMAX + ", generation #%" PRIu64, + (uintmax_t)offset, write_gen); + + current->write_gen = write_gen; + current->offset = offset; + current->size = size; + current->checksum = checksum; + + /* + * The file size is in a fixed-size chunk of data, although it's + * packed (for portability). + */ + t = p; + WT_BLOCK_SKIP(__wt_vunpack_uint(&t, 0, ¤t->file_size)); + p += WT_INTPACK64_MAXSIZE; + + /* Save a copy of the metadata. */ + __wt_free(session, current->metadata); + WT_BLOCK_SKIP(__wt_vunpack_uint(&p, 0, &len)); + WT_ERR(__wt_strndup(session, p, len, ¤t->metadata)); + p += len; + + /* Save a copy of the checkpoint list. */ + __wt_free(session, current->checkpoint_list); + WT_BLOCK_SKIP(__wt_vunpack_uint(&p, 0, &len)); + WT_ERR(__wt_strndup( + session, p, len, ¤t->checkpoint_list)); + p += len; + + /* Save a copy of the checkpoint information. */ + WT_BLOCK_SKIP(__wt_vunpack_uint(&p, 0, &len)); + WT_ERR(__wt_buf_set(session, current->checkpoint, p, len)); + + /* A new winner, swap the "best" and "current" information. */ + saved_tmp = best; + best = current; + current = saved_tmp; + found = true; + } + + if (!found) + WT_ERR_MSG(session, WT_NOTFOUND, + "%s: no final checkpoint found in file scan", + block->name); + + /* Correct the checkpoint. */ + WT_ERR(__block_checkpoint_update(session, block, best)); + + /* + * Copy the information out to our caller. Do the WT_ITEM first, it's + * the only thing left that can fail and simplifies error handling. + */ + WT_ERR(__wt_buf_set(session, + checkpoint, best->checkpoint->data, best->checkpoint->size)); + *metadatap = best->metadata; + best->metadata = NULL; + *checkpoint_listp = best->checkpoint_list; + best->checkpoint_list = NULL; + +err: + __wt_free(session, best->metadata); + __wt_free(session, best->checkpoint_list); + __wt_scr_free(session, &best->checkpoint); + __wt_free(session, current->metadata); + __wt_free(session, current->checkpoint_list); + __wt_scr_free(session, ¤t->checkpoint); + + __wt_scr_free(session, &tmp); + + F_CLR(session, WT_SESSION_QUIET_CORRUPT_FILE); + return (ret); +} diff --git a/src/third_party/wiredtiger/src/block/block_ext.c b/src/third_party/wiredtiger/src/block/block_ext.c index 003f0489586..82e85658e22 100644 --- a/src/third_party/wiredtiger/src/block/block_ext.c +++ b/src/third_party/wiredtiger/src/block/block_ext.c @@ -1186,15 +1186,8 @@ __wt_block_extlist_read(WT_SESSION_IMPL *session, WT_ERR(__wt_block_read_off( session, block, tmp, el->offset, el->size, el->checksum)); -#define WT_EXTLIST_READ(p, v) do { \ - uint64_t _v; \ - WT_ERR(__wt_vunpack_uint(&(p), 0, &_v)); \ - (v) = (wt_off_t)_v; \ -} while (0) - p = WT_BLOCK_HEADER_BYTE(tmp->mem); - WT_EXTLIST_READ(p, off); - WT_EXTLIST_READ(p, size); + WT_ERR(__wt_extlist_read_pair(&p, &off, &size)); if (off != WT_BLOCK_EXTLIST_MAGIC || size != 0) goto corrupted; @@ -1210,8 +1203,7 @@ __wt_block_extlist_read(WT_SESSION_IMPL *session, */ func = el->track_size == 0 ? __block_append : __block_merge; for (;;) { - WT_EXTLIST_READ(p, off); - WT_EXTLIST_READ(p, size); + WT_ERR(__wt_extlist_read_pair(&p, &off, &size)); if (off == WT_BLOCK_INVALID_OFFSET) break; @@ -1263,10 +1255,11 @@ __wt_block_extlist_write(WT_SESSION_IMPL *session, /* * Figure out how many entries we're writing -- if there aren't any - * entries, we're done. + * entries, there's nothing to write, unless we still have to write + * the extent list to include the checkpoint recovery information. */ entries = el->entries + (additional == NULL ? 0 : additional->entries); - if (entries == 0) { + if (entries == 0 && block->final_ckpt == NULL) { el->offset = WT_BLOCK_INVALID_OFFSET; el->checksum = el->size = 0; return (0); @@ -1286,25 +1279,21 @@ __wt_block_extlist_write(WT_SESSION_IMPL *session, dsk = tmp->mem; memset(dsk, 0, WT_BLOCK_HEADER_BYTE_SIZE); dsk->type = WT_PAGE_BLOCK_MANAGER; - -#define WT_EXTLIST_WRITE(p, v) \ - WT_ERR(__wt_vpack_uint(&(p), 0, (uint64_t)(v))) + dsk->version = WT_PAGE_VERSION_TS; /* Fill the page's data. */ p = WT_BLOCK_HEADER_BYTE(dsk); - WT_EXTLIST_WRITE(p, WT_BLOCK_EXTLIST_MAGIC); /* Initial value */ - WT_EXTLIST_WRITE(p, 0); - WT_EXT_FOREACH(ext, el->off) { /* Free ranges */ - WT_EXTLIST_WRITE(p, ext->off); - WT_EXTLIST_WRITE(p, ext->size); - } + /* Extent list starts */ + WT_ERR(__wt_extlist_write_pair(&p, WT_BLOCK_EXTLIST_MAGIC, 0)); + WT_EXT_FOREACH(ext, el->off) /* Free ranges */ + WT_ERR(__wt_extlist_write_pair(&p, ext->off, ext->size)); if (additional != NULL) - WT_EXT_FOREACH(ext, additional->off) { /* Free ranges */ - WT_EXTLIST_WRITE(p, ext->off); - WT_EXTLIST_WRITE(p, ext->size); - } - WT_EXTLIST_WRITE(p, WT_BLOCK_INVALID_OFFSET); /* Ending value */ - WT_EXTLIST_WRITE(p, 0); + WT_EXT_FOREACH(ext, additional->off) /* Free ranges */ + WT_ERR( + __wt_extlist_write_pair(&p, ext->off, ext->size)); + /* Extent list stops */ + WT_ERR(__wt_extlist_write_pair(&p, WT_BLOCK_INVALID_OFFSET, + block->final_ckpt == NULL ? 0 : WT_BLOCK_EXTLIST_VERSION_CKPT)); dsk->u.datalen = WT_PTRDIFF32(p, WT_BLOCK_HEADER_BYTE(dsk)); tmp->size = dsk->mem_size = WT_PTRDIFF32(p, dsk); diff --git a/src/third_party/wiredtiger/src/block/block_mgr.c b/src/third_party/wiredtiger/src/block/block_mgr.c index 7211e5cfa24..80662ec6634 100644 --- a/src/third_party/wiredtiger/src/block/block_mgr.c +++ b/src/third_party/wiredtiger/src/block/block_mgr.c @@ -69,6 +69,18 @@ __bm_checkpoint(WT_BM *bm, WT_SESSION_IMPL *session, } /* + * __bm_checkpoint_last -- + * Return information for the last known file checkpoint. + */ +static int +__bm_checkpoint_last(WT_BM *bm, WT_SESSION_IMPL *session, + char **metadatap, char **checkpoint_listp, WT_ITEM *checkpoint) +{ + return (__wt_block_checkpoint_last( + session, bm->block, metadatap, checkpoint_listp, checkpoint)); +} + +/* * __bm_checkpoint_readonly -- * Write a buffer into a block, creating a checkpoint; readonly version. */ @@ -562,6 +574,7 @@ __bm_method_set(WT_BM *bm, bool readonly) bm->addr_string = __bm_addr_string; bm->block_header = __bm_block_header; bm->checkpoint = __bm_checkpoint; + bm->checkpoint_last = __bm_checkpoint_last; bm->checkpoint_load = __bm_checkpoint_load; bm->checkpoint_resolve = __bm_checkpoint_resolve; bm->checkpoint_start = __bm_checkpoint_start; diff --git a/src/third_party/wiredtiger/src/block/block_open.c b/src/third_party/wiredtiger/src/block/block_open.c index 68f7224ff76..9e4689d7b7e 100644 --- a/src/third_party/wiredtiger/src/block/block_open.c +++ b/src/third_party/wiredtiger/src/block/block_open.c @@ -8,7 +8,7 @@ #include "wt_internal.h" -static int __desc_read(WT_SESSION_IMPL *, WT_BLOCK *); +static int __desc_read(WT_SESSION_IMPL *, uint32_t allocsize, WT_BLOCK *); /* * __wt_block_manager_drop -- @@ -228,7 +228,7 @@ __wt_block_open(WT_SESSION_IMPL *session, * look at anything, including the description information. */ if (!forced_salvage) - WT_ERR(__desc_read(session, block)); + WT_ERR(__desc_read(session, allocsize, block)); *blockp = block; __wt_spin_unlock(session, &conn->block_lock); @@ -314,7 +314,7 @@ __wt_desc_write(WT_SESSION_IMPL *session, WT_FH *fh, uint32_t allocsize) * Read and verify the file's metadata. */ static int -__desc_read(WT_SESSION_IMPL *session, WT_BLOCK *block) +__desc_read(WT_SESSION_IMPL *session, uint32_t allocsize, WT_BLOCK *block) { WT_BLOCK_DESC *desc; WT_DECL_ITEM(buf); @@ -326,11 +326,11 @@ __desc_read(WT_SESSION_IMPL *session, WT_BLOCK *block) return (0); /* Use a scratch buffer to get correct alignment for direct I/O. */ - WT_RET(__wt_scr_alloc(session, block->allocsize, &buf)); + WT_RET(__wt_scr_alloc(session, allocsize, &buf)); /* Read the first allocation-sized block and verify the file format. */ WT_ERR(__wt_read(session, - block->fh, (wt_off_t)0, (size_t)block->allocsize, buf->mem)); + block->fh, (wt_off_t)0, (size_t)allocsize, buf->mem)); /* * Handle little- and big-endian objects. Objects are written in little- @@ -342,7 +342,7 @@ __desc_read(WT_SESSION_IMPL *session, WT_BLOCK *block) desc = buf->mem; checksum_tmp = desc->checksum; desc->checksum = 0; - checksum_calculate = __wt_checksum(desc, block->allocsize); + checksum_calculate = __wt_checksum(desc, allocsize); desc->checksum = checksum_tmp; __wt_block_desc_byteswap(desc); diff --git a/src/third_party/wiredtiger/src/block/block_write.c b/src/third_party/wiredtiger/src/block/block_write.c index 55f9d4ca57c..b678c148668 100644 --- a/src/third_party/wiredtiger/src/block/block_write.c +++ b/src/third_party/wiredtiger/src/block/block_write.c @@ -232,6 +232,7 @@ __block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, wt_off_t offset; size_t align_size; uint32_t checksum; + uint8_t *file_sizep; bool local_locked; *offsetp = 0; /* -Werror=maybe-uninitialized */ @@ -240,13 +241,6 @@ __block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, fh = block->fh; - /* - * Clear the block header to ensure all of it is initialized, even the - * unused fields. - */ - blk = WT_BLOCK_HEADER_REF(buf->mem); - memset(blk, 0, sizeof(*blk)); - /* Buffers should be aligned for writing. */ if (!F_ISSET(buf, WT_ITEM_ALIGNED)) { WT_ASSERT(session, F_ISSET(buf, WT_ITEM_ALIGNED)); @@ -255,6 +249,14 @@ __block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, } /* + * File checkpoint/recovery magic: done before sizing the buffer as it + * may grow the buffer. + */ + if (block->final_ckpt != NULL) + WT_RET(__wt_block_checkpoint_final( + session, block, buf, &file_sizep)); + + /* * Align the size to an allocation unit. * * The buffer must be big enough for us to zero to the next allocsize @@ -273,10 +275,46 @@ __block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, "buffer size check: write buffer too large to write"); } + /* Pre-allocate some number of extension structures. */ + WT_RET(__wt_block_ext_prealloc(session, 5)); + + /* + * Acquire a lock, if we don't already hold one. + * Allocate space for the write, and optionally extend the file (note + * the block-extend function may release the lock). + * Release any locally acquired lock. + */ + local_locked = false; + if (!caller_locked) { + __wt_spin_lock(session, &block->live_lock); + local_locked = true; + } + ret = __wt_block_alloc(session, block, &offset, (wt_off_t)align_size); + if (ret == 0) + ret = __wt_block_extend( + session, block, fh, offset, align_size, &local_locked); + if (local_locked) + __wt_spin_unlock(session, &block->live_lock); + WT_RET(ret); + + /* + * The file has finished changing size. If this is the final write in a + * checkpoint, update the checkpoint's information inline. + */ + if (block->final_ckpt != NULL) + WT_RET(__wt_vpack_uint(&file_sizep, 0, (uint64_t)block->size)); + /* Zero out any unused bytes at the end of the buffer. */ memset((uint8_t *)buf->mem + buf->size, 0, align_size - buf->size); /* + * Clear the block header to ensure all of it is initialized, even the + * unused fields. + */ + blk = WT_BLOCK_HEADER_REF(buf->mem); + memset(blk, 0, sizeof(*blk)); + + /* * Set the disk size so we don't have to incrementally read blocks * during salvage. */ @@ -310,28 +348,6 @@ __block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, blk->checksum = __wt_bswap32(blk->checksum); #endif - /* Pre-allocate some number of extension structures. */ - WT_RET(__wt_block_ext_prealloc(session, 5)); - - /* - * Acquire a lock, if we don't already hold one. - * Allocate space for the write, and optionally extend the file (note - * the block-extend function may release the lock). - * Release any locally acquired lock. - */ - local_locked = false; - if (!caller_locked) { - __wt_spin_lock(session, &block->live_lock); - local_locked = true; - } - ret = __wt_block_alloc(session, block, &offset, (wt_off_t)align_size); - if (ret == 0) - ret = __wt_block_extend( - session, block, fh, offset, align_size, &local_locked); - if (local_locked) - __wt_spin_unlock(session, &block->live_lock); - WT_RET(ret); - /* Write the block. */ if ((ret = __wt_write(session, fh, offset, align_size, buf->mem)) != 0) { diff --git a/src/third_party/wiredtiger/src/btree/bt_cursor.c b/src/third_party/wiredtiger/src/btree/bt_cursor.c index e75432f7836..b7176c74adf 100644 --- a/src/third_party/wiredtiger/src/btree/bt_cursor.c +++ b/src/third_party/wiredtiger/src/btree/bt_cursor.c @@ -1438,12 +1438,13 @@ __cursor_chain_exceeded(WT_CURSOR_BTREE *cbt) upd != NULL && upd->type == WT_UPDATE_MODIFY; ++i, upd = upd->next) { upd_size += WT_UPDATE_MEMSIZE(upd); - if (upd_size >= WT_MODIFY_MEM_FACTOR * cursor->value.size) + if (i >= WT_MAX_MODIFY_UPDATE && + upd_size * WT_MODIFY_MEM_FRACTION >= cursor->value.size) return (true); } - if (upd != NULL && upd->type == WT_UPDATE_STANDARD && - __wt_txn_upd_visible_all(session, upd) && - i >= WT_MAX_MODIFY_UPDATE) + if (i >= WT_MAX_MODIFY_UPDATE && upd != NULL && + upd->type == WT_UPDATE_STANDARD && + __wt_txn_upd_visible_all(session, upd)) return (true); return (false); } diff --git a/src/third_party/wiredtiger/src/btree/bt_handle.c b/src/third_party/wiredtiger/src/btree/bt_handle.c index f179fa38d91..168a0152f17 100644 --- a/src/third_party/wiredtiger/src/btree/bt_handle.c +++ b/src/third_party/wiredtiger/src/btree/bt_handle.c @@ -93,6 +93,7 @@ __wt_btree_open(WT_SESSION_IMPL *session, const char *op_cfg[]) WT_CKPT ckpt; WT_CONFIG_ITEM cval; WT_DATA_HANDLE *dhandle; + WT_DECL_ITEM(tmp); WT_DECL_RET; size_t root_addr_size; uint8_t root_addr[WT_BTREE_MAX_ADDR_COOKIE]; @@ -239,6 +240,7 @@ err: WT_TRET(__wt_btree_close(session)); } __wt_meta_checkpoint_free(session, &ckpt); + __wt_scr_free(session, &tmp); return (ret); } @@ -324,6 +326,44 @@ __wt_btree_discard(WT_SESSION_IMPL *session) } /* + * __wt_btree_config_encryptor -- + * Return an encryptor handle based on the configuration. + */ +int +__wt_btree_config_encryptor(WT_SESSION_IMPL *session, + const char **cfg, WT_KEYED_ENCRYPTOR **kencryptorp) +{ + WT_CONFIG_ITEM cval, enc, keyid; + WT_DECL_RET; + const char *enc_cfg[] = { NULL, NULL }; + + /* + * We do not use __wt_config_gets_none here because "none" and the empty + * string have different meanings. The empty string means inherit the + * system encryption setting and "none" means this table is in the clear + * even if the database is encrypted. + */ + WT_RET(__wt_config_gets(session, cfg, "encryption.name", &cval)); + if (cval.len == 0) + *kencryptorp = S2C(session)->kencryptor; + else if (WT_STRING_MATCH("none", cval.str, cval.len)) + *kencryptorp = NULL; + else { + WT_RET(__wt_config_gets_none( + session, cfg, "encryption.keyid", &keyid)); + WT_RET(__wt_config_gets(session, cfg, "encryption", &enc)); + if (enc.len != 0) + WT_RET(__wt_strndup(session, enc.str, enc.len, + &enc_cfg[0])); + ret = __wt_encryptor_config(session, &cval, &keyid, + (WT_CONFIG_ARG *)enc_cfg, kencryptorp); + __wt_free(session, enc_cfg[0]); + WT_RET(ret); + } + return (0); +} + +/* * __btree_conf -- * Configure a WT_BTREE structure. */ @@ -331,12 +371,11 @@ static int __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt) { WT_BTREE *btree; - WT_CONFIG_ITEM cval, enc, keyid, metadata; + WT_CONFIG_ITEM cval, metadata; WT_CONNECTION_IMPL *conn; - WT_DECL_RET; int64_t maj_version, min_version; uint32_t bitcnt; - const char **cfg, *enc_cfg[] = { NULL, NULL }; + const char **cfg; bool fixed; btree = S2BT(session); @@ -548,29 +587,8 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt) } } - /* - * We do not use __wt_config_gets_none here because "none" and the empty - * string have different meanings. The empty string means inherit the - * system encryption setting and "none" means this table is in the clear - * even if the database is encrypted. - */ - WT_RET(__wt_config_gets(session, cfg, "encryption.name", &cval)); - if (cval.len == 0) - btree->kencryptor = conn->kencryptor; - else if (WT_STRING_MATCH("none", cval.str, cval.len)) - btree->kencryptor = NULL; - else { - WT_RET(__wt_config_gets_none( - session, cfg, "encryption.keyid", &keyid)); - WT_RET(__wt_config_gets(session, cfg, "encryption", &enc)); - if (enc.len != 0) - WT_RET(__wt_strndup(session, enc.str, enc.len, - &enc_cfg[0])); - ret = __wt_encryptor_config(session, &cval, &keyid, - (WT_CONFIG_ARG *)enc_cfg, &btree->kencryptor); - __wt_free(session, enc_cfg[0]); - WT_RET(ret); - } + /* Configure encryption. */ + WT_RET(__wt_btree_config_encryptor(session, cfg, &btree->kencryptor)); /* Initialize locks. */ WT_RET(__wt_rwlock_init(session, &btree->ovfl_lock)); diff --git a/src/third_party/wiredtiger/src/btree/bt_import.c b/src/third_party/wiredtiger/src/btree/bt_import.c new file mode 100644 index 00000000000..69dcda8c1c5 --- /dev/null +++ b/src/third_party/wiredtiger/src/btree/bt_import.c @@ -0,0 +1,169 @@ +/*- + * Copyright (c) 2014-2019 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_import -- + * Import a WiredTiger file into the database. + */ +int +__wt_import(WT_SESSION_IMPL *session, const char *uri) +{ + WT_BM *bm; + WT_CKPT *ckpt, *ckptbase; + WT_CONFIG_ITEM v; + WT_DECL_ITEM(a); + WT_DECL_ITEM(b); + WT_DECL_ITEM(checkpoint); + WT_DECL_RET; + WT_KEYED_ENCRYPTOR *kencryptor; + const char *filename; + const char *filecfg[] = { + WT_CONFIG_BASE(session, file_meta), NULL, NULL, NULL, NULL, NULL }; + char *checkpoint_list, *fileconf, *metadata, fileid[64]; + + ckptbase = NULL; + checkpoint_list = fileconf = metadata = NULL; + + WT_ERR(__wt_scr_alloc(session, 0, &a)); + WT_ERR(__wt_scr_alloc(session, 0, &b)); + WT_ERR(__wt_scr_alloc(session, 0, &checkpoint)); + + WT_ASSERT(session, WT_PREFIX_MATCH(uri, "file:")); + filename = uri; + WT_PREFIX_SKIP(filename, "file:"); + + /* + * Open the file, request block manager checkpoint information. + * We don't know the allocation size, but 512B allows us to read + * the descriptor block and that's all we care about. + */ + WT_ERR(__wt_block_manager_open( + session, filename, filecfg, false, true, 512, &bm)); + ret = bm->checkpoint_last( + bm, session, &metadata, &checkpoint_list, checkpoint); + WT_TRET(bm->close(bm, session)); + WT_ERR(ret); + __wt_verbose(session, + WT_VERB_CHECKPOINT, "import metadata: %s", metadata); + __wt_verbose(session, + WT_VERB_CHECKPOINT, "import checkpoint-list: %s", checkpoint_list); + + /* + * The metadata may have been encrypted, in which case it's also + * hexadecimal encoded. The checkpoint included a boolean value + * set if the metadata was encrypted for easier failure diagnosis. + */ + WT_ERR(__wt_config_getones( + session, metadata, "block_metadata_encrypted", &v)); + WT_ERR(__wt_btree_config_encryptor(session, filecfg, &kencryptor)); + if ((kencryptor == NULL && v.val != 0) || + (kencryptor != NULL && v.val == 0)) + WT_ERR_MSG(session, EINVAL, + "%s: loaded object's encryption configuration doesn't " + "match the database's encryption configuration", + filename); + /* + * The metadata was quoted to avoid configuration string characters + * acting as separators. Discard any quote characters. + */ + WT_ERR(__wt_config_getones(session, metadata, "block_metadata", &v)); + if (v.len > 0 && (v.str[0] == '[' || v.str[0] == '(')) { + ++v.str; + v.len -= 2; + } + if (kencryptor == NULL) { + WT_ERR(__wt_buf_grow(session, a, v.len + 1)); + WT_ERR(__wt_buf_set(session, a, v.str, v.len)); + ((uint8_t *)a->data)[a->size] = '\0'; + } else { + WT_ERR(__wt_buf_grow(session, b, v.len)); + WT_ERR(__wt_nhex_to_raw(session, v.str, v.len, b)); + WT_ERR(__wt_buf_grow(session, a, b->size + 1)); + WT_ERR(__wt_decrypt(session, kencryptor->encryptor, 0, b, a)); + ((uint8_t *)a->data)[a->size] = '\0'; + } + + /* + * OK, we've now got three chunks of data: the file's metadata from when + * the last checkpoint started, the array of checkpoints as of when the + * last checkpoint was almost complete (everything written but the avail + * list), and fixed-up checkpoint information from the last checkpoint. + * + * Build and flatten the metadata and the checkpoint list, then insert + * it into the metadata for this file. + * + * Strip out the checkpoint-LSN, an imported file isn't associated + * with any log files. + * Assign a unique file ID. + */ + filecfg[1] = a->data; + filecfg[2] = checkpoint_list; + filecfg[3] = "checkpoint_lsn="; + WT_WITH_SCHEMA_LOCK(session, ret = + __wt_snprintf(fileid, sizeof(fileid), + "id=%" PRIu32, ++S2C(session)->next_file_id)); + WT_ERR(ret); + filecfg[4] = fileid; + WT_ERR(__wt_config_collapse(session, filecfg, &fileconf)); + WT_ERR(__wt_metadata_insert(session, uri, fileconf)); + __wt_verbose(session, + WT_VERB_CHECKPOINT, "import configuration: %s/%s", uri, fileconf); + + /* + * The just inserted metadata was correct as of immediately before the + * before the final checkpoint, but it's not quite right. The block + * manager returned the corrected final checkpoint, put it all together. + * + * Get the checkpoint information from the file's metadata as an array + * of WT_CKPT structures. + * + * XXX + * There's a problem here. If a file is imported from our future (leaf + * pages with unstable entries that have write-generations ahead of the + * current database's base write generation), we'll read the values and + * treat them as stable. A restart will fix this: when we added the + * imported file to our metadata, the write generation in the imported + * file's checkpoints updated our database's maximum write generation, + * and so a restart will have a maximum generation newer than the + * imported file's write generation. An alternative solution is to add + * a "base write generation" value to the imported file's metadata, and + * use that value instead of the connection's base write generation when + * deciding what page items should be read. Since all future writes to + * the imported file would be ahead of that write generation, it would + * have the effect we want. + * + * Update the last checkpoint with the corrected information. + * Update the file's metadata with the new checkpoint information. + */ + WT_ERR(__wt_meta_ckptlist_get(session, uri, false, &ckptbase)); + WT_CKPT_FOREACH(ckptbase, ckpt) + if (ckpt->name == NULL || (ckpt + 1)->name == NULL) + break; + if (ckpt->name == NULL) + WT_ERR_MSG(session, EINVAL, + "no checkpoint information available to import"); + F_SET(ckpt, WT_CKPT_UPDATE); + WT_ERR(__wt_buf_set( + session, &ckpt->raw, checkpoint->data, checkpoint->size)); + WT_ERR(__wt_meta_ckptlist_set(session, uri, ckptbase, NULL)); + +err: + __wt_meta_ckptlist_free(session, &ckptbase); + + __wt_free(session, fileconf); + __wt_free(session, metadata); + __wt_free(session, checkpoint_list); + + __wt_scr_free(session, &a); + __wt_scr_free(session, &b); + __wt_scr_free(session, &checkpoint); + + return (ret); +} diff --git a/src/third_party/wiredtiger/src/btree/bt_slvg.c b/src/third_party/wiredtiger/src/btree/bt_slvg.c index d2b0472a11e..3063af2de0a 100644 --- a/src/third_party/wiredtiger/src/btree/bt_slvg.c +++ b/src/third_party/wiredtiger/src/btree/bt_slvg.c @@ -157,11 +157,88 @@ static int __slvg_trk_ovfl(WT_SESSION_IMPL *, const WT_PAGE_HEADER *, uint8_t *, size_t, WT_STUFF *); /* - * __wt_bt_salvage -- + * __slvg_checkpoint -- + * Create the post-salvage checkpoint. + */ +static int +__slvg_checkpoint(WT_SESSION_IMPL *session, WT_REF *root) +{ + WT_BTREE *btree; + WT_CKPT *ckptbase; + WT_DATA_HANDLE *dhandle; + WT_DECL_RET; + char *config; + + btree = S2BT(session); + ckptbase = NULL; + dhandle = session->dhandle; + config = NULL; + + /* + * XXX + * The salvage process reads and discards previous checkpoints, so the + * underlying block manager has to ignore any previous checkpoint + * entries when creating a new checkpoint. In other words, we can't use + * the metadata checkpoint list, it lists the previous checkpoints and + * we don't care about them. Build a clean checkpoint list and use it + * instead. + * + * Don't first clear the metadata checkpoint list and call the function + * to get a list of checkpoints: a crash between clearing the metadata + * checkpoint list and creating a new checkpoint list would look like a + * create or open of a file without a checkpoint to roll-forward from, + * and the contents of the file would be discarded. + */ + WT_RET(__wt_calloc_def(session, 2, &ckptbase)); + WT_ERR(__wt_strdup(session, WT_CHECKPOINT, &ckptbase->name)); + ckptbase->order = 1; + __wt_seconds(session, &ckptbase->sec); + WT_ERR(__wt_metadata_search(session, dhandle->name, &config)); + WT_ERR(__wt_meta_block_metadata(session, config, ckptbase)); + ckptbase->newest_durable_ts = WT_TS_NONE; + ckptbase->oldest_start_ts = WT_TS_NONE; + ckptbase->oldest_start_txn = WT_TXN_NONE; + ckptbase->newest_stop_ts = WT_TS_MAX; + ckptbase->newest_stop_txn = WT_TXN_MAX; + F_SET(ckptbase, WT_CKPT_ADD); + + /* + * We may not have found any pages during salvage and there's no tree + * to flush. + */ + if (root->page != NULL) { + btree->ckpt = ckptbase; + ret = __wt_evict( + session, root, WT_REF_MEM, WT_EVICT_CALL_CLOSING); + root->page = NULL; + btree->ckpt = NULL; + WT_ERR(ret); + } + + /* + * If no checkpoint was created, clear all recorded checkpoints for the + * file. This is expected if we didn't find any leaf pages to salvage. + * + * If a checkpoint was created, life is good, replace any existing list + * of checkpoints with the single new one. + */ + if (ckptbase->raw.data == NULL) + WT_TRET(__wt_meta_checkpoint_clear(session, dhandle->name)); + else + WT_ERR(__wt_meta_ckptlist_set( + session, dhandle->name, ckptbase, NULL)); + +err: __wt_meta_ckptlist_free(session, &ckptbase); + __wt_free(session, config); + return (ret); +} + +/* + * __wt_salvage -- * Salvage a Btree. */ int -__wt_bt_salvage(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, const char *cfg[]) +__wt_salvage(WT_SESSION_IMPL *session, const char *cfg[]) { WT_BM *bm; WT_BTREE *btree; @@ -324,15 +401,9 @@ __wt_bt_salvage(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, const char *cfg[]) /* * Step 9: - * Evict the newly created root page, creating a checkpoint. + * Evict any newly created root page, creating a checkpoint. */ - if (ss->root_ref.page != NULL) { - btree->ckpt = ckptbase; - ret = __wt_evict(session, &ss->root_ref, WT_REF_MEM, - WT_EVICT_CALL_CLOSING); - ss->root_ref.page = NULL; - btree->ckpt = NULL; - } + WT_ERR(__slvg_checkpoint(session, &ss->root_ref)); /* * Step 10: diff --git a/src/third_party/wiredtiger/src/btree/bt_split.c b/src/third_party/wiredtiger/src/btree/bt_split.c index b1d8dce75a5..8bd58c17975 100644 --- a/src/third_party/wiredtiger/src/btree/bt_split.c +++ b/src/third_party/wiredtiger/src/btree/bt_split.c @@ -1412,6 +1412,25 @@ err: if (parent != NULL) return (0); } +#ifdef HAVE_DIAGNOSTIC +/* + * __check_upd_list -- + * Sanity check an update list. + * In particular, make sure there no birthmarks. + */ +static void +__check_upd_list(WT_SESSION_IMPL *session, WT_UPDATE *upd) +{ + int birthmark_count; + + for (birthmark_count = 0; upd != NULL; upd = upd->next) + if (upd->type == WT_UPDATE_BIRTHMARK) + ++birthmark_count; + + WT_ASSERT(session, birthmark_count <= 1); +} +#endif + /* * __split_multi_inmem -- * Instantiate a page from a disk image. @@ -1511,6 +1530,10 @@ __split_multi_inmem( key->size = WT_INSERT_KEY_SIZE(supd->ins); } +#ifdef HAVE_DIAGNOSTIC + __check_upd_list(session, upd); +#endif + /* Search the page. */ WT_ERR(__wt_row_search( session, key, ref, &cbt, true, true)); @@ -1817,9 +1840,11 @@ __split_insert(WT_SESSION_IMPL *session, WT_REF *ref) WT_SKIP_FIRST(WT_ROW_INSERT_SMALLEST(page))) != NULL) { key->data = WT_INSERT_KEY(ins); key->size = WT_INSERT_KEY_SIZE(ins); - } else + } else { + WT_ASSERT(session, page->entries > 0); WT_ERR(__wt_row_leaf_key( session, page, &page->pg_row[0], key, true)); + } WT_ERR(__wt_row_ikey(session, 0, key->data, key->size, child)); parent_incr += sizeof(WT_IKEY) + key->size; __wt_scr_free(session, &key); diff --git a/src/third_party/wiredtiger/src/btree/bt_vrfy.c b/src/third_party/wiredtiger/src/btree/bt_vrfy.c index 9078e74674b..6a07ba5aa55 100644 --- a/src/third_party/wiredtiger/src/btree/bt_vrfy.c +++ b/src/third_party/wiredtiger/src/btree/bt_vrfy.c @@ -162,11 +162,13 @@ __wt_verify(WT_SESSION_IMPL *session, const char *cfg[]) WT_VSTUFF *vs, _vstuff; size_t root_addr_size; uint8_t root_addr[WT_BTREE_MAX_ADDR_COOKIE]; + const char *name; bool bm_start, quit; btree = S2BT(session); bm = btree->bm; ckptbase = NULL; + name = session->dhandle->name; bm_start = false; WT_CLEAR(_vstuff); @@ -186,9 +188,16 @@ __wt_verify(WT_SESSION_IMPL *session, const char *cfg[]) if (quit) goto done; - /* Get a list of the checkpoints for this file. */ - WT_ERR( - __wt_meta_ckptlist_get(session, btree->dhandle->name, &ckptbase)); + /* + * Get a list of the checkpoints for this file. Empty objects have no + * checkpoints, in which case there's no work to do. + */ + ret = __wt_meta_ckptlist_get(session, name, false, &ckptbase); + if (ret == WT_NOTFOUND) { + ret = 0; + goto done; + } + WT_ERR(ret); /* Inform the underlying block manager we're verifying. */ WT_ERR(bm->verify_start(bm, session, ckptbase, cfg)); @@ -197,7 +206,7 @@ __wt_verify(WT_SESSION_IMPL *session, const char *cfg[]) /* Loop through the file's checkpoints, verifying each one. */ WT_CKPT_FOREACH(ckptbase, ckpt) { __wt_verbose(session, WT_VERB_VERIFY, - "%s: checkpoint %s", btree->dhandle->name, ckpt->name); + "%s: checkpoint %s", name, ckpt->name); /* Fake checkpoints require no work. */ if (F_ISSET(ckpt, WT_CKPT_FAKE)) @@ -209,7 +218,7 @@ __wt_verify(WT_SESSION_IMPL *session, const char *cfg[]) if (WT_VRFY_DUMP(vs)) { WT_ERR(__wt_msg(session, "%s", WT_DIVIDER)); WT_ERR(__wt_msg(session, "%s: checkpoint %s", - btree->dhandle->name, ckpt->name)); + name, ckpt->name)); } /* Load the checkpoint. */ diff --git a/src/third_party/wiredtiger/src/cache/cache_las.c b/src/third_party/wiredtiger/src/cache/cache_las.c index 87d55c1ac5e..d743f4f2eeb 100644 --- a/src/third_party/wiredtiger/src/cache/cache_las.c +++ b/src/third_party/wiredtiger/src/cache/cache_las.c @@ -660,7 +660,7 @@ __wt_las_insert_block(WT_CURSOR *cursor, WT_SAVE_UPD *list; WT_SESSION_IMPL *session; WT_TXN_ISOLATION saved_isolation; - WT_UPDATE *upd; + WT_UPDATE *first_upd, *upd; wt_off_t las_size; uint64_t insert_cnt, las_counter, las_pageid, max_las_size; uint64_t prepared_insert_cnt; @@ -741,7 +741,7 @@ __wt_las_insert_block(WT_CURSOR *cursor, slot = page->type == WT_PAGE_ROW_LEAF ? WT_ROW_SLOT(page, list->ripcip) : WT_COL_SLOT(page, list->ripcip); - upd = list->ins == NULL ? + first_upd = upd = list->ins == NULL ? page->modify->mod_row_update[slot] : list->ins->upd; /* @@ -760,6 +760,9 @@ __wt_las_insert_block(WT_CURSOR *cursor, las_value.size = upd->size; break; case WT_UPDATE_BIRTHMARK: + WT_ASSERT(session, upd != first_upd || + multi->page_las.skew_newest); + /* FALLTHROUGH */ case WT_UPDATE_TOMBSTONE: las_value.size = 0; break; @@ -780,6 +783,8 @@ __wt_las_insert_block(WT_CURSOR *cursor, (upd->type == WT_UPDATE_STANDARD || upd->type == WT_UPDATE_MODIFY)) { las_value.size = 0; + WT_ASSERT(session, upd != first_upd || + multi->page_las.skew_newest); cursor->set_value(cursor, upd->txnid, upd->start_ts, upd->durable_ts, upd->prepare_state, WT_UPDATE_BIRTHMARK, @@ -835,6 +840,7 @@ err: /* Resolve the transaction. */ __las_insert_block_verbose(session, btree, multi); } + WT_UNUSED(first_upd); return (ret); } diff --git a/src/third_party/wiredtiger/src/config/config_collapse.c b/src/third_party/wiredtiger/src/config/config_collapse.c index 0c4dfe7c604..3c21e0e224f 100644 --- a/src/third_party/wiredtiger/src/config/config_collapse.c +++ b/src/third_party/wiredtiger/src/config/config_collapse.c @@ -82,6 +82,103 @@ err: __wt_scr_free(session, &tmp); } /* + * __wt_config_discard_defaults -- + * Copy non-default configuration strings into newly allocated memory. + * + * This function strips out entries from a configuration string that aren't the + * default values. It takes a NULL-terminated list of configuration strings, the + * defaults, and a configuration string, and copies into allocated memory the + * strings from the configuration string that aren't the same as the defaults. + */ +int +__wt_config_discard_defaults(WT_SESSION_IMPL *session, + const char **cfg, const char *config, char **config_ret) +{ + WT_CONFIG cparser; + WT_CONFIG_ITEM k, v, vtmp; + WT_DECL_ITEM(tmp); + WT_DECL_RET; + + *config_ret = NULL; + + WT_RET(__wt_scr_alloc(session, 0, &tmp)); + + /* + * Walk the configuration string, search the default configuration for + * each entry, and discard any matches. + */ + __wt_config_init(session, &cparser, config); + while ((ret = __wt_config_next(&cparser, &k, &v)) == 0) { + if (k.type != WT_CONFIG_ITEM_STRING && + k.type != WT_CONFIG_ITEM_ID) + WT_ERR_MSG(session, EINVAL, + "Invalid configuration key found: '%s'", k.str); + + /* + * Get the default value. There isn't a default value in some + * cases, so not finding one isn't an error. + */ + if ((ret = + __wt_config_get(session, cfg, &k, &vtmp)) == WT_NOTFOUND) + goto keep; + WT_ERR(ret); + + /* + * Skip exact matches and simple things we can catch like "none" + * and an empty string, "true" and 1, "false" and 0. + */ + if (v.type == WT_CONFIG_ITEM_STRUCT && + vtmp.type == WT_CONFIG_ITEM_STRUCT && + v.len == vtmp.len && memcmp(v.str, vtmp.str, v.len) == 0) + continue; + if ((v.type == WT_CONFIG_ITEM_BOOL || + v.type == WT_CONFIG_ITEM_NUM) && + (vtmp.type == WT_CONFIG_ITEM_BOOL || + vtmp.type == WT_CONFIG_ITEM_NUM) && v.val == vtmp.val) + continue; + if ((v.type == WT_CONFIG_ITEM_ID || + v.type == WT_CONFIG_ITEM_STRING) && + (vtmp.type == WT_CONFIG_ITEM_ID || + vtmp.type == WT_CONFIG_ITEM_STRING) && + v.len == vtmp.len && memcmp(v.str, vtmp.str, v.len) == 0) + continue; + if (vtmp.len == 0 && v.len == strlen("none") && + WT_STRING_MATCH("none", v.str, v.len)) + continue; + + /* Include the quotes around string keys/values. */ +keep: if (k.type == WT_CONFIG_ITEM_STRING) { + --k.str; + k.len += 2; + } + if (v.type == WT_CONFIG_ITEM_STRING) { + --v.str; + v.len += 2; + } + WT_ERR(__wt_buf_catfmt(session, tmp, "%.*s=%.*s,", + (int)k.len, k.str, (int)v.len, v.str)); + } + + /* We loop until error, and the expected error is WT_NOTFOUND. */ + if (ret != WT_NOTFOUND) + goto err; + + /* + * If the caller passes us only default configuration strings, we get + * here with no bytes to copy -- that's OK, the underlying string copy + * can handle empty strings. + * + * Strip any trailing comma. + */ + if (tmp->size != 0) + --tmp->size; + ret = __wt_strndup(session, tmp->data, tmp->size, config_ret); + +err: __wt_scr_free(session, &tmp); + return (ret); +} + +/* * We need a character that can't appear in a key as a separator. */ #undef SEP /* separator key, character */ diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 11b41b07787..a8152f08d16 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -84,6 +84,7 @@ static const WT_CONFIG_CHECK { "checkpoint_retention", "int", NULL, "min=0,max=1024", NULL, 0 }, + { "eviction", "boolean", NULL, NULL, NULL, 0 }, { "rollback_error", "int", NULL, "min=0,max=10M", NULL, 0 }, { "table_logging", "boolean", NULL, NULL, NULL, 0 }, { NULL, NULL, NULL, NULL, NULL, 0 } @@ -177,7 +178,7 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { confchk_WT_CONNECTION_reconfigure_compatibility_subconfigs, 1 }, { "debug_mode", "category", NULL, NULL, - confchk_wiredtiger_open_debug_mode_subconfigs, 3 }, + confchk_wiredtiger_open_debug_mode_subconfigs, 4 }, { "error_prefix", "string", NULL, NULL, NULL, 0 }, { "eviction", "category", NULL, NULL, @@ -894,7 +895,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { { "create", "boolean", NULL, NULL, NULL, 0 }, { "debug_mode", "category", NULL, NULL, - confchk_wiredtiger_open_debug_mode_subconfigs, 3 }, + confchk_wiredtiger_open_debug_mode_subconfigs, 4 }, { "direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0 }, @@ -1009,7 +1010,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { { "create", "boolean", NULL, NULL, NULL, 0 }, { "debug_mode", "category", NULL, NULL, - confchk_wiredtiger_open_debug_mode_subconfigs, 3 }, + confchk_wiredtiger_open_debug_mode_subconfigs, 4 }, { "direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0 }, @@ -1123,7 +1124,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { confchk_wiredtiger_open_compatibility_subconfigs, 3 }, { "debug_mode", "category", NULL, NULL, - confchk_wiredtiger_open_debug_mode_subconfigs, 3 }, + confchk_wiredtiger_open_debug_mode_subconfigs, 4 }, { "direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0 }, @@ -1233,7 +1234,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { confchk_wiredtiger_open_compatibility_subconfigs, 3 }, { "debug_mode", "category", NULL, NULL, - confchk_wiredtiger_open_debug_mode_subconfigs, 3 }, + confchk_wiredtiger_open_debug_mode_subconfigs, 4 }, { "direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0 }, @@ -1372,12 +1373,12 @@ static const WT_CONFIG_ENTRY config_entries[] = { "async=(enabled=false,ops_max=1024,threads=2),cache_max_wait_ms=0" ",cache_overflow=(file_max=0),cache_overhead=8,cache_size=100MB," "checkpoint=(log_size=0,wait=0),compatibility=(release=)," - "debug_mode=(checkpoint_retention=0,rollback_error=0," - "table_logging=false),error_prefix=,eviction=(threads_max=8," - "threads_min=1),eviction_checkpoint_target=1," - "eviction_dirty_target=5,eviction_dirty_trigger=20," - "eviction_target=80,eviction_trigger=95," - "file_manager=(close_handle_minimum=250,close_idle_time=30," + "debug_mode=(checkpoint_retention=0,eviction=false," + "rollback_error=0,table_logging=false),error_prefix=," + "eviction=(threads_max=8,threads_min=1)," + "eviction_checkpoint_target=1,eviction_dirty_target=5," + "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" + ",file_manager=(close_handle_minimum=250,close_idle_time=30," "close_scan_interval=10),io_capacity=(total=0),log=(archive=true," "os_cache_dirty_pct=0,prealloc=true,zero_fill=false)," "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true," @@ -1465,6 +1466,10 @@ static const WT_CONFIG_ENTRY config_entries[] = { "remove_files=true", confchk_WT_SESSION_drop, 4 }, + { "WT_SESSION.import", + "", + NULL, 0 + }, { "WT_SESSION.join", "bloom_bit_count=16,bloom_false_positives=false," "bloom_hash_count=8,compare=\"eq\",count=,operation=\"and\"," @@ -1627,10 +1632,11 @@ static const WT_CONFIG_ENTRY config_entries[] = { ",cache_size=100MB,checkpoint=(log_size=0,wait=0)," "checkpoint_sync=true,compatibility=(release=,require_max=," "require_min=),config_base=true,create=false," - "debug_mode=(checkpoint_retention=0,rollback_error=0," - "table_logging=false),direct_io=,encryption=(keyid=,name=," - "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" - ",eviction_checkpoint_target=1,eviction_dirty_target=5," + "debug_mode=(checkpoint_retention=0,eviction=false," + "rollback_error=0,table_logging=false),direct_io=," + "encryption=(keyid=,name=,secretkey=),error_prefix=," + "eviction=(threads_max=8,threads_min=1)," + "eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",exclusive=false,extensions=,file_extend=," "file_manager=(close_handle_minimum=250,close_idle_time=30," @@ -1657,10 +1663,11 @@ static const WT_CONFIG_ENTRY config_entries[] = { ",cache_size=100MB,checkpoint=(log_size=0,wait=0)," "checkpoint_sync=true,compatibility=(release=,require_max=," "require_min=),config_base=true,create=false," - "debug_mode=(checkpoint_retention=0,rollback_error=0," - "table_logging=false),direct_io=,encryption=(keyid=,name=," - "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" - ",eviction_checkpoint_target=1,eviction_dirty_target=5," + "debug_mode=(checkpoint_retention=0,eviction=false," + "rollback_error=0,table_logging=false),direct_io=," + "encryption=(keyid=,name=,secretkey=),error_prefix=," + "eviction=(threads_max=8,threads_min=1)," + "eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",exclusive=false,extensions=,file_extend=," "file_manager=(close_handle_minimum=250,close_idle_time=30," @@ -1686,7 +1693,7 @@ static const WT_CONFIG_ENTRY config_entries[] = { "cache_max_wait_ms=0,cache_overflow=(file_max=0),cache_overhead=8" ",cache_size=100MB,checkpoint=(log_size=0,wait=0)," "checkpoint_sync=true,compatibility=(release=,require_max=," - "require_min=),debug_mode=(checkpoint_retention=0," + "require_min=),debug_mode=(checkpoint_retention=0,eviction=false," "rollback_error=0,table_logging=false),direct_io=," "encryption=(keyid=,name=,secretkey=),error_prefix=," "eviction=(threads_max=8,threads_min=1)," @@ -1714,7 +1721,7 @@ static const WT_CONFIG_ENTRY config_entries[] = { "cache_max_wait_ms=0,cache_overflow=(file_max=0),cache_overhead=8" ",cache_size=100MB,checkpoint=(log_size=0,wait=0)," "checkpoint_sync=true,compatibility=(release=,require_max=," - "require_min=),debug_mode=(checkpoint_retention=0," + "require_min=),debug_mode=(checkpoint_retention=0,eviction=false," "rollback_error=0,table_logging=false),direct_io=," "encryption=(keyid=,name=,secretkey=),error_prefix=," "eviction=(threads_max=8,threads_min=1)," diff --git a/src/third_party/wiredtiger/src/conn/api_calc_modify.c b/src/third_party/wiredtiger/src/conn/api_calc_modify.c index 4a435a85ef1..a8091498ee6 100644 --- a/src/third_party/wiredtiger/src/conn/api_calc_modify.c +++ b/src/third_party/wiredtiger/src/conn/api_calc_modify.c @@ -69,16 +69,31 @@ static void __cm_extend(WT_CM_STATE *cms, const uint8_t *m1, const uint8_t *m2, WT_CM_MATCH *match) { + ptrdiff_t n; const uint8_t *p1, *p2; - /* Step past the end and before the beginning of the matching block. */ + p1 = m1; + p2 = m2; + + /* + * Keep skipping half of the remaining bytes while they compare equal. + * This is significantly faster than our byte-at-a-time loop below. + */ for (p1 = m1, p2 = m2; - p1 < cms->e1 && p2 < cms->e2 && *p1 == *p2; - p1++, p2++) + (n = WT_MIN(cms->e1 - p1, cms->e2 - p2) / 2) > 8 && + memcmp(p1, p2, (size_t)n) == 0; + p1 += n, p2 += n) + ; + + /* Step past the end and before the beginning of the matching block. */ + for (n = WT_MIN(cms->e1 - p1, cms->e2 - p2); + n > 0 && *p1 == *p2; + n--, p1++, p2++) ; - for (; m1 >= cms->used1 && m2 >= cms->used2 && *m1 == *m2; - m1--, m2--) + for (n = WT_MIN(m1 - cms->used1, m2 - cms->used2); + n > 0 && *m1 == *m2; + n--, m1--, m2--) ; match->m1 = m1 + 1; diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index 7a854ee596f..e68fb6c15f8 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -1830,25 +1830,16 @@ err: /* int __wt_debug_mode_config(WT_SESSION_IMPL *session, const char *cfg[]) { + WT_CACHE *cache; WT_CONFIG_ITEM cval; WT_CONNECTION_IMPL *conn; WT_TXN_GLOBAL *txn_global; conn = S2C(session); + cache = conn->cache; txn_global = &conn->txn_global; WT_RET(__wt_config_gets(session, - cfg, "debug_mode.rollback_error", &cval)); - txn_global->debug_rollback = (uint64_t)cval.val; - - WT_RET(__wt_config_gets(session, - cfg, "debug_mode.table_logging", &cval)); - if (cval.val) - FLD_SET(conn->log_flags, WT_CONN_LOG_DEBUG_MODE); - else - FLD_CLR(conn->log_flags, WT_CONN_LOG_DEBUG_MODE); - - WT_RET(__wt_config_gets(session, cfg, "debug_mode.checkpoint_retention", &cval)); conn->debug_ckpt_cnt = (uint32_t)cval.val; if (cval.val == 0) { @@ -1862,6 +1853,24 @@ __wt_debug_mode_config(WT_SESSION_IMPL *session, const char *cfg[]) WT_RET(__wt_calloc_def(session, conn->debug_ckpt_cnt, &conn->debug_ckpt)); + WT_RET(__wt_config_gets(session, + cfg, "debug_mode.eviction", &cval)); + if (cval.val) + F_SET(cache, WT_CACHE_EVICT_DEBUG_MODE); + else + F_CLR(cache, WT_CACHE_EVICT_DEBUG_MODE); + + WT_RET(__wt_config_gets(session, + cfg, "debug_mode.rollback_error", &cval)); + txn_global->debug_rollback = (uint64_t)cval.val; + + WT_RET(__wt_config_gets(session, + cfg, "debug_mode.table_logging", &cval)); + if (cval.val) + FLD_SET(conn->log_flags, WT_CONN_LOG_DEBUG_MODE); + else + FLD_CLR(conn->log_flags, WT_CONN_LOG_DEBUG_MODE); + return (0); } @@ -2637,7 +2646,6 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, session, cval.str, cval.len, &conn->error_prefix)); } WT_ERR(__wt_verbose_config(session, cfg)); - WT_ERR(__wt_debug_mode_config(session, cfg)); WT_ERR(__wt_timing_stress_config(session, cfg)); __wt_btree_page_version_config(session); @@ -2762,6 +2770,12 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, session = conn->default_session; /* + * This function expects the cache to be created so parse this after + * the rest of the connection is set up. + */ + WT_ERR(__wt_debug_mode_config(session, cfg)); + + /* * Load the extensions after initialization completes; extensions expect * everything else to be in place, and the extensions call back into the * library. diff --git a/src/third_party/wiredtiger/src/docs/command-line.dox b/src/third_party/wiredtiger/src/docs/command-line.dox index df52324f8f8..852e07dd70a 100644 --- a/src/third_party/wiredtiger/src/docs/command-line.dox +++ b/src/third_party/wiredtiger/src/docs/command-line.dox @@ -342,7 +342,7 @@ data that cannot be recovered. Underlying files are re-written in place, overwriting the original file contents. @subsection util_salvage_synopsis Synopsis -<code>wt [-RVv] [-C config] [-E secretkey ] [-h directory] salvage [-F force] uri</code> +<code>wt [-RVv] [-C config] [-E secretkey ] [-h directory] salvage [-F] uri</code> @subsection util_salvage_options Options The following are command-specific options for the \c salvage command: diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 7d696a20831..ff2840b722c 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -107,6 +107,25 @@ __evict_entry_priority(WT_SESSION_IMPL *session, WT_REF *ref) } /* + * __evict_lru_cmp_debug -- + * Qsort function: sort the eviction array. + * Version for eviction debug mode. + */ +static int WT_CDECL +__evict_lru_cmp_debug(const void *a_arg, const void *b_arg) +{ + const WT_EVICT_ENTRY *a, *b; + uint64_t a_score, b_score; + + a = a_arg; + b = b_arg; + a_score = (a->ref == NULL ? UINT64_MAX : 0); + b_score = (b->ref == NULL ? UINT64_MAX : 0); + + return ((a_score < b_score) ? -1 : (a_score == b_score) ? 0 : 1); +} + +/* * __evict_lru_cmp -- * Qsort function: sort the eviction array. */ @@ -1257,8 +1276,17 @@ __evict_lru_walk(WT_SESSION_IMPL *session) queue->evict_current = NULL; entries = queue->evict_entries; - __wt_qsort(queue->evict_queue, - entries, sizeof(WT_EVICT_ENTRY), __evict_lru_cmp); + /* + * Style note: __wt_qsort is a macro that can leave a dangling + * else. Full curly braces are needed here for the compiler. + */ + if (F_ISSET(cache, WT_CACHE_EVICT_DEBUG_MODE)) { + __wt_qsort(queue->evict_queue, + entries, sizeof(WT_EVICT_ENTRY), __evict_lru_cmp_debug); + } else { + __wt_qsort(queue->evict_queue, + entries, sizeof(WT_EVICT_ENTRY), __evict_lru_cmp); + } /* Trim empty entries from the end. */ while (entries > 0 && queue->evict_queue[entries - 1].ref == NULL) @@ -2057,12 +2085,14 @@ __evict_walk_tree(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, * cache (indicated by seeing an internal page that is the * parent of the last page we saw). * - * Also skip internal page unless we get aggressive or the tree - * is idle (indicated by the tree being skipped for walks). + * Also skip internal page unless we get aggressive, the tree + * is idle (indicated by the tree being skipped for walks), + * or we are in eviction debug mode. * The goal here is that if trees become completely idle, we * eventually push them out of cache completely. */ - if (WT_PAGE_IS_INTERNAL(page)) { + if (!F_ISSET(cache, WT_CACHE_EVICT_DEBUG_MODE) && + WT_PAGE_IS_INTERNAL(page)) { if (page == last_parent) continue; if (btree->evict_walk_period == 0 && diff --git a/src/third_party/wiredtiger/src/evict/evict_page.c b/src/third_party/wiredtiger/src/evict/evict_page.c index 5618220774b..2510815401f 100644 --- a/src/third_party/wiredtiger/src/evict/evict_page.c +++ b/src/third_party/wiredtiger/src/evict/evict_page.c @@ -656,7 +656,13 @@ __evict_review( else if (!WT_IS_METADATA(session->dhandle)) { LF_SET(WT_REC_UPDATE_RESTORE); - if (F_ISSET(cache, WT_CACHE_EVICT_SCRUB)) + /* + * Scrub if we're supposed to or toss it in sometimes + * if we are in debugging mode. + */ + if (F_ISSET(cache, WT_CACHE_EVICT_SCRUB) || + (F_ISSET(cache, WT_CACHE_EVICT_DEBUG_MODE) && + __wt_random(&session->rnd) % 3 == 0)) LF_SET(WT_REC_SCRUB); /* @@ -665,8 +671,16 @@ __evict_review( * suggests trying the lookaside table. */ if (F_ISSET(cache, WT_CACHE_EVICT_LOOKASIDE) && - !F_ISSET(conn, WT_CONN_EVICTION_NO_LOOKASIDE)) + !F_ISSET(conn, WT_CONN_EVICTION_NO_LOOKASIDE)) { + if (F_ISSET(cache, + WT_CACHE_EVICT_DEBUG_MODE) && + __wt_random(&session->rnd) % 10 == 0) { + LF_CLR(WT_REC_SCRUB | + WT_REC_UPDATE_RESTORE); + LF_SET(WT_REC_LOOKASIDE); + } lookaside_retryp = &lookaside_retry; + } } } diff --git a/src/third_party/wiredtiger/src/include/block.h b/src/third_party/wiredtiger/src/include/block.h index 8efaf10dd2b..4cfe07f759d 100644 --- a/src/third_party/wiredtiger/src/include/block.h +++ b/src/third_party/wiredtiger/src/include/block.h @@ -125,6 +125,21 @@ struct __wt_size { */ #define WT_BM_CHECKPOINT_VERSION 1 /* Checkpoint format version */ #define WT_BLOCK_EXTLIST_MAGIC 71002 /* Identify a list */ + +/* + * There are two versions of the extent list blocks: the original, and a second + * version where current checkpoint information is appended to the avail extent + * list. + */ +#define WT_BLOCK_EXTLIST_VERSION_ORIG 0 /* Original version */ +#define WT_BLOCK_EXTLIST_VERSION_CKPT 1 /* Checkpoint in avail output */ + +/* + * Maximum buffer required to store a checkpoint: 1 version byte followed by + * 14 packed 8B values. + */ +#define WT_BLOCK_CHECKPOINT_BUFFER (1 + 14 * WT_INTPACK64_MAXSIZE) + struct __wt_block_ckpt { uint8_t version; /* Version */ @@ -163,6 +178,8 @@ struct __wt_bm { u_int (*block_header)(WT_BM *); int (*checkpoint) (WT_BM *, WT_SESSION_IMPL *, WT_ITEM *, WT_CKPT *, bool); + int (*checkpoint_last) + (WT_BM *, WT_SESSION_IMPL *, char **, char **, WT_ITEM *); int (*checkpoint_load)(WT_BM *, WT_SESSION_IMPL *, const uint8_t *, size_t, uint8_t *, size_t *, bool); int (*checkpoint_resolve)(WT_BM *, WT_SESSION_IMPL *, bool); @@ -254,6 +271,8 @@ struct __wt_block { enum { WT_CKPT_NONE=0, WT_CKPT_INPROGRESS, WT_CKPT_PANIC_ON_FAILURE, WT_CKPT_SALVAGE } ckpt_state; + WT_CKPT *final_ckpt; /* Final live checkpoint write */ + /* Compaction support */ int compact_pct_tenths; /* Percent to compact */ uint64_t compact_pages_reviewed;/* Pages reviewed */ diff --git a/src/third_party/wiredtiger/src/include/block.i b/src/third_party/wiredtiger/src/include/block.i new file mode 100644 index 00000000000..3b9183a19fa --- /dev/null +++ b/src/third_party/wiredtiger/src/include/block.i @@ -0,0 +1,39 @@ +/*- + * Copyright (c) 2014-2019 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +/* + * WiredTiger's block manager interface. + */ + +/* + * __wt_extlist_write_pair -- + * Write an extent list pair. + */ +static inline int +__wt_extlist_write_pair(uint8_t **p, wt_off_t off, wt_off_t size) +{ + WT_RET(__wt_vpack_uint(p, 0, (uint64_t)(off))); + WT_RET(__wt_vpack_uint(p, 0, (uint64_t)(size))); + return (0); +} + +/* + * __wt_extlist_read_pair -- + * Read an extent list pair. + */ +static inline int +__wt_extlist_read_pair(const uint8_t **p, wt_off_t *offp, wt_off_t *sizep) +{ + uint64_t v; + + WT_RET(__wt_vunpack_uint(p, 0, &v)); + *offp = (wt_off_t)v; + WT_RET(__wt_vunpack_uint(p, 0, &v)); + *sizep = (wt_off_t)v; + return (0); +} diff --git a/src/third_party/wiredtiger/src/include/btmem.h b/src/third_party/wiredtiger/src/include/btmem.h index 71a26862e4a..7729c9887d5 100644 --- a/src/third_party/wiredtiger/src/include/btmem.h +++ b/src/third_party/wiredtiger/src/include/btmem.h @@ -1130,9 +1130,9 @@ struct __wt_update { /* * WT_MODIFY_MEM_FACTOR -- - * Limit update chains to a factor of the base document size. + * Limit update chains to a fraction of the base document size. */ -#define WT_MODIFY_MEM_FACTOR 1 +#define WT_MODIFY_MEM_FRACTION 10 /* * WT_INSERT -- diff --git a/src/third_party/wiredtiger/src/include/btree.h b/src/third_party/wiredtiger/src/include/btree.h index 0becfded2a9..44fae885ae1 100644 --- a/src/third_party/wiredtiger/src/include/btree.h +++ b/src/third_party/wiredtiger/src/include/btree.h @@ -16,6 +16,8 @@ #define WT_BTREE_MAJOR_VERSION_MAX 1 /* Newest version supported */ #define WT_BTREE_MINOR_VERSION_MAX 1 +#define WT_BTREE_MIN_ALLOC_SIZE 512 + /* * The maximum btree leaf and internal page size is 512MB (2^29). The limit * is enforced in software, it could be larger, specifically, the underlying diff --git a/src/third_party/wiredtiger/src/include/btree.i b/src/third_party/wiredtiger/src/include/btree.i index 162ea524b09..2fb9afd14a6 100644 --- a/src/third_party/wiredtiger/src/include/btree.i +++ b/src/third_party/wiredtiger/src/include/btree.i @@ -922,6 +922,7 @@ __wt_row_leaf_key_set_cell(WT_PAGE *page, WT_ROW *rip, WT_CELL *cell) */ v = WT_CELL_ENCODE_OFFSET(WT_PAGE_DISK_OFFSET(page, cell)) | WT_CELL_FLAG; + WT_ASSERT(NULL, WT_ROW_SLOT(page, rip) < page->entries); WT_ROW_KEY_SET(rip, v); } @@ -941,6 +942,7 @@ __wt_row_leaf_key_set(WT_PAGE *page, WT_ROW *rip, WT_CELL_UNPACK *unpack) v = WT_K_ENCODE_KEY_LEN(unpack->size) | WT_K_ENCODE_KEY_OFFSET(WT_PAGE_DISK_OFFSET(page, unpack->data)) | WT_K_FLAG; + WT_ASSERT(NULL, WT_ROW_SLOT(page, rip) < page->entries); WT_ROW_KEY_SET(rip, v); } @@ -979,6 +981,7 @@ __wt_row_leaf_value_set(WT_PAGE *page, WT_ROW *rip, WT_CELL_UNPACK *unpack) WT_KV_ENCODE_VALUE_LEN(unpack->size) | WT_KV_ENCODE_KEY_OFFSET(key_offset) | WT_KV_ENCODE_VALUE_OFFSET(value_offset) | WT_KV_FLAG; + WT_ASSERT(NULL, WT_ROW_SLOT(page, rip) < page->entries); WT_ROW_KEY_SET(rip, v); } diff --git a/src/third_party/wiredtiger/src/include/cache.h b/src/third_party/wiredtiger/src/include/cache.h index 07d25d13ec2..2146a00a76d 100644 --- a/src/third_party/wiredtiger/src/include/cache.h +++ b/src/third_party/wiredtiger/src/include/cache.h @@ -170,7 +170,7 @@ struct __wt_cache { * Score of how aggressive eviction should be about selecting eviction * candidates. If eviction is struggling to make progress, this score * rises (up to a maximum of 100), at which point the cache is "stuck" - * and transaction will be rolled back. + * and transactions will be rolled back. */ uint32_t evict_aggressive_score; @@ -250,11 +250,12 @@ struct __wt_cache { /* AUTOMATIC FLAG VALUE GENERATION START */ #define WT_CACHE_EVICT_CLEAN 0x01u /* Evict clean pages */ #define WT_CACHE_EVICT_CLEAN_HARD 0x02u /* Clean % blocking app threads */ -#define WT_CACHE_EVICT_DIRTY 0x04u /* Evict dirty pages */ -#define WT_CACHE_EVICT_DIRTY_HARD 0x08u /* Dirty % blocking app threads */ -#define WT_CACHE_EVICT_LOOKASIDE 0x10u /* Try lookaside eviction */ -#define WT_CACHE_EVICT_SCRUB 0x20u /* Scrub dirty pages */ -#define WT_CACHE_EVICT_URGENT 0x40u /* Pages are in the urgent queue */ +#define WT_CACHE_EVICT_DEBUG_MODE 0x04u /* Aggressive debugging mode */ +#define WT_CACHE_EVICT_DIRTY 0x08u /* Evict dirty pages */ +#define WT_CACHE_EVICT_DIRTY_HARD 0x10u /* Dirty % blocking app threads */ +#define WT_CACHE_EVICT_LOOKASIDE 0x20u /* Try lookaside eviction */ +#define WT_CACHE_EVICT_SCRUB 0x40u /* Scrub dirty pages */ +#define WT_CACHE_EVICT_URGENT 0x80u /* Pages are in the urgent queue */ /* AUTOMATIC FLAG VALUE GENERATION STOP */ #define WT_CACHE_EVICT_ALL (WT_CACHE_EVICT_CLEAN | WT_CACHE_EVICT_DIRTY) uint32_t flags; diff --git a/src/third_party/wiredtiger/src/include/config.h b/src/third_party/wiredtiger/src/include/config.h index 64c55d7223b..847ddef1b2e 100644 --- a/src/third_party/wiredtiger/src/include/config.h +++ b/src/third_party/wiredtiger/src/include/config.h @@ -80,35 +80,36 @@ struct __wt_config_parser_impl { #define WT_CONFIG_ENTRY_WT_SESSION_compact 22 #define WT_CONFIG_ENTRY_WT_SESSION_create 23 #define WT_CONFIG_ENTRY_WT_SESSION_drop 24 -#define WT_CONFIG_ENTRY_WT_SESSION_join 25 -#define WT_CONFIG_ENTRY_WT_SESSION_log_flush 26 -#define WT_CONFIG_ENTRY_WT_SESSION_log_printf 27 -#define WT_CONFIG_ENTRY_WT_SESSION_open_cursor 28 -#define WT_CONFIG_ENTRY_WT_SESSION_prepare_transaction 29 -#define WT_CONFIG_ENTRY_WT_SESSION_query_timestamp 30 -#define WT_CONFIG_ENTRY_WT_SESSION_rebalance 31 -#define WT_CONFIG_ENTRY_WT_SESSION_reconfigure 32 -#define WT_CONFIG_ENTRY_WT_SESSION_rename 33 -#define WT_CONFIG_ENTRY_WT_SESSION_reset 34 -#define WT_CONFIG_ENTRY_WT_SESSION_rollback_transaction 35 -#define WT_CONFIG_ENTRY_WT_SESSION_salvage 36 -#define WT_CONFIG_ENTRY_WT_SESSION_snapshot 37 -#define WT_CONFIG_ENTRY_WT_SESSION_strerror 38 -#define WT_CONFIG_ENTRY_WT_SESSION_timestamp_transaction 39 -#define WT_CONFIG_ENTRY_WT_SESSION_transaction_sync 40 -#define WT_CONFIG_ENTRY_WT_SESSION_truncate 41 -#define WT_CONFIG_ENTRY_WT_SESSION_upgrade 42 -#define WT_CONFIG_ENTRY_WT_SESSION_verify 43 -#define WT_CONFIG_ENTRY_colgroup_meta 44 -#define WT_CONFIG_ENTRY_file_config 45 -#define WT_CONFIG_ENTRY_file_meta 46 -#define WT_CONFIG_ENTRY_index_meta 47 -#define WT_CONFIG_ENTRY_lsm_meta 48 -#define WT_CONFIG_ENTRY_table_meta 49 -#define WT_CONFIG_ENTRY_wiredtiger_open 50 -#define WT_CONFIG_ENTRY_wiredtiger_open_all 51 -#define WT_CONFIG_ENTRY_wiredtiger_open_basecfg 52 -#define WT_CONFIG_ENTRY_wiredtiger_open_usercfg 53 +#define WT_CONFIG_ENTRY_WT_SESSION_import 25 +#define WT_CONFIG_ENTRY_WT_SESSION_join 26 +#define WT_CONFIG_ENTRY_WT_SESSION_log_flush 27 +#define WT_CONFIG_ENTRY_WT_SESSION_log_printf 28 +#define WT_CONFIG_ENTRY_WT_SESSION_open_cursor 29 +#define WT_CONFIG_ENTRY_WT_SESSION_prepare_transaction 30 +#define WT_CONFIG_ENTRY_WT_SESSION_query_timestamp 31 +#define WT_CONFIG_ENTRY_WT_SESSION_rebalance 32 +#define WT_CONFIG_ENTRY_WT_SESSION_reconfigure 33 +#define WT_CONFIG_ENTRY_WT_SESSION_rename 34 +#define WT_CONFIG_ENTRY_WT_SESSION_reset 35 +#define WT_CONFIG_ENTRY_WT_SESSION_rollback_transaction 36 +#define WT_CONFIG_ENTRY_WT_SESSION_salvage 37 +#define WT_CONFIG_ENTRY_WT_SESSION_snapshot 38 +#define WT_CONFIG_ENTRY_WT_SESSION_strerror 39 +#define WT_CONFIG_ENTRY_WT_SESSION_timestamp_transaction 40 +#define WT_CONFIG_ENTRY_WT_SESSION_transaction_sync 41 +#define WT_CONFIG_ENTRY_WT_SESSION_truncate 42 +#define WT_CONFIG_ENTRY_WT_SESSION_upgrade 43 +#define WT_CONFIG_ENTRY_WT_SESSION_verify 44 +#define WT_CONFIG_ENTRY_colgroup_meta 45 +#define WT_CONFIG_ENTRY_file_config 46 +#define WT_CONFIG_ENTRY_file_meta 47 +#define WT_CONFIG_ENTRY_index_meta 48 +#define WT_CONFIG_ENTRY_lsm_meta 49 +#define WT_CONFIG_ENTRY_table_meta 50 +#define WT_CONFIG_ENTRY_wiredtiger_open 51 +#define WT_CONFIG_ENTRY_wiredtiger_open_all 52 +#define WT_CONFIG_ENTRY_wiredtiger_open_basecfg 53 +#define WT_CONFIG_ENTRY_wiredtiger_open_usercfg 54 /* * configuration section: END * DO NOT EDIT: automatically built by dist/flags.py. diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index f122c5e8f84..370f467e930 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -15,7 +15,8 @@ extern int __wt_block_addr_invalid(WT_SESSION_IMPL *session, WT_BLOCK *block, co extern int __wt_block_addr_string(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_buffer_to_ckpt(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *p, WT_BLOCK_CKPT *ci) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_ckpt_decode(WT_SESSION *wt_session, size_t allocsize, const uint8_t *p, WT_BLOCK_CKPT *ci) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_block_ckpt_to_buffer(WT_SESSION_IMPL *session, WT_BLOCK *block, uint8_t **pp, WT_BLOCK_CKPT *ci) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_ckpt_to_buffer(WT_SESSION_IMPL *session, WT_BLOCK *block, uint8_t **pp, WT_BLOCK_CKPT *ci, bool skip_avail) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_ckpt_verbose(WT_SESSION_IMPL *session, WT_BLOCK *block, const char *tag, const char *ckpt_name, const uint8_t *ckpt_string); extern int __wt_block_ckpt_init(WT_SESSION_IMPL *session, WT_BLOCK_CKPT *ci, const char *name) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_checkpoint_load(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size, uint8_t *root_addr, size_t *root_addr_sizep, bool checkpoint) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_checkpoint_unload(WT_SESSION_IMPL *session, WT_BLOCK *block, bool checkpoint) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -23,6 +24,8 @@ extern void __wt_block_ckpt_destroy(WT_SESSION_IMPL *session, WT_BLOCK_CKPT *ci) extern int __wt_block_checkpoint_start(WT_SESSION_IMPL *session, WT_BLOCK *block) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_checkpoint(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, WT_CKPT *ckptbase, bool data_checksum) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_checkpoint_resolve(WT_SESSION_IMPL *session, WT_BLOCK *block, bool failed) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_checkpoint_final(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint8_t **file_sizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_checkpoint_last(WT_SESSION_IMPL *session, WT_BLOCK *block, char **metadatap, char **checkpoint_listp, WT_ITEM *checkpoint) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_compact_start(WT_SESSION_IMPL *session, WT_BLOCK *block) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_compact_end(WT_SESSION_IMPL *session, WT_BLOCK *block) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_compact_skip(WT_SESSION_IMPL *session, WT_BLOCK *block, bool *skipp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -141,12 +144,14 @@ extern void __wt_btree_page_version_config(WT_SESSION_IMPL *session); extern int __wt_btree_open(WT_SESSION_IMPL *session, const char *op_cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_btree_close(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_btree_discard(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_btree_config_encryptor(WT_SESSION_IMPL *session, const char **cfg, WT_KEYED_ENCRYPTOR **kencryptorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_root_ref_init(WT_SESSION_IMPL *session, WT_REF *root_ref, WT_PAGE *root, bool is_recno); extern int __wt_btree_tree_open(WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_btree_new_leaf_page(WT_SESSION_IMPL *session, WT_PAGE **pagep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern bool __wt_btree_immediately_durable(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_btree_huffman_open(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_btree_huffman_close(WT_SESSION_IMPL *session); +extern int __wt_import(WT_SESSION_IMPL *session, const char *uri) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bt_read(WT_SESSION_IMPL *session, WT_ITEM *buf, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bt_write(WT_SESSION_IMPL *session, WT_ITEM *buf, uint8_t *addr, size_t *addr_sizep, size_t *compressed_sizep, bool checkpoint, bool checkpoint_io, bool compressed) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern const char *__wt_page_type_string(u_int type) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); @@ -172,7 +177,7 @@ extern int __wt_bt_rebalance(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC extern int __wt_value_return_upd(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd, bool ignore_visibility) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_key_return(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_value_return(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_bt_salvage(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_salvage(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_multi_to_ref(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi, WT_REF **refp, size_t *incrp, bool closing) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_split_insert(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_split_multi(WT_SESSION_IMPL *session, WT_REF *ref, int closing) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -237,6 +242,7 @@ extern void __wt_conn_foc_discard(WT_SESSION_IMPL *session); extern int __wt_configure_method(WT_SESSION_IMPL *session, const char *method, const char *uri, const char *config, const char *type, const char *check) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_config_check(WT_SESSION_IMPL *session, const WT_CONFIG_ENTRY *entry, const char *config, size_t config_len) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_config_collapse(WT_SESSION_IMPL *session, const char **cfg, char **config_ret) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_config_discard_defaults(WT_SESSION_IMPL *session, const char **cfg, const char *config, char **config_ret) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_config_merge(WT_SESSION_IMPL *session, const char **cfg, const char *cfg_strip, const char **config_ret) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_conn_config_init(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_conn_config_discard(WT_SESSION_IMPL *session); @@ -516,8 +522,10 @@ extern int __wt_meta_apply_all(WT_SESSION_IMPL *session, int (*file_func)(WT_SES extern int __wt_meta_checkpoint(WT_SESSION_IMPL *session, const char *fname, const char *checkpoint, WT_CKPT *ckpt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_meta_checkpoint_last_name(WT_SESSION_IMPL *session, const char *fname, const char **namep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_meta_checkpoint_clear(WT_SESSION_IMPL *session, const char *fname) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_meta_ckptlist_get(WT_SESSION_IMPL *session, const char *fname, WT_CKPT **ckptbasep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_meta_block_metadata(WT_SESSION_IMPL *session, const char *config, WT_CKPT *ckpt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_meta_ckptlist_get(WT_SESSION_IMPL *session, const char *fname, bool update, WT_CKPT **ckptbasep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_metadata_set_base_write_gen(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_meta_ckptlist_to_meta(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_meta_ckptlist_set(WT_SESSION_IMPL *session, const char *fname, WT_CKPT *ckptbase, WT_LSN *ckptlsn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_meta_ckptlist_free(WT_SESSION_IMPL *session, WT_CKPT **ckptbasep); extern void __wt_meta_checkpoint_free(WT_SESSION_IMPL *session, WT_CKPT *ckpt); @@ -702,7 +710,6 @@ extern int __wt_session_get_btree_ckpt(WT_SESSION_IMPL *session, const char *uri extern void __wt_session_close_cache(WT_SESSION_IMPL *session); extern int __wt_session_get_dhandle(WT_SESSION_IMPL *session, const char *uri, const char *checkpoint, const char *cfg[], uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_session_lock_checkpoint(WT_SESSION_IMPL *session, const char *checkpoint) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_salvage(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_cond_auto_alloc(WT_SESSION_IMPL *session, const char *name, uint64_t min, uint64_t max, WT_CONDVAR **condp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_cond_auto_wait_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond, bool progress, bool (*run_func)(WT_SESSION_IMPL *), bool *signalled); extern void __wt_cond_auto_wait(WT_SESSION_IMPL *session, WT_CONDVAR *cond, bool progress, bool (*run_func)(WT_SESSION_IMPL *)); diff --git a/src/third_party/wiredtiger/src/include/meta.h b/src/third_party/wiredtiger/src/include/meta.h index 0b08b318c3d..ac9e0be3c20 100644 --- a/src/third_party/wiredtiger/src/include/meta.h +++ b/src/third_party/wiredtiger/src/include/meta.h @@ -66,9 +66,19 @@ struct __wt_ckpt { char *name; /* Name or NULL */ - WT_ITEM addr; /* Checkpoint cookie string */ - WT_ITEM raw; /* Checkpoint cookie raw */ - + /* + * Each internal checkpoint name is appended with a generation + * to make it a unique name. We're solving two problems: when + * two checkpoints are taken quickly, the timer may not be + * unique and/or we can even see time travel on the second + * checkpoint if we snapshot the time in-between nanoseconds + * rolling over. Second, if we reset the generational counter + * when new checkpoints arrive, we could logically re-create + * specific checkpoints, racing with cursors open on those + * checkpoints. I can't think of any way to return incorrect + * results by racing with those cursors, but it's simpler not + * to worry about it. + */ int64_t order; /* Checkpoint order */ uint64_t sec; /* Wall clock time */ @@ -77,6 +87,9 @@ struct __wt_ckpt { uint64_t write_gen; /* Write generation */ + char *block_metadata; /* Block-stored metadata */ + char *block_checkpoint; /* Block-stored checkpoint */ + /* Validity window */ wt_timestamp_t newest_durable_ts; wt_timestamp_t oldest_start_ts; @@ -84,6 +97,9 @@ struct __wt_ckpt { wt_timestamp_t newest_stop_ts; uint64_t newest_stop_txn; + WT_ITEM addr; /* Checkpoint cookie string */ + WT_ITEM raw; /* Checkpoint cookie raw */ + void *bpriv; /* Block manager private */ /* AUTOMATIC FLAG VALUE GENERATION START */ diff --git a/src/third_party/wiredtiger/src/include/misc.i b/src/third_party/wiredtiger/src/include/misc.i index ab24e87f34a..b552f9b1f3e 100644 --- a/src/third_party/wiredtiger/src/include/misc.i +++ b/src/third_party/wiredtiger/src/include/misc.i @@ -255,13 +255,26 @@ __wt_spin_backoff(uint64_t *yield_count, uint64_t *sleep_usecs) static inline void __wt_timing_stress(WT_SESSION_IMPL *session, u_int flag) { - uint64_t i; + double pct; + uint64_t i, max; /* Optionally only sleep when a specified configuration flag is set. */ if (flag != 0 && !FLD_ISSET(S2C(session)->timing_stress_flags, flag)) return; /* + * If there is a lot of cache pressure, don't let the sleep time + * get too large. If the cache is totally full, return. + */ + pct = 0.0; + if (__wt_eviction_needed(session, false, false, &pct)) + max = 5; + else + max = 9; + if (pct > 100.0) + return; + + /* * We need a fast way to choose a sleep time. We want to sleep a short * period most of the time, but occasionally wait longer. Divide the * maximum period of time into 10 buckets (where bucket 0 doesn't sleep @@ -269,7 +282,7 @@ __wt_timing_stress(WT_SESSION_IMPL *session, u_int flag) * That means we'll hit the maximum roughly every 1K calls. */ for (i = 0;;) - if (__wt_random(&session->rnd) & 0x1 || ++i > 9) + if (__wt_random(&session->rnd) & 0x1 || ++i > max) break; if (i == 0) diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index e752ac05c0d..26c030009e6 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -633,6 +633,8 @@ struct __wt_connection_stats { int64_t session_table_create_success; int64_t session_table_drop_fail; int64_t session_table_drop_success; + int64_t session_table_import_fail; + int64_t session_table_import_success; int64_t session_table_rebalance_fail; int64_t session_table_rebalance_success; int64_t session_table_rename_fail; diff --git a/src/third_party/wiredtiger/src/include/txn.h b/src/third_party/wiredtiger/src/include/txn.h index 29ad33f3b76..2a514629006 100644 --- a/src/third_party/wiredtiger/src/include/txn.h +++ b/src/third_party/wiredtiger/src/include/txn.h @@ -242,6 +242,11 @@ struct __wt_txn_op { } mode; } truncate_row; } u; + +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_TXN_OP_KEY_REPEATED 0x1u +/* AUTOMATIC FLAG VALUE GENERATION STOP */ + uint32_t flags; }; /* @@ -305,14 +310,6 @@ struct __wt_txn { WT_TXN_OP *mod; size_t mod_alloc; u_int mod_count; -#ifdef HAVE_DIAGNOSTIC - /* - * Reference count of multiple updates processed, as part of a single - * transaction operation processing for resolving the indirect update - * references in a prepared transaction as part of commit. - */ - u_int multi_update_count; -#endif /* Scratch buffer for in-memory log records. */ WT_ITEM *logrec; diff --git a/src/third_party/wiredtiger/src/include/txn.i b/src/third_party/wiredtiger/src/include/txn.i index 0c8e22c90d1..1aea1e854d3 100644 --- a/src/third_party/wiredtiger/src/include/txn.i +++ b/src/third_party/wiredtiger/src/include/txn.i @@ -171,17 +171,21 @@ __txn_resolve_prepared_update(WT_SESSION_IMPL *session, WT_UPDATE *upd) /* * __wt_txn_resolve_prepared_op -- - * Resolve a transaction operation indirect references. + * Resolve a transaction's operations indirect references. * * In case of prepared transactions, the prepared updates could be evicted * using cache overflow mechanism. Transaction operations referring to * these prepared updates would be referring to them using indirect * references (i.e keys/recnos), which need to be resolved as part of that * transaction commit/rollback. + * + * If no updates are resolved throw an error. Increment resolved update + * count for each resolved update count we locate. */ static inline int __wt_txn_resolve_prepared_op( - WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit) + WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit, + int64_t *resolved_update_countp) { WT_CURSOR *cursor; WT_DECL_RET; @@ -218,37 +222,29 @@ __wt_txn_resolve_prepared_op( (WT_CURSOR_BTREE *)cursor, &upd)); WT_ERR(ret); -#ifdef HAVE_DIAGNOSTIC - /* - * Do what we can to ensure that finding prepared updates from a key - * is working as expected. In the case where a transaction has updated - * the same key multiple times, it's possible to resolve all updates - * for the key when processing the first op structure, and then have - * eviction free those updates before subsequent ops are processed, - * which means a search could reasonably not find an update in that - * case. - * We track the update count only for commit, but not for rollback, as - * our tracking is based on transaction id, and in case of rollback, we - * set it to aborted. - */ - if (upd == NULL && commit) { - /* - * FIXME: - * WT_ASSERT(session, txn->multi_update_count > 0); - */ - --txn->multi_update_count; + /* If we haven't found anything then there's an error. */ + if (upd == NULL) { + WT_ASSERT(session, upd != NULL); + WT_ERR(WT_NOTFOUND); } -#endif - - WT_STAT_CONN_INCR(session, txn_prepared_updates_resolved); for (; upd != NULL; upd = upd->next) { - if (upd->txnid != txn->id) + /* + * Aborted updates can exist in the update chain of our txn. + * Generally this will occur due to a reserved update. + * As such we should skip over these updates. If the txn + * id is then different and not aborted we know we've + * reached the end of our update chain and can exit. + */ + if (upd->txnid == WT_TXN_ABORTED) continue; - + if (upd->txnid != txn->id) + break; if (op->u.op_upd == NULL) op->u.op_upd = upd; + ++(*resolved_update_countp); + if (!commit) { upd->txnid = WT_TXN_ABORTED; continue; @@ -282,52 +278,9 @@ __wt_txn_resolve_prepared_op( * thing as part of "txn_op2". */ -#ifdef HAVE_DIAGNOSTIC - /* - * When an update is not identified for resolution of a - * transaction operation, it might have been already processed - * during the resolution of a previous update belonging to the - * same key. To ascertain transaction tracks multiple extra - * updates processed in resolution of an transaction operation. - */ - if (upd->prepare_state == WT_PREPARE_RESOLVED) { - /* - * FIXME: - * WT_ASSERT(session, txn->multi_update_count > 0); - */ - --txn->multi_update_count; - } else if (upd != op->u.op_upd) - ++txn->multi_update_count; -#endif - - if (upd->prepare_state == WT_PREPARE_RESOLVED) - break; - /* Resolve the prepared update to be committed update. */ __txn_resolve_prepared_update(session, upd); } - - /* FIXME: it isn't safe to walk updates after they are resolved. */ -#if 0 && defined(HAVE_DIAGNOSTIC) - upd = op->u.op_upd; - /* Ensure that we have not missed any of this transaction updates. */ - for (; upd != NULL; upd = upd->next) { - /* - * Should not have an unprocessed uncommitted update of this - * transaction. For commit, no uncommitted update of this - * transaction should be in prepared state. For rollback, there - * should not be any more uncommitted updates from this - * transaction. - */ - if (commit && upd->txnid == txn->id) - WT_ASSERT(session, - upd->prepare_state != WT_PREPARE_INPROGRESS); - else - WT_ASSERT(session, upd->txnid != txn->id); - - } -#endif - err: WT_TRET(cursor->close(cursor)); return (ret); } @@ -443,6 +396,43 @@ __wt_txn_op_apply_prepare_state( } /* + * __wt_txn_op_delete_commit_apply_timestamps -- + * Apply the correct start and durable timestamps to any + * updates in the page del update list. + */ +static inline void +__wt_txn_op_delete_commit_apply_timestamps( + WT_SESSION_IMPL *session, WT_REF *ref) +{ + WT_TXN *txn; + WT_UPDATE **updp; + uint32_t previous_state; + + txn = &session->txn; + + /* + * Lock the ref to ensure we don't race with eviction freeing the page + * deleted update list or with a page instantiate. + */ + for (;; __wt_yield()) { + previous_state = ref->state; + WT_ASSERT(session, previous_state != WT_REF_READING); + if (previous_state != WT_REF_LOCKED && WT_REF_CAS_STATE( + session, ref, previous_state, WT_REF_LOCKED)) + break; + } + + for (updp = ref->page_del->update_list; + updp != NULL && *updp != NULL; ++updp) { + (*updp)->start_ts = txn->commit_timestamp; + (*updp)->durable_ts = txn->durable_timestamp; + } + + /* Unlock the page by setting it back to it's previous state */ + WT_REF_SET_STATE(ref, previous_state); +} + +/* * __wt_txn_op_set_timestamp -- * Decide whether to copy a commit timestamp into an update. If the op * structure doesn't have a populated update or ref field or in prepared @@ -496,6 +486,10 @@ __wt_txn_op_set_timestamp(WT_SESSION_IMPL *session, WT_TXN_OP *op) &op->u.op_upd->durable_ts; *timestamp = txn->durable_timestamp; } + + if (op->type == WT_TXN_OP_REF_DELETE) + __wt_txn_op_delete_commit_apply_timestamps( + session, op->u.ref); } } @@ -511,9 +505,14 @@ __wt_txn_modify(WT_SESSION_IMPL *session, WT_UPDATE *upd) txn = &session->txn; - if (F_ISSET(txn, WT_TXN_READONLY)) + if (F_ISSET(txn, WT_TXN_READONLY)) { + if (F_ISSET(txn, WT_TXN_IGNORE_PREPARE)) + WT_RET_MSG(session, ENOTSUP, + "Transactions with ignore_prepare=true" + " cannot perform updates"); WT_RET_MSG(session, WT_ROLLBACK, "Attempt to update in a read-only transaction"); + } WT_RET(__txn_next_op(session, &op)); if (F_ISSET(session, WT_SESSION_LOGGING_INMEM)) { diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 9830d79a190..484175dd43e 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -1448,6 +1448,21 @@ struct __wt_session { int __F(create)(WT_SESSION *session, const char *name, const char *config); +#if !defined(SWIG) && !defined(DOXYGEN) + /*! + * Import a file. + * + * @snippet ex_all.c Import a file + * + * @param session the session handle + * @param name the URI of the object to import + * @configempty{WT_SESSION.import, see dist/api_data.py} + * @errors + */ + int __F(import)(WT_SESSION *session, + const char *name, const char *config); +#endif + /*! * Compact a live row- or column-store btree or LSM tree. * @@ -2284,16 +2299,23 @@ struct __wt_connection { * checkpoints. Zero or one means perform normal archiving., an integer * between 0 and 1024; default \c 0.} * @config{ - * rollback_error, return a WT_ROLLBACK error from a transaction - * operation about every Nth operation to simulate a collision., an - * integer between 0 and 10M; default \c 0.} - * @config{ table_logging, if true\, write - * transaction related information to the log for all operations\, even - * operations for tables with logging turned off. This setting - * introduces a log format change that may break older versions of - * WiredTiger. These operations are informational and skipped in - * recovery., a boolean flag; default \c false.} - * @config{ ),,} + * eviction, if true\, modify internal algorithms to change skew to + * force lookaside eviction to happen more aggressively. This includes + * but is not limited to not skewing newest\, not favoring leaf pages\, + * and modifying the eviction score mechanism., a boolean flag; default + * \c false.} + * @config{ rollback_error, return a + * WT_ROLLBACK error from a transaction operation about every Nth + * operation to simulate a collision., an integer between 0 and 10M; + * default \c 0.} + * @config{ table_logging, if + * true\, write transaction related information to the log for all + * operations\, even operations for tables with logging turned off. + * This setting introduces a log format change that may break older + * versions of WiredTiger. These operations are informational and + * skipped in recovery., a boolean flag; default \c false.} + * @config{ + * ),,} * @config{error_prefix, prefix string for error messages., a string; * default empty.} * @config{eviction = (, eviction configuration options., a set of @@ -2928,14 +2950,21 @@ struct __wt_connection { * @config{ checkpoint_retention, adjust log archiving to * retain the log records of this number of checkpoints. Zero or one means * perform normal archiving., an integer between 0 and 1024; default \c 0.} - * @config{ rollback_error, return a WT_ROLLBACK error - * from a transaction operation about every Nth operation to simulate a - * collision., an integer between 0 and 10M; default \c 0.} - * @config{ table_logging, if true\, write transaction - * related information to the log for all operations\, even operations for - * tables with logging turned off. This setting introduces a log format change - * that may break older versions of WiredTiger. These operations are - * informational and skipped in recovery., a boolean flag; default \c false.} + * @config{ eviction, if true\, modify internal + * algorithms to change skew to force lookaside eviction to happen more + * aggressively. This includes but is not limited to not skewing newest\, not + * favoring leaf pages\, and modifying the eviction score mechanism., a boolean + * flag; default \c false.} + * @config{ rollback_error, + * return a WT_ROLLBACK error from a transaction operation about every Nth + * operation to simulate a collision., an integer between 0 and 10M; default \c + * 0.} + * @config{ table_logging, if true\, write + * transaction related information to the log for all operations\, even + * operations for tables with logging turned off. This setting introduces a log + * format change that may break older versions of WiredTiger. These operations + * are informational and skipped in recovery., a boolean flag; default \c + * false.} * @config{ ),,} * @config{direct_io, Use \c O_DIRECT on POSIX systems\, and \c * FILE_FLAG_NO_BUFFERING on Windows to access files. Options are given as a @@ -5697,201 +5726,205 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1307 /*! session: table drop successful calls */ #define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1308 +/*! session: table import failed calls */ +#define WT_STAT_CONN_SESSION_TABLE_IMPORT_FAIL 1309 +/*! session: table import successful calls */ +#define WT_STAT_CONN_SESSION_TABLE_IMPORT_SUCCESS 1310 /*! session: table rebalance failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1309 +#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1311 /*! session: table rebalance successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1310 +#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1312 /*! session: table rename failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1311 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1313 /*! session: table rename successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1312 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1314 /*! session: table salvage failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1313 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1315 /*! session: table salvage successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1314 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1316 /*! session: table truncate failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1315 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1317 /*! session: table truncate successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1316 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1318 /*! session: table verify failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1317 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1319 /*! session: table verify successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1318 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1320 /*! thread-state: active filesystem fsync calls */ -#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1319 +#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1321 /*! thread-state: active filesystem read calls */ -#define WT_STAT_CONN_THREAD_READ_ACTIVE 1320 +#define WT_STAT_CONN_THREAD_READ_ACTIVE 1322 /*! thread-state: active filesystem write calls */ -#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1321 +#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1323 /*! thread-yield: application thread time evicting (usecs) */ -#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1322 +#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1324 /*! thread-yield: application thread time waiting for cache (usecs) */ -#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1323 +#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1325 /*! * thread-yield: connection close blocked waiting for transaction state * stabilization */ -#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1324 +#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1326 /*! thread-yield: connection close yielded for lsm manager shutdown */ -#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1325 +#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1327 /*! thread-yield: data handle lock yielded */ -#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1326 +#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1328 /*! * thread-yield: get reference for page index and slot time sleeping * (usecs) */ -#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1327 +#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1329 /*! thread-yield: log server sync yielded for log write */ -#define WT_STAT_CONN_LOG_SERVER_SYNC_BLOCKED 1328 +#define WT_STAT_CONN_LOG_SERVER_SYNC_BLOCKED 1330 /*! thread-yield: page access yielded due to prepare state change */ -#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1329 +#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1331 /*! thread-yield: page acquire busy blocked */ -#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1330 +#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1332 /*! thread-yield: page acquire eviction blocked */ -#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1331 +#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1333 /*! thread-yield: page acquire locked blocked */ -#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1332 +#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1334 /*! thread-yield: page acquire read blocked */ -#define WT_STAT_CONN_PAGE_READ_BLOCKED 1333 +#define WT_STAT_CONN_PAGE_READ_BLOCKED 1335 /*! thread-yield: page acquire time sleeping (usecs) */ -#define WT_STAT_CONN_PAGE_SLEEP 1334 +#define WT_STAT_CONN_PAGE_SLEEP 1336 /*! * thread-yield: page delete rollback time sleeping for state change * (usecs) */ -#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1335 +#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1337 /*! thread-yield: page reconciliation yielded due to child modification */ -#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1336 +#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1338 /*! transaction: Number of prepared updates */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COUNT 1337 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COUNT 1339 /*! transaction: Number of prepared updates added to cache overflow */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES_LOOKASIDE_INSERTS 1338 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_LOOKASIDE_INSERTS 1340 /*! transaction: Number of prepared updates resolved */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES_RESOLVED 1339 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_RESOLVED 1341 /*! transaction: commit timestamp queue entries walked */ -#define WT_STAT_CONN_TXN_COMMIT_QUEUE_WALKED 1340 +#define WT_STAT_CONN_TXN_COMMIT_QUEUE_WALKED 1342 /*! transaction: commit timestamp queue insert to empty */ -#define WT_STAT_CONN_TXN_COMMIT_QUEUE_EMPTY 1341 +#define WT_STAT_CONN_TXN_COMMIT_QUEUE_EMPTY 1343 /*! transaction: commit timestamp queue inserts to head */ -#define WT_STAT_CONN_TXN_COMMIT_QUEUE_HEAD 1342 +#define WT_STAT_CONN_TXN_COMMIT_QUEUE_HEAD 1344 /*! transaction: commit timestamp queue inserts total */ -#define WT_STAT_CONN_TXN_COMMIT_QUEUE_INSERTS 1343 +#define WT_STAT_CONN_TXN_COMMIT_QUEUE_INSERTS 1345 /*! transaction: commit timestamp queue length */ -#define WT_STAT_CONN_TXN_COMMIT_QUEUE_LEN 1344 +#define WT_STAT_CONN_TXN_COMMIT_QUEUE_LEN 1346 /*! transaction: number of named snapshots created */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1345 +#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1347 /*! transaction: number of named snapshots dropped */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1346 +#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1348 /*! transaction: prepared transactions */ -#define WT_STAT_CONN_TXN_PREPARE 1347 +#define WT_STAT_CONN_TXN_PREPARE 1349 /*! transaction: prepared transactions committed */ -#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1348 +#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1350 /*! transaction: prepared transactions currently active */ -#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1349 +#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1351 /*! transaction: prepared transactions rolled back */ -#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1350 +#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1352 /*! transaction: query timestamp calls */ -#define WT_STAT_CONN_TXN_QUERY_TS 1351 +#define WT_STAT_CONN_TXN_QUERY_TS 1353 /*! transaction: read timestamp queue entries walked */ -#define WT_STAT_CONN_TXN_READ_QUEUE_WALKED 1352 +#define WT_STAT_CONN_TXN_READ_QUEUE_WALKED 1354 /*! transaction: read timestamp queue insert to empty */ -#define WT_STAT_CONN_TXN_READ_QUEUE_EMPTY 1353 +#define WT_STAT_CONN_TXN_READ_QUEUE_EMPTY 1355 /*! transaction: read timestamp queue inserts to head */ -#define WT_STAT_CONN_TXN_READ_QUEUE_HEAD 1354 +#define WT_STAT_CONN_TXN_READ_QUEUE_HEAD 1356 /*! transaction: read timestamp queue inserts total */ -#define WT_STAT_CONN_TXN_READ_QUEUE_INSERTS 1355 +#define WT_STAT_CONN_TXN_READ_QUEUE_INSERTS 1357 /*! transaction: read timestamp queue length */ -#define WT_STAT_CONN_TXN_READ_QUEUE_LEN 1356 +#define WT_STAT_CONN_TXN_READ_QUEUE_LEN 1358 /*! transaction: rollback to stable calls */ -#define WT_STAT_CONN_TXN_ROLLBACK_TO_STABLE 1357 +#define WT_STAT_CONN_TXN_ROLLBACK_TO_STABLE 1359 /*! transaction: rollback to stable updates aborted */ -#define WT_STAT_CONN_TXN_ROLLBACK_UPD_ABORTED 1358 +#define WT_STAT_CONN_TXN_ROLLBACK_UPD_ABORTED 1360 /*! transaction: rollback to stable updates removed from cache overflow */ -#define WT_STAT_CONN_TXN_ROLLBACK_LAS_REMOVED 1359 +#define WT_STAT_CONN_TXN_ROLLBACK_LAS_REMOVED 1361 /*! transaction: set timestamp calls */ -#define WT_STAT_CONN_TXN_SET_TS 1360 +#define WT_STAT_CONN_TXN_SET_TS 1362 /*! transaction: set timestamp commit calls */ -#define WT_STAT_CONN_TXN_SET_TS_COMMIT 1361 +#define WT_STAT_CONN_TXN_SET_TS_COMMIT 1363 /*! transaction: set timestamp commit updates */ -#define WT_STAT_CONN_TXN_SET_TS_COMMIT_UPD 1362 +#define WT_STAT_CONN_TXN_SET_TS_COMMIT_UPD 1364 /*! transaction: set timestamp oldest calls */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1363 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1365 /*! transaction: set timestamp oldest updates */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1364 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1366 /*! transaction: set timestamp stable calls */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE 1365 +#define WT_STAT_CONN_TXN_SET_TS_STABLE 1367 /*! transaction: set timestamp stable updates */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1366 +#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1368 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1367 +#define WT_STAT_CONN_TXN_BEGIN 1369 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1368 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1370 /*! transaction: transaction checkpoint generation */ -#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1369 +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1371 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1370 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1372 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1371 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1373 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1372 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1374 /*! transaction: transaction checkpoint scrub dirty target */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1373 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1375 /*! transaction: transaction checkpoint scrub time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1374 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1376 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1375 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1377 /*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1376 +#define WT_STAT_CONN_TXN_CHECKPOINT 1378 /*! * transaction: transaction checkpoints skipped because database was * clean */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1377 +#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1379 /*! transaction: transaction failures due to cache overflow */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1378 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1380 /*! * transaction: transaction fsync calls for checkpoint after allocating * the transaction ID */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1379 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1381 /*! * transaction: transaction fsync duration for checkpoint after * allocating the transaction ID (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1380 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1382 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1381 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1383 /*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1382 +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1384 /*! * transaction: transaction range of IDs currently pinned by named * snapshots */ -#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1383 +#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1385 /*! transaction: transaction range of timestamps currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1384 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1386 /*! transaction: transaction range of timestamps pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1385 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1387 /*! * transaction: transaction range of timestamps pinned by the oldest * active read timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1386 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1388 /*! * transaction: transaction range of timestamps pinned by the oldest * timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1387 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1389 /*! transaction: transaction read timestamp of the oldest active reader */ -#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1388 +#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1390 /*! transaction: transaction sync calls */ -#define WT_STAT_CONN_TXN_SYNC 1389 +#define WT_STAT_CONN_TXN_SYNC 1391 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1390 +#define WT_STAT_CONN_TXN_COMMIT 1392 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1391 +#define WT_STAT_CONN_TXN_ROLLBACK 1393 /*! transaction: update conflicts */ -#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1392 +#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1394 /*! * @} diff --git a/src/third_party/wiredtiger/src/include/wt_internal.h b/src/third_party/wiredtiger/src/include/wt_internal.h index 8f18828d59b..5d2205f7718 100644 --- a/src/third_party/wiredtiger/src/include/wt_internal.h +++ b/src/third_party/wiredtiger/src/include/wt_internal.h @@ -407,17 +407,18 @@ typedef uint64_t wt_timestamp_t; #endif #include "verify_build.h" +#include "cache.i" /* required by misc.i */ #include "ctype.i" /* required by packing.i */ #include "intpack.i" /* required by cell.i, packing.i */ #include "misc.i" /* required by mutex.i */ #include "buf.i" /* required by cell.i */ -#include "cache.i" /* required by txn.i */ #include "cell.i" /* required by btree.i */ #include "mutex.i" /* required by btree.i */ #include "txn.i" /* required by btree.i */ #include "bitstring.i" +#include "block.i" #include "btree.i" /* required by cursor.i */ #include "btree_cmp.i" #include "column.i" diff --git a/src/third_party/wiredtiger/src/meta/meta_ckpt.c b/src/third_party/wiredtiger/src/meta/meta_ckpt.c index 9ac9aa11994..7b0f9a3e621 100644 --- a/src/third_party/wiredtiger/src/meta/meta_ckpt.c +++ b/src/third_party/wiredtiger/src/meta/meta_ckpt.c @@ -8,14 +8,14 @@ #include "wt_internal.h" -static int __ckpt_last(WT_SESSION_IMPL *, const char *, WT_CKPT *); -static int __ckpt_last_name(WT_SESSION_IMPL *, const char *, const char **); -static int __ckpt_load(WT_SESSION_IMPL *, +static int __ckpt_last(WT_SESSION_IMPL *, const char *, WT_CKPT *); +static int __ckpt_last_name(WT_SESSION_IMPL *, const char *, const char **); +static int __ckpt_load(WT_SESSION_IMPL *, WT_CONFIG_ITEM *, WT_CONFIG_ITEM *, WT_CKPT *); -static int __ckpt_named( +static int __ckpt_named( WT_SESSION_IMPL *, const char *, const char *, WT_CKPT *); -static int __ckpt_set(WT_SESSION_IMPL *, const char *, const char *); -static int __ckpt_version_chk(WT_SESSION_IMPL *, const char *, const char *); +static int __ckpt_set(WT_SESSION_IMPL *, const char *, const char *); +static int __ckpt_version_chk(WT_SESSION_IMPL *, const char *, const char *); /* * __wt_meta_checkpoint -- @@ -229,6 +229,83 @@ err: __wt_free(session, *namep); } /* + * __wt_meta_block_metadata -- + * Build a version of the file's metadata for the block manager to store. + */ +int +__wt_meta_block_metadata( + WT_SESSION_IMPL *session, const char *config, WT_CKPT *ckpt) +{ + WT_CONFIG_ITEM cval; + WT_DECL_ITEM(a); + WT_DECL_ITEM(b); + WT_DECL_RET; + WT_KEYED_ENCRYPTOR *kencryptor; + size_t encrypt_size, metadata_len; + const char *metadata, *filecfg[] = { + WT_CONFIG_BASE(session, file_meta), NULL, NULL }; + char *min_config; + + min_config = NULL; + WT_ERR(__wt_scr_alloc(session, 0, &a)); + WT_ERR(__wt_scr_alloc(session, 0, &b)); + + /* + * The metadata has to be encrypted because it contains private data + * (for example, column names). We pass the block manager text that + * describes the metadata (the encryption information), and the + * possibly encrypted metadata encoded as a hexadecimal string. + * configuration string. + * + * Get a minimal configuration string, just the non-default entries. + */ + WT_ERR(__wt_config_discard_defaults( + session, filecfg, config, &min_config)); + + /* Fill out the configuration array for normal retrieval. */ + filecfg[1] = config; + + /* + * Find out if this file is encrypted. If encrypting, encrypt and encode + * the minimal configuration. + */ + WT_ERR(__wt_btree_config_encryptor(session, filecfg, &kencryptor)); + if (kencryptor == NULL) { + metadata = min_config; + metadata_len = strlen(min_config); + } else { + __wt_buf_set(session, a, min_config, strlen(min_config)); + __wt_encrypt_size(session, kencryptor, a->size, &encrypt_size); + WT_ERR(__wt_buf_grow(session, b, encrypt_size)); + WT_ERR(__wt_encrypt(session, kencryptor, 0, a, b)); + WT_ERR(__wt_buf_grow(session, a, b->size * 2)); + __wt_fill_hex(b->mem, b->size, a->mem, a->memsize, &a->size); + + metadata = a->data; + metadata_len = a->size; + } + + /* + * Get a copy of the encryption information and flag if we're doing + * encryption. The latter isn't necessary, but it makes it easier to + * diagnose issues during the load. + */ + WT_ERR(__wt_config_gets(session, filecfg, "encryption", &cval)); + WT_ERR(__wt_buf_fmt(session, b, + "encryption=%.*s," + "block_metadata_encrypted=%s,block_metadata=[%.*s]", + (int)cval.len, cval.str, kencryptor == NULL ? "false" : "true", + (int)metadata_len, metadata)); + WT_ERR(__wt_strndup(session, b->data, b->size, &ckpt->block_metadata)); + +err: + __wt_free(session, min_config); + __wt_scr_free(session, &a); + __wt_scr_free(session, &b); + return (ret); +} + +/* * __ckpt_compare_order -- * Qsort comparison routine for the checkpoint list. */ @@ -248,15 +325,15 @@ __ckpt_compare_order(const void *a, const void *b) * Load all available checkpoint information for a file. */ int -__wt_meta_ckptlist_get( - WT_SESSION_IMPL *session, const char *fname, WT_CKPT **ckptbasep) +__wt_meta_ckptlist_get(WT_SESSION_IMPL *session, + const char *fname, bool update, WT_CKPT **ckptbasep) { WT_CKPT *ckpt, *ckptbase; WT_CONFIG ckptconf; WT_CONFIG_ITEM k, v; - WT_DECL_ITEM(buf); WT_DECL_RET; size_t allocated, slot; + int64_t maxorder; char *config; *ckptbasep = NULL; @@ -269,8 +346,8 @@ __wt_meta_ckptlist_get( WT_RET(__wt_metadata_search(session, fname, &config)); /* Load any existing checkpoints into the array. */ - WT_ERR(__wt_scr_alloc(session, 0, &buf)); - if (__wt_config_getones(session, config, "checkpoint", &v) == 0) { + if ((ret = + __wt_config_getones(session, config, "checkpoint", &v)) == 0) { __wt_config_subinit(session, &ckptconf, &v); for (; __wt_config_next(&ckptconf, &k, &v) == 0; ++slot) { WT_ERR(__wt_realloc_def( @@ -280,22 +357,42 @@ __wt_meta_ckptlist_get( WT_ERR(__ckpt_load(session, &k, &v, ckpt)); } } - - /* - * Allocate an extra slot for a new value, plus a slot to mark the end. - * - * This isn't very clean, but there's necessary cooperation between the - * schema layer (that maintains the list of checkpoints), the btree - * layer (that knows when the root page is written, creating a new - * checkpoint), and the block manager (which actually creates the - * checkpoint). All of that cooperation is handled in the WT_CKPT - * structure referenced from the WT_BTREE structure. - */ - WT_ERR(__wt_realloc_def(session, &allocated, slot + 2, &ckptbase)); + WT_ERR_NOTFOUND_OK(ret); + if (!update && slot == 0) + WT_ERR(WT_NOTFOUND); /* Sort in creation-order. */ __wt_qsort(ckptbase, slot, sizeof(WT_CKPT), __ckpt_compare_order); + if (update) { + /* + * Allocate an extra slot for a new value, plus a slot to mark + * mark the end. + * + * This isn't clean, but there's necessary cooperation between + * the schema layer (that maintains the list of checkpoints), + * the btree layer (that knows when the root page is written, + * creating a new checkpoint), and the block manager (which + * actually creates the checkpoint). All of that cooperation is + * handled in the array of checkpoint structures referenced from + * the WT_BTREE structure. + */ + WT_ERR(__wt_realloc_def( + session, &allocated, slot + 2, &ckptbase)); + + /* The caller may be adding a value, initialize it. */ + maxorder = 0; + WT_CKPT_FOREACH(ckptbase, ckpt) + if (ckpt->order > maxorder) + maxorder = ckpt->order; + ckpt->order = maxorder + 1; + __wt_seconds(session, &ckpt->sec); + + WT_ERR(__wt_meta_block_metadata(session, config, ckpt)); + + F_SET(ckpt, WT_CKPT_ADD); + } + /* Return the array to our caller. */ *ckptbasep = ckptbase; @@ -303,7 +400,6 @@ __wt_meta_ckptlist_get( err: __wt_meta_ckptlist_free(session, &ckptbase); } __wt_free(session, config); - __wt_scr_free(session, &buf); return (ret); } @@ -362,7 +458,7 @@ __ckpt_load(WT_SESSION_IMPL *session, ret = __wt_config_subgets(session, v, "oldest_start_txn", &a); WT_RET_NOTFOUND_OK(ret); ckpt->oldest_start_txn = - ret == WT_NOTFOUND || a.len == 0 ? WT_TS_NONE : (uint64_t)a.val; + ret == WT_NOTFOUND || a.len == 0 ? WT_TXN_NONE : (uint64_t)a.val; ret = __wt_config_subgets(session, v, "newest_stop_ts", &a); WT_RET_NOTFOUND_OK(ret); ckpt->newest_stop_ts = @@ -370,7 +466,7 @@ __ckpt_load(WT_SESSION_IMPL *session, ret = __wt_config_subgets(session, v, "newest_stop_txn", &a); WT_RET_NOTFOUND_OK(ret); ckpt->newest_stop_txn = - ret == WT_NOTFOUND || a.len == 0 ? WT_TS_MAX : (uint64_t)a.val; + ret == WT_NOTFOUND || a.len == 0 ? WT_TXN_MAX : (uint64_t)a.val; __wt_check_addr_validity(session, ckpt->oldest_start_ts, ckpt->oldest_start_txn, ckpt->newest_stop_ts, ckpt->newest_stop_txn); @@ -443,45 +539,24 @@ __ckptlist_review_write_gen(WT_SESSION_IMPL *session, WT_CKPT *ckpt) * If checkpointing the metadata file, update its write generation to * be the maximum we've seen. */ - if (WT_IS_METADATA(session->dhandle) && ckpt->write_gen < v) + if (session->dhandle != NULL && + WT_IS_METADATA(session->dhandle) && ckpt->write_gen < v) ckpt->write_gen = v; } /* - * __wt_meta_ckptlist_set -- - * Set a file's checkpoint value from the WT_CKPT list. + * __wt_meta_ckptlist_to_meta -- + * Convert a checkpoint list into its metadata representation. */ int -__wt_meta_ckptlist_set(WT_SESSION_IMPL *session, - const char *fname, WT_CKPT *ckptbase, WT_LSN *ckptlsn) +__wt_meta_ckptlist_to_meta( + WT_SESSION_IMPL *session, WT_CKPT *ckptbase, WT_ITEM *buf) { WT_CKPT *ckpt; - WT_DECL_ITEM(buf); - WT_DECL_RET; - int64_t maxorder; const char *sep; - /* - * Each internal checkpoint name is appended with a generation to make - * it a unique name. We're solving two problems: when two checkpoints - * are taken quickly, the timer may not be unique and/or we can even - * see time travel on the second checkpoint if we snapshot in-between - * nanoseconds rolling over. Second, if we reset the generational - * counter when new checkpoints arrive, we could logically re-create - * specific checkpoints, racing with cursors open on those checkpoints. - * I can't think of any way to return incorrect results by racing with - * those cursors, but it's simpler not to worry about it. - * - * Determine the current maximum checkpoint generation. - */ - maxorder = 0; - WT_CKPT_FOREACH(ckptbase, ckpt) - if (ckpt->order > maxorder) - maxorder = ckpt->order; - - WT_ERR(__wt_scr_alloc(session, 0, &buf)); sep = ""; - WT_ERR(__wt_buf_fmt(session, buf, "checkpoint=(")); + WT_RET(__wt_buf_fmt(session, buf, "checkpoint=(")); WT_CKPT_FOREACH(ckptbase, ckpt) { /* Skip deleted checkpoints. */ if (F_ISSET(ckpt, WT_CKPT_DELETE)) @@ -496,36 +571,27 @@ __wt_meta_ckptlist_set(WT_SESSION_IMPL *session, if (ckpt->raw.size == 0) ckpt->addr.size = 0; else - WT_ERR(__wt_raw_to_hex(session, + WT_RET(__wt_raw_to_hex(session, ckpt->raw.data, ckpt->raw.size, &ckpt->addr)); - - /* Set the order and timestamp. */ - if (F_ISSET(ckpt, WT_CKPT_ADD)) - ckpt->order = ++maxorder; - - __wt_seconds(session, &ckpt->sec); } __wt_check_addr_validity(session, ckpt->oldest_start_ts, ckpt->oldest_start_txn, ckpt->newest_stop_ts, ckpt->newest_stop_txn); - WT_ERR(__wt_buf_catfmt(session, buf, "%s%s", sep, ckpt->name)); + WT_RET(__wt_buf_catfmt(session, buf, "%s%s", sep, ckpt->name)); sep = ","; if (strcmp(ckpt->name, WT_CHECKPOINT) == 0) - WT_ERR(__wt_buf_catfmt(session, buf, + WT_RET(__wt_buf_catfmt(session, buf, ".%" PRId64, ckpt->order)); - /* Review the checkpoint's write generation. */ - __ckptlist_review_write_gen(session, ckpt); - /* * Use PRId64 formats: WiredTiger's configuration code handles * signed 8B values. */ - WT_ERR(__wt_buf_catfmt(session, buf, + WT_RET(__wt_buf_catfmt(session, buf, "=(addr=\"%.*s\",order=%" PRId64 ",time=%" PRIu64 ",size=%" PRId64 @@ -546,13 +612,38 @@ __wt_meta_ckptlist_set(WT_SESSION_IMPL *session, (int64_t)ckpt->newest_stop_txn, (int64_t)ckpt->write_gen)); } - WT_ERR(__wt_buf_catfmt(session, buf, ")")); + WT_RET(__wt_buf_catfmt(session, buf, ")")); + + return (0); +} + +/* + * __wt_meta_ckptlist_set -- + * Set a file's checkpoint value from the WT_CKPT list. + */ +int +__wt_meta_ckptlist_set(WT_SESSION_IMPL *session, + const char *fname, WT_CKPT *ckptbase, WT_LSN *ckptlsn) +{ + WT_CKPT *ckpt; + WT_DECL_ITEM(buf); + WT_DECL_RET; + + WT_RET(__wt_scr_alloc(session, 1024, &buf)); + + WT_ERR(__wt_meta_ckptlist_to_meta(session, ckptbase, buf)); + if (ckptlsn != NULL) WT_ERR(__wt_buf_catfmt(session, buf, ",checkpoint_lsn=(%" PRIu32 ",%" PRIuMAX ")", ckptlsn->l.file, (uintmax_t)ckptlsn->l.offset)); + WT_ERR(__ckpt_set(session, fname, buf->mem)); + /* Review the checkpoint's write generation. */ + WT_CKPT_FOREACH(ckptbase, ckpt) + __ckptlist_review_write_gen(session, ckpt); + err: __wt_scr_free(session, &buf); return (ret); } @@ -585,6 +676,8 @@ __wt_meta_checkpoint_free(WT_SESSION_IMPL *session, WT_CKPT *ckpt) return; __wt_free(session, ckpt->name); + __wt_free(session, ckpt->block_metadata); + __wt_free(session, ckpt->block_checkpoint); __wt_buf_free(session, &ckpt->addr); __wt_buf_free(session, &ckpt->raw); __wt_free(session, ckpt->bpriv); diff --git a/src/third_party/wiredtiger/src/meta/meta_ext.c b/src/third_party/wiredtiger/src/meta/meta_ext.c index 9404bd016a0..80d0d7c2d5a 100644 --- a/src/third_party/wiredtiger/src/meta/meta_ext.c +++ b/src/third_party/wiredtiger/src/meta/meta_ext.c @@ -91,7 +91,7 @@ __wt_metadata_get_ckptlist( WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { return (__wt_meta_ckptlist_get( - (WT_SESSION_IMPL *)session, name, ckptbasep)); + (WT_SESSION_IMPL *)session, name, false, ckptbasep)); } /* diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index 84f07e317ff..51cabeda029 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -9,6 +9,7 @@ #include "wt_internal.h" static void __rec_cleanup(WT_SESSION_IMPL *, WT_RECONCILE *); +static void __rec_destroy(WT_SESSION_IMPL *, void *); static int __rec_destroy_session(WT_SESSION_IMPL *); static int __rec_init(WT_SESSION_IMPL *, WT_REF *, uint32_t, WT_SALVAGE_COOKIE *, void *); @@ -513,6 +514,7 @@ __rec_init(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags, WT_SALVAGE_COOKIE *salvage, void *reconcilep) { WT_BTREE *btree; + WT_DECL_RET; WT_PAGE *page; WT_RECONCILE *r; WT_TXN_GLOBAL *txn_global; @@ -522,8 +524,6 @@ __rec_init(WT_SESSION_IMPL *session, if ((r = *(WT_RECONCILE **)reconcilep) == NULL) { WT_RET(__wt_calloc_one(session, &r)); - - *(WT_RECONCILE **)reconcilep = r; session->reconcile_cleanup = __rec_destroy_session; /* Connect pointers/buffers. */ @@ -575,6 +575,12 @@ __rec_init(WT_SESSION_IMPL *session, * history, or the stable timestamp hasn't changed since last time this * page was successfully, skew oldest instead. */ + if (F_ISSET(S2C(session)->cache, WT_CACHE_EVICT_DEBUG_MODE) && + __wt_random(&session->rnd) % 3 == 0) + r->las_skew_newest = false; + else + r->las_skew_newest = + LF_ISSET(WT_REC_LOOKASIDE) && LF_ISSET(WT_REC_VISIBLE_ALL); r->las_skew_newest = LF_ISSET(WT_REC_LOOKASIDE) && LF_ISSET(WT_REC_VISIBLE_ALL); if (r->las_skew_newest && @@ -675,7 +681,7 @@ __rec_init(WT_SESSION_IMPL *session, * Sanity check the size: 100 slots is the smallest dictionary we use. */ if (btree->dictionary != 0 && btree->dictionary > r->dictionary_slots) - WT_RET(__wt_rec_dictionary_init(session, + WT_ERR(__wt_rec_dictionary_init(session, r, btree->dictionary < 100 ? 100 : btree->dictionary)); __wt_rec_dictionary_reset(r); @@ -716,7 +722,20 @@ __rec_init(WT_SESSION_IMPL *session, r->update_modify_cbt.ref = ref; r->update_modify_cbt.iface.value_format = btree->value_format; - return (0); + /* + * If we allocated the reconciliation structure and there was an error, + * clean up. If our caller passed in a structure, they own it. + */ +err: if (*(WT_RECONCILE **)reconcilep == NULL) { + if (ret == 0) + *(WT_RECONCILE **)reconcilep = r; + else { + __rec_cleanup(session, r); + __rec_destroy(session, &r); + } + } + + return (ret); } /* @@ -2139,6 +2158,7 @@ int __wt_bulk_wrapup(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) { WT_BTREE *btree; + WT_DECL_RET; WT_PAGE *parent; WT_RECONCILE *r; @@ -2155,25 +2175,25 @@ __wt_bulk_wrapup(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) break; case BTREE_COL_VAR: if (cbulk->rle != 0) - WT_RET(__wt_bulk_insert_var(session, cbulk, false)); + WT_ERR(__wt_bulk_insert_var(session, cbulk, false)); break; case BTREE_ROW: break; } - WT_RET(__wt_rec_split_finish(session, r)); - WT_RET(__rec_write_wrapup(session, r, r->page)); + WT_ERR(__wt_rec_split_finish(session, r)); + WT_ERR(__rec_write_wrapup(session, r, r->page)); __rec_write_page_status(session, r); /* Mark the page's parent and the tree dirty. */ parent = r->ref->home; - WT_RET(__wt_page_modify_init(session, parent)); + WT_ERR(__wt_page_modify_init(session, parent)); __wt_page_modify_set(session, parent); - __rec_cleanup(session, r); +err: __rec_cleanup(session, r); __rec_destroy(session, &cbulk->reconcile); - return (0); + return (ret); } /* @@ -2373,11 +2393,11 @@ __rec_write_wrapup(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) */ ref = r->ref; if (__wt_ref_is_root(ref)) { - WT_RET(bm->checkpoint( - bm, session, NULL, btree->ckpt, false)); __wt_checkpoint_tree_reconcile_update(session, WT_TS_NONE, WT_TS_NONE, WT_TXN_NONE, WT_TS_MAX, WT_TXN_MAX); + WT_RET(bm->checkpoint( + bm, session, NULL, btree->ckpt, false)); } /* @@ -2419,16 +2439,16 @@ __rec_write_wrapup(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) r->multi->disk_image = NULL; mod->mod_page_las = r->multi->page_las; } else { - WT_RET(__wt_bt_write(session, r->wrapup_checkpoint, - NULL, NULL, NULL, - true, F_ISSET(r, WT_REC_CHECKPOINT), - r->wrapup_checkpoint_compressed)); __wt_checkpoint_tree_reconcile_update(session, r->multi->addr.newest_durable_ts, r->multi->addr.oldest_start_ts, r->multi->addr.oldest_start_txn, r->multi->addr.newest_stop_ts, r->multi->addr.newest_stop_txn); + WT_RET(__wt_bt_write(session, r->wrapup_checkpoint, + NULL, NULL, NULL, + true, F_ISSET(r, WT_REC_CHECKPOINT), + r->wrapup_checkpoint_compressed)); } mod->rec_result = WT_PM_REC_REPLACE; diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c index 7d3b4b9d81a..4e1742a89db 100644 --- a/src/third_party/wiredtiger/src/session/session_api.c +++ b/src/third_party/wiredtiger/src/session/session_api.c @@ -1115,6 +1115,65 @@ err: API_END_RET(session, ret); } /* + * __session_import -- + * WT_SESSION->import method. + */ +static int +__session_import(WT_SESSION *wt_session, const char *uri, const char *config) +{ + WT_DECL_RET; + WT_SESSION_IMPL *session; + char *value; + + WT_UNUSED(config); + + session = (WT_SESSION_IMPL *)wt_session; + SESSION_API_CALL_NOCONF(session, import); + + WT_ERR(__wt_inmem_unsupported_op(session, NULL)); + + if (!WT_PREFIX_MATCH(uri, "file:")) + WT_ERR(__wt_bad_object_type(session, uri)); + + if ((ret = __wt_metadata_search(session, uri, &value)) == 0) + WT_ERR_MSG(session, EINVAL, + "an object named \"%s\" already exists in the database", + uri); + WT_ERR_NOTFOUND_OK(ret); + + WT_ERR(__wt_import(session, uri)); + +err: + if (ret != 0) + WT_STAT_CONN_INCR(session, session_table_import_fail); + else + WT_STAT_CONN_INCR(session, session_table_import_success); + API_END_RET_NOTFOUND_MAP(session, ret); +} + +/* + * __session_import_readonly -- + * WT_SESSION->import method; readonly version. + */ +static int +__session_import_readonly( + WT_SESSION *wt_session, const char *uri, const char *config) +{ + WT_DECL_RET; + WT_SESSION_IMPL *session; + + WT_UNUSED(uri); + WT_UNUSED(config); + + session = (WT_SESSION_IMPL *)wt_session; + SESSION_API_CALL_NOCONF(session, import); + + WT_STAT_CONN_INCR(session, session_table_import_fail); + ret = __wt_session_notsup(session); +err: API_END_RET(session, ret); +} + +/* * __session_join -- * WT_SESSION->join method. */ @@ -2092,6 +2151,7 @@ __open_session(WT_CONNECTION_IMPL *conn, __session_open_cursor, __session_alter, __session_create, + __session_import, __wt_session_compact, __session_drop, __session_join, @@ -2124,6 +2184,7 @@ __open_session(WT_CONNECTION_IMPL *conn, __session_open_cursor, __session_alter_readonly, __session_create_readonly, + __session_import_readonly, __wt_session_compact_readonly, __session_drop_readonly, __session_join, diff --git a/src/third_party/wiredtiger/src/session/session_salvage.c b/src/third_party/wiredtiger/src/session/session_salvage.c deleted file mode 100644 index 79e67475958..00000000000 --- a/src/third_party/wiredtiger/src/session/session_salvage.c +++ /dev/null @@ -1,59 +0,0 @@ -/*- - * Copyright (c) 2014-2019 MongoDB, Inc. - * Copyright (c) 2008-2014 WiredTiger, Inc. - * All rights reserved. - * - * See the file LICENSE for redistribution information. - */ - -#include "wt_internal.h" - -/* - * __wt_salvage -- - * Salvage a single file. - */ -int -__wt_salvage(WT_SESSION_IMPL *session, const char *cfg[]) -{ - WT_CKPT *ckptbase; - WT_DATA_HANDLE *dhandle; - WT_DECL_RET; - - dhandle = session->dhandle; - - /* - * XXX - * The salvage process reads and discards previous checkpoints, so the - * underlying block manager has to ignore any previous checkpoint - * entries when creating a new checkpoint, in other words, we can't use - * the metadata checkpoint list, it has all of those checkpoint listed - * and we don't care about them. Build a clean checkpoint list and use - * it instead. - * - * Don't first clear the metadata checkpoint list and call the function - * to get a list of checkpoints: a crash between clearing the metadata - * checkpoint list and creating a new checkpoint list would look like a - * create or open of a file without a checkpoint to roll-forward from, - * and the contents of the file would be discarded. - */ - WT_RET(__wt_calloc_def(session, 2, &ckptbase)); - WT_ERR(__wt_strdup(session, WT_CHECKPOINT, &ckptbase[0].name)); - F_SET(&ckptbase[0], WT_CKPT_ADD); - - WT_ERR(__wt_bt_salvage(session, ckptbase, cfg)); - - /* - * If no checkpoint was created, well, it's probably bad news, but there - * is nothing to do but clear any recorded checkpoints for the file. If - * a checkpoint was created, life is good, replace any existing list of - * checkpoints with the single new one. - */ - if (ckptbase[0].raw.data == NULL) - WT_ERR(__wt_meta_checkpoint_clear(session, dhandle->name)); - else - WT_ERR(__wt_meta_ckptlist_set( - session, dhandle->name, ckptbase, NULL)); - -err: __wt_meta_ckptlist_free(session, &ckptbase); - return (ret); -} diff --git a/src/third_party/wiredtiger/src/support/huffman.c b/src/third_party/wiredtiger/src/support/huffman.c index b7bd9e24bb6..85e0800e170 100644 --- a/src/third_party/wiredtiger/src/support/huffman.c +++ b/src/third_party/wiredtiger/src/support/huffman.c @@ -261,8 +261,9 @@ static void make_table(WT_SESSION_IMPL *session, uint8_t *code2symbol, uint16_t max_depth, WT_HUFFMAN_CODE *codes, u_int symcnt) { + u_int i; uint32_t j, c1, c2; /* Exceeds uint16_t bounds at loop boundary. */ - uint16_t c, i; + uint16_t c; uint8_t len, shift; /* Zero out, for assertion below. */ @@ -321,8 +322,8 @@ __wt_huffman_open(WT_SESSION_IMPL *session, WT_DECL_RET; WT_FREQTREE_NODE *node, *node2, **refnode, *tempnode; WT_HUFFMAN_OBJ *huffman; + u_int i; uint64_t w1, w2; - uint16_t i; indexed_freqs = NULL; combined_nodes = leaves = NULL; diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index 32241675a9e..e76930c3b38 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -1074,6 +1074,8 @@ static const char * const __stats_connection_desc[] = { "session: table create successful calls", "session: table drop failed calls", "session: table drop successful calls", + "session: table import failed calls", + "session: table import successful calls", "session: table rebalance failed calls", "session: table rebalance successful calls", "session: table rename failed calls", @@ -1509,6 +1511,8 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) /* not clearing session_table_create_success */ /* not clearing session_table_drop_fail */ /* not clearing session_table_drop_success */ + /* not clearing session_table_import_fail */ + /* not clearing session_table_import_success */ /* not clearing session_table_rebalance_fail */ /* not clearing session_table_rebalance_success */ /* not clearing session_table_rename_fail */ @@ -2064,6 +2068,10 @@ __wt_stat_connection_aggregate( WT_STAT_READ(from, session_table_drop_fail); to->session_table_drop_success += WT_STAT_READ(from, session_table_drop_success); + to->session_table_import_fail += + WT_STAT_READ(from, session_table_import_fail); + to->session_table_import_success += + WT_STAT_READ(from, session_table_import_success); to->session_table_rebalance_fail += WT_STAT_READ(from, session_table_rebalance_fail); to->session_table_rebalance_success += diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index 4056d33a2b7..2449c1ccabe 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -529,9 +529,11 @@ __wt_txn_config(WT_SESSION_IMPL *session, const char *cfg[]) /* Check if prepared updates should be ignored during reads. */ WT_RET(__wt_config_gets_def(session, cfg, "ignore_prepare", 0, &cval)); - if (cval.val || - (cval.len > 0 && WT_STRING_MATCH("force", cval.str, cval.len))) + if (cval.len > 0 && + WT_STRING_MATCH("force", cval.str, cval.len)) F_SET(txn, WT_TXN_IGNORE_PREPARE); + else if (cval.val) + F_SET(txn, WT_TXN_IGNORE_PREPARE | WT_TXN_READONLY); /* * Check if the prepare timestamp and the commit timestamp of a @@ -636,9 +638,6 @@ __wt_txn_release(WT_SESSION_IMPL *session) */ __wt_txn_release_snapshot(session); txn->isolation = session->isolation; -#ifdef HAVE_DIAGNOSTIC - txn->multi_update_count = 0; -#endif txn->rollback_reason = NULL; @@ -801,6 +800,7 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) WT_TXN_OP *op; WT_UPDATE *upd; wt_timestamp_t prev_commit_timestamp; + int64_t resolved_update_count, visited_update_count; uint32_t fileid; u_int i; bool locked, prepare, readonly, update_timestamp; @@ -810,6 +810,7 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) txn_global = &conn->txn_global; prev_commit_timestamp = 0; /* -Wconditional-uninitialized */ locked = false; + resolved_update_count = visited_update_count = 0; WT_ASSERT(session, F_ISSET(txn, WT_TXN_RUNNING)); WT_ASSERT(session, !F_ISSET(txn, WT_TXN_ERROR) || @@ -978,8 +979,27 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) __wt_txn_op_set_timestamp(session, op); } else { - WT_ERR(__wt_txn_resolve_prepared_op( - session, op, true)); + visited_update_count++; + /* + * If we have set the key repeated flag + * we can skip resolving prepared updates as + * it would have happened on a previous + * modification in this txn. + */ + if (!F_ISSET(op, WT_TXN_OP_KEY_REPEATED)) + WT_ERR(__wt_txn_resolve_prepared_op( + session, op, true, + &resolved_update_count)); + /* + * We should resolve at least one or more + * updates each time we call + * __wt_txn_resolve_prepared_op, as such + * resolved update count should never be less + * than visited update count. + */ + WT_ASSERT(session, + resolved_update_count >= + visited_update_count); } break; @@ -994,12 +1014,10 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) __wt_txn_op_free(session, op); } + WT_ASSERT(session, resolved_update_count == visited_update_count); + WT_STAT_CONN_INCRV(session, txn_prepared_updates_resolved, + resolved_update_count); - /* - * FIXME: I think we want to say that all prepared updates were - * resolved. - * WT_ASSERT(session, txn->multi_update_count == 0); - */ txn->mod_count = 0; /* @@ -1141,6 +1159,13 @@ __wt_txn_prepare(WT_SESSION_IMPL *session, const char *cfg[]) WT_PUBLISH(upd->prepare_state, WT_PREPARE_INPROGRESS); op->u.op_upd = NULL; WT_STAT_CONN_INCR(session, txn_prepared_updates_count); + /* + * Set the key repeated flag which tells us that we've + * got multiple updates to the same key by the same txn. + * This is later used in txn commit. + */ + if (upd->next != NULL && upd->txnid == upd->next->txnid) + F_SET(op, WT_TXN_OP_KEY_REPEATED); break; case WT_TXN_OP_REF_DELETE: __wt_txn_op_apply_prepare_state( @@ -1180,11 +1205,12 @@ __wt_txn_rollback(WT_SESSION_IMPL *session, const char *cfg[]) WT_TXN *txn; WT_TXN_OP *op; WT_UPDATE *upd; + int64_t resolved_update_count, visited_update_count; u_int i; bool readonly; WT_UNUSED(cfg); - + resolved_update_count = visited_update_count = 0; txn = &session->txn; readonly = txn->mod_count == 0; WT_ASSERT(session, F_ISSET(txn, WT_TXN_RUNNING)); @@ -1218,10 +1244,29 @@ __wt_txn_rollback(WT_SESSION_IMPL *session, const char *cfg[]) * Need to resolve indirect references of transaction * operation, in case of prepared transaction. */ - if (F_ISSET(txn, WT_TXN_PREPARE)) - WT_RET(__wt_txn_resolve_prepared_op( - session, op, false)); - else { + if (F_ISSET(txn, WT_TXN_PREPARE)) { + visited_update_count++; + /* + * If we have set the key repeated flag + * we can skip resolving prepared updates as + * it would have happened on a previous + * modification in this txn. + */ + if (!F_ISSET(op, WT_TXN_OP_KEY_REPEATED)) + WT_RET(__wt_txn_resolve_prepared_op( + session, op, false, + &resolved_update_count)); + /* + * We should resolve at least one or more + * updates each time we call + * __wt_txn_resolve_prepared_op, as such + * resolved update count should never be less + * than visited update count. + */ + WT_ASSERT(session, + resolved_update_count >= + visited_update_count); + } else { WT_ASSERT(session, upd->txnid == txn->id || upd->txnid == WT_TXN_ABORTED); upd->txnid = WT_TXN_ABORTED; @@ -1243,6 +1288,10 @@ __wt_txn_rollback(WT_SESSION_IMPL *session, const char *cfg[]) __wt_txn_op_free(session, op); } + WT_ASSERT(session, resolved_update_count == visited_update_count); + WT_STAT_CONN_INCRV(session, txn_prepared_updates_resolved, + resolved_update_count); + txn->mod_count = 0; __wt_txn_release(session); diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c index 143bec4e445..0b4e9394f9f 100644 --- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c +++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c @@ -1400,7 +1400,7 @@ __checkpoint_lock_dirty_tree(WT_SESSION_IMPL *session, WT_IS_METADATA(dhandle) || WT_META_TRACKING(session)); /* Get the list of checkpoints for this file. */ - WT_RET(__wt_meta_ckptlist_get(session, dhandle->name, &ckptbase)); + WT_RET(__wt_meta_ckptlist_get(session, dhandle->name, true, &ckptbase)); /* This may be a named checkpoint, check the configuration. */ cval.len = 0; @@ -1449,16 +1449,10 @@ __checkpoint_lock_dirty_tree(WT_SESSION_IMPL *session, /* Drop checkpoints with the same name as the one we're taking. */ __drop(ckptbase, name, strlen(name)); - /* Add a new checkpoint entry at the end of the list. */ + /* Set the name of the new entry at the end of the list. */ WT_CKPT_FOREACH(ckptbase, ckpt) ; WT_ERR(__wt_strdup(session, name, &ckpt->name)); - /* - * We are now done with the local use of the name. Free the local - * allocation, if needed. - */ - __wt_free(session, name_alloc); - F_SET(ckpt, WT_CKPT_ADD); /* * There is some interaction between backups and checkpoints. Perform @@ -1476,9 +1470,10 @@ __checkpoint_lock_dirty_tree(WT_SESSION_IMPL *session, !F_ISSET(btree, WT_BTREE_SKIP_CKPT)); btree->ckpt = ckptbase; - return (0); - -err: __wt_meta_ckptlist_free(session, &ckptbase); + if (0) { +err: + __wt_meta_ckptlist_free(session, &ckptbase); + } __wt_free(session, name_alloc); return (ret); diff --git a/src/third_party/wiredtiger/src/txn/txn_log.c b/src/third_party/wiredtiger/src/txn/txn_log.c index 6398cea7e4e..b9d9c26d941 100644 --- a/src/third_party/wiredtiger/src/txn/txn_log.c +++ b/src/third_party/wiredtiger/src/txn/txn_log.c @@ -38,6 +38,7 @@ __txn_op_log_row_key_check(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) if (cbt->ins == NULL) { session = (WT_SESSION_IMPL *)cbt->iface.session; page = cbt->ref->page; + WT_ASSERT(session, cbt->slot < page->entries); rip = &page->pg_row[cbt->slot]; WT_ASSERT(session, __wt_row_leaf_key(session, page, rip, &key, false) == 0); @@ -183,6 +184,7 @@ __wt_txn_op_free(WT_SESSION_IMPL *session, WT_TXN_OP *op) (void)__wt_atomic_subi32(&op->btree->dhandle->session_inuse, 1); op->type = WT_TXN_OP_NONE; + op->flags = 0; } /* diff --git a/src/third_party/wiredtiger/src/txn/txn_recover.c b/src/third_party/wiredtiger/src/txn/txn_recover.c index b68a4415e76..ff4c2b8b9cb 100644 --- a/src/third_party/wiredtiger/src/txn/txn_recover.c +++ b/src/third_party/wiredtiger/src/txn/txn_recover.c @@ -457,6 +457,11 @@ __recovery_setup_file(WT_RECOVERY *r, const char *uri, const char *config) r->nfiles = fileid + 1; } + if (r->files[fileid].uri != NULL) + WT_PANIC_RET(r->session, WT_PANIC, + "metadata corruption: files %s and %s have the same " + "file ID %u", + uri, r->files[fileid].uri, fileid); WT_RET(__wt_strdup(r->session, uri, &r->files[fileid].uri)); WT_RET( __wt_config_getones(r->session, config, "checkpoint_lsn", &cval)); diff --git a/src/third_party/wiredtiger/src/txn/txn_timestamp.c b/src/third_party/wiredtiger/src/txn/txn_timestamp.c index 7e26bfe8fe5..6046829def3 100644 --- a/src/third_party/wiredtiger/src/txn/txn_timestamp.c +++ b/src/third_party/wiredtiger/src/txn/txn_timestamp.c @@ -928,10 +928,22 @@ __wt_txn_set_read_timestamp( did_roundup_to_oldest = true; } else { __wt_readunlock(session, &txn_global->rwlock); - WT_RET_MSG(session, EINVAL, "read timestamp " + + /* + * In some cases, MongoDB sets a read timestamp older + * than the oldest timestamp, relying on WiredTiger's + * concurrency to detect and fail the set. In other + * cases it's a bug and MongoDB wants error context to + * make it easier to find those problems. Don't output + * an error message because that logs a MongoDB error, + * use an informational message to provide the context + * instead. + */ + WT_RET(__wt_msg(session, "read timestamp " "%s less than the oldest timestamp %s", __wt_timestamp_to_string(read_ts, ts_string[0]), - __wt_timestamp_to_string(ts_oldest, ts_string[1])); + __wt_timestamp_to_string(ts_oldest, ts_string[1]))); + return (EINVAL); } } else txn->read_timestamp = read_ts; diff --git a/src/third_party/wiredtiger/src/utilities/util.h b/src/third_party/wiredtiger/src/utilities/util.h index be58f8eae0e..3b12d9be98a 100644 --- a/src/third_party/wiredtiger/src/utilities/util.h +++ b/src/third_party/wiredtiger/src/utilities/util.h @@ -38,6 +38,7 @@ int util_dump(WT_SESSION *, int, char *[]); int util_err(WT_SESSION *, int, const char *, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))); int util_flush(WT_SESSION *, const char *); +int util_import(WT_SESSION *, int, char *[]); int util_list(WT_SESSION *, int, char *[]); int util_load(WT_SESSION *, int, char *[]); int util_loadtext(WT_SESSION *, int, char *[]); diff --git a/src/third_party/wiredtiger/src/utilities/util_import.c b/src/third_party/wiredtiger/src/utilities/util_import.c new file mode 100644 index 00000000000..995a09cfb93 --- /dev/null +++ b/src/third_party/wiredtiger/src/utilities/util_import.c @@ -0,0 +1,49 @@ +/*- + * Copyright (c) 2014-2019 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "util.h" + +static int +usage(void) +{ + (void)fprintf(stderr, + "usage: %s %s " + "import uri\n", + progname, usage_prefix); + return (1); +} + +int +util_import(WT_SESSION *session, int argc, char *argv[]) +{ + WT_DECL_RET; + int ch; + char *uri; + + uri = NULL; + while ((ch = __wt_getopt(progname, argc, argv, "")) != EOF) + switch (ch) { + case '?': + default: + return (usage()); + } + argc -= __wt_optind; + argv += __wt_optind; + + /* The remaining argument is the file URI. */ + if (argc != 1) + return (usage()); + if ((uri = util_uri(session, *argv, "file")) == NULL) + return (1); + + if ((ret = session->import(session, uri, NULL)) != 0) + (void)util_err(session, ret, "WT_SESSION.import: %s", uri); + + free(uri); + return (ret); +} diff --git a/src/third_party/wiredtiger/src/utilities/util_list.c b/src/third_party/wiredtiger/src/utilities/util_list.c index e34fee663dc..ede0a91f979 100644 --- a/src/third_party/wiredtiger/src/utilities/util_list.c +++ b/src/third_party/wiredtiger/src/utilities/util_list.c @@ -71,27 +71,31 @@ list_get_allocsize(WT_SESSION *session, const char *key, size_t *allocsize) *allocsize = 0; + parser = NULL; + config = NULL; + wt_api = session->connection->get_extension_api(session->connection); if ((ret = wt_api->metadata_search(wt_api, session, key, &config)) != 0) - return (util_err( + WT_ERR(util_err( session, ret, "%s: WT_EXTENSION_API.metadata_search", key)); if ((ret = wt_api->config_parser_open(wt_api, session, config, strlen(config), &parser)) != 0) - return (util_err( + WT_ERR(util_err( session, ret, "WT_EXTENSION_API.config_parser_open")); - if ((ret = parser->get(parser, "allocation_size", &szvalue)) != 0) { - if (ret == WT_NOTFOUND) - ret = 0; - else - ret = util_err(session, ret, "WT_CONFIG_PARSER.get"); - if ((tret = parser->close(parser)) != 0) - (void)util_err(session, tret, "WT_CONFIG_PARSER.close"); - return (ret); + if ((ret = parser->get(parser, "allocation_size", &szvalue)) == 0) + *allocsize = (size_t)szvalue.val; + else + ret = ret == WT_NOTFOUND ? + 0 : util_err(session, ret, "WT_CONFIG_PARSER.get"); +err: + if (parser != NULL && (tret = parser->close(parser)) != 0) { + tret = util_err(session, tret, "WT_CONFIG_PARSER.close"); + if (ret == 0) + ret = tret; } - if ((ret = parser->close(parser)) != 0) - return (util_err(session, ret, "WT_CONFIG_PARSER.close")); - *allocsize = (size_t)szvalue.val; - return (0); + + free(config); + return (ret); } /* @@ -172,6 +176,27 @@ list_print(WT_SESSION *session, const char *uri, bool cflag, bool vflag) } /* + * list_print_size -- + * List a size found in the checkpoint information. + */ +static void +list_print_size(uint64_t v) +{ + if (v >= WT_PETABYTE) + printf("%" PRIu64 " PB", v / WT_PETABYTE); + else if (v >= WT_TERABYTE) + printf("%" PRIu64 " TB", v / WT_TERABYTE); + else if (v >= WT_GIGABYTE) + printf("%" PRIu64 " GB", v / WT_GIGABYTE); + else if (v >= WT_MEGABYTE) + printf("%" PRIu64 " MB", v / WT_MEGABYTE); + else if (v >= WT_KILOBYTE) + printf("%" PRIu64 " KB", v / WT_KILOBYTE); + else + printf("%" PRIu64 " B", v); +} + +/* * list_print_checkpoint -- * List the checkpoint information. */ @@ -183,7 +208,6 @@ list_print_checkpoint(WT_SESSION *session, const char *key) WT_DECL_RET; size_t allocsize, len; time_t t; - uint64_t v; /* * We may not find any checkpoints for this file, in which case we don't @@ -210,38 +234,55 @@ list_print_checkpoint(WT_SESSION *session, const char *key) * Call ctime, not ctime_r; ctime_r has portability problems, * the Solaris version is different from the POSIX standard. */ + if (ckpt != ckptbase) + printf("\n"); t = (time_t)ckpt->sec; printf("\t%*s: %.24s", (int)len, ckpt->name, ctime(&t)); - v = ckpt->size; - if (v >= WT_PETABYTE) - printf(" (%" PRIu64 " PB)\n", v / WT_PETABYTE); - else if (v >= WT_TERABYTE) - printf(" (%" PRIu64 " TB)\n", v / WT_TERABYTE); - else if (v >= WT_GIGABYTE) - printf(" (%" PRIu64 " GB)\n", v / WT_GIGABYTE); - else if (v >= WT_MEGABYTE) - printf(" (%" PRIu64 " MB)\n", v / WT_MEGABYTE); - else if (v >= WT_KILOBYTE) - printf(" (%" PRIu64 " KB)\n", v / WT_KILOBYTE); - else - printf(" (%" PRIu64 " B)\n", v); + printf(" (size "); + list_print_size(ckpt->size); + printf(")\n"); /* Decode the checkpoint block. */ if (ckpt->raw.data == NULL) continue; if ((ret = __wt_block_ckpt_decode( session, allocsize, ckpt->raw.data, &ci)) == 0) { - printf("\t\t" "root offset: %" PRIuMAX - " (0x%" PRIxMAX ")\n", - (uintmax_t)ci.root_offset, - (uintmax_t)ci.root_offset); - printf("\t\t" "root size: %" PRIu32 - " (0x%" PRIx32 ")\n", - ci.root_size, ci.root_size); - printf("\t\t" "root checksum: %" PRIu32 - " (0x%" PRIx32 ")\n", + printf("\t\t" "file-size: "); + list_print_size((uint64_t)ci.file_size); + printf(", checkpoint-size: "); + list_print_size(ci.ckpt_size); + printf("\n\n"); + + printf("\t\t" " offset, size, checksum\n"); + printf( + "\t\t" "root " + ": %" PRIuMAX + ", %" PRIu32 + ", %" PRIu32 " (%#" PRIx32 ")\n", + (uintmax_t)ci.root_offset, ci.root_size, ci.root_checksum, ci.root_checksum); + printf( + "\t\t" "alloc " + ": %" PRIuMAX + ", %" PRIu32 + ", %" PRIu32 " (%#" PRIx32 ")\n", + (uintmax_t)ci.alloc.offset, ci.alloc.size, + ci.alloc.checksum, ci.alloc.checksum); + printf( + "\t\t" "discard " + ": %" PRIuMAX + ", %" PRIu32 + ", %" PRIu32 " (%#" PRIx32 ")\n", + (uintmax_t)ci.discard.offset, ci.discard.size, + ci.discard.checksum, ci.discard.checksum); + printf( + "\t\t" "avail " + ": %" PRIuMAX + ", %" PRIu32 + ", %" PRIu32 " (%#" PRIx32 ")\n", + (uintmax_t)ci.avail.offset, ci.avail.size, + ci.avail.checksum, ci.avail.checksum); } else { /* Ignore the error and continue if damaged. */ (void)util_err(session, ret, "__wt_block_ckpt_decode"); diff --git a/src/third_party/wiredtiger/src/utilities/util_main.c b/src/third_party/wiredtiger/src/utilities/util_main.c index 81ce18c3598..fb2b1990166 100644 --- a/src/third_party/wiredtiger/src/utilities/util_main.c +++ b/src/third_party/wiredtiger/src/utilities/util_main.c @@ -47,6 +47,10 @@ usage(void) "\t" "downgrade downgrade a database\n" "\t" "drop\t drop an object\n" "\t" "dump\t dump an object\n" + /* + * Import is not documented. + * "\t" "import\t import an object\n" + */ "\t" "list\t list database objects\n" "\t" "load\t load an object\n" "\t" "loadtext load an object from a text file\n" @@ -193,6 +197,10 @@ main(int argc, char *argv[]) else if (strcmp(command, "dump") == 0) func = util_dump; break; + case 'i': + if (strcmp(command, "import") == 0) + func = util_import; + break; case 'l': if (strcmp(command, "list") == 0) func = util_list; diff --git a/src/third_party/wiredtiger/test/checkpoint/smoke.sh b/src/third_party/wiredtiger/test/checkpoint/smoke.sh index dba60babb92..e1bc3819c74 100755 --- a/src/third_party/wiredtiger/test/checkpoint/smoke.sh +++ b/src/third_party/wiredtiger/test/checkpoint/smoke.sh @@ -25,6 +25,9 @@ echo "checkpoint: 6 row-store tables, named checkpoint" $TEST_WRAPPER ./t -c 'TeSt' -T 6 -t r echo "checkpoint: row-store tables, stress LAS. Sweep and timestamps" +$TEST_WRAPPER ./t -t r -W 3 -r 2 -D -s -x -n 100000 -k 100000 -C cache_size=100MB + +echo "checkpoint: row-store tables, Sweep and timestamps" $TEST_WRAPPER ./t -t r -W 3 -r 2 -s -x -n 100000 -k 100000 -C cache_size=100MB echo "checkpoint: 3 mixed tables, with sweep" diff --git a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c index 461b6334b27..c30af666c5c 100644 --- a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c +++ b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c @@ -57,6 +57,7 @@ main(int argc, char *argv[]) working_dir = NULL; ttype = MIX; g.checkpoint_name = "WiredTigerCheckpoint"; + g.debug_mode = false; g.home = dmalloc(512); g.nkeys = 10000; g.nops = 100000; @@ -66,7 +67,7 @@ main(int argc, char *argv[]) runs = 1; while ((ch = __wt_getopt( - progname, argc, argv, "C:c:h:k:l:n:r:sT:t:W:x")) != EOF) + progname, argc, argv, "C:c:Dh:k:l:n:r:sT:t:W:x")) != EOF) switch (ch) { case 'c': g.checkpoint_name = __wt_optarg; @@ -74,6 +75,9 @@ main(int argc, char *argv[]) case 'C': /* wiredtiger_open config */ config_open = __wt_optarg; break; + case 'D': + g.debug_mode = true; + break; case 'h': /* wiredtiger_open config */ working_dir = __wt_optarg; break; @@ -182,6 +186,8 @@ main(int argc, char *argv[]) return (g.status); } +#define DEBUG_MODE_CFG \ +",debug_mode=(eviction=true,table_logging=true)" /* * wt_connect -- * Configure the WiredTiger connection. @@ -208,16 +214,18 @@ wt_connect(const char *config_open) "statistics_log=(json,wait=1),error_prefix=\"%s\"," \ "file_manager=(close_handle_minimum=1,close_idle_time=1,"\ "close_scan_interval=1),log=(enabled),cache_size=1GB,"\ - "timing_stress_for_test=(aggressive_sweep)%s%s", + "timing_stress_for_test=(aggressive_sweep)%s%s%s", progname, + g.debug_mode ? DEBUG_MODE_CFG : "", config_open == NULL ? "" : ",", config_open == NULL ? "" : config_open)); else testutil_check(__wt_snprintf(config, sizeof(config), "create,cache_cursors=false,statistics=(fast)," \ "statistics_log=(json,wait=1),error_prefix=\"%s\"" \ - "%s%s", + "%s%s%s", progname, + g.debug_mode ? DEBUG_MODE_CFG : "", config_open == NULL ? "" : ",", config_open == NULL ? "" : config_open)); diff --git a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h index b579f5cf9b9..3d4375cd137 100644 --- a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h +++ b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h @@ -55,6 +55,7 @@ typedef struct { char *home; /* Home directory */ const char *checkpoint_name; /* Checkpoint name */ WT_CONNECTION *conn; /* WiredTiger connection */ + bool debug_mode; /* Lookaside stress test */ u_int nkeys; /* Keys to load */ u_int nops; /* Operations per thread */ FILE *logfp; /* Message log file. */ diff --git a/src/third_party/wiredtiger/test/csuite/Makefile.am b/src/third_party/wiredtiger/test/csuite/Makefile.am index 8b29565b596..f737bc97bed 100644 --- a/src/third_party/wiredtiger/test/csuite/Makefile.am +++ b/src/third_party/wiredtiger/test/csuite/Makefile.am @@ -7,6 +7,9 @@ AM_LDFLAGS = -static all_TESTS= noinst_PROGRAMS= +# The import test is only a shell script +all_TESTS += import/smoke.sh + test_random_abort_SOURCES = random_abort/main.c noinst_PROGRAMS += test_random_abort all_TESTS += random_abort/smoke.sh @@ -137,7 +140,7 @@ all_TESTS += test_wt4803_cache_overflow_abort # Run this during a "make check" smoke test. TESTS = $(all_TESTS) -LOG_COMPILER = $(TEST_WRAPPER) +LOG_COMPILER = env top_builddir=$(top_builddir) top_srcdir=$(top_srcdir) $(TEST_WRAPPER) clean-local: rm -rf WT_TEST.* core.* *.core diff --git a/src/third_party/wiredtiger/test/csuite/import/smoke.sh b/src/third_party/wiredtiger/test/csuite/import/smoke.sh new file mode 100755 index 00000000000..624e3b26336 --- /dev/null +++ b/src/third_party/wiredtiger/test/csuite/import/smoke.sh @@ -0,0 +1,115 @@ +#! /bin/sh + +set -e + +# Bypass this test for slow machines, valgrind +test "$TESTUTIL_SLOW_MACHINE" = "1" && exit 0 +test "$TESTUTIL_BYPASS_VALGRIND" = "1" && exit 0 + +# If $top_builddir/$top_srcdir aren't set, default to building in build_posix +# and running in test/csuite. +top_builddir=${top_builddir:-../../build_posix} +top_srcdir=${top_srcdir:-../..} + +dir=WT_TEST.import + +rundir=$dir/RUNDIR +foreign=$dir/FOREIGN + +mo=$dir/metadata.orig +mi=$dir/metadata.import +co=$dir/ckpt.orig +ci=$dir/ckpt.import + +EXT="extensions=[\ +$top_builddir/ext/encryptors/rotn/.libs/libwiredtiger_rotn.so,\ +$top_builddir/ext/collators/reverse/.libs/libwiredtiger_reverse_collator.so]" + +wt="$top_builddir/wt" + +# Run test/format to create an object. +format() +{ + rm -rf $rundir + + $top_builddir/test/format/t \ + -1q \ + -C "$EXT" \ + -c $top_srcdir/test/format/CONFIG.stress \ + -h $rundir \ + backups=0 \ + checkpoints=1 \ + data_source=file \ + ops=0 \ + rebalance=0 \ + salvage=0 \ + threads=4 \ + timer=2 \ + verify=1 +} + +import() +{ + # Update the extensions if the run included encryption. + egrep 'encryption=none' $rundir/CONFIG > /dev/null || + EXT="encryption=(name=rotn,keyid=7),$EXT" + + # Dump the original metadata. + echo; echo 'dumping the original metadata' + $wt -C "$EXT" -h $rundir list -cv file:wt + $wt -C "$EXT" -h $rundir list -v file:wt | sed 1d > $mo + + # Create a stub datbase and copy in the table. + rm -rf $foreign && mkdir $foreign + $wt -C "$EXT" -h $foreign create file:xxx + cp $rundir/wt $foreign/yyy + + # Import the table. + $wt -C "$EXT" -h $foreign import file:yyy + + # Dump the imported metadata. + echo; echo 'dumping the imported metadata' + $wt -C "$EXT" -h $foreign list -cv file:yyy + $wt -C "$EXT" -h $foreign list -v file:yyy | sed 1d > $mi +} + +compare_checkpoints() +{ + sed -e 's/.*\(checkpoint=.*))\).*/\1/' < $mo > $co + sed -e 's/.*\(checkpoint=.*))\).*/\1/' < $mi > $ci + echo; echo 'original checkpoint' + cat $co + echo; echo 'imported checkpoint' + cat $ci + + echo; echo 'comparing the original and imported checkpoints' + cmp $co $ci && echo 'comparison succeeded' +} + +verify() +{ + echo; echo 'verifying the imported file' + $wt -C "$EXT" -h $foreign verify file:yyy && echo 'verify succeeded' +} + +# If verify fails, you can repeatedly run the import, checkpoint comparison and +# verify process using the -r option for debugging. +readonly=0 +while : + do case "$1" in + -r) + readonly=1 + shift;; + *) + break;; + esac +done + +if test $readonly -eq 0; then + rm -rf $dir && mkdir $dir + format +fi +import +compare_checkpoints +verify +exit 0 diff --git a/src/third_party/wiredtiger/test/csuite/random_abort/smoke.sh b/src/third_party/wiredtiger/test/csuite/random_abort/smoke.sh index e36ad62dd8b..713b000b4f1 100755 --- a/src/third_party/wiredtiger/test/csuite/random_abort/smoke.sh +++ b/src/third_party/wiredtiger/test/csuite/random_abort/smoke.sh @@ -4,7 +4,12 @@ set -e # Smoke-test random-abort as part of running "make check". -$TEST_WRAPPER ./test_random_abort -t 10 -T 5 -$TEST_WRAPPER ./test_random_abort -m -t 10 -T 5 -$TEST_WRAPPER ./test_random_abort -C -t 10 -T 5 -$TEST_WRAPPER ./test_random_abort -C -m -t 10 -T 5 +# If $top_builddir/$top_srcdir aren't set, default to building in build_posix +# and running in test/csuite. +top_builddir=${top_builddir:-../../build_posix} +top_srcdir=${top_srcdir:-../..} + +$TEST_WRAPPER $top_builddir/test/csuite/test_random_abort -t 10 -T 5 +$TEST_WRAPPER $top_builddir/test/csuite/test_random_abort -m -t 10 -T 5 +$TEST_WRAPPER $top_builddir/test/csuite/test_random_abort -C -t 10 -T 5 +$TEST_WRAPPER $top_builddir/test/csuite/test_random_abort -C -m -t 10 -T 5 diff --git a/src/third_party/wiredtiger/test/csuite/random_directio/smoke.sh b/src/third_party/wiredtiger/test/csuite/random_directio/smoke.sh index f871ff4684d..815b8a21c9f 100755 --- a/src/third_party/wiredtiger/test/csuite/random_directio/smoke.sh +++ b/src/third_party/wiredtiger/test/csuite/random_directio/smoke.sh @@ -4,7 +4,12 @@ set -e # Smoke-test random_directio as part of running "make check". -RUN_TEST_CMD="$TEST_WRAPPER ./test_random_directio" +# If $top_builddir/$top_srcdir aren't set, default to building in build_posix +# and running in test/csuite. +top_builddir=${top_builddir:-../../build_posix} +top_srcdir=${top_srcdir:-../..} + +RUN_TEST_CMD="$TEST_WRAPPER $top_builddir/test/csuite/test_random_directio" # Replace for more complete testing #TEST_THREADS="1 5 10" diff --git a/src/third_party/wiredtiger/test/csuite/schema_abort/smoke.sh b/src/third_party/wiredtiger/test/csuite/schema_abort/smoke.sh index 41d702d40cd..2e52b6f438c 100755 --- a/src/third_party/wiredtiger/test/csuite/schema_abort/smoke.sh +++ b/src/third_party/wiredtiger/test/csuite/schema_abort/smoke.sh @@ -4,9 +4,14 @@ set -e # Smoke-test schema-abort as part of running "make check". -$TEST_WRAPPER ./test_schema_abort -t 10 -T 5 -$TEST_WRAPPER ./test_schema_abort -m -t 10 -T 5 -$TEST_WRAPPER ./test_schema_abort -C -t 10 -T 5 -$TEST_WRAPPER ./test_schema_abort -C -m -t 10 -T 5 -$TEST_WRAPPER ./test_schema_abort -m -t 10 -T 5 -z -$TEST_WRAPPER ./test_schema_abort -m -t 10 -T 5 -x +# If $top_builddir/$top_srcdir aren't set, default to building in build_posix +# and running in test/csuite. +top_builddir=${top_builddir:-../../build_posix} +top_srcdir=${top_srcdir:-../..} + +$TEST_WRAPPER $top_builddir/test/csuite/test_schema_abort -t 10 -T 5 +$TEST_WRAPPER $top_builddir/test/csuite/test_schema_abort -m -t 10 -T 5 +$TEST_WRAPPER $top_builddir/test/csuite/test_schema_abort -C -t 10 -T 5 +$TEST_WRAPPER $top_builddir/test/csuite/test_schema_abort -C -m -t 10 -T 5 +$TEST_WRAPPER $top_builddir/test/csuite/test_schema_abort -m -t 10 -T 5 -z +$TEST_WRAPPER $top_builddir/test/csuite/test_schema_abort -m -t 10 -T 5 -x diff --git a/src/third_party/wiredtiger/test/csuite/timestamp_abort/smoke.sh b/src/third_party/wiredtiger/test/csuite/timestamp_abort/smoke.sh index 661261eb1bb..324e981dd70 100755 --- a/src/third_party/wiredtiger/test/csuite/timestamp_abort/smoke.sh +++ b/src/third_party/wiredtiger/test/csuite/timestamp_abort/smoke.sh @@ -4,9 +4,14 @@ set -e # Smoke-test timestamp-abort as part of running "make check". -$TEST_WRAPPER ./test_timestamp_abort -t 10 -T 5 -#$TEST_WRAPPER ./test_timestamp_abort -t 10 -T 5 -L -$TEST_WRAPPER ./test_timestamp_abort -m -t 10 -T 5 -#$TEST_WRAPPER ./test_timestamp_abort -m -t 10 -T 5 -L -$TEST_WRAPPER ./test_timestamp_abort -C -t 10 -T 5 -$TEST_WRAPPER ./test_timestamp_abort -C -m -t 10 -T 5 +# If $top_builddir/$top_srcdir aren't set, default to building in build_posix +# and running in test/csuite. +top_builddir=${top_builddir:-../../build_posix} +top_srcdir=${top_srcdir:-../..} + + $TEST_WRAPPER $top_builddir/test/csuite/test_timestamp_abort -t 10 -T 5 +#$TEST_WRAPPER $top_builddir/test/csuite/test_timestamp_abort -t 10 -T 5 -L + $TEST_WRAPPER $top_builddir/test/csuite/test_timestamp_abort -m -t 10 -T 5 +#$TEST_WRAPPER $top_builddir/test/csuite/test_timestamp_abort -m -t 10 -T 5 -L + $TEST_WRAPPER $top_builddir/test/csuite/test_timestamp_abort -C -t 10 -T 5 + $TEST_WRAPPER $top_builddir/test/csuite/test_timestamp_abort -C -m -t 10 -T 5 diff --git a/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c b/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c index bec69105679..54b56d597a4 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c @@ -102,8 +102,6 @@ static void subtest_populate(TEST_OPTS *, bool); extern int __wt_optind; -#define WT_FAIL_FS_LIB "../../ext/test/fail_fs/.libs/libwiredtiger_fail_fs.so" - /* * check_results -- * Check all the tables and verify the results. @@ -318,7 +316,7 @@ run_check_subtest(TEST_OPTS *opts, const char *debugger, uint64_t nops, subtest_args[narg++] = (char *)"--"; } - subtest_args[narg++] = (char *)opts->progname; + subtest_args[narg++] = (char *)opts->argv0; /* "subtest" must appear before arguments */ if (close_test) subtest_args[narg++] = (char *)"subtest_close"; @@ -342,7 +340,7 @@ run_check_subtest(TEST_OPTS *opts, const char *debugger, uint64_t nops, " operations until fail...\n", nops); testutil_clean_work_dir(opts->home); testutil_check(run_process( - opts, debugger != NULL ? debugger : opts->progname, + opts, debugger != NULL ? debugger : opts->argv0, subtest_args, &estatus)); if (opts->verbose) printf("process exited %d\n", estatus); @@ -524,6 +522,7 @@ subtest_main(int argc, char *argv[], bool close_test) TEST_OPTS *opts, _opts; WT_SESSION *session; char config[1024], filename[1024]; + const char *p; opts = &_opts; memset(opts, 0, sizeof(*opts)); @@ -541,17 +540,25 @@ subtest_main(int argc, char *argv[], bool close_test) testutil_check(__wt_snprintf( filename, sizeof(filename), "%s/%s", opts->home, STDOUT_FILE)); testutil_assert(freopen(filename, "a", stdout) != NULL); + + /* + * Use $top_builddir if it's available, otherwise assume we're building + * in build_posix and running in the test/csuite directory. + */ +#define WT_FAIL_FS_LIB "ext/test/fail_fs/.libs/libwiredtiger_fail_fs.so" + if ((p = getenv("top_builddir")) == NULL) + p = "../../build_posix"; testutil_check(__wt_snprintf(config, sizeof(config), "create,cache_size=250M,log=(enabled)," - "transaction_sync=(enabled,method=none),extensions=(" - WT_FAIL_FS_LIB - "=(early_load,config={environment=true,verbose=true})]")); - + "transaction_sync=(enabled,method=none)," + "extensions=(%s/%s=" + "(early_load,config={environment=true,verbose=true}))", + p, WT_FAIL_FS_LIB)); testutil_check( wiredtiger_open(opts->home, &event_handler, config, &opts->conn)); + testutil_check( opts->conn->open_session(opts->conn, NULL, NULL, &session)); - testutil_check(session->create(session, "table:subtest", "key_format=i,value_format=iiiS," "columns=(id,v0,v1,v2,big)")); diff --git a/src/third_party/wiredtiger/test/csuite/wt3120_filesys/main.c b/src/third_party/wiredtiger/test/csuite/wt3120_filesys/main.c index aa8d8a4e521..148d0062ddd 100644 --- a/src/third_party/wiredtiger/test/csuite/wt3120_filesys/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt3120_filesys/main.c @@ -36,24 +36,29 @@ * than that. */ -#define WT_FAIL_FS_LIB "../../ext/test/fail_fs/.libs/libwiredtiger_fail_fs.so" - int main(int argc, char *argv[]) { TEST_OPTS *opts, _opts; WT_CURSOR *cursor; WT_SESSION *session; - char buf[1024]; - char *kstr, *vstr; + char *kstr, *vstr, buf[1024]; + const char *p; opts = &_opts; memset(opts, 0, sizeof(*opts)); testutil_check(testutil_parse_opts(argc, argv, opts)); testutil_make_work_dir(opts->home); + /* + * Use $top_builddir if it's available, otherwise assume we're building + * in build_posix and running in the test/csuite directory. + */ +#define WT_FAIL_FS_LIB "ext/test/fail_fs/.libs/libwiredtiger_fail_fs.so" + if ((p = getenv("top_builddir")) == NULL) + p = "../../build_posix"; testutil_check(__wt_snprintf(buf, sizeof(buf), - "create,extensions=(" WT_FAIL_FS_LIB "=(early_load=true))")); + "create,extensions=(%s/%s=(early_load=true))", p, WT_FAIL_FS_LIB)); testutil_check(wiredtiger_open(opts->home, NULL, buf, &opts->conn)); testutil_check( opts->conn->open_session(opts->conn, NULL, NULL, &session)); diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index 99933cff595..e9ae4d1e37b 100644 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -364,6 +364,21 @@ tasks: # Start of csuite test tasks + - name: csuite-import-test + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "compile wiredtiger" + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix" + script: | + set -o errexit + set -o verbose + + ${test_env_vars|} $(pwd)/../test/csuite/import/smoke.sh 2>&1 + - name: csuite-random-abort-test depends_on: - name: compile @@ -371,12 +386,12 @@ tasks: - func: "fetch artifacts" - command: shell.exec params: - working_dir: "wiredtiger/build_posix/test/csuite" + working_dir: "wiredtiger/build_posix" script: | set -o errexit set -o verbose - ${test_env_vars|} $(pwd)/../../../test/csuite/random_abort/smoke.sh 2>&1 + ${test_env_vars|} $(pwd)/../test/csuite/random_abort/smoke.sh 2>&1 - name: csuite-random-directio-test depends_on: @@ -385,12 +400,12 @@ tasks: - func: "fetch artifacts" - command: shell.exec params: - working_dir: "wiredtiger/build_posix/test/csuite" + working_dir: "wiredtiger/build_posix" script: | set -o errexit set -o verbose - ${test_env_vars|} $(pwd)/../../../test/csuite/random_directio/smoke.sh 2>&1 + ${test_env_vars|} $(pwd)/../test/csuite/random_directio/smoke.sh 2>&1 - name: csuite-schema-abort-test depends_on: @@ -399,12 +414,12 @@ tasks: - func: "fetch artifacts" - command: shell.exec params: - working_dir: "wiredtiger/build_posix/test/csuite" + working_dir: "wiredtiger/build_posix" script: | set -o errexit set -o verbose - ${test_env_vars|} $(pwd)/../../../test/csuite/schema_abort/smoke.sh 2>&1 + ${test_env_vars|} $(pwd)/../test/csuite/schema_abort/smoke.sh 2>&1 - name: csuite-timestamp-abort-test depends_on: @@ -413,12 +428,12 @@ tasks: - func: "fetch artifacts" - command: shell.exec params: - working_dir: "wiredtiger/build_posix/test/csuite" + working_dir: "wiredtiger/build_posix" script: | set -o errexit set -o verbose - ${test_env_vars|} $(pwd)/../../../test/csuite/timestamp_abort/smoke.sh 2>&1 + ${test_env_vars|} $(pwd)/../test/csuite/timestamp_abort/smoke.sh 2>&1 - name: csuite-scope-test depends_on: @@ -427,12 +442,12 @@ tasks: - func: "fetch artifacts" - command: shell.exec params: - working_dir: "wiredtiger/build_posix/test/csuite" + working_dir: "wiredtiger/build_posix" script: | set -o errexit set -o verbose - ${test_env_vars|} $(pwd)/test_scope 2>&1 + ${test_env_vars|} $(pwd)/test/csuite/test_scope 2>&1 - name: csuite-truncated-log-test depends_on: @@ -441,12 +456,12 @@ tasks: - func: "fetch artifacts" - command: shell.exec params: - working_dir: "wiredtiger/build_posix/test/csuite" + working_dir: "wiredtiger/build_posix" script: | set -o errexit set -o verbose - ${test_env_vars|} $(pwd)/test_truncated_log 2>&1 + ${test_env_vars|} $(pwd)/test/csuite/test_truncated_log 2>&1 - name: csuite-wt1965-col-efficiency-test depends_on: @@ -455,12 +470,12 @@ tasks: - func: "fetch artifacts" - command: shell.exec params: - working_dir: "wiredtiger/build_posix/test/csuite" + working_dir: "wiredtiger/build_posix" script: | set -o errexit set -o verbose - ${test_env_vars|} $(pwd)/test_wt1965_col_efficiency 2>&1 + ${test_env_vars|} $(pwd)/test/csuite/test_wt1965_col_efficiency 2>&1 - name: csuite-wt2403-lsm-workload-test depends_on: @@ -469,12 +484,12 @@ tasks: - func: "fetch artifacts" - command: shell.exec params: - working_dir: "wiredtiger/build_posix/test/csuite" + working_dir: "wiredtiger/build_posix" script: | set -o errexit set -o verbose - ${test_env_vars|} $(pwd)/test_wt2403_lsm_workload 2>&1 + ${test_env_vars|} $(pwd)/test/csuite/test_wt2403_lsm_workload 2>&1 - name: csuite-wt2447-join-main-table-test depends_on: @@ -483,12 +498,12 @@ tasks: - func: "fetch artifacts" - command: shell.exec params: - working_dir: "wiredtiger/build_posix/test/csuite" + working_dir: "wiredtiger/build_posix" script: | set -o errexit set -o verbose - ${test_env_vars|} $(pwd)/test_wt2447_join_main_table 2>&1 + ${test_env_vars|} $(pwd)/test/csuite/test_wt2447_join_main_table 2>&1 - name: csuite-wt2695-checksum-test depends_on: @@ -497,12 +512,12 @@ tasks: - func: "fetch artifacts" - command: shell.exec params: - working_dir: "wiredtiger/build_posix/test/csuite" + working_dir: "wiredtiger/build_posix" script: | set -o errexit set -o verbose - ${test_env_vars|} $(pwd)/test_wt2695_checksum 2>&1 + ${test_env_vars|} $(pwd)/test/csuite/test_wt2695_checksum 2>&1 - name: csuite-wt2592-join-schema-test depends_on: @@ -511,12 +526,12 @@ tasks: - func: "fetch artifacts" - command: shell.exec params: - working_dir: "wiredtiger/build_posix/test/csuite" + working_dir: "wiredtiger/build_posix" script: | set -o errexit set -o verbose - ${test_env_vars|} $(pwd)/test_wt2592_join_schema 2>&1 + ${test_env_vars|} $(pwd)/test/csuite/test_wt2592_join_schema 2>&1 - name: csuite-wt2719-reconfig-test depends_on: @@ -525,12 +540,12 @@ tasks: - func: "fetch artifacts" - command: shell.exec params: - working_dir: "wiredtiger/build_posix/test/csuite" + working_dir: "wiredtiger/build_posix" script: | set -o errexit set -o verbose - ${test_env_vars|} $(pwd)/test_wt2719_reconfig 2>&1 + ${test_env_vars|} $(pwd)/test/csuite/test_wt2719_reconfig 2>&1 - name: csuite-wt2999-join-extractor-test depends_on: @@ -539,12 +554,12 @@ tasks: - func: "fetch artifacts" - command: shell.exec params: - working_dir: "wiredtiger/build_posix/test/csuite" + working_dir: "wiredtiger/build_posix" script: | set -o errexit set -o verbose - ${test_env_vars|} $(pwd)/test_wt2999_join_extractor 2>&1 + ${test_env_vars|} $(pwd)/test/csuite/test_wt2999_join_extractor 2>&1 - name: csuite-wt3120-filesys-test depends_on: @@ -553,12 +568,12 @@ tasks: - func: "fetch artifacts" - command: shell.exec params: - working_dir: "wiredtiger/build_posix/test/csuite" + working_dir: "wiredtiger/build_posix" script: | set -o errexit set -o verbose - ${test_env_vars|} $(pwd)/test_wt3120_filesys 2>&1 + ${test_env_vars|} $(pwd)/test/csuite/test_wt3120_filesys 2>&1 - name: csuite-wt3135-search-near-collator-test depends_on: @@ -567,12 +582,12 @@ tasks: - func: "fetch artifacts" - command: shell.exec params: - working_dir: "wiredtiger/build_posix/test/csuite" + working_dir: "wiredtiger/build_posix" script: | set -o errexit set -o verbose - ${test_env_vars|} $(pwd)/test_wt3135_search_near_collator 2>&1 + ${test_env_vars|} $(pwd)/test/csuite/test_wt3135_search_near_collator 2>&1 - name: csuite-wt3184-dup-index-collator-test depends_on: @@ -581,12 +596,12 @@ tasks: - func: "fetch artifacts" - command: shell.exec params: - working_dir: "wiredtiger/build_posix/test/csuite" + working_dir: "wiredtiger/build_posix" script: | set -o errexit set -o verbose - ${test_env_vars|} $(pwd)/test_wt3184_dup_index_collator 2>&1 + ${test_env_vars|} $(pwd)/test/csuite/test_wt3184_dup_index_collator 2>&1 - name: csuite-wt3363-checkpoint-op-races-test depends_on: @@ -595,12 +610,12 @@ tasks: - func: "fetch artifacts" - command: shell.exec params: - working_dir: "wiredtiger/build_posix/test/csuite" + working_dir: "wiredtiger/build_posix" script: | set -o errexit set -o verbose - ${test_env_vars|} $(pwd)/test_wt3363_checkpoint_op_races 2>&1 + ${test_env_vars|} $(pwd)/test/csuite/test_wt3363_checkpoint_op_races 2>&1 - name: csuite-wt3874-pad-byte-collator-test depends_on: @@ -609,12 +624,12 @@ tasks: - func: "fetch artifacts" - command: shell.exec params: - working_dir: "wiredtiger/build_posix/test/csuite" + working_dir: "wiredtiger/build_posix" script: | set -o errexit set -o verbose - ${test_env_vars|} $(pwd)/test_wt3874_pad_byte_collator 2>&1 + ${test_env_vars|} $(pwd)/test/csuite/test_wt3874_pad_byte_collator 2>&1 - name: csuite-wt4105-large-doc-small-upd-test depends_on: @@ -623,12 +638,12 @@ tasks: - func: "fetch artifacts" - command: shell.exec params: - working_dir: "wiredtiger/build_posix/test/csuite" + working_dir: "wiredtiger/build_posix" script: | set -o errexit set -o verbose - ${test_env_vars|} $(pwd)/test_wt4105_large_doc_small_upd 2>&1 + ${test_env_vars|} $(pwd)/test/csuite/test_wt4105_large_doc_small_upd 2>&1 - name: csuite-wt4117-checksum-test depends_on: @@ -637,12 +652,12 @@ tasks: - func: "fetch artifacts" - command: shell.exec params: - working_dir: "wiredtiger/build_posix/test/csuite" + working_dir: "wiredtiger/build_posix" script: | set -o errexit set -o verbose - ${test_env_vars|} $(pwd)/test_wt4117_checksum 2>&1 + ${test_env_vars|} $(pwd)/test/csuite/test_wt4117_checksum 2>&1 - name: csuite-wt4156-metadata-salvage-test depends_on: @@ -651,12 +666,12 @@ tasks: - func: "fetch artifacts" - command: shell.exec params: - working_dir: "wiredtiger/build_posix/test/csuite" + working_dir: "wiredtiger/build_posix" script: | set -o errexit set -o verbose - ${test_env_vars|} $(pwd)/test_wt4156_metadata_salvage 2>&1 + ${test_env_vars|} $(pwd)/test/csuite/test_wt4156_metadata_salvage 2>&1 - name: csuite-wt4699-json-test depends_on: @@ -665,12 +680,12 @@ tasks: - func: "fetch artifacts" - command: shell.exec params: - working_dir: "wiredtiger/build_posix/test/csuite" + working_dir: "wiredtiger/build_posix" script: | set -o errexit set -o verbose - ${test_env_vars|} $(pwd)/test_wt4699_json 2>&1 + ${test_env_vars|} $(pwd)/test/csuite/test_wt4699_json 2>&1 - name: csuite-wt4803-cache-overflow-abort-test depends_on: @@ -679,12 +694,12 @@ tasks: - func: "fetch artifacts" - command: shell.exec params: - working_dir: "wiredtiger/build_posix/test/csuite" + working_dir: "wiredtiger/build_posix" script: | set -o errexit set -o verbose - ${test_env_vars|} $(pwd)/test_wt4803_cache_overflow_abort 2>&1 + ${test_env_vars|} $(pwd)/test/csuite/test_wt4803_cache_overflow_abort 2>&1 - name: csuite-rwlock-test depends_on: @@ -693,12 +708,12 @@ tasks: - func: "fetch artifacts" - command: shell.exec params: - working_dir: "wiredtiger/build_posix/test/csuite" + working_dir: "wiredtiger/build_posix" script: | set -o errexit set -o verbose - ${test_env_vars|} $(pwd)/test_rwlock 2>&1 + ${test_env_vars|} $(pwd)/test/csuite/test_rwlock 2>&1 - name: csuite-wt2246-col-append-test depends_on: @@ -707,12 +722,12 @@ tasks: - func: "fetch artifacts" - command: shell.exec params: - working_dir: "wiredtiger/build_posix/test/csuite" + working_dir: "wiredtiger/build_posix" script: | set -o errexit set -o verbose - ${test_env_vars|} $(pwd)/test_wt2246_col_append 2>&1 + ${test_env_vars|} $(pwd)/test/csuite/test_wt2246_col_append 2>&1 - name: csuite-wt2323-join-visibility-test depends_on: @@ -721,12 +736,12 @@ tasks: - func: "fetch artifacts" - command: shell.exec params: - working_dir: "wiredtiger/build_posix/test/csuite" + working_dir: "wiredtiger/build_posix" script: | set -o errexit set -o verbose - ${test_env_vars|} $(pwd)/test_wt2323_join_visibility 2>&1 + ${test_env_vars|} $(pwd)/test/csuite/test_wt2323_join_visibility 2>&1 - name: csuite-wt2535-insert-race-test depends_on: @@ -735,12 +750,12 @@ tasks: - func: "fetch artifacts" - command: shell.exec params: - working_dir: "wiredtiger/build_posix/test/csuite" + working_dir: "wiredtiger/build_posix" script: | set -o errexit set -o verbose - ${test_env_vars|} $(pwd)/test_wt2535_insert_race 2>&1 + ${test_env_vars|} $(pwd)/test/csuite/test_wt2535_insert_race 2>&1 - name: csuite-wt2834-join-bloom-fix-test depends_on: @@ -749,12 +764,12 @@ tasks: - func: "fetch artifacts" - command: shell.exec params: - working_dir: "wiredtiger/build_posix/test/csuite" + working_dir: "wiredtiger/build_posix" script: | set -o errexit set -o verbose - ${test_env_vars|} $(pwd)/test_wt2834_join_bloom_fix 2>&1 + ${test_env_vars|} $(pwd)/test/csuite/test_wt2834_join_bloom_fix 2>&1 - name: csuite-wt2853-perf-test depends_on: @@ -763,12 +778,12 @@ tasks: - func: "fetch artifacts" - command: shell.exec params: - working_dir: "wiredtiger/build_posix/test/csuite" + working_dir: "wiredtiger/build_posix" script: | set -o errexit set -o verbose - ${test_env_vars|} $(pwd)/test_wt2853_perf 2>&1 + ${test_env_vars|} $(pwd)/test/csuite/test_wt2853_perf 2>&1 - name: csuite-wt2909-checkpoint-integrity-test depends_on: @@ -777,12 +792,12 @@ tasks: - func: "fetch artifacts" - command: shell.exec params: - working_dir: "wiredtiger/build_posix/test/csuite" + working_dir: "wiredtiger/build_posix" script: | set -o errexit set -o verbose - ${test_env_vars|} $(pwd)/test_wt2909_checkpoint_integrity 2>&1 + ${test_env_vars|} $(pwd)/test/csuite/test_wt2909_checkpoint_integrity 2>&1 - name: csuite-wt3338-partial-update-test depends_on: @@ -791,12 +806,12 @@ tasks: - func: "fetch artifacts" - command: shell.exec params: - working_dir: "wiredtiger/build_posix/test/csuite" + working_dir: "wiredtiger/build_posix" script: | set -o errexit set -o verbose - ${test_env_vars|} $(pwd)/test_wt3338_partial_update 2>&1 + ${test_env_vars|} $(pwd)/test/csuite/test_wt3338_partial_update 2>&1 - name: csuite-wt4333-handle-locks-test depends_on: @@ -805,12 +820,12 @@ tasks: - func: "fetch artifacts" - command: shell.exec params: - working_dir: "wiredtiger/build_posix/test/csuite" + working_dir: "wiredtiger/build_posix" script: | set -o errexit set -o verbose - ${test_env_vars|} $(pwd)/test_wt4333_handle_locks 2>&1 + ${test_env_vars|} $(pwd)/test/csuite/test_wt4333_handle_locks 2>&1 # End of csuite test tasks @@ -1158,7 +1173,7 @@ buildvariants: - ubuntu1404-test expansions: # It's ugly, but we need the absolute path here, not the relative - test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs + test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs top_srcdir=$(pwd)/.. top_builddir=$(pwd) smp_command: -j $(grep -c ^processor /proc/cpuinfo) configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/gcc CXX=/opt/mongodbtoolchain/v3/bin/g++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH make_command: PATH=/opt/mongodbtoolchain/v3/bin:$PATH make @@ -1178,6 +1193,7 @@ buildvariants: - name: salvage-test - name: thread-test - name: bench-wtperf-test + - name: csuite-import-test - name: csuite-random-abort-test - name: csuite-random-directio-test - name: csuite-schema-abort-test @@ -1224,7 +1240,7 @@ buildvariants: run_on: - ubuntu1404-test expansions: - test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs + test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs top_srcdir=$(pwd)/.. top_builddir=$(pwd) smp_command: -j $(grep -c ^processor /proc/cpuinfo) configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/gcc CXX=/opt/mongodbtoolchain/v3/bin/g++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH configure_python_setting: PYTHON=python3 @@ -1285,7 +1301,7 @@ buildvariants: smp_command: -j $(sysctl -n hw.logicalcpu) configure_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH make_command: PATH=/opt/mongodbtoolchain/v3/bin:$PATH ARCHFLAGS=-Wno-error=unused-command-line-argument-hard-error-in-future make - test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH DYLD_LIBRARY_PATH=$(pwd)/.libs + test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH DYLD_LIBRARY_PATH=$(pwd)/.libs top_srcdir=$(pwd)/.. top_builddir=$(pwd) tasks: - name: compile - name: make-check-test @@ -1300,7 +1316,7 @@ buildvariants: expansions: smp_command: -j $(grep -c ^processor /proc/cpuinfo) configure_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH - test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs + test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs top_srcdir=$(pwd)/.. top_builddir=$(pwd) tasks: - name: compile - name: generate-datafile-little-endian @@ -1317,7 +1333,7 @@ buildvariants: expansions: smp_command: -j $(grep -c ^processor /proc/cpuinfo) configure_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH - test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs + test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.lib top_srcdir=$(pwd)/.. top_builddir=$(pwd) tasks: - name: compile - name: generate-datafile-big-endian diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c index 8f2c25429c9..df3222b072a 100644 --- a/src/third_party/wiredtiger/test/format/config.c +++ b/src/third_party/wiredtiger/test/format/config.c @@ -106,7 +106,7 @@ config_setup(void) * cache problems, don't configure LSM if those set. * * XXX - * Remove the timestamp test when WT-4067 resolved. + * Remove the timestamp test when WT-4162 resolved. */ if (g.type != ROW || g.c_in_memory) break; @@ -563,8 +563,7 @@ config_lsm_reset(void) * LSM doesn't currently play nicely with timestamps, don't choose the * pair unless forced to. If we turn off timestamps, make sure we turn * off prepare as well, it requires timestamps. Remove this code with - * WT-4067. - * + * WT-4162. */ if (!config_is_perm("prepare") && !config_is_perm("transaction_timestamps")) { diff --git a/src/third_party/wiredtiger/test/suite/run.py b/src/third_party/wiredtiger/test/suite/run.py index 5895325a3d7..3de273c8ebf 100755 --- a/src/third_party/wiredtiger/test/suite/run.py +++ b/src/third_party/wiredtiger/test/suite/run.py @@ -31,7 +31,7 @@ # from __future__ import print_function -import glob, json, os, re, sys +import glob, json, os, random, re, sys try: xrange @@ -118,6 +118,8 @@ Options:\n\ -j N | --parallel N run all tests in parallel using N processes\n\ -l | --long run the entire test suite\n\ -p | --preserve preserve output files in WT_TEST/<testname>\n\ + -r N | --random-sample N randomly sort scenarios to be run, then\n\ + execute every Nth (2<=N<=1000) scenario.\n\ -s N | --scenario N use scenario N (N can be number or symbolic)\n\ -t | --timestamp name WT_TEST according to timestamp\n\ -v N | --verbose N set verboseness to N (0<=N<=3, default=1)\n\ @@ -269,6 +271,7 @@ if __name__ == '__main__': # Turn numbers and ranges into test module names preserve = timestamp = debug = dryRun = gdbSub = lldbSub = longtest = False parallel = 0 + random_sample = 0 configfile = None configwrite = False dirarg = None @@ -307,6 +310,15 @@ if __name__ == '__main__': if option == '-long' or option == 'l': longtest = True continue + if option == '-random-sample' or option == 'r': + if len(args) == 0: + usage() + sys.exit(2) + random_sample = int(args.pop(0)) + if random_sample < 2 or random_sample > 1000: + usage() + sys.exit(2) + continue if option == '-parallel' or option == 'j': if parallel != 0 or len(args) == 0: usage() @@ -376,6 +388,14 @@ if __name__ == '__main__': for arg in testargs: testsFromArg(tests, loader, arg, scenario) + # Shuffle the tests and create a new suite containing every Nth test from + # the original suite + if random_sample > 0: + random_sample_tests = [] + for test in tests: + random_sample_tests.append(test) + random.shuffle(random_sample_tests) + tests = unittest.TestSuite(random_sample_tests[::random_sample]) if debug: import pdb pdb.set_trace() diff --git a/src/third_party/wiredtiger/test/suite/test_debug_mode04.py b/src/third_party/wiredtiger/test/suite/test_debug_mode04.py new file mode 100644 index 00000000000..1f5429495e8 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_debug_mode04.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# +# Public Domain 2034-2039 MongoDB, Inc. +# Public Domain 2008-2034 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest + +# test_debug_mode04.py +# Test the debug mode settings. Test eviction use. +class test_debug_mode04(wttest.WiredTigerTestCase): + conn_config = 'log=(enabled=true,file_max=100K),debug_mode=(eviction=true)' + uri = 'file:test_debug' + entries = 100 + value = b'\x01\x02abcd\x03\x04' + + def add_data(self): + keys = range(0, self.entries) + c = self.session.open_cursor(self.uri, None) + for k in keys: + c[k] = self.value + c.close() + + # Just test turning it on and off. There really isn't something + # specific to verify. + def test_table_logging(self): + self.session.create(self.uri, 'key_format=i,value_format=u') + self.add_data() + + def test_table_logging_off(self): + self.conn.reconfigure("debug_mode=(eviction=false)") + self.session.create(self.uri, 'key_format=i,value_format=u') + self.add_data() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_empty.py b/src/third_party/wiredtiger/test/suite/test_empty.py index f7949871a11..b41ae3540c9 100755 --- a/src/third_party/wiredtiger/test/suite/test_empty.py +++ b/src/third_party/wiredtiger/test/suite/test_empty.py @@ -28,7 +28,6 @@ import os import wiredtiger, wttest -from wtdataset import simple_key from wtscenario import make_scenarios # test_empty.py @@ -53,47 +52,5 @@ class test_empty(wttest.WiredTigerTestCase): name = name + '.wt' self.assertEquals(os.stat(name).st_size, 4*1024) - # Open a new session, add a few rows to an object and then remove them, - # then close the object. We open/close the object so it's flushed from - # the underlying cache each time. - def empty(self): - uri = self.type + self.name - self.session = self.conn.open_session() - self.session.create(uri, 'key_format=' + self.fmt + ',value_format=S') - - # Add a few records to the object and remove them. - cursor = self.session.open_cursor(uri, None, None) - for i in range(1,5): - key = simple_key(cursor, i) - cursor[key] = "XXX" - del cursor[key] - - # Perform a checkpoint (we shouldn't write any underlying pages because - # of a checkpoint, either). - self.session.checkpoint("name=ckpt") - - # Open and close a checkpoint cursor. - cursor = self.session.open_cursor(uri, None, "checkpoint=ckpt") - cursor.close() - - self.session.close() - - # The file should not have grown. - name = self.name - if self.type == "table:": - name = name + '.wt' - self.assertEquals(os.stat(name).st_size, 4*1024) - - # Creating an object, inserting and removing records (that is, building an - # empty, dirty tree), shouldn't write any blocks. This doesn't work for - # column-store objects, though, because deleting an object modifies the name - # space, which requires a write. - def test_empty(self): - if self.fmt == 'r': - return - - for i in range(1,5): - self.empty() - if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_encrypt06.py b/src/third_party/wiredtiger/test/suite/test_encrypt06.py index 7663a8d6d7e..520c18c04d6 100755 --- a/src/third_party/wiredtiger/test/suite/test_encrypt06.py +++ b/src/third_party/wiredtiger/test/suite/test_encrypt06.py @@ -45,7 +45,7 @@ class test_encrypt06(wttest.WiredTigerTestCase): # testing a potential misuse of the API: a table is opened with # with its own encryption options (different from the system), # but the indices and column groups do not specify encryption, - # so they'll get the system encryptor. + # so they may get the system encryptor. storagetype = [ ('table', dict( uriprefix='table:', use_cg=False, use_index=False, match=True)), @@ -66,25 +66,25 @@ class test_encrypt06(wttest.WiredTigerTestCase): ] encrypt = [ ('none', dict( - sys_encrypt='none', sys_encrypt_args='', encryptmeta=False, - file0_encrypt='none', file0_encrypt_args='', encrypt0=False, - file1_encrypt='none', file1_encrypt_args='', encrypt1=False)), + sys_encrypt='none', sys_encrypt_args='', + table0_encrypt='none', table0_encrypt_args='', + table1_encrypt='none', table1_encrypt_args='')), ('rotn-implied', dict( - sys_encrypt='rotn', sys_encrypt_args=key11, encryptmeta=True, - file0_encrypt=None, file0_encrypt_args='', encrypt0=True, - file1_encrypt=None, file1_encrypt_args='', encrypt1=True)), + sys_encrypt='rotn', sys_encrypt_args=key11, + table0_encrypt=None, table0_encrypt_args='', + table1_encrypt=None, table1_encrypt_args='')), ('rotn-all', dict( - sys_encrypt='rotn', sys_encrypt_args=key11, encryptmeta=True, - file0_encrypt='rotn', file0_encrypt_args=key13, encrypt0=True, - file1_encrypt='rotn', file1_encrypt_args=key13, encrypt1=True)), + sys_encrypt='rotn', sys_encrypt_args=key11, + table0_encrypt='rotn', table0_encrypt_args=key13, + table1_encrypt='rotn', table1_encrypt_args=key13)), ('rotn-sys', dict( - sys_encrypt='rotn', sys_encrypt_args=key11, encryptmeta=True, - file0_encrypt='none', file0_encrypt_args='', encrypt0=False, - file1_encrypt='none', file1_encrypt_args='', encrypt1=False)), - ('rotn-file0', dict( - sys_encrypt='rotn', sys_encrypt_args=key11, encryptmeta=True, - file0_encrypt='rotn', file0_encrypt_args=key13, encrypt0=True, - file1_encrypt='none', file1_encrypt_args='', encrypt1=False)), + sys_encrypt='rotn', sys_encrypt_args=key11, + table0_encrypt='none', table0_encrypt_args='', + table1_encrypt='none', table1_encrypt_args='')), + ('rotn-table0', dict( + sys_encrypt='rotn', sys_encrypt_args=key11, + table0_encrypt='rotn', table0_encrypt_args=key13, + table1_encrypt='none', table1_encrypt_args='')), ] scenarios = make_scenarios(encrypt, storagetype) nrecords = 1000 @@ -92,14 +92,14 @@ class test_encrypt06(wttest.WiredTigerTestCase): def conn_extensions(self, extlist): extlist.skip_if_missing = True extlist.extension('encryptors', self.sys_encrypt) - extlist.extension('encryptors', self.file0_encrypt) - extlist.extension('encryptors', self.file1_encrypt) + extlist.extension('encryptors', self.table0_encrypt) + extlist.extension('encryptors', self.table1_encrypt) def conn_config(self): return 'encryption=(name={0}{1}),'.format( self.sys_encrypt, self.sys_encrypt_args) - def encrypt_file_params(self, name, args): + def encrypt_table_params(self, name, args): if name == None: return '' else: @@ -116,63 +116,85 @@ class test_encrypt06(wttest.WiredTigerTestCase): return True return False - def expected_encryption(self, exp): - expect = exp - # If we're expecting it to be unencrypted, but we (errantly) - # did not specify encryption on indices/columngroups, - # then column groups (if they exist) will be encrypted - - # there will be no data in the main table to be unencrypted. - if self.sys_encrypt != 'none' and not self.match and self.use_cg: - expect = True - return expect + def visible_data(self, table_setting): + if table_setting == None: + # No table encryption explicitly set, so we use the system setting + visible = (self.sys_encrypt == 'none') + else: + visible = (table_setting == 'none') + return visible + + def visible_name(self, table_setting, iskey): + if table_setting == None: + # No table encryption explicitly set, so we use the system setting + visible = (self.sys_encrypt == 'none') + else: + visible = (table_setting == 'none') + + # If we have everything in a column group, the key name will not + # be stored in the column group files. It will be stored in the + # system metadata, if that is not encrypted. + if iskey and self.use_cg and self.sys_encrypt != 'none': + visible = False + return visible # Create a table, add key/values with specific lengths, then verify them. def test_encrypt(self): name0 = 'test_encrypt06-0' name1 = 'test_encrypt06-1' - enc0 = self.encrypt_file_params(self.file0_encrypt, - self.file0_encrypt_args) - enc1 = self.encrypt_file_params(self.file1_encrypt, - self.file1_encrypt_args) + enc0 = self.encrypt_table_params(self.table0_encrypt, + self.table0_encrypt_args) + enc1 = self.encrypt_table_params(self.table1_encrypt, + self.table1_encrypt_args) # This is the clear text that we'll be looking for txt0 = 'AbCdEfG' txt1 = 'aBcDeFg' + keyname0 = 'MyKey0Name' + keyname1 = 'MyKey1Name' + valname0 = 'MyValue0Name' + valname1 = 'MyValue1Name' # Make a bunch of column group and indices, # we want to see if any information is leaked anywhere. + # The key column and one of the value columns is given a name + # we will look for as clear text. sharedparam = 'key_format=S,value_format=SSSS,' + \ - 'columns=(MyKeyName,v0,v1,v2,v3),' + 'columns=({},{},v1,v2,v3),' s = self.session pfx = self.uriprefix cgparam = 'colgroups=(g00,g01)' if self.use_cg else '' - s.create(pfx + name0, sharedparam + cgparam + enc0) + s.create(pfx + name0, sharedparam.format(keyname0, valname0) + \ + cgparam + enc0) - # Having unmatched encryption for colgroup or index is - # not recommended, but we check it. if not self.match: enc0 = '' if self.use_cg: - s.create('colgroup:' + name0 + ':g00', 'columns=(v0,v1)' + enc0) + s.create('colgroup:' + name0 + ':g00', + 'columns=({},v1)'.format(valname0) + enc0) s.create('colgroup:' + name0 + ':g01', 'columns=(v2,v3)' + enc0) if self.use_index: - s.create('index:' + name0 + ':i00', 'columns=(v0)' + enc0) + s.create('index:' + name0 + ':i00', + 'columns=({})'.format(valname0) + enc0) s.create('index:' + name0 + ':i01', 'columns=(v1,v2)' + enc0) s.create('index:' + name0 + ':i02', 'columns=(v3)' + enc0) cgparam = 'colgroups=(g10,g11)' if self.use_cg else '' - s.create(pfx + name1, sharedparam + cgparam + enc1) + s.create(pfx + name1, sharedparam.format(keyname1, valname1) + \ + cgparam + enc1) if not self.match: enc1 = '' if self.use_cg: - s.create('colgroup:' + name1 + ':g10', 'columns=(v0,v1)' + enc1) + s.create('colgroup:' + name1 + ':g10', + 'columns=({},v1)'.format(valname1) + enc1) s.create('colgroup:' + name1 + ':g11', 'columns=(v2,v3)' + enc1) if self.use_index: - s.create('index:' + name1 + ':i10', 'columns=(v0)' + enc1) + s.create('index:' + name1 + ':i10', + 'columns=({})'.format(valname1) + enc1) s.create('index:' + name1 + ':i11', 'columns=(v1,v2)' + enc1) s.create('index:' + name1 + ':i12', 'columns=(v3)' + enc1) @@ -192,13 +214,40 @@ class test_encrypt06(wttest.WiredTigerTestCase): # Force everything to disk so we can examine it self.close_conn() - self.assertEqual(self.encryptmeta, - not self.match_string_in_rundir('MyKeyName')) - - self.assertEqual(self.expected_encryption(self.encrypt0), - not self.match_string_in_rundir(txt0)) - self.assertEqual(self.expected_encryption(self.encrypt1), - not self.match_string_in_rundir(txt1)) + if self.match: + # Key and value names are encrypted according to the + # encryption level on the associated table. + self.assertEqual(self.visible_data(self.table0_encrypt), + self.match_string_in_rundir(txt0)) + self.assertEqual(self.visible_name(self.table0_encrypt, True), + self.match_string_in_rundir(keyname0)) + self.assertEqual(self.visible_name(self.table0_encrypt, False), + self.match_string_in_rundir(valname0)) + + self.assertEqual(self.visible_data(self.table1_encrypt), + self.match_string_in_rundir(txt1)) + self.assertEqual(self.visible_name(self.table1_encrypt, True), + self.match_string_in_rundir(keyname1)) + self.assertEqual(self.visible_name(self.table1_encrypt, False), + self.match_string_in_rundir(valname1)) + else: + # If the encryption config for indices and column groups is blank, + # we make a conservative check - if we specified encryption on the + # table, none of our data or key/value names should be exposed. + # + # If we have system encryption on, set table encryption to 'none', + # and set the index or column group config to blank, we technically + # should get no encryption for names or data. That currently + # doesn't work (CGs and indices instead will be encrypted), + # so we don't cover that case. + if self.table0_encrypt != 'none': + self.assertFalse(self.match_string_in_rundir(txt0)) + self.assertFalse(self.match_string_in_rundir(keyname0)) + self.assertFalse(self.match_string_in_rundir(valname0)) + if self.table1_encrypt != 'none': + self.assertFalse(self.match_string_in_rundir(txt1)) + self.assertFalse(self.match_string_in_rundir(keyname1)) + self.assertFalse(self.match_string_in_rundir(valname1)) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp09.py b/src/third_party/wiredtiger/test/suite/test_timestamp09.py index c4b545fe2c1..efc081c29f4 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp09.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp09.py @@ -165,10 +165,9 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): # Read timestamp >= Oldest timestamp self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(7) + ',stable_timestamp=' + timestamp_str(7)) - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.begin_transaction('read_timestamp=' + - timestamp_str(6)), - '/less than the oldest timestamp/') + with self.expectedStdoutPattern('less than the oldest timestamp'): + self.assertRaisesException(wiredtiger.WiredTigerError, + lambda: self.session.begin_transaction('read_timestamp=' + timestamp_str(6))) # c[8] is not visible at read_timestamp < 8 self.session.begin_transaction('read_timestamp=' + timestamp_str(7)) @@ -189,10 +188,9 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): # We can move the oldest timestamp backwards with "force" self.conn.set_timestamp( 'oldest_timestamp=' + timestamp_str(5) + ',force') - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.begin_transaction('read_timestamp=' + - timestamp_str(4)), - '/less than the oldest timestamp/') + with self.expectedStdoutPattern('less than the oldest timestamp'): + self.assertRaisesException(wiredtiger.WiredTigerError, + lambda: self.session.begin_transaction('read_timestamp=' + timestamp_str(4))) self.session.begin_transaction('read_timestamp=' + timestamp_str(6)) self.assertTimestampsEqual( self.conn.query_timestamp('get=oldest_reader'), timestamp_str(6)) diff --git a/src/third_party/wiredtiger/test/utility/parse_opts.c b/src/third_party/wiredtiger/test/utility/parse_opts.c index c5bb1a83148..06d9ea31538 100644 --- a/src/third_party/wiredtiger/test/utility/parse_opts.c +++ b/src/third_party/wiredtiger/test/utility/parse_opts.c @@ -44,6 +44,7 @@ testutil_parse_opts(int argc, char * const *argv, TEST_OPTS *opts) opts->running = true; opts->verbose = false; + opts->argv0 = argv[0]; opts->progname = testutil_set_progname(argv); testutil_print_command_line(argc, argv); diff --git a/src/third_party/wiredtiger/test/utility/test_util.h b/src/third_party/wiredtiger/test/utility/test_util.h index 27310de0006..7aa4eaecc0f 100644 --- a/src/third_party/wiredtiger/test/utility/test_util.h +++ b/src/third_party/wiredtiger/test/utility/test_util.h @@ -49,7 +49,9 @@ /* Generic option parsing structure shared by all test cases. */ typedef struct { char *home; - const char *progname; + const char *argv0; /* Exec name */ + const char *progname; /* Truncated program name */ + enum { TABLE_COL=1, /* Fixed-length column store */ TABLE_FIX=2, /* Variable-length column store */ TABLE_ROW=3 /* Row-store */ diff --git a/src/third_party/wiredtiger/tools/optrack/find-latency-spikes.py b/src/third_party/wiredtiger/tools/optrack/find-latency-spikes.py index 7409ab62243..1c030721787 100755 --- a/src/third_party/wiredtiger/tools/optrack/find-latency-spikes.py +++ b/src/third_party/wiredtiger/tools/optrack/find-latency-spikes.py @@ -864,7 +864,7 @@ def generateTSSlicesForBuckets(): numBuckets, "Generating timeline charts"); for i, fname in returnValues.items(): - bucketFilenames.append(str(fname.value)); + bucketFilenames.append(fname.value.decode()); print(color.END); return bucketFilenames; |