summaryrefslogtreecommitdiff
path: root/src/third_party/wiredtiger
diff options
context:
space:
mode:
authorLuke Chen <luke.chen@mongodb.com>2019-06-17 14:02:48 +1000
committerLuke Chen <luke.chen@mongodb.com>2019-06-17 14:02:48 +1000
commit896af5ad5b522aaad90b48515958c3dcce513144 (patch)
tree5e05359a7f34d5d0d8fe51219a3d35eb83240b3e /src/third_party/wiredtiger
parent40c8a913892eaf3cd88f55cfa23e6f7f8b68aa95 (diff)
downloadmongo-896af5ad5b522aaad90b48515958c3dcce513144.tar.gz
Import wiredtiger: 40e3225e55ffbf0c9819af138fa4a014c0639c72 from branch mongodb-4.2
ref: 9b85ad8968..40e3225e55 for: 4.2.0 WT-4477 Add eviction debug mode and extra checks WT-4494 Support import of of standalone WiredTiger files. WT-4580 Abort transactions that perform updates with ignore_prepare=true WT-4719 Non-zero update count detected when committing prepared transaction WT-4787 Coverity: Potential infinite loops WT-4790 Coverity: Memory leak within __wt_curbulk_init WT-4827 Apply commit timestamps when a truncated page is read WT-4829 File names are generated incorrectly in operation tracking scripts WT-4837 Disabled cache during verify WT-4839 Add a feature to python testsuite to run a random subset of tests WT-4844 Only log an informational message when a set read-timestamp is older than the oldest timestamp. WT-4848 Fix perf regression when calculating differences WT-4850 Fix __slvg_checkpoint() metadata memory leak WT-4851 heap-use-after-free when block manager grows buffer during final checkpoint WT-4852 Evergreen runs of test/format require shared library extensions WT-4853 standalone recovery code cannot handle deleted checkpoints WT-4854 wt utility list command has a memory leak and minor error handling bugs. WT-4855 WiredTiger recovery should detect files without unique IDs WT-4859 Fix test/format program path issue for endian tests
Diffstat (limited to 'src/third_party/wiredtiger')
-rw-r--r--src/third_party/wiredtiger/SConstruct3
-rw-r--r--src/third_party/wiredtiger/build_posix/Make.base1
-rw-r--r--src/third_party/wiredtiger/build_posix/Make.subdirs1
-rw-r--r--src/third_party/wiredtiger/dist/api_data.py8
-rw-r--r--src/third_party/wiredtiger/dist/filelist3
-rw-r--r--src/third_party/wiredtiger/dist/s_define.list1
-rw-r--r--src/third_party/wiredtiger/dist/s_string.ok2
-rw-r--r--src/third_party/wiredtiger/dist/stat_data.py2
-rw-r--r--src/third_party/wiredtiger/examples/c/ex_all.c41
-rw-r--r--src/third_party/wiredtiger/import.data2
-rw-r--r--src/third_party/wiredtiger/lang/java/java_doc.i1
-rw-r--r--src/third_party/wiredtiger/src/block/block_addr.c87
-rw-r--r--src/third_party/wiredtiger/src/block/block_ckpt.c171
-rw-r--r--src/third_party/wiredtiger/src/block/block_ckpt_scan.c417
-rw-r--r--src/third_party/wiredtiger/src/block/block_ext.c43
-rw-r--r--src/third_party/wiredtiger/src/block/block_mgr.c13
-rw-r--r--src/third_party/wiredtiger/src/block/block_open.c12
-rw-r--r--src/third_party/wiredtiger/src/block/block_write.c74
-rw-r--r--src/third_party/wiredtiger/src/btree/bt_cursor.c9
-rw-r--r--src/third_party/wiredtiger/src/btree/bt_handle.c70
-rw-r--r--src/third_party/wiredtiger/src/btree/bt_import.c169
-rw-r--r--src/third_party/wiredtiger/src/btree/bt_slvg.c91
-rw-r--r--src/third_party/wiredtiger/src/btree/bt_split.c27
-rw-r--r--src/third_party/wiredtiger/src/btree/bt_vrfy.c19
-rw-r--r--src/third_party/wiredtiger/src/cache/cache_las.c10
-rw-r--r--src/third_party/wiredtiger/src/config/config_collapse.c97
-rw-r--r--src/third_party/wiredtiger/src/config/config_def.c49
-rw-r--r--src/third_party/wiredtiger/src/conn/api_calc_modify.c25
-rw-r--r--src/third_party/wiredtiger/src/conn/conn_api.c38
-rw-r--r--src/third_party/wiredtiger/src/docs/command-line.dox2
-rw-r--r--src/third_party/wiredtiger/src/evict/evict_lru.c40
-rw-r--r--src/third_party/wiredtiger/src/evict/evict_page.c18
-rw-r--r--src/third_party/wiredtiger/src/include/block.h19
-rw-r--r--src/third_party/wiredtiger/src/include/block.i39
-rw-r--r--src/third_party/wiredtiger/src/include/btmem.h4
-rw-r--r--src/third_party/wiredtiger/src/include/btree.h2
-rw-r--r--src/third_party/wiredtiger/src/include/btree.i3
-rw-r--r--src/third_party/wiredtiger/src/include/cache.h13
-rw-r--r--src/third_party/wiredtiger/src/include/config.h59
-rw-r--r--src/third_party/wiredtiger/src/include/extern.h15
-rw-r--r--src/third_party/wiredtiger/src/include/meta.h22
-rw-r--r--src/third_party/wiredtiger/src/include/misc.i17
-rw-r--r--src/third_party/wiredtiger/src/include/stat.h2
-rw-r--r--src/third_party/wiredtiger/src/include/txn.h13
-rw-r--r--src/third_party/wiredtiger/src/include/txn.i139
-rw-r--r--src/third_party/wiredtiger/src/include/wiredtiger.in237
-rw-r--r--src/third_party/wiredtiger/src/include/wt_internal.h3
-rw-r--r--src/third_party/wiredtiger/src/meta/meta_ckpt.c229
-rw-r--r--src/third_party/wiredtiger/src/meta/meta_ext.c2
-rw-r--r--src/third_party/wiredtiger/src/reconcile/rec_write.c52
-rw-r--r--src/third_party/wiredtiger/src/session/session_api.c61
-rw-r--r--src/third_party/wiredtiger/src/session/session_salvage.c59
-rw-r--r--src/third_party/wiredtiger/src/support/huffman.c5
-rw-r--r--src/third_party/wiredtiger/src/support/stat.c8
-rw-r--r--src/third_party/wiredtiger/src/txn/txn.c83
-rw-r--r--src/third_party/wiredtiger/src/txn/txn_ckpt.c17
-rw-r--r--src/third_party/wiredtiger/src/txn/txn_log.c2
-rw-r--r--src/third_party/wiredtiger/src/txn/txn_recover.c5
-rw-r--r--src/third_party/wiredtiger/src/txn/txn_timestamp.c16
-rw-r--r--src/third_party/wiredtiger/src/utilities/util.h1
-rw-r--r--src/third_party/wiredtiger/src/utilities/util_import.c49
-rw-r--r--src/third_party/wiredtiger/src/utilities/util_list.c115
-rw-r--r--src/third_party/wiredtiger/src/utilities/util_main.c8
-rwxr-xr-xsrc/third_party/wiredtiger/test/checkpoint/smoke.sh3
-rw-r--r--src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c14
-rw-r--r--src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h1
-rw-r--r--src/third_party/wiredtiger/test/csuite/Makefile.am5
-rwxr-xr-xsrc/third_party/wiredtiger/test/csuite/import/smoke.sh115
-rwxr-xr-xsrc/third_party/wiredtiger/test/csuite/random_abort/smoke.sh13
-rwxr-xr-xsrc/third_party/wiredtiger/test/csuite/random_directio/smoke.sh7
-rwxr-xr-xsrc/third_party/wiredtiger/test/csuite/schema_abort/smoke.sh17
-rwxr-xr-xsrc/third_party/wiredtiger/test/csuite/timestamp_abort/smoke.sh17
-rw-r--r--src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c25
-rw-r--r--src/third_party/wiredtiger/test/csuite/wt3120_filesys/main.c15
-rw-r--r--src/third_party/wiredtiger/test/evergreen.yml154
-rw-r--r--src/third_party/wiredtiger/test/format/config.c5
-rwxr-xr-xsrc/third_party/wiredtiger/test/suite/run.py22
-rw-r--r--src/third_party/wiredtiger/test/suite/test_debug_mode04.py58
-rwxr-xr-xsrc/third_party/wiredtiger/test/suite/test_empty.py43
-rwxr-xr-xsrc/third_party/wiredtiger/test/suite/test_encrypt06.py147
-rw-r--r--src/third_party/wiredtiger/test/suite/test_timestamp09.py14
-rw-r--r--src/third_party/wiredtiger/test/utility/parse_opts.c1
-rw-r--r--src/third_party/wiredtiger/test/utility/test_util.h4
-rwxr-xr-xsrc/third_party/wiredtiger/tools/optrack/find-latency-spikes.py2
84 files changed, 2589 insertions, 938 deletions
diff --git a/src/third_party/wiredtiger/SConstruct b/src/third_party/wiredtiger/SConstruct
index 20916ac8da0..786dff93a6d 100644
--- a/src/third_party/wiredtiger/SConstruct
+++ b/src/third_party/wiredtiger/SConstruct
@@ -290,12 +290,13 @@ Default(wtlib, wtdll)
wtbin = env.Program("wt", [
"src/utilities/util_alter.c",
"src/utilities/util_backup.c",
- "src/utilities/util_cpyright.c",
"src/utilities/util_compact.c",
+ "src/utilities/util_cpyright.c",
"src/utilities/util_create.c",
"src/utilities/util_downgrade.c",
"src/utilities/util_drop.c",
"src/utilities/util_dump.c",
+ "src/utilities/util_import.c",
"src/utilities/util_list.c",
"src/utilities/util_load.c",
"src/utilities/util_load_json.c",
diff --git a/src/third_party/wiredtiger/build_posix/Make.base b/src/third_party/wiredtiger/build_posix/Make.base
index 7543b346ea5..db8ae2fabf4 100644
--- a/src/third_party/wiredtiger/build_posix/Make.base
+++ b/src/third_party/wiredtiger/build_posix/Make.base
@@ -19,6 +19,7 @@ wt_SOURCES =\
src/utilities/util_downgrade.c \
src/utilities/util_drop.c \
src/utilities/util_dump.c \
+ src/utilities/util_import.c \
src/utilities/util_list.c \
src/utilities/util_load.c \
src/utilities/util_load_json.c \
diff --git a/src/third_party/wiredtiger/build_posix/Make.subdirs b/src/third_party/wiredtiger/build_posix/Make.subdirs
index d598c74c631..a7fc291fc61 100644
--- a/src/third_party/wiredtiger/build_posix/Make.subdirs
+++ b/src/third_party/wiredtiger/build_posix/Make.subdirs
@@ -37,6 +37,7 @@ test/cursor_order
test/fops
test/format
test/huge
+test/import
test/manydbs
test/packing
test/readonly
diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py
index 37b84df00e1..146d6c06668 100644
--- a/src/third_party/wiredtiger/dist/api_data.py
+++ b/src/third_party/wiredtiger/dist/api_data.py
@@ -495,6 +495,12 @@ connection_runtime_config = [
adjust log archiving to retain the log records of this number
of checkpoints. Zero or one means perform normal archiving.''',
min='0', max='1024'),
+ Config('eviction', 'false', r'''
+ if true, modify internal algorithms to change skew to force
+ lookaside eviction to happen more aggressively. This includes but
+ is not limited to not skewing newest, not favoring leaf pages,
+ and modifying the eviction score mechanism.''',
+ type='boolean'),
Config('rollback_error', '0', r'''
return a WT_ROLLBACK error from a transaction operation about
every Nth operation to simulate a collision''',
@@ -1109,6 +1115,8 @@ methods = {
type='boolean'),
]),
+'WT_SESSION.import' : Method([]),
+
'WT_SESSION.join' : Method([
Config('compare', '"eq"', r'''
modifies the set of items to be returned so that the index key
diff --git a/src/third_party/wiredtiger/dist/filelist b/src/third_party/wiredtiger/dist/filelist
index 036b1a8b1a9..c529ebc0366 100644
--- a/src/third_party/wiredtiger/dist/filelist
+++ b/src/third_party/wiredtiger/dist/filelist
@@ -6,6 +6,7 @@ src/async/async_op.c
src/async/async_worker.c
src/block/block_addr.c
src/block/block_ckpt.c
+src/block/block_ckpt_scan.c
src/block/block_compact.c
src/block/block_ext.c
src/block/block_map.c
@@ -26,6 +27,7 @@ src/btree/bt_delete.c
src/btree/bt_discard.c
src/btree/bt_handle.c
src/btree/bt_huffman.c
+src/btree/bt_import.c
src/btree/bt_io.c
src/btree/bt_misc.c
src/btree/bt_ovfl.c
@@ -185,7 +187,6 @@ src/schema/schema_worker.c
src/session/session_api.c
src/session/session_compact.c
src/session/session_dhandle.c
-src/session/session_salvage.c
src/support/cond_auto.c
src/support/crypto.c
src/support/err.c
diff --git a/src/third_party/wiredtiger/dist/s_define.list b/src/third_party/wiredtiger/dist/s_define.list
index e82ef2ad31f..1c9121da3f2 100644
--- a/src/third_party/wiredtiger/dist/s_define.list
+++ b/src/third_party/wiredtiger/dist/s_define.list
@@ -16,6 +16,7 @@ WT_ATOMIC_CAS
WT_ATOMIC_CAS_FUNC
WT_ATOMIC_FUNC
WT_BLOCK_DESC_SIZE
+WT_BLOCK_EXTLIST_VERSION_ORIG
WT_BLOCK_HEADER_SIZE
WT_CACHE_LINE_ALIGNMENT
WT_CACHE_LINE_PAD_BEGIN
diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok
index 06e58b46236..d8f89a9e6b9 100644
--- a/src/third_party/wiredtiger/dist/s_string.ok
+++ b/src/third_party/wiredtiger/dist/s_string.ok
@@ -105,6 +105,7 @@ DeadStores
Decrement
Decrypt
DeleteFileW
+Dh
EACCES
EAGAIN
EB
@@ -526,6 +527,7 @@ buf
bufsz
bugfix
bugfixes
+builddir
builtin
builtins
bytelock
diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py
index 478830108f3..92008c8f7e0 100644
--- a/src/third_party/wiredtiger/dist/stat_data.py
+++ b/src/third_party/wiredtiger/dist/stat_data.py
@@ -514,6 +514,8 @@ connection_stats = [
SessionOpStat('session_table_create_success', 'table create successful calls', 'no_clear,no_scale'),
SessionOpStat('session_table_drop_fail', 'table drop failed calls', 'no_clear,no_scale'),
SessionOpStat('session_table_drop_success', 'table drop successful calls', 'no_clear,no_scale'),
+ SessionOpStat('session_table_import_fail', 'table import failed calls', 'no_clear,no_scale'),
+ SessionOpStat('session_table_import_success', 'table import successful calls', 'no_clear,no_scale'),
SessionOpStat('session_table_rebalance_fail', 'table rebalance failed calls', 'no_clear,no_scale'),
SessionOpStat('session_table_rebalance_success', 'table rebalance successful calls', 'no_clear,no_scale'),
SessionOpStat('session_table_rename_fail', 'table rename failed calls', 'no_clear,no_scale'),
diff --git a/src/third_party/wiredtiger/examples/c/ex_all.c b/src/third_party/wiredtiger/examples/c/ex_all.c
index 9f1e1df17f3..5e5dbee6c03 100644
--- a/src/third_party/wiredtiger/examples/c/ex_all.c
+++ b/src/third_party/wiredtiger/examples/c/ex_all.c
@@ -603,16 +603,8 @@ named_snapshot_ops(WT_SESSION *session)
}
static void
-session_ops(WT_SESSION *session)
+session_ops_create(WT_SESSION *session)
{
- WT_CONNECTION *conn;
-
- conn = session->connection;
-
- /*! [Reconfigure a session] */
- error_check(session->reconfigure(session, "isolation=snapshot"));
- /*! [Reconfigure a session] */
-
/*! [Create a table] */
error_check(session->create(session,
"table:mytable", "key_format=S,value_format=S"));
@@ -622,12 +614,6 @@ session_ops(WT_SESSION *session)
/*! [Create a column-store table] */
error_check(session->create(session,
"table:mytable", "key_format=r,value_format=S"));
-
- /*! [Alter a table] */
- error_check(session->alter(session,
- "table:mytable", "access_pattern_hint=random"));
- /*! [Alter a table] */
-
/*! [Create a column-store table] */
error_check(session->drop(session, "table:mytable", NULL));
@@ -736,16 +722,41 @@ session_ops(WT_SESSION *session)
"key_format=r,value_format=S,cache_resident=true"));
/*! [Create a cache-resident object] */
error_check(session->drop(session, "table:mytable", NULL));
+}
+static void
+session_ops(WT_SESSION *session)
+{
+ WT_CONNECTION *conn;
+
+ conn = session->connection;
+
+ /* WT_SESSION.create operations. */
+ session_ops_create(session);
+
+ /*! [Reconfigure a session] */
+ error_check(session->reconfigure(session, "isolation=snapshot"));
+ /*! [Reconfigure a session] */
{
/* Create a table for the session operations. */
error_check(session->create(
session, "table:mytable", "key_format=S,value_format=S"));
+ /*! [Alter a table] */
+ error_check(session->alter(session,
+ "table:mytable", "access_pattern_hint=random"));
+ /*! [Alter a table] */
+
/*! [Compact a table] */
error_check(session->compact(session, "table:mytable", NULL));
/*! [Compact a table] */
+#ifdef MIGHT_NOT_RUN
+ /*! [Import a file] */
+ error_check(session->import(session, "file:import", NULL));
+ /*! [Import a file] */
+#endif
+
/*! [Rebalance a table] */
error_check(session->rebalance(session, "table:mytable", NULL));
/*! [Rebalance a table] */
diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index 22114815861..d5a8a1b8330 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -1,5 +1,5 @@
{
- "commit": "9b85ad89688bd72b8a649d844a7e458832955764",
+ "commit": "40e3225e55ffbf0c9819af138fa4a014c0639c72",
"github": "wiredtiger/wiredtiger.git",
"vendor": "wiredtiger",
"branch": "mongodb-4.2"
diff --git a/src/third_party/wiredtiger/lang/java/java_doc.i b/src/third_party/wiredtiger/lang/java/java_doc.i
index 8b363ee4cf7..fa6a4c883c4 100644
--- a/src/third_party/wiredtiger/lang/java/java_doc.i
+++ b/src/third_party/wiredtiger/lang/java/java_doc.i
@@ -36,6 +36,7 @@ COPYDOC(__wt_session, WT_SESSION, reconfigure)
COPYDOC(__wt_session, WT_SESSION, open_cursor)
COPYDOC(__wt_session, WT_SESSION, alter)
COPYDOC(__wt_session, WT_SESSION, create)
+COPYDOC(__wt_session, WT_SESSION, import)
COPYDOC(__wt_session, WT_SESSION, compact)
COPYDOC(__wt_session, WT_SESSION, drop)
COPYDOC(__wt_session, WT_SESSION, join)
diff --git a/src/third_party/wiredtiger/src/block/block_addr.c b/src/third_party/wiredtiger/src/block/block_addr.c
index f08e4430a73..d7b0c56bd56 100644
--- a/src/third_party/wiredtiger/src/block/block_addr.c
+++ b/src/third_party/wiredtiger/src/block/block_addr.c
@@ -206,7 +206,7 @@ __wt_block_ckpt_decode(WT_SESSION *wt_session,
*/
int
__wt_block_ckpt_to_buffer(WT_SESSION_IMPL *session,
- WT_BLOCK *block, uint8_t **pp, WT_BLOCK_CKPT *ci)
+ WT_BLOCK *block, uint8_t **pp, WT_BLOCK_CKPT *ci, bool skip_avail)
{
uint64_t a;
@@ -220,8 +220,11 @@ __wt_block_ckpt_to_buffer(WT_SESSION_IMPL *session,
ci->root_offset, ci->root_size, ci->root_checksum));
WT_RET(__wt_block_addr_to_buffer(block, pp,
ci->alloc.offset, ci->alloc.size, ci->alloc.checksum));
- WT_RET(__wt_block_addr_to_buffer(block, pp,
- ci->avail.offset, ci->avail.size, ci->avail.checksum));
+ if (skip_avail)
+ WT_RET(__wt_block_addr_to_buffer(block, pp, 0, 0, 0));
+ else
+ WT_RET(__wt_block_addr_to_buffer(block, pp,
+ ci->avail.offset, ci->avail.size, ci->avail.checksum));
WT_RET(__wt_block_addr_to_buffer(block, pp,
ci->discard.offset, ci->discard.size, ci->discard.checksum));
a = (uint64_t)ci->file_size;
@@ -231,3 +234,81 @@ __wt_block_ckpt_to_buffer(WT_SESSION_IMPL *session,
return (0);
}
+
+/*
+ * __wt_ckpt_verbose --
+ * Display a printable string representation of a checkpoint.
+ */
+void
+__wt_ckpt_verbose(WT_SESSION_IMPL *session, WT_BLOCK *block,
+ const char *tag, const char *ckpt_name, const uint8_t *ckpt_string)
+{
+ WT_BLOCK_CKPT *ci, _ci;
+ WT_DECL_ITEM(tmp);
+ WT_DECL_RET;
+
+ if (ckpt_string == NULL) {
+ __wt_verbose_worker(session,
+ "%s: %s: %s%s[Empty]", block->name, tag,
+ ckpt_name ? ckpt_name : "",
+ ckpt_name ? ": " : "");
+ return;
+ }
+
+ /* Initialize the checkpoint, crack the cookie. */
+ ci = &_ci;
+ WT_ERR(__wt_block_ckpt_init(session, ci, "string"));
+ WT_ERR(__wt_block_buffer_to_ckpt(session, block, ckpt_string, ci));
+
+ WT_ERR(__wt_scr_alloc(session, 0, &tmp));
+ WT_ERR(__wt_buf_fmt(session, tmp, "version=%" PRIu8, ci->version));
+ if (ci->root_offset == WT_BLOCK_INVALID_OFFSET)
+ WT_ERR(__wt_buf_catfmt(session, tmp, ", root=[Empty]"));
+ else
+ WT_ERR(__wt_buf_catfmt(session, tmp,
+ ", root=[%"
+ PRIuMAX "-%" PRIuMAX ", %" PRIu32 ", %" PRIu32 "]",
+ (uintmax_t)ci->root_offset,
+ (uintmax_t)(ci->root_offset + ci->root_size),
+ ci->root_size, ci->root_checksum));
+ if (ci->alloc.offset == WT_BLOCK_INVALID_OFFSET)
+ WT_ERR(__wt_buf_catfmt(session, tmp, ", alloc=[Empty]"));
+ else
+ WT_ERR(__wt_buf_catfmt(session, tmp,
+ ", alloc=[%"
+ PRIuMAX "-%" PRIuMAX ", %" PRIu32 ", %" PRIu32 "]",
+ (uintmax_t)ci->alloc.offset,
+ (uintmax_t)(ci->alloc.offset + ci->alloc.size),
+ ci->alloc.size, ci->alloc.checksum));
+ if (ci->avail.offset == WT_BLOCK_INVALID_OFFSET)
+ WT_ERR(__wt_buf_catfmt(session, tmp, ", avail=[Empty]"));
+ else
+ WT_ERR(__wt_buf_catfmt(session, tmp,
+ ", avail=[%"
+ PRIuMAX "-%" PRIuMAX ", %" PRIu32 ", %" PRIu32 "]",
+ (uintmax_t)ci->avail.offset,
+ (uintmax_t)(ci->avail.offset + ci->avail.size),
+ ci->avail.size, ci->avail.checksum));
+ if (ci->discard.offset == WT_BLOCK_INVALID_OFFSET)
+ WT_ERR(__wt_buf_catfmt(session, tmp, ", discard=[Empty]"));
+ else
+ WT_ERR(__wt_buf_catfmt(session, tmp,
+ ", discard=[%"
+ PRIuMAX "-%" PRIuMAX ", %" PRIu32 ", %" PRIu32 "]",
+ (uintmax_t)ci->discard.offset,
+ (uintmax_t)(ci->discard.offset + ci->discard.size),
+ ci->discard.size, ci->discard.checksum));
+ WT_ERR(__wt_buf_catfmt(session, tmp,
+ ", file size=%" PRIuMAX, (uintmax_t)ci->file_size));
+ WT_ERR(__wt_buf_catfmt(session, tmp,
+ ", checkpoint size=%" PRIu64, ci->ckpt_size));
+
+ __wt_verbose_worker(session,
+ "%s: %s: %s%s%s",
+ block->name, tag,
+ ckpt_name ? ckpt_name : "",
+ ckpt_name ? ": " : "", (const char *)tmp->data);
+
+err: __wt_scr_free(session, &tmp);
+ __wt_block_ckpt_destroy(session, ci);
+}
diff --git a/src/third_party/wiredtiger/src/block/block_ckpt.c b/src/third_party/wiredtiger/src/block/block_ckpt.c
index 40b0fef5a29..db6b458d671 100644
--- a/src/third_party/wiredtiger/src/block/block_ckpt.c
+++ b/src/third_party/wiredtiger/src/block/block_ckpt.c
@@ -9,10 +9,8 @@
#include "wt_internal.h"
static int __ckpt_process(WT_SESSION_IMPL *, WT_BLOCK *, WT_CKPT *);
-static int __ckpt_string(
- WT_SESSION_IMPL *, WT_BLOCK *, const uint8_t *, WT_ITEM *);
-static int __ckpt_update(
- WT_SESSION_IMPL *, WT_BLOCK *, WT_CKPT *, WT_BLOCK_CKPT *, bool);
+static int __ckpt_update(WT_SESSION_IMPL *,
+ WT_BLOCK *, WT_CKPT *, WT_CKPT *, WT_BLOCK_CKPT *, bool);
/*
* __wt_block_ckpt_init --
@@ -49,7 +47,6 @@ __wt_block_checkpoint_load(WT_SESSION_IMPL *session, WT_BLOCK *block,
uint8_t *root_addr, size_t *root_addr_sizep, bool checkpoint)
{
WT_BLOCK_CKPT *ci, _ci;
- WT_DECL_ITEM(tmp);
WT_DECL_RET;
uint8_t *endp;
@@ -62,15 +59,8 @@ __wt_block_checkpoint_load(WT_SESSION_IMPL *session, WT_BLOCK *block,
ci = NULL;
- if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT)) {
- if (addr != NULL) {
- WT_ERR(__wt_scr_alloc(session, 0, &tmp));
- WT_ERR(__ckpt_string(session, block, addr, tmp));
- }
- __wt_verbose(session, WT_VERB_CHECKPOINT,
- "%s: load-checkpoint: %s", block->name,
- addr == NULL ? "[Empty]" : (const char *)tmp->data);
- }
+ if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT))
+ __wt_ckpt_verbose(session, block, "load", NULL, addr);
/*
* There's a single checkpoint in the file that can be written, all of
@@ -153,7 +143,6 @@ err: /*
if (checkpoint && ci != NULL)
__wt_block_ckpt_destroy(session, ci);
- __wt_scr_free(session, &tmp);
return (ret);
}
@@ -388,7 +377,6 @@ __ckpt_process(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckptbase)
{
WT_BLOCK_CKPT *a, *b, *ci;
WT_CKPT *ckpt, *next_ckpt;
- WT_DECL_ITEM(tmp);
WT_DECL_RET;
uint64_t ckpt_size;
bool deleting, fatal, locked;
@@ -544,15 +532,10 @@ __ckpt_process(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckptbase)
!F_ISSET(ckpt, WT_CKPT_DELETE))
continue;
- if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT)) {
- if (tmp == NULL)
- WT_ERR(__wt_scr_alloc(session, 0, &tmp));
- WT_ERR(__ckpt_string(
- session, block, ckpt->raw.data, tmp));
- __wt_verbose(session, WT_VERB_CHECKPOINT,
- "%s: delete-checkpoint: %s: %s",
- block->name, ckpt->name, (const char *)tmp->data);
- }
+ if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT))
+ __wt_ckpt_verbose(session,
+ block, "delete", ckpt->name, ckpt->raw.data);
+
/*
* Find the checkpoint into which we'll roll this checkpoint's
* blocks: it's the next real checkpoint in the list, and it
@@ -642,8 +625,8 @@ __ckpt_process(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckptbase)
/* Update checkpoints marked for update. */
WT_CKPT_FOREACH(ckptbase, ckpt)
if (F_ISSET(ckpt, WT_CKPT_UPDATE))
- WT_ERR(__ckpt_update(
- session, block, ckpt, ckpt->bpriv, false));
+ WT_ERR(__ckpt_update(session,
+ block, ckptbase, ckpt, ckpt->bpriv, false));
live_update:
/* Truncate the file if that's possible. */
@@ -686,7 +669,8 @@ live_update:
ci->ckpt_size =
WT_MIN(ckpt_size, (uint64_t)block->size);
- WT_ERR(__ckpt_update(session, block, ckpt, ci, true));
+ WT_ERR(__ckpt_update(
+ session, block, ckptbase, ckpt, ci, true));
}
/*
@@ -733,7 +717,6 @@ err: if (ret != 0 && fatal) {
if ((ci = ckpt->bpriv) != NULL)
__wt_block_ckpt_destroy(session, ci);
- __wt_scr_free(session, &tmp);
return (ret);
}
@@ -742,10 +725,10 @@ err: if (ret != 0 && fatal) {
* Update a checkpoint.
*/
static int
-__ckpt_update(WT_SESSION_IMPL *session,
- WT_BLOCK *block, WT_CKPT *ckpt, WT_BLOCK_CKPT *ci, bool is_live)
+__ckpt_update(WT_SESSION_IMPL *session, WT_BLOCK *block,
+ WT_CKPT *ckptbase, WT_CKPT *ckpt, WT_BLOCK_CKPT *ci, bool is_live)
{
- WT_DECL_ITEM(tmp);
+ WT_DECL_ITEM(a);
WT_DECL_RET;
uint8_t *endp;
@@ -764,6 +747,37 @@ __ckpt_update(WT_SESSION_IMPL *session,
WT_RET(__wt_block_extlist_write(session, block, &ci->discard, NULL));
/*
+ * If this is the final block, we append an incomplete copy of the
+ * checkpoint information to the avail list for standalone retrieval.
+ */
+ if (is_live) {
+ /*
+ * Copy the INCOMPLETE checkpoint information into the
+ * checkpoint.
+ */
+ WT_RET(__wt_buf_init(
+ session, &ckpt->raw, WT_BLOCK_CHECKPOINT_BUFFER));
+ endp = ckpt->raw.mem;
+ WT_RET(__wt_block_ckpt_to_buffer(
+ session, block, &endp, ci, true));
+ ckpt->raw.size = WT_PTRDIFF(endp, ckpt->raw.mem);
+
+ /*
+ * Convert the INCOMPLETE checkpoint array into its metadata
+ * representation. This must match what is eventually written
+ * into the metadata file, in other words, everything must be
+ * initialized before the block manager does the checkpoint.
+ */
+ WT_RET(__wt_scr_alloc(session, 8 * 1024, &a));
+ ret = __wt_meta_ckptlist_to_meta(session, ckptbase, a);
+ if (ret == 0)
+ ret = __wt_strndup(
+ session, a->data, a->size, &ckpt->block_checkpoint);
+ __wt_scr_free(session, &a);
+ WT_RET(ret);
+ }
+
+ /*
* We only write an avail list for the live system, other checkpoint's
* avail lists are static and never change.
*
@@ -775,9 +789,13 @@ __ckpt_update(WT_SESSION_IMPL *session,
* it's not truly available until the new checkpoint locations have been
* saved to the metadata.
*/
- if (is_live)
- WT_RET(__wt_block_extlist_write(
- session, block, &ci->avail, &ci->ckpt_avail));
+ if (is_live) {
+ block->final_ckpt = ckpt;
+ ret = __wt_block_extlist_write(
+ session, block, &ci->avail, &ci->ckpt_avail);
+ block->final_ckpt = NULL;
+ WT_RET(ret);
+ }
/*
* Set the file size for the live system.
@@ -800,25 +818,17 @@ __ckpt_update(WT_SESSION_IMPL *session,
if (is_live)
ci->file_size = block->size;
- /*
- * Copy the checkpoint information into the checkpoint array's address
- * cookie.
- */
- WT_RET(__wt_buf_init(session, &ckpt->raw, WT_BTREE_MAX_ADDR_COOKIE));
+ /* Copy the COMPLETE checkpoint information into the checkpoint. */
+ WT_RET(__wt_buf_init(session, &ckpt->raw, WT_BLOCK_CHECKPOINT_BUFFER));
endp = ckpt->raw.mem;
- WT_RET(__wt_block_ckpt_to_buffer(session, block, &endp, ci));
+ WT_RET(__wt_block_ckpt_to_buffer(session, block, &endp, ci, false));
ckpt->raw.size = WT_PTRDIFF(endp, ckpt->raw.mem);
- if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT)) {
- WT_RET(__wt_scr_alloc(session, 0, &tmp));
- WT_ERR(__ckpt_string(session, block, ckpt->raw.data, tmp));
- __wt_verbose(session, WT_VERB_CHECKPOINT,
- "%s: create-checkpoint: %s: %s",
- block->name, ckpt->name, (const char *)tmp->data);
- }
+ if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT))
+ __wt_ckpt_verbose(
+ session, block, "create", ckpt->name, ckpt->raw.data);
-err: __wt_scr_free(session, &tmp);
- return (ret);
+ return (0);
}
/*
@@ -882,64 +892,3 @@ err: __wt_spin_unlock(session, &block->live_lock);
return (ret);
}
-
-/*
- * __ckpt_string --
- * Return a printable string representation of a checkpoint address cookie.
- */
-static int
-__ckpt_string(WT_SESSION_IMPL *session,
- WT_BLOCK *block, const uint8_t *addr, WT_ITEM *buf)
-{
- WT_BLOCK_CKPT *ci, _ci;
-
- /* Initialize the checkpoint, crack the cookie. */
- ci = &_ci;
- WT_RET(__wt_block_ckpt_init(session, ci, "string"));
- WT_RET(__wt_block_buffer_to_ckpt(session, block, addr, ci));
-
- WT_RET(__wt_buf_fmt(session, buf,
- "version=%" PRIu8, ci->version));
- if (ci->root_offset == WT_BLOCK_INVALID_OFFSET)
- WT_RET(__wt_buf_catfmt(session, buf, ", root=[Empty]"));
- else
- WT_RET(__wt_buf_catfmt(session, buf,
- ", root=[%"
- PRIuMAX "-%" PRIuMAX ", %" PRIu32 ", %" PRIu32 "]",
- (uintmax_t)ci->root_offset,
- (uintmax_t)(ci->root_offset + ci->root_size),
- ci->root_size, ci->root_checksum));
- if (ci->alloc.offset == WT_BLOCK_INVALID_OFFSET)
- WT_RET(__wt_buf_catfmt(session, buf, ", alloc=[Empty]"));
- else
- WT_RET(__wt_buf_catfmt(session, buf,
- ", alloc=[%"
- PRIuMAX "-%" PRIuMAX ", %" PRIu32 ", %" PRIu32 "]",
- (uintmax_t)ci->alloc.offset,
- (uintmax_t)(ci->alloc.offset + ci->alloc.size),
- ci->alloc.size, ci->alloc.checksum));
- if (ci->avail.offset == WT_BLOCK_INVALID_OFFSET)
- WT_RET(__wt_buf_catfmt(session, buf, ", avail=[Empty]"));
- else
- WT_RET(__wt_buf_catfmt(session, buf,
- ", avail=[%"
- PRIuMAX "-%" PRIuMAX ", %" PRIu32 ", %" PRIu32 "]",
- (uintmax_t)ci->avail.offset,
- (uintmax_t)(ci->avail.offset + ci->avail.size),
- ci->avail.size, ci->avail.checksum));
- if (ci->discard.offset == WT_BLOCK_INVALID_OFFSET)
- WT_RET(__wt_buf_catfmt(session, buf, ", discard=[Empty]"));
- else
- WT_RET(__wt_buf_catfmt(session, buf,
- ", discard=[%"
- PRIuMAX "-%" PRIuMAX ", %" PRIu32 ", %" PRIu32 "]",
- (uintmax_t)ci->discard.offset,
- (uintmax_t)(ci->discard.offset + ci->discard.size),
- ci->discard.size, ci->discard.checksum));
- WT_RET(__wt_buf_catfmt(session, buf,
- ", file size=%" PRIuMAX, (uintmax_t)ci->file_size));
-
- __wt_block_ckpt_destroy(session, ci);
-
- return (0);
-}
diff --git a/src/third_party/wiredtiger/src/block/block_ckpt_scan.c b/src/third_party/wiredtiger/src/block/block_ckpt_scan.c
new file mode 100644
index 00000000000..5db299c399e
--- /dev/null
+++ b/src/third_party/wiredtiger/src/block/block_ckpt_scan.c
@@ -0,0 +1,417 @@
+/*-
+ * Copyright (c) 2014-2019 MongoDB, Inc.
+ * Copyright (c) 2008-2014 WiredTiger, Inc.
+ * All rights reserved.
+ *
+ * See the file LICENSE for redistribution information.
+ */
+
+#include "wt_internal.h"
+
+/*
+ * It wasn't possible to open standalone files in historic WiredTiger databases,
+ * you're done if you lose the file's associated metadata. That was a mistake
+ * and this code is the workaround. What we need to crack a file is database
+ * metadata plus a list of active checkpoints as of the file's clean shutdown
+ * (normally stored in the database metadata). The last write done in a block
+ * manager's checkpoint is the avail list. If current metadata and checkpoint
+ * information is included in that write, we're close. We can open the file,
+ * read the blocks, scan until we find the avail list, and read the metadata
+ * and checkpoint information from there.
+ * Two problems remain: first, the checkpoint information isn't correct
+ * until we write the avail list and the checkpoint information has to include
+ * the avail list address plus the final file size after the write. Fortunately,
+ * when scanning the file for the avail lists, we're figuring out exactly the
+ * information needed to fix up the checkpoint information we wrote, that is,
+ * the avail list's offset, size and checksum triplet. As for the final file
+ * size, we allocate all space in the file before we calculate block checksums,
+ * so we can do that space allocation, then fill in the final file size before
+ * calculating the checksum and writing the actual block.
+ * The second problem is we have to be able to find the avail lists that
+ * include checkpoint information (ignoring previous files created by previous
+ * releases, and, of course, making upgrade/downgrade work seamlessly). Extent
+ * lists are written to their own pages, and we could version this change using
+ * the page header version. Extent lists have WT_PAGE_BLOCK_MANAGER page types,
+ * we could version this change using the upcoming WT_PAGE_VERSION_TS upgrade.
+ * However, that requires waiting a release (we would have to first release a
+ * version that ignores those new page header versions so downgrade works), and
+ * we're not planning a release that writes WT_PAGE_VERSION_TS page headers for
+ * awhile. Happily, historic WiredTiger releases have a bug. Extent lists
+ * consist of a set of offset/size pairs, with magic offset/size pairs at the
+ * beginning and end of the list. Historic releases only verified the offset of
+ * the special pair at the end of the list, ignoring the size. To detect avail
+ * lists that include appended metadata and checkpoint information, this change
+ * adds a version to the extent list: if size is WT_BLOCK_EXTLIST_VERSION_CKPT,
+ * then metadata/checkpoint information follows.
+ */
+
+/*
+ * __wt_block_checkpoint_final --
+ * Append metadata and checkpoint information to a buffer.
+ */
+int
+__wt_block_checkpoint_final(WT_SESSION_IMPL *session,
+ WT_BLOCK *block, WT_ITEM *buf, uint8_t **file_sizep)
+{
+ WT_CKPT *ckpt;
+ size_t align_size, file_size_offset, len, size;
+ uint8_t *p;
+
+ *file_sizep = 0;
+
+ ckpt = block->final_ckpt;
+ p = (uint8_t *)buf->mem + buf->size;
+
+ /*
+ * First, add in a counter to uniquely order checkpoints at our level.
+ * There's order and time information in the checkpoint itself, but the
+ * order isn't written and the time is only at second granularity.
+ * I'm using the Btree write generation for this purpose. That's
+ * safe and guaranteed correct because everything is locked down for the
+ * checkpoint, we're the only writer. Plus, because we use the write
+ * generation as a database connection generation, it's guaranteed to
+ * move forward and never repeat.
+ * It's a layering violation though, this is the only place the
+ * block manager uses the write generation. The alternative would be to
+ * add our own write-generation scheme in the block manager, storing a
+ * value and recovering it when we open the file. We could do that, as
+ * reading the final avail list when a file is opened is unavoidable,
+ * so we can retrieve the value written here when we open the file, but
+ * this approach is simpler.
+ */
+ size = buf->size + WT_INTPACK64_MAXSIZE;
+ WT_RET(__wt_buf_extend(session, buf, size));
+ p = (uint8_t *)buf->mem + buf->size;
+ WT_RET(__wt_vpack_uint(&p, 0, ++S2BT(session)->write_gen));
+ buf->size = WT_PTRDIFF(p, buf->mem);
+
+ /*
+ * Second, add space for the final file size as a packed value. We don't
+ * know how large it will be so skip the maximum required space.
+ */
+ size = buf->size + WT_INTPACK64_MAXSIZE;
+ WT_RET(__wt_buf_extend(session, buf, size));
+ file_size_offset = buf->size;
+ buf->size = size;
+
+ /* 3a, copy the metadata length into the buffer. */
+ len = strlen(ckpt->block_metadata);
+ size = buf->size + WT_INTPACK64_MAXSIZE;
+ WT_RET(__wt_buf_extend(session, buf, size));
+ p = (uint8_t *)buf->mem + buf->size;
+ WT_RET(__wt_vpack_uint(&p, 0, (uint64_t)len));
+ buf->size = WT_PTRDIFF(p, buf->mem);
+
+ /* 3b, copy the metadata into the buffer. */
+ size = buf->size + len;
+ WT_RET(__wt_buf_extend(session, buf, size));
+ p = (uint8_t *)buf->mem + buf->size;
+ memcpy(p, ckpt->block_metadata, len);
+ buf->size = size;
+
+ /* 4a, copy the checkpoint list length into the buffer. */
+ len = strlen(ckpt->block_checkpoint);
+ size = buf->size + WT_INTPACK64_MAXSIZE;
+ WT_RET(__wt_buf_extend(session, buf, size));
+ p = (uint8_t *)buf->mem + buf->size;
+ WT_RET(__wt_vpack_uint(&p, 0, (uint64_t)len));
+ buf->size = WT_PTRDIFF(p, buf->mem);
+
+ /* 4b, copy the checkpoint list into the buffer. */
+ size = buf->size + len;
+ WT_RET(__wt_buf_extend(session, buf, size));
+ p = (uint8_t *)buf->mem + buf->size;
+ memcpy(p, ckpt->block_checkpoint, len);
+ buf->size = size;
+
+ /*
+ * 5a, copy the not-quite-right checkpoint information length into the
+ * buffer.
+ */
+ len = ckpt->raw.size;
+ size = buf->size + WT_INTPACK64_MAXSIZE;
+ WT_RET(__wt_buf_extend(session, buf, size));
+ p = (uint8_t *)buf->mem + buf->size;
+ WT_RET(__wt_vpack_uint(&p, 0, (uint64_t)len));
+ buf->size = WT_PTRDIFF(p, buf->mem);
+
+ /*
+ * 5b, copy the not-quite-right checkpoint information into the buffer.
+ */
+ size = buf->size + len;
+ WT_RET(__wt_buf_extend(session, buf, size));
+ p = (uint8_t *)buf->mem + buf->size;
+ memcpy(p, ckpt->raw.data, len);
+ buf->size = size;
+
+ /*
+ * We might have grown the buffer beyond the original allocation size,
+ * make sure that we're still in compliance.
+ */
+ align_size = WT_ALIGN(buf->size, block->allocsize);
+ if (align_size > buf->memsize)
+ WT_RET(__wt_buf_extend(session, buf, align_size));
+
+ *file_sizep = (uint8_t *)buf->mem + file_size_offset;
+
+ return (0);
+}
+
+struct saved_block_info {
+ uint64_t write_gen;
+ wt_off_t offset;
+ uint32_t size;
+ uint32_t checksum;
+ uint64_t file_size;
+
+ char *metadata;
+ char *checkpoint_list;
+
+ WT_ITEM *checkpoint;
+};
+
+/*
+ * __block_checkpoint_update --
+ * Update the checkpoint information for the file.
+ */
+static int
+__block_checkpoint_update(
+ WT_SESSION_IMPL *session, WT_BLOCK *block, struct saved_block_info *info)
+{
+ WT_BLOCK_CKPT ci;
+ WT_ITEM *checkpoint;
+ uint8_t *endp;
+
+ memset(&ci, 0, sizeof(ci));
+ checkpoint = info->checkpoint;
+
+ if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT))
+ __wt_ckpt_verbose(
+ session, block, "import original", NULL, checkpoint->mem);
+
+ /*
+ * Convert the final checkpoint data blob to a WT_BLOCK_CKPT structure,
+ * update it with the avail list information, and convert it back to a
+ * data blob.
+ */
+ WT_RET(__wt_block_buffer_to_ckpt(
+ session, block, checkpoint->data, &ci));
+ ci.avail.offset = info->offset;
+ ci.avail.size = info->size;
+ ci.avail.checksum = info->checksum;
+ ci.file_size = (wt_off_t)info->file_size;
+ WT_RET(__wt_buf_extend(
+ session, checkpoint, WT_BLOCK_CHECKPOINT_BUFFER));
+ endp = checkpoint->mem;
+ WT_RET(__wt_block_ckpt_to_buffer(session, block, &endp, &ci, false));
+ checkpoint->size = WT_PTRDIFF(endp, checkpoint->mem);
+
+ if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT))
+ __wt_ckpt_verbose(
+ session, block, "import replace", NULL, checkpoint->mem);
+
+ return (0);
+}
+
+#define WT_BLOCK_SKIP(a) do { \
+ if ((a) != 0) \
+ continue; \
+} while (0)
+
+/*
+ * __wt_block_checkpoint_last --
+ * Scan a file for checkpoints, returning the last one we find.
+ */
+int
+__wt_block_checkpoint_last(WT_SESSION_IMPL *session, WT_BLOCK *block,
+ char **metadatap, char **checkpoint_listp, WT_ITEM *checkpoint)
+{
+ struct saved_block_info *best, _best, *current, _current, *saved_tmp;
+ WT_BLOCK_HEADER *blk;
+ WT_DECL_ITEM(tmp);
+ WT_DECL_RET;
+ WT_FH *fh;
+ const WT_PAGE_HEADER *dsk;
+ wt_off_t ext_off, ext_size, offset;
+ uint64_t len, nblocks, write_gen;
+ uint32_t checksum, size;
+ const uint8_t *p, *t;
+ bool found;
+
+ *metadatap = *checkpoint_listp = NULL;
+ __wt_buf_init(session, checkpoint, WT_BLOCK_CHECKPOINT_BUFFER);
+
+ /*
+ * Initialize a pair of structures that track the best and current
+ * checkpoints found so far. This is a little trickier than normal
+ * because we don't want to start saving a checkpoint only to find
+ * out it's not one we can use. I doubt that can happen and it
+ * suggests corruption, but half-a-checkpoint isn't a good place to
+ * be. Only swap to a new "best" checkpoint if we read the whole
+ * thing successfully.
+ *
+ * Don't re-order these lines: it's done this way so the WT_ITEMs
+ * are always initialized and error handling works.
+ */
+ memset((best = &_best), 0, sizeof(_best));
+ memset((current = &_current), 0, sizeof(_current));
+ WT_ERR(__wt_scr_alloc(session, 0, &best->checkpoint));
+ WT_ERR(__wt_scr_alloc(session, 0, &current->checkpoint));
+
+ found = false;
+ ext_off = 0; /* [-Werror=maybe-uninitialized] */
+ ext_size = 0;
+ len = write_gen = 0;
+
+ WT_ERR(__wt_scr_alloc(session, 64 * 1024, &tmp));
+
+ F_SET(session, WT_SESSION_QUIET_CORRUPT_FILE);
+
+ /*
+ * Scan the file for pages, using the minimum possible WiredTiger
+ * allocation size.
+ */
+ fh = block->fh;
+ for (nblocks = 0, offset = 0; offset < block->size; offset += size) {
+ /* Report progress occasionally. */
+#define WT_CHECKPOINT_LIST_PROGRESS_INTERVAL 100
+ if (++nblocks % WT_CHECKPOINT_LIST_PROGRESS_INTERVAL == 0)
+ WT_ERR(__wt_progress(session, NULL, nblocks));
+
+ /*
+ * Read the start of a possible page and get a block length from
+ * it. Move to the next allocation sized boundary, we'll never
+ * consider this one again.
+ */
+ if ((ret = __wt_read(session, fh,
+ offset, (size_t)WT_BTREE_MIN_ALLOC_SIZE, tmp->mem)) != 0)
+ break;
+ blk = WT_BLOCK_HEADER_REF(tmp->mem);
+ __wt_block_header_byteswap(blk);
+ size = blk->disk_size;
+ checksum = blk->checksum;
+
+ /*
+ * Check the block size: if it's not insane, read the block.
+ * Reading the block validates any checksum. The file might
+ * reasonably have garbage at the end, and we're not here to
+ * detect that. Ignore problems, subsequent file verification
+ * can deal with any corruption. If the block isn't valid,
+ * skip to the next possible block.
+ */
+ if (__wt_block_offset_invalid(block, offset, size) ||
+ __wt_block_read_off(
+ session, block, tmp, offset, size, checksum) != 0) {
+ size = WT_BTREE_MIN_ALLOC_SIZE;
+ continue;
+ }
+
+ dsk = tmp->mem;
+ if (dsk->type != WT_PAGE_BLOCK_MANAGER)
+ continue;
+
+ p = WT_BLOCK_HEADER_BYTE(tmp->mem);
+ WT_BLOCK_SKIP(__wt_extlist_read_pair(&p, &ext_off, &ext_size));
+ if (ext_off != WT_BLOCK_EXTLIST_MAGIC || ext_size != 0)
+ continue;
+ for (;;) {
+ if ((ret = __wt_extlist_read_pair(
+ &p, &ext_off, &ext_size)) != 0)
+ break;
+ if (ext_off == WT_BLOCK_INVALID_OFFSET)
+ break;
+ }
+ if (ret != 0) {
+ ret = 0;
+ continue;
+ }
+ /*
+ * Note the less-than check of WT_BLOCK_EXTLIST_VERSION_CKPT,
+ * that way we can extend this with additional values in the
+ * future.
+ */
+ if (ext_size < WT_BLOCK_EXTLIST_VERSION_CKPT)
+ continue;
+
+ /*
+ * Skip any entries that aren't the most recent we've seen so
+ * far.
+ */
+ WT_BLOCK_SKIP(__wt_vunpack_uint(&p, 0, &write_gen));
+ if (write_gen < best->write_gen)
+ continue;
+
+ __wt_verbose(session, WT_VERB_CHECKPOINT,
+ "scan: checkpoint block at offset %" PRIuMAX
+ ", generation #%" PRIu64,
+ (uintmax_t)offset, write_gen);
+
+ current->write_gen = write_gen;
+ current->offset = offset;
+ current->size = size;
+ current->checksum = checksum;
+
+ /*
+ * The file size is in a fixed-size chunk of data, although it's
+ * packed (for portability).
+ */
+ t = p;
+ WT_BLOCK_SKIP(__wt_vunpack_uint(&t, 0, &current->file_size));
+ p += WT_INTPACK64_MAXSIZE;
+
+ /* Save a copy of the metadata. */
+ __wt_free(session, current->metadata);
+ WT_BLOCK_SKIP(__wt_vunpack_uint(&p, 0, &len));
+ WT_ERR(__wt_strndup(session, p, len, &current->metadata));
+ p += len;
+
+ /* Save a copy of the checkpoint list. */
+ __wt_free(session, current->checkpoint_list);
+ WT_BLOCK_SKIP(__wt_vunpack_uint(&p, 0, &len));
+ WT_ERR(__wt_strndup(
+ session, p, len, &current->checkpoint_list));
+ p += len;
+
+ /* Save a copy of the checkpoint information. */
+ WT_BLOCK_SKIP(__wt_vunpack_uint(&p, 0, &len));
+ WT_ERR(__wt_buf_set(session, current->checkpoint, p, len));
+
+ /* A new winner, swap the "best" and "current" information. */
+ saved_tmp = best;
+ best = current;
+ current = saved_tmp;
+ found = true;
+ }
+
+ if (!found)
+ WT_ERR_MSG(session, WT_NOTFOUND,
+ "%s: no final checkpoint found in file scan",
+ block->name);
+
+ /* Correct the checkpoint. */
+ WT_ERR(__block_checkpoint_update(session, block, best));
+
+ /*
+ * Copy the information out to our caller. Do the WT_ITEM first, it's
+ * the only thing left that can fail and simplifies error handling.
+ */
+ WT_ERR(__wt_buf_set(session,
+ checkpoint, best->checkpoint->data, best->checkpoint->size));
+ *metadatap = best->metadata;
+ best->metadata = NULL;
+ *checkpoint_listp = best->checkpoint_list;
+ best->checkpoint_list = NULL;
+
+err:
+ __wt_free(session, best->metadata);
+ __wt_free(session, best->checkpoint_list);
+ __wt_scr_free(session, &best->checkpoint);
+ __wt_free(session, current->metadata);
+ __wt_free(session, current->checkpoint_list);
+ __wt_scr_free(session, &current->checkpoint);
+
+ __wt_scr_free(session, &tmp);
+
+ F_CLR(session, WT_SESSION_QUIET_CORRUPT_FILE);
+ return (ret);
+}
diff --git a/src/third_party/wiredtiger/src/block/block_ext.c b/src/third_party/wiredtiger/src/block/block_ext.c
index 003f0489586..82e85658e22 100644
--- a/src/third_party/wiredtiger/src/block/block_ext.c
+++ b/src/third_party/wiredtiger/src/block/block_ext.c
@@ -1186,15 +1186,8 @@ __wt_block_extlist_read(WT_SESSION_IMPL *session,
WT_ERR(__wt_block_read_off(
session, block, tmp, el->offset, el->size, el->checksum));
-#define WT_EXTLIST_READ(p, v) do { \
- uint64_t _v; \
- WT_ERR(__wt_vunpack_uint(&(p), 0, &_v)); \
- (v) = (wt_off_t)_v; \
-} while (0)
-
p = WT_BLOCK_HEADER_BYTE(tmp->mem);
- WT_EXTLIST_READ(p, off);
- WT_EXTLIST_READ(p, size);
+ WT_ERR(__wt_extlist_read_pair(&p, &off, &size));
if (off != WT_BLOCK_EXTLIST_MAGIC || size != 0)
goto corrupted;
@@ -1210,8 +1203,7 @@ __wt_block_extlist_read(WT_SESSION_IMPL *session,
*/
func = el->track_size == 0 ? __block_append : __block_merge;
for (;;) {
- WT_EXTLIST_READ(p, off);
- WT_EXTLIST_READ(p, size);
+ WT_ERR(__wt_extlist_read_pair(&p, &off, &size));
if (off == WT_BLOCK_INVALID_OFFSET)
break;
@@ -1263,10 +1255,11 @@ __wt_block_extlist_write(WT_SESSION_IMPL *session,
/*
* Figure out how many entries we're writing -- if there aren't any
- * entries, we're done.
+ * entries, there's nothing to write, unless we still have to write
+ * the extent list to include the checkpoint recovery information.
*/
entries = el->entries + (additional == NULL ? 0 : additional->entries);
- if (entries == 0) {
+ if (entries == 0 && block->final_ckpt == NULL) {
el->offset = WT_BLOCK_INVALID_OFFSET;
el->checksum = el->size = 0;
return (0);
@@ -1286,25 +1279,21 @@ __wt_block_extlist_write(WT_SESSION_IMPL *session,
dsk = tmp->mem;
memset(dsk, 0, WT_BLOCK_HEADER_BYTE_SIZE);
dsk->type = WT_PAGE_BLOCK_MANAGER;
-
-#define WT_EXTLIST_WRITE(p, v) \
- WT_ERR(__wt_vpack_uint(&(p), 0, (uint64_t)(v)))
+ dsk->version = WT_PAGE_VERSION_TS;
/* Fill the page's data. */
p = WT_BLOCK_HEADER_BYTE(dsk);
- WT_EXTLIST_WRITE(p, WT_BLOCK_EXTLIST_MAGIC); /* Initial value */
- WT_EXTLIST_WRITE(p, 0);
- WT_EXT_FOREACH(ext, el->off) { /* Free ranges */
- WT_EXTLIST_WRITE(p, ext->off);
- WT_EXTLIST_WRITE(p, ext->size);
- }
+ /* Extent list starts */
+ WT_ERR(__wt_extlist_write_pair(&p, WT_BLOCK_EXTLIST_MAGIC, 0));
+ WT_EXT_FOREACH(ext, el->off) /* Free ranges */
+ WT_ERR(__wt_extlist_write_pair(&p, ext->off, ext->size));
if (additional != NULL)
- WT_EXT_FOREACH(ext, additional->off) { /* Free ranges */
- WT_EXTLIST_WRITE(p, ext->off);
- WT_EXTLIST_WRITE(p, ext->size);
- }
- WT_EXTLIST_WRITE(p, WT_BLOCK_INVALID_OFFSET); /* Ending value */
- WT_EXTLIST_WRITE(p, 0);
+ WT_EXT_FOREACH(ext, additional->off) /* Free ranges */
+ WT_ERR(
+ __wt_extlist_write_pair(&p, ext->off, ext->size));
+ /* Extent list stops */
+ WT_ERR(__wt_extlist_write_pair(&p, WT_BLOCK_INVALID_OFFSET,
+ block->final_ckpt == NULL ? 0 : WT_BLOCK_EXTLIST_VERSION_CKPT));
dsk->u.datalen = WT_PTRDIFF32(p, WT_BLOCK_HEADER_BYTE(dsk));
tmp->size = dsk->mem_size = WT_PTRDIFF32(p, dsk);
diff --git a/src/third_party/wiredtiger/src/block/block_mgr.c b/src/third_party/wiredtiger/src/block/block_mgr.c
index 7211e5cfa24..80662ec6634 100644
--- a/src/third_party/wiredtiger/src/block/block_mgr.c
+++ b/src/third_party/wiredtiger/src/block/block_mgr.c
@@ -69,6 +69,18 @@ __bm_checkpoint(WT_BM *bm, WT_SESSION_IMPL *session,
}
/*
+ * __bm_checkpoint_last --
+ * Return information for the last known file checkpoint.
+ */
+static int
+__bm_checkpoint_last(WT_BM *bm, WT_SESSION_IMPL *session,
+ char **metadatap, char **checkpoint_listp, WT_ITEM *checkpoint)
+{
+ return (__wt_block_checkpoint_last(
+ session, bm->block, metadatap, checkpoint_listp, checkpoint));
+}
+
+/*
* __bm_checkpoint_readonly --
* Write a buffer into a block, creating a checkpoint; readonly version.
*/
@@ -562,6 +574,7 @@ __bm_method_set(WT_BM *bm, bool readonly)
bm->addr_string = __bm_addr_string;
bm->block_header = __bm_block_header;
bm->checkpoint = __bm_checkpoint;
+ bm->checkpoint_last = __bm_checkpoint_last;
bm->checkpoint_load = __bm_checkpoint_load;
bm->checkpoint_resolve = __bm_checkpoint_resolve;
bm->checkpoint_start = __bm_checkpoint_start;
diff --git a/src/third_party/wiredtiger/src/block/block_open.c b/src/third_party/wiredtiger/src/block/block_open.c
index 68f7224ff76..9e4689d7b7e 100644
--- a/src/third_party/wiredtiger/src/block/block_open.c
+++ b/src/third_party/wiredtiger/src/block/block_open.c
@@ -8,7 +8,7 @@
#include "wt_internal.h"
-static int __desc_read(WT_SESSION_IMPL *, WT_BLOCK *);
+static int __desc_read(WT_SESSION_IMPL *, uint32_t allocsize, WT_BLOCK *);
/*
* __wt_block_manager_drop --
@@ -228,7 +228,7 @@ __wt_block_open(WT_SESSION_IMPL *session,
* look at anything, including the description information.
*/
if (!forced_salvage)
- WT_ERR(__desc_read(session, block));
+ WT_ERR(__desc_read(session, allocsize, block));
*blockp = block;
__wt_spin_unlock(session, &conn->block_lock);
@@ -314,7 +314,7 @@ __wt_desc_write(WT_SESSION_IMPL *session, WT_FH *fh, uint32_t allocsize)
* Read and verify the file's metadata.
*/
static int
-__desc_read(WT_SESSION_IMPL *session, WT_BLOCK *block)
+__desc_read(WT_SESSION_IMPL *session, uint32_t allocsize, WT_BLOCK *block)
{
WT_BLOCK_DESC *desc;
WT_DECL_ITEM(buf);
@@ -326,11 +326,11 @@ __desc_read(WT_SESSION_IMPL *session, WT_BLOCK *block)
return (0);
/* Use a scratch buffer to get correct alignment for direct I/O. */
- WT_RET(__wt_scr_alloc(session, block->allocsize, &buf));
+ WT_RET(__wt_scr_alloc(session, allocsize, &buf));
/* Read the first allocation-sized block and verify the file format. */
WT_ERR(__wt_read(session,
- block->fh, (wt_off_t)0, (size_t)block->allocsize, buf->mem));
+ block->fh, (wt_off_t)0, (size_t)allocsize, buf->mem));
/*
* Handle little- and big-endian objects. Objects are written in little-
@@ -342,7 +342,7 @@ __desc_read(WT_SESSION_IMPL *session, WT_BLOCK *block)
desc = buf->mem;
checksum_tmp = desc->checksum;
desc->checksum = 0;
- checksum_calculate = __wt_checksum(desc, block->allocsize);
+ checksum_calculate = __wt_checksum(desc, allocsize);
desc->checksum = checksum_tmp;
__wt_block_desc_byteswap(desc);
diff --git a/src/third_party/wiredtiger/src/block/block_write.c b/src/third_party/wiredtiger/src/block/block_write.c
index 55f9d4ca57c..b678c148668 100644
--- a/src/third_party/wiredtiger/src/block/block_write.c
+++ b/src/third_party/wiredtiger/src/block/block_write.c
@@ -232,6 +232,7 @@ __block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block,
wt_off_t offset;
size_t align_size;
uint32_t checksum;
+ uint8_t *file_sizep;
bool local_locked;
*offsetp = 0; /* -Werror=maybe-uninitialized */
@@ -240,13 +241,6 @@ __block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block,
fh = block->fh;
- /*
- * Clear the block header to ensure all of it is initialized, even the
- * unused fields.
- */
- blk = WT_BLOCK_HEADER_REF(buf->mem);
- memset(blk, 0, sizeof(*blk));
-
/* Buffers should be aligned for writing. */
if (!F_ISSET(buf, WT_ITEM_ALIGNED)) {
WT_ASSERT(session, F_ISSET(buf, WT_ITEM_ALIGNED));
@@ -255,6 +249,14 @@ __block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block,
}
/*
+ * File checkpoint/recovery magic: done before sizing the buffer as it
+ * may grow the buffer.
+ */
+ if (block->final_ckpt != NULL)
+ WT_RET(__wt_block_checkpoint_final(
+ session, block, buf, &file_sizep));
+
+ /*
* Align the size to an allocation unit.
*
* The buffer must be big enough for us to zero to the next allocsize
@@ -273,10 +275,46 @@ __block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block,
"buffer size check: write buffer too large to write");
}
+ /* Pre-allocate some number of extension structures. */
+ WT_RET(__wt_block_ext_prealloc(session, 5));
+
+ /*
+ * Acquire a lock, if we don't already hold one.
+ * Allocate space for the write, and optionally extend the file (note
+ * the block-extend function may release the lock).
+ * Release any locally acquired lock.
+ */
+ local_locked = false;
+ if (!caller_locked) {
+ __wt_spin_lock(session, &block->live_lock);
+ local_locked = true;
+ }
+ ret = __wt_block_alloc(session, block, &offset, (wt_off_t)align_size);
+ if (ret == 0)
+ ret = __wt_block_extend(
+ session, block, fh, offset, align_size, &local_locked);
+ if (local_locked)
+ __wt_spin_unlock(session, &block->live_lock);
+ WT_RET(ret);
+
+ /*
+ * The file has finished changing size. If this is the final write in a
+ * checkpoint, update the checkpoint's information inline.
+ */
+ if (block->final_ckpt != NULL)
+ WT_RET(__wt_vpack_uint(&file_sizep, 0, (uint64_t)block->size));
+
/* Zero out any unused bytes at the end of the buffer. */
memset((uint8_t *)buf->mem + buf->size, 0, align_size - buf->size);
/*
+ * Clear the block header to ensure all of it is initialized, even the
+ * unused fields.
+ */
+ blk = WT_BLOCK_HEADER_REF(buf->mem);
+ memset(blk, 0, sizeof(*blk));
+
+ /*
* Set the disk size so we don't have to incrementally read blocks
* during salvage.
*/
@@ -310,28 +348,6 @@ __block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block,
blk->checksum = __wt_bswap32(blk->checksum);
#endif
- /* Pre-allocate some number of extension structures. */
- WT_RET(__wt_block_ext_prealloc(session, 5));
-
- /*
- * Acquire a lock, if we don't already hold one.
- * Allocate space for the write, and optionally extend the file (note
- * the block-extend function may release the lock).
- * Release any locally acquired lock.
- */
- local_locked = false;
- if (!caller_locked) {
- __wt_spin_lock(session, &block->live_lock);
- local_locked = true;
- }
- ret = __wt_block_alloc(session, block, &offset, (wt_off_t)align_size);
- if (ret == 0)
- ret = __wt_block_extend(
- session, block, fh, offset, align_size, &local_locked);
- if (local_locked)
- __wt_spin_unlock(session, &block->live_lock);
- WT_RET(ret);
-
/* Write the block. */
if ((ret =
__wt_write(session, fh, offset, align_size, buf->mem)) != 0) {
diff --git a/src/third_party/wiredtiger/src/btree/bt_cursor.c b/src/third_party/wiredtiger/src/btree/bt_cursor.c
index e75432f7836..b7176c74adf 100644
--- a/src/third_party/wiredtiger/src/btree/bt_cursor.c
+++ b/src/third_party/wiredtiger/src/btree/bt_cursor.c
@@ -1438,12 +1438,13 @@ __cursor_chain_exceeded(WT_CURSOR_BTREE *cbt)
upd != NULL && upd->type == WT_UPDATE_MODIFY;
++i, upd = upd->next) {
upd_size += WT_UPDATE_MEMSIZE(upd);
- if (upd_size >= WT_MODIFY_MEM_FACTOR * cursor->value.size)
+ if (i >= WT_MAX_MODIFY_UPDATE &&
+ upd_size * WT_MODIFY_MEM_FRACTION >= cursor->value.size)
return (true);
}
- if (upd != NULL && upd->type == WT_UPDATE_STANDARD &&
- __wt_txn_upd_visible_all(session, upd) &&
- i >= WT_MAX_MODIFY_UPDATE)
+ if (i >= WT_MAX_MODIFY_UPDATE && upd != NULL &&
+ upd->type == WT_UPDATE_STANDARD &&
+ __wt_txn_upd_visible_all(session, upd))
return (true);
return (false);
}
diff --git a/src/third_party/wiredtiger/src/btree/bt_handle.c b/src/third_party/wiredtiger/src/btree/bt_handle.c
index f179fa38d91..168a0152f17 100644
--- a/src/third_party/wiredtiger/src/btree/bt_handle.c
+++ b/src/third_party/wiredtiger/src/btree/bt_handle.c
@@ -93,6 +93,7 @@ __wt_btree_open(WT_SESSION_IMPL *session, const char *op_cfg[])
WT_CKPT ckpt;
WT_CONFIG_ITEM cval;
WT_DATA_HANDLE *dhandle;
+ WT_DECL_ITEM(tmp);
WT_DECL_RET;
size_t root_addr_size;
uint8_t root_addr[WT_BTREE_MAX_ADDR_COOKIE];
@@ -239,6 +240,7 @@ err: WT_TRET(__wt_btree_close(session));
}
__wt_meta_checkpoint_free(session, &ckpt);
+ __wt_scr_free(session, &tmp);
return (ret);
}
@@ -324,6 +326,44 @@ __wt_btree_discard(WT_SESSION_IMPL *session)
}
/*
+ * __wt_btree_config_encryptor --
+ * Return an encryptor handle based on the configuration.
+ */
+int
+__wt_btree_config_encryptor(WT_SESSION_IMPL *session,
+ const char **cfg, WT_KEYED_ENCRYPTOR **kencryptorp)
+{
+ WT_CONFIG_ITEM cval, enc, keyid;
+ WT_DECL_RET;
+ const char *enc_cfg[] = { NULL, NULL };
+
+ /*
+ * We do not use __wt_config_gets_none here because "none" and the empty
+ * string have different meanings. The empty string means inherit the
+ * system encryption setting and "none" means this table is in the clear
+ * even if the database is encrypted.
+ */
+ WT_RET(__wt_config_gets(session, cfg, "encryption.name", &cval));
+ if (cval.len == 0)
+ *kencryptorp = S2C(session)->kencryptor;
+ else if (WT_STRING_MATCH("none", cval.str, cval.len))
+ *kencryptorp = NULL;
+ else {
+ WT_RET(__wt_config_gets_none(
+ session, cfg, "encryption.keyid", &keyid));
+ WT_RET(__wt_config_gets(session, cfg, "encryption", &enc));
+ if (enc.len != 0)
+ WT_RET(__wt_strndup(session, enc.str, enc.len,
+ &enc_cfg[0]));
+ ret = __wt_encryptor_config(session, &cval, &keyid,
+ (WT_CONFIG_ARG *)enc_cfg, kencryptorp);
+ __wt_free(session, enc_cfg[0]);
+ WT_RET(ret);
+ }
+ return (0);
+}
+
+/*
* __btree_conf --
* Configure a WT_BTREE structure.
*/
@@ -331,12 +371,11 @@ static int
__btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt)
{
WT_BTREE *btree;
- WT_CONFIG_ITEM cval, enc, keyid, metadata;
+ WT_CONFIG_ITEM cval, metadata;
WT_CONNECTION_IMPL *conn;
- WT_DECL_RET;
int64_t maj_version, min_version;
uint32_t bitcnt;
- const char **cfg, *enc_cfg[] = { NULL, NULL };
+ const char **cfg;
bool fixed;
btree = S2BT(session);
@@ -548,29 +587,8 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt)
}
}
- /*
- * We do not use __wt_config_gets_none here because "none" and the empty
- * string have different meanings. The empty string means inherit the
- * system encryption setting and "none" means this table is in the clear
- * even if the database is encrypted.
- */
- WT_RET(__wt_config_gets(session, cfg, "encryption.name", &cval));
- if (cval.len == 0)
- btree->kencryptor = conn->kencryptor;
- else if (WT_STRING_MATCH("none", cval.str, cval.len))
- btree->kencryptor = NULL;
- else {
- WT_RET(__wt_config_gets_none(
- session, cfg, "encryption.keyid", &keyid));
- WT_RET(__wt_config_gets(session, cfg, "encryption", &enc));
- if (enc.len != 0)
- WT_RET(__wt_strndup(session, enc.str, enc.len,
- &enc_cfg[0]));
- ret = __wt_encryptor_config(session, &cval, &keyid,
- (WT_CONFIG_ARG *)enc_cfg, &btree->kencryptor);
- __wt_free(session, enc_cfg[0]);
- WT_RET(ret);
- }
+ /* Configure encryption. */
+ WT_RET(__wt_btree_config_encryptor(session, cfg, &btree->kencryptor));
/* Initialize locks. */
WT_RET(__wt_rwlock_init(session, &btree->ovfl_lock));
diff --git a/src/third_party/wiredtiger/src/btree/bt_import.c b/src/third_party/wiredtiger/src/btree/bt_import.c
new file mode 100644
index 00000000000..69dcda8c1c5
--- /dev/null
+++ b/src/third_party/wiredtiger/src/btree/bt_import.c
@@ -0,0 +1,169 @@
+/*-
+ * Copyright (c) 2014-2019 MongoDB, Inc.
+ * Copyright (c) 2008-2014 WiredTiger, Inc.
+ * All rights reserved.
+ *
+ * See the file LICENSE for redistribution information.
+ */
+
+#include "wt_internal.h"
+
+/*
+ * __wt_import --
+ * Import a WiredTiger file into the database.
+ */
+int
+__wt_import(WT_SESSION_IMPL *session, const char *uri)
+{
+ WT_BM *bm;
+ WT_CKPT *ckpt, *ckptbase;
+ WT_CONFIG_ITEM v;
+ WT_DECL_ITEM(a);
+ WT_DECL_ITEM(b);
+ WT_DECL_ITEM(checkpoint);
+ WT_DECL_RET;
+ WT_KEYED_ENCRYPTOR *kencryptor;
+ const char *filename;
+ const char *filecfg[] = {
+ WT_CONFIG_BASE(session, file_meta), NULL, NULL, NULL, NULL, NULL };
+ char *checkpoint_list, *fileconf, *metadata, fileid[64];
+
+ ckptbase = NULL;
+ checkpoint_list = fileconf = metadata = NULL;
+
+ WT_ERR(__wt_scr_alloc(session, 0, &a));
+ WT_ERR(__wt_scr_alloc(session, 0, &b));
+ WT_ERR(__wt_scr_alloc(session, 0, &checkpoint));
+
+ WT_ASSERT(session, WT_PREFIX_MATCH(uri, "file:"));
+ filename = uri;
+ WT_PREFIX_SKIP(filename, "file:");
+
+ /*
+ * Open the file, request block manager checkpoint information.
+ * We don't know the allocation size, but 512B allows us to read
+ * the descriptor block and that's all we care about.
+ */
+ WT_ERR(__wt_block_manager_open(
+ session, filename, filecfg, false, true, 512, &bm));
+ ret = bm->checkpoint_last(
+ bm, session, &metadata, &checkpoint_list, checkpoint);
+ WT_TRET(bm->close(bm, session));
+ WT_ERR(ret);
+ __wt_verbose(session,
+ WT_VERB_CHECKPOINT, "import metadata: %s", metadata);
+ __wt_verbose(session,
+ WT_VERB_CHECKPOINT, "import checkpoint-list: %s", checkpoint_list);
+
+ /*
+ * The metadata may have been encrypted, in which case it's also
+ * hexadecimal encoded. The checkpoint included a boolean value
+ * set if the metadata was encrypted for easier failure diagnosis.
+ */
+ WT_ERR(__wt_config_getones(
+ session, metadata, "block_metadata_encrypted", &v));
+ WT_ERR(__wt_btree_config_encryptor(session, filecfg, &kencryptor));
+ if ((kencryptor == NULL && v.val != 0) ||
+ (kencryptor != NULL && v.val == 0))
+ WT_ERR_MSG(session, EINVAL,
+ "%s: loaded object's encryption configuration doesn't "
+ "match the database's encryption configuration",
+ filename);
+ /*
+ * The metadata was quoted to avoid configuration string characters
+ * acting as separators. Discard any quote characters.
+ */
+ WT_ERR(__wt_config_getones(session, metadata, "block_metadata", &v));
+ if (v.len > 0 && (v.str[0] == '[' || v.str[0] == '(')) {
+ ++v.str;
+ v.len -= 2;
+ }
+ if (kencryptor == NULL) {
+ WT_ERR(__wt_buf_grow(session, a, v.len + 1));
+ WT_ERR(__wt_buf_set(session, a, v.str, v.len));
+ ((uint8_t *)a->data)[a->size] = '\0';
+ } else {
+ WT_ERR(__wt_buf_grow(session, b, v.len));
+ WT_ERR(__wt_nhex_to_raw(session, v.str, v.len, b));
+ WT_ERR(__wt_buf_grow(session, a, b->size + 1));
+ WT_ERR(__wt_decrypt(session, kencryptor->encryptor, 0, b, a));
+ ((uint8_t *)a->data)[a->size] = '\0';
+ }
+
+ /*
+ * OK, we've now got three chunks of data: the file's metadata from when
+ * the last checkpoint started, the array of checkpoints as of when the
+ * last checkpoint was almost complete (everything written but the avail
+ * list), and fixed-up checkpoint information from the last checkpoint.
+ *
+ * Build and flatten the metadata and the checkpoint list, then insert
+ * it into the metadata for this file.
+ *
+ * Strip out the checkpoint-LSN, an imported file isn't associated
+ * with any log files.
+ * Assign a unique file ID.
+ */
+ filecfg[1] = a->data;
+ filecfg[2] = checkpoint_list;
+ filecfg[3] = "checkpoint_lsn=";
+ WT_WITH_SCHEMA_LOCK(session, ret =
+ __wt_snprintf(fileid, sizeof(fileid),
+ "id=%" PRIu32, ++S2C(session)->next_file_id));
+ WT_ERR(ret);
+ filecfg[4] = fileid;
+ WT_ERR(__wt_config_collapse(session, filecfg, &fileconf));
+ WT_ERR(__wt_metadata_insert(session, uri, fileconf));
+ __wt_verbose(session,
+ WT_VERB_CHECKPOINT, "import configuration: %s/%s", uri, fileconf);
+
+ /*
+ * The just inserted metadata was correct as of immediately before the
+ * before the final checkpoint, but it's not quite right. The block
+ * manager returned the corrected final checkpoint, put it all together.
+ *
+ * Get the checkpoint information from the file's metadata as an array
+ * of WT_CKPT structures.
+ *
+ * XXX
+ * There's a problem here. If a file is imported from our future (leaf
+ * pages with unstable entries that have write-generations ahead of the
+ * current database's base write generation), we'll read the values and
+ * treat them as stable. A restart will fix this: when we added the
+ * imported file to our metadata, the write generation in the imported
+ * file's checkpoints updated our database's maximum write generation,
+ * and so a restart will have a maximum generation newer than the
+ * imported file's write generation. An alternative solution is to add
+ * a "base write generation" value to the imported file's metadata, and
+ * use that value instead of the connection's base write generation when
+ * deciding what page items should be read. Since all future writes to
+ * the imported file would be ahead of that write generation, it would
+ * have the effect we want.
+ *
+ * Update the last checkpoint with the corrected information.
+ * Update the file's metadata with the new checkpoint information.
+ */
+ WT_ERR(__wt_meta_ckptlist_get(session, uri, false, &ckptbase));
+ WT_CKPT_FOREACH(ckptbase, ckpt)
+ if (ckpt->name == NULL || (ckpt + 1)->name == NULL)
+ break;
+ if (ckpt->name == NULL)
+ WT_ERR_MSG(session, EINVAL,
+ "no checkpoint information available to import");
+ F_SET(ckpt, WT_CKPT_UPDATE);
+ WT_ERR(__wt_buf_set(
+ session, &ckpt->raw, checkpoint->data, checkpoint->size));
+ WT_ERR(__wt_meta_ckptlist_set(session, uri, ckptbase, NULL));
+
+err:
+ __wt_meta_ckptlist_free(session, &ckptbase);
+
+ __wt_free(session, fileconf);
+ __wt_free(session, metadata);
+ __wt_free(session, checkpoint_list);
+
+ __wt_scr_free(session, &a);
+ __wt_scr_free(session, &b);
+ __wt_scr_free(session, &checkpoint);
+
+ return (ret);
+}
diff --git a/src/third_party/wiredtiger/src/btree/bt_slvg.c b/src/third_party/wiredtiger/src/btree/bt_slvg.c
index d2b0472a11e..3063af2de0a 100644
--- a/src/third_party/wiredtiger/src/btree/bt_slvg.c
+++ b/src/third_party/wiredtiger/src/btree/bt_slvg.c
@@ -157,11 +157,88 @@ static int __slvg_trk_ovfl(WT_SESSION_IMPL *,
const WT_PAGE_HEADER *, uint8_t *, size_t, WT_STUFF *);
/*
- * __wt_bt_salvage --
+ * __slvg_checkpoint --
+ * Create the post-salvage checkpoint.
+ */
+static int
+__slvg_checkpoint(WT_SESSION_IMPL *session, WT_REF *root)
+{
+ WT_BTREE *btree;
+ WT_CKPT *ckptbase;
+ WT_DATA_HANDLE *dhandle;
+ WT_DECL_RET;
+ char *config;
+
+ btree = S2BT(session);
+ ckptbase = NULL;
+ dhandle = session->dhandle;
+ config = NULL;
+
+ /*
+ * XXX
+ * The salvage process reads and discards previous checkpoints, so the
+ * underlying block manager has to ignore any previous checkpoint
+ * entries when creating a new checkpoint. In other words, we can't use
+ * the metadata checkpoint list, it lists the previous checkpoints and
+ * we don't care about them. Build a clean checkpoint list and use it
+ * instead.
+ *
+ * Don't first clear the metadata checkpoint list and call the function
+ * to get a list of checkpoints: a crash between clearing the metadata
+ * checkpoint list and creating a new checkpoint list would look like a
+ * create or open of a file without a checkpoint to roll-forward from,
+ * and the contents of the file would be discarded.
+ */
+ WT_RET(__wt_calloc_def(session, 2, &ckptbase));
+ WT_ERR(__wt_strdup(session, WT_CHECKPOINT, &ckptbase->name));
+ ckptbase->order = 1;
+ __wt_seconds(session, &ckptbase->sec);
+ WT_ERR(__wt_metadata_search(session, dhandle->name, &config));
+ WT_ERR(__wt_meta_block_metadata(session, config, ckptbase));
+ ckptbase->newest_durable_ts = WT_TS_NONE;
+ ckptbase->oldest_start_ts = WT_TS_NONE;
+ ckptbase->oldest_start_txn = WT_TXN_NONE;
+ ckptbase->newest_stop_ts = WT_TS_MAX;
+ ckptbase->newest_stop_txn = WT_TXN_MAX;
+ F_SET(ckptbase, WT_CKPT_ADD);
+
+ /*
+ * We may not have found any pages during salvage and there's no tree
+ * to flush.
+ */
+ if (root->page != NULL) {
+ btree->ckpt = ckptbase;
+ ret = __wt_evict(
+ session, root, WT_REF_MEM, WT_EVICT_CALL_CLOSING);
+ root->page = NULL;
+ btree->ckpt = NULL;
+ WT_ERR(ret);
+ }
+
+ /*
+ * If no checkpoint was created, clear all recorded checkpoints for the
+ * file. This is expected if we didn't find any leaf pages to salvage.
+ *
+ * If a checkpoint was created, life is good, replace any existing list
+ * of checkpoints with the single new one.
+ */
+ if (ckptbase->raw.data == NULL)
+ WT_TRET(__wt_meta_checkpoint_clear(session, dhandle->name));
+ else
+ WT_ERR(__wt_meta_ckptlist_set(
+ session, dhandle->name, ckptbase, NULL));
+
+err: __wt_meta_ckptlist_free(session, &ckptbase);
+ __wt_free(session, config);
+ return (ret);
+}
+
+/*
+ * __wt_salvage --
* Salvage a Btree.
*/
int
-__wt_bt_salvage(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, const char *cfg[])
+__wt_salvage(WT_SESSION_IMPL *session, const char *cfg[])
{
WT_BM *bm;
WT_BTREE *btree;
@@ -324,15 +401,9 @@ __wt_bt_salvage(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, const char *cfg[])
/*
* Step 9:
- * Evict the newly created root page, creating a checkpoint.
+ * Evict any newly created root page, creating a checkpoint.
*/
- if (ss->root_ref.page != NULL) {
- btree->ckpt = ckptbase;
- ret = __wt_evict(session, &ss->root_ref, WT_REF_MEM,
- WT_EVICT_CALL_CLOSING);
- ss->root_ref.page = NULL;
- btree->ckpt = NULL;
- }
+ WT_ERR(__slvg_checkpoint(session, &ss->root_ref));
/*
* Step 10:
diff --git a/src/third_party/wiredtiger/src/btree/bt_split.c b/src/third_party/wiredtiger/src/btree/bt_split.c
index b1d8dce75a5..8bd58c17975 100644
--- a/src/third_party/wiredtiger/src/btree/bt_split.c
+++ b/src/third_party/wiredtiger/src/btree/bt_split.c
@@ -1412,6 +1412,25 @@ err: if (parent != NULL)
return (0);
}
+#ifdef HAVE_DIAGNOSTIC
+/*
+ * __check_upd_list --
+ * Sanity check an update list.
+ * In particular, make sure there no birthmarks.
+ */
+static void
+__check_upd_list(WT_SESSION_IMPL *session, WT_UPDATE *upd)
+{
+ int birthmark_count;
+
+ for (birthmark_count = 0; upd != NULL; upd = upd->next)
+ if (upd->type == WT_UPDATE_BIRTHMARK)
+ ++birthmark_count;
+
+ WT_ASSERT(session, birthmark_count <= 1);
+}
+#endif
+
/*
* __split_multi_inmem --
* Instantiate a page from a disk image.
@@ -1511,6 +1530,10 @@ __split_multi_inmem(
key->size = WT_INSERT_KEY_SIZE(supd->ins);
}
+#ifdef HAVE_DIAGNOSTIC
+ __check_upd_list(session, upd);
+#endif
+
/* Search the page. */
WT_ERR(__wt_row_search(
session, key, ref, &cbt, true, true));
@@ -1817,9 +1840,11 @@ __split_insert(WT_SESSION_IMPL *session, WT_REF *ref)
WT_SKIP_FIRST(WT_ROW_INSERT_SMALLEST(page))) != NULL) {
key->data = WT_INSERT_KEY(ins);
key->size = WT_INSERT_KEY_SIZE(ins);
- } else
+ } else {
+ WT_ASSERT(session, page->entries > 0);
WT_ERR(__wt_row_leaf_key(
session, page, &page->pg_row[0], key, true));
+ }
WT_ERR(__wt_row_ikey(session, 0, key->data, key->size, child));
parent_incr += sizeof(WT_IKEY) + key->size;
__wt_scr_free(session, &key);
diff --git a/src/third_party/wiredtiger/src/btree/bt_vrfy.c b/src/third_party/wiredtiger/src/btree/bt_vrfy.c
index 9078e74674b..6a07ba5aa55 100644
--- a/src/third_party/wiredtiger/src/btree/bt_vrfy.c
+++ b/src/third_party/wiredtiger/src/btree/bt_vrfy.c
@@ -162,11 +162,13 @@ __wt_verify(WT_SESSION_IMPL *session, const char *cfg[])
WT_VSTUFF *vs, _vstuff;
size_t root_addr_size;
uint8_t root_addr[WT_BTREE_MAX_ADDR_COOKIE];
+ const char *name;
bool bm_start, quit;
btree = S2BT(session);
bm = btree->bm;
ckptbase = NULL;
+ name = session->dhandle->name;
bm_start = false;
WT_CLEAR(_vstuff);
@@ -186,9 +188,16 @@ __wt_verify(WT_SESSION_IMPL *session, const char *cfg[])
if (quit)
goto done;
- /* Get a list of the checkpoints for this file. */
- WT_ERR(
- __wt_meta_ckptlist_get(session, btree->dhandle->name, &ckptbase));
+ /*
+ * Get a list of the checkpoints for this file. Empty objects have no
+ * checkpoints, in which case there's no work to do.
+ */
+ ret = __wt_meta_ckptlist_get(session, name, false, &ckptbase);
+ if (ret == WT_NOTFOUND) {
+ ret = 0;
+ goto done;
+ }
+ WT_ERR(ret);
/* Inform the underlying block manager we're verifying. */
WT_ERR(bm->verify_start(bm, session, ckptbase, cfg));
@@ -197,7 +206,7 @@ __wt_verify(WT_SESSION_IMPL *session, const char *cfg[])
/* Loop through the file's checkpoints, verifying each one. */
WT_CKPT_FOREACH(ckptbase, ckpt) {
__wt_verbose(session, WT_VERB_VERIFY,
- "%s: checkpoint %s", btree->dhandle->name, ckpt->name);
+ "%s: checkpoint %s", name, ckpt->name);
/* Fake checkpoints require no work. */
if (F_ISSET(ckpt, WT_CKPT_FAKE))
@@ -209,7 +218,7 @@ __wt_verify(WT_SESSION_IMPL *session, const char *cfg[])
if (WT_VRFY_DUMP(vs)) {
WT_ERR(__wt_msg(session, "%s", WT_DIVIDER));
WT_ERR(__wt_msg(session, "%s: checkpoint %s",
- btree->dhandle->name, ckpt->name));
+ name, ckpt->name));
}
/* Load the checkpoint. */
diff --git a/src/third_party/wiredtiger/src/cache/cache_las.c b/src/third_party/wiredtiger/src/cache/cache_las.c
index 87d55c1ac5e..d743f4f2eeb 100644
--- a/src/third_party/wiredtiger/src/cache/cache_las.c
+++ b/src/third_party/wiredtiger/src/cache/cache_las.c
@@ -660,7 +660,7 @@ __wt_las_insert_block(WT_CURSOR *cursor,
WT_SAVE_UPD *list;
WT_SESSION_IMPL *session;
WT_TXN_ISOLATION saved_isolation;
- WT_UPDATE *upd;
+ WT_UPDATE *first_upd, *upd;
wt_off_t las_size;
uint64_t insert_cnt, las_counter, las_pageid, max_las_size;
uint64_t prepared_insert_cnt;
@@ -741,7 +741,7 @@ __wt_las_insert_block(WT_CURSOR *cursor,
slot = page->type == WT_PAGE_ROW_LEAF ?
WT_ROW_SLOT(page, list->ripcip) :
WT_COL_SLOT(page, list->ripcip);
- upd = list->ins == NULL ?
+ first_upd = upd = list->ins == NULL ?
page->modify->mod_row_update[slot] : list->ins->upd;
/*
@@ -760,6 +760,9 @@ __wt_las_insert_block(WT_CURSOR *cursor,
las_value.size = upd->size;
break;
case WT_UPDATE_BIRTHMARK:
+ WT_ASSERT(session, upd != first_upd ||
+ multi->page_las.skew_newest);
+ /* FALLTHROUGH */
case WT_UPDATE_TOMBSTONE:
las_value.size = 0;
break;
@@ -780,6 +783,8 @@ __wt_las_insert_block(WT_CURSOR *cursor,
(upd->type == WT_UPDATE_STANDARD ||
upd->type == WT_UPDATE_MODIFY)) {
las_value.size = 0;
+ WT_ASSERT(session, upd != first_upd ||
+ multi->page_las.skew_newest);
cursor->set_value(cursor, upd->txnid,
upd->start_ts, upd->durable_ts,
upd->prepare_state, WT_UPDATE_BIRTHMARK,
@@ -835,6 +840,7 @@ err: /* Resolve the transaction. */
__las_insert_block_verbose(session, btree, multi);
}
+ WT_UNUSED(first_upd);
return (ret);
}
diff --git a/src/third_party/wiredtiger/src/config/config_collapse.c b/src/third_party/wiredtiger/src/config/config_collapse.c
index 0c4dfe7c604..3c21e0e224f 100644
--- a/src/third_party/wiredtiger/src/config/config_collapse.c
+++ b/src/third_party/wiredtiger/src/config/config_collapse.c
@@ -82,6 +82,103 @@ err: __wt_scr_free(session, &tmp);
}
/*
+ * __wt_config_discard_defaults --
+ * Copy non-default configuration strings into newly allocated memory.
+ *
+ * This function strips out entries from a configuration string that aren't the
+ * default values. It takes a NULL-terminated list of configuration strings, the
+ * defaults, and a configuration string, and copies into allocated memory the
+ * strings from the configuration string that aren't the same as the defaults.
+ */
+int
+__wt_config_discard_defaults(WT_SESSION_IMPL *session,
+ const char **cfg, const char *config, char **config_ret)
+{
+ WT_CONFIG cparser;
+ WT_CONFIG_ITEM k, v, vtmp;
+ WT_DECL_ITEM(tmp);
+ WT_DECL_RET;
+
+ *config_ret = NULL;
+
+ WT_RET(__wt_scr_alloc(session, 0, &tmp));
+
+ /*
+ * Walk the configuration string, search the default configuration for
+ * each entry, and discard any matches.
+ */
+ __wt_config_init(session, &cparser, config);
+ while ((ret = __wt_config_next(&cparser, &k, &v)) == 0) {
+ if (k.type != WT_CONFIG_ITEM_STRING &&
+ k.type != WT_CONFIG_ITEM_ID)
+ WT_ERR_MSG(session, EINVAL,
+ "Invalid configuration key found: '%s'", k.str);
+
+ /*
+ * Get the default value. There isn't a default value in some
+ * cases, so not finding one isn't an error.
+ */
+ if ((ret =
+ __wt_config_get(session, cfg, &k, &vtmp)) == WT_NOTFOUND)
+ goto keep;
+ WT_ERR(ret);
+
+ /*
+ * Skip exact matches and simple things we can catch like "none"
+ * and an empty string, "true" and 1, "false" and 0.
+ */
+ if (v.type == WT_CONFIG_ITEM_STRUCT &&
+ vtmp.type == WT_CONFIG_ITEM_STRUCT &&
+ v.len == vtmp.len && memcmp(v.str, vtmp.str, v.len) == 0)
+ continue;
+ if ((v.type == WT_CONFIG_ITEM_BOOL ||
+ v.type == WT_CONFIG_ITEM_NUM) &&
+ (vtmp.type == WT_CONFIG_ITEM_BOOL ||
+ vtmp.type == WT_CONFIG_ITEM_NUM) && v.val == vtmp.val)
+ continue;
+ if ((v.type == WT_CONFIG_ITEM_ID ||
+ v.type == WT_CONFIG_ITEM_STRING) &&
+ (vtmp.type == WT_CONFIG_ITEM_ID ||
+ vtmp.type == WT_CONFIG_ITEM_STRING) &&
+ v.len == vtmp.len && memcmp(v.str, vtmp.str, v.len) == 0)
+ continue;
+ if (vtmp.len == 0 && v.len == strlen("none") &&
+ WT_STRING_MATCH("none", v.str, v.len))
+ continue;
+
+ /* Include the quotes around string keys/values. */
+keep: if (k.type == WT_CONFIG_ITEM_STRING) {
+ --k.str;
+ k.len += 2;
+ }
+ if (v.type == WT_CONFIG_ITEM_STRING) {
+ --v.str;
+ v.len += 2;
+ }
+ WT_ERR(__wt_buf_catfmt(session, tmp, "%.*s=%.*s,",
+ (int)k.len, k.str, (int)v.len, v.str));
+ }
+
+ /* We loop until error, and the expected error is WT_NOTFOUND. */
+ if (ret != WT_NOTFOUND)
+ goto err;
+
+ /*
+ * If the caller passes us only default configuration strings, we get
+ * here with no bytes to copy -- that's OK, the underlying string copy
+ * can handle empty strings.
+ *
+ * Strip any trailing comma.
+ */
+ if (tmp->size != 0)
+ --tmp->size;
+ ret = __wt_strndup(session, tmp->data, tmp->size, config_ret);
+
+err: __wt_scr_free(session, &tmp);
+ return (ret);
+}
+
+/*
* We need a character that can't appear in a key as a separator.
*/
#undef SEP /* separator key, character */
diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c
index 11b41b07787..a8152f08d16 100644
--- a/src/third_party/wiredtiger/src/config/config_def.c
+++ b/src/third_party/wiredtiger/src/config/config_def.c
@@ -84,6 +84,7 @@ static const WT_CONFIG_CHECK
{ "checkpoint_retention", "int",
NULL, "min=0,max=1024",
NULL, 0 },
+ { "eviction", "boolean", NULL, NULL, NULL, 0 },
{ "rollback_error", "int", NULL, "min=0,max=10M", NULL, 0 },
{ "table_logging", "boolean", NULL, NULL, NULL, 0 },
{ NULL, NULL, NULL, NULL, NULL, 0 }
@@ -177,7 +178,7 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = {
confchk_WT_CONNECTION_reconfigure_compatibility_subconfigs, 1 },
{ "debug_mode", "category",
NULL, NULL,
- confchk_wiredtiger_open_debug_mode_subconfigs, 3 },
+ confchk_wiredtiger_open_debug_mode_subconfigs, 4 },
{ "error_prefix", "string", NULL, NULL, NULL, 0 },
{ "eviction", "category",
NULL, NULL,
@@ -894,7 +895,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = {
{ "create", "boolean", NULL, NULL, NULL, 0 },
{ "debug_mode", "category",
NULL, NULL,
- confchk_wiredtiger_open_debug_mode_subconfigs, 3 },
+ confchk_wiredtiger_open_debug_mode_subconfigs, 4 },
{ "direct_io", "list",
NULL, "choices=[\"checkpoint\",\"data\",\"log\"]",
NULL, 0 },
@@ -1009,7 +1010,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = {
{ "create", "boolean", NULL, NULL, NULL, 0 },
{ "debug_mode", "category",
NULL, NULL,
- confchk_wiredtiger_open_debug_mode_subconfigs, 3 },
+ confchk_wiredtiger_open_debug_mode_subconfigs, 4 },
{ "direct_io", "list",
NULL, "choices=[\"checkpoint\",\"data\",\"log\"]",
NULL, 0 },
@@ -1123,7 +1124,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = {
confchk_wiredtiger_open_compatibility_subconfigs, 3 },
{ "debug_mode", "category",
NULL, NULL,
- confchk_wiredtiger_open_debug_mode_subconfigs, 3 },
+ confchk_wiredtiger_open_debug_mode_subconfigs, 4 },
{ "direct_io", "list",
NULL, "choices=[\"checkpoint\",\"data\",\"log\"]",
NULL, 0 },
@@ -1233,7 +1234,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = {
confchk_wiredtiger_open_compatibility_subconfigs, 3 },
{ "debug_mode", "category",
NULL, NULL,
- confchk_wiredtiger_open_debug_mode_subconfigs, 3 },
+ confchk_wiredtiger_open_debug_mode_subconfigs, 4 },
{ "direct_io", "list",
NULL, "choices=[\"checkpoint\",\"data\",\"log\"]",
NULL, 0 },
@@ -1372,12 +1373,12 @@ static const WT_CONFIG_ENTRY config_entries[] = {
"async=(enabled=false,ops_max=1024,threads=2),cache_max_wait_ms=0"
",cache_overflow=(file_max=0),cache_overhead=8,cache_size=100MB,"
"checkpoint=(log_size=0,wait=0),compatibility=(release=),"
- "debug_mode=(checkpoint_retention=0,rollback_error=0,"
- "table_logging=false),error_prefix=,eviction=(threads_max=8,"
- "threads_min=1),eviction_checkpoint_target=1,"
- "eviction_dirty_target=5,eviction_dirty_trigger=20,"
- "eviction_target=80,eviction_trigger=95,"
- "file_manager=(close_handle_minimum=250,close_idle_time=30,"
+ "debug_mode=(checkpoint_retention=0,eviction=false,"
+ "rollback_error=0,table_logging=false),error_prefix=,"
+ "eviction=(threads_max=8,threads_min=1),"
+ "eviction_checkpoint_target=1,eviction_dirty_target=5,"
+ "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95"
+ ",file_manager=(close_handle_minimum=250,close_idle_time=30,"
"close_scan_interval=10),io_capacity=(total=0),log=(archive=true,"
"os_cache_dirty_pct=0,prealloc=true,zero_fill=false),"
"lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true,"
@@ -1465,6 +1466,10 @@ static const WT_CONFIG_ENTRY config_entries[] = {
"remove_files=true",
confchk_WT_SESSION_drop, 4
},
+ { "WT_SESSION.import",
+ "",
+ NULL, 0
+ },
{ "WT_SESSION.join",
"bloom_bit_count=16,bloom_false_positives=false,"
"bloom_hash_count=8,compare=\"eq\",count=,operation=\"and\","
@@ -1627,10 +1632,11 @@ static const WT_CONFIG_ENTRY config_entries[] = {
",cache_size=100MB,checkpoint=(log_size=0,wait=0),"
"checkpoint_sync=true,compatibility=(release=,require_max=,"
"require_min=),config_base=true,create=false,"
- "debug_mode=(checkpoint_retention=0,rollback_error=0,"
- "table_logging=false),direct_io=,encryption=(keyid=,name=,"
- "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)"
- ",eviction_checkpoint_target=1,eviction_dirty_target=5,"
+ "debug_mode=(checkpoint_retention=0,eviction=false,"
+ "rollback_error=0,table_logging=false),direct_io=,"
+ "encryption=(keyid=,name=,secretkey=),error_prefix=,"
+ "eviction=(threads_max=8,threads_min=1),"
+ "eviction_checkpoint_target=1,eviction_dirty_target=5,"
"eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95"
",exclusive=false,extensions=,file_extend=,"
"file_manager=(close_handle_minimum=250,close_idle_time=30,"
@@ -1657,10 +1663,11 @@ static const WT_CONFIG_ENTRY config_entries[] = {
",cache_size=100MB,checkpoint=(log_size=0,wait=0),"
"checkpoint_sync=true,compatibility=(release=,require_max=,"
"require_min=),config_base=true,create=false,"
- "debug_mode=(checkpoint_retention=0,rollback_error=0,"
- "table_logging=false),direct_io=,encryption=(keyid=,name=,"
- "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)"
- ",eviction_checkpoint_target=1,eviction_dirty_target=5,"
+ "debug_mode=(checkpoint_retention=0,eviction=false,"
+ "rollback_error=0,table_logging=false),direct_io=,"
+ "encryption=(keyid=,name=,secretkey=),error_prefix=,"
+ "eviction=(threads_max=8,threads_min=1),"
+ "eviction_checkpoint_target=1,eviction_dirty_target=5,"
"eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95"
",exclusive=false,extensions=,file_extend=,"
"file_manager=(close_handle_minimum=250,close_idle_time=30,"
@@ -1686,7 +1693,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {
"cache_max_wait_ms=0,cache_overflow=(file_max=0),cache_overhead=8"
",cache_size=100MB,checkpoint=(log_size=0,wait=0),"
"checkpoint_sync=true,compatibility=(release=,require_max=,"
- "require_min=),debug_mode=(checkpoint_retention=0,"
+ "require_min=),debug_mode=(checkpoint_retention=0,eviction=false,"
"rollback_error=0,table_logging=false),direct_io=,"
"encryption=(keyid=,name=,secretkey=),error_prefix=,"
"eviction=(threads_max=8,threads_min=1),"
@@ -1714,7 +1721,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {
"cache_max_wait_ms=0,cache_overflow=(file_max=0),cache_overhead=8"
",cache_size=100MB,checkpoint=(log_size=0,wait=0),"
"checkpoint_sync=true,compatibility=(release=,require_max=,"
- "require_min=),debug_mode=(checkpoint_retention=0,"
+ "require_min=),debug_mode=(checkpoint_retention=0,eviction=false,"
"rollback_error=0,table_logging=false),direct_io=,"
"encryption=(keyid=,name=,secretkey=),error_prefix=,"
"eviction=(threads_max=8,threads_min=1),"
diff --git a/src/third_party/wiredtiger/src/conn/api_calc_modify.c b/src/third_party/wiredtiger/src/conn/api_calc_modify.c
index 4a435a85ef1..a8091498ee6 100644
--- a/src/third_party/wiredtiger/src/conn/api_calc_modify.c
+++ b/src/third_party/wiredtiger/src/conn/api_calc_modify.c
@@ -69,16 +69,31 @@ static void
__cm_extend(WT_CM_STATE *cms,
const uint8_t *m1, const uint8_t *m2, WT_CM_MATCH *match)
{
+ ptrdiff_t n;
const uint8_t *p1, *p2;
- /* Step past the end and before the beginning of the matching block. */
+ p1 = m1;
+ p2 = m2;
+
+ /*
+ * Keep skipping half of the remaining bytes while they compare equal.
+ * This is significantly faster than our byte-at-a-time loop below.
+ */
for (p1 = m1, p2 = m2;
- p1 < cms->e1 && p2 < cms->e2 && *p1 == *p2;
- p1++, p2++)
+ (n = WT_MIN(cms->e1 - p1, cms->e2 - p2) / 2) > 8 &&
+ memcmp(p1, p2, (size_t)n) == 0;
+ p1 += n, p2 += n)
+ ;
+
+ /* Step past the end and before the beginning of the matching block. */
+ for (n = WT_MIN(cms->e1 - p1, cms->e2 - p2);
+ n > 0 && *p1 == *p2;
+ n--, p1++, p2++)
;
- for (; m1 >= cms->used1 && m2 >= cms->used2 && *m1 == *m2;
- m1--, m2--)
+ for (n = WT_MIN(m1 - cms->used1, m2 - cms->used2);
+ n > 0 && *m1 == *m2;
+ n--, m1--, m2--)
;
match->m1 = m1 + 1;
diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c
index 7a854ee596f..e68fb6c15f8 100644
--- a/src/third_party/wiredtiger/src/conn/conn_api.c
+++ b/src/third_party/wiredtiger/src/conn/conn_api.c
@@ -1830,25 +1830,16 @@ err: /*
int
__wt_debug_mode_config(WT_SESSION_IMPL *session, const char *cfg[])
{
+ WT_CACHE *cache;
WT_CONFIG_ITEM cval;
WT_CONNECTION_IMPL *conn;
WT_TXN_GLOBAL *txn_global;
conn = S2C(session);
+ cache = conn->cache;
txn_global = &conn->txn_global;
WT_RET(__wt_config_gets(session,
- cfg, "debug_mode.rollback_error", &cval));
- txn_global->debug_rollback = (uint64_t)cval.val;
-
- WT_RET(__wt_config_gets(session,
- cfg, "debug_mode.table_logging", &cval));
- if (cval.val)
- FLD_SET(conn->log_flags, WT_CONN_LOG_DEBUG_MODE);
- else
- FLD_CLR(conn->log_flags, WT_CONN_LOG_DEBUG_MODE);
-
- WT_RET(__wt_config_gets(session,
cfg, "debug_mode.checkpoint_retention", &cval));
conn->debug_ckpt_cnt = (uint32_t)cval.val;
if (cval.val == 0) {
@@ -1862,6 +1853,24 @@ __wt_debug_mode_config(WT_SESSION_IMPL *session, const char *cfg[])
WT_RET(__wt_calloc_def(session,
conn->debug_ckpt_cnt, &conn->debug_ckpt));
+ WT_RET(__wt_config_gets(session,
+ cfg, "debug_mode.eviction", &cval));
+ if (cval.val)
+ F_SET(cache, WT_CACHE_EVICT_DEBUG_MODE);
+ else
+ F_CLR(cache, WT_CACHE_EVICT_DEBUG_MODE);
+
+ WT_RET(__wt_config_gets(session,
+ cfg, "debug_mode.rollback_error", &cval));
+ txn_global->debug_rollback = (uint64_t)cval.val;
+
+ WT_RET(__wt_config_gets(session,
+ cfg, "debug_mode.table_logging", &cval));
+ if (cval.val)
+ FLD_SET(conn->log_flags, WT_CONN_LOG_DEBUG_MODE);
+ else
+ FLD_CLR(conn->log_flags, WT_CONN_LOG_DEBUG_MODE);
+
return (0);
}
@@ -2637,7 +2646,6 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler,
session, cval.str, cval.len, &conn->error_prefix));
}
WT_ERR(__wt_verbose_config(session, cfg));
- WT_ERR(__wt_debug_mode_config(session, cfg));
WT_ERR(__wt_timing_stress_config(session, cfg));
__wt_btree_page_version_config(session);
@@ -2762,6 +2770,12 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler,
session = conn->default_session;
/*
+ * This function expects the cache to be created so parse this after
+ * the rest of the connection is set up.
+ */
+ WT_ERR(__wt_debug_mode_config(session, cfg));
+
+ /*
* Load the extensions after initialization completes; extensions expect
* everything else to be in place, and the extensions call back into the
* library.
diff --git a/src/third_party/wiredtiger/src/docs/command-line.dox b/src/third_party/wiredtiger/src/docs/command-line.dox
index df52324f8f8..852e07dd70a 100644
--- a/src/third_party/wiredtiger/src/docs/command-line.dox
+++ b/src/third_party/wiredtiger/src/docs/command-line.dox
@@ -342,7 +342,7 @@ data that cannot be recovered. Underlying files are re-written in place,
overwriting the original file contents.
@subsection util_salvage_synopsis Synopsis
-<code>wt [-RVv] [-C config] [-E secretkey ] [-h directory] salvage [-F force] uri</code>
+<code>wt [-RVv] [-C config] [-E secretkey ] [-h directory] salvage [-F] uri</code>
@subsection util_salvage_options Options
The following are command-specific options for the \c salvage command:
diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c
index 7d696a20831..ff2840b722c 100644
--- a/src/third_party/wiredtiger/src/evict/evict_lru.c
+++ b/src/third_party/wiredtiger/src/evict/evict_lru.c
@@ -107,6 +107,25 @@ __evict_entry_priority(WT_SESSION_IMPL *session, WT_REF *ref)
}
/*
+ * __evict_lru_cmp_debug --
+ * Qsort function: sort the eviction array.
+ * Version for eviction debug mode.
+ */
+static int WT_CDECL
+__evict_lru_cmp_debug(const void *a_arg, const void *b_arg)
+{
+ const WT_EVICT_ENTRY *a, *b;
+ uint64_t a_score, b_score;
+
+ a = a_arg;
+ b = b_arg;
+ a_score = (a->ref == NULL ? UINT64_MAX : 0);
+ b_score = (b->ref == NULL ? UINT64_MAX : 0);
+
+ return ((a_score < b_score) ? -1 : (a_score == b_score) ? 0 : 1);
+}
+
+/*
* __evict_lru_cmp --
* Qsort function: sort the eviction array.
*/
@@ -1257,8 +1276,17 @@ __evict_lru_walk(WT_SESSION_IMPL *session)
queue->evict_current = NULL;
entries = queue->evict_entries;
- __wt_qsort(queue->evict_queue,
- entries, sizeof(WT_EVICT_ENTRY), __evict_lru_cmp);
+ /*
+ * Style note: __wt_qsort is a macro that can leave a dangling
+ * else. Full curly braces are needed here for the compiler.
+ */
+ if (F_ISSET(cache, WT_CACHE_EVICT_DEBUG_MODE)) {
+ __wt_qsort(queue->evict_queue,
+ entries, sizeof(WT_EVICT_ENTRY), __evict_lru_cmp_debug);
+ } else {
+ __wt_qsort(queue->evict_queue,
+ entries, sizeof(WT_EVICT_ENTRY), __evict_lru_cmp);
+ }
/* Trim empty entries from the end. */
while (entries > 0 && queue->evict_queue[entries - 1].ref == NULL)
@@ -2057,12 +2085,14 @@ __evict_walk_tree(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue,
* cache (indicated by seeing an internal page that is the
* parent of the last page we saw).
*
- * Also skip internal page unless we get aggressive or the tree
- * is idle (indicated by the tree being skipped for walks).
+ * Also skip internal page unless we get aggressive, the tree
+ * is idle (indicated by the tree being skipped for walks),
+ * or we are in eviction debug mode.
* The goal here is that if trees become completely idle, we
* eventually push them out of cache completely.
*/
- if (WT_PAGE_IS_INTERNAL(page)) {
+ if (!F_ISSET(cache, WT_CACHE_EVICT_DEBUG_MODE) &&
+ WT_PAGE_IS_INTERNAL(page)) {
if (page == last_parent)
continue;
if (btree->evict_walk_period == 0 &&
diff --git a/src/third_party/wiredtiger/src/evict/evict_page.c b/src/third_party/wiredtiger/src/evict/evict_page.c
index 5618220774b..2510815401f 100644
--- a/src/third_party/wiredtiger/src/evict/evict_page.c
+++ b/src/third_party/wiredtiger/src/evict/evict_page.c
@@ -656,7 +656,13 @@ __evict_review(
else if (!WT_IS_METADATA(session->dhandle)) {
LF_SET(WT_REC_UPDATE_RESTORE);
- if (F_ISSET(cache, WT_CACHE_EVICT_SCRUB))
+ /*
+ * Scrub if we're supposed to or toss it in sometimes
+ * if we are in debugging mode.
+ */
+ if (F_ISSET(cache, WT_CACHE_EVICT_SCRUB) ||
+ (F_ISSET(cache, WT_CACHE_EVICT_DEBUG_MODE) &&
+ __wt_random(&session->rnd) % 3 == 0))
LF_SET(WT_REC_SCRUB);
/*
@@ -665,8 +671,16 @@ __evict_review(
* suggests trying the lookaside table.
*/
if (F_ISSET(cache, WT_CACHE_EVICT_LOOKASIDE) &&
- !F_ISSET(conn, WT_CONN_EVICTION_NO_LOOKASIDE))
+ !F_ISSET(conn, WT_CONN_EVICTION_NO_LOOKASIDE)) {
+ if (F_ISSET(cache,
+ WT_CACHE_EVICT_DEBUG_MODE) &&
+ __wt_random(&session->rnd) % 10 == 0) {
+ LF_CLR(WT_REC_SCRUB |
+ WT_REC_UPDATE_RESTORE);
+ LF_SET(WT_REC_LOOKASIDE);
+ }
lookaside_retryp = &lookaside_retry;
+ }
}
}
diff --git a/src/third_party/wiredtiger/src/include/block.h b/src/third_party/wiredtiger/src/include/block.h
index 8efaf10dd2b..4cfe07f759d 100644
--- a/src/third_party/wiredtiger/src/include/block.h
+++ b/src/third_party/wiredtiger/src/include/block.h
@@ -125,6 +125,21 @@ struct __wt_size {
*/
#define WT_BM_CHECKPOINT_VERSION 1 /* Checkpoint format version */
#define WT_BLOCK_EXTLIST_MAGIC 71002 /* Identify a list */
+
+/*
+ * There are two versions of the extent list blocks: the original, and a second
+ * version where current checkpoint information is appended to the avail extent
+ * list.
+ */
+#define WT_BLOCK_EXTLIST_VERSION_ORIG 0 /* Original version */
+#define WT_BLOCK_EXTLIST_VERSION_CKPT 1 /* Checkpoint in avail output */
+
+/*
+ * Maximum buffer required to store a checkpoint: 1 version byte followed by
+ * 14 packed 8B values.
+ */
+#define WT_BLOCK_CHECKPOINT_BUFFER (1 + 14 * WT_INTPACK64_MAXSIZE)
+
struct __wt_block_ckpt {
uint8_t version; /* Version */
@@ -163,6 +178,8 @@ struct __wt_bm {
u_int (*block_header)(WT_BM *);
int (*checkpoint)
(WT_BM *, WT_SESSION_IMPL *, WT_ITEM *, WT_CKPT *, bool);
+ int (*checkpoint_last)
+ (WT_BM *, WT_SESSION_IMPL *, char **, char **, WT_ITEM *);
int (*checkpoint_load)(WT_BM *, WT_SESSION_IMPL *,
const uint8_t *, size_t, uint8_t *, size_t *, bool);
int (*checkpoint_resolve)(WT_BM *, WT_SESSION_IMPL *, bool);
@@ -254,6 +271,8 @@ struct __wt_block {
enum { WT_CKPT_NONE=0, WT_CKPT_INPROGRESS,
WT_CKPT_PANIC_ON_FAILURE, WT_CKPT_SALVAGE } ckpt_state;
+ WT_CKPT *final_ckpt; /* Final live checkpoint write */
+
/* Compaction support */
int compact_pct_tenths; /* Percent to compact */
uint64_t compact_pages_reviewed;/* Pages reviewed */
diff --git a/src/third_party/wiredtiger/src/include/block.i b/src/third_party/wiredtiger/src/include/block.i
new file mode 100644
index 00000000000..3b9183a19fa
--- /dev/null
+++ b/src/third_party/wiredtiger/src/include/block.i
@@ -0,0 +1,39 @@
+/*-
+ * Copyright (c) 2014-2019 MongoDB, Inc.
+ * Copyright (c) 2008-2014 WiredTiger, Inc.
+ * All rights reserved.
+ *
+ * See the file LICENSE for redistribution information.
+ */
+
+/*
+ * WiredTiger's block manager interface.
+ */
+
+/*
+ * __wt_extlist_write_pair --
+ * Write an extent list pair.
+ */
+static inline int
+__wt_extlist_write_pair(uint8_t **p, wt_off_t off, wt_off_t size)
+{
+ WT_RET(__wt_vpack_uint(p, 0, (uint64_t)(off)));
+ WT_RET(__wt_vpack_uint(p, 0, (uint64_t)(size)));
+ return (0);
+}
+
+/*
+ * __wt_extlist_read_pair --
+ * Read an extent list pair.
+ */
+static inline int
+__wt_extlist_read_pair(const uint8_t **p, wt_off_t *offp, wt_off_t *sizep)
+{
+ uint64_t v;
+
+ WT_RET(__wt_vunpack_uint(p, 0, &v));
+ *offp = (wt_off_t)v;
+ WT_RET(__wt_vunpack_uint(p, 0, &v));
+ *sizep = (wt_off_t)v;
+ return (0);
+}
diff --git a/src/third_party/wiredtiger/src/include/btmem.h b/src/third_party/wiredtiger/src/include/btmem.h
index 71a26862e4a..7729c9887d5 100644
--- a/src/third_party/wiredtiger/src/include/btmem.h
+++ b/src/third_party/wiredtiger/src/include/btmem.h
@@ -1130,9 +1130,9 @@ struct __wt_update {
/*
* WT_MODIFY_MEM_FACTOR --
- * Limit update chains to a factor of the base document size.
+ * Limit update chains to a fraction of the base document size.
*/
-#define WT_MODIFY_MEM_FACTOR 1
+#define WT_MODIFY_MEM_FRACTION 10
/*
* WT_INSERT --
diff --git a/src/third_party/wiredtiger/src/include/btree.h b/src/third_party/wiredtiger/src/include/btree.h
index 0becfded2a9..44fae885ae1 100644
--- a/src/third_party/wiredtiger/src/include/btree.h
+++ b/src/third_party/wiredtiger/src/include/btree.h
@@ -16,6 +16,8 @@
#define WT_BTREE_MAJOR_VERSION_MAX 1 /* Newest version supported */
#define WT_BTREE_MINOR_VERSION_MAX 1
+#define WT_BTREE_MIN_ALLOC_SIZE 512
+
/*
* The maximum btree leaf and internal page size is 512MB (2^29). The limit
* is enforced in software, it could be larger, specifically, the underlying
diff --git a/src/third_party/wiredtiger/src/include/btree.i b/src/third_party/wiredtiger/src/include/btree.i
index 162ea524b09..2fb9afd14a6 100644
--- a/src/third_party/wiredtiger/src/include/btree.i
+++ b/src/third_party/wiredtiger/src/include/btree.i
@@ -922,6 +922,7 @@ __wt_row_leaf_key_set_cell(WT_PAGE *page, WT_ROW *rip, WT_CELL *cell)
*/
v = WT_CELL_ENCODE_OFFSET(WT_PAGE_DISK_OFFSET(page, cell)) |
WT_CELL_FLAG;
+ WT_ASSERT(NULL, WT_ROW_SLOT(page, rip) < page->entries);
WT_ROW_KEY_SET(rip, v);
}
@@ -941,6 +942,7 @@ __wt_row_leaf_key_set(WT_PAGE *page, WT_ROW *rip, WT_CELL_UNPACK *unpack)
v = WT_K_ENCODE_KEY_LEN(unpack->size) |
WT_K_ENCODE_KEY_OFFSET(WT_PAGE_DISK_OFFSET(page, unpack->data)) |
WT_K_FLAG;
+ WT_ASSERT(NULL, WT_ROW_SLOT(page, rip) < page->entries);
WT_ROW_KEY_SET(rip, v);
}
@@ -979,6 +981,7 @@ __wt_row_leaf_value_set(WT_PAGE *page, WT_ROW *rip, WT_CELL_UNPACK *unpack)
WT_KV_ENCODE_VALUE_LEN(unpack->size) |
WT_KV_ENCODE_KEY_OFFSET(key_offset) |
WT_KV_ENCODE_VALUE_OFFSET(value_offset) | WT_KV_FLAG;
+ WT_ASSERT(NULL, WT_ROW_SLOT(page, rip) < page->entries);
WT_ROW_KEY_SET(rip, v);
}
diff --git a/src/third_party/wiredtiger/src/include/cache.h b/src/third_party/wiredtiger/src/include/cache.h
index 07d25d13ec2..2146a00a76d 100644
--- a/src/third_party/wiredtiger/src/include/cache.h
+++ b/src/third_party/wiredtiger/src/include/cache.h
@@ -170,7 +170,7 @@ struct __wt_cache {
* Score of how aggressive eviction should be about selecting eviction
* candidates. If eviction is struggling to make progress, this score
* rises (up to a maximum of 100), at which point the cache is "stuck"
- * and transaction will be rolled back.
+ * and transactions will be rolled back.
*/
uint32_t evict_aggressive_score;
@@ -250,11 +250,12 @@ struct __wt_cache {
/* AUTOMATIC FLAG VALUE GENERATION START */
#define WT_CACHE_EVICT_CLEAN 0x01u /* Evict clean pages */
#define WT_CACHE_EVICT_CLEAN_HARD 0x02u /* Clean % blocking app threads */
-#define WT_CACHE_EVICT_DIRTY 0x04u /* Evict dirty pages */
-#define WT_CACHE_EVICT_DIRTY_HARD 0x08u /* Dirty % blocking app threads */
-#define WT_CACHE_EVICT_LOOKASIDE 0x10u /* Try lookaside eviction */
-#define WT_CACHE_EVICT_SCRUB 0x20u /* Scrub dirty pages */
-#define WT_CACHE_EVICT_URGENT 0x40u /* Pages are in the urgent queue */
+#define WT_CACHE_EVICT_DEBUG_MODE 0x04u /* Aggressive debugging mode */
+#define WT_CACHE_EVICT_DIRTY 0x08u /* Evict dirty pages */
+#define WT_CACHE_EVICT_DIRTY_HARD 0x10u /* Dirty % blocking app threads */
+#define WT_CACHE_EVICT_LOOKASIDE 0x20u /* Try lookaside eviction */
+#define WT_CACHE_EVICT_SCRUB 0x40u /* Scrub dirty pages */
+#define WT_CACHE_EVICT_URGENT 0x80u /* Pages are in the urgent queue */
/* AUTOMATIC FLAG VALUE GENERATION STOP */
#define WT_CACHE_EVICT_ALL (WT_CACHE_EVICT_CLEAN | WT_CACHE_EVICT_DIRTY)
uint32_t flags;
diff --git a/src/third_party/wiredtiger/src/include/config.h b/src/third_party/wiredtiger/src/include/config.h
index 64c55d7223b..847ddef1b2e 100644
--- a/src/third_party/wiredtiger/src/include/config.h
+++ b/src/third_party/wiredtiger/src/include/config.h
@@ -80,35 +80,36 @@ struct __wt_config_parser_impl {
#define WT_CONFIG_ENTRY_WT_SESSION_compact 22
#define WT_CONFIG_ENTRY_WT_SESSION_create 23
#define WT_CONFIG_ENTRY_WT_SESSION_drop 24
-#define WT_CONFIG_ENTRY_WT_SESSION_join 25
-#define WT_CONFIG_ENTRY_WT_SESSION_log_flush 26
-#define WT_CONFIG_ENTRY_WT_SESSION_log_printf 27
-#define WT_CONFIG_ENTRY_WT_SESSION_open_cursor 28
-#define WT_CONFIG_ENTRY_WT_SESSION_prepare_transaction 29
-#define WT_CONFIG_ENTRY_WT_SESSION_query_timestamp 30
-#define WT_CONFIG_ENTRY_WT_SESSION_rebalance 31
-#define WT_CONFIG_ENTRY_WT_SESSION_reconfigure 32
-#define WT_CONFIG_ENTRY_WT_SESSION_rename 33
-#define WT_CONFIG_ENTRY_WT_SESSION_reset 34
-#define WT_CONFIG_ENTRY_WT_SESSION_rollback_transaction 35
-#define WT_CONFIG_ENTRY_WT_SESSION_salvage 36
-#define WT_CONFIG_ENTRY_WT_SESSION_snapshot 37
-#define WT_CONFIG_ENTRY_WT_SESSION_strerror 38
-#define WT_CONFIG_ENTRY_WT_SESSION_timestamp_transaction 39
-#define WT_CONFIG_ENTRY_WT_SESSION_transaction_sync 40
-#define WT_CONFIG_ENTRY_WT_SESSION_truncate 41
-#define WT_CONFIG_ENTRY_WT_SESSION_upgrade 42
-#define WT_CONFIG_ENTRY_WT_SESSION_verify 43
-#define WT_CONFIG_ENTRY_colgroup_meta 44
-#define WT_CONFIG_ENTRY_file_config 45
-#define WT_CONFIG_ENTRY_file_meta 46
-#define WT_CONFIG_ENTRY_index_meta 47
-#define WT_CONFIG_ENTRY_lsm_meta 48
-#define WT_CONFIG_ENTRY_table_meta 49
-#define WT_CONFIG_ENTRY_wiredtiger_open 50
-#define WT_CONFIG_ENTRY_wiredtiger_open_all 51
-#define WT_CONFIG_ENTRY_wiredtiger_open_basecfg 52
-#define WT_CONFIG_ENTRY_wiredtiger_open_usercfg 53
+#define WT_CONFIG_ENTRY_WT_SESSION_import 25
+#define WT_CONFIG_ENTRY_WT_SESSION_join 26
+#define WT_CONFIG_ENTRY_WT_SESSION_log_flush 27
+#define WT_CONFIG_ENTRY_WT_SESSION_log_printf 28
+#define WT_CONFIG_ENTRY_WT_SESSION_open_cursor 29
+#define WT_CONFIG_ENTRY_WT_SESSION_prepare_transaction 30
+#define WT_CONFIG_ENTRY_WT_SESSION_query_timestamp 31
+#define WT_CONFIG_ENTRY_WT_SESSION_rebalance 32
+#define WT_CONFIG_ENTRY_WT_SESSION_reconfigure 33
+#define WT_CONFIG_ENTRY_WT_SESSION_rename 34
+#define WT_CONFIG_ENTRY_WT_SESSION_reset 35
+#define WT_CONFIG_ENTRY_WT_SESSION_rollback_transaction 36
+#define WT_CONFIG_ENTRY_WT_SESSION_salvage 37
+#define WT_CONFIG_ENTRY_WT_SESSION_snapshot 38
+#define WT_CONFIG_ENTRY_WT_SESSION_strerror 39
+#define WT_CONFIG_ENTRY_WT_SESSION_timestamp_transaction 40
+#define WT_CONFIG_ENTRY_WT_SESSION_transaction_sync 41
+#define WT_CONFIG_ENTRY_WT_SESSION_truncate 42
+#define WT_CONFIG_ENTRY_WT_SESSION_upgrade 43
+#define WT_CONFIG_ENTRY_WT_SESSION_verify 44
+#define WT_CONFIG_ENTRY_colgroup_meta 45
+#define WT_CONFIG_ENTRY_file_config 46
+#define WT_CONFIG_ENTRY_file_meta 47
+#define WT_CONFIG_ENTRY_index_meta 48
+#define WT_CONFIG_ENTRY_lsm_meta 49
+#define WT_CONFIG_ENTRY_table_meta 50
+#define WT_CONFIG_ENTRY_wiredtiger_open 51
+#define WT_CONFIG_ENTRY_wiredtiger_open_all 52
+#define WT_CONFIG_ENTRY_wiredtiger_open_basecfg 53
+#define WT_CONFIG_ENTRY_wiredtiger_open_usercfg 54
/*
* configuration section: END
* DO NOT EDIT: automatically built by dist/flags.py.
diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h
index f122c5e8f84..370f467e930 100644
--- a/src/third_party/wiredtiger/src/include/extern.h
+++ b/src/third_party/wiredtiger/src/include/extern.h
@@ -15,7 +15,8 @@ extern int __wt_block_addr_invalid(WT_SESSION_IMPL *session, WT_BLOCK *block, co
extern int __wt_block_addr_string(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_block_buffer_to_ckpt(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *p, WT_BLOCK_CKPT *ci) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_block_ckpt_decode(WT_SESSION *wt_session, size_t allocsize, const uint8_t *p, WT_BLOCK_CKPT *ci) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
-extern int __wt_block_ckpt_to_buffer(WT_SESSION_IMPL *session, WT_BLOCK *block, uint8_t **pp, WT_BLOCK_CKPT *ci) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
+extern int __wt_block_ckpt_to_buffer(WT_SESSION_IMPL *session, WT_BLOCK *block, uint8_t **pp, WT_BLOCK_CKPT *ci, bool skip_avail) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
+extern void __wt_ckpt_verbose(WT_SESSION_IMPL *session, WT_BLOCK *block, const char *tag, const char *ckpt_name, const uint8_t *ckpt_string);
extern int __wt_block_ckpt_init(WT_SESSION_IMPL *session, WT_BLOCK_CKPT *ci, const char *name) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_block_checkpoint_load(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size, uint8_t *root_addr, size_t *root_addr_sizep, bool checkpoint) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_block_checkpoint_unload(WT_SESSION_IMPL *session, WT_BLOCK *block, bool checkpoint) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
@@ -23,6 +24,8 @@ extern void __wt_block_ckpt_destroy(WT_SESSION_IMPL *session, WT_BLOCK_CKPT *ci)
extern int __wt_block_checkpoint_start(WT_SESSION_IMPL *session, WT_BLOCK *block) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_block_checkpoint(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, WT_CKPT *ckptbase, bool data_checksum) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_block_checkpoint_resolve(WT_SESSION_IMPL *session, WT_BLOCK *block, bool failed) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
+extern int __wt_block_checkpoint_final(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint8_t **file_sizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
+extern int __wt_block_checkpoint_last(WT_SESSION_IMPL *session, WT_BLOCK *block, char **metadatap, char **checkpoint_listp, WT_ITEM *checkpoint) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_block_compact_start(WT_SESSION_IMPL *session, WT_BLOCK *block) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_block_compact_end(WT_SESSION_IMPL *session, WT_BLOCK *block) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_block_compact_skip(WT_SESSION_IMPL *session, WT_BLOCK *block, bool *skipp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
@@ -141,12 +144,14 @@ extern void __wt_btree_page_version_config(WT_SESSION_IMPL *session);
extern int __wt_btree_open(WT_SESSION_IMPL *session, const char *op_cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_btree_close(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_btree_discard(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
+extern int __wt_btree_config_encryptor(WT_SESSION_IMPL *session, const char **cfg, WT_KEYED_ENCRYPTOR **kencryptorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern void __wt_root_ref_init(WT_SESSION_IMPL *session, WT_REF *root_ref, WT_PAGE *root, bool is_recno);
extern int __wt_btree_tree_open(WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_btree_new_leaf_page(WT_SESSION_IMPL *session, WT_PAGE **pagep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern bool __wt_btree_immediately_durable(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_btree_huffman_open(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern void __wt_btree_huffman_close(WT_SESSION_IMPL *session);
+extern int __wt_import(WT_SESSION_IMPL *session, const char *uri) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_bt_read(WT_SESSION_IMPL *session, WT_ITEM *buf, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_bt_write(WT_SESSION_IMPL *session, WT_ITEM *buf, uint8_t *addr, size_t *addr_sizep, size_t *compressed_sizep, bool checkpoint, bool checkpoint_io, bool compressed) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern const char *__wt_page_type_string(u_int type) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default")));
@@ -172,7 +177,7 @@ extern int __wt_bt_rebalance(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC
extern int __wt_value_return_upd(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd, bool ignore_visibility) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_key_return(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_value_return(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
-extern int __wt_bt_salvage(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
+extern int __wt_salvage(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_multi_to_ref(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi, WT_REF **refp, size_t *incrp, bool closing) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_split_insert(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_split_multi(WT_SESSION_IMPL *session, WT_REF *ref, int closing) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
@@ -237,6 +242,7 @@ extern void __wt_conn_foc_discard(WT_SESSION_IMPL *session);
extern int __wt_configure_method(WT_SESSION_IMPL *session, const char *method, const char *uri, const char *config, const char *type, const char *check) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_config_check(WT_SESSION_IMPL *session, const WT_CONFIG_ENTRY *entry, const char *config, size_t config_len) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_config_collapse(WT_SESSION_IMPL *session, const char **cfg, char **config_ret) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
+extern int __wt_config_discard_defaults(WT_SESSION_IMPL *session, const char **cfg, const char *config, char **config_ret) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_config_merge(WT_SESSION_IMPL *session, const char **cfg, const char *cfg_strip, const char **config_ret) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_conn_config_init(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern void __wt_conn_config_discard(WT_SESSION_IMPL *session);
@@ -516,8 +522,10 @@ extern int __wt_meta_apply_all(WT_SESSION_IMPL *session, int (*file_func)(WT_SES
extern int __wt_meta_checkpoint(WT_SESSION_IMPL *session, const char *fname, const char *checkpoint, WT_CKPT *ckpt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_meta_checkpoint_last_name(WT_SESSION_IMPL *session, const char *fname, const char **namep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_meta_checkpoint_clear(WT_SESSION_IMPL *session, const char *fname) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
-extern int __wt_meta_ckptlist_get(WT_SESSION_IMPL *session, const char *fname, WT_CKPT **ckptbasep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
+extern int __wt_meta_block_metadata(WT_SESSION_IMPL *session, const char *config, WT_CKPT *ckpt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
+extern int __wt_meta_ckptlist_get(WT_SESSION_IMPL *session, const char *fname, bool update, WT_CKPT **ckptbasep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_metadata_set_base_write_gen(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
+extern int __wt_meta_ckptlist_to_meta(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_meta_ckptlist_set(WT_SESSION_IMPL *session, const char *fname, WT_CKPT *ckptbase, WT_LSN *ckptlsn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern void __wt_meta_ckptlist_free(WT_SESSION_IMPL *session, WT_CKPT **ckptbasep);
extern void __wt_meta_checkpoint_free(WT_SESSION_IMPL *session, WT_CKPT *ckpt);
@@ -702,7 +710,6 @@ extern int __wt_session_get_btree_ckpt(WT_SESSION_IMPL *session, const char *uri
extern void __wt_session_close_cache(WT_SESSION_IMPL *session);
extern int __wt_session_get_dhandle(WT_SESSION_IMPL *session, const char *uri, const char *checkpoint, const char *cfg[], uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_session_lock_checkpoint(WT_SESSION_IMPL *session, const char *checkpoint) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
-extern int __wt_salvage(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_cond_auto_alloc(WT_SESSION_IMPL *session, const char *name, uint64_t min, uint64_t max, WT_CONDVAR **condp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern void __wt_cond_auto_wait_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond, bool progress, bool (*run_func)(WT_SESSION_IMPL *), bool *signalled);
extern void __wt_cond_auto_wait(WT_SESSION_IMPL *session, WT_CONDVAR *cond, bool progress, bool (*run_func)(WT_SESSION_IMPL *));
diff --git a/src/third_party/wiredtiger/src/include/meta.h b/src/third_party/wiredtiger/src/include/meta.h
index 0b08b318c3d..ac9e0be3c20 100644
--- a/src/third_party/wiredtiger/src/include/meta.h
+++ b/src/third_party/wiredtiger/src/include/meta.h
@@ -66,9 +66,19 @@
struct __wt_ckpt {
char *name; /* Name or NULL */
- WT_ITEM addr; /* Checkpoint cookie string */
- WT_ITEM raw; /* Checkpoint cookie raw */
-
+ /*
+ * Each internal checkpoint name is appended with a generation
+ * to make it a unique name. We're solving two problems: when
+ * two checkpoints are taken quickly, the timer may not be
+ * unique and/or we can even see time travel on the second
+ * checkpoint if we snapshot the time in-between nanoseconds
+ * rolling over. Second, if we reset the generational counter
+ * when new checkpoints arrive, we could logically re-create
+ * specific checkpoints, racing with cursors open on those
+ * checkpoints. I can't think of any way to return incorrect
+ * results by racing with those cursors, but it's simpler not
+ * to worry about it.
+ */
int64_t order; /* Checkpoint order */
uint64_t sec; /* Wall clock time */
@@ -77,6 +87,9 @@ struct __wt_ckpt {
uint64_t write_gen; /* Write generation */
+ char *block_metadata; /* Block-stored metadata */
+ char *block_checkpoint; /* Block-stored checkpoint */
+
/* Validity window */
wt_timestamp_t newest_durable_ts;
wt_timestamp_t oldest_start_ts;
@@ -84,6 +97,9 @@ struct __wt_ckpt {
wt_timestamp_t newest_stop_ts;
uint64_t newest_stop_txn;
+ WT_ITEM addr; /* Checkpoint cookie string */
+ WT_ITEM raw; /* Checkpoint cookie raw */
+
void *bpriv; /* Block manager private */
/* AUTOMATIC FLAG VALUE GENERATION START */
diff --git a/src/third_party/wiredtiger/src/include/misc.i b/src/third_party/wiredtiger/src/include/misc.i
index ab24e87f34a..b552f9b1f3e 100644
--- a/src/third_party/wiredtiger/src/include/misc.i
+++ b/src/third_party/wiredtiger/src/include/misc.i
@@ -255,13 +255,26 @@ __wt_spin_backoff(uint64_t *yield_count, uint64_t *sleep_usecs)
static inline void
__wt_timing_stress(WT_SESSION_IMPL *session, u_int flag)
{
- uint64_t i;
+ double pct;
+ uint64_t i, max;
/* Optionally only sleep when a specified configuration flag is set. */
if (flag != 0 && !FLD_ISSET(S2C(session)->timing_stress_flags, flag))
return;
/*
+ * If there is a lot of cache pressure, don't let the sleep time
+ * get too large. If the cache is totally full, return.
+ */
+ pct = 0.0;
+ if (__wt_eviction_needed(session, false, false, &pct))
+ max = 5;
+ else
+ max = 9;
+ if (pct > 100.0)
+ return;
+
+ /*
* We need a fast way to choose a sleep time. We want to sleep a short
* period most of the time, but occasionally wait longer. Divide the
* maximum period of time into 10 buckets (where bucket 0 doesn't sleep
@@ -269,7 +282,7 @@ __wt_timing_stress(WT_SESSION_IMPL *session, u_int flag)
* That means we'll hit the maximum roughly every 1K calls.
*/
for (i = 0;;)
- if (__wt_random(&session->rnd) & 0x1 || ++i > 9)
+ if (__wt_random(&session->rnd) & 0x1 || ++i > max)
break;
if (i == 0)
diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h
index e752ac05c0d..26c030009e6 100644
--- a/src/third_party/wiredtiger/src/include/stat.h
+++ b/src/third_party/wiredtiger/src/include/stat.h
@@ -633,6 +633,8 @@ struct __wt_connection_stats {
int64_t session_table_create_success;
int64_t session_table_drop_fail;
int64_t session_table_drop_success;
+ int64_t session_table_import_fail;
+ int64_t session_table_import_success;
int64_t session_table_rebalance_fail;
int64_t session_table_rebalance_success;
int64_t session_table_rename_fail;
diff --git a/src/third_party/wiredtiger/src/include/txn.h b/src/third_party/wiredtiger/src/include/txn.h
index 29ad33f3b76..2a514629006 100644
--- a/src/third_party/wiredtiger/src/include/txn.h
+++ b/src/third_party/wiredtiger/src/include/txn.h
@@ -242,6 +242,11 @@ struct __wt_txn_op {
} mode;
} truncate_row;
} u;
+
+/* AUTOMATIC FLAG VALUE GENERATION START */
+#define WT_TXN_OP_KEY_REPEATED 0x1u
+/* AUTOMATIC FLAG VALUE GENERATION STOP */
+ uint32_t flags;
};
/*
@@ -305,14 +310,6 @@ struct __wt_txn {
WT_TXN_OP *mod;
size_t mod_alloc;
u_int mod_count;
-#ifdef HAVE_DIAGNOSTIC
- /*
- * Reference count of multiple updates processed, as part of a single
- * transaction operation processing for resolving the indirect update
- * references in a prepared transaction as part of commit.
- */
- u_int multi_update_count;
-#endif
/* Scratch buffer for in-memory log records. */
WT_ITEM *logrec;
diff --git a/src/third_party/wiredtiger/src/include/txn.i b/src/third_party/wiredtiger/src/include/txn.i
index 0c8e22c90d1..1aea1e854d3 100644
--- a/src/third_party/wiredtiger/src/include/txn.i
+++ b/src/third_party/wiredtiger/src/include/txn.i
@@ -171,17 +171,21 @@ __txn_resolve_prepared_update(WT_SESSION_IMPL *session, WT_UPDATE *upd)
/*
* __wt_txn_resolve_prepared_op --
- * Resolve a transaction operation indirect references.
+ * Resolve a transaction's operations indirect references.
*
* In case of prepared transactions, the prepared updates could be evicted
* using cache overflow mechanism. Transaction operations referring to
* these prepared updates would be referring to them using indirect
* references (i.e keys/recnos), which need to be resolved as part of that
* transaction commit/rollback.
+ *
+ * If no updates are resolved throw an error. Increment resolved update
+ * count for each resolved update count we locate.
*/
static inline int
__wt_txn_resolve_prepared_op(
- WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit)
+ WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit,
+ int64_t *resolved_update_countp)
{
WT_CURSOR *cursor;
WT_DECL_RET;
@@ -218,37 +222,29 @@ __wt_txn_resolve_prepared_op(
(WT_CURSOR_BTREE *)cursor, &upd));
WT_ERR(ret);
-#ifdef HAVE_DIAGNOSTIC
- /*
- * Do what we can to ensure that finding prepared updates from a key
- * is working as expected. In the case where a transaction has updated
- * the same key multiple times, it's possible to resolve all updates
- * for the key when processing the first op structure, and then have
- * eviction free those updates before subsequent ops are processed,
- * which means a search could reasonably not find an update in that
- * case.
- * We track the update count only for commit, but not for rollback, as
- * our tracking is based on transaction id, and in case of rollback, we
- * set it to aborted.
- */
- if (upd == NULL && commit) {
- /*
- * FIXME:
- * WT_ASSERT(session, txn->multi_update_count > 0);
- */
- --txn->multi_update_count;
+ /* If we haven't found anything then there's an error. */
+ if (upd == NULL) {
+ WT_ASSERT(session, upd != NULL);
+ WT_ERR(WT_NOTFOUND);
}
-#endif
-
- WT_STAT_CONN_INCR(session, txn_prepared_updates_resolved);
for (; upd != NULL; upd = upd->next) {
- if (upd->txnid != txn->id)
+ /*
+ * Aborted updates can exist in the update chain of our txn.
+ * Generally this will occur due to a reserved update.
+ * As such we should skip over these updates. If the txn
+ * id is then different and not aborted we know we've
+ * reached the end of our update chain and can exit.
+ */
+ if (upd->txnid == WT_TXN_ABORTED)
continue;
-
+ if (upd->txnid != txn->id)
+ break;
if (op->u.op_upd == NULL)
op->u.op_upd = upd;
+ ++(*resolved_update_countp);
+
if (!commit) {
upd->txnid = WT_TXN_ABORTED;
continue;
@@ -282,52 +278,9 @@ __wt_txn_resolve_prepared_op(
* thing as part of "txn_op2".
*/
-#ifdef HAVE_DIAGNOSTIC
- /*
- * When an update is not identified for resolution of a
- * transaction operation, it might have been already processed
- * during the resolution of a previous update belonging to the
- * same key. To ascertain transaction tracks multiple extra
- * updates processed in resolution of an transaction operation.
- */
- if (upd->prepare_state == WT_PREPARE_RESOLVED) {
- /*
- * FIXME:
- * WT_ASSERT(session, txn->multi_update_count > 0);
- */
- --txn->multi_update_count;
- } else if (upd != op->u.op_upd)
- ++txn->multi_update_count;
-#endif
-
- if (upd->prepare_state == WT_PREPARE_RESOLVED)
- break;
-
/* Resolve the prepared update to be committed update. */
__txn_resolve_prepared_update(session, upd);
}
-
- /* FIXME: it isn't safe to walk updates after they are resolved. */
-#if 0 && defined(HAVE_DIAGNOSTIC)
- upd = op->u.op_upd;
- /* Ensure that we have not missed any of this transaction updates. */
- for (; upd != NULL; upd = upd->next) {
- /*
- * Should not have an unprocessed uncommitted update of this
- * transaction. For commit, no uncommitted update of this
- * transaction should be in prepared state. For rollback, there
- * should not be any more uncommitted updates from this
- * transaction.
- */
- if (commit && upd->txnid == txn->id)
- WT_ASSERT(session,
- upd->prepare_state != WT_PREPARE_INPROGRESS);
- else
- WT_ASSERT(session, upd->txnid != txn->id);
-
- }
-#endif
-
err: WT_TRET(cursor->close(cursor));
return (ret);
}
@@ -443,6 +396,43 @@ __wt_txn_op_apply_prepare_state(
}
/*
+ * __wt_txn_op_delete_commit_apply_timestamps --
+ * Apply the correct start and durable timestamps to any
+ * updates in the page del update list.
+ */
+static inline void
+__wt_txn_op_delete_commit_apply_timestamps(
+ WT_SESSION_IMPL *session, WT_REF *ref)
+{
+ WT_TXN *txn;
+ WT_UPDATE **updp;
+ uint32_t previous_state;
+
+ txn = &session->txn;
+
+ /*
+ * Lock the ref to ensure we don't race with eviction freeing the page
+ * deleted update list or with a page instantiate.
+ */
+ for (;; __wt_yield()) {
+ previous_state = ref->state;
+ WT_ASSERT(session, previous_state != WT_REF_READING);
+ if (previous_state != WT_REF_LOCKED && WT_REF_CAS_STATE(
+ session, ref, previous_state, WT_REF_LOCKED))
+ break;
+ }
+
+ for (updp = ref->page_del->update_list;
+ updp != NULL && *updp != NULL; ++updp) {
+ (*updp)->start_ts = txn->commit_timestamp;
+ (*updp)->durable_ts = txn->durable_timestamp;
+ }
+
+ /* Unlock the page by setting it back to it's previous state */
+ WT_REF_SET_STATE(ref, previous_state);
+}
+
+/*
* __wt_txn_op_set_timestamp --
* Decide whether to copy a commit timestamp into an update. If the op
* structure doesn't have a populated update or ref field or in prepared
@@ -496,6 +486,10 @@ __wt_txn_op_set_timestamp(WT_SESSION_IMPL *session, WT_TXN_OP *op)
&op->u.op_upd->durable_ts;
*timestamp = txn->durable_timestamp;
}
+
+ if (op->type == WT_TXN_OP_REF_DELETE)
+ __wt_txn_op_delete_commit_apply_timestamps(
+ session, op->u.ref);
}
}
@@ -511,9 +505,14 @@ __wt_txn_modify(WT_SESSION_IMPL *session, WT_UPDATE *upd)
txn = &session->txn;
- if (F_ISSET(txn, WT_TXN_READONLY))
+ if (F_ISSET(txn, WT_TXN_READONLY)) {
+ if (F_ISSET(txn, WT_TXN_IGNORE_PREPARE))
+ WT_RET_MSG(session, ENOTSUP,
+ "Transactions with ignore_prepare=true"
+ " cannot perform updates");
WT_RET_MSG(session, WT_ROLLBACK,
"Attempt to update in a read-only transaction");
+ }
WT_RET(__txn_next_op(session, &op));
if (F_ISSET(session, WT_SESSION_LOGGING_INMEM)) {
diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in
index 9830d79a190..484175dd43e 100644
--- a/src/third_party/wiredtiger/src/include/wiredtiger.in
+++ b/src/third_party/wiredtiger/src/include/wiredtiger.in
@@ -1448,6 +1448,21 @@ struct __wt_session {
int __F(create)(WT_SESSION *session,
const char *name, const char *config);
+#if !defined(SWIG) && !defined(DOXYGEN)
+ /*!
+ * Import a file.
+ *
+ * @snippet ex_all.c Import a file
+ *
+ * @param session the session handle
+ * @param name the URI of the object to import
+ * @configempty{WT_SESSION.import, see dist/api_data.py}
+ * @errors
+ */
+ int __F(import)(WT_SESSION *session,
+ const char *name, const char *config);
+#endif
+
/*!
* Compact a live row- or column-store btree or LSM tree.
*
@@ -2284,16 +2299,23 @@ struct __wt_connection {
* checkpoints. Zero or one means perform normal archiving., an integer
* between 0 and 1024; default \c 0.}
* @config{&nbsp;&nbsp;&nbsp;&nbsp;
- * rollback_error, return a WT_ROLLBACK error from a transaction
- * operation about every Nth operation to simulate a collision., an
- * integer between 0 and 10M; default \c 0.}
- * @config{&nbsp;&nbsp;&nbsp;&nbsp;table_logging, if true\, write
- * transaction related information to the log for all operations\, even
- * operations for tables with logging turned off. This setting
- * introduces a log format change that may break older versions of
- * WiredTiger. These operations are informational and skipped in
- * recovery., a boolean flag; default \c false.}
- * @config{ ),,}
+ * eviction, if true\, modify internal algorithms to change skew to
+ * force lookaside eviction to happen more aggressively. This includes
+ * but is not limited to not skewing newest\, not favoring leaf pages\,
+ * and modifying the eviction score mechanism., a boolean flag; default
+ * \c false.}
+ * @config{&nbsp;&nbsp;&nbsp;&nbsp;rollback_error, return a
+ * WT_ROLLBACK error from a transaction operation about every Nth
+ * operation to simulate a collision., an integer between 0 and 10M;
+ * default \c 0.}
+ * @config{&nbsp;&nbsp;&nbsp;&nbsp;table_logging, if
+ * true\, write transaction related information to the log for all
+ * operations\, even operations for tables with logging turned off.
+ * This setting introduces a log format change that may break older
+ * versions of WiredTiger. These operations are informational and
+ * skipped in recovery., a boolean flag; default \c false.}
+ * @config{
+ * ),,}
* @config{error_prefix, prefix string for error messages., a string;
* default empty.}
* @config{eviction = (, eviction configuration options., a set of
@@ -2928,14 +2950,21 @@ struct __wt_connection {
* @config{&nbsp;&nbsp;&nbsp;&nbsp;checkpoint_retention, adjust log archiving to
* retain the log records of this number of checkpoints. Zero or one means
* perform normal archiving., an integer between 0 and 1024; default \c 0.}
- * @config{&nbsp;&nbsp;&nbsp;&nbsp;rollback_error, return a WT_ROLLBACK error
- * from a transaction operation about every Nth operation to simulate a
- * collision., an integer between 0 and 10M; default \c 0.}
- * @config{&nbsp;&nbsp;&nbsp;&nbsp;table_logging, if true\, write transaction
- * related information to the log for all operations\, even operations for
- * tables with logging turned off. This setting introduces a log format change
- * that may break older versions of WiredTiger. These operations are
- * informational and skipped in recovery., a boolean flag; default \c false.}
+ * @config{&nbsp;&nbsp;&nbsp;&nbsp;eviction, if true\, modify internal
+ * algorithms to change skew to force lookaside eviction to happen more
+ * aggressively. This includes but is not limited to not skewing newest\, not
+ * favoring leaf pages\, and modifying the eviction score mechanism., a boolean
+ * flag; default \c false.}
+ * @config{&nbsp;&nbsp;&nbsp;&nbsp;rollback_error,
+ * return a WT_ROLLBACK error from a transaction operation about every Nth
+ * operation to simulate a collision., an integer between 0 and 10M; default \c
+ * 0.}
+ * @config{&nbsp;&nbsp;&nbsp;&nbsp;table_logging, if true\, write
+ * transaction related information to the log for all operations\, even
+ * operations for tables with logging turned off. This setting introduces a log
+ * format change that may break older versions of WiredTiger. These operations
+ * are informational and skipped in recovery., a boolean flag; default \c
+ * false.}
* @config{ ),,}
* @config{direct_io, Use \c O_DIRECT on POSIX systems\, and \c
* FILE_FLAG_NO_BUFFERING on Windows to access files. Options are given as a
@@ -5697,201 +5726,205 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection);
#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1307
/*! session: table drop successful calls */
#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1308
+/*! session: table import failed calls */
+#define WT_STAT_CONN_SESSION_TABLE_IMPORT_FAIL 1309
+/*! session: table import successful calls */
+#define WT_STAT_CONN_SESSION_TABLE_IMPORT_SUCCESS 1310
/*! session: table rebalance failed calls */
-#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1309
+#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1311
/*! session: table rebalance successful calls */
-#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1310
+#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1312
/*! session: table rename failed calls */
-#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1311
+#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1313
/*! session: table rename successful calls */
-#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1312
+#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1314
/*! session: table salvage failed calls */
-#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1313
+#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1315
/*! session: table salvage successful calls */
-#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1314
+#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1316
/*! session: table truncate failed calls */
-#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1315
+#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1317
/*! session: table truncate successful calls */
-#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1316
+#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1318
/*! session: table verify failed calls */
-#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1317
+#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1319
/*! session: table verify successful calls */
-#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1318
+#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1320
/*! thread-state: active filesystem fsync calls */
-#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1319
+#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1321
/*! thread-state: active filesystem read calls */
-#define WT_STAT_CONN_THREAD_READ_ACTIVE 1320
+#define WT_STAT_CONN_THREAD_READ_ACTIVE 1322
/*! thread-state: active filesystem write calls */
-#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1321
+#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1323
/*! thread-yield: application thread time evicting (usecs) */
-#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1322
+#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1324
/*! thread-yield: application thread time waiting for cache (usecs) */
-#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1323
+#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1325
/*!
* thread-yield: connection close blocked waiting for transaction state
* stabilization
*/
-#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1324
+#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1326
/*! thread-yield: connection close yielded for lsm manager shutdown */
-#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1325
+#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1327
/*! thread-yield: data handle lock yielded */
-#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1326
+#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1328
/*!
* thread-yield: get reference for page index and slot time sleeping
* (usecs)
*/
-#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1327
+#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1329
/*! thread-yield: log server sync yielded for log write */
-#define WT_STAT_CONN_LOG_SERVER_SYNC_BLOCKED 1328
+#define WT_STAT_CONN_LOG_SERVER_SYNC_BLOCKED 1330
/*! thread-yield: page access yielded due to prepare state change */
-#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1329
+#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1331
/*! thread-yield: page acquire busy blocked */
-#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1330
+#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1332
/*! thread-yield: page acquire eviction blocked */
-#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1331
+#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1333
/*! thread-yield: page acquire locked blocked */
-#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1332
+#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1334
/*! thread-yield: page acquire read blocked */
-#define WT_STAT_CONN_PAGE_READ_BLOCKED 1333
+#define WT_STAT_CONN_PAGE_READ_BLOCKED 1335
/*! thread-yield: page acquire time sleeping (usecs) */
-#define WT_STAT_CONN_PAGE_SLEEP 1334
+#define WT_STAT_CONN_PAGE_SLEEP 1336
/*!
* thread-yield: page delete rollback time sleeping for state change
* (usecs)
*/
-#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1335
+#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1337
/*! thread-yield: page reconciliation yielded due to child modification */
-#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1336
+#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1338
/*! transaction: Number of prepared updates */
-#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COUNT 1337
+#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COUNT 1339
/*! transaction: Number of prepared updates added to cache overflow */
-#define WT_STAT_CONN_TXN_PREPARED_UPDATES_LOOKASIDE_INSERTS 1338
+#define WT_STAT_CONN_TXN_PREPARED_UPDATES_LOOKASIDE_INSERTS 1340
/*! transaction: Number of prepared updates resolved */
-#define WT_STAT_CONN_TXN_PREPARED_UPDATES_RESOLVED 1339
+#define WT_STAT_CONN_TXN_PREPARED_UPDATES_RESOLVED 1341
/*! transaction: commit timestamp queue entries walked */
-#define WT_STAT_CONN_TXN_COMMIT_QUEUE_WALKED 1340
+#define WT_STAT_CONN_TXN_COMMIT_QUEUE_WALKED 1342
/*! transaction: commit timestamp queue insert to empty */
-#define WT_STAT_CONN_TXN_COMMIT_QUEUE_EMPTY 1341
+#define WT_STAT_CONN_TXN_COMMIT_QUEUE_EMPTY 1343
/*! transaction: commit timestamp queue inserts to head */
-#define WT_STAT_CONN_TXN_COMMIT_QUEUE_HEAD 1342
+#define WT_STAT_CONN_TXN_COMMIT_QUEUE_HEAD 1344
/*! transaction: commit timestamp queue inserts total */
-#define WT_STAT_CONN_TXN_COMMIT_QUEUE_INSERTS 1343
+#define WT_STAT_CONN_TXN_COMMIT_QUEUE_INSERTS 1345
/*! transaction: commit timestamp queue length */
-#define WT_STAT_CONN_TXN_COMMIT_QUEUE_LEN 1344
+#define WT_STAT_CONN_TXN_COMMIT_QUEUE_LEN 1346
/*! transaction: number of named snapshots created */
-#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1345
+#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1347
/*! transaction: number of named snapshots dropped */
-#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1346
+#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1348
/*! transaction: prepared transactions */
-#define WT_STAT_CONN_TXN_PREPARE 1347
+#define WT_STAT_CONN_TXN_PREPARE 1349
/*! transaction: prepared transactions committed */
-#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1348
+#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1350
/*! transaction: prepared transactions currently active */
-#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1349
+#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1351
/*! transaction: prepared transactions rolled back */
-#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1350
+#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1352
/*! transaction: query timestamp calls */
-#define WT_STAT_CONN_TXN_QUERY_TS 1351
+#define WT_STAT_CONN_TXN_QUERY_TS 1353
/*! transaction: read timestamp queue entries walked */
-#define WT_STAT_CONN_TXN_READ_QUEUE_WALKED 1352
+#define WT_STAT_CONN_TXN_READ_QUEUE_WALKED 1354
/*! transaction: read timestamp queue insert to empty */
-#define WT_STAT_CONN_TXN_READ_QUEUE_EMPTY 1353
+#define WT_STAT_CONN_TXN_READ_QUEUE_EMPTY 1355
/*! transaction: read timestamp queue inserts to head */
-#define WT_STAT_CONN_TXN_READ_QUEUE_HEAD 1354
+#define WT_STAT_CONN_TXN_READ_QUEUE_HEAD 1356
/*! transaction: read timestamp queue inserts total */
-#define WT_STAT_CONN_TXN_READ_QUEUE_INSERTS 1355
+#define WT_STAT_CONN_TXN_READ_QUEUE_INSERTS 1357
/*! transaction: read timestamp queue length */
-#define WT_STAT_CONN_TXN_READ_QUEUE_LEN 1356
+#define WT_STAT_CONN_TXN_READ_QUEUE_LEN 1358
/*! transaction: rollback to stable calls */
-#define WT_STAT_CONN_TXN_ROLLBACK_TO_STABLE 1357
+#define WT_STAT_CONN_TXN_ROLLBACK_TO_STABLE 1359
/*! transaction: rollback to stable updates aborted */
-#define WT_STAT_CONN_TXN_ROLLBACK_UPD_ABORTED 1358
+#define WT_STAT_CONN_TXN_ROLLBACK_UPD_ABORTED 1360
/*! transaction: rollback to stable updates removed from cache overflow */
-#define WT_STAT_CONN_TXN_ROLLBACK_LAS_REMOVED 1359
+#define WT_STAT_CONN_TXN_ROLLBACK_LAS_REMOVED 1361
/*! transaction: set timestamp calls */
-#define WT_STAT_CONN_TXN_SET_TS 1360
+#define WT_STAT_CONN_TXN_SET_TS 1362
/*! transaction: set timestamp commit calls */
-#define WT_STAT_CONN_TXN_SET_TS_COMMIT 1361
+#define WT_STAT_CONN_TXN_SET_TS_COMMIT 1363
/*! transaction: set timestamp commit updates */
-#define WT_STAT_CONN_TXN_SET_TS_COMMIT_UPD 1362
+#define WT_STAT_CONN_TXN_SET_TS_COMMIT_UPD 1364
/*! transaction: set timestamp oldest calls */
-#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1363
+#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1365
/*! transaction: set timestamp oldest updates */
-#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1364
+#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1366
/*! transaction: set timestamp stable calls */
-#define WT_STAT_CONN_TXN_SET_TS_STABLE 1365
+#define WT_STAT_CONN_TXN_SET_TS_STABLE 1367
/*! transaction: set timestamp stable updates */
-#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1366
+#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1368
/*! transaction: transaction begins */
-#define WT_STAT_CONN_TXN_BEGIN 1367
+#define WT_STAT_CONN_TXN_BEGIN 1369
/*! transaction: transaction checkpoint currently running */
-#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1368
+#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1370
/*! transaction: transaction checkpoint generation */
-#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1369
+#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1371
/*! transaction: transaction checkpoint max time (msecs) */
-#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1370
+#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1372
/*! transaction: transaction checkpoint min time (msecs) */
-#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1371
+#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1373
/*! transaction: transaction checkpoint most recent time (msecs) */
-#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1372
+#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1374
/*! transaction: transaction checkpoint scrub dirty target */
-#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1373
+#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1375
/*! transaction: transaction checkpoint scrub time (msecs) */
-#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1374
+#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1376
/*! transaction: transaction checkpoint total time (msecs) */
-#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1375
+#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1377
/*! transaction: transaction checkpoints */
-#define WT_STAT_CONN_TXN_CHECKPOINT 1376
+#define WT_STAT_CONN_TXN_CHECKPOINT 1378
/*!
* transaction: transaction checkpoints skipped because database was
* clean
*/
-#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1377
+#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1379
/*! transaction: transaction failures due to cache overflow */
-#define WT_STAT_CONN_TXN_FAIL_CACHE 1378
+#define WT_STAT_CONN_TXN_FAIL_CACHE 1380
/*!
* transaction: transaction fsync calls for checkpoint after allocating
* the transaction ID
*/
-#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1379
+#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1381
/*!
* transaction: transaction fsync duration for checkpoint after
* allocating the transaction ID (usecs)
*/
-#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1380
+#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1382
/*! transaction: transaction range of IDs currently pinned */
-#define WT_STAT_CONN_TXN_PINNED_RANGE 1381
+#define WT_STAT_CONN_TXN_PINNED_RANGE 1383
/*! transaction: transaction range of IDs currently pinned by a checkpoint */
-#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1382
+#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1384
/*!
* transaction: transaction range of IDs currently pinned by named
* snapshots
*/
-#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1383
+#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1385
/*! transaction: transaction range of timestamps currently pinned */
-#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1384
+#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1386
/*! transaction: transaction range of timestamps pinned by a checkpoint */
-#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1385
+#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1387
/*!
* transaction: transaction range of timestamps pinned by the oldest
* active read timestamp
*/
-#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1386
+#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1388
/*!
* transaction: transaction range of timestamps pinned by the oldest
* timestamp
*/
-#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1387
+#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1389
/*! transaction: transaction read timestamp of the oldest active reader */
-#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1388
+#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1390
/*! transaction: transaction sync calls */
-#define WT_STAT_CONN_TXN_SYNC 1389
+#define WT_STAT_CONN_TXN_SYNC 1391
/*! transaction: transactions committed */
-#define WT_STAT_CONN_TXN_COMMIT 1390
+#define WT_STAT_CONN_TXN_COMMIT 1392
/*! transaction: transactions rolled back */
-#define WT_STAT_CONN_TXN_ROLLBACK 1391
+#define WT_STAT_CONN_TXN_ROLLBACK 1393
/*! transaction: update conflicts */
-#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1392
+#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1394
/*!
* @}
diff --git a/src/third_party/wiredtiger/src/include/wt_internal.h b/src/third_party/wiredtiger/src/include/wt_internal.h
index 8f18828d59b..5d2205f7718 100644
--- a/src/third_party/wiredtiger/src/include/wt_internal.h
+++ b/src/third_party/wiredtiger/src/include/wt_internal.h
@@ -407,17 +407,18 @@ typedef uint64_t wt_timestamp_t;
#endif
#include "verify_build.h"
+#include "cache.i" /* required by misc.i */
#include "ctype.i" /* required by packing.i */
#include "intpack.i" /* required by cell.i, packing.i */
#include "misc.i" /* required by mutex.i */
#include "buf.i" /* required by cell.i */
-#include "cache.i" /* required by txn.i */
#include "cell.i" /* required by btree.i */
#include "mutex.i" /* required by btree.i */
#include "txn.i" /* required by btree.i */
#include "bitstring.i"
+#include "block.i"
#include "btree.i" /* required by cursor.i */
#include "btree_cmp.i"
#include "column.i"
diff --git a/src/third_party/wiredtiger/src/meta/meta_ckpt.c b/src/third_party/wiredtiger/src/meta/meta_ckpt.c
index 9ac9aa11994..7b0f9a3e621 100644
--- a/src/third_party/wiredtiger/src/meta/meta_ckpt.c
+++ b/src/third_party/wiredtiger/src/meta/meta_ckpt.c
@@ -8,14 +8,14 @@
#include "wt_internal.h"
-static int __ckpt_last(WT_SESSION_IMPL *, const char *, WT_CKPT *);
-static int __ckpt_last_name(WT_SESSION_IMPL *, const char *, const char **);
-static int __ckpt_load(WT_SESSION_IMPL *,
+static int __ckpt_last(WT_SESSION_IMPL *, const char *, WT_CKPT *);
+static int __ckpt_last_name(WT_SESSION_IMPL *, const char *, const char **);
+static int __ckpt_load(WT_SESSION_IMPL *,
WT_CONFIG_ITEM *, WT_CONFIG_ITEM *, WT_CKPT *);
-static int __ckpt_named(
+static int __ckpt_named(
WT_SESSION_IMPL *, const char *, const char *, WT_CKPT *);
-static int __ckpt_set(WT_SESSION_IMPL *, const char *, const char *);
-static int __ckpt_version_chk(WT_SESSION_IMPL *, const char *, const char *);
+static int __ckpt_set(WT_SESSION_IMPL *, const char *, const char *);
+static int __ckpt_version_chk(WT_SESSION_IMPL *, const char *, const char *);
/*
* __wt_meta_checkpoint --
@@ -229,6 +229,83 @@ err: __wt_free(session, *namep);
}
/*
+ * __wt_meta_block_metadata --
+ * Build a version of the file's metadata for the block manager to store.
+ */
+int
+__wt_meta_block_metadata(
+ WT_SESSION_IMPL *session, const char *config, WT_CKPT *ckpt)
+{
+ WT_CONFIG_ITEM cval;
+ WT_DECL_ITEM(a);
+ WT_DECL_ITEM(b);
+ WT_DECL_RET;
+ WT_KEYED_ENCRYPTOR *kencryptor;
+ size_t encrypt_size, metadata_len;
+ const char *metadata, *filecfg[] = {
+ WT_CONFIG_BASE(session, file_meta), NULL, NULL };
+ char *min_config;
+
+ min_config = NULL;
+ WT_ERR(__wt_scr_alloc(session, 0, &a));
+ WT_ERR(__wt_scr_alloc(session, 0, &b));
+
+ /*
+ * The metadata has to be encrypted because it contains private data
+ * (for example, column names). We pass the block manager text that
+ * describes the metadata (the encryption information), and the
+ * possibly encrypted metadata encoded as a hexadecimal string.
+ * configuration string.
+ *
+ * Get a minimal configuration string, just the non-default entries.
+ */
+ WT_ERR(__wt_config_discard_defaults(
+ session, filecfg, config, &min_config));
+
+ /* Fill out the configuration array for normal retrieval. */
+ filecfg[1] = config;
+
+ /*
+ * Find out if this file is encrypted. If encrypting, encrypt and encode
+ * the minimal configuration.
+ */
+ WT_ERR(__wt_btree_config_encryptor(session, filecfg, &kencryptor));
+ if (kencryptor == NULL) {
+ metadata = min_config;
+ metadata_len = strlen(min_config);
+ } else {
+ __wt_buf_set(session, a, min_config, strlen(min_config));
+ __wt_encrypt_size(session, kencryptor, a->size, &encrypt_size);
+ WT_ERR(__wt_buf_grow(session, b, encrypt_size));
+ WT_ERR(__wt_encrypt(session, kencryptor, 0, a, b));
+ WT_ERR(__wt_buf_grow(session, a, b->size * 2));
+ __wt_fill_hex(b->mem, b->size, a->mem, a->memsize, &a->size);
+
+ metadata = a->data;
+ metadata_len = a->size;
+ }
+
+ /*
+ * Get a copy of the encryption information and flag if we're doing
+ * encryption. The latter isn't necessary, but it makes it easier to
+ * diagnose issues during the load.
+ */
+ WT_ERR(__wt_config_gets(session, filecfg, "encryption", &cval));
+ WT_ERR(__wt_buf_fmt(session, b,
+ "encryption=%.*s,"
+ "block_metadata_encrypted=%s,block_metadata=[%.*s]",
+ (int)cval.len, cval.str, kencryptor == NULL ? "false" : "true",
+ (int)metadata_len, metadata));
+ WT_ERR(__wt_strndup(session, b->data, b->size, &ckpt->block_metadata));
+
+err:
+ __wt_free(session, min_config);
+ __wt_scr_free(session, &a);
+ __wt_scr_free(session, &b);
+ return (ret);
+}
+
+/*
* __ckpt_compare_order --
* Qsort comparison routine for the checkpoint list.
*/
@@ -248,15 +325,15 @@ __ckpt_compare_order(const void *a, const void *b)
* Load all available checkpoint information for a file.
*/
int
-__wt_meta_ckptlist_get(
- WT_SESSION_IMPL *session, const char *fname, WT_CKPT **ckptbasep)
+__wt_meta_ckptlist_get(WT_SESSION_IMPL *session,
+ const char *fname, bool update, WT_CKPT **ckptbasep)
{
WT_CKPT *ckpt, *ckptbase;
WT_CONFIG ckptconf;
WT_CONFIG_ITEM k, v;
- WT_DECL_ITEM(buf);
WT_DECL_RET;
size_t allocated, slot;
+ int64_t maxorder;
char *config;
*ckptbasep = NULL;
@@ -269,8 +346,8 @@ __wt_meta_ckptlist_get(
WT_RET(__wt_metadata_search(session, fname, &config));
/* Load any existing checkpoints into the array. */
- WT_ERR(__wt_scr_alloc(session, 0, &buf));
- if (__wt_config_getones(session, config, "checkpoint", &v) == 0) {
+ if ((ret =
+ __wt_config_getones(session, config, "checkpoint", &v)) == 0) {
__wt_config_subinit(session, &ckptconf, &v);
for (; __wt_config_next(&ckptconf, &k, &v) == 0; ++slot) {
WT_ERR(__wt_realloc_def(
@@ -280,22 +357,42 @@ __wt_meta_ckptlist_get(
WT_ERR(__ckpt_load(session, &k, &v, ckpt));
}
}
-
- /*
- * Allocate an extra slot for a new value, plus a slot to mark the end.
- *
- * This isn't very clean, but there's necessary cooperation between the
- * schema layer (that maintains the list of checkpoints), the btree
- * layer (that knows when the root page is written, creating a new
- * checkpoint), and the block manager (which actually creates the
- * checkpoint). All of that cooperation is handled in the WT_CKPT
- * structure referenced from the WT_BTREE structure.
- */
- WT_ERR(__wt_realloc_def(session, &allocated, slot + 2, &ckptbase));
+ WT_ERR_NOTFOUND_OK(ret);
+ if (!update && slot == 0)
+ WT_ERR(WT_NOTFOUND);
/* Sort in creation-order. */
__wt_qsort(ckptbase, slot, sizeof(WT_CKPT), __ckpt_compare_order);
+ if (update) {
+ /*
+ * Allocate an extra slot for a new value, plus a slot to mark
+ * mark the end.
+ *
+ * This isn't clean, but there's necessary cooperation between
+ * the schema layer (that maintains the list of checkpoints),
+ * the btree layer (that knows when the root page is written,
+ * creating a new checkpoint), and the block manager (which
+ * actually creates the checkpoint). All of that cooperation is
+ * handled in the array of checkpoint structures referenced from
+ * the WT_BTREE structure.
+ */
+ WT_ERR(__wt_realloc_def(
+ session, &allocated, slot + 2, &ckptbase));
+
+ /* The caller may be adding a value, initialize it. */
+ maxorder = 0;
+ WT_CKPT_FOREACH(ckptbase, ckpt)
+ if (ckpt->order > maxorder)
+ maxorder = ckpt->order;
+ ckpt->order = maxorder + 1;
+ __wt_seconds(session, &ckpt->sec);
+
+ WT_ERR(__wt_meta_block_metadata(session, config, ckpt));
+
+ F_SET(ckpt, WT_CKPT_ADD);
+ }
+
/* Return the array to our caller. */
*ckptbasep = ckptbase;
@@ -303,7 +400,6 @@ __wt_meta_ckptlist_get(
err: __wt_meta_ckptlist_free(session, &ckptbase);
}
__wt_free(session, config);
- __wt_scr_free(session, &buf);
return (ret);
}
@@ -362,7 +458,7 @@ __ckpt_load(WT_SESSION_IMPL *session,
ret = __wt_config_subgets(session, v, "oldest_start_txn", &a);
WT_RET_NOTFOUND_OK(ret);
ckpt->oldest_start_txn =
- ret == WT_NOTFOUND || a.len == 0 ? WT_TS_NONE : (uint64_t)a.val;
+ ret == WT_NOTFOUND || a.len == 0 ? WT_TXN_NONE : (uint64_t)a.val;
ret = __wt_config_subgets(session, v, "newest_stop_ts", &a);
WT_RET_NOTFOUND_OK(ret);
ckpt->newest_stop_ts =
@@ -370,7 +466,7 @@ __ckpt_load(WT_SESSION_IMPL *session,
ret = __wt_config_subgets(session, v, "newest_stop_txn", &a);
WT_RET_NOTFOUND_OK(ret);
ckpt->newest_stop_txn =
- ret == WT_NOTFOUND || a.len == 0 ? WT_TS_MAX : (uint64_t)a.val;
+ ret == WT_NOTFOUND || a.len == 0 ? WT_TXN_MAX : (uint64_t)a.val;
__wt_check_addr_validity(session,
ckpt->oldest_start_ts, ckpt->oldest_start_txn,
ckpt->newest_stop_ts, ckpt->newest_stop_txn);
@@ -443,45 +539,24 @@ __ckptlist_review_write_gen(WT_SESSION_IMPL *session, WT_CKPT *ckpt)
* If checkpointing the metadata file, update its write generation to
* be the maximum we've seen.
*/
- if (WT_IS_METADATA(session->dhandle) && ckpt->write_gen < v)
+ if (session->dhandle != NULL &&
+ WT_IS_METADATA(session->dhandle) && ckpt->write_gen < v)
ckpt->write_gen = v;
}
/*
- * __wt_meta_ckptlist_set --
- * Set a file's checkpoint value from the WT_CKPT list.
+ * __wt_meta_ckptlist_to_meta --
+ * Convert a checkpoint list into its metadata representation.
*/
int
-__wt_meta_ckptlist_set(WT_SESSION_IMPL *session,
- const char *fname, WT_CKPT *ckptbase, WT_LSN *ckptlsn)
+__wt_meta_ckptlist_to_meta(
+ WT_SESSION_IMPL *session, WT_CKPT *ckptbase, WT_ITEM *buf)
{
WT_CKPT *ckpt;
- WT_DECL_ITEM(buf);
- WT_DECL_RET;
- int64_t maxorder;
const char *sep;
- /*
- * Each internal checkpoint name is appended with a generation to make
- * it a unique name. We're solving two problems: when two checkpoints
- * are taken quickly, the timer may not be unique and/or we can even
- * see time travel on the second checkpoint if we snapshot in-between
- * nanoseconds rolling over. Second, if we reset the generational
- * counter when new checkpoints arrive, we could logically re-create
- * specific checkpoints, racing with cursors open on those checkpoints.
- * I can't think of any way to return incorrect results by racing with
- * those cursors, but it's simpler not to worry about it.
- *
- * Determine the current maximum checkpoint generation.
- */
- maxorder = 0;
- WT_CKPT_FOREACH(ckptbase, ckpt)
- if (ckpt->order > maxorder)
- maxorder = ckpt->order;
-
- WT_ERR(__wt_scr_alloc(session, 0, &buf));
sep = "";
- WT_ERR(__wt_buf_fmt(session, buf, "checkpoint=("));
+ WT_RET(__wt_buf_fmt(session, buf, "checkpoint=("));
WT_CKPT_FOREACH(ckptbase, ckpt) {
/* Skip deleted checkpoints. */
if (F_ISSET(ckpt, WT_CKPT_DELETE))
@@ -496,36 +571,27 @@ __wt_meta_ckptlist_set(WT_SESSION_IMPL *session,
if (ckpt->raw.size == 0)
ckpt->addr.size = 0;
else
- WT_ERR(__wt_raw_to_hex(session,
+ WT_RET(__wt_raw_to_hex(session,
ckpt->raw.data,
ckpt->raw.size, &ckpt->addr));
-
- /* Set the order and timestamp. */
- if (F_ISSET(ckpt, WT_CKPT_ADD))
- ckpt->order = ++maxorder;
-
- __wt_seconds(session, &ckpt->sec);
}
__wt_check_addr_validity(session,
ckpt->oldest_start_ts, ckpt->oldest_start_txn,
ckpt->newest_stop_ts, ckpt->newest_stop_txn);
- WT_ERR(__wt_buf_catfmt(session, buf, "%s%s", sep, ckpt->name));
+ WT_RET(__wt_buf_catfmt(session, buf, "%s%s", sep, ckpt->name));
sep = ",";
if (strcmp(ckpt->name, WT_CHECKPOINT) == 0)
- WT_ERR(__wt_buf_catfmt(session, buf,
+ WT_RET(__wt_buf_catfmt(session, buf,
".%" PRId64, ckpt->order));
- /* Review the checkpoint's write generation. */
- __ckptlist_review_write_gen(session, ckpt);
-
/*
* Use PRId64 formats: WiredTiger's configuration code handles
* signed 8B values.
*/
- WT_ERR(__wt_buf_catfmt(session, buf,
+ WT_RET(__wt_buf_catfmt(session, buf,
"=(addr=\"%.*s\",order=%" PRId64
",time=%" PRIu64
",size=%" PRId64
@@ -546,13 +612,38 @@ __wt_meta_ckptlist_set(WT_SESSION_IMPL *session,
(int64_t)ckpt->newest_stop_txn,
(int64_t)ckpt->write_gen));
}
- WT_ERR(__wt_buf_catfmt(session, buf, ")"));
+ WT_RET(__wt_buf_catfmt(session, buf, ")"));
+
+ return (0);
+}
+
+/*
+ * __wt_meta_ckptlist_set --
+ * Set a file's checkpoint value from the WT_CKPT list.
+ */
+int
+__wt_meta_ckptlist_set(WT_SESSION_IMPL *session,
+ const char *fname, WT_CKPT *ckptbase, WT_LSN *ckptlsn)
+{
+ WT_CKPT *ckpt;
+ WT_DECL_ITEM(buf);
+ WT_DECL_RET;
+
+ WT_RET(__wt_scr_alloc(session, 1024, &buf));
+
+ WT_ERR(__wt_meta_ckptlist_to_meta(session, ckptbase, buf));
+
if (ckptlsn != NULL)
WT_ERR(__wt_buf_catfmt(session, buf,
",checkpoint_lsn=(%" PRIu32 ",%" PRIuMAX ")",
ckptlsn->l.file, (uintmax_t)ckptlsn->l.offset));
+
WT_ERR(__ckpt_set(session, fname, buf->mem));
+ /* Review the checkpoint's write generation. */
+ WT_CKPT_FOREACH(ckptbase, ckpt)
+ __ckptlist_review_write_gen(session, ckpt);
+
err: __wt_scr_free(session, &buf);
return (ret);
}
@@ -585,6 +676,8 @@ __wt_meta_checkpoint_free(WT_SESSION_IMPL *session, WT_CKPT *ckpt)
return;
__wt_free(session, ckpt->name);
+ __wt_free(session, ckpt->block_metadata);
+ __wt_free(session, ckpt->block_checkpoint);
__wt_buf_free(session, &ckpt->addr);
__wt_buf_free(session, &ckpt->raw);
__wt_free(session, ckpt->bpriv);
diff --git a/src/third_party/wiredtiger/src/meta/meta_ext.c b/src/third_party/wiredtiger/src/meta/meta_ext.c
index 9404bd016a0..80d0d7c2d5a 100644
--- a/src/third_party/wiredtiger/src/meta/meta_ext.c
+++ b/src/third_party/wiredtiger/src/meta/meta_ext.c
@@ -91,7 +91,7 @@ __wt_metadata_get_ckptlist(
WT_GCC_FUNC_ATTRIBUTE((visibility("default")))
{
return (__wt_meta_ckptlist_get(
- (WT_SESSION_IMPL *)session, name, ckptbasep));
+ (WT_SESSION_IMPL *)session, name, false, ckptbasep));
}
/*
diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c
index 84f07e317ff..51cabeda029 100644
--- a/src/third_party/wiredtiger/src/reconcile/rec_write.c
+++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c
@@ -9,6 +9,7 @@
#include "wt_internal.h"
static void __rec_cleanup(WT_SESSION_IMPL *, WT_RECONCILE *);
+static void __rec_destroy(WT_SESSION_IMPL *, void *);
static int __rec_destroy_session(WT_SESSION_IMPL *);
static int __rec_init(WT_SESSION_IMPL *,
WT_REF *, uint32_t, WT_SALVAGE_COOKIE *, void *);
@@ -513,6 +514,7 @@ __rec_init(WT_SESSION_IMPL *session,
WT_REF *ref, uint32_t flags, WT_SALVAGE_COOKIE *salvage, void *reconcilep)
{
WT_BTREE *btree;
+ WT_DECL_RET;
WT_PAGE *page;
WT_RECONCILE *r;
WT_TXN_GLOBAL *txn_global;
@@ -522,8 +524,6 @@ __rec_init(WT_SESSION_IMPL *session,
if ((r = *(WT_RECONCILE **)reconcilep) == NULL) {
WT_RET(__wt_calloc_one(session, &r));
-
- *(WT_RECONCILE **)reconcilep = r;
session->reconcile_cleanup = __rec_destroy_session;
/* Connect pointers/buffers. */
@@ -575,6 +575,12 @@ __rec_init(WT_SESSION_IMPL *session,
* history, or the stable timestamp hasn't changed since last time this
* page was successfully, skew oldest instead.
*/
+ if (F_ISSET(S2C(session)->cache, WT_CACHE_EVICT_DEBUG_MODE) &&
+ __wt_random(&session->rnd) % 3 == 0)
+ r->las_skew_newest = false;
+ else
+ r->las_skew_newest =
+ LF_ISSET(WT_REC_LOOKASIDE) && LF_ISSET(WT_REC_VISIBLE_ALL);
r->las_skew_newest =
LF_ISSET(WT_REC_LOOKASIDE) && LF_ISSET(WT_REC_VISIBLE_ALL);
if (r->las_skew_newest &&
@@ -675,7 +681,7 @@ __rec_init(WT_SESSION_IMPL *session,
* Sanity check the size: 100 slots is the smallest dictionary we use.
*/
if (btree->dictionary != 0 && btree->dictionary > r->dictionary_slots)
- WT_RET(__wt_rec_dictionary_init(session,
+ WT_ERR(__wt_rec_dictionary_init(session,
r, btree->dictionary < 100 ? 100 : btree->dictionary));
__wt_rec_dictionary_reset(r);
@@ -716,7 +722,20 @@ __rec_init(WT_SESSION_IMPL *session,
r->update_modify_cbt.ref = ref;
r->update_modify_cbt.iface.value_format = btree->value_format;
- return (0);
+ /*
+ * If we allocated the reconciliation structure and there was an error,
+ * clean up. If our caller passed in a structure, they own it.
+ */
+err: if (*(WT_RECONCILE **)reconcilep == NULL) {
+ if (ret == 0)
+ *(WT_RECONCILE **)reconcilep = r;
+ else {
+ __rec_cleanup(session, r);
+ __rec_destroy(session, &r);
+ }
+ }
+
+ return (ret);
}
/*
@@ -2139,6 +2158,7 @@ int
__wt_bulk_wrapup(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk)
{
WT_BTREE *btree;
+ WT_DECL_RET;
WT_PAGE *parent;
WT_RECONCILE *r;
@@ -2155,25 +2175,25 @@ __wt_bulk_wrapup(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk)
break;
case BTREE_COL_VAR:
if (cbulk->rle != 0)
- WT_RET(__wt_bulk_insert_var(session, cbulk, false));
+ WT_ERR(__wt_bulk_insert_var(session, cbulk, false));
break;
case BTREE_ROW:
break;
}
- WT_RET(__wt_rec_split_finish(session, r));
- WT_RET(__rec_write_wrapup(session, r, r->page));
+ WT_ERR(__wt_rec_split_finish(session, r));
+ WT_ERR(__rec_write_wrapup(session, r, r->page));
__rec_write_page_status(session, r);
/* Mark the page's parent and the tree dirty. */
parent = r->ref->home;
- WT_RET(__wt_page_modify_init(session, parent));
+ WT_ERR(__wt_page_modify_init(session, parent));
__wt_page_modify_set(session, parent);
- __rec_cleanup(session, r);
+err: __rec_cleanup(session, r);
__rec_destroy(session, &cbulk->reconcile);
- return (0);
+ return (ret);
}
/*
@@ -2373,11 +2393,11 @@ __rec_write_wrapup(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page)
*/
ref = r->ref;
if (__wt_ref_is_root(ref)) {
- WT_RET(bm->checkpoint(
- bm, session, NULL, btree->ckpt, false));
__wt_checkpoint_tree_reconcile_update(session,
WT_TS_NONE, WT_TS_NONE, WT_TXN_NONE,
WT_TS_MAX, WT_TXN_MAX);
+ WT_RET(bm->checkpoint(
+ bm, session, NULL, btree->ckpt, false));
}
/*
@@ -2419,16 +2439,16 @@ __rec_write_wrapup(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page)
r->multi->disk_image = NULL;
mod->mod_page_las = r->multi->page_las;
} else {
- WT_RET(__wt_bt_write(session, r->wrapup_checkpoint,
- NULL, NULL, NULL,
- true, F_ISSET(r, WT_REC_CHECKPOINT),
- r->wrapup_checkpoint_compressed));
__wt_checkpoint_tree_reconcile_update(session,
r->multi->addr.newest_durable_ts,
r->multi->addr.oldest_start_ts,
r->multi->addr.oldest_start_txn,
r->multi->addr.newest_stop_ts,
r->multi->addr.newest_stop_txn);
+ WT_RET(__wt_bt_write(session, r->wrapup_checkpoint,
+ NULL, NULL, NULL,
+ true, F_ISSET(r, WT_REC_CHECKPOINT),
+ r->wrapup_checkpoint_compressed));
}
mod->rec_result = WT_PM_REC_REPLACE;
diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c
index 7d3b4b9d81a..4e1742a89db 100644
--- a/src/third_party/wiredtiger/src/session/session_api.c
+++ b/src/third_party/wiredtiger/src/session/session_api.c
@@ -1115,6 +1115,65 @@ err: API_END_RET(session, ret);
}
/*
+ * __session_import --
+ * WT_SESSION->import method.
+ */
+static int
+__session_import(WT_SESSION *wt_session, const char *uri, const char *config)
+{
+ WT_DECL_RET;
+ WT_SESSION_IMPL *session;
+ char *value;
+
+ WT_UNUSED(config);
+
+ session = (WT_SESSION_IMPL *)wt_session;
+ SESSION_API_CALL_NOCONF(session, import);
+
+ WT_ERR(__wt_inmem_unsupported_op(session, NULL));
+
+ if (!WT_PREFIX_MATCH(uri, "file:"))
+ WT_ERR(__wt_bad_object_type(session, uri));
+
+ if ((ret = __wt_metadata_search(session, uri, &value)) == 0)
+ WT_ERR_MSG(session, EINVAL,
+ "an object named \"%s\" already exists in the database",
+ uri);
+ WT_ERR_NOTFOUND_OK(ret);
+
+ WT_ERR(__wt_import(session, uri));
+
+err:
+ if (ret != 0)
+ WT_STAT_CONN_INCR(session, session_table_import_fail);
+ else
+ WT_STAT_CONN_INCR(session, session_table_import_success);
+ API_END_RET_NOTFOUND_MAP(session, ret);
+}
+
+/*
+ * __session_import_readonly --
+ * WT_SESSION->import method; readonly version.
+ */
+static int
+__session_import_readonly(
+ WT_SESSION *wt_session, const char *uri, const char *config)
+{
+ WT_DECL_RET;
+ WT_SESSION_IMPL *session;
+
+ WT_UNUSED(uri);
+ WT_UNUSED(config);
+
+ session = (WT_SESSION_IMPL *)wt_session;
+ SESSION_API_CALL_NOCONF(session, import);
+
+ WT_STAT_CONN_INCR(session, session_table_import_fail);
+ ret = __wt_session_notsup(session);
+err: API_END_RET(session, ret);
+}
+
+/*
* __session_join --
* WT_SESSION->join method.
*/
@@ -2092,6 +2151,7 @@ __open_session(WT_CONNECTION_IMPL *conn,
__session_open_cursor,
__session_alter,
__session_create,
+ __session_import,
__wt_session_compact,
__session_drop,
__session_join,
@@ -2124,6 +2184,7 @@ __open_session(WT_CONNECTION_IMPL *conn,
__session_open_cursor,
__session_alter_readonly,
__session_create_readonly,
+ __session_import_readonly,
__wt_session_compact_readonly,
__session_drop_readonly,
__session_join,
diff --git a/src/third_party/wiredtiger/src/session/session_salvage.c b/src/third_party/wiredtiger/src/session/session_salvage.c
deleted file mode 100644
index 79e67475958..00000000000
--- a/src/third_party/wiredtiger/src/session/session_salvage.c
+++ /dev/null
@@ -1,59 +0,0 @@
-/*-
- * Copyright (c) 2014-2019 MongoDB, Inc.
- * Copyright (c) 2008-2014 WiredTiger, Inc.
- * All rights reserved.
- *
- * See the file LICENSE for redistribution information.
- */
-
-#include "wt_internal.h"
-
-/*
- * __wt_salvage --
- * Salvage a single file.
- */
-int
-__wt_salvage(WT_SESSION_IMPL *session, const char *cfg[])
-{
- WT_CKPT *ckptbase;
- WT_DATA_HANDLE *dhandle;
- WT_DECL_RET;
-
- dhandle = session->dhandle;
-
- /*
- * XXX
- * The salvage process reads and discards previous checkpoints, so the
- * underlying block manager has to ignore any previous checkpoint
- * entries when creating a new checkpoint, in other words, we can't use
- * the metadata checkpoint list, it has all of those checkpoint listed
- * and we don't care about them. Build a clean checkpoint list and use
- * it instead.
- *
- * Don't first clear the metadata checkpoint list and call the function
- * to get a list of checkpoints: a crash between clearing the metadata
- * checkpoint list and creating a new checkpoint list would look like a
- * create or open of a file without a checkpoint to roll-forward from,
- * and the contents of the file would be discarded.
- */
- WT_RET(__wt_calloc_def(session, 2, &ckptbase));
- WT_ERR(__wt_strdup(session, WT_CHECKPOINT, &ckptbase[0].name));
- F_SET(&ckptbase[0], WT_CKPT_ADD);
-
- WT_ERR(__wt_bt_salvage(session, ckptbase, cfg));
-
- /*
- * If no checkpoint was created, well, it's probably bad news, but there
- * is nothing to do but clear any recorded checkpoints for the file. If
- * a checkpoint was created, life is good, replace any existing list of
- * checkpoints with the single new one.
- */
- if (ckptbase[0].raw.data == NULL)
- WT_ERR(__wt_meta_checkpoint_clear(session, dhandle->name));
- else
- WT_ERR(__wt_meta_ckptlist_set(
- session, dhandle->name, ckptbase, NULL));
-
-err: __wt_meta_ckptlist_free(session, &ckptbase);
- return (ret);
-}
diff --git a/src/third_party/wiredtiger/src/support/huffman.c b/src/third_party/wiredtiger/src/support/huffman.c
index b7bd9e24bb6..85e0800e170 100644
--- a/src/third_party/wiredtiger/src/support/huffman.c
+++ b/src/third_party/wiredtiger/src/support/huffman.c
@@ -261,8 +261,9 @@ static void
make_table(WT_SESSION_IMPL *session, uint8_t *code2symbol,
uint16_t max_depth, WT_HUFFMAN_CODE *codes, u_int symcnt)
{
+ u_int i;
uint32_t j, c1, c2; /* Exceeds uint16_t bounds at loop boundary. */
- uint16_t c, i;
+ uint16_t c;
uint8_t len, shift;
/* Zero out, for assertion below. */
@@ -321,8 +322,8 @@ __wt_huffman_open(WT_SESSION_IMPL *session,
WT_DECL_RET;
WT_FREQTREE_NODE *node, *node2, **refnode, *tempnode;
WT_HUFFMAN_OBJ *huffman;
+ u_int i;
uint64_t w1, w2;
- uint16_t i;
indexed_freqs = NULL;
combined_nodes = leaves = NULL;
diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c
index 32241675a9e..e76930c3b38 100644
--- a/src/third_party/wiredtiger/src/support/stat.c
+++ b/src/third_party/wiredtiger/src/support/stat.c
@@ -1074,6 +1074,8 @@ static const char * const __stats_connection_desc[] = {
"session: table create successful calls",
"session: table drop failed calls",
"session: table drop successful calls",
+ "session: table import failed calls",
+ "session: table import successful calls",
"session: table rebalance failed calls",
"session: table rebalance successful calls",
"session: table rename failed calls",
@@ -1509,6 +1511,8 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats)
/* not clearing session_table_create_success */
/* not clearing session_table_drop_fail */
/* not clearing session_table_drop_success */
+ /* not clearing session_table_import_fail */
+ /* not clearing session_table_import_success */
/* not clearing session_table_rebalance_fail */
/* not clearing session_table_rebalance_success */
/* not clearing session_table_rename_fail */
@@ -2064,6 +2068,10 @@ __wt_stat_connection_aggregate(
WT_STAT_READ(from, session_table_drop_fail);
to->session_table_drop_success +=
WT_STAT_READ(from, session_table_drop_success);
+ to->session_table_import_fail +=
+ WT_STAT_READ(from, session_table_import_fail);
+ to->session_table_import_success +=
+ WT_STAT_READ(from, session_table_import_success);
to->session_table_rebalance_fail +=
WT_STAT_READ(from, session_table_rebalance_fail);
to->session_table_rebalance_success +=
diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c
index 4056d33a2b7..2449c1ccabe 100644
--- a/src/third_party/wiredtiger/src/txn/txn.c
+++ b/src/third_party/wiredtiger/src/txn/txn.c
@@ -529,9 +529,11 @@ __wt_txn_config(WT_SESSION_IMPL *session, const char *cfg[])
/* Check if prepared updates should be ignored during reads. */
WT_RET(__wt_config_gets_def(session, cfg, "ignore_prepare", 0, &cval));
- if (cval.val ||
- (cval.len > 0 && WT_STRING_MATCH("force", cval.str, cval.len)))
+ if (cval.len > 0 &&
+ WT_STRING_MATCH("force", cval.str, cval.len))
F_SET(txn, WT_TXN_IGNORE_PREPARE);
+ else if (cval.val)
+ F_SET(txn, WT_TXN_IGNORE_PREPARE | WT_TXN_READONLY);
/*
* Check if the prepare timestamp and the commit timestamp of a
@@ -636,9 +638,6 @@ __wt_txn_release(WT_SESSION_IMPL *session)
*/
__wt_txn_release_snapshot(session);
txn->isolation = session->isolation;
-#ifdef HAVE_DIAGNOSTIC
- txn->multi_update_count = 0;
-#endif
txn->rollback_reason = NULL;
@@ -801,6 +800,7 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[])
WT_TXN_OP *op;
WT_UPDATE *upd;
wt_timestamp_t prev_commit_timestamp;
+ int64_t resolved_update_count, visited_update_count;
uint32_t fileid;
u_int i;
bool locked, prepare, readonly, update_timestamp;
@@ -810,6 +810,7 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[])
txn_global = &conn->txn_global;
prev_commit_timestamp = 0; /* -Wconditional-uninitialized */
locked = false;
+ resolved_update_count = visited_update_count = 0;
WT_ASSERT(session, F_ISSET(txn, WT_TXN_RUNNING));
WT_ASSERT(session, !F_ISSET(txn, WT_TXN_ERROR) ||
@@ -978,8 +979,27 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[])
__wt_txn_op_set_timestamp(session, op);
} else {
- WT_ERR(__wt_txn_resolve_prepared_op(
- session, op, true));
+ visited_update_count++;
+ /*
+ * If we have set the key repeated flag
+ * we can skip resolving prepared updates as
+ * it would have happened on a previous
+ * modification in this txn.
+ */
+ if (!F_ISSET(op, WT_TXN_OP_KEY_REPEATED))
+ WT_ERR(__wt_txn_resolve_prepared_op(
+ session, op, true,
+ &resolved_update_count));
+ /*
+ * We should resolve at least one or more
+ * updates each time we call
+ * __wt_txn_resolve_prepared_op, as such
+ * resolved update count should never be less
+ * than visited update count.
+ */
+ WT_ASSERT(session,
+ resolved_update_count >=
+ visited_update_count);
}
break;
@@ -994,12 +1014,10 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[])
__wt_txn_op_free(session, op);
}
+ WT_ASSERT(session, resolved_update_count == visited_update_count);
+ WT_STAT_CONN_INCRV(session, txn_prepared_updates_resolved,
+ resolved_update_count);
- /*
- * FIXME: I think we want to say that all prepared updates were
- * resolved.
- * WT_ASSERT(session, txn->multi_update_count == 0);
- */
txn->mod_count = 0;
/*
@@ -1141,6 +1159,13 @@ __wt_txn_prepare(WT_SESSION_IMPL *session, const char *cfg[])
WT_PUBLISH(upd->prepare_state, WT_PREPARE_INPROGRESS);
op->u.op_upd = NULL;
WT_STAT_CONN_INCR(session, txn_prepared_updates_count);
+ /*
+ * Set the key repeated flag which tells us that we've
+ * got multiple updates to the same key by the same txn.
+ * This is later used in txn commit.
+ */
+ if (upd->next != NULL && upd->txnid == upd->next->txnid)
+ F_SET(op, WT_TXN_OP_KEY_REPEATED);
break;
case WT_TXN_OP_REF_DELETE:
__wt_txn_op_apply_prepare_state(
@@ -1180,11 +1205,12 @@ __wt_txn_rollback(WT_SESSION_IMPL *session, const char *cfg[])
WT_TXN *txn;
WT_TXN_OP *op;
WT_UPDATE *upd;
+ int64_t resolved_update_count, visited_update_count;
u_int i;
bool readonly;
WT_UNUSED(cfg);
-
+ resolved_update_count = visited_update_count = 0;
txn = &session->txn;
readonly = txn->mod_count == 0;
WT_ASSERT(session, F_ISSET(txn, WT_TXN_RUNNING));
@@ -1218,10 +1244,29 @@ __wt_txn_rollback(WT_SESSION_IMPL *session, const char *cfg[])
* Need to resolve indirect references of transaction
* operation, in case of prepared transaction.
*/
- if (F_ISSET(txn, WT_TXN_PREPARE))
- WT_RET(__wt_txn_resolve_prepared_op(
- session, op, false));
- else {
+ if (F_ISSET(txn, WT_TXN_PREPARE)) {
+ visited_update_count++;
+ /*
+ * If we have set the key repeated flag
+ * we can skip resolving prepared updates as
+ * it would have happened on a previous
+ * modification in this txn.
+ */
+ if (!F_ISSET(op, WT_TXN_OP_KEY_REPEATED))
+ WT_RET(__wt_txn_resolve_prepared_op(
+ session, op, false,
+ &resolved_update_count));
+ /*
+ * We should resolve at least one or more
+ * updates each time we call
+ * __wt_txn_resolve_prepared_op, as such
+ * resolved update count should never be less
+ * than visited update count.
+ */
+ WT_ASSERT(session,
+ resolved_update_count >=
+ visited_update_count);
+ } else {
WT_ASSERT(session, upd->txnid == txn->id ||
upd->txnid == WT_TXN_ABORTED);
upd->txnid = WT_TXN_ABORTED;
@@ -1243,6 +1288,10 @@ __wt_txn_rollback(WT_SESSION_IMPL *session, const char *cfg[])
__wt_txn_op_free(session, op);
}
+ WT_ASSERT(session, resolved_update_count == visited_update_count);
+ WT_STAT_CONN_INCRV(session, txn_prepared_updates_resolved,
+ resolved_update_count);
+
txn->mod_count = 0;
__wt_txn_release(session);
diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c
index 143bec4e445..0b4e9394f9f 100644
--- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c
+++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c
@@ -1400,7 +1400,7 @@ __checkpoint_lock_dirty_tree(WT_SESSION_IMPL *session,
WT_IS_METADATA(dhandle) || WT_META_TRACKING(session));
/* Get the list of checkpoints for this file. */
- WT_RET(__wt_meta_ckptlist_get(session, dhandle->name, &ckptbase));
+ WT_RET(__wt_meta_ckptlist_get(session, dhandle->name, true, &ckptbase));
/* This may be a named checkpoint, check the configuration. */
cval.len = 0;
@@ -1449,16 +1449,10 @@ __checkpoint_lock_dirty_tree(WT_SESSION_IMPL *session,
/* Drop checkpoints with the same name as the one we're taking. */
__drop(ckptbase, name, strlen(name));
- /* Add a new checkpoint entry at the end of the list. */
+ /* Set the name of the new entry at the end of the list. */
WT_CKPT_FOREACH(ckptbase, ckpt)
;
WT_ERR(__wt_strdup(session, name, &ckpt->name));
- /*
- * We are now done with the local use of the name. Free the local
- * allocation, if needed.
- */
- __wt_free(session, name_alloc);
- F_SET(ckpt, WT_CKPT_ADD);
/*
* There is some interaction between backups and checkpoints. Perform
@@ -1476,9 +1470,10 @@ __checkpoint_lock_dirty_tree(WT_SESSION_IMPL *session,
!F_ISSET(btree, WT_BTREE_SKIP_CKPT));
btree->ckpt = ckptbase;
- return (0);
-
-err: __wt_meta_ckptlist_free(session, &ckptbase);
+ if (0) {
+err:
+ __wt_meta_ckptlist_free(session, &ckptbase);
+ }
__wt_free(session, name_alloc);
return (ret);
diff --git a/src/third_party/wiredtiger/src/txn/txn_log.c b/src/third_party/wiredtiger/src/txn/txn_log.c
index 6398cea7e4e..b9d9c26d941 100644
--- a/src/third_party/wiredtiger/src/txn/txn_log.c
+++ b/src/third_party/wiredtiger/src/txn/txn_log.c
@@ -38,6 +38,7 @@ __txn_op_log_row_key_check(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt)
if (cbt->ins == NULL) {
session = (WT_SESSION_IMPL *)cbt->iface.session;
page = cbt->ref->page;
+ WT_ASSERT(session, cbt->slot < page->entries);
rip = &page->pg_row[cbt->slot];
WT_ASSERT(session,
__wt_row_leaf_key(session, page, rip, &key, false) == 0);
@@ -183,6 +184,7 @@ __wt_txn_op_free(WT_SESSION_IMPL *session, WT_TXN_OP *op)
(void)__wt_atomic_subi32(&op->btree->dhandle->session_inuse, 1);
op->type = WT_TXN_OP_NONE;
+ op->flags = 0;
}
/*
diff --git a/src/third_party/wiredtiger/src/txn/txn_recover.c b/src/third_party/wiredtiger/src/txn/txn_recover.c
index b68a4415e76..ff4c2b8b9cb 100644
--- a/src/third_party/wiredtiger/src/txn/txn_recover.c
+++ b/src/third_party/wiredtiger/src/txn/txn_recover.c
@@ -457,6 +457,11 @@ __recovery_setup_file(WT_RECOVERY *r, const char *uri, const char *config)
r->nfiles = fileid + 1;
}
+ if (r->files[fileid].uri != NULL)
+ WT_PANIC_RET(r->session, WT_PANIC,
+ "metadata corruption: files %s and %s have the same "
+ "file ID %u",
+ uri, r->files[fileid].uri, fileid);
WT_RET(__wt_strdup(r->session, uri, &r->files[fileid].uri));
WT_RET(
__wt_config_getones(r->session, config, "checkpoint_lsn", &cval));
diff --git a/src/third_party/wiredtiger/src/txn/txn_timestamp.c b/src/third_party/wiredtiger/src/txn/txn_timestamp.c
index 7e26bfe8fe5..6046829def3 100644
--- a/src/third_party/wiredtiger/src/txn/txn_timestamp.c
+++ b/src/third_party/wiredtiger/src/txn/txn_timestamp.c
@@ -928,10 +928,22 @@ __wt_txn_set_read_timestamp(
did_roundup_to_oldest = true;
} else {
__wt_readunlock(session, &txn_global->rwlock);
- WT_RET_MSG(session, EINVAL, "read timestamp "
+
+ /*
+ * In some cases, MongoDB sets a read timestamp older
+ * than the oldest timestamp, relying on WiredTiger's
+ * concurrency to detect and fail the set. In other
+ * cases it's a bug and MongoDB wants error context to
+ * make it easier to find those problems. Don't output
+ * an error message because that logs a MongoDB error,
+ * use an informational message to provide the context
+ * instead.
+ */
+ WT_RET(__wt_msg(session, "read timestamp "
"%s less than the oldest timestamp %s",
__wt_timestamp_to_string(read_ts, ts_string[0]),
- __wt_timestamp_to_string(ts_oldest, ts_string[1]));
+ __wt_timestamp_to_string(ts_oldest, ts_string[1])));
+ return (EINVAL);
}
} else
txn->read_timestamp = read_ts;
diff --git a/src/third_party/wiredtiger/src/utilities/util.h b/src/third_party/wiredtiger/src/utilities/util.h
index be58f8eae0e..3b12d9be98a 100644
--- a/src/third_party/wiredtiger/src/utilities/util.h
+++ b/src/third_party/wiredtiger/src/utilities/util.h
@@ -38,6 +38,7 @@ int util_dump(WT_SESSION *, int, char *[]);
int util_err(WT_SESSION *, int, const char *, ...)
WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4)));
int util_flush(WT_SESSION *, const char *);
+int util_import(WT_SESSION *, int, char *[]);
int util_list(WT_SESSION *, int, char *[]);
int util_load(WT_SESSION *, int, char *[]);
int util_loadtext(WT_SESSION *, int, char *[]);
diff --git a/src/third_party/wiredtiger/src/utilities/util_import.c b/src/third_party/wiredtiger/src/utilities/util_import.c
new file mode 100644
index 00000000000..995a09cfb93
--- /dev/null
+++ b/src/third_party/wiredtiger/src/utilities/util_import.c
@@ -0,0 +1,49 @@
+/*-
+ * Copyright (c) 2014-2019 MongoDB, Inc.
+ * Copyright (c) 2008-2014 WiredTiger, Inc.
+ * All rights reserved.
+ *
+ * See the file LICENSE for redistribution information.
+ */
+
+#include "util.h"
+
+static int
+usage(void)
+{
+ (void)fprintf(stderr,
+ "usage: %s %s "
+ "import uri\n",
+ progname, usage_prefix);
+ return (1);
+}
+
+int
+util_import(WT_SESSION *session, int argc, char *argv[])
+{
+ WT_DECL_RET;
+ int ch;
+ char *uri;
+
+ uri = NULL;
+ while ((ch = __wt_getopt(progname, argc, argv, "")) != EOF)
+ switch (ch) {
+ case '?':
+ default:
+ return (usage());
+ }
+ argc -= __wt_optind;
+ argv += __wt_optind;
+
+ /* The remaining argument is the file URI. */
+ if (argc != 1)
+ return (usage());
+ if ((uri = util_uri(session, *argv, "file")) == NULL)
+ return (1);
+
+ if ((ret = session->import(session, uri, NULL)) != 0)
+ (void)util_err(session, ret, "WT_SESSION.import: %s", uri);
+
+ free(uri);
+ return (ret);
+}
diff --git a/src/third_party/wiredtiger/src/utilities/util_list.c b/src/third_party/wiredtiger/src/utilities/util_list.c
index e34fee663dc..ede0a91f979 100644
--- a/src/third_party/wiredtiger/src/utilities/util_list.c
+++ b/src/third_party/wiredtiger/src/utilities/util_list.c
@@ -71,27 +71,31 @@ list_get_allocsize(WT_SESSION *session, const char *key, size_t *allocsize)
*allocsize = 0;
+ parser = NULL;
+ config = NULL;
+
wt_api = session->connection->get_extension_api(session->connection);
if ((ret = wt_api->metadata_search(wt_api, session, key, &config)) != 0)
- return (util_err(
+ WT_ERR(util_err(
session, ret, "%s: WT_EXTENSION_API.metadata_search", key));
if ((ret = wt_api->config_parser_open(wt_api, session, config,
strlen(config), &parser)) != 0)
- return (util_err(
+ WT_ERR(util_err(
session, ret, "WT_EXTENSION_API.config_parser_open"));
- if ((ret = parser->get(parser, "allocation_size", &szvalue)) != 0) {
- if (ret == WT_NOTFOUND)
- ret = 0;
- else
- ret = util_err(session, ret, "WT_CONFIG_PARSER.get");
- if ((tret = parser->close(parser)) != 0)
- (void)util_err(session, tret, "WT_CONFIG_PARSER.close");
- return (ret);
+ if ((ret = parser->get(parser, "allocation_size", &szvalue)) == 0)
+ *allocsize = (size_t)szvalue.val;
+ else
+ ret = ret == WT_NOTFOUND ?
+ 0 : util_err(session, ret, "WT_CONFIG_PARSER.get");
+err:
+ if (parser != NULL && (tret = parser->close(parser)) != 0) {
+ tret = util_err(session, tret, "WT_CONFIG_PARSER.close");
+ if (ret == 0)
+ ret = tret;
}
- if ((ret = parser->close(parser)) != 0)
- return (util_err(session, ret, "WT_CONFIG_PARSER.close"));
- *allocsize = (size_t)szvalue.val;
- return (0);
+
+ free(config);
+ return (ret);
}
/*
@@ -172,6 +176,27 @@ list_print(WT_SESSION *session, const char *uri, bool cflag, bool vflag)
}
/*
+ * list_print_size --
+ * List a size found in the checkpoint information.
+ */
+static void
+list_print_size(uint64_t v)
+{
+ if (v >= WT_PETABYTE)
+ printf("%" PRIu64 " PB", v / WT_PETABYTE);
+ else if (v >= WT_TERABYTE)
+ printf("%" PRIu64 " TB", v / WT_TERABYTE);
+ else if (v >= WT_GIGABYTE)
+ printf("%" PRIu64 " GB", v / WT_GIGABYTE);
+ else if (v >= WT_MEGABYTE)
+ printf("%" PRIu64 " MB", v / WT_MEGABYTE);
+ else if (v >= WT_KILOBYTE)
+ printf("%" PRIu64 " KB", v / WT_KILOBYTE);
+ else
+ printf("%" PRIu64 " B", v);
+}
+
+/*
* list_print_checkpoint --
* List the checkpoint information.
*/
@@ -183,7 +208,6 @@ list_print_checkpoint(WT_SESSION *session, const char *key)
WT_DECL_RET;
size_t allocsize, len;
time_t t;
- uint64_t v;
/*
* We may not find any checkpoints for this file, in which case we don't
@@ -210,38 +234,55 @@ list_print_checkpoint(WT_SESSION *session, const char *key)
* Call ctime, not ctime_r; ctime_r has portability problems,
* the Solaris version is different from the POSIX standard.
*/
+ if (ckpt != ckptbase)
+ printf("\n");
t = (time_t)ckpt->sec;
printf("\t%*s: %.24s", (int)len, ckpt->name, ctime(&t));
- v = ckpt->size;
- if (v >= WT_PETABYTE)
- printf(" (%" PRIu64 " PB)\n", v / WT_PETABYTE);
- else if (v >= WT_TERABYTE)
- printf(" (%" PRIu64 " TB)\n", v / WT_TERABYTE);
- else if (v >= WT_GIGABYTE)
- printf(" (%" PRIu64 " GB)\n", v / WT_GIGABYTE);
- else if (v >= WT_MEGABYTE)
- printf(" (%" PRIu64 " MB)\n", v / WT_MEGABYTE);
- else if (v >= WT_KILOBYTE)
- printf(" (%" PRIu64 " KB)\n", v / WT_KILOBYTE);
- else
- printf(" (%" PRIu64 " B)\n", v);
+ printf(" (size ");
+ list_print_size(ckpt->size);
+ printf(")\n");
/* Decode the checkpoint block. */
if (ckpt->raw.data == NULL)
continue;
if ((ret = __wt_block_ckpt_decode(
session, allocsize, ckpt->raw.data, &ci)) == 0) {
- printf("\t\t" "root offset: %" PRIuMAX
- " (0x%" PRIxMAX ")\n",
- (uintmax_t)ci.root_offset,
- (uintmax_t)ci.root_offset);
- printf("\t\t" "root size: %" PRIu32
- " (0x%" PRIx32 ")\n",
- ci.root_size, ci.root_size);
- printf("\t\t" "root checksum: %" PRIu32
- " (0x%" PRIx32 ")\n",
+ printf("\t\t" "file-size: ");
+ list_print_size((uint64_t)ci.file_size);
+ printf(", checkpoint-size: ");
+ list_print_size(ci.ckpt_size);
+ printf("\n\n");
+
+ printf("\t\t" " offset, size, checksum\n");
+ printf(
+ "\t\t" "root "
+ ": %" PRIuMAX
+ ", %" PRIu32
+ ", %" PRIu32 " (%#" PRIx32 ")\n",
+ (uintmax_t)ci.root_offset, ci.root_size,
ci.root_checksum, ci.root_checksum);
+ printf(
+ "\t\t" "alloc "
+ ": %" PRIuMAX
+ ", %" PRIu32
+ ", %" PRIu32 " (%#" PRIx32 ")\n",
+ (uintmax_t)ci.alloc.offset, ci.alloc.size,
+ ci.alloc.checksum, ci.alloc.checksum);
+ printf(
+ "\t\t" "discard "
+ ": %" PRIuMAX
+ ", %" PRIu32
+ ", %" PRIu32 " (%#" PRIx32 ")\n",
+ (uintmax_t)ci.discard.offset, ci.discard.size,
+ ci.discard.checksum, ci.discard.checksum);
+ printf(
+ "\t\t" "avail "
+ ": %" PRIuMAX
+ ", %" PRIu32
+ ", %" PRIu32 " (%#" PRIx32 ")\n",
+ (uintmax_t)ci.avail.offset, ci.avail.size,
+ ci.avail.checksum, ci.avail.checksum);
} else {
/* Ignore the error and continue if damaged. */
(void)util_err(session, ret, "__wt_block_ckpt_decode");
diff --git a/src/third_party/wiredtiger/src/utilities/util_main.c b/src/third_party/wiredtiger/src/utilities/util_main.c
index 81ce18c3598..fb2b1990166 100644
--- a/src/third_party/wiredtiger/src/utilities/util_main.c
+++ b/src/third_party/wiredtiger/src/utilities/util_main.c
@@ -47,6 +47,10 @@ usage(void)
"\t" "downgrade downgrade a database\n"
"\t" "drop\t drop an object\n"
"\t" "dump\t dump an object\n"
+ /*
+ * Import is not documented.
+ * "\t" "import\t import an object\n"
+ */
"\t" "list\t list database objects\n"
"\t" "load\t load an object\n"
"\t" "loadtext load an object from a text file\n"
@@ -193,6 +197,10 @@ main(int argc, char *argv[])
else if (strcmp(command, "dump") == 0)
func = util_dump;
break;
+ case 'i':
+ if (strcmp(command, "import") == 0)
+ func = util_import;
+ break;
case 'l':
if (strcmp(command, "list") == 0)
func = util_list;
diff --git a/src/third_party/wiredtiger/test/checkpoint/smoke.sh b/src/third_party/wiredtiger/test/checkpoint/smoke.sh
index dba60babb92..e1bc3819c74 100755
--- a/src/third_party/wiredtiger/test/checkpoint/smoke.sh
+++ b/src/third_party/wiredtiger/test/checkpoint/smoke.sh
@@ -25,6 +25,9 @@ echo "checkpoint: 6 row-store tables, named checkpoint"
$TEST_WRAPPER ./t -c 'TeSt' -T 6 -t r
echo "checkpoint: row-store tables, stress LAS. Sweep and timestamps"
+$TEST_WRAPPER ./t -t r -W 3 -r 2 -D -s -x -n 100000 -k 100000 -C cache_size=100MB
+
+echo "checkpoint: row-store tables, Sweep and timestamps"
$TEST_WRAPPER ./t -t r -W 3 -r 2 -s -x -n 100000 -k 100000 -C cache_size=100MB
echo "checkpoint: 3 mixed tables, with sweep"
diff --git a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c
index 461b6334b27..c30af666c5c 100644
--- a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c
+++ b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c
@@ -57,6 +57,7 @@ main(int argc, char *argv[])
working_dir = NULL;
ttype = MIX;
g.checkpoint_name = "WiredTigerCheckpoint";
+ g.debug_mode = false;
g.home = dmalloc(512);
g.nkeys = 10000;
g.nops = 100000;
@@ -66,7 +67,7 @@ main(int argc, char *argv[])
runs = 1;
while ((ch = __wt_getopt(
- progname, argc, argv, "C:c:h:k:l:n:r:sT:t:W:x")) != EOF)
+ progname, argc, argv, "C:c:Dh:k:l:n:r:sT:t:W:x")) != EOF)
switch (ch) {
case 'c':
g.checkpoint_name = __wt_optarg;
@@ -74,6 +75,9 @@ main(int argc, char *argv[])
case 'C': /* wiredtiger_open config */
config_open = __wt_optarg;
break;
+ case 'D':
+ g.debug_mode = true;
+ break;
case 'h': /* wiredtiger_open config */
working_dir = __wt_optarg;
break;
@@ -182,6 +186,8 @@ main(int argc, char *argv[])
return (g.status);
}
+#define DEBUG_MODE_CFG \
+",debug_mode=(eviction=true,table_logging=true)"
/*
* wt_connect --
* Configure the WiredTiger connection.
@@ -208,16 +214,18 @@ wt_connect(const char *config_open)
"statistics_log=(json,wait=1),error_prefix=\"%s\"," \
"file_manager=(close_handle_minimum=1,close_idle_time=1,"\
"close_scan_interval=1),log=(enabled),cache_size=1GB,"\
- "timing_stress_for_test=(aggressive_sweep)%s%s",
+ "timing_stress_for_test=(aggressive_sweep)%s%s%s",
progname,
+ g.debug_mode ? DEBUG_MODE_CFG : "",
config_open == NULL ? "" : ",",
config_open == NULL ? "" : config_open));
else
testutil_check(__wt_snprintf(config, sizeof(config),
"create,cache_cursors=false,statistics=(fast)," \
"statistics_log=(json,wait=1),error_prefix=\"%s\"" \
- "%s%s",
+ "%s%s%s",
progname,
+ g.debug_mode ? DEBUG_MODE_CFG : "",
config_open == NULL ? "" : ",",
config_open == NULL ? "" : config_open));
diff --git a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h
index b579f5cf9b9..3d4375cd137 100644
--- a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h
+++ b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h
@@ -55,6 +55,7 @@ typedef struct {
char *home; /* Home directory */
const char *checkpoint_name; /* Checkpoint name */
WT_CONNECTION *conn; /* WiredTiger connection */
+ bool debug_mode; /* Lookaside stress test */
u_int nkeys; /* Keys to load */
u_int nops; /* Operations per thread */
FILE *logfp; /* Message log file. */
diff --git a/src/third_party/wiredtiger/test/csuite/Makefile.am b/src/third_party/wiredtiger/test/csuite/Makefile.am
index 8b29565b596..f737bc97bed 100644
--- a/src/third_party/wiredtiger/test/csuite/Makefile.am
+++ b/src/third_party/wiredtiger/test/csuite/Makefile.am
@@ -7,6 +7,9 @@ AM_LDFLAGS = -static
all_TESTS=
noinst_PROGRAMS=
+# The import test is only a shell script
+all_TESTS += import/smoke.sh
+
test_random_abort_SOURCES = random_abort/main.c
noinst_PROGRAMS += test_random_abort
all_TESTS += random_abort/smoke.sh
@@ -137,7 +140,7 @@ all_TESTS += test_wt4803_cache_overflow_abort
# Run this during a "make check" smoke test.
TESTS = $(all_TESTS)
-LOG_COMPILER = $(TEST_WRAPPER)
+LOG_COMPILER = env top_builddir=$(top_builddir) top_srcdir=$(top_srcdir) $(TEST_WRAPPER)
clean-local:
rm -rf WT_TEST.* core.* *.core
diff --git a/src/third_party/wiredtiger/test/csuite/import/smoke.sh b/src/third_party/wiredtiger/test/csuite/import/smoke.sh
new file mode 100755
index 00000000000..624e3b26336
--- /dev/null
+++ b/src/third_party/wiredtiger/test/csuite/import/smoke.sh
@@ -0,0 +1,115 @@
+#! /bin/sh
+
+set -e
+
+# Bypass this test for slow machines, valgrind
+test "$TESTUTIL_SLOW_MACHINE" = "1" && exit 0
+test "$TESTUTIL_BYPASS_VALGRIND" = "1" && exit 0
+
+# If $top_builddir/$top_srcdir aren't set, default to building in build_posix
+# and running in test/csuite.
+top_builddir=${top_builddir:-../../build_posix}
+top_srcdir=${top_srcdir:-../..}
+
+dir=WT_TEST.import
+
+rundir=$dir/RUNDIR
+foreign=$dir/FOREIGN
+
+mo=$dir/metadata.orig
+mi=$dir/metadata.import
+co=$dir/ckpt.orig
+ci=$dir/ckpt.import
+
+EXT="extensions=[\
+$top_builddir/ext/encryptors/rotn/.libs/libwiredtiger_rotn.so,\
+$top_builddir/ext/collators/reverse/.libs/libwiredtiger_reverse_collator.so]"
+
+wt="$top_builddir/wt"
+
+# Run test/format to create an object.
+format()
+{
+ rm -rf $rundir
+
+ $top_builddir/test/format/t \
+ -1q \
+ -C "$EXT" \
+ -c $top_srcdir/test/format/CONFIG.stress \
+ -h $rundir \
+ backups=0 \
+ checkpoints=1 \
+ data_source=file \
+ ops=0 \
+ rebalance=0 \
+ salvage=0 \
+ threads=4 \
+ timer=2 \
+ verify=1
+}
+
+import()
+{
+ # Update the extensions if the run included encryption.
+ egrep 'encryption=none' $rundir/CONFIG > /dev/null ||
+ EXT="encryption=(name=rotn,keyid=7),$EXT"
+
+ # Dump the original metadata.
+ echo; echo 'dumping the original metadata'
+ $wt -C "$EXT" -h $rundir list -cv file:wt
+ $wt -C "$EXT" -h $rundir list -v file:wt | sed 1d > $mo
+
+ # Create a stub datbase and copy in the table.
+ rm -rf $foreign && mkdir $foreign
+ $wt -C "$EXT" -h $foreign create file:xxx
+ cp $rundir/wt $foreign/yyy
+
+ # Import the table.
+ $wt -C "$EXT" -h $foreign import file:yyy
+
+ # Dump the imported metadata.
+ echo; echo 'dumping the imported metadata'
+ $wt -C "$EXT" -h $foreign list -cv file:yyy
+ $wt -C "$EXT" -h $foreign list -v file:yyy | sed 1d > $mi
+}
+
+compare_checkpoints()
+{
+ sed -e 's/.*\(checkpoint=.*))\).*/\1/' < $mo > $co
+ sed -e 's/.*\(checkpoint=.*))\).*/\1/' < $mi > $ci
+ echo; echo 'original checkpoint'
+ cat $co
+ echo; echo 'imported checkpoint'
+ cat $ci
+
+ echo; echo 'comparing the original and imported checkpoints'
+ cmp $co $ci && echo 'comparison succeeded'
+}
+
+verify()
+{
+ echo; echo 'verifying the imported file'
+ $wt -C "$EXT" -h $foreign verify file:yyy && echo 'verify succeeded'
+}
+
+# If verify fails, you can repeatedly run the import, checkpoint comparison and
+# verify process using the -r option for debugging.
+readonly=0
+while :
+ do case "$1" in
+ -r)
+ readonly=1
+ shift;;
+ *)
+ break;;
+ esac
+done
+
+if test $readonly -eq 0; then
+ rm -rf $dir && mkdir $dir
+ format
+fi
+import
+compare_checkpoints
+verify
+exit 0
diff --git a/src/third_party/wiredtiger/test/csuite/random_abort/smoke.sh b/src/third_party/wiredtiger/test/csuite/random_abort/smoke.sh
index e36ad62dd8b..713b000b4f1 100755
--- a/src/third_party/wiredtiger/test/csuite/random_abort/smoke.sh
+++ b/src/third_party/wiredtiger/test/csuite/random_abort/smoke.sh
@@ -4,7 +4,12 @@ set -e
# Smoke-test random-abort as part of running "make check".
-$TEST_WRAPPER ./test_random_abort -t 10 -T 5
-$TEST_WRAPPER ./test_random_abort -m -t 10 -T 5
-$TEST_WRAPPER ./test_random_abort -C -t 10 -T 5
-$TEST_WRAPPER ./test_random_abort -C -m -t 10 -T 5
+# If $top_builddir/$top_srcdir aren't set, default to building in build_posix
+# and running in test/csuite.
+top_builddir=${top_builddir:-../../build_posix}
+top_srcdir=${top_srcdir:-../..}
+
+$TEST_WRAPPER $top_builddir/test/csuite/test_random_abort -t 10 -T 5
+$TEST_WRAPPER $top_builddir/test/csuite/test_random_abort -m -t 10 -T 5
+$TEST_WRAPPER $top_builddir/test/csuite/test_random_abort -C -t 10 -T 5
+$TEST_WRAPPER $top_builddir/test/csuite/test_random_abort -C -m -t 10 -T 5
diff --git a/src/third_party/wiredtiger/test/csuite/random_directio/smoke.sh b/src/third_party/wiredtiger/test/csuite/random_directio/smoke.sh
index f871ff4684d..815b8a21c9f 100755
--- a/src/third_party/wiredtiger/test/csuite/random_directio/smoke.sh
+++ b/src/third_party/wiredtiger/test/csuite/random_directio/smoke.sh
@@ -4,7 +4,12 @@ set -e
# Smoke-test random_directio as part of running "make check".
-RUN_TEST_CMD="$TEST_WRAPPER ./test_random_directio"
+# If $top_builddir/$top_srcdir aren't set, default to building in build_posix
+# and running in test/csuite.
+top_builddir=${top_builddir:-../../build_posix}
+top_srcdir=${top_srcdir:-../..}
+
+RUN_TEST_CMD="$TEST_WRAPPER $top_builddir/test/csuite/test_random_directio"
# Replace for more complete testing
#TEST_THREADS="1 5 10"
diff --git a/src/third_party/wiredtiger/test/csuite/schema_abort/smoke.sh b/src/third_party/wiredtiger/test/csuite/schema_abort/smoke.sh
index 41d702d40cd..2e52b6f438c 100755
--- a/src/third_party/wiredtiger/test/csuite/schema_abort/smoke.sh
+++ b/src/third_party/wiredtiger/test/csuite/schema_abort/smoke.sh
@@ -4,9 +4,14 @@ set -e
# Smoke-test schema-abort as part of running "make check".
-$TEST_WRAPPER ./test_schema_abort -t 10 -T 5
-$TEST_WRAPPER ./test_schema_abort -m -t 10 -T 5
-$TEST_WRAPPER ./test_schema_abort -C -t 10 -T 5
-$TEST_WRAPPER ./test_schema_abort -C -m -t 10 -T 5
-$TEST_WRAPPER ./test_schema_abort -m -t 10 -T 5 -z
-$TEST_WRAPPER ./test_schema_abort -m -t 10 -T 5 -x
+# If $top_builddir/$top_srcdir aren't set, default to building in build_posix
+# and running in test/csuite.
+top_builddir=${top_builddir:-../../build_posix}
+top_srcdir=${top_srcdir:-../..}
+
+$TEST_WRAPPER $top_builddir/test/csuite/test_schema_abort -t 10 -T 5
+$TEST_WRAPPER $top_builddir/test/csuite/test_schema_abort -m -t 10 -T 5
+$TEST_WRAPPER $top_builddir/test/csuite/test_schema_abort -C -t 10 -T 5
+$TEST_WRAPPER $top_builddir/test/csuite/test_schema_abort -C -m -t 10 -T 5
+$TEST_WRAPPER $top_builddir/test/csuite/test_schema_abort -m -t 10 -T 5 -z
+$TEST_WRAPPER $top_builddir/test/csuite/test_schema_abort -m -t 10 -T 5 -x
diff --git a/src/third_party/wiredtiger/test/csuite/timestamp_abort/smoke.sh b/src/third_party/wiredtiger/test/csuite/timestamp_abort/smoke.sh
index 661261eb1bb..324e981dd70 100755
--- a/src/third_party/wiredtiger/test/csuite/timestamp_abort/smoke.sh
+++ b/src/third_party/wiredtiger/test/csuite/timestamp_abort/smoke.sh
@@ -4,9 +4,14 @@ set -e
# Smoke-test timestamp-abort as part of running "make check".
-$TEST_WRAPPER ./test_timestamp_abort -t 10 -T 5
-#$TEST_WRAPPER ./test_timestamp_abort -t 10 -T 5 -L
-$TEST_WRAPPER ./test_timestamp_abort -m -t 10 -T 5
-#$TEST_WRAPPER ./test_timestamp_abort -m -t 10 -T 5 -L
-$TEST_WRAPPER ./test_timestamp_abort -C -t 10 -T 5
-$TEST_WRAPPER ./test_timestamp_abort -C -m -t 10 -T 5
+# If $top_builddir/$top_srcdir aren't set, default to building in build_posix
+# and running in test/csuite.
+top_builddir=${top_builddir:-../../build_posix}
+top_srcdir=${top_srcdir:-../..}
+
+ $TEST_WRAPPER $top_builddir/test/csuite/test_timestamp_abort -t 10 -T 5
+#$TEST_WRAPPER $top_builddir/test/csuite/test_timestamp_abort -t 10 -T 5 -L
+ $TEST_WRAPPER $top_builddir/test/csuite/test_timestamp_abort -m -t 10 -T 5
+#$TEST_WRAPPER $top_builddir/test/csuite/test_timestamp_abort -m -t 10 -T 5 -L
+ $TEST_WRAPPER $top_builddir/test/csuite/test_timestamp_abort -C -t 10 -T 5
+ $TEST_WRAPPER $top_builddir/test/csuite/test_timestamp_abort -C -m -t 10 -T 5
diff --git a/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c b/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c
index bec69105679..54b56d597a4 100644
--- a/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c
+++ b/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c
@@ -102,8 +102,6 @@ static void subtest_populate(TEST_OPTS *, bool);
extern int __wt_optind;
-#define WT_FAIL_FS_LIB "../../ext/test/fail_fs/.libs/libwiredtiger_fail_fs.so"
-
/*
* check_results --
* Check all the tables and verify the results.
@@ -318,7 +316,7 @@ run_check_subtest(TEST_OPTS *opts, const char *debugger, uint64_t nops,
subtest_args[narg++] = (char *)"--";
}
- subtest_args[narg++] = (char *)opts->progname;
+ subtest_args[narg++] = (char *)opts->argv0;
/* "subtest" must appear before arguments */
if (close_test)
subtest_args[narg++] = (char *)"subtest_close";
@@ -342,7 +340,7 @@ run_check_subtest(TEST_OPTS *opts, const char *debugger, uint64_t nops,
" operations until fail...\n", nops);
testutil_clean_work_dir(opts->home);
testutil_check(run_process(
- opts, debugger != NULL ? debugger : opts->progname,
+ opts, debugger != NULL ? debugger : opts->argv0,
subtest_args, &estatus));
if (opts->verbose)
printf("process exited %d\n", estatus);
@@ -524,6 +522,7 @@ subtest_main(int argc, char *argv[], bool close_test)
TEST_OPTS *opts, _opts;
WT_SESSION *session;
char config[1024], filename[1024];
+ const char *p;
opts = &_opts;
memset(opts, 0, sizeof(*opts));
@@ -541,17 +540,25 @@ subtest_main(int argc, char *argv[], bool close_test)
testutil_check(__wt_snprintf(
filename, sizeof(filename), "%s/%s", opts->home, STDOUT_FILE));
testutil_assert(freopen(filename, "a", stdout) != NULL);
+
+ /*
+ * Use $top_builddir if it's available, otherwise assume we're building
+ * in build_posix and running in the test/csuite directory.
+ */
+#define WT_FAIL_FS_LIB "ext/test/fail_fs/.libs/libwiredtiger_fail_fs.so"
+ if ((p = getenv("top_builddir")) == NULL)
+ p = "../../build_posix";
testutil_check(__wt_snprintf(config, sizeof(config),
"create,cache_size=250M,log=(enabled),"
- "transaction_sync=(enabled,method=none),extensions=("
- WT_FAIL_FS_LIB
- "=(early_load,config={environment=true,verbose=true})]"));
-
+ "transaction_sync=(enabled,method=none),"
+ "extensions=(%s/%s="
+ "(early_load,config={environment=true,verbose=true}))",
+ p, WT_FAIL_FS_LIB));
testutil_check(
wiredtiger_open(opts->home, &event_handler, config, &opts->conn));
+
testutil_check(
opts->conn->open_session(opts->conn, NULL, NULL, &session));
-
testutil_check(session->create(session, "table:subtest",
"key_format=i,value_format=iiiS,"
"columns=(id,v0,v1,v2,big)"));
diff --git a/src/third_party/wiredtiger/test/csuite/wt3120_filesys/main.c b/src/third_party/wiredtiger/test/csuite/wt3120_filesys/main.c
index aa8d8a4e521..148d0062ddd 100644
--- a/src/third_party/wiredtiger/test/csuite/wt3120_filesys/main.c
+++ b/src/third_party/wiredtiger/test/csuite/wt3120_filesys/main.c
@@ -36,24 +36,29 @@
* than that.
*/
-#define WT_FAIL_FS_LIB "../../ext/test/fail_fs/.libs/libwiredtiger_fail_fs.so"
-
int
main(int argc, char *argv[])
{
TEST_OPTS *opts, _opts;
WT_CURSOR *cursor;
WT_SESSION *session;
- char buf[1024];
- char *kstr, *vstr;
+ char *kstr, *vstr, buf[1024];
+ const char *p;
opts = &_opts;
memset(opts, 0, sizeof(*opts));
testutil_check(testutil_parse_opts(argc, argv, opts));
testutil_make_work_dir(opts->home);
+ /*
+ * Use $top_builddir if it's available, otherwise assume we're building
+ * in build_posix and running in the test/csuite directory.
+ */
+#define WT_FAIL_FS_LIB "ext/test/fail_fs/.libs/libwiredtiger_fail_fs.so"
+ if ((p = getenv("top_builddir")) == NULL)
+ p = "../../build_posix";
testutil_check(__wt_snprintf(buf, sizeof(buf),
- "create,extensions=(" WT_FAIL_FS_LIB "=(early_load=true))"));
+ "create,extensions=(%s/%s=(early_load=true))", p, WT_FAIL_FS_LIB));
testutil_check(wiredtiger_open(opts->home, NULL, buf, &opts->conn));
testutil_check(
opts->conn->open_session(opts->conn, NULL, NULL, &session));
diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml
index 99933cff595..e9ae4d1e37b 100644
--- a/src/third_party/wiredtiger/test/evergreen.yml
+++ b/src/third_party/wiredtiger/test/evergreen.yml
@@ -364,6 +364,21 @@ tasks:
# Start of csuite test tasks
+ - name: csuite-import-test
+ depends_on:
+ - name: compile
+ commands:
+ - func: "fetch artifacts"
+ - func: "compile wiredtiger"
+ - command: shell.exec
+ params:
+ working_dir: "wiredtiger/build_posix"
+ script: |
+ set -o errexit
+ set -o verbose
+
+ ${test_env_vars|} $(pwd)/../test/csuite/import/smoke.sh 2>&1
+
- name: csuite-random-abort-test
depends_on:
- name: compile
@@ -371,12 +386,12 @@ tasks:
- func: "fetch artifacts"
- command: shell.exec
params:
- working_dir: "wiredtiger/build_posix/test/csuite"
+ working_dir: "wiredtiger/build_posix"
script: |
set -o errexit
set -o verbose
- ${test_env_vars|} $(pwd)/../../../test/csuite/random_abort/smoke.sh 2>&1
+ ${test_env_vars|} $(pwd)/../test/csuite/random_abort/smoke.sh 2>&1
- name: csuite-random-directio-test
depends_on:
@@ -385,12 +400,12 @@ tasks:
- func: "fetch artifacts"
- command: shell.exec
params:
- working_dir: "wiredtiger/build_posix/test/csuite"
+ working_dir: "wiredtiger/build_posix"
script: |
set -o errexit
set -o verbose
- ${test_env_vars|} $(pwd)/../../../test/csuite/random_directio/smoke.sh 2>&1
+ ${test_env_vars|} $(pwd)/../test/csuite/random_directio/smoke.sh 2>&1
- name: csuite-schema-abort-test
depends_on:
@@ -399,12 +414,12 @@ tasks:
- func: "fetch artifacts"
- command: shell.exec
params:
- working_dir: "wiredtiger/build_posix/test/csuite"
+ working_dir: "wiredtiger/build_posix"
script: |
set -o errexit
set -o verbose
- ${test_env_vars|} $(pwd)/../../../test/csuite/schema_abort/smoke.sh 2>&1
+ ${test_env_vars|} $(pwd)/../test/csuite/schema_abort/smoke.sh 2>&1
- name: csuite-timestamp-abort-test
depends_on:
@@ -413,12 +428,12 @@ tasks:
- func: "fetch artifacts"
- command: shell.exec
params:
- working_dir: "wiredtiger/build_posix/test/csuite"
+ working_dir: "wiredtiger/build_posix"
script: |
set -o errexit
set -o verbose
- ${test_env_vars|} $(pwd)/../../../test/csuite/timestamp_abort/smoke.sh 2>&1
+ ${test_env_vars|} $(pwd)/../test/csuite/timestamp_abort/smoke.sh 2>&1
- name: csuite-scope-test
depends_on:
@@ -427,12 +442,12 @@ tasks:
- func: "fetch artifacts"
- command: shell.exec
params:
- working_dir: "wiredtiger/build_posix/test/csuite"
+ working_dir: "wiredtiger/build_posix"
script: |
set -o errexit
set -o verbose
- ${test_env_vars|} $(pwd)/test_scope 2>&1
+ ${test_env_vars|} $(pwd)/test/csuite/test_scope 2>&1
- name: csuite-truncated-log-test
depends_on:
@@ -441,12 +456,12 @@ tasks:
- func: "fetch artifacts"
- command: shell.exec
params:
- working_dir: "wiredtiger/build_posix/test/csuite"
+ working_dir: "wiredtiger/build_posix"
script: |
set -o errexit
set -o verbose
- ${test_env_vars|} $(pwd)/test_truncated_log 2>&1
+ ${test_env_vars|} $(pwd)/test/csuite/test_truncated_log 2>&1
- name: csuite-wt1965-col-efficiency-test
depends_on:
@@ -455,12 +470,12 @@ tasks:
- func: "fetch artifacts"
- command: shell.exec
params:
- working_dir: "wiredtiger/build_posix/test/csuite"
+ working_dir: "wiredtiger/build_posix"
script: |
set -o errexit
set -o verbose
- ${test_env_vars|} $(pwd)/test_wt1965_col_efficiency 2>&1
+ ${test_env_vars|} $(pwd)/test/csuite/test_wt1965_col_efficiency 2>&1
- name: csuite-wt2403-lsm-workload-test
depends_on:
@@ -469,12 +484,12 @@ tasks:
- func: "fetch artifacts"
- command: shell.exec
params:
- working_dir: "wiredtiger/build_posix/test/csuite"
+ working_dir: "wiredtiger/build_posix"
script: |
set -o errexit
set -o verbose
- ${test_env_vars|} $(pwd)/test_wt2403_lsm_workload 2>&1
+ ${test_env_vars|} $(pwd)/test/csuite/test_wt2403_lsm_workload 2>&1
- name: csuite-wt2447-join-main-table-test
depends_on:
@@ -483,12 +498,12 @@ tasks:
- func: "fetch artifacts"
- command: shell.exec
params:
- working_dir: "wiredtiger/build_posix/test/csuite"
+ working_dir: "wiredtiger/build_posix"
script: |
set -o errexit
set -o verbose
- ${test_env_vars|} $(pwd)/test_wt2447_join_main_table 2>&1
+ ${test_env_vars|} $(pwd)/test/csuite/test_wt2447_join_main_table 2>&1
- name: csuite-wt2695-checksum-test
depends_on:
@@ -497,12 +512,12 @@ tasks:
- func: "fetch artifacts"
- command: shell.exec
params:
- working_dir: "wiredtiger/build_posix/test/csuite"
+ working_dir: "wiredtiger/build_posix"
script: |
set -o errexit
set -o verbose
- ${test_env_vars|} $(pwd)/test_wt2695_checksum 2>&1
+ ${test_env_vars|} $(pwd)/test/csuite/test_wt2695_checksum 2>&1
- name: csuite-wt2592-join-schema-test
depends_on:
@@ -511,12 +526,12 @@ tasks:
- func: "fetch artifacts"
- command: shell.exec
params:
- working_dir: "wiredtiger/build_posix/test/csuite"
+ working_dir: "wiredtiger/build_posix"
script: |
set -o errexit
set -o verbose
- ${test_env_vars|} $(pwd)/test_wt2592_join_schema 2>&1
+ ${test_env_vars|} $(pwd)/test/csuite/test_wt2592_join_schema 2>&1
- name: csuite-wt2719-reconfig-test
depends_on:
@@ -525,12 +540,12 @@ tasks:
- func: "fetch artifacts"
- command: shell.exec
params:
- working_dir: "wiredtiger/build_posix/test/csuite"
+ working_dir: "wiredtiger/build_posix"
script: |
set -o errexit
set -o verbose
- ${test_env_vars|} $(pwd)/test_wt2719_reconfig 2>&1
+ ${test_env_vars|} $(pwd)/test/csuite/test_wt2719_reconfig 2>&1
- name: csuite-wt2999-join-extractor-test
depends_on:
@@ -539,12 +554,12 @@ tasks:
- func: "fetch artifacts"
- command: shell.exec
params:
- working_dir: "wiredtiger/build_posix/test/csuite"
+ working_dir: "wiredtiger/build_posix"
script: |
set -o errexit
set -o verbose
- ${test_env_vars|} $(pwd)/test_wt2999_join_extractor 2>&1
+ ${test_env_vars|} $(pwd)/test/csuite/test_wt2999_join_extractor 2>&1
- name: csuite-wt3120-filesys-test
depends_on:
@@ -553,12 +568,12 @@ tasks:
- func: "fetch artifacts"
- command: shell.exec
params:
- working_dir: "wiredtiger/build_posix/test/csuite"
+ working_dir: "wiredtiger/build_posix"
script: |
set -o errexit
set -o verbose
- ${test_env_vars|} $(pwd)/test_wt3120_filesys 2>&1
+ ${test_env_vars|} $(pwd)/test/csuite/test_wt3120_filesys 2>&1
- name: csuite-wt3135-search-near-collator-test
depends_on:
@@ -567,12 +582,12 @@ tasks:
- func: "fetch artifacts"
- command: shell.exec
params:
- working_dir: "wiredtiger/build_posix/test/csuite"
+ working_dir: "wiredtiger/build_posix"
script: |
set -o errexit
set -o verbose
- ${test_env_vars|} $(pwd)/test_wt3135_search_near_collator 2>&1
+ ${test_env_vars|} $(pwd)/test/csuite/test_wt3135_search_near_collator 2>&1
- name: csuite-wt3184-dup-index-collator-test
depends_on:
@@ -581,12 +596,12 @@ tasks:
- func: "fetch artifacts"
- command: shell.exec
params:
- working_dir: "wiredtiger/build_posix/test/csuite"
+ working_dir: "wiredtiger/build_posix"
script: |
set -o errexit
set -o verbose
- ${test_env_vars|} $(pwd)/test_wt3184_dup_index_collator 2>&1
+ ${test_env_vars|} $(pwd)/test/csuite/test_wt3184_dup_index_collator 2>&1
- name: csuite-wt3363-checkpoint-op-races-test
depends_on:
@@ -595,12 +610,12 @@ tasks:
- func: "fetch artifacts"
- command: shell.exec
params:
- working_dir: "wiredtiger/build_posix/test/csuite"
+ working_dir: "wiredtiger/build_posix"
script: |
set -o errexit
set -o verbose
- ${test_env_vars|} $(pwd)/test_wt3363_checkpoint_op_races 2>&1
+ ${test_env_vars|} $(pwd)/test/csuite/test_wt3363_checkpoint_op_races 2>&1
- name: csuite-wt3874-pad-byte-collator-test
depends_on:
@@ -609,12 +624,12 @@ tasks:
- func: "fetch artifacts"
- command: shell.exec
params:
- working_dir: "wiredtiger/build_posix/test/csuite"
+ working_dir: "wiredtiger/build_posix"
script: |
set -o errexit
set -o verbose
- ${test_env_vars|} $(pwd)/test_wt3874_pad_byte_collator 2>&1
+ ${test_env_vars|} $(pwd)/test/csuite/test_wt3874_pad_byte_collator 2>&1
- name: csuite-wt4105-large-doc-small-upd-test
depends_on:
@@ -623,12 +638,12 @@ tasks:
- func: "fetch artifacts"
- command: shell.exec
params:
- working_dir: "wiredtiger/build_posix/test/csuite"
+ working_dir: "wiredtiger/build_posix"
script: |
set -o errexit
set -o verbose
- ${test_env_vars|} $(pwd)/test_wt4105_large_doc_small_upd 2>&1
+ ${test_env_vars|} $(pwd)/test/csuite/test_wt4105_large_doc_small_upd 2>&1
- name: csuite-wt4117-checksum-test
depends_on:
@@ -637,12 +652,12 @@ tasks:
- func: "fetch artifacts"
- command: shell.exec
params:
- working_dir: "wiredtiger/build_posix/test/csuite"
+ working_dir: "wiredtiger/build_posix"
script: |
set -o errexit
set -o verbose
- ${test_env_vars|} $(pwd)/test_wt4117_checksum 2>&1
+ ${test_env_vars|} $(pwd)/test/csuite/test_wt4117_checksum 2>&1
- name: csuite-wt4156-metadata-salvage-test
depends_on:
@@ -651,12 +666,12 @@ tasks:
- func: "fetch artifacts"
- command: shell.exec
params:
- working_dir: "wiredtiger/build_posix/test/csuite"
+ working_dir: "wiredtiger/build_posix"
script: |
set -o errexit
set -o verbose
- ${test_env_vars|} $(pwd)/test_wt4156_metadata_salvage 2>&1
+ ${test_env_vars|} $(pwd)/test/csuite/test_wt4156_metadata_salvage 2>&1
- name: csuite-wt4699-json-test
depends_on:
@@ -665,12 +680,12 @@ tasks:
- func: "fetch artifacts"
- command: shell.exec
params:
- working_dir: "wiredtiger/build_posix/test/csuite"
+ working_dir: "wiredtiger/build_posix"
script: |
set -o errexit
set -o verbose
- ${test_env_vars|} $(pwd)/test_wt4699_json 2>&1
+ ${test_env_vars|} $(pwd)/test/csuite/test_wt4699_json 2>&1
- name: csuite-wt4803-cache-overflow-abort-test
depends_on:
@@ -679,12 +694,12 @@ tasks:
- func: "fetch artifacts"
- command: shell.exec
params:
- working_dir: "wiredtiger/build_posix/test/csuite"
+ working_dir: "wiredtiger/build_posix"
script: |
set -o errexit
set -o verbose
- ${test_env_vars|} $(pwd)/test_wt4803_cache_overflow_abort 2>&1
+ ${test_env_vars|} $(pwd)/test/csuite/test_wt4803_cache_overflow_abort 2>&1
- name: csuite-rwlock-test
depends_on:
@@ -693,12 +708,12 @@ tasks:
- func: "fetch artifacts"
- command: shell.exec
params:
- working_dir: "wiredtiger/build_posix/test/csuite"
+ working_dir: "wiredtiger/build_posix"
script: |
set -o errexit
set -o verbose
- ${test_env_vars|} $(pwd)/test_rwlock 2>&1
+ ${test_env_vars|} $(pwd)/test/csuite/test_rwlock 2>&1
- name: csuite-wt2246-col-append-test
depends_on:
@@ -707,12 +722,12 @@ tasks:
- func: "fetch artifacts"
- command: shell.exec
params:
- working_dir: "wiredtiger/build_posix/test/csuite"
+ working_dir: "wiredtiger/build_posix"
script: |
set -o errexit
set -o verbose
- ${test_env_vars|} $(pwd)/test_wt2246_col_append 2>&1
+ ${test_env_vars|} $(pwd)/test/csuite/test_wt2246_col_append 2>&1
- name: csuite-wt2323-join-visibility-test
depends_on:
@@ -721,12 +736,12 @@ tasks:
- func: "fetch artifacts"
- command: shell.exec
params:
- working_dir: "wiredtiger/build_posix/test/csuite"
+ working_dir: "wiredtiger/build_posix"
script: |
set -o errexit
set -o verbose
- ${test_env_vars|} $(pwd)/test_wt2323_join_visibility 2>&1
+ ${test_env_vars|} $(pwd)/test/csuite/test_wt2323_join_visibility 2>&1
- name: csuite-wt2535-insert-race-test
depends_on:
@@ -735,12 +750,12 @@ tasks:
- func: "fetch artifacts"
- command: shell.exec
params:
- working_dir: "wiredtiger/build_posix/test/csuite"
+ working_dir: "wiredtiger/build_posix"
script: |
set -o errexit
set -o verbose
- ${test_env_vars|} $(pwd)/test_wt2535_insert_race 2>&1
+ ${test_env_vars|} $(pwd)/test/csuite/test_wt2535_insert_race 2>&1
- name: csuite-wt2834-join-bloom-fix-test
depends_on:
@@ -749,12 +764,12 @@ tasks:
- func: "fetch artifacts"
- command: shell.exec
params:
- working_dir: "wiredtiger/build_posix/test/csuite"
+ working_dir: "wiredtiger/build_posix"
script: |
set -o errexit
set -o verbose
- ${test_env_vars|} $(pwd)/test_wt2834_join_bloom_fix 2>&1
+ ${test_env_vars|} $(pwd)/test/csuite/test_wt2834_join_bloom_fix 2>&1
- name: csuite-wt2853-perf-test
depends_on:
@@ -763,12 +778,12 @@ tasks:
- func: "fetch artifacts"
- command: shell.exec
params:
- working_dir: "wiredtiger/build_posix/test/csuite"
+ working_dir: "wiredtiger/build_posix"
script: |
set -o errexit
set -o verbose
- ${test_env_vars|} $(pwd)/test_wt2853_perf 2>&1
+ ${test_env_vars|} $(pwd)/test/csuite/test_wt2853_perf 2>&1
- name: csuite-wt2909-checkpoint-integrity-test
depends_on:
@@ -777,12 +792,12 @@ tasks:
- func: "fetch artifacts"
- command: shell.exec
params:
- working_dir: "wiredtiger/build_posix/test/csuite"
+ working_dir: "wiredtiger/build_posix"
script: |
set -o errexit
set -o verbose
- ${test_env_vars|} $(pwd)/test_wt2909_checkpoint_integrity 2>&1
+ ${test_env_vars|} $(pwd)/test/csuite/test_wt2909_checkpoint_integrity 2>&1
- name: csuite-wt3338-partial-update-test
depends_on:
@@ -791,12 +806,12 @@ tasks:
- func: "fetch artifacts"
- command: shell.exec
params:
- working_dir: "wiredtiger/build_posix/test/csuite"
+ working_dir: "wiredtiger/build_posix"
script: |
set -o errexit
set -o verbose
- ${test_env_vars|} $(pwd)/test_wt3338_partial_update 2>&1
+ ${test_env_vars|} $(pwd)/test/csuite/test_wt3338_partial_update 2>&1
- name: csuite-wt4333-handle-locks-test
depends_on:
@@ -805,12 +820,12 @@ tasks:
- func: "fetch artifacts"
- command: shell.exec
params:
- working_dir: "wiredtiger/build_posix/test/csuite"
+ working_dir: "wiredtiger/build_posix"
script: |
set -o errexit
set -o verbose
- ${test_env_vars|} $(pwd)/test_wt4333_handle_locks 2>&1
+ ${test_env_vars|} $(pwd)/test/csuite/test_wt4333_handle_locks 2>&1
# End of csuite test tasks
@@ -1158,7 +1173,7 @@ buildvariants:
- ubuntu1404-test
expansions:
# It's ugly, but we need the absolute path here, not the relative
- test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs
+ test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs top_srcdir=$(pwd)/.. top_builddir=$(pwd)
smp_command: -j $(grep -c ^processor /proc/cpuinfo)
configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/gcc CXX=/opt/mongodbtoolchain/v3/bin/g++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH
make_command: PATH=/opt/mongodbtoolchain/v3/bin:$PATH make
@@ -1178,6 +1193,7 @@ buildvariants:
- name: salvage-test
- name: thread-test
- name: bench-wtperf-test
+ - name: csuite-import-test
- name: csuite-random-abort-test
- name: csuite-random-directio-test
- name: csuite-schema-abort-test
@@ -1224,7 +1240,7 @@ buildvariants:
run_on:
- ubuntu1404-test
expansions:
- test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs
+ test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs top_srcdir=$(pwd)/.. top_builddir=$(pwd)
smp_command: -j $(grep -c ^processor /proc/cpuinfo)
configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/gcc CXX=/opt/mongodbtoolchain/v3/bin/g++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH
configure_python_setting: PYTHON=python3
@@ -1285,7 +1301,7 @@ buildvariants:
smp_command: -j $(sysctl -n hw.logicalcpu)
configure_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH
make_command: PATH=/opt/mongodbtoolchain/v3/bin:$PATH ARCHFLAGS=-Wno-error=unused-command-line-argument-hard-error-in-future make
- test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH DYLD_LIBRARY_PATH=$(pwd)/.libs
+ test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH DYLD_LIBRARY_PATH=$(pwd)/.libs top_srcdir=$(pwd)/.. top_builddir=$(pwd)
tasks:
- name: compile
- name: make-check-test
@@ -1300,7 +1316,7 @@ buildvariants:
expansions:
smp_command: -j $(grep -c ^processor /proc/cpuinfo)
configure_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH
- test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs
+ test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs top_srcdir=$(pwd)/.. top_builddir=$(pwd)
tasks:
- name: compile
- name: generate-datafile-little-endian
@@ -1317,7 +1333,7 @@ buildvariants:
expansions:
smp_command: -j $(grep -c ^processor /proc/cpuinfo)
configure_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH
- test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs
+ test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.lib top_srcdir=$(pwd)/.. top_builddir=$(pwd)
tasks:
- name: compile
- name: generate-datafile-big-endian
diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c
index 8f2c25429c9..df3222b072a 100644
--- a/src/third_party/wiredtiger/test/format/config.c
+++ b/src/third_party/wiredtiger/test/format/config.c
@@ -106,7 +106,7 @@ config_setup(void)
* cache problems, don't configure LSM if those set.
*
* XXX
- * Remove the timestamp test when WT-4067 resolved.
+ * Remove the timestamp test when WT-4162 resolved.
*/
if (g.type != ROW || g.c_in_memory)
break;
@@ -563,8 +563,7 @@ config_lsm_reset(void)
* LSM doesn't currently play nicely with timestamps, don't choose the
* pair unless forced to. If we turn off timestamps, make sure we turn
* off prepare as well, it requires timestamps. Remove this code with
- * WT-4067.
- *
+ * WT-4162.
*/
if (!config_is_perm("prepare") &&
!config_is_perm("transaction_timestamps")) {
diff --git a/src/third_party/wiredtiger/test/suite/run.py b/src/third_party/wiredtiger/test/suite/run.py
index 5895325a3d7..3de273c8ebf 100755
--- a/src/third_party/wiredtiger/test/suite/run.py
+++ b/src/third_party/wiredtiger/test/suite/run.py
@@ -31,7 +31,7 @@
#
from __future__ import print_function
-import glob, json, os, re, sys
+import glob, json, os, random, re, sys
try:
xrange
@@ -118,6 +118,8 @@ Options:\n\
-j N | --parallel N run all tests in parallel using N processes\n\
-l | --long run the entire test suite\n\
-p | --preserve preserve output files in WT_TEST/<testname>\n\
+ -r N | --random-sample N randomly sort scenarios to be run, then\n\
+ execute every Nth (2<=N<=1000) scenario.\n\
-s N | --scenario N use scenario N (N can be number or symbolic)\n\
-t | --timestamp name WT_TEST according to timestamp\n\
-v N | --verbose N set verboseness to N (0<=N<=3, default=1)\n\
@@ -269,6 +271,7 @@ if __name__ == '__main__':
# Turn numbers and ranges into test module names
preserve = timestamp = debug = dryRun = gdbSub = lldbSub = longtest = False
parallel = 0
+ random_sample = 0
configfile = None
configwrite = False
dirarg = None
@@ -307,6 +310,15 @@ if __name__ == '__main__':
if option == '-long' or option == 'l':
longtest = True
continue
+ if option == '-random-sample' or option == 'r':
+ if len(args) == 0:
+ usage()
+ sys.exit(2)
+ random_sample = int(args.pop(0))
+ if random_sample < 2 or random_sample > 1000:
+ usage()
+ sys.exit(2)
+ continue
if option == '-parallel' or option == 'j':
if parallel != 0 or len(args) == 0:
usage()
@@ -376,6 +388,14 @@ if __name__ == '__main__':
for arg in testargs:
testsFromArg(tests, loader, arg, scenario)
+ # Shuffle the tests and create a new suite containing every Nth test from
+ # the original suite
+ if random_sample > 0:
+ random_sample_tests = []
+ for test in tests:
+ random_sample_tests.append(test)
+ random.shuffle(random_sample_tests)
+ tests = unittest.TestSuite(random_sample_tests[::random_sample])
if debug:
import pdb
pdb.set_trace()
diff --git a/src/third_party/wiredtiger/test/suite/test_debug_mode04.py b/src/third_party/wiredtiger/test/suite/test_debug_mode04.py
new file mode 100644
index 00000000000..1f5429495e8
--- /dev/null
+++ b/src/third_party/wiredtiger/test/suite/test_debug_mode04.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+#
+# Public Domain 2034-2039 MongoDB, Inc.
+# Public Domain 2008-2034 WiredTiger, Inc.
+#
+# This is free and unencumbered software released into the public domain.
+#
+# Anyone is free to copy, modify, publish, use, compile, sell, or
+# distribute this software, either in source code form or as a compiled
+# binary, for any purpose, commercial or non-commercial, and by any
+# means.
+#
+# In jurisdictions that recognize copyright laws, the author or authors
+# of this software dedicate any and all copyright interest in the
+# software to the public domain. We make this dedication for the benefit
+# of the public at large and to the detriment of our heirs and
+# successors. We intend this dedication to be an overt act of
+# relinquishment in perpetuity of all present and future rights to this
+# software under copyright law.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+
+import wiredtiger, wttest
+
+# test_debug_mode04.py
+# Test the debug mode settings. Test eviction use.
+class test_debug_mode04(wttest.WiredTigerTestCase):
+ conn_config = 'log=(enabled=true,file_max=100K),debug_mode=(eviction=true)'
+ uri = 'file:test_debug'
+ entries = 100
+ value = b'\x01\x02abcd\x03\x04'
+
+ def add_data(self):
+ keys = range(0, self.entries)
+ c = self.session.open_cursor(self.uri, None)
+ for k in keys:
+ c[k] = self.value
+ c.close()
+
+ # Just test turning it on and off. There really isn't something
+ # specific to verify.
+ def test_table_logging(self):
+ self.session.create(self.uri, 'key_format=i,value_format=u')
+ self.add_data()
+
+ def test_table_logging_off(self):
+ self.conn.reconfigure("debug_mode=(eviction=false)")
+ self.session.create(self.uri, 'key_format=i,value_format=u')
+ self.add_data()
+
+if __name__ == '__main__':
+ wttest.run()
diff --git a/src/third_party/wiredtiger/test/suite/test_empty.py b/src/third_party/wiredtiger/test/suite/test_empty.py
index f7949871a11..b41ae3540c9 100755
--- a/src/third_party/wiredtiger/test/suite/test_empty.py
+++ b/src/third_party/wiredtiger/test/suite/test_empty.py
@@ -28,7 +28,6 @@
import os
import wiredtiger, wttest
-from wtdataset import simple_key
from wtscenario import make_scenarios
# test_empty.py
@@ -53,47 +52,5 @@ class test_empty(wttest.WiredTigerTestCase):
name = name + '.wt'
self.assertEquals(os.stat(name).st_size, 4*1024)
- # Open a new session, add a few rows to an object and then remove them,
- # then close the object. We open/close the object so it's flushed from
- # the underlying cache each time.
- def empty(self):
- uri = self.type + self.name
- self.session = self.conn.open_session()
- self.session.create(uri, 'key_format=' + self.fmt + ',value_format=S')
-
- # Add a few records to the object and remove them.
- cursor = self.session.open_cursor(uri, None, None)
- for i in range(1,5):
- key = simple_key(cursor, i)
- cursor[key] = "XXX"
- del cursor[key]
-
- # Perform a checkpoint (we shouldn't write any underlying pages because
- # of a checkpoint, either).
- self.session.checkpoint("name=ckpt")
-
- # Open and close a checkpoint cursor.
- cursor = self.session.open_cursor(uri, None, "checkpoint=ckpt")
- cursor.close()
-
- self.session.close()
-
- # The file should not have grown.
- name = self.name
- if self.type == "table:":
- name = name + '.wt'
- self.assertEquals(os.stat(name).st_size, 4*1024)
-
- # Creating an object, inserting and removing records (that is, building an
- # empty, dirty tree), shouldn't write any blocks. This doesn't work for
- # column-store objects, though, because deleting an object modifies the name
- # space, which requires a write.
- def test_empty(self):
- if self.fmt == 'r':
- return
-
- for i in range(1,5):
- self.empty()
-
if __name__ == '__main__':
wttest.run()
diff --git a/src/third_party/wiredtiger/test/suite/test_encrypt06.py b/src/third_party/wiredtiger/test/suite/test_encrypt06.py
index 7663a8d6d7e..520c18c04d6 100755
--- a/src/third_party/wiredtiger/test/suite/test_encrypt06.py
+++ b/src/third_party/wiredtiger/test/suite/test_encrypt06.py
@@ -45,7 +45,7 @@ class test_encrypt06(wttest.WiredTigerTestCase):
# testing a potential misuse of the API: a table is opened with
# with its own encryption options (different from the system),
# but the indices and column groups do not specify encryption,
- # so they'll get the system encryptor.
+ # so they may get the system encryptor.
storagetype = [
('table', dict(
uriprefix='table:', use_cg=False, use_index=False, match=True)),
@@ -66,25 +66,25 @@ class test_encrypt06(wttest.WiredTigerTestCase):
]
encrypt = [
('none', dict(
- sys_encrypt='none', sys_encrypt_args='', encryptmeta=False,
- file0_encrypt='none', file0_encrypt_args='', encrypt0=False,
- file1_encrypt='none', file1_encrypt_args='', encrypt1=False)),
+ sys_encrypt='none', sys_encrypt_args='',
+ table0_encrypt='none', table0_encrypt_args='',
+ table1_encrypt='none', table1_encrypt_args='')),
('rotn-implied', dict(
- sys_encrypt='rotn', sys_encrypt_args=key11, encryptmeta=True,
- file0_encrypt=None, file0_encrypt_args='', encrypt0=True,
- file1_encrypt=None, file1_encrypt_args='', encrypt1=True)),
+ sys_encrypt='rotn', sys_encrypt_args=key11,
+ table0_encrypt=None, table0_encrypt_args='',
+ table1_encrypt=None, table1_encrypt_args='')),
('rotn-all', dict(
- sys_encrypt='rotn', sys_encrypt_args=key11, encryptmeta=True,
- file0_encrypt='rotn', file0_encrypt_args=key13, encrypt0=True,
- file1_encrypt='rotn', file1_encrypt_args=key13, encrypt1=True)),
+ sys_encrypt='rotn', sys_encrypt_args=key11,
+ table0_encrypt='rotn', table0_encrypt_args=key13,
+ table1_encrypt='rotn', table1_encrypt_args=key13)),
('rotn-sys', dict(
- sys_encrypt='rotn', sys_encrypt_args=key11, encryptmeta=True,
- file0_encrypt='none', file0_encrypt_args='', encrypt0=False,
- file1_encrypt='none', file1_encrypt_args='', encrypt1=False)),
- ('rotn-file0', dict(
- sys_encrypt='rotn', sys_encrypt_args=key11, encryptmeta=True,
- file0_encrypt='rotn', file0_encrypt_args=key13, encrypt0=True,
- file1_encrypt='none', file1_encrypt_args='', encrypt1=False)),
+ sys_encrypt='rotn', sys_encrypt_args=key11,
+ table0_encrypt='none', table0_encrypt_args='',
+ table1_encrypt='none', table1_encrypt_args='')),
+ ('rotn-table0', dict(
+ sys_encrypt='rotn', sys_encrypt_args=key11,
+ table0_encrypt='rotn', table0_encrypt_args=key13,
+ table1_encrypt='none', table1_encrypt_args='')),
]
scenarios = make_scenarios(encrypt, storagetype)
nrecords = 1000
@@ -92,14 +92,14 @@ class test_encrypt06(wttest.WiredTigerTestCase):
def conn_extensions(self, extlist):
extlist.skip_if_missing = True
extlist.extension('encryptors', self.sys_encrypt)
- extlist.extension('encryptors', self.file0_encrypt)
- extlist.extension('encryptors', self.file1_encrypt)
+ extlist.extension('encryptors', self.table0_encrypt)
+ extlist.extension('encryptors', self.table1_encrypt)
def conn_config(self):
return 'encryption=(name={0}{1}),'.format(
self.sys_encrypt, self.sys_encrypt_args)
- def encrypt_file_params(self, name, args):
+ def encrypt_table_params(self, name, args):
if name == None:
return ''
else:
@@ -116,63 +116,85 @@ class test_encrypt06(wttest.WiredTigerTestCase):
return True
return False
- def expected_encryption(self, exp):
- expect = exp
- # If we're expecting it to be unencrypted, but we (errantly)
- # did not specify encryption on indices/columngroups,
- # then column groups (if they exist) will be encrypted -
- # there will be no data in the main table to be unencrypted.
- if self.sys_encrypt != 'none' and not self.match and self.use_cg:
- expect = True
- return expect
+ def visible_data(self, table_setting):
+ if table_setting == None:
+ # No table encryption explicitly set, so we use the system setting
+ visible = (self.sys_encrypt == 'none')
+ else:
+ visible = (table_setting == 'none')
+ return visible
+
+ def visible_name(self, table_setting, iskey):
+ if table_setting == None:
+ # No table encryption explicitly set, so we use the system setting
+ visible = (self.sys_encrypt == 'none')
+ else:
+ visible = (table_setting == 'none')
+
+ # If we have everything in a column group, the key name will not
+ # be stored in the column group files. It will be stored in the
+ # system metadata, if that is not encrypted.
+ if iskey and self.use_cg and self.sys_encrypt != 'none':
+ visible = False
+ return visible
# Create a table, add key/values with specific lengths, then verify them.
def test_encrypt(self):
name0 = 'test_encrypt06-0'
name1 = 'test_encrypt06-1'
- enc0 = self.encrypt_file_params(self.file0_encrypt,
- self.file0_encrypt_args)
- enc1 = self.encrypt_file_params(self.file1_encrypt,
- self.file1_encrypt_args)
+ enc0 = self.encrypt_table_params(self.table0_encrypt,
+ self.table0_encrypt_args)
+ enc1 = self.encrypt_table_params(self.table1_encrypt,
+ self.table1_encrypt_args)
# This is the clear text that we'll be looking for
txt0 = 'AbCdEfG'
txt1 = 'aBcDeFg'
+ keyname0 = 'MyKey0Name'
+ keyname1 = 'MyKey1Name'
+ valname0 = 'MyValue0Name'
+ valname1 = 'MyValue1Name'
# Make a bunch of column group and indices,
# we want to see if any information is leaked anywhere.
+ # The key column and one of the value columns is given a name
+ # we will look for as clear text.
sharedparam = 'key_format=S,value_format=SSSS,' + \
- 'columns=(MyKeyName,v0,v1,v2,v3),'
+ 'columns=({},{},v1,v2,v3),'
s = self.session
pfx = self.uriprefix
cgparam = 'colgroups=(g00,g01)' if self.use_cg else ''
- s.create(pfx + name0, sharedparam + cgparam + enc0)
+ s.create(pfx + name0, sharedparam.format(keyname0, valname0) + \
+ cgparam + enc0)
- # Having unmatched encryption for colgroup or index is
- # not recommended, but we check it.
if not self.match:
enc0 = ''
if self.use_cg:
- s.create('colgroup:' + name0 + ':g00', 'columns=(v0,v1)' + enc0)
+ s.create('colgroup:' + name0 + ':g00',
+ 'columns=({},v1)'.format(valname0) + enc0)
s.create('colgroup:' + name0 + ':g01', 'columns=(v2,v3)' + enc0)
if self.use_index:
- s.create('index:' + name0 + ':i00', 'columns=(v0)' + enc0)
+ s.create('index:' + name0 + ':i00',
+ 'columns=({})'.format(valname0) + enc0)
s.create('index:' + name0 + ':i01', 'columns=(v1,v2)' + enc0)
s.create('index:' + name0 + ':i02', 'columns=(v3)' + enc0)
cgparam = 'colgroups=(g10,g11)' if self.use_cg else ''
- s.create(pfx + name1, sharedparam + cgparam + enc1)
+ s.create(pfx + name1, sharedparam.format(keyname1, valname1) + \
+ cgparam + enc1)
if not self.match:
enc1 = ''
if self.use_cg:
- s.create('colgroup:' + name1 + ':g10', 'columns=(v0,v1)' + enc1)
+ s.create('colgroup:' + name1 + ':g10',
+ 'columns=({},v1)'.format(valname1) + enc1)
s.create('colgroup:' + name1 + ':g11', 'columns=(v2,v3)' + enc1)
if self.use_index:
- s.create('index:' + name1 + ':i10', 'columns=(v0)' + enc1)
+ s.create('index:' + name1 + ':i10',
+ 'columns=({})'.format(valname1) + enc1)
s.create('index:' + name1 + ':i11', 'columns=(v1,v2)' + enc1)
s.create('index:' + name1 + ':i12', 'columns=(v3)' + enc1)
@@ -192,13 +214,40 @@ class test_encrypt06(wttest.WiredTigerTestCase):
# Force everything to disk so we can examine it
self.close_conn()
- self.assertEqual(self.encryptmeta,
- not self.match_string_in_rundir('MyKeyName'))
-
- self.assertEqual(self.expected_encryption(self.encrypt0),
- not self.match_string_in_rundir(txt0))
- self.assertEqual(self.expected_encryption(self.encrypt1),
- not self.match_string_in_rundir(txt1))
+ if self.match:
+ # Key and value names are encrypted according to the
+ # encryption level on the associated table.
+ self.assertEqual(self.visible_data(self.table0_encrypt),
+ self.match_string_in_rundir(txt0))
+ self.assertEqual(self.visible_name(self.table0_encrypt, True),
+ self.match_string_in_rundir(keyname0))
+ self.assertEqual(self.visible_name(self.table0_encrypt, False),
+ self.match_string_in_rundir(valname0))
+
+ self.assertEqual(self.visible_data(self.table1_encrypt),
+ self.match_string_in_rundir(txt1))
+ self.assertEqual(self.visible_name(self.table1_encrypt, True),
+ self.match_string_in_rundir(keyname1))
+ self.assertEqual(self.visible_name(self.table1_encrypt, False),
+ self.match_string_in_rundir(valname1))
+ else:
+ # If the encryption config for indices and column groups is blank,
+ # we make a conservative check - if we specified encryption on the
+ # table, none of our data or key/value names should be exposed.
+ #
+ # If we have system encryption on, set table encryption to 'none',
+ # and set the index or column group config to blank, we technically
+ # should get no encryption for names or data. That currently
+ # doesn't work (CGs and indices instead will be encrypted),
+ # so we don't cover that case.
+ if self.table0_encrypt != 'none':
+ self.assertFalse(self.match_string_in_rundir(txt0))
+ self.assertFalse(self.match_string_in_rundir(keyname0))
+ self.assertFalse(self.match_string_in_rundir(valname0))
+ if self.table1_encrypt != 'none':
+ self.assertFalse(self.match_string_in_rundir(txt1))
+ self.assertFalse(self.match_string_in_rundir(keyname1))
+ self.assertFalse(self.match_string_in_rundir(valname1))
if __name__ == '__main__':
wttest.run()
diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp09.py b/src/third_party/wiredtiger/test/suite/test_timestamp09.py
index c4b545fe2c1..efc081c29f4 100644
--- a/src/third_party/wiredtiger/test/suite/test_timestamp09.py
+++ b/src/third_party/wiredtiger/test/suite/test_timestamp09.py
@@ -165,10 +165,9 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess):
# Read timestamp >= Oldest timestamp
self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(7) +
',stable_timestamp=' + timestamp_str(7))
- self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
- lambda: self.session.begin_transaction('read_timestamp=' +
- timestamp_str(6)),
- '/less than the oldest timestamp/')
+ with self.expectedStdoutPattern('less than the oldest timestamp'):
+ self.assertRaisesException(wiredtiger.WiredTigerError,
+ lambda: self.session.begin_transaction('read_timestamp=' + timestamp_str(6)))
# c[8] is not visible at read_timestamp < 8
self.session.begin_transaction('read_timestamp=' + timestamp_str(7))
@@ -189,10 +188,9 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess):
# We can move the oldest timestamp backwards with "force"
self.conn.set_timestamp(
'oldest_timestamp=' + timestamp_str(5) + ',force')
- self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
- lambda: self.session.begin_transaction('read_timestamp=' +
- timestamp_str(4)),
- '/less than the oldest timestamp/')
+ with self.expectedStdoutPattern('less than the oldest timestamp'):
+ self.assertRaisesException(wiredtiger.WiredTigerError,
+ lambda: self.session.begin_transaction('read_timestamp=' + timestamp_str(4)))
self.session.begin_transaction('read_timestamp=' + timestamp_str(6))
self.assertTimestampsEqual(
self.conn.query_timestamp('get=oldest_reader'), timestamp_str(6))
diff --git a/src/third_party/wiredtiger/test/utility/parse_opts.c b/src/third_party/wiredtiger/test/utility/parse_opts.c
index c5bb1a83148..06d9ea31538 100644
--- a/src/third_party/wiredtiger/test/utility/parse_opts.c
+++ b/src/third_party/wiredtiger/test/utility/parse_opts.c
@@ -44,6 +44,7 @@ testutil_parse_opts(int argc, char * const *argv, TEST_OPTS *opts)
opts->running = true;
opts->verbose = false;
+ opts->argv0 = argv[0];
opts->progname = testutil_set_progname(argv);
testutil_print_command_line(argc, argv);
diff --git a/src/third_party/wiredtiger/test/utility/test_util.h b/src/third_party/wiredtiger/test/utility/test_util.h
index 27310de0006..7aa4eaecc0f 100644
--- a/src/third_party/wiredtiger/test/utility/test_util.h
+++ b/src/third_party/wiredtiger/test/utility/test_util.h
@@ -49,7 +49,9 @@
/* Generic option parsing structure shared by all test cases. */
typedef struct {
char *home;
- const char *progname;
+ const char *argv0; /* Exec name */
+ const char *progname; /* Truncated program name */
+
enum { TABLE_COL=1, /* Fixed-length column store */
TABLE_FIX=2, /* Variable-length column store */
TABLE_ROW=3 /* Row-store */
diff --git a/src/third_party/wiredtiger/tools/optrack/find-latency-spikes.py b/src/third_party/wiredtiger/tools/optrack/find-latency-spikes.py
index 7409ab62243..1c030721787 100755
--- a/src/third_party/wiredtiger/tools/optrack/find-latency-spikes.py
+++ b/src/third_party/wiredtiger/tools/optrack/find-latency-spikes.py
@@ -864,7 +864,7 @@ def generateTSSlicesForBuckets():
numBuckets, "Generating timeline charts");
for i, fname in returnValues.items():
- bucketFilenames.append(str(fname.value));
+ bucketFilenames.append(fname.value.decode());
print(color.END);
return bucketFilenames;