Import wiredtiger: 40e3225e55ffbf0c9819af138fa4a014c0639c72 from branch mongodb-4.2

ref: 9b85ad8968..40e3225e55
for: 4.3.1

WT-4477       Add eviction debug mode and extra checks
WT-4494       Support import of of standalone WiredTiger files
WT-4580       Abort transactions that perform updates with ignore_prepare=true
WT-4719       Non-zero update count detected when committing prepared transaction
WT-4787       Coverity: Potential infinite loops
WT-4790       Coverity: Memory leak within __wt_curbulk_init
WT-4827       Apply commit timestamps when a truncated page is read
WT-4829       File names are generated incorrectly in operation tracking scripts
WT-4837       Disabled cache during verify
WT-4839       Add a feature to python testsuite to run a random subset of tests
WT-4844       Only log an informational message when a set read-timestamp is older than the oldest timestamp
WT-4848       Fix perf regression when calculating differences
WT-4850       Fix __slvg_checkpoint() metadata memory leak
WT-4851       heap-use-after-free when block manager grows buffer during final checkpoint
WT-4852       Evergreen runs of test/format require shared library extensions
WT-4853       Standalone recovery code cannot handle deleted checkpoints
WT-4854       wt utility list command has a memory leak and minor error handling bugs
WT-4855       WiredTiger recovery should detect files without unique IDs
WT-4859       Fix test/format program path issue for endian tests

1 files changed, 60 insertions, 111 deletions

diff --git a/src/third_party/wiredtiger/src/block/block_ckpt.c b/src/third_party/wiredtiger/src/block/block_ckpt.c
index 40b0fef5a29..db6b458d671 100644
--- a/src/third_party/wiredtiger/src/block/block_ckpt.c
+++ b/src/third_party/wiredtiger/src/block/block_ckpt.c
@@ -9,10 +9,8 @@
 #include "wt_internal.h"
 
 static int __ckpt_process(WT_SESSION_IMPL *, WT_BLOCK *, WT_CKPT *);
-static int __ckpt_string(
-	WT_SESSION_IMPL *, WT_BLOCK *, const uint8_t *, WT_ITEM *);
-static int __ckpt_update(
-	WT_SESSION_IMPL *, WT_BLOCK *, WT_CKPT *, WT_BLOCK_CKPT *, bool);
+static int __ckpt_update(WT_SESSION_IMPL *,
+	WT_BLOCK *, WT_CKPT *, WT_CKPT *, WT_BLOCK_CKPT *, bool);
 
 /*
  * __wt_block_ckpt_init --
@@ -49,7 +47,6 @@ __wt_block_checkpoint_load(WT_SESSION_IMPL *session, WT_BLOCK *block,
     uint8_t *root_addr, size_t *root_addr_sizep, bool checkpoint)
 {
 	WT_BLOCK_CKPT *ci, _ci;
-	WT_DECL_ITEM(tmp);
 	WT_DECL_RET;
 	uint8_t *endp;
 
@@ -62,15 +59,8 @@ __wt_block_checkpoint_load(WT_SESSION_IMPL *session, WT_BLOCK *block,
 
 	ci = NULL;
 
-	if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT)) {
-		if (addr != NULL) {
-			WT_ERR(__wt_scr_alloc(session, 0, &tmp));
-			WT_ERR(__ckpt_string(session, block, addr, tmp));
-		}
-		__wt_verbose(session, WT_VERB_CHECKPOINT,
-		    "%s: load-checkpoint: %s", block->name,
-		    addr == NULL ? "[Empty]" : (const char *)tmp->data);
-	}
+	if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT))
+		__wt_ckpt_verbose(session, block, "load", NULL, addr);
 
 	/*
 	 * There's a single checkpoint in the file that can be written, all of
@@ -153,7 +143,6 @@ err:		/*
 	if (checkpoint && ci != NULL)
 		__wt_block_ckpt_destroy(session, ci);
 
-	__wt_scr_free(session, &tmp);
 	return (ret);
 }
 
@@ -388,7 +377,6 @@ __ckpt_process(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckptbase)
 {
 	WT_BLOCK_CKPT *a, *b, *ci;
 	WT_CKPT *ckpt, *next_ckpt;
-	WT_DECL_ITEM(tmp);
 	WT_DECL_RET;
 	uint64_t ckpt_size;
 	bool deleting, fatal, locked;
@@ -544,15 +532,10 @@ __ckpt_process(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckptbase)
 		    !F_ISSET(ckpt, WT_CKPT_DELETE))
 			continue;
 
-		if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT)) {
-			if (tmp == NULL)
-				WT_ERR(__wt_scr_alloc(session, 0, &tmp));
-			WT_ERR(__ckpt_string(
-			    session, block, ckpt->raw.data, tmp));
-			__wt_verbose(session, WT_VERB_CHECKPOINT,
-			    "%s: delete-checkpoint: %s: %s",
-			    block->name, ckpt->name, (const char *)tmp->data);
-		}
+		if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT))
+			__wt_ckpt_verbose(session,
+			    block, "delete", ckpt->name, ckpt->raw.data);
+
 		/*
 		 * Find the checkpoint into which we'll roll this checkpoint's
 		 * blocks: it's the next real checkpoint in the list, and it
@@ -642,8 +625,8 @@ __ckpt_process(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckptbase)
 	/* Update checkpoints marked for update. */
 	WT_CKPT_FOREACH(ckptbase, ckpt)
 		if (F_ISSET(ckpt, WT_CKPT_UPDATE))
-			WT_ERR(__ckpt_update(
-			    session, block, ckpt, ckpt->bpriv, false));
+			WT_ERR(__ckpt_update(session,
+			    block, ckptbase, ckpt, ckpt->bpriv, false));
 
 live_update:
 	/* Truncate the file if that's possible. */
@@ -686,7 +669,8 @@ live_update:
 			ci->ckpt_size =
 			    WT_MIN(ckpt_size, (uint64_t)block->size);
 
-			WT_ERR(__ckpt_update(session, block, ckpt, ci, true));
+			WT_ERR(__ckpt_update(
+			    session, block, ckptbase, ckpt, ci, true));
 		}
 
 	/*
@@ -733,7 +717,6 @@ err:	if (ret != 0 && fatal) {
 		if ((ci = ckpt->bpriv) != NULL)
 			__wt_block_ckpt_destroy(session, ci);
 
-	__wt_scr_free(session, &tmp);
 	return (ret);
 }
 
@@ -742,10 +725,10 @@ err:	if (ret != 0 && fatal) {
  *	Update a checkpoint.
  */
 static int
-__ckpt_update(WT_SESSION_IMPL *session,
-    WT_BLOCK *block, WT_CKPT *ckpt, WT_BLOCK_CKPT *ci, bool is_live)
+__ckpt_update(WT_SESSION_IMPL *session, WT_BLOCK *block,
+    WT_CKPT *ckptbase, WT_CKPT *ckpt, WT_BLOCK_CKPT *ci, bool is_live)
 {
-	WT_DECL_ITEM(tmp);
+	WT_DECL_ITEM(a);
 	WT_DECL_RET;
 	uint8_t *endp;
 
@@ -764,6 +747,37 @@ __ckpt_update(WT_SESSION_IMPL *session,
 	WT_RET(__wt_block_extlist_write(session, block, &ci->discard, NULL));
 
 	/*
+	 * If this is the final block, we append an incomplete copy of the
+	 * checkpoint information to the avail list for standalone retrieval.
+	 */
+	if (is_live) {
+		/*
+		 * Copy the INCOMPLETE checkpoint information into the
+		 * checkpoint.
+		 */
+		WT_RET(__wt_buf_init(
+		    session, &ckpt->raw, WT_BLOCK_CHECKPOINT_BUFFER));
+		endp = ckpt->raw.mem;
+		WT_RET(__wt_block_ckpt_to_buffer(
+		    session, block, &endp, ci, true));
+		ckpt->raw.size = WT_PTRDIFF(endp, ckpt->raw.mem);
+
+		/*
+		 * Convert the INCOMPLETE checkpoint array into its metadata
+		 * representation. This must match what is eventually written
+		 * into the metadata file, in other words, everything must be
+		 * initialized before the block manager does the checkpoint.
+		 */
+		WT_RET(__wt_scr_alloc(session, 8 * 1024, &a));
+		ret = __wt_meta_ckptlist_to_meta(session, ckptbase, a);
+		if (ret == 0)
+			ret = __wt_strndup(
+			    session, a->data, a->size, &ckpt->block_checkpoint);
+		__wt_scr_free(session, &a);
+		WT_RET(ret);
+	}
+
+	/*
 	 * We only write an avail list for the live system, other checkpoint's
 	 * avail lists are static and never change.
 	 *
@@ -775,9 +789,13 @@ __ckpt_update(WT_SESSION_IMPL *session,
 	 * it's not truly available until the new checkpoint locations have been
 	 * saved to the metadata.
 	 */
-	if (is_live)
-		WT_RET(__wt_block_extlist_write(
-		    session, block, &ci->avail, &ci->ckpt_avail));
+	if (is_live) {
+		block->final_ckpt = ckpt;
+		ret = __wt_block_extlist_write(
+		    session, block, &ci->avail, &ci->ckpt_avail);
+		block->final_ckpt = NULL;
+		WT_RET(ret);
+	}
 
 	/*
 	 * Set the file size for the live system.
@@ -800,25 +818,17 @@ __ckpt_update(WT_SESSION_IMPL *session,
 	if (is_live)
 		ci->file_size = block->size;
 
-	/*
-	 * Copy the checkpoint information into the checkpoint array's address
-	 * cookie.
-	 */
-	WT_RET(__wt_buf_init(session, &ckpt->raw, WT_BTREE_MAX_ADDR_COOKIE));
+	/* Copy the COMPLETE checkpoint information into the checkpoint. */
+	WT_RET(__wt_buf_init(session, &ckpt->raw, WT_BLOCK_CHECKPOINT_BUFFER));
 	endp = ckpt->raw.mem;
-	WT_RET(__wt_block_ckpt_to_buffer(session, block, &endp, ci));
+	WT_RET(__wt_block_ckpt_to_buffer(session, block, &endp, ci, false));
 	ckpt->raw.size = WT_PTRDIFF(endp, ckpt->raw.mem);
 
-	if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT)) {
-		WT_RET(__wt_scr_alloc(session, 0, &tmp));
-		WT_ERR(__ckpt_string(session, block, ckpt->raw.data, tmp));
-		__wt_verbose(session, WT_VERB_CHECKPOINT,
-		    "%s: create-checkpoint: %s: %s",
-		    block->name, ckpt->name, (const char *)tmp->data);
-	}
+	if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT))
+		__wt_ckpt_verbose(
+		    session, block, "create", ckpt->name, ckpt->raw.data);
 
-err:	__wt_scr_free(session, &tmp);
-	return (ret);
+	return (0);
 }
 
 /*
@@ -882,64 +892,3 @@ err:	__wt_spin_unlock(session, &block->live_lock);
 
 	return (ret);
 }
-
-/*
- * __ckpt_string --
- *	Return a printable string representation of a checkpoint address cookie.
- */
-static int
-__ckpt_string(WT_SESSION_IMPL *session,
-    WT_BLOCK *block, const uint8_t *addr, WT_ITEM *buf)
-{
-	WT_BLOCK_CKPT *ci, _ci;
-
-	/* Initialize the checkpoint, crack the cookie. */
-	ci = &_ci;
-	WT_RET(__wt_block_ckpt_init(session, ci, "string"));
-	WT_RET(__wt_block_buffer_to_ckpt(session, block, addr, ci));
-
-	WT_RET(__wt_buf_fmt(session, buf,
-	    "version=%" PRIu8, ci->version));
-	if (ci->root_offset == WT_BLOCK_INVALID_OFFSET)
-		WT_RET(__wt_buf_catfmt(session, buf, ", root=[Empty]"));
-	else
-		WT_RET(__wt_buf_catfmt(session, buf,
-		    ", root=[%"
-		    PRIuMAX "-%" PRIuMAX ", %" PRIu32 ", %" PRIu32 "]",
-		    (uintmax_t)ci->root_offset,
-		    (uintmax_t)(ci->root_offset + ci->root_size),
-		    ci->root_size, ci->root_checksum));
-	if (ci->alloc.offset == WT_BLOCK_INVALID_OFFSET)
-		WT_RET(__wt_buf_catfmt(session, buf, ", alloc=[Empty]"));
-	else
-		WT_RET(__wt_buf_catfmt(session, buf,
-		    ", alloc=[%"
-		    PRIuMAX "-%" PRIuMAX ", %" PRIu32 ", %" PRIu32 "]",
-		    (uintmax_t)ci->alloc.offset,
-		    (uintmax_t)(ci->alloc.offset + ci->alloc.size),
-		    ci->alloc.size, ci->alloc.checksum));
-	if (ci->avail.offset == WT_BLOCK_INVALID_OFFSET)
-		WT_RET(__wt_buf_catfmt(session, buf, ", avail=[Empty]"));
-	else
-		WT_RET(__wt_buf_catfmt(session, buf,
-		    ", avail=[%"
-		    PRIuMAX "-%" PRIuMAX ", %" PRIu32 ", %" PRIu32 "]",
-		    (uintmax_t)ci->avail.offset,
-		    (uintmax_t)(ci->avail.offset + ci->avail.size),
-		    ci->avail.size, ci->avail.checksum));
-	if (ci->discard.offset == WT_BLOCK_INVALID_OFFSET)
-		WT_RET(__wt_buf_catfmt(session, buf, ", discard=[Empty]"));
-	else
-		WT_RET(__wt_buf_catfmt(session, buf,
-		    ", discard=[%"
-		    PRIuMAX "-%" PRIuMAX ", %" PRIu32 ", %" PRIu32 "]",
-		    (uintmax_t)ci->discard.offset,
-		    (uintmax_t)(ci->discard.offset + ci->discard.size),
-		    ci->discard.size, ci->discard.checksum));
-	WT_RET(__wt_buf_catfmt(session, buf,
-	    ", file size=%" PRIuMAX, (uintmax_t)ci->file_size));
-
-	__wt_block_ckpt_destroy(session, ci);
-
-	return (0);
-}