Import wiredtiger: 6173a98979715ed727c432c1a31da64ea8a37048 from branch mongodb-3.6

ref: ff10db8811..6173a98979
for: 3.5.11

WT-3039       Enhance logging so new log files no longer need to wait for the previous log file to be synced
WT-3138       Enhance eviction statistics
WT-3140       Enhance json mode statistics to better support per-dhandle statistics
WT-3310       Add support to WT_SESSION::alter to change table log setting
WT-3329       With a uniform workload and a number of small collections, eviction does a poor job of selecting candidates for eviction.
WT-3381       Improve concurrency in the transaction subsystem
WT-3394       Build WiredTiger with gcc7
WT-3396       test/csuite/wt3363_checkpoint_op_races, test_utility cleanups
WT-3397       Coverity lint
WT-3398       in-memory configurations don't support schema_worker operations
WT-3401       Lint and minor cleanup
WT-3402       Move cached overflow records to the update list.
WT-3403       Automated tests timeout running csuite
WT-3409       WiredTiger generations can silently self-deadlock.
WT-3413       Add more aggressive compile warning flags to Jenkins Windows job
WT-3415       setting timestamp in txn_state requires the global write lock
WT-3421       Compilation error unreachable code in test/csuite/wt3363_checkpoint_op_races
WT-3424       additional gcc 7.1 compile warnings.
WT-3425       workgen: add ability to reopen cursors
WT-3426       Add new wtperf configuration to automated performance testing
WT-3438       Don't tune eviction thread count when the count is fixed

Also explicitly disable v2 log records via a configuration setting in MongoDB -
this will later be plumbed into the feature compatability version mechanism.

8 files changed, 147 insertions, 51 deletions

diff --git a/src/third_party/wiredtiger/src/btree/bt_debug.c b/src/third_party/wiredtiger/src/btree/bt_debug.c
index 394ac6c7b84..c0aaf3f42d9 100644
--- a/src/third_party/wiredtiger/src/btree/bt_debug.c
+++ b/src/third_party/wiredtiger/src/btree/bt_debug.c
@@ -445,18 +445,28 @@ static char *
 __debug_tree_shape_info(WT_PAGE *page)
 {
 	uint64_t v;
-	static char buf[32];
+	static char buf[128];
+	const char *unit;
 
 	v = page->memory_footprint;
-	if (v >= WT_GIGABYTE)
-		(void)__wt_snprintf(buf, sizeof(buf),
-		    "(%p %" PRIu64 "G)", (void *)page, v / WT_GIGABYTE);
-	else if (v >= WT_MEGABYTE)
-		(void)__wt_snprintf(buf, sizeof(buf),
-		    "(%p %" PRIu64 "M)", (void *)page, v / WT_MEGABYTE);
-	else
-		(void)__wt_snprintf(buf, sizeof(buf),
-		    "(%p %" PRIu64 ")", (void *)page, v);
+
+	if (v > WT_GIGABYTE) {
+		v /= WT_GIGABYTE;
+		unit = "G";
+	} else if (v > WT_MEGABYTE) {
+		v /= WT_MEGABYTE;
+		unit = "M";
+	} else if (v > WT_KILOBYTE) {
+		v /= WT_KILOBYTE;
+		unit = "K";
+	} else {
+		unit = "B";
+	}
+
+	(void)__wt_snprintf(buf, sizeof(buf), "(%p, %" PRIu64
+	    "%s, evict gen %" PRIu64 ", create gen %" PRIu64 ")",
+	    (void *)page, v, unit,
+	    page->evict_pass_gen, page->cache_create_gen);
 	return (buf);
 }
 
@@ -982,7 +992,7 @@ __debug_row_skip(WT_DBG *ds, WT_INSERT_HEAD *head)
 static int
 __debug_update(WT_DBG *ds, WT_UPDATE *upd, bool hexbyte)
 {
-	for (; upd != NULL; upd = upd->next)
+	for (; upd != NULL; upd = upd->next) {
 		if (upd->type == WT_UPDATE_DELETED)
 			WT_RET(ds->f(ds, "\tvalue {deleted}\n"));
 		else if (upd->type == WT_UPDATE_RESERVED)
@@ -995,6 +1005,30 @@ __debug_update(WT_DBG *ds, WT_UPDATE *upd, bool hexbyte)
 		} else
 			WT_RET(__debug_item(ds,
 			    "value", WT_UPDATE_DATA(upd), upd->size));
+		WT_RET(ds->f(ds, "\t" "txn id %" PRIu64, upd->txnid));
+
+#ifdef HAVE_TIMESTAMPS
+		if (!__wt_timestamp_iszero(upd->timestamp)) {
+#if WT_TIMESTAMP_SIZE == 8
+			{
+			uint64_t ts;
+			__wt_timestamp_set(
+			    (uint8_t *)&ts, (uint8_t *)&upd->timestamp[0]);
+			ts = __wt_bswap64(ts);
+			WT_RET(ds->f(ds, ", stamp %" PRIu64, ts));
+			}
+#else
+			{
+			int i;
+			WT_RET(ds->f(ds, ", stamp 0x"));
+			for (i = 0; i < WT_TIMESTAMP_SIZE; ++i)
+				WT_RET(ds->f(ds, "%" PRIx8, upd->timestamp[i]));
+			}
+#endif
+		}
+#endif
+		WT_RET(ds->f(ds, "\n"));
+	}
 	return (0);
 }
 
diff --git a/src/third_party/wiredtiger/src/btree/bt_discard.c b/src/third_party/wiredtiger/src/btree/bt_discard.c
index bfa8eb25aac..bc9356e2669 100644
--- a/src/third_party/wiredtiger/src/btree/bt_discard.c
+++ b/src/third_party/wiredtiger/src/btree/bt_discard.c
@@ -249,7 +249,8 @@ __free_page_modify(WT_SESSION_IMPL *session, WT_PAGE *page)
 
 	/* Free the overflow on-page, reuse and transaction-cache skiplists. */
 	__wt_ovfl_reuse_free(session, page);
-	__wt_ovfl_txnc_free(session, page);
+	if (mod->ovfl_track != NULL)
+		__wt_free(session, mod->ovfl_track->remove);
 	__wt_ovfl_discard_free(session, page);
 
 	__wt_free(session, page->modify->ovfl_track);
diff --git a/src/third_party/wiredtiger/src/btree/bt_handle.c b/src/third_party/wiredtiger/src/btree/bt_handle.c
index 06fbd6b74c7..5fad76849dc 100644
--- a/src/third_party/wiredtiger/src/btree/bt_handle.c
+++ b/src/third_party/wiredtiger/src/btree/bt_handle.c
@@ -279,6 +279,9 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt)
 	bool fixed;
 	const char **cfg, *enc_cfg[] = { NULL, NULL };
 
+	WT_UNUSED(maj_version);				/* !HAVE_VERBOSE */
+	WT_UNUSED(min_version);				/* !HAVE_VERBOSE */
+
 	btree = S2BT(session);
 	cfg = btree->dhandle->cfg;
 	conn = S2C(session);
@@ -290,7 +293,7 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt)
 		WT_RET(__wt_config_gets(session, cfg, "version.minor", &cval));
 		min_version = cval.val;
 		__wt_verbose(session, WT_VERB_VERSION,
-		    "%" PRIu64 ".%" PRIu64, maj_version, min_version);
+		    "%" PRId64 ".%" PRId64, maj_version, min_version);
 	}
 
 	/* Get the file ID. */
diff --git a/src/third_party/wiredtiger/src/btree/bt_ovfl.c b/src/third_party/wiredtiger/src/btree/bt_ovfl.c
index 76d1bfd121c..a0b1ff65006 100644
--- a/src/third_party/wiredtiger/src/btree/bt_ovfl.c
+++ b/src/third_party/wiredtiger/src/btree/bt_ovfl.c
@@ -48,6 +48,9 @@ __wt_ovfl_read(WT_SESSION_IMPL *session,
     WT_PAGE *page, WT_CELL_UNPACK *unpack, WT_ITEM *store)
 {
 	WT_DECL_RET;
+	WT_OVFL_TRACK *track;
+	WT_UPDATE *upd;
+	size_t i;
 
 	/*
 	 * If no page specified, there's no need to lock and there's no cache
@@ -61,16 +64,24 @@ __wt_ovfl_read(WT_SESSION_IMPL *session,
 	 * WT_CELL_VALUE_OVFL_RM cells: If reconciliation deleted an overflow
 	 * value, but there was still a reader in the system that might need it,
 	 * the on-page cell type will have been reset to WT_CELL_VALUE_OVFL_RM
-	 * and we will be passed a page so we can look-aside into the cache of
-	 * such values.
+	 * and we will be passed a page so we can check the on-page cell.
 	 *
 	 * Acquire the overflow lock, and retest the on-page cell's value inside
 	 * the lock.
 	 */
 	__wt_readlock(session, &S2BT(session)->ovfl_lock);
-	ret = __wt_cell_type_raw(unpack->cell) == WT_CELL_VALUE_OVFL_RM ?
-	    __wt_ovfl_txnc_search(page, unpack->data, unpack->size, store) :
-	    __ovfl_read(session, unpack->data, unpack->size, store);
+	if (__wt_cell_type_raw(unpack->cell) == WT_CELL_VALUE_OVFL_RM) {
+		track = page->modify->ovfl_track;
+		for (upd = NULL, i = 0; i < track->remove_next; ++i)
+			if (track->remove[i].cell == unpack->cell) {
+				upd = track->remove[i].upd;
+				break;
+			}
+		WT_ASSERT(session, i < track->remove_next);
+		store->data = WT_UPDATE_DATA(upd);
+		store->size = upd->size;
+	} else
+		ret = __ovfl_read(session, unpack->data, unpack->size, store);
 	__wt_readunlock(session, &S2BT(session)->ovfl_lock);
 
 	return (ret);
@@ -98,8 +109,7 @@ __ovfl_cache_col_visible(
 	 * the value.
 	 */
 	if (__wt_cell_rle(unpack) == 1 &&
-	    upd != NULL &&		/* Sanity: upd should always be set. */
-	    __wt_txn_upd_visible_all(session, upd))
+	    WT_UPDATE_DATA_VALUE(upd) && __wt_txn_upd_visible_all(session, upd))
 		return (true);
 	return (false);
 }
@@ -109,51 +119,99 @@ __ovfl_cache_col_visible(
  *	row-store: check for a globally visible update.
  */
 static bool
-__ovfl_cache_row_visible(WT_SESSION_IMPL *session, WT_PAGE *page, WT_ROW *rip)
+__ovfl_cache_row_visible(WT_SESSION_IMPL *session, WT_UPDATE *upd)
 {
-	WT_UPDATE *upd;
-
 	/* Check to see if there's a globally visible update. */
-	for (upd = WT_ROW_UPDATE(page, rip); upd != NULL; upd = upd->next)
-		if (__wt_txn_upd_visible_all(session, upd))
+	for (; upd != NULL; upd = upd->next)
+		 if (WT_UPDATE_DATA_VALUE(upd) &&
+		     __wt_txn_upd_visible_all(session, upd))
 			return (true);
 
 	return (false);
 }
 
 /*
- * __ovfl_cache --
- *	Cache a deleted overflow value.
+ * __ovfl_cache_append_update --
+ *	Append an overflow value to the update list.
  */
 static int
-__ovfl_cache(WT_SESSION_IMPL *session, WT_PAGE *page, WT_CELL_UNPACK *unpack)
+__ovfl_cache_append_update(WT_SESSION_IMPL *session, WT_PAGE *page,
+    WT_UPDATE *upd_list, WT_CELL_UNPACK *unpack, WT_UPDATE **updp)
 {
 	WT_DECL_ITEM(tmp);
 	WT_DECL_RET;
-	size_t addr_size;
-	const uint8_t *addr;
+	WT_UPDATE *append, *upd;
+	size_t size;
 
-	addr = unpack->data;
-	addr_size = unpack->size;
+	*updp = NULL;
 
+	/* Read the overflow value. */
 	WT_RET(__wt_scr_alloc(session, 1024, &tmp));
+	WT_ERR(__ovfl_read(session, unpack->data, unpack->size, tmp));
+
+	/*
+	 * Create an update entry with no transaction ID to ensure global
+	 * visibility, append it to the update list.
+	 *
+	 * We don't need locks or barriers in this function: any thread reading
+	 * the update list will see our newly appended record or not, it doesn't
+	 * matter until the on-page cell is set to WT_CELL_VALUE_OVFL_RM. That
+	 * involves atomic operations which will act as our barrier. Regardless,
+	 * we update the page footprint as part of this operation, which acts as
+	 * a barrier as well.
+	 */
+	WT_ERR(__wt_update_alloc(
+	    session, tmp, &append, &size, WT_UPDATE_STANDARD));
+	append->txnid = WT_TXN_NONE;
+	for (upd = upd_list; upd->next != NULL; upd = upd->next)
+		;
+	WT_PUBLISH(upd->next, append);
+
+	__wt_cache_page_inmem_incr(session, page, size);
 
-	/* Enter the value into the overflow cache. */
-	WT_ERR(__ovfl_read(session, addr, addr_size, tmp));
-	WT_ERR(__wt_ovfl_txnc_add(
-	    session, page, addr, addr_size, tmp->data, tmp->size));
+	*updp = append;
 
 err:	__wt_scr_free(session, &tmp);
 	return (ret);
 }
 
 /*
- * __wt_ovfl_cache --
- *	Handle deletion of an overflow value.
+ * __ovfl_cache --
+ *	Cache an overflow value.
+ */
+static int
+__ovfl_cache(WT_SESSION_IMPL *session,
+    WT_PAGE *page, WT_UPDATE *upd_list, WT_CELL_UNPACK *unpack)
+{
+	WT_OVFL_TRACK *track;
+	WT_UPDATE *upd;
+
+	/* Append a copy of the overflow value to the update list. */
+	WT_RET(__ovfl_cache_append_update(
+	    session, page, upd_list, unpack, &upd));
+
+	/* Allocating tracking structures as necessary. */
+	if (page->modify->ovfl_track == NULL)
+		WT_RET(__wt_ovfl_track_init(session, page));
+	track = page->modify->ovfl_track;
+
+	/* Add the value's information to the update list. */
+	WT_RET(__wt_realloc_def(session,
+	    &track->remove_allocated, track->remove_next + 1, &track->remove));
+	track->remove[track->remove_next].cell = unpack->cell;
+	track->remove[track->remove_next].upd = upd;
+	++track->remove_next;
+
+	return (0);
+}
+
+/*
+ * __wt_ovfl_remove --
+ *	Remove an overflow value.
  */
 int
-__wt_ovfl_cache(WT_SESSION_IMPL *session,
-    WT_PAGE *page, void *cookie, WT_CELL_UNPACK *vpack)
+__wt_ovfl_remove(WT_SESSION_IMPL *session,
+    WT_PAGE *page, WT_UPDATE *upd_list, WT_CELL_UNPACK *unpack)
 {
 	bool visible;
 
@@ -195,10 +253,10 @@ __wt_ovfl_cache(WT_SESSION_IMPL *session,
 	 */
 	switch (page->type) {
 	case WT_PAGE_COL_VAR:
-		visible = __ovfl_cache_col_visible(session, cookie, vpack);
+		visible = __ovfl_cache_col_visible(session, upd_list, unpack);
 		break;
 	case WT_PAGE_ROW_LEAF:
-		visible = __ovfl_cache_row_visible(session, page, cookie);
+		visible = __ovfl_cache_row_visible(session, upd_list);
 		break;
 	WT_ILLEGAL_VALUE(session);
 	}
@@ -207,18 +265,15 @@ __wt_ovfl_cache(WT_SESSION_IMPL *session,
 	 * If there's no globally visible update, there's a reader in the system
 	 * that might try and read the old value, cache it.
 	 */
-	if (!visible) {
-		WT_RET(__ovfl_cache(session, page, vpack));
-		WT_STAT_CONN_INCR(session, cache_overflow_value);
-		WT_STAT_DATA_INCR(session, cache_overflow_value);
-	}
+	if (!visible)
+		WT_RET(__ovfl_cache(session, page, upd_list, unpack));
 
 	/*
 	 * Queue the on-page cell to be set to WT_CELL_VALUE_OVFL_RM and the
 	 * underlying overflow value's blocks to be freed when reconciliation
 	 * completes.
 	 */
-	return (__wt_ovfl_discard_add(session, page, vpack->cell));
+	return (__wt_ovfl_discard_add(session, page, unpack->cell));
 }
 
 /*
diff --git a/src/third_party/wiredtiger/src/btree/bt_page.c b/src/third_party/wiredtiger/src/btree/bt_page.c
index ca5f05fe3dc..d7643e9ad26 100644
--- a/src/third_party/wiredtiger/src/btree/bt_page.c
+++ b/src/third_party/wiredtiger/src/btree/bt_page.c
@@ -116,6 +116,7 @@ err:			if ((pindex = WT_INTL_INDEX_GET_SAFE(page)) != NULL) {
 	__wt_cache_page_inmem_incr(session, page, size);
 	(void)__wt_atomic_add64(&cache->bytes_read, size);
 	(void)__wt_atomic_add64(&cache->pages_inmem, 1);
+	page->cache_create_gen = cache->evict_pass_gen;
 
 	*pagep = page;
 	return (0);
diff --git a/src/third_party/wiredtiger/src/btree/bt_read.c b/src/third_party/wiredtiger/src/btree/bt_read.c
index 5f8cf3a45e1..6a89f505c31 100644
--- a/src/third_party/wiredtiger/src/btree/bt_read.c
+++ b/src/third_party/wiredtiger/src/btree/bt_read.c
@@ -710,7 +710,7 @@ __btree_verbose_lookaside_read(WT_SESSION_IMPL *session)
 		if (__wt_atomic_casv64(&conn->las_verb_gen_read,
 			ckpt_gen_last, ckpt_gen_current)) {
 			__wt_verbose(session, WT_VERB_LOOKASIDE,
-			    "Read from lookaside file triggered.");
+			    "%s", "Read from lookaside file triggered.");
 		}
 	}
 #else
diff --git a/src/third_party/wiredtiger/src/btree/bt_slvg.c b/src/third_party/wiredtiger/src/btree/bt_slvg.c
index eb39301abc7..1f686d904ec 100644
--- a/src/third_party/wiredtiger/src/btree/bt_slvg.c
+++ b/src/third_party/wiredtiger/src/btree/bt_slvg.c
@@ -1885,6 +1885,8 @@ __slvg_row_build_leaf(
 	uint32_t i, skip_start, skip_stop;
 	int cmp;
 
+	WT_UNUSED(ss);					/* !HAVE_VERBOSE */
+
 	btree = S2BT(session);
 	page = NULL;
 
diff --git a/src/third_party/wiredtiger/src/btree/row_srch.c b/src/third_party/wiredtiger/src/btree/row_srch.c
index 76bebde7de7..3a9a6eb0f9b 100644
--- a/src/third_party/wiredtiger/src/btree/row_srch.c
+++ b/src/third_party/wiredtiger/src/btree/row_srch.c
@@ -591,13 +591,13 @@ leaf_match:	cbt->compare = 0;
 	 */
 	if (base == 0) {
 		cbt->compare = 1;
-		cbt->slot = WT_ROW_SLOT(page, page->pg_row);
+		cbt->slot = 0;
 
 		F_SET(cbt, WT_CBT_SEARCH_SMALLEST);
 		ins_head = WT_ROW_INSERT_SMALLEST(page);
 	} else {
 		cbt->compare = -1;
-		cbt->slot = WT_ROW_SLOT(page, page->pg_row + (base - 1));
+		cbt->slot = base - 1;
 
 		ins_head = WT_ROW_INSERT_SLOT(page, cbt->slot);
 	}