Import wiredtiger: 6173a98979715ed727c432c1a31da64ea8a37048 from branch mongodb-3.6

ref: ff10db8811..6173a98979
for: 3.5.11

WT-3039       Enhance logging so new log files no longer need to wait for the previous log file to be synced
WT-3138       Enhance eviction statistics
WT-3140       Enhance json mode statistics to better support per-dhandle statistics
WT-3310       Add support to WT_SESSION::alter to change table log setting
WT-3329       With a uniform workload and a number of small collections, eviction does a poor job of selecting candidates for eviction.
WT-3381       Improve concurrency in the transaction subsystem
WT-3394       Build WiredTiger with gcc7
WT-3396       test/csuite/wt3363_checkpoint_op_races, test_utility cleanups
WT-3397       Coverity lint
WT-3398       in-memory configurations don't support schema_worker operations
WT-3401       Lint and minor cleanup
WT-3402       Move cached overflow records to the update list.
WT-3403       Automated tests timeout running csuite
WT-3409       WiredTiger generations can silently self-deadlock.
WT-3413       Add more aggressive compile warning flags to Jenkins Windows job
WT-3415       setting timestamp in txn_state requires the global write lock
WT-3421       Compilation error unreachable code in test/csuite/wt3363_checkpoint_op_races
WT-3424       additional gcc 7.1 compile warnings.
WT-3425       workgen: add ability to reopen cursors
WT-3426       Add new wtperf configuration to automated performance testing
WT-3438       Don't tune eviction thread count when the count is fixed

Also explicitly disable v2 log records via a configuration setting in MongoDB -
this will later be plumbed into the feature compatability version mechanism.

2 files changed, 128 insertions, 52 deletions

diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c
index 4f4ecdf286c..1f26949c94f 100644
--- a/src/third_party/wiredtiger/src/evict/evict_lru.c
+++ b/src/third_party/wiredtiger/src/evict/evict_lru.c
@@ -305,12 +305,14 @@ __wt_evict_thread_run(WT_SESSION_IMPL *session, WT_THREAD *thread)
 			    F_ISSET(thread, WT_THREAD_RUN))
 				__wt_yield();
 		else {
-			__wt_verbose(session, WT_VERB_EVICTSERVER, "sleeping");
+			__wt_verbose(session,
+			    WT_VERB_EVICTSERVER, "%s", "sleeping");
 
 			/* Don't rely on signals: check periodically. */
 			__wt_cond_auto_wait(session,
 			    cache->evict_cond, did_work, NULL);
-			__wt_verbose(session, WT_VERB_EVICTSERVER, "waking");
+			__wt_verbose(session,
+			    WT_VERB_EVICTSERVER, "%s", "waking");
 		}
 	} else
 		WT_ERR(__evict_lru_pages(session, false));
@@ -351,8 +353,8 @@ __wt_evict_thread_stop(WT_SESSION_IMPL *session, WT_THREAD *thread)
 	 */
 	WT_ASSERT(session, F_ISSET(conn, WT_CONN_CLOSING | WT_CONN_RECOVERING));
 
-	__wt_verbose(
-	    session, WT_VERB_EVICTSERVER, "cache eviction thread exiting");
+	__wt_verbose(session,
+	    WT_VERB_EVICTSERVER, "%s", "cache eviction thread exiting");
 
 	if (0) {
 err:		WT_PANIC_MSG(session, ret, "cache eviction thread error");
@@ -515,7 +517,7 @@ __wt_evict_destroy(WT_SESSION_IMPL *session)
 	__wt_evict_server_wake(session);
 
 	__wt_verbose(
-	    session, WT_VERB_EVICTSERVER, "waiting for helper threads");
+	    session, WT_VERB_EVICTSERVER, "%s", "waiting for helper threads");
 
 	/*
 	 * We call the destroy function still holding the write lock.
@@ -740,7 +742,7 @@ __evict_pass(WT_SESSION_IMPL *session)
 
 			WT_STAT_CONN_INCR(session, cache_eviction_slow);
 			__wt_verbose(session, WT_VERB_EVICTSERVER,
-			    "unable to reach eviction goal");
+			    "%s", "unable to reach eviction goal");
 			break;
 		} else {
 			if (cache->evict_aggressive_score > 0) {
@@ -779,6 +781,7 @@ __evict_clear_walk(WT_SESSION_IMPL *session)
 		return (0);
 
 	WT_STAT_CONN_INCR(session, cache_eviction_walks_abandoned);
+	WT_STAT_DATA_INCR(session, cache_eviction_walks_abandoned);
 
 	/*
 	 * Clear evict_ref before releasing it in case that forces eviction (we
@@ -957,6 +960,13 @@ __evict_tune_workers(WT_SESSION_IMPL *session)
 	conn = S2C(session);
 	cache = conn->cache;
 
+	/*
+	 * If we have a fixed number of eviction threads, there is no value in
+	 * calculating if we should do any tuning.
+	 */
+       if (conn->evict_threads_max == conn->evict_threads_min)
+		return;
+
 	WT_ASSERT(session, conn->evict_threads.threads[0]->session == session);
 	pgs_evicted_cur = 0;
 
@@ -1106,8 +1116,8 @@ __evict_tune_workers(WT_SESSION_IMPL *session)
 			    &conn->evict_threads, false);
 			WT_STAT_CONN_INCR(session,
 			    cache_eviction_worker_created);
-			__wt_verbose(session, WT_VERB_EVICTSERVER,
-			    "added worker thread");
+			__wt_verbose(session,
+			    WT_VERB_EVICTSERVER, "%s", "added worker thread");
 		}
 		conn->evict_tune_last_action_time = current_time;
 	}
@@ -1641,26 +1651,16 @@ __evict_walk_file(WT_SESSION_IMPL *session,
 	    QUEUE_FILLS_PER_PASS;
 
 	/*
-	 * Randomly walk trees with a small fraction of the cache in case there
-	 * are so many trees that none of them use enough of the cache to be
-	 * allocated slots.
-	 *
-	 * The chance of walking a tree is equal to the chance that a random
-	 * byte in cache belongs to the tree, weighted by how many times we
-	 * want to fill queues during a pass through all the trees in cache.
+	 * Walk trees with a small fraction of the cache in case there are so
+	 * many trees that none of them use enough of the cache to be allocated
+	 * slots.  Only skip a tree if it has no bytes of interest.
 	 */
 	if (target_pages == 0) {
-		if (F_ISSET(cache, WT_CACHE_EVICT_CLEAN)) {
-			btree_inuse = __wt_btree_bytes_evictable(session);
-			cache_inuse = __wt_cache_bytes_inuse(cache);
-		} else {
-			btree_inuse = __wt_btree_dirty_leaf_inuse(session);
-			cache_inuse = __wt_cache_dirty_leaf_inuse(cache);
-		}
-		if (btree_inuse == 0 || cache_inuse == 0)
-			return (0);
-		if (__wt_random64(&session->rnd) % cache_inuse >
-		    btree_inuse * QUEUE_FILLS_PER_PASS)
+		btree_inuse = F_ISSET(cache, WT_CACHE_EVICT_CLEAN) ?
+		    __wt_btree_bytes_evictable(session) :
+		    __wt_btree_dirty_leaf_inuse(session);
+
+		if (btree_inuse == 0)
 			return (0);
 	}
 
@@ -1678,6 +1678,32 @@ __evict_walk_file(WT_SESSION_IMPL *session,
 	if (F_ISSET(session->dhandle, WT_DHANDLE_DEAD) ||
 	    target_pages > remaining_slots)
 		target_pages = remaining_slots;
+
+	/*
+	 * These statistics generate a histogram of the number of pages targeted
+	 * for eviction each round. The range of values here start at
+	 * MIN_PAGES_PER_TREE as this is the smallest number of pages we can
+	 * target, unless there are fewer slots available. The aim is to cover
+	 * the likely ranges of target pages in as few statistics as possible to
+	 * reduce the overall overhead.
+	 */
+	if (target_pages < MIN_PAGES_PER_TREE) {
+		WT_STAT_CONN_INCR(session, cache_eviction_target_page_lt10);
+		WT_STAT_DATA_INCR(session, cache_eviction_target_page_lt10);
+	} else if (target_pages < 32) {
+		WT_STAT_CONN_INCR(session, cache_eviction_target_page_lt32);
+		WT_STAT_DATA_INCR(session, cache_eviction_target_page_lt32);
+	} else if (target_pages < 64) {
+		WT_STAT_CONN_INCR(session, cache_eviction_target_page_lt64);
+		WT_STAT_DATA_INCR(session, cache_eviction_target_page_lt64);
+	} else if (target_pages < 128) {
+		WT_STAT_CONN_INCR(session, cache_eviction_target_page_lt128);
+		WT_STAT_DATA_INCR(session, cache_eviction_target_page_lt128);
+	} else {
+		WT_STAT_CONN_INCR(session, cache_eviction_target_page_ge128);
+		WT_STAT_DATA_INCR(session, cache_eviction_target_page_ge128);
+	}
+
 	end = start + target_pages;
 
 	/*
@@ -1690,6 +1716,14 @@ __evict_walk_file(WT_SESSION_IMPL *session,
 	    !F_ISSET(cache, WT_CACHE_EVICT_CLEAN))
 		min_pages *= 10;
 
+	if (btree->evict_ref == NULL) {
+		WT_STAT_CONN_INCR(session, cache_eviction_walk_from_root);
+		WT_STAT_DATA_INCR(session, cache_eviction_walk_from_root);
+	} else {
+		WT_STAT_CONN_INCR(session, cache_eviction_walk_saved_pos);
+		WT_STAT_DATA_INCR(session, cache_eviction_walk_saved_pos);
+	}
+
 	walk_flags =
 	    WT_READ_CACHE | WT_READ_NO_EVICT | WT_READ_NO_GEN | WT_READ_NO_WAIT;
 
@@ -1760,12 +1794,37 @@ __evict_walk_file(WT_SESSION_IMPL *session,
 			btree->evict_start_type =
 			    (btree->evict_start_type + 1) %
 			    WT_EVICT_WALK_START_NUM;
+			/*
+			 * We differentiate the reasons we gave up on this walk
+			 * and increment the stats accordingly.
+			 */
+			if (pages_queued == 0) {
+				WT_STAT_CONN_INCR(session,
+				    cache_eviction_walks_gave_up_no_targets);
+				WT_STAT_DATA_INCR(session,
+				    cache_eviction_walks_gave_up_no_targets);
+			} else {
+				WT_STAT_CONN_INCR(session,
+				    cache_eviction_walks_gave_up_ratio);
+				WT_STAT_DATA_INCR(session,
+				    cache_eviction_walks_gave_up_ratio);
+			}
 			break;
 		}
 
 		if (ref == NULL) {
-			if (++restarts == 2)
+			WT_STAT_CONN_INCR(
+			    session, cache_eviction_walks_ended);
+			WT_STAT_DATA_INCR(
+			    session, cache_eviction_walks_ended);
+
+			if (++restarts == 2) {
+				WT_STAT_CONN_INCR(
+				    session, cache_eviction_walks_stopped);
+				WT_STAT_DATA_INCR(
+				    session, cache_eviction_walks_stopped);
 				break;
+			}
 			WT_STAT_CONN_INCR(
 			    session, cache_eviction_walks_started);
 			continue;
@@ -1919,6 +1978,9 @@ fast:		/* If the page can't be evicted, give up. */
 
 	WT_STAT_CONN_INCRV(session, cache_eviction_walk, refs_walked);
 	WT_STAT_CONN_INCRV(session, cache_eviction_pages_seen, pages_seen);
+	WT_STAT_DATA_INCRV(session, cache_eviction_pages_seen, pages_seen);
+	WT_STAT_CONN_INCRV(session, cache_eviction_walk_passes, 1);
+	WT_STAT_DATA_INCRV(session, cache_eviction_walk_passes, 1);
 
 	return (0);
 }
diff --git a/src/third_party/wiredtiger/src/evict/evict_stat.c b/src/third_party/wiredtiger/src/evict/evict_stat.c
index 276e737ebbb..63d0aff60f1 100644
--- a/src/third_party/wiredtiger/src/evict/evict_stat.c
+++ b/src/third_party/wiredtiger/src/evict/evict_stat.c
@@ -16,28 +16,30 @@ static void
 __evict_stat_walk(WT_SESSION_IMPL *session)
 {
 	WT_BTREE *btree;
+	WT_CACHE *cache;
 	WT_PAGE *page;
 	WT_REF *next_walk;
-	uint64_t dsk_size, gen_gap, size;
-	uint64_t written_size_cnt, written_size_sum;
-	uint64_t gen_gap_cnt, gen_gap_max, gen_gap_sum;
-	uint64_t max_pagesize, min_written_size;
-	uint64_t num_memory, num_queued, num_not_queueable, num_smaller_allocsz;
-	uint64_t pages_clean, pages_dirty, pages_internal, pages_leaf;
-	uint64_t seen_count, walk_count;
+	uint64_t dsk_size, gen_gap, gen_gap_max, gen_gap_sum, max_pagesize;
+	uint64_t min_written_size, num_memory, num_not_queueable, num_queued;
+	uint64_t num_smaller_allocsz, pages_clean, pages_dirty, pages_internal;
+	uint64_t pages_leaf, seen_count, size, visited_count;
+	uint64_t visited_age_gap_sum, unvisited_count, unvisited_age_gap_sum;
+	uint64_t walk_count, written_size_cnt, written_size_sum;
 
 	btree = S2BT(session);
+	cache = S2C(session)->cache;
 	next_walk = NULL;
-	written_size_cnt = written_size_sum = 0;
-	gen_gap_cnt = gen_gap_max = gen_gap_sum = 0;
-	max_pagesize = 0;
-	num_memory = num_queued = num_not_queueable = num_smaller_allocsz = 0;
-	pages_clean = pages_dirty = pages_internal = pages_leaf = 0;
-	seen_count = walk_count = 0;
+	dsk_size = gen_gap = gen_gap_max = gen_gap_sum = max_pagesize = 0;
+	num_memory = num_not_queueable = num_queued = 0;
+	num_smaller_allocsz = pages_clean = pages_dirty = pages_internal = 0;
+	pages_leaf = seen_count = size = visited_count = 0;
+	visited_age_gap_sum = unvisited_count = unvisited_age_gap_sum = 0;
+	walk_count = written_size_cnt = written_size_sum = 0;
 	min_written_size = UINT64_MAX;
 
 	while (__wt_tree_walk_count(session, &next_walk, &walk_count,
-	    WT_READ_CACHE | WT_READ_NO_EVICT | WT_READ_NO_WAIT) == 0 &&
+	    WT_READ_CACHE | WT_READ_NO_EVICT |
+	    WT_READ_NO_GEN | WT_READ_NO_WAIT) == 0 &&
 	    next_walk != NULL) {
 		++seen_count;
 		page = next_walk->page;
@@ -78,19 +80,29 @@ __evict_stat_walk(WT_SESSION_IMPL *session)
 		if (__wt_ref_is_root(next_walk))
 			continue;
 
-		gen_gap =
-		    S2C(session)->cache->evict_pass_gen - page->evict_pass_gen;
-		if (gen_gap > gen_gap_max)
-			gen_gap_max = gen_gap;
-		gen_gap_sum += gen_gap;
-		++gen_gap_cnt;
+		if (page->evict_pass_gen == 0) {
+			unvisited_age_gap_sum +=
+			    (cache->evict_pass_gen - page->cache_create_gen);
+			++unvisited_count;
+		} else {
+			visited_age_gap_sum +=
+			    (cache->evict_pass_gen - page->cache_create_gen);
+			gen_gap = cache->evict_pass_gen - page->evict_pass_gen;
+			if (gen_gap > gen_gap_max)
+				gen_gap_max = gen_gap;
+			gen_gap_sum += gen_gap;
+			++visited_count;
+		}
 	}
 
+	WT_STAT_DATA_SET(session, cache_state_gen_avg_gap,
+	    visited_count == 0 ? 0 : gen_gap_sum / visited_count);
+	WT_STAT_DATA_SET(session, cache_state_avg_unvisited_age,
+	    unvisited_count == 0 ? 0 : unvisited_age_gap_sum / unvisited_count);
+	WT_STAT_DATA_SET(session, cache_state_avg_visited_age,
+	    visited_count == 0 ? 0 : visited_age_gap_sum / visited_count);
 	WT_STAT_DATA_SET(session, cache_state_avg_written_size,
 	    written_size_cnt == 0 ? 0 : written_size_sum / written_size_cnt);
-	WT_STAT_DATA_SET(session, cache_state_gen_avg_gap,
-	    gen_gap_cnt == 0 ? 0 : gen_gap_sum / gen_gap_cnt);
-
 	WT_STAT_DATA_SET(session, cache_state_gen_max_gap, gen_gap_max);
 	WT_STAT_DATA_SET(session, cache_state_max_pagesize, max_pagesize);
 	WT_STAT_DATA_SET(session,
@@ -98,8 +110,6 @@ __evict_stat_walk(WT_SESSION_IMPL *session)
 	WT_STAT_DATA_SET(session, cache_state_memory, num_memory);
 	WT_STAT_DATA_SET(session, cache_state_queued, num_queued);
 	WT_STAT_DATA_SET(session, cache_state_not_queueable, num_not_queueable);
-	WT_STAT_DATA_SET(session,
-	    cache_state_smaller_alloc_size, num_smaller_allocsz);
 	WT_STAT_DATA_SET(session, cache_state_pages, walk_count);
 	WT_STAT_DATA_SET(session, cache_state_pages_clean, pages_clean);
 	WT_STAT_DATA_SET(session, cache_state_pages_dirty, pages_dirty);
@@ -107,6 +117,10 @@ __evict_stat_walk(WT_SESSION_IMPL *session)
 	WT_STAT_DATA_SET(session, cache_state_pages_leaf, pages_leaf);
 	WT_STAT_DATA_SET(session,
 	    cache_state_refs_skipped, walk_count - seen_count);
+	WT_STAT_DATA_SET(session,
+	    cache_state_smaller_alloc_size, num_smaller_allocsz);
+	WT_STAT_DATA_SET(session,
+	    cache_state_unvisited_count, unvisited_count);
 }
 
 /*