Import wiredtiger: 04447c57d565903849f0797fa391cd60f5fc7992 from branch mongodb-3.6

ref: 629316c271..04447c57d5
for: 3.6.14

Revert "WT-4869 When eviction isn't keeping up, stop adding pages to cache. (#4699)"
Revert "WT-4881 Soften the restrictions on reentering reconciliation. (#4724)"
Revert "WT-4893 Fix a race between internal page child-page eviction checks and cursors in the tree(#4749)"

8 files changed, 54 insertions, 177 deletions

diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index d8c8b0dbb03..b16f253e705 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -1,5 +1,5 @@
 {
-    "commit": "629316c2714322e1fbd26765aa22ba431ce977c1", 
+    "commit": "04447c57d565903849f0797fa391cd60f5fc7992", 
     "github": "wiredtiger/wiredtiger.git", 
     "vendor": "wiredtiger", 
     "branch": "mongodb-3.6"
diff --git a/src/third_party/wiredtiger/src/btree/bt_read.c b/src/third_party/wiredtiger/src/btree/bt_read.c
index 69b0f95d205..bbf60aabc20 100644
--- a/src/third_party/wiredtiger/src/btree/bt_read.c
+++ b/src/third_party/wiredtiger/src/btree/bt_read.c
@@ -689,8 +689,7 @@ read:			/*
 			 * we "acquire" it.
 			 */
 			wont_need = LF_ISSET(WT_READ_WONT_NEED) ||
-			    F_ISSET(session, WT_SESSION_READ_WONT_NEED) ||
-			    F_ISSET(S2C(session)->cache, WT_CACHE_EVICT_NOKEEP);
+			    F_ISSET(session, WT_SESSION_READ_WONT_NEED);
 			continue;
 		case WT_REF_READING:
 			if (LF_ISSET(WT_READ_CACHE))
diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c
index bb310369d82..ff3772533ae 100644
--- a/src/third_party/wiredtiger/src/evict/evict_lru.c
+++ b/src/third_party/wiredtiger/src/evict/evict_lru.c
@@ -559,7 +559,6 @@ __evict_update_work(WT_SESSION_IMPL *session)
 	WT_CONNECTION_IMPL *conn;
 	double dirty_target, dirty_trigger, target, trigger;
 	uint64_t bytes_inuse, bytes_max, dirty_inuse;
-	uint32_t flags;
 
 	conn = S2C(session);
 	cache = conn->cache;
@@ -569,16 +568,14 @@ __evict_update_work(WT_SESSION_IMPL *session)
 	target = cache->eviction_target;
 	trigger = cache->eviction_trigger;
 
-	/* Build up the new state. */
-	flags = 0;
+	/* Clear previous state. */
+	cache->flags = 0;
 
-	if (!F_ISSET(conn, WT_CONN_EVICTION_RUN)) {
-		cache->flags = 0;
+	if (!F_ISSET(conn, WT_CONN_EVICTION_RUN))
 		return (false);
-	}
 
 	if (!__evict_queue_empty(cache->evict_urgent_queue, false))
-		LF_SET(WT_CACHE_EVICT_URGENT);
+		F_SET(cache, WT_CACHE_EVICT_URGENT);
 
 	if (F_ISSET(conn, WT_CONN_LOOKASIDE_OPEN)) {
 		WT_ASSERT(session,
@@ -597,38 +594,32 @@ __evict_update_work(WT_SESSION_IMPL *session)
 	bytes_max = conn->cache_size + 1;
 	bytes_inuse = __wt_cache_bytes_inuse(cache);
 	if (__wt_eviction_clean_needed(session, NULL))
-		LF_SET(WT_CACHE_EVICT_CLEAN | WT_CACHE_EVICT_CLEAN_HARD);
+		F_SET(cache, WT_CACHE_EVICT_CLEAN | WT_CACHE_EVICT_CLEAN_HARD);
 	else if (bytes_inuse > (target * bytes_max) / 100)
-		LF_SET(WT_CACHE_EVICT_CLEAN);
+		F_SET(cache, WT_CACHE_EVICT_CLEAN);
 
 	dirty_inuse = __wt_cache_dirty_leaf_inuse(cache);
 	if (__wt_eviction_dirty_needed(session, NULL))
-		LF_SET(WT_CACHE_EVICT_DIRTY | WT_CACHE_EVICT_DIRTY_HARD);
+		F_SET(cache, WT_CACHE_EVICT_DIRTY | WT_CACHE_EVICT_DIRTY_HARD);
 	else if (dirty_inuse > (uint64_t)(dirty_target * bytes_max) / 100)
-		LF_SET(WT_CACHE_EVICT_DIRTY);
+		F_SET(cache, WT_CACHE_EVICT_DIRTY);
 
 	/*
 	 * If application threads are blocked by the total volume of data in
 	 * cache, try dirty pages as well.
 	 */
 	if (__wt_cache_aggressive(session) &&
-	    LF_ISSET(WT_CACHE_EVICT_CLEAN_HARD))
-		LF_SET(WT_CACHE_EVICT_DIRTY);
-
-	/* When we stop looking for dirty pages, reduce the lookaside score. */
-	if (!LF_ISSET(WT_CACHE_EVICT_DIRTY))
-		__wt_cache_update_lookaside_score(session, 1, 0);
+	    F_ISSET(cache, WT_CACHE_EVICT_CLEAN_HARD))
+		F_SET(cache, WT_CACHE_EVICT_DIRTY);
 
 	/*
 	 * Scrub dirty pages and keep them in cache if we are less than half
 	 * way to the clean or dirty trigger.
 	 */
-	if (bytes_inuse < (uint64_t)((target + trigger) * bytes_max) / 200) {
-		if (dirty_inuse < (uint64_t)
-		    ((dirty_target + dirty_trigger) * bytes_max) / 200)
-			LF_SET(WT_CACHE_EVICT_SCRUB);
-	} else
-		LF_SET(WT_CACHE_EVICT_NOKEEP);
+	if (bytes_inuse < (uint64_t)((target + trigger) * bytes_max) / 200 &&
+	    dirty_inuse <
+	    (uint64_t)((dirty_target + dirty_trigger) * bytes_max) / 200)
+		F_SET(cache, WT_CACHE_EVICT_SCRUB);
 
 	/*
 	 * Try lookaside evict when:
@@ -641,23 +632,20 @@ __evict_update_work(WT_SESSION_IMPL *session)
 	    (__wt_cache_lookaside_score(cache) > 80 &&
 	    dirty_inuse >
 	    (uint64_t)((dirty_target + dirty_trigger) * bytes_max) / 200))
-		LF_SET(WT_CACHE_EVICT_LOOKASIDE);
+		F_SET(cache, WT_CACHE_EVICT_LOOKASIDE);
 
 	/*
 	 * With an in-memory cache, we only do dirty eviction in order to scrub
 	 * pages.
 	 */
 	if (F_ISSET(conn, WT_CONN_IN_MEMORY)) {
-		if (LF_ISSET(WT_CACHE_EVICT_CLEAN))
-			LF_SET(WT_CACHE_EVICT_DIRTY);
-		if (LF_ISSET(WT_CACHE_EVICT_CLEAN_HARD))
-			LF_SET(WT_CACHE_EVICT_DIRTY_HARD);
-		LF_CLR(WT_CACHE_EVICT_CLEAN | WT_CACHE_EVICT_CLEAN_HARD);
+		if (F_ISSET(cache, WT_CACHE_EVICT_CLEAN))
+			F_SET(cache, WT_CACHE_EVICT_DIRTY);
+		if (F_ISSET(cache, WT_CACHE_EVICT_CLEAN_HARD))
+			F_SET(cache, WT_CACHE_EVICT_DIRTY_HARD);
+		F_CLR(cache, WT_CACHE_EVICT_CLEAN | WT_CACHE_EVICT_CLEAN_HARD);
 	}
 
-	/* Update the global eviction state. */
-	cache->flags = flags;
-
 	return (F_ISSET(cache, WT_CACHE_EVICT_ALL | WT_CACHE_EVICT_URGENT));
 }
 
diff --git a/src/third_party/wiredtiger/src/evict/evict_page.c b/src/third_party/wiredtiger/src/evict/evict_page.c
index a3fbe66b4a9..44c3bbb8f78 100644
--- a/src/third_party/wiredtiger/src/evict/evict_page.c
+++ b/src/third_party/wiredtiger/src/evict/evict_page.c
@@ -461,46 +461,11 @@ __evict_child_check(WT_SESSION_IMPL *session, WT_REF *parent)
 	WT_REF *child;
 	bool active;
 
-	/*
-	 * There may be cursors in the tree walking the list of child pages.
-	 * The parent is locked, so all we care about is cursors already in the
-	 * child pages, no thread can enter them. Any cursor moving through the
-	 * child pages must be hazard pointer coupling between pages, where the
-	 * page on which it currently has a hazard pointer must be in a state
-	 * other than on-disk. Walk the child list forward, then backward, to
-	 * ensure we don't race with a cursor walking in the opposite direction
-	 * from our check.
-	 */
-	WT_INTL_FOREACH_BEGIN(session, parent->page, child) {
-		switch (child->state) {
-		case WT_REF_DISK:		/* On-disk */
-		case WT_REF_DELETED:		/* On-disk, deleted */
-		case WT_REF_LOOKASIDE:		/* On-disk, lookaside */
-			break;
-		default:
-			return (__wt_set_return(session, EBUSY));
-		}
-	} WT_INTL_FOREACH_END;
-	WT_INTL_FOREACH_REVERSE_BEGIN(session, parent->page, child) {
-		switch (child->state) {
-		case WT_REF_DISK:		/* On-disk */
-		case WT_REF_DELETED:		/* On-disk, deleted */
-		case WT_REF_LOOKASIDE:		/* On-disk, lookaside */
-			break;
-		default:
-			return (__wt_set_return(session, EBUSY));
-		}
-	} WT_INTL_FOREACH_END;
-
-	/*
-	 * The fast check is done and there are no cursors in the child pages.
-	 * Make sure the child WT_REF structures pages can be discarded.
-	 */
 	WT_INTL_FOREACH_BEGIN(session, parent->page, child) {
 		switch (child->state) {
 		case WT_REF_DISK:		/* On-disk */
 			break;
-		case WT_REF_DELETED:		/* On-disk, deleted */
+		case WT_REF_DELETED:		/* Deleted */
 			/*
 			 * If the child page was part of a truncate,
 			 * transaction rollback might switch this page into its
@@ -524,7 +489,7 @@ __evict_child_check(WT_SESSION_IMPL *session, WT_REF *parent)
 			if (active)
 				return (__wt_set_return(session, EBUSY));
 			break;
-		case WT_REF_LOOKASIDE:		/* On-disk, lookaside */
+		case WT_REF_LOOKASIDE:
 			/*
 			 * If the lookaside history is obsolete, the reference
 			 * can be ignored.
diff --git a/src/third_party/wiredtiger/src/include/btmem.h b/src/third_party/wiredtiger/src/include/btmem.h
index 27d8217061c..dcfb59d2ce9 100644
--- a/src/third_party/wiredtiger/src/include/btmem.h
+++ b/src/third_party/wiredtiger/src/include/btmem.h
@@ -577,14 +577,6 @@ struct __wt_page {
 	for (__refp = __pindex->index,					\
 	    __entries = __pindex->entries; __entries > 0; --__entries) {\
 		(ref) = *__refp++;
-#define	WT_INTL_FOREACH_REVERSE_BEGIN(session, page, ref) do {		\
-	WT_PAGE_INDEX *__pindex;					\
-	WT_REF **__refp;						\
-	uint32_t __entries;						\
-	WT_INTL_INDEX_GET(session, page, __pindex);			\
-	for (__refp = __pindex->index + __pindex->entries,		\
-	    __entries = __pindex->entries; __entries > 0; --__entries) {\
-		(ref) = *--__refp;
 #define	WT_INTL_FOREACH_END						\
 	}								\
 } while (0)
diff --git a/src/third_party/wiredtiger/src/include/cache.h b/src/third_party/wiredtiger/src/include/cache.h
index b0620091a23..9e849bf4d7f 100644
--- a/src/third_party/wiredtiger/src/include/cache.h
+++ b/src/third_party/wiredtiger/src/include/cache.h
@@ -249,14 +249,13 @@ struct __wt_cache {
 	uint32_t pool_flags;		/* Cache pool flags */
 
 /* AUTOMATIC FLAG VALUE GENERATION START */
-#define	WT_CACHE_EVICT_CLEAN	  0x01u /* Evict clean pages */
-#define	WT_CACHE_EVICT_CLEAN_HARD 0x02u /* Clean % blocking app threads */
-#define	WT_CACHE_EVICT_DIRTY	  0x04u /* Evict dirty pages */
-#define	WT_CACHE_EVICT_DIRTY_HARD 0x08u /* Dirty % blocking app threads */
-#define	WT_CACHE_EVICT_LOOKASIDE  0x10u /* Try lookaside eviction */
-#define	WT_CACHE_EVICT_NOKEEP	  0x20u /* Don't add read pages to cache */
-#define	WT_CACHE_EVICT_SCRUB	  0x40u /* Scrub dirty pages */
-#define	WT_CACHE_EVICT_URGENT	  0x80u /* Pages are in the urgent queue */
+#define	WT_CACHE_EVICT_CLEAN	  0x01u	/* Evict clean pages */
+#define	WT_CACHE_EVICT_CLEAN_HARD 0x02u	/* Clean % blocking app threads */
+#define	WT_CACHE_EVICT_DIRTY	  0x04u	/* Evict dirty pages */
+#define	WT_CACHE_EVICT_DIRTY_HARD 0x08u	/* Dirty % blocking app threads */
+#define	WT_CACHE_EVICT_LOOKASIDE  0x10u	/* Try lookaside eviction */
+#define	WT_CACHE_EVICT_SCRUB	  0x20u	/* Scrub dirty pages */
+#define	WT_CACHE_EVICT_URGENT	  0x40u	/* Pages are in the urgent queue */
 /* AUTOMATIC FLAG VALUE GENERATION STOP */
 #define	WT_CACHE_EVICT_ALL	(WT_CACHE_EVICT_CLEAN | WT_CACHE_EVICT_DIRTY)
 	uint32_t flags;
diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c
index 6ee0f45e808..37b0581af6b 100644
--- a/src/third_party/wiredtiger/src/reconcile/rec_write.c
+++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c
@@ -340,8 +340,6 @@ static void __rec_write_page_status(WT_SESSION_IMPL *, WT_RECONCILE *);
 static int  __rec_write_wrapup(WT_SESSION_IMPL *, WT_RECONCILE *, WT_PAGE *);
 static int  __rec_write_wrapup_err(
 		WT_SESSION_IMPL *, WT_RECONCILE *, WT_PAGE *);
-static int __reconcile(WT_SESSION_IMPL *,
-		WT_REF *, WT_SALVAGE_COOKIE *, uint32_t, bool *, bool *);
 
 static void __rec_dictionary_free(WT_SESSION_IMPL *, WT_RECONCILE *);
 static int  __rec_dictionary_init(WT_SESSION_IMPL *, WT_RECONCILE *, u_int);
@@ -357,15 +355,19 @@ int
 __wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref,
     WT_SALVAGE_COOKIE *salvage, uint32_t flags, bool *lookaside_retryp)
 {
+	WT_BTREE *btree;
 	WT_DECL_RET;
 	WT_PAGE *page;
-	bool no_reconcile_set, page_locked;
+	WT_PAGE_MODIFY *mod;
+	WT_RECONCILE *r;
+	uint64_t oldest_id;
 
+	btree = S2BT(session);
+	page = ref->page;
+	mod = page->modify;
 	if (lookaside_retryp != NULL)
 		*lookaside_retryp = false;
 
-	page = ref->page;
-
 	__wt_verbose(session, WT_VERB_RECONCILE,
 	    "%p reconcile %s (%s%s%s)",
 	    (void *)ref, __wt_page_type_string(page->type),
@@ -394,19 +396,10 @@ __wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref,
 	    LF_ISSET(WT_REC_VISIBLE_ALL) ||
 	    F_ISSET(&session->txn, WT_TXN_HAS_SNAPSHOT));
 
-	/* It's an error to be called with a clean page. */
+	/* We shouldn't get called with a clean page, that's an error. */
 	WT_ASSERT(session, __wt_page_is_modified(page));
 
 	/*
-	 * Reconciliation acquires and releases pages, and in rare cases that
-	 * page release triggers eviction. If the page is dirty, eviction can
-	 * trigger reconciliation, and we re-enter this code. Reconciliation
-	 * isn't re-entrant, so we need to ensure that doesn't happen.
-	 */
-	no_reconcile_set = F_ISSET(session, WT_SESSION_NO_RECONCILE);
-	F_SET(session, WT_SESSION_NO_RECONCILE);
-
-	/*
 	 * Reconciliation locks the page for three reasons:
 	 *    Reconciliation reads the lists of page updates, obsolete updates
 	 * cannot be discarded while reconciliation is in progress;
@@ -416,7 +409,6 @@ __wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref,
 	 * a child page splitting during the reconciliation.
 	 */
 	WT_PAGE_LOCK(session, page);
-	page_locked = true;
 
 	/*
 	 * Now that the page is locked, if attempting to evict it, check again
@@ -424,37 +416,20 @@ __wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref,
 	 * while we were waiting to acquire the lock (e.g., the page could have
 	 * split).
 	 */
-	if (LF_ISSET(WT_REC_EVICT) && !__wt_page_can_evict(session, ref, NULL))
-		WT_ERR(__wt_set_return(session, EBUSY));
-
-	/*
-	 * Reconcile the page. The reconciliation code unlocks the page as soon
-	 * as possible, and returns that information.
-	 */
-	ret = __reconcile(session, ref,
-	    salvage, flags, lookaside_retryp, &page_locked);
-
-err:
-	if (page_locked)
+	if (LF_ISSET(WT_REC_EVICT) &&
+	    !__wt_page_can_evict(session, ref, NULL)) {
 		WT_PAGE_UNLOCK(session, page);
-	if (!no_reconcile_set)
-		F_CLR(session, WT_SESSION_NO_RECONCILE);
-	return (ret);
-}
+		return (__wt_set_return(session, EBUSY));
+	}
 
-/*
- * __reconcile_save_evict_state --
- *	Save the transaction state that causes history to be pinned, whether
- * reconciliation succeeds or fails.
- */
-static void
-__reconcile_save_evict_state(
-    WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags)
-{
-	WT_PAGE_MODIFY *mod;
-	uint64_t oldest_id;
+	/* Initialize the reconciliation structure for each new run. */
+	if ((ret = __rec_init(
+	    session, ref, flags, salvage, &session->reconcile)) != 0) {
+		WT_PAGE_UNLOCK(session, page);
+		return (ret);
+	}
+	r = session->reconcile;
 
-	mod = ref->page->modify;
 	oldest_id = __wt_txn_oldest_id(session);
 
 	/*
@@ -482,36 +457,6 @@ __reconcile_save_evict_state(
 	WT_ASSERT(session, WT_TXNID_LE(mod->last_oldest_id, oldest_id));
 	mod->last_oldest_id = oldest_id;
 #endif
-}
-
-/*
- * __reconcile --
- *	Reconcile an in-memory page into its on-disk format, and write it.
- */
-static int
-__reconcile(WT_SESSION_IMPL *session, WT_REF *ref, WT_SALVAGE_COOKIE *salvage,
-    uint32_t flags, bool *lookaside_retryp, bool *page_lockedp)
-{
-	WT_BTREE *btree;
-	WT_DECL_RET;
-	WT_PAGE *page;
-#ifdef HAVE_TIMESTAMPS
-	WT_PAGE_MODIFY *mod;
-#endif
-	WT_RECONCILE *r;
-
-	btree = S2BT(session);
-	page = ref->page;
-
-#ifdef HAVE_TIMESTAMPS
-	mod = page->modify;
-#endif
-	/* Save the eviction state. */
-	__reconcile_save_evict_state(session, ref, flags);
-
-	/* Initialize the reconciliation structure for each new run. */
-	WT_RET(__rec_init(session, ref, flags, salvage, &session->reconcile));
-	r = session->reconcile;
 
 	/* Reconcile the page. */
 	switch (page->type) {
@@ -570,7 +515,6 @@ __reconcile(WT_SESSION_IMPL *session, WT_REF *ref, WT_SALVAGE_COOKIE *salvage,
 #endif
 
 	/* Release the reconciliation lock. */
-	*page_lockedp = false;
 	WT_PAGE_UNLOCK(session, page);
 
 	/* Update statistics. */
@@ -960,16 +904,7 @@ __rec_init(WT_SESSION_IMPL *session,
 	btree = S2BT(session);
 	page = ref->page;
 
-	/*
-	 * Reconciliation is not re-entrant, make sure that doesn't happen. Our
-	 * caller sets WT_SESSION_IMPL.WT_SESSION_NO_RECONCILE to prevent it,
-	 * but it's been a problem in the past, check to be sure.
-	 */
-	r = *(WT_RECONCILE **)reconcilep;
-	if (r != NULL && r->ref != NULL)
-		WT_RET_MSG(session, WT_ERROR, "reconciliation re-entered");
-
-	if (r == NULL) {
+	if ((r = *(WT_RECONCILE **)reconcilep) == NULL) {
 		WT_RET(__wt_calloc_one(session, &r));
 
 		*(WT_RECONCILE **)reconcilep = r;
@@ -984,6 +919,9 @@ __rec_init(WT_SESSION_IMPL *session,
 		F_SET(&r->chunkB.image, WT_ITEM_ALIGNED);
 	}
 
+	/* Reconciliation is not re-entrant, make sure that doesn't happen. */
+	WT_ASSERT(session, r->ref == NULL);
+
 	/* Remember the configuration. */
 	r->ref = ref;
 	r->page = page;
diff --git a/src/third_party/wiredtiger/src/support/hazard.c b/src/third_party/wiredtiger/src/support/hazard.c
index 1c6487ef07f..eb65c00741c 100644
--- a/src/third_party/wiredtiger/src/support/hazard.c
+++ b/src/third_party/wiredtiger/src/support/hazard.c
@@ -329,10 +329,6 @@ __wt_hazard_check(WT_SESSION_IMPL *session, WT_REF *ref)
 	WT_SESSION_IMPL *s;
 	uint32_t i, j, hazard_inuse, max, session_cnt, walk_cnt;
 
-	/* If a file can never be evicted, hazard pointers aren't required. */
-	if (F_ISSET(S2BT(session), WT_BTREE_IN_MEMORY))
-		return (NULL);
-
 	conn = S2C(session);
 
 	WT_STAT_CONN_INCR(session, cache_hazard_checks);