WT-3203 bulk-load state changes can race (#3318)

* The bulk-load state change (where multiple threads of control turn off the possibility of bulk-load in a tree), has always been able to race, but it's potentially dangerous now that turning off bulk-load involves calling `__wt_evict_file_exclusive_off()`. In the current tree, there's a diagnostic-only test that might fail because of this race.

* The WT_BTREE_NO_EVICTION flag is no longer set other than through the __wt_evict_file_exclusive_on/off functions; remove that flag and use the WT_BTREE.evict_disabled counter by itself.

2 files changed, 14 insertions, 32 deletions

diff --git a/src/evict/evict_file.c b/src/evict/evict_file.c
index 3bc8fe36e5e..3d8f4a61ca7 100644
--- a/src/evict/evict_file.c
+++ b/src/evict/evict_file.c
@@ -27,7 +27,7 @@ __wt_evict_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop)
 	 * page. Assert eviction has been locked out.
 	 */
 	WT_ASSERT(session,
-	    F_ISSET(btree, WT_BTREE_NO_EVICTION) ||
+	    btree->evict_disabled > 0 ||
 	    !F_ISSET(session->dhandle, WT_DHANDLE_OPEN));
 
 	/*
diff --git a/src/evict/evict_lru.c b/src/evict/evict_lru.c
index e59a6c2f2d9..6863533acfb 100644
--- a/src/evict/evict_lru.c
+++ b/src/evict/evict_lru.c
@@ -824,31 +824,19 @@ __wt_evict_file_exclusive_on(WT_SESSION_IMPL *session)
 	btree = S2BT(session);
 	cache = S2C(session)->cache;
 
-	/*
-	 * Hold the walk lock to set the no-eviction flag.
-	 *
-	 * The no-eviction flag can be set permanently, in which case we never
-	 * increment the no-eviction count.
-	 */
+	/* Hold the walk lock to turn off eviction. */
 	__wt_spin_lock(session, &cache->evict_walk_lock);
-	if (F_ISSET(btree, WT_BTREE_NO_EVICTION)) {
-		if (btree->evict_disabled != 0)
-			++btree->evict_disabled;
+	if (++btree->evict_disabled > 1) {
 		__wt_spin_unlock(session, &cache->evict_walk_lock);
 		return (0);
 	}
-	++btree->evict_disabled;
 
 	/*
 	 * Ensure no new pages from the file will be queued for eviction after
-	 * this point.
+	 * this point, then clear any existing LRU eviction walk for the file.
 	 */
-	F_SET(btree, WT_BTREE_NO_EVICTION);
 	(void)__wt_atomic_addv32(&cache->pass_intr, 1);
-
-	/* Clear any existing LRU eviction walk for the file. */
-	WT_WITH_PASS_LOCK(session,
-	    ret = __evict_clear_walk(session));
+	WT_WITH_PASS_LOCK(session, ret = __evict_clear_walk(session));
 	(void)__wt_atomic_subv32(&cache->pass_intr, 1);
 	WT_ERR(ret);
 
@@ -879,7 +867,6 @@ __wt_evict_file_exclusive_on(WT_SESSION_IMPL *session)
 
 	if (0) {
 err:		--btree->evict_disabled;
-		F_CLR(btree, WT_BTREE_NO_EVICTION);
 	}
 	__wt_spin_unlock(session, &cache->evict_walk_lock);
 	return (ret);
@@ -904,16 +891,11 @@ __wt_evict_file_exclusive_off(WT_SESSION_IMPL *session)
 	 */
 	WT_DIAGNOSTIC_YIELD;
 
-	WT_ASSERT(session,
-	    btree->evict_ref == NULL && F_ISSET(btree, WT_BTREE_NO_EVICTION));
-
-	/*
-	 * The no-eviction flag can be set permanently, in which case we never
-	 * increment the no-eviction count.
-	 */
+	/* Hold the walk lock to turn on eviction. */
 	__wt_spin_lock(session, &cache->evict_walk_lock);
-	if (btree->evict_disabled > 0 && --btree->evict_disabled == 0)
-		F_CLR(btree, WT_BTREE_NO_EVICTION);
+	WT_ASSERT(session,
+	    btree->evict_ref == NULL && btree->evict_disabled > 0);
+	--btree->evict_disabled;
 	__wt_spin_unlock(session, &cache->evict_walk_lock);
 }
 
@@ -1372,7 +1354,7 @@ retry:	while (slot < max_entries) {
 
 		/* Skip files that don't allow eviction. */
 		btree = dhandle->handle;
-		if (F_ISSET(btree, WT_BTREE_NO_EVICTION))
+		if (btree->evict_disabled > 0)
 			continue;
 
 		/*
@@ -1428,9 +1410,9 @@ retry:	while (slot < max_entries) {
 		 * the tree's current eviction point, and part of the process is
 		 * waiting on this thread to acknowledge that action.
 		 */
-		if (!F_ISSET(btree, WT_BTREE_NO_EVICTION) &&
+		if (btree->evict_disabled == 0 &&
 		    !__wt_spin_trylock(session, &cache->evict_walk_lock)) {
-			if (!F_ISSET(btree, WT_BTREE_NO_EVICTION)) {
+			if (btree->evict_disabled == 0) {
 				/*
 				 * Assert the handle has a root page: eviction
 				 * should have been locked out if the tree is
@@ -2249,7 +2231,7 @@ __wt_page_evict_urgent(WT_SESSION_IMPL *session, WT_REF *ref)
 
 	page = ref->page;
 	if (F_ISSET_ATOMIC(page, WT_PAGE_EVICT_LRU) ||
-	    F_ISSET(S2BT(session), WT_BTREE_NO_EVICTION))
+	    S2BT(session)->evict_disabled > 0)
 		return (false);
 
 	/* Append to the urgent queue if we can. */
@@ -2259,7 +2241,7 @@ __wt_page_evict_urgent(WT_SESSION_IMPL *session, WT_REF *ref)
 
 	__wt_spin_lock(session, &cache->evict_queue_lock);
 	if (F_ISSET_ATOMIC(page, WT_PAGE_EVICT_LRU) ||
-	    F_ISSET(S2BT(session), WT_BTREE_NO_EVICTION))
+	    S2BT(session)->evict_disabled > 0)
 		goto done;
 
 	__wt_spin_lock(session, &urgent_queue->evict_lock);