WT-3115 Convert the dhandle list lock into a read/write lock. (#3236)

It was a spinlock, but most acquirers only need shared access and it can be a contention point in many-table workloads.

Split uses of the handle list lock into small operations. In particular, only hold the handle list lock to get the "next" handle, not for loops over all the handles in the system.

Update statistics around handle list lock and corresponding doc.

5 files changed, 41 insertions, 46 deletions

diff --git a/src/lsm/lsm_cursor.c b/src/lsm/lsm_cursor.c
index a2511f48e2b..60afbc99ade 100644
--- a/src/lsm/lsm_cursor.c
+++ b/src/lsm/lsm_cursor.c
@@ -1692,8 +1692,8 @@ __wt_clsm_open(WT_SESSION_IMPL *session,
 	bulk = cval.val != 0;
 
 	/* Get the LSM tree. */
-	WT_WITH_HANDLE_LIST_LOCK(session,
-	    ret = __wt_lsm_tree_get(session, uri, bulk, &lsm_tree));
+	ret = __wt_lsm_tree_get(session, uri, bulk, &lsm_tree);
+
 	/*
 	 * Check whether the exclusive open for a bulk load succeeded, and
 	 * if it did ensure that it's safe to bulk load into the tree.
diff --git a/src/lsm/lsm_manager.c b/src/lsm/lsm_manager.c
index cbd83a5cd30..6dc06146179 100644
--- a/src/lsm/lsm_manager.c
+++ b/src/lsm/lsm_manager.c
@@ -387,8 +387,8 @@ __lsm_manager_run_server(WT_SESSION_IMPL *session)
 		__wt_sleep(0, 10000);
 		if (TAILQ_EMPTY(&conn->lsmqh))
 			continue;
-		__wt_spin_lock(session, &conn->dhandle_lock);
-		F_SET(session, WT_SESSION_LOCKED_HANDLE_LIST);
+		__wt_readlock(session, &conn->dhandle_lock);
+		F_SET(session, WT_SESSION_LOCKED_HANDLE_LIST_READ);
 		dhandle_locked = true;
 		TAILQ_FOREACH(lsm_tree, &S2C(session)->lsmqh, q) {
 			if (!lsm_tree->active)
@@ -448,14 +448,14 @@ __lsm_manager_run_server(WT_SESSION_IMPL *session)
 				    session, WT_LSM_WORK_MERGE, 0, lsm_tree));
 			}
 		}
-		__wt_spin_unlock(session, &conn->dhandle_lock);
-		F_CLR(session, WT_SESSION_LOCKED_HANDLE_LIST);
+		__wt_readunlock(session, &conn->dhandle_lock);
+		F_CLR(session, WT_SESSION_LOCKED_HANDLE_LIST_READ);
 		dhandle_locked = false;
 	}
 
 err:	if (dhandle_locked) {
-		__wt_spin_unlock(session, &conn->dhandle_lock);
-		F_CLR(session, WT_SESSION_LOCKED_HANDLE_LIST);
+		__wt_readunlock(session, &conn->dhandle_lock);
+		F_CLR(session, WT_SESSION_LOCKED_HANDLE_LIST_READ);
 	}
 	return (ret);
 }
diff --git a/src/lsm/lsm_stat.c b/src/lsm/lsm_stat.c
index 150de968722..21e8991be94 100644
--- a/src/lsm/lsm_stat.c
+++ b/src/lsm/lsm_stat.c
@@ -33,9 +33,7 @@ __curstat_lsm_init(
 	   "checkpoint=" WT_CHECKPOINT, NULL, NULL };
 
 	locked = false;
-	WT_WITH_HANDLE_LIST_LOCK(session,
-	    ret = __wt_lsm_tree_get(session, uri, false, &lsm_tree));
-	WT_RET(ret);
+	WT_RET(__wt_lsm_tree_get(session, uri, false, &lsm_tree));
 	WT_ERR(__wt_scr_alloc(session, 0, &uribuf));
 
 	/* Propagate all, fast and/or clear to the cursors we open. */
diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c
index 71a981a6284..a9275976023 100644
--- a/src/lsm/lsm_tree.c
+++ b/src/lsm/lsm_tree.c
@@ -38,7 +38,7 @@ __lsm_tree_discard(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, bool final)
 	/* We may be destroying an lsm_tree before it was added. */
 	if (F_ISSET(lsm_tree, WT_LSM_TREE_OPEN)) {
 		WT_ASSERT(session, final ||
-		    F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST));
+		    F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST_WRITE));
 		TAILQ_REMOVE(&S2C(session)->lsmqh, lsm_tree, q);
 	}
 
@@ -321,9 +321,7 @@ __wt_lsm_tree_create(WT_SESSION_IMPL *session,
 	metadata = NULL;
 
 	/* If the tree can be opened, it already exists. */
-	WT_WITH_HANDLE_LIST_LOCK(session,
-	    ret = __wt_lsm_tree_get(session, uri, false, &lsm_tree));
-	if (ret == 0) {
+	if ((ret = __wt_lsm_tree_get(session, uri, false, &lsm_tree)) == 0) {
 		__wt_lsm_tree_release(session, lsm_tree);
 		return (exclusive ? EEXIST : 0);
 	}
@@ -339,7 +337,7 @@ __wt_lsm_tree_create(WT_SESSION_IMPL *session,
 	 * error: the returned handle is NULL on error, and the metadata
 	 * tracking macros handle cleaning up on failure.
 	 */
-	WT_WITH_HANDLE_LIST_LOCK(session,
+	WT_WITH_HANDLE_LIST_WRITE_LOCK(session,
 	    ret = __lsm_tree_open(session, uri, true, &lsm_tree));
 	if (ret == 0)
 		__wt_lsm_tree_release(session, lsm_tree);
@@ -404,6 +402,9 @@ __lsm_tree_find(WT_SESSION_IMPL *session,
 			}
 
 			*treep = lsm_tree;
+
+			WT_ASSERT(session, lsm_tree->excl_session ==
+			     (exclusive ? session : NULL));
 			return (0);
 		}
 
@@ -456,7 +457,8 @@ __lsm_tree_open(WT_SESSION_IMPL *session,
 	conn = S2C(session);
 	lsm_tree = NULL;
 
-	WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST));
+	WT_ASSERT(session,
+	    F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST_WRITE));
 
 	/* Start the LSM manager thread if it isn't running. */
 	if (__wt_atomic_cas32(&conn->lsm_manager.lsm_workers, 0, 1))
@@ -520,14 +522,21 @@ __wt_lsm_tree_get(WT_SESSION_IMPL *session,
 {
 	WT_DECL_RET;
 
-	WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST));
-
-	ret = __lsm_tree_find(session, uri, exclusive, treep);
+	/*
+	 * Dropping and re-acquiring the lock is safe here, since the tree open
+	 * call checks to see if another thread beat it to opening the tree
+	 * before proceeding.
+	 */
+	if (exclusive)
+		WT_WITH_HANDLE_LIST_WRITE_LOCK(session,
+		    ret = __lsm_tree_find(session, uri, exclusive, treep));
+	else
+		WT_WITH_HANDLE_LIST_READ_LOCK(session,
+		    ret = __lsm_tree_find(session, uri, exclusive, treep));
 	if (ret == WT_NOTFOUND)
-		ret = __lsm_tree_open(session, uri, exclusive, treep);
+		WT_WITH_HANDLE_LIST_WRITE_LOCK(session,
+		    ret = __lsm_tree_open(session, uri, exclusive, treep));
 
-	WT_ASSERT(session, ret != 0 ||
-	     (*treep)->excl_session == (exclusive ? session : NULL));
 	return (ret);
 }
 
@@ -857,9 +866,7 @@ __wt_lsm_tree_alter(
 	locked = false;
 
 	/* Get the LSM tree. */
-	WT_WITH_HANDLE_LIST_LOCK(session,
-	    ret = __wt_lsm_tree_get(session, uri, false, &lsm_tree));
-	WT_RET(ret);
+	WT_RET(__wt_lsm_tree_get(session, uri, false, &lsm_tree));
 
 	/* Prevent any new opens. */
 	__wt_lsm_tree_writelock(session, lsm_tree);
@@ -899,9 +906,7 @@ __wt_lsm_tree_drop(
 	locked = false;
 
 	/* Get the LSM tree. */
-	WT_WITH_HANDLE_LIST_LOCK(session,
-	    ret = __wt_lsm_tree_get(session, name, true, &lsm_tree));
-	WT_RET(ret);
+	WT_RET(__wt_lsm_tree_get(session, name, true, &lsm_tree));
 	WT_ASSERT(session, !lsm_tree->active);
 
 	/* Prevent any new opens. */
@@ -934,7 +939,7 @@ __wt_lsm_tree_drop(
 	WT_ASSERT(session, !lsm_tree->active);
 err:	if (locked)
 		__wt_lsm_tree_writeunlock(session, lsm_tree);
-	WT_WITH_HANDLE_LIST_LOCK(session,
+	WT_WITH_HANDLE_LIST_WRITE_LOCK(session,
 	    tret = __lsm_tree_discard(session, lsm_tree, false));
 	WT_TRET(tret);
 	return (ret);
@@ -960,9 +965,7 @@ __wt_lsm_tree_rename(WT_SESSION_IMPL *session,
 	locked = false;
 
 	/* Get the LSM tree. */
-	WT_WITH_HANDLE_LIST_LOCK(session,
-	    ret = __wt_lsm_tree_get(session, olduri, true, &lsm_tree));
-	WT_RET(ret);
+	WT_RET(__wt_lsm_tree_get(session, olduri, true, &lsm_tree));
 
 	/* Prevent any new opens. */
 	__wt_lsm_tree_writelock(session, lsm_tree);
@@ -1007,7 +1010,7 @@ err:	if (locked)
 	 * Discard this LSM tree structure. The first operation on the renamed
 	 * tree will create a new one.
 	 */
-	WT_WITH_HANDLE_LIST_LOCK(session,
+	WT_WITH_HANDLE_LIST_WRITE_LOCK(session,
 	    tret = __lsm_tree_discard(session, lsm_tree, false));
 	WT_TRET(tret);
 	return (ret);
@@ -1032,9 +1035,7 @@ __wt_lsm_tree_truncate(
 	locked = false;
 
 	/* Get the LSM tree. */
-	WT_WITH_HANDLE_LIST_LOCK(session,
-	    ret = __wt_lsm_tree_get(session, name, true, &lsm_tree));
-	WT_RET(ret);
+	WT_RET(__wt_lsm_tree_get(session, name, true, &lsm_tree));
 
 	/* Prevent any new opens. */
 	__wt_lsm_tree_writelock(session, lsm_tree);
@@ -1068,7 +1069,7 @@ err:	if (locked)
 		 * the last good version of the metadata will be used, resulting
 		 * in a valid (not truncated) tree.
 		 */
-		WT_WITH_HANDLE_LIST_LOCK(session,
+		WT_WITH_HANDLE_LIST_WRITE_LOCK(session,
 		    tret = __lsm_tree_discard(session, lsm_tree, false));
 		WT_TRET(tret);
 	}
@@ -1157,9 +1158,7 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, bool *skipp)
 	/* Tell __wt_schema_worker not to look inside the LSM tree. */
 	*skipp = true;
 
-	WT_WITH_HANDLE_LIST_LOCK(session,
-	    ret = __wt_lsm_tree_get(session, name, false, &lsm_tree));
-	WT_RET(ret);
+	WT_RET(__wt_lsm_tree_get(session, name, false, &lsm_tree));
 
 	if (!F_ISSET(S2C(session), WT_CONN_LSM_MERGE))
 		WT_ERR_MSG(session, EINVAL,
@@ -1356,9 +1355,7 @@ __wt_lsm_tree_worker(WT_SESSION_IMPL *session,
 
 	locked = false;
 	exclusive = FLD_ISSET(open_flags, WT_DHANDLE_EXCLUSIVE);
-	WT_WITH_HANDLE_LIST_LOCK(session,
-	    ret = __wt_lsm_tree_get(session, uri, exclusive, &lsm_tree));
-	WT_RET(ret);
+	WT_RET(__wt_lsm_tree_get(session, uri, exclusive, &lsm_tree));
 
 	/*
 	 * We mark that we're busy using the tree to coordinate
diff --git a/src/lsm/lsm_work_unit.c b/src/lsm/lsm_work_unit.c
index d9c185a3f58..4349acf7b55 100644
--- a/src/lsm/lsm_work_unit.c
+++ b/src/lsm/lsm_work_unit.c
@@ -276,7 +276,7 @@ __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session,
 	if (F_ISSET(chunk, WT_LSM_CHUNK_ONDISK) &&
 	    !F_ISSET(chunk, WT_LSM_CHUNK_STABLE) &&
 	    !chunk->evicted) {
-		WT_WITH_HANDLE_LIST_LOCK(session,
+		WT_WITH_HANDLE_LIST_WRITE_LOCK(session,
 		    ret = __lsm_discard_handle(session, chunk->uri, NULL));
 		if (ret == 0)
 			chunk->evicted = 1;
@@ -517,7 +517,7 @@ __lsm_drop_file(WT_SESSION_IMPL *session, const char *uri)
 	 *
 	 * This will fail with EBUSY if the file is still in use.
 	 */
-	WT_WITH_HANDLE_LIST_LOCK(session,
+	WT_WITH_HANDLE_LIST_WRITE_LOCK(session,
 	   ret = __lsm_discard_handle(session, uri, WT_CHECKPOINT));
 	WT_RET(ret);