Only reopen handles when exclusive access is required, leave them closed on the

list so that any future accesses open them with whatever flags are required.
Spin trying to get exclusive access to a handle in the open path: we don't want block waiting on open cursors if we race to open the file.

2 files changed, 70 insertions, 38 deletions

diff --git a/src/conn/conn_btree.c b/src/conn/conn_btree.c
index 556905644b7..e12dfcc9e18 100644
--- a/src/conn/conn_btree.c
+++ b/src/conn/conn_btree.c
@@ -46,23 +46,46 @@ __wt_conn_btree_open(WT_SESSION_IMPL *session,
 	if (matched) {
 		__wt_spin_unlock(session, &conn->spinlock);
 
-		/* Check that the handle is open. */
-		__wt_readlock(session, btree->rwlock);
-		matched = F_ISSET(btree, WT_BTREE_OPEN) ? 1 : 0;
-		__wt_rwunlock(session, btree->rwlock);
-
-		if (!matched) {
-			__wt_writelock(session, btree->rwlock);
-			if (!F_ISSET(btree, WT_BTREE_OPEN)) {
-				/* We're going to overwrite the old config. */
-				__wt_free(session, btree->config);
+		/*
+		 * Check that the handle is open.  We've already incremented
+		 * the reference count, so once the handle is open it won't be
+		 * closed by another thread.
+		 *
+		 * If we can see the WT_BTREE_OPEN flag set while holding a
+		 * lock on the handle, then it's really open and we can start
+		 * using it.  Alternatively, if we can get an exclusive lock
+		 * and WT_BTREE_OPEN is still not set, we need to do the open.
+		 */
+		for (;;) {
+			__wt_readlock(session, btree->rwlock);
+			if (F_ISSET(btree, WT_BTREE_OPEN))
+				break;
+
+			/*
+			 * Try to upgrade to an exclusive lock.  There is some
+			 * subtlety here: if we race with another thread that
+			 * successfully opens the file, we don't want to block
+			 * waiting to get exclusive access.
+			 */
+			__wt_rwunlock(session, btree->rwlock);
+			if (__wt_try_writelock(session, btree->rwlock) == 0) {
+				/* Was it opened while we waited? */
+				if (F_ISSET(btree, WT_BTREE_OPEN))
+					break;
+
+				/*
+				 * We've got the exclusive handle lock, it's
+				 * our job to open the file.
+				 */
 				goto conf;
 			}
 
-			/* It was opened while we waited. */
-			__wt_rwunlock(session, btree->rwlock);
+			/* Give other threads a chance to make progress. */
+			__wt_yield();
 		}
 
+		__wt_rwunlock(session, btree->rwlock);
+
 		/* The config string will not be needed: free it now. */
 		__wt_free(session, config);
 
@@ -112,6 +135,8 @@ __wt_conn_btree_open(WT_SESSION_IMPL *session,
 
 	/* Open the underlying file. */
 conf:	session->btree = btree;
+	/* Free any old config. */
+	__wt_free(session, btree->config);
 	btree->config = config;
 
 	ret = __wt_btree_open(session, cfg, flags);
@@ -139,16 +164,17 @@ __wt_conn_btree_close(WT_SESSION_IMPL *session, int locked)
 	conn = S2C(session);
 	ret = 0;
 
-	if (F_ISSET(btree, WT_BTREE_OPEN))
+	if (F_ISSET(btree, WT_BTREE_OPEN)) {
 		WT_STAT_DECR(conn->stats, file_open);
 
-	/*
-	 * If it looks like we are the last reference, sync the file.  This
-	 * should make the close call fast (while we are holding an exclusive
-	 * lock on the handle).
-	 */
-	if (btree->refcnt == 1)
-		WT_RET(__wt_btree_sync(session, NULL));
+		/*
+		 * If it looks like we are the last reference, sync the file.
+		 * This should make the close call fast (while we are holding
+		 * an exclusive lock on the handle).
+		 */
+		if (btree->refcnt == 1)
+			WT_RET(__wt_btree_sync(session, NULL));
+	}
 
 	/*
 	 * Decrement the reference count.  If we really are the last reference,
@@ -161,8 +187,10 @@ __wt_conn_btree_close(WT_SESSION_IMPL *session, int locked)
 	__wt_spin_unlock(session, &conn->spinlock);
 
 	if (!inuse) {
-		ret = __wt_btree_close(session);
-		F_CLR(btree, WT_BTREE_OPEN);
+		if (F_ISSET(btree, WT_BTREE_OPEN)) {
+			ret = __wt_btree_close(session);
+			F_CLR(btree, WT_BTREE_OPEN);
+		}
 		if (!locked)
 			__wt_rwunlock(session, btree->rwlock);
 	}
diff --git a/src/session/session_btree.c b/src/session/session_btree.c
index 8feacf63cb5..75f3499dd2a 100644
--- a/src/session/session_btree.c
+++ b/src/session/session_btree.c
@@ -37,8 +37,11 @@ __wt_session_lock_btree(
     WT_SESSION_IMPL *session, const char *cfg[], uint32_t flags)
 {
 	WT_BTREE *btree;
+	uint32_t open_flags;
+	int ret;
 
 	btree = session->btree;
+	ret = 0;
 
 	if (LF_ISSET(WT_BTREE_EXCLUSIVE)) {
 		/*
@@ -51,18 +54,19 @@ __wt_session_lock_btree(
 		WT_RET(__wt_try_writelock(session, btree->rwlock));
 
 		/*
-		 * Check if the handle needs to be reopened for this operation.
-		 * We do need to pick up the flags anyway, for example to set
-		 * WT_BTREE_BULK so the handle is closed correctly.
+		 * Reopen the handle for this operation to set any special
+		 * flags.  For example, set WT_BTREE_BULK so the handle is
+		 * closed correctly.
 		 */
-		if (LF_ISSET(WT_BTREE_BULK |
-		    WT_BTREE_SALVAGE | WT_BTREE_UPGRADE | WT_BTREE_VERIFY))
-			return (__wt_conn_btree_reopen(session, cfg, flags));
-		F_SET(btree, flags);
+		open_flags = LF_ISSET(WT_BTREE_BULK |
+		    WT_BTREE_SALVAGE | WT_BTREE_UPGRADE | WT_BTREE_VERIFY);
+		if (open_flags != 0)
+			ret = __wt_conn_btree_reopen(session, cfg, open_flags);
+		F_SET(btree, WT_BTREE_EXCLUSIVE);
 	} else if (!LF_ISSET(WT_BTREE_NO_LOCK))
 		__wt_readlock(session, btree->rwlock);
 
-	return (0);
+	return (ret);
 }
 
 /*
@@ -79,17 +83,17 @@ __wt_session_release_btree(WT_SESSION_IMPL *session)
 	ret = 0;
 
 	/*
-	 * If we had exclusive access, reopen the tree without special flags so
-	 * that other threads can use it (note the reopen call sets the flags).
+	 * If we had exclusive access, close the handle so that other threads
+	 * can use it (the next thread to access the handle will open it with
+	 * any special flags as required.  The handle stays in our cache, so
+	 * we don't want to go through __wt_conn_btree_close.
 	 */
-	if (F_ISSET(btree, WT_BTREE_BULK |
-	    WT_BTREE_SALVAGE | WT_BTREE_UPGRADE | WT_BTREE_VERIFY)) {
-		WT_ASSERT(session, F_ISSET(btree, WT_BTREE_EXCLUSIVE));
-		ret = __wt_conn_btree_reopen(session, NULL, 0);
+	if (F_ISSET(btree, WT_BTREE_EXCLUSIVE)) {
+		ret = __wt_btree_close(session);
+		F_CLR(btree, WT_BTREE_EXCLUSIVE | WT_BTREE_OPEN);
 	}
-	F_CLR(btree, WT_BTREE_EXCLUSIVE);
 
-	__wt_rwunlock(session, session->btree->rwlock);
+	__wt_rwunlock(session, btree->rwlock);
 
 	return (ret);
 }