Fix a deadlock related to handle locking.

If one thread does a __wt_session_release_btree of a handle at the same time
as another thread does a __wt_session_get_btree both wanting exclusive access
to the file. It was possible for one thread to get the dhandle list lock and
wait on the handle lock, which another thread held the handle lock waiting for
the handle list lock.

Temporarily fix by doing a try-lock on the __wt_session_get_btree path, long term
the solution is to get rid of the __conn_dhandle_open_lock method, and split get
and lock into two passes.

Refs BF-716

1 files changed, 11 insertions, 3 deletions

diff --git a/src/conn/conn_dhandle.c b/src/conn/conn_dhandle.c
index 385280116a6..0eb3389de34 100644
--- a/src/conn/conn_dhandle.c
+++ b/src/conn/conn_dhandle.c
@@ -70,9 +70,9 @@ __conn_dhandle_open_lock(
 			return (EBUSY);
 
 		/*
-		 * If the handle is open, get a read lock and recheck.
+		 * If the handle is open, try to get a read lock and recheck.
 		 *
-		 * Wait for a read lock if we want exclusive access and failed
+		 * Try to get a read lock if we want exclusive access and failed
 		 * to get it: the sweep server may be closing this handle, and
 		 * we need to wait for it to complete.  If we want exclusive
 		 * access and find the handle open once we get the read lock,
@@ -80,7 +80,15 @@ __conn_dhandle_open_lock(
 		 */
 		if (F_ISSET(dhandle, WT_DHANDLE_OPEN) &&
 		    (!want_exclusive || lock_busy)) {
-			WT_RET(__wt_readlock(session, dhandle->rwlock));
+			ret = __wt_try_readlock(session, dhandle->rwlock);
+			if (ret == EBUSY) {
+				if (want_exclusive &&
+				    F_ISSET(dhandle, WT_DHANDLE_OPEN))
+					return (ret);
+				__wt_yield();
+				continue;
+			}
+			WT_RET(ret);
 			is_open = F_ISSET(dhandle, WT_DHANDLE_OPEN) ? 1 : 0;
 			if (is_open && !want_exclusive)
 				return (0);