Sweep old handles more aggressively:

1. don't have checkpoint or other periodic operations like statistics logging keep old handles alive;
2. don't wait for all sessions to empty the file from their cache before closing;
3. only update the time of death from the sweep thread.

8 files changed, 70 insertions, 77 deletions

diff --git a/src/conn/conn_dhandle.c b/src/conn/conn_dhandle.c
index 088ff2f3d2c..9741800477c 100644
--- a/src/conn/conn_dhandle.c
+++ b/src/conn/conn_dhandle.c
@@ -665,7 +665,7 @@ __conn_dhandle_remove(WT_SESSION_IMPL *session, int final)
 	WT_ASSERT(session, F_ISSET(session, WT_SESSION_HANDLE_LIST_LOCKED));
 
 	/* Check if the handle was reacquired by a session while we waited. */
-	if (!final && dhandle->session_ref != 0)
+	if (!final && dhandle->session_inuse != 0)
 		return (EBUSY);
 
 	WT_CONN_DHANDLE_REMOVE(conn, dhandle, bucket);
diff --git a/src/conn/conn_sweep.c b/src/conn/conn_sweep.c
index 35f03cbe980..8e2fa620f34 100644
--- a/src/conn/conn_sweep.c
+++ b/src/conn/conn_sweep.c
@@ -27,8 +27,12 @@ __sweep(WT_SESSION_IMPL *session)
 	dhandle = SLIST_FIRST(&conn->dhlh);
 	for (; dhandle != NULL; dhandle = dhandle_next) {
 		dhandle_next = SLIST_NEXT(dhandle, l);
-		if (dhandle->session_ref != 0 ||
-		    now - dhandle->timeofdeath <= WT_DHANDLE_SWEEP_WAIT)
+		if (dhandle->session_inuse == 0 && dhandle->timeofdeath == 0) {
+			dhandle->timeofdeath = now;
+			continue;
+		}
+		if (dhandle->session_inuse != 0 ||
+		    now <= dhandle->timeofdeath + WT_DHANDLE_SWEEP_WAIT)
 			continue;
 
 		/*
@@ -50,8 +54,8 @@ __sweep(WT_SESSION_IMPL *session)
 			WT_RET(ret);
 
 			/* Re-check that this looks like a good candidate. */
-			if (dhandle->session_ref != 0 ||
-			    now - dhandle->timeofdeath <= WT_DHANDLE_SWEEP_WAIT)
+			if (dhandle->session_inuse != 0 ||
+			    now <= dhandle->timeofdeath + WT_DHANDLE_SWEEP_WAIT)
 				continue;
 		}
 
@@ -65,13 +69,19 @@ __sweep(WT_SESSION_IMPL *session)
 			continue;
 		WT_RET(ret);
 
+		/* If the handle is open, try to close it. */
+		if (F_ISSET(dhandle, WT_DHANDLE_OPEN))
+			WT_TRET(__wt_conn_btree_sync_and_close(session, 0));
+
 		/*
-		 * Attempt to discard the handle.  The called function
+		 * If there are no longer any references to the handle in any
+		 * sessions, attempt to discard it.  The called function
 		 * re-checks that the handle is not in use, which is why we
 		 * don't do any special handling of EBUSY returns above.
 		 */
-		WT_WITH_DHANDLE(session, dhandle,
-		    ret = __wt_conn_dhandle_discard_single(session, 0));
+		if (ret == 0 && dhandle->session_ref == 0)
+			WT_WITH_DHANDLE(session, dhandle,
+			    ret = __wt_conn_dhandle_discard_single(session, 0));
 
 		/* If the handle wasn't discarded, drop our lock. */
 		if (ret != 0)
diff --git a/src/cursor/cur_file.c b/src/cursor/cur_file.c
index e8428b76691..5d6072ceb3e 100644
--- a/src/cursor/cur_file.c
+++ b/src/cursor/cur_file.c
@@ -328,8 +328,11 @@ __curfile_close(WT_CURSOR *cursor)
 	cbt = (WT_CURSOR_BTREE *)cursor;
 	CURSOR_API_CALL(cursor, session, close, cbt->btree);
 	WT_TRET(__wt_btcur_close(cbt));
-	if (cbt->btree != NULL)
+	if (cbt->btree != NULL) {
+		/* Increment the data-source's in-use counter. */
+		__wt_cursor_dhandle_decr_use(session);
 		WT_TRET(__wt_session_release_btree(session));
+	}
 	/* The URI is owned by the btree handle. */
 	cursor->internal_uri = NULL;
 	WT_TRET(__wt_cursor_close(cursor));
@@ -475,6 +478,8 @@ __wt_curfile_open(WT_SESSION_IMPL *session, const char *uri,
 
 	WT_ERR(__wt_curfile_create(session, owner, cfg, bulk, bitmap, cursorp));
 
+	/* Increment the data-source's in-use counter. */
+	__wt_cursor_dhandle_incr_use(session);
 	return (0);
 
 err:	/* If the cursor could not be opened, release the handle. */
diff --git a/src/evict/evict_lru.c b/src/evict/evict_lru.c
index 2af9f0024db..b20d03feef0 100644
--- a/src/evict/evict_lru.c
+++ b/src/evict/evict_lru.c
@@ -886,8 +886,8 @@ retry:	while (slot < max_entries && ret == 0) {
 			dhandle = SLIST_FIRST(&conn->dhlh);
 		else {
 			if (incr) {
-				WT_ASSERT(session, dhandle->session_ref > 0);
-				(void)WT_ATOMIC_SUB4(dhandle->session_ref, 1);
+				WT_ASSERT(session, dhandle->session_inuse > 0);
+				(void)WT_ATOMIC_SUB4(dhandle->session_inuse, 1);
 				incr = 0;
 			}
 			dhandle = SLIST_NEXT(dhandle, l);
@@ -935,7 +935,7 @@ retry:	while (slot < max_entries && ret == 0) {
 		btree->evict_walk_skips = 0;
 		old_slot = slot;
 
-		(void)WT_ATOMIC_ADD4(dhandle->session_ref, 1);
+		(void)WT_ATOMIC_ADD4(dhandle->session_inuse, 1);
 		incr = 1;
 		__wt_spin_unlock(session, &conn->dhandle_lock);
 		dhandle_locked = 0;
@@ -965,8 +965,8 @@ retry:	while (slot < max_entries && ret == 0) {
 	}
 
 	if (incr) {
-		WT_ASSERT(session, dhandle->session_ref > 0);
-		(void)WT_ATOMIC_SUB4(dhandle->session_ref, 1);
+		WT_ASSERT(session, dhandle->session_inuse > 0);
+		(void)WT_ATOMIC_SUB4(dhandle->session_inuse, 1);
 		incr = 0;
 	}
 
diff --git a/src/include/cursor.i b/src/include/cursor.i
index 7716a16fe0b..737ac8dcdba 100644
--- a/src/include/cursor.i
+++ b/src/include/cursor.i
@@ -136,6 +136,38 @@ __curfile_leave(WT_CURSOR_BTREE *cbt)
 }
 
 /*
+ * __wt_cursor_dhandle_incr_use --
+ *	Increment the in-use counter in cursor's data source.
+ */
+static inline void
+__wt_cursor_dhandle_incr_use(WT_SESSION_IMPL *session)
+{
+	WT_DATA_HANDLE *dhandle;
+
+	dhandle = session->dhandle;
+
+	/* If we open a handle with a time of death set, clear it. */
+	if (WT_ATOMIC_ADD4(dhandle->session_inuse, 1) == 1 &&
+	    dhandle->timeofdeath != 0)
+		dhandle->timeofdeath = 0;
+}
+
+/*
+ * __wt_cursor_dhandle_decr_use --
+ *	Decrement the in-use counter in cursor's data source.
+ */
+static inline void
+__wt_cursor_dhandle_decr_use(WT_SESSION_IMPL *session)
+{
+	WT_DATA_HANDLE *dhandle;
+
+	dhandle = session->dhandle;
+
+	WT_ASSERT(session, dhandle->session_inuse > 0);
+	(void)WT_ATOMIC_SUB4(dhandle->session_inuse, 1);
+}
+
+/*
  * __cursor_func_init --
  *	Cursor call setup.
  */
diff --git a/src/include/extern.h b/src/include/extern.h
index a8bc6a11cef..c982376ea46 100644
--- a/src/include/extern.h
+++ b/src/include/extern.h
@@ -548,7 +548,6 @@ extern int __wt_open_internal_session(WT_CONNECTION_IMPL *conn, const char *name
 extern int __wt_open_session(WT_CONNECTION_IMPL *conn, WT_EVENT_HANDLER *event_handler, const char *config, WT_SESSION_IMPL **sessionp);
 extern int __wt_compact_uri_analyze(WT_SESSION_IMPL *session, const char *uri, int *skip);
 extern int __wt_session_compact( WT_SESSION *wt_session, const char *uri, const char *config);
-extern void __wt_session_dhandle_incr_use(WT_SESSION_IMPL *session);
 extern int __wt_session_lock_dhandle(WT_SESSION_IMPL *session, uint32_t flags);
 extern int __wt_session_release_btree(WT_SESSION_IMPL *session);
 extern int __wt_session_get_btree_ckpt(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], uint32_t flags);
diff --git a/src/meta/meta_table.c b/src/meta/meta_table.c
index d1e0b2f4bfe..0f2d37a2e55 100644
--- a/src/meta/meta_table.c
+++ b/src/meta/meta_table.c
@@ -68,12 +68,11 @@ __wt_metadata_cursor(
 
 	/* 
 	 * We use the metadata a lot, so we have a handle cached; lock it and
-	 * increment the in-use counter.
+	 * increment the in-use counter once the cursor is open.
 	 */
 	WT_ERR(__wt_session_lock_dhandle(session, 0));
-	__wt_session_dhandle_incr_use(session);
-
-	ret = __wt_curfile_create(session, NULL, cfg, 0, 0, cursorp);
+	WT_ERR(__wt_curfile_create(session, NULL, cfg, 0, 0, cursorp));
+	__wt_cursor_dhandle_incr_use(session);
 
 	/* Restore the caller's btree. */
 err:	session->dhandle = saved_dhandle;
diff --git a/src/session/session_dhandle.c b/src/session/session_dhandle.c
index 51835cf260c..e28e277d5f6 100644
--- a/src/session/session_dhandle.c
+++ b/src/session/session_dhandle.c
@@ -7,41 +7,7 @@
 
 #include "wt_internal.h"
 
-/*
- * __wt_session_dhandle_incr_use --
- *	Increment the session data source's in-use counter.
- */
-void
-__wt_session_dhandle_incr_use(WT_SESSION_IMPL *session)
-{
-	WT_DATA_HANDLE *dhandle;
-
-	dhandle = session->dhandle;
-
-	(void)WT_ATOMIC_ADD4(dhandle->session_inuse, 1);
-}
-
-/*
- * __session_dhandle_decr_use --
- *	Decrement the session data source's in-use counter.
- */
-static int
-__session_dhandle_decr_use(WT_SESSION_IMPL *session)
-{
-	WT_DATA_HANDLE *dhandle;
-	WT_DECL_RET;
-
-	dhandle = session->dhandle;
-
-	/*
-	 * Decrement the in-use count on the underlying data-source -- if we're
-	 * the last reference, set the time-of-death timestamp.
-	 */
-	WT_ASSERT(session, dhandle->session_inuse > 0);
-	if (WT_ATOMIC_SUB4(dhandle->session_inuse, 1) == 0)
-		WT_TRET(__wt_seconds(session, &dhandle->timeofdeath));
-	return (0);
-}
+static int __session_dhandle_sweep(WT_SESSION_IMPL *);
 
 /*
  * __session_add_dhandle --
@@ -65,7 +31,9 @@ __session_add_dhandle(
 		*dhandle_cachep = dhandle_cache;
 
 	(void)WT_ATOMIC_ADD4(session->dhandle->session_ref, 1);
-	return (0);
+
+	/* Sweep the handle list to remove any dead handles. */
+	return (__session_dhandle_sweep(session));
 }
 
 /*
@@ -165,13 +133,6 @@ __wt_session_release_btree(WT_SESSION_IMPL *session)
 	btree = S2BT(session);
 	dhandle = session->dhandle;
 
-	/*
-	 * Decrement the data-source's in-use counter. We ignore errors because
-	 * they're insignificant and handling them complicates error handling in
-	 * this function more than I'm willing to live with.
-	 */
-	(void)__session_dhandle_decr_use(session);
-
 	locked = F_ISSET(dhandle, WT_DHANDLE_EXCLUSIVE) ? WRITELOCK : READLOCK;
 	if (F_ISSET(dhandle, WT_DHANDLE_DISCARD_CLOSE)) {
 		/*
@@ -324,20 +285,13 @@ __wt_session_close_cache(WT_SESSION_IMPL *session)
  *	Discard any session dhandles that are not open.
  */
 static int
-__session_dhandle_sweep(WT_SESSION_IMPL *session, uint32_t flags)
+__session_dhandle_sweep(WT_SESSION_IMPL *session)
 {
 	WT_DATA_HANDLE *dhandle;
 	WT_DATA_HANDLE_CACHE *dhandle_cache, *dhandle_cache_next;
 	time_t now;
 
 	/*
-	 * Check the local flag WT_DHANDLE_LOCK_ONLY; a common caller with that
-	 * flag is in the path to discard the handle, don't sweep in that case.
-	 */
-	if (LF_ISSET(WT_DHANDLE_LOCK_ONLY))
-		return (0);
-
-	/*
 	 * Periodically sweep for dead handles; if we've swept recently, don't
 	 * do it again.
 	 */
@@ -447,16 +401,10 @@ __wt_session_get_btree(WT_SESSION_IMPL *session,
 	if (!LF_ISSET(WT_DHANDLE_HAVE_REF))
 		WT_RET(__session_add_dhandle(session, NULL));
 
-	/* Sweep the handle list to remove any dead handles. */
-	WT_RET(__session_dhandle_sweep(session, flags));
-
 	WT_ASSERT(session, LF_ISSET(WT_DHANDLE_LOCK_ONLY) ||
 	    F_ISSET(session->dhandle, WT_DHANDLE_OPEN));
 
-done:	/* Increment the data-source's in-use counter. */
-	__wt_session_dhandle_incr_use(session);
-
-	WT_ASSERT(session, LF_ISSET(WT_DHANDLE_EXCLUSIVE) ==
+done:	WT_ASSERT(session, LF_ISSET(WT_DHANDLE_EXCLUSIVE) ==
 	    F_ISSET(session->dhandle, WT_DHANDLE_EXCLUSIVE));
 	F_SET(session->dhandle, LF_ISSET(WT_DHANDLE_DISCARD_CLOSE));