WT-2036 Make handle sweeps more robust

Merge pull request #2113 from wiredtiger/sweep-robustness
(cherry picked from commit 36310d45b73e0d97f69efcee075141ba81153563)

2 files changed, 73 insertions, 68 deletions

diff --git a/src/conn/conn_sweep.c b/src/conn/conn_sweep.c
index e13bcd1487a..d658ed8d3f3 100644
--- a/src/conn/conn_sweep.c
+++ b/src/conn/conn_sweep.c
@@ -14,38 +14,34 @@
  *	handles.
  */
 static int
-__sweep_mark(WT_SESSION_IMPL *session, int *dead_handlesp)
+__sweep_mark(WT_SESSION_IMPL *session, time_t now)
 {
 	WT_CONNECTION_IMPL *conn;
 	WT_DATA_HANDLE *dhandle;
-	time_t now;
 
 	conn = S2C(session);
-	*dead_handlesp = 0;
-
-	/* Don't discard handles that have been open recently. */
-	WT_RET(__wt_seconds(session, &now));
 
 	WT_STAT_FAST_CONN_INCR(session, dh_conn_sweeps);
 	SLIST_FOREACH(dhandle, &conn->dhlh, l) {
 		if (WT_IS_METADATA(dhandle))
 			continue;
-		if (F_ISSET(dhandle, WT_DHANDLE_DEAD)) {
-			++*dead_handlesp;
-			continue;
-		}
-		if (dhandle->session_inuse != 0 ||
-		    now <= dhandle->timeofdeath + conn->sweep_idle_time ||
-		    conn->sweep_idle_time == 0)
-			continue;
-		if (dhandle->timeofdeath == 0) {
-			dhandle->timeofdeath = now;
-			WT_STAT_FAST_CONN_INCR(session, dh_conn_tod);
+
+		/*
+		 * There are some internal increments of the in-use count such
+		 * as eviction.  Don't keep handles alive because of those
+		 * cases, but if we see multiple cursors open, clear the time
+		 * of death.
+		 */
+		if (dhandle->session_inuse > 1)
+			dhandle->timeofdeath = 0;
+
+		if (F_ISSET(dhandle, WT_DHANDLE_DEAD) ||
+		    dhandle->session_inuse != 0 ||
+		    dhandle->timeofdeath != 0)
 			continue;
-		}
 
-		/* We now have a candidate to close. */
-		++*dead_handlesp;
+		dhandle->timeofdeath = now;
+		WT_STAT_FAST_CONN_INCR(session, dh_conn_tod);
 	}
 
 	return (0);
@@ -113,37 +109,26 @@ err:	WT_TRET(__wt_writeunlock(session, dhandle->rwlock));
  *	until we have reached the configured minimum number of handles.
  */
 static int
-__sweep_expire(WT_SESSION_IMPL *session)
+__sweep_expire(WT_SESSION_IMPL *session, time_t now)
 {
 	WT_CONNECTION_IMPL *conn;
 	WT_DATA_HANDLE *dhandle;
 	WT_DECL_RET;
-	time_t now;
 
 	conn = S2C(session);
 
-	/* If sweep_idle_time is 0, then we won't expire any cursors */
-	if (conn->sweep_idle_time == 0)
-		return (0);
-
-	/* Don't discard handles that have been open recently. */
-	WT_RET(__wt_seconds(session, &now));
-
 	WT_STAT_FAST_CONN_INCR(session, dh_conn_sweeps);
 	SLIST_FOREACH(dhandle, &conn->dhlh, l) {
 		/*
-		 * Ignore open files once the open file count reaches the
+		 * Ignore open files once the btree file count is below the
 		 * minimum number of handles.
 		 */
-		if (conn->open_file_count < conn->sweep_handles_min)
+		if (conn->open_btree_count < conn->sweep_handles_min)
 			break;
 
-		if (WT_IS_METADATA(dhandle))
-			continue;
-		if (!F_ISSET(dhandle, WT_DHANDLE_OPEN) ||
-		    F_ISSET(dhandle, WT_DHANDLE_DEAD))
-			continue;
-		if (dhandle->session_inuse != 0 ||
+		if (WT_IS_METADATA(dhandle) ||
+		    dhandle->session_inuse != 0 ||
+		    dhandle->timeofdeath == 0 ||
 		    now <= dhandle->timeofdeath + conn->sweep_idle_time)
 			continue;
 
@@ -156,11 +141,12 @@ __sweep_expire(WT_SESSION_IMPL *session)
 }
 
 /*
- * __sweep_flush --
- *	Flush pages from dead trees.
+ * __sweep_discard_trees --
+ *	Discard pages from dead trees.
  */
 static int
-__sweep_flush(WT_SESSION_IMPL *session)
+__sweep_discard_trees(
+    WT_SESSION_IMPL *session, time_t now, u_int *dead_handlesp)
 {
 	WT_CONNECTION_IMPL *conn;
 	WT_DATA_HANDLE *dhandle;
@@ -168,8 +154,15 @@ __sweep_flush(WT_SESSION_IMPL *session)
 
 	conn = S2C(session);
 
+	*dead_handlesp = 0;
+
 	WT_STAT_FAST_CONN_INCR(session, dh_conn_sweeps);
 	SLIST_FOREACH(dhandle, &conn->dhlh, l) {
+		if (!F_ISSET(dhandle, WT_DHANDLE_OPEN | WT_DHANDLE_EXCLUSIVE) &&
+		    (dhandle->timeofdiscard == 0 ||
+		    now <= dhandle->timeofdiscard + conn->sweep_idle_time))
+			++*dead_handlesp;
+
 		if (!F_ISSET(dhandle, WT_DHANDLE_OPEN) ||
 		    !F_ISSET(dhandle, WT_DHANDLE_DEAD))
 			continue;
@@ -178,9 +171,11 @@ __sweep_flush(WT_SESSION_IMPL *session)
 		WT_WITH_DHANDLE(session, dhandle, ret =
 		    __wt_conn_btree_sync_and_close(session, 0, 0));
 
-		/* We closed the btree handle, bump the statistic. */
-		if (ret == 0)
+		/* We closed the btree handle. */
+		if (ret == 0) {
 			WT_STAT_FAST_CONN_INCR(session, dh_conn_handles);
+			++*dead_handlesp;
+		}
 
 		WT_RET_BUSY_OK(ret);
 	}
@@ -193,7 +188,7 @@ __sweep_flush(WT_SESSION_IMPL *session)
  *	Remove closed dhandles from the connection list.
  */
 static int
-__sweep_remove_handles(WT_SESSION_IMPL *session)
+__sweep_remove_handles(WT_SESSION_IMPL *session, time_t now)
 {
 	WT_CONNECTION_IMPL *conn;
 	WT_DATA_HANDLE *dhandle, *dhandle_next;
@@ -206,22 +201,26 @@ __sweep_remove_handles(WT_SESSION_IMPL *session)
 		dhandle_next = SLIST_NEXT(dhandle, l);
 		if (WT_IS_METADATA(dhandle))
 			continue;
-		if (F_ISSET(dhandle, WT_DHANDLE_OPEN) ||
-		    dhandle->session_inuse != 0 ||
-		    dhandle->session_ref != 0)
+		if (F_ISSET(dhandle, WT_DHANDLE_EXCLUSIVE | WT_DHANDLE_OPEN) ||
+		    dhandle->session_inuse != 0 || dhandle->session_ref != 0)
+			continue;
+		if (dhandle->timeofdiscard != 0 &&
+		    now <= dhandle->timeofdiscard + conn->sweep_idle_time)
 			continue;
 
 		/* Make sure we get exclusive access. */
 		if ((ret =
-		    __wt_try_writelock(session, dhandle->rwlock)) == EBUSY)
+		    __wt_try_writelock(session, dhandle->rwlock)) == EBUSY) {
+			dhandle->timeofdiscard = now;
 			continue;
+		}
 		WT_RET(ret);
 
 		/*
 		 * If there are no longer any references to the handle in any
 		 * sessions, attempt to discard it.
 		 */
-		if (F_ISSET(dhandle, WT_DHANDLE_OPEN) ||
+		if (F_ISSET(dhandle, WT_DHANDLE_EXCLUSIVE | WT_DHANDLE_OPEN) ||
 		    dhandle->session_inuse != 0 || dhandle->session_ref != 0) {
 			WT_RET(__wt_writeunlock(session, dhandle->rwlock));
 			continue;
@@ -230,9 +229,14 @@ __sweep_remove_handles(WT_SESSION_IMPL *session)
 		WT_WITH_DHANDLE(session, dhandle,
 		    ret = __wt_conn_dhandle_discard_single(session, 0, 1));
 
-		/* If the handle was not successfully discarded, unlock it. */
-		if (ret != 0)
+		/*
+		 * If the handle was not successfully discarded, unlock it and
+		 * don't retry the discard until it times out again.
+		 */
+		if (ret != 0) {
+			dhandle->timeofdiscard = now;
 			WT_TRET(__wt_writeunlock(session, dhandle->rwlock));
+		}
 		WT_RET_BUSY_OK(ret);
 		WT_STAT_FAST_CONN_INCR(session, dh_conn_ref);
 	}
@@ -250,7 +254,8 @@ __sweep_server(void *arg)
 	WT_CONNECTION_IMPL *conn;
 	WT_DECL_RET;
 	WT_SESSION_IMPL *session;
-	int dead_handles;
+	time_t now;
+	u_int dead_handles;
 
 	session = arg;
 	conn = S2C(session);
@@ -263,35 +268,34 @@ __sweep_server(void *arg)
 		/* Wait until the next event. */
 		WT_ERR(__wt_cond_wait(session, conn->sweep_cond,
 		    (uint64_t)conn->sweep_interval * WT_MILLION));
+		WT_ERR(__wt_seconds(session, &now));
 
 		/*
 		 * Mark handles with a time of death, and report whether any
-		 * handles are marked dead.
+		 * handles are marked dead.  If sweep_idle_time is 0, handles
+		 * never become idle.
 		 */
-		WT_ERR(__sweep_mark(session, &dead_handles));
+		if (conn->sweep_idle_time != 0)
+			WT_ERR(__sweep_mark(session, now));
 
 		/*
-		 * We only want to flush and expire if there are no dead handles
-		 * and if either the sweep_idle_time is not 0, or if we have
-		 * reached the configured limit of handles.
+		 * Close handles if we have reached the configured limit.
+		 * If sweep_idle_time is 0, handles never become idle.
 		 */
-		if (dead_handles == 0 &&
-		    (conn->open_file_count < conn->sweep_handles_min ||
-		    conn->sweep_idle_time != 0))
-			continue;
-
-		/* Close handles if we have reached the configured limit */
-		if (conn->open_file_count >= conn->sweep_handles_min) {
+		if (conn->sweep_idle_time != 0 &&
+		    conn->open_btree_count >= conn->sweep_handles_min) {
 			WT_WITH_DHANDLE_LOCK(session,
-			    ret = __sweep_expire(session));
+			    ret = __sweep_expire(session, now));
 			WT_ERR(ret);
 		}
 
-		WT_ERR(__sweep_flush(session));
+		WT_ERR(__sweep_discard_trees(session, now, &dead_handles));
 
-		WT_WITH_DHANDLE_LOCK(session,
-		    ret = __sweep_remove_handles(session));
-		WT_ERR(ret);
+		if (dead_handles > 0) {
+			WT_WITH_DHANDLE_LOCK(session,
+			    ret = __sweep_remove_handles(session, now));
+			WT_ERR(ret);
+		}
 	}
 
 	if (0) {
diff --git a/src/include/dhandle.h b/src/include/dhandle.h
index 034db30a0a2..b188c19566b 100644
--- a/src/include/dhandle.h
+++ b/src/include/dhandle.h
@@ -45,6 +45,7 @@ struct __wt_data_handle {
 	uint32_t session_ref;		/* Sessions referencing this handle */
 	int32_t	 session_inuse;		/* Sessions using this handle */
 	time_t	 timeofdeath;		/* Use count went to 0 */
+	time_t	 timeofdiscard;		/* Time of last failed discard */
 
 	uint64_t name_hash;		/* Hash of name */
 	const char *name;		/* Object name as a URI */