From 58bc810b91b7496ee2db388a562f069a1ea05b14 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Mon, 10 Aug 2015 15:30:33 +1000 Subject: WT-2036 Make handle sweeps more robust Merge pull request #2113 from wiredtiger/sweep-robustness (cherry picked from commit 36310d45b73e0d97f69efcee075141ba81153563) --- src/conn/conn_sweep.c | 140 ++++++++++++++++++++++++++------------------------ src/include/dhandle.h | 1 + 2 files changed, 73 insertions(+), 68 deletions(-) diff --git a/src/conn/conn_sweep.c b/src/conn/conn_sweep.c index e13bcd1487a..d658ed8d3f3 100644 --- a/src/conn/conn_sweep.c +++ b/src/conn/conn_sweep.c @@ -14,38 +14,34 @@ * handles. */ static int -__sweep_mark(WT_SESSION_IMPL *session, int *dead_handlesp) +__sweep_mark(WT_SESSION_IMPL *session, time_t now) { WT_CONNECTION_IMPL *conn; WT_DATA_HANDLE *dhandle; - time_t now; conn = S2C(session); - *dead_handlesp = 0; - - /* Don't discard handles that have been open recently. */ - WT_RET(__wt_seconds(session, &now)); WT_STAT_FAST_CONN_INCR(session, dh_conn_sweeps); SLIST_FOREACH(dhandle, &conn->dhlh, l) { if (WT_IS_METADATA(dhandle)) continue; - if (F_ISSET(dhandle, WT_DHANDLE_DEAD)) { - ++*dead_handlesp; - continue; - } - if (dhandle->session_inuse != 0 || - now <= dhandle->timeofdeath + conn->sweep_idle_time || - conn->sweep_idle_time == 0) - continue; - if (dhandle->timeofdeath == 0) { - dhandle->timeofdeath = now; - WT_STAT_FAST_CONN_INCR(session, dh_conn_tod); + + /* + * There are some internal increments of the in-use count such + * as eviction. Don't keep handles alive because of those + * cases, but if we see multiple cursors open, clear the time + * of death. + */ + if (dhandle->session_inuse > 1) + dhandle->timeofdeath = 0; + + if (F_ISSET(dhandle, WT_DHANDLE_DEAD) || + dhandle->session_inuse != 0 || + dhandle->timeofdeath != 0) continue; - } - /* We now have a candidate to close. */ - ++*dead_handlesp; + dhandle->timeofdeath = now; + WT_STAT_FAST_CONN_INCR(session, dh_conn_tod); } return (0); @@ -113,37 +109,26 @@ err: WT_TRET(__wt_writeunlock(session, dhandle->rwlock)); * until we have reached the configured minimum number of handles. */ static int -__sweep_expire(WT_SESSION_IMPL *session) +__sweep_expire(WT_SESSION_IMPL *session, time_t now) { WT_CONNECTION_IMPL *conn; WT_DATA_HANDLE *dhandle; WT_DECL_RET; - time_t now; conn = S2C(session); - /* If sweep_idle_time is 0, then we won't expire any cursors */ - if (conn->sweep_idle_time == 0) - return (0); - - /* Don't discard handles that have been open recently. */ - WT_RET(__wt_seconds(session, &now)); - WT_STAT_FAST_CONN_INCR(session, dh_conn_sweeps); SLIST_FOREACH(dhandle, &conn->dhlh, l) { /* - * Ignore open files once the open file count reaches the + * Ignore open files once the btree file count is below the * minimum number of handles. */ - if (conn->open_file_count < conn->sweep_handles_min) + if (conn->open_btree_count < conn->sweep_handles_min) break; - if (WT_IS_METADATA(dhandle)) - continue; - if (!F_ISSET(dhandle, WT_DHANDLE_OPEN) || - F_ISSET(dhandle, WT_DHANDLE_DEAD)) - continue; - if (dhandle->session_inuse != 0 || + if (WT_IS_METADATA(dhandle) || + dhandle->session_inuse != 0 || + dhandle->timeofdeath == 0 || now <= dhandle->timeofdeath + conn->sweep_idle_time) continue; @@ -156,11 +141,12 @@ __sweep_expire(WT_SESSION_IMPL *session) } /* - * __sweep_flush -- - * Flush pages from dead trees. + * __sweep_discard_trees -- + * Discard pages from dead trees. */ static int -__sweep_flush(WT_SESSION_IMPL *session) +__sweep_discard_trees( + WT_SESSION_IMPL *session, time_t now, u_int *dead_handlesp) { WT_CONNECTION_IMPL *conn; WT_DATA_HANDLE *dhandle; @@ -168,8 +154,15 @@ __sweep_flush(WT_SESSION_IMPL *session) conn = S2C(session); + *dead_handlesp = 0; + WT_STAT_FAST_CONN_INCR(session, dh_conn_sweeps); SLIST_FOREACH(dhandle, &conn->dhlh, l) { + if (!F_ISSET(dhandle, WT_DHANDLE_OPEN | WT_DHANDLE_EXCLUSIVE) && + (dhandle->timeofdiscard == 0 || + now <= dhandle->timeofdiscard + conn->sweep_idle_time)) + ++*dead_handlesp; + if (!F_ISSET(dhandle, WT_DHANDLE_OPEN) || !F_ISSET(dhandle, WT_DHANDLE_DEAD)) continue; @@ -178,9 +171,11 @@ __sweep_flush(WT_SESSION_IMPL *session) WT_WITH_DHANDLE(session, dhandle, ret = __wt_conn_btree_sync_and_close(session, 0, 0)); - /* We closed the btree handle, bump the statistic. */ - if (ret == 0) + /* We closed the btree handle. */ + if (ret == 0) { WT_STAT_FAST_CONN_INCR(session, dh_conn_handles); + ++*dead_handlesp; + } WT_RET_BUSY_OK(ret); } @@ -193,7 +188,7 @@ __sweep_flush(WT_SESSION_IMPL *session) * Remove closed dhandles from the connection list. */ static int -__sweep_remove_handles(WT_SESSION_IMPL *session) +__sweep_remove_handles(WT_SESSION_IMPL *session, time_t now) { WT_CONNECTION_IMPL *conn; WT_DATA_HANDLE *dhandle, *dhandle_next; @@ -206,22 +201,26 @@ __sweep_remove_handles(WT_SESSION_IMPL *session) dhandle_next = SLIST_NEXT(dhandle, l); if (WT_IS_METADATA(dhandle)) continue; - if (F_ISSET(dhandle, WT_DHANDLE_OPEN) || - dhandle->session_inuse != 0 || - dhandle->session_ref != 0) + if (F_ISSET(dhandle, WT_DHANDLE_EXCLUSIVE | WT_DHANDLE_OPEN) || + dhandle->session_inuse != 0 || dhandle->session_ref != 0) + continue; + if (dhandle->timeofdiscard != 0 && + now <= dhandle->timeofdiscard + conn->sweep_idle_time) continue; /* Make sure we get exclusive access. */ if ((ret = - __wt_try_writelock(session, dhandle->rwlock)) == EBUSY) + __wt_try_writelock(session, dhandle->rwlock)) == EBUSY) { + dhandle->timeofdiscard = now; continue; + } WT_RET(ret); /* * If there are no longer any references to the handle in any * sessions, attempt to discard it. */ - if (F_ISSET(dhandle, WT_DHANDLE_OPEN) || + if (F_ISSET(dhandle, WT_DHANDLE_EXCLUSIVE | WT_DHANDLE_OPEN) || dhandle->session_inuse != 0 || dhandle->session_ref != 0) { WT_RET(__wt_writeunlock(session, dhandle->rwlock)); continue; @@ -230,9 +229,14 @@ __sweep_remove_handles(WT_SESSION_IMPL *session) WT_WITH_DHANDLE(session, dhandle, ret = __wt_conn_dhandle_discard_single(session, 0, 1)); - /* If the handle was not successfully discarded, unlock it. */ - if (ret != 0) + /* + * If the handle was not successfully discarded, unlock it and + * don't retry the discard until it times out again. + */ + if (ret != 0) { + dhandle->timeofdiscard = now; WT_TRET(__wt_writeunlock(session, dhandle->rwlock)); + } WT_RET_BUSY_OK(ret); WT_STAT_FAST_CONN_INCR(session, dh_conn_ref); } @@ -250,7 +254,8 @@ __sweep_server(void *arg) WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_SESSION_IMPL *session; - int dead_handles; + time_t now; + u_int dead_handles; session = arg; conn = S2C(session); @@ -263,35 +268,34 @@ __sweep_server(void *arg) /* Wait until the next event. */ WT_ERR(__wt_cond_wait(session, conn->sweep_cond, (uint64_t)conn->sweep_interval * WT_MILLION)); + WT_ERR(__wt_seconds(session, &now)); /* * Mark handles with a time of death, and report whether any - * handles are marked dead. + * handles are marked dead. If sweep_idle_time is 0, handles + * never become idle. */ - WT_ERR(__sweep_mark(session, &dead_handles)); + if (conn->sweep_idle_time != 0) + WT_ERR(__sweep_mark(session, now)); /* - * We only want to flush and expire if there are no dead handles - * and if either the sweep_idle_time is not 0, or if we have - * reached the configured limit of handles. + * Close handles if we have reached the configured limit. + * If sweep_idle_time is 0, handles never become idle. */ - if (dead_handles == 0 && - (conn->open_file_count < conn->sweep_handles_min || - conn->sweep_idle_time != 0)) - continue; - - /* Close handles if we have reached the configured limit */ - if (conn->open_file_count >= conn->sweep_handles_min) { + if (conn->sweep_idle_time != 0 && + conn->open_btree_count >= conn->sweep_handles_min) { WT_WITH_DHANDLE_LOCK(session, - ret = __sweep_expire(session)); + ret = __sweep_expire(session, now)); WT_ERR(ret); } - WT_ERR(__sweep_flush(session)); + WT_ERR(__sweep_discard_trees(session, now, &dead_handles)); - WT_WITH_DHANDLE_LOCK(session, - ret = __sweep_remove_handles(session)); - WT_ERR(ret); + if (dead_handles > 0) { + WT_WITH_DHANDLE_LOCK(session, + ret = __sweep_remove_handles(session, now)); + WT_ERR(ret); + } } if (0) { diff --git a/src/include/dhandle.h b/src/include/dhandle.h index 034db30a0a2..b188c19566b 100644 --- a/src/include/dhandle.h +++ b/src/include/dhandle.h @@ -45,6 +45,7 @@ struct __wt_data_handle { uint32_t session_ref; /* Sessions referencing this handle */ int32_t session_inuse; /* Sessions using this handle */ time_t timeofdeath; /* Use count went to 0 */ + time_t timeofdiscard; /* Time of last failed discard */ uint64_t name_hash; /* Hash of name */ const char *name; /* Object name as a URI */ -- cgit v1.2.1