WT-3268 Failure to close cursor can get wiredtiger stuck in a cursor-close loop (#3378)

Traversing lists to close handles could turn into an infinite loop on error, the underlying close functions could return without unlinking the cursor from its linked list.

Add a macro pair, `WT_TAILQ_SAFE_REMOVE_BEGIN/END`, that include a test that if we see the same element twice on a linked list, we remove it so we don't loop forever.

Clean up various loops that remove elements from lists to either use the standard `TAILQ_FOREACH_SAFE`, the new macro or a pattern where the `TAILQ_REMOVE` is explicit in the loop.

15 files changed, 119 insertions, 99 deletions

diff --git a/examples/c/ex_file_system.c b/examples/c/ex_file_system.c
index 40f1d66cbc1..e454d228c39 100644
--- a/examples/c/ex_file_system.c
+++ b/examples/c/ex_file_system.c
@@ -583,13 +583,13 @@ demo_fs_size(WT_FILE_SYSTEM *file_system,
 static int
 demo_fs_terminate(WT_FILE_SYSTEM *file_system, WT_SESSION *session)
 {
-	DEMO_FILE_HANDLE *demo_fh;
+	DEMO_FILE_HANDLE *demo_fh, *demo_fh_tmp;
 	DEMO_FILE_SYSTEM *demo_fs;
 	int ret = 0, tret;
 
 	demo_fs = (DEMO_FILE_SYSTEM *)file_system;
 
-	while ((demo_fh = TAILQ_FIRST(&demo_fs->fileq)) != NULL)
+	TAILQ_FOREACH_SAFE(demo_fh, &demo_fs->fileq, q, demo_fh_tmp)
 		if ((tret =
 		    demo_handle_remove(session, demo_fh)) != 0 && ret == 0)
 			ret = tret;
diff --git a/ext/test/fail_fs/fail_fs.c b/ext/test/fail_fs/fail_fs.c
index b4add92be94..fd01ec66c68 100644
--- a/ext/test/fail_fs/fail_fs.c
+++ b/ext/test/fail_fs/fail_fs.c
@@ -740,12 +740,12 @@ fail_fs_size(WT_FILE_SYSTEM *file_system,
 static int
 fail_fs_terminate(WT_FILE_SYSTEM *file_system, WT_SESSION *session)
 {
-	FAIL_FILE_HANDLE *fail_fh;
+	FAIL_FILE_HANDLE *fail_fh, *fail_fh_tmp;
 	FAIL_FILE_SYSTEM *fail_fs;
 
 	fail_fs = (FAIL_FILE_SYSTEM *)file_system;
 
-	while ((fail_fh = TAILQ_FIRST(&fail_fs->fileq)) != NULL)
+	TAILQ_FOREACH_SAFE(fail_fh, &fail_fs->fileq, q, fail_fh_tmp)
 		fail_file_handle_remove(session, fail_fh);
 
 	fail_fs_destroy_lock(&fail_fs->lock);
diff --git a/src/async/async_api.c b/src/async/async_api.c
index ef3af8d15d3..e4943e61ed4 100644
--- a/src/async/async_api.c
+++ b/src/async/async_api.c
@@ -420,7 +420,7 @@ int
 __wt_async_destroy(WT_SESSION_IMPL *session)
 {
 	WT_ASYNC *async;
-	WT_ASYNC_FORMAT *af, *afnext;
+	WT_ASYNC_FORMAT *af;
 	WT_ASYNC_OP *op;
 	WT_CONNECTION_IMPL *conn;
 	WT_DECL_RET;
@@ -459,15 +459,13 @@ __wt_async_destroy(WT_SESSION_IMPL *session)
 	}
 
 	/* Free format resources */
-	af = TAILQ_FIRST(&async->formatqh);
-	while (af != NULL) {
-		afnext = TAILQ_NEXT(af, q);
+	while ((af = TAILQ_FIRST(&async->formatqh)) != NULL) {
+		TAILQ_REMOVE(&async->formatqh, af, q);
 		__wt_free(session, af->uri);
 		__wt_free(session, af->config);
 		__wt_free(session, af->key_format);
 		__wt_free(session, af->value_format);
 		__wt_free(session, af);
-		af = afnext;
 	}
 	__wt_free(session, async->async_queue);
 	__wt_free(session, async->async_ops);
diff --git a/src/async/async_worker.c b/src/async/async_worker.c
index 2390d9e47cd..ff00ca1c9d1 100644
--- a/src/async/async_worker.c
+++ b/src/async/async_worker.c
@@ -282,7 +282,7 @@ WT_THREAD_RET
 __wt_async_worker(void *arg)
 {
 	WT_ASYNC *async;
-	WT_ASYNC_CURSOR *ac, *acnext;
+	WT_ASYNC_CURSOR *ac;
 	WT_ASYNC_OP_IMPL *op;
 	WT_ASYNC_WORKER_STATE worker;
 	WT_CONNECTION_IMPL *conn;
@@ -341,12 +341,10 @@ err:		WT_PANIC_MSG(session, ret, "async worker error");
 	 * Worker thread cleanup, close our cached cursors and free all the
 	 * WT_ASYNC_CURSOR structures.
 	 */
-	ac = TAILQ_FIRST(&worker.cursorqh);
-	while (ac != NULL) {
-		acnext = TAILQ_NEXT(ac, q);
+	while ((ac = TAILQ_FIRST(&worker.cursorqh)) != NULL) {
+		TAILQ_REMOVE(&worker.cursorqh, ac, q);
 		WT_TRET(ac->c->close(ac->c));
 		__wt_free(session, ac);
-		ac = acnext;
 	}
 	return (WT_THREAD_RET_VALUE);
 }
diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c
index 44333ceec3f..c0a1f5c0920 100644
--- a/src/conn/conn_api.c
+++ b/src/conn/conn_api.c
@@ -175,13 +175,13 @@ __wt_conn_remove_collator(WT_SESSION_IMPL *session)
 	conn = S2C(session);
 
 	while ((ncoll = TAILQ_FIRST(&conn->collqh)) != NULL) {
+		/* Remove from the connection's list, free memory. */
+		TAILQ_REMOVE(&conn->collqh, ncoll, q);
 		/* Call any termination method. */
 		if (ncoll->collator->terminate != NULL)
 			WT_TRET(ncoll->collator->terminate(
 			    ncoll->collator, (WT_SESSION *)session));
 
-		/* Remove from the connection's list, free memory. */
-		TAILQ_REMOVE(&conn->collqh, ncoll, q);
 		__wt_free(session, ncoll->name);
 		__wt_free(session, ncoll);
 	}
@@ -281,13 +281,13 @@ __wt_conn_remove_compressor(WT_SESSION_IMPL *session)
 	conn = S2C(session);
 
 	while ((ncomp = TAILQ_FIRST(&conn->compqh)) != NULL) {
+		/* Remove from the connection's list, free memory. */
+		TAILQ_REMOVE(&conn->compqh, ncomp, q);
 		/* Call any termination method. */
 		if (ncomp->compressor->terminate != NULL)
 			WT_TRET(ncomp->compressor->terminate(
 			    ncomp->compressor, (WT_SESSION *)session));
 
-		/* Remove from the connection's list, free memory. */
-		TAILQ_REMOVE(&conn->compqh, ncomp, q);
 		__wt_free(session, ncomp->name);
 		__wt_free(session, ncomp);
 	}
@@ -346,13 +346,13 @@ __wt_conn_remove_data_source(WT_SESSION_IMPL *session)
 	conn = S2C(session);
 
 	while ((ndsrc = TAILQ_FIRST(&conn->dsrcqh)) != NULL) {
+		/* Remove from the connection's list, free memory. */
+		TAILQ_REMOVE(&conn->dsrcqh, ndsrc, q);
 		/* Call any termination method. */
 		if (ndsrc->dsrc->terminate != NULL)
 			WT_TRET(ndsrc->dsrc->terminate(
 			    ndsrc->dsrc, (WT_SESSION *)session));
 
-		/* Remove from the connection's list, free memory. */
-		TAILQ_REMOVE(&conn->dsrcqh, ndsrc, q);
 		__wt_free(session, ndsrc->prefix);
 		__wt_free(session, ndsrc);
 	}
@@ -536,14 +536,16 @@ __wt_conn_remove_encryptor(WT_SESSION_IMPL *session)
 	conn = S2C(session);
 
 	while ((nenc = TAILQ_FIRST(&conn->encryptqh)) != NULL) {
+		/* Remove from the connection's list, free memory. */
+		TAILQ_REMOVE(&conn->encryptqh, nenc, q);
 		while ((kenc = TAILQ_FIRST(&nenc->keyedqh)) != NULL) {
+			/* Remove from the connection's list, free memory. */
+			TAILQ_REMOVE(&nenc->keyedqh, kenc, q);
 			/* Call any termination method. */
 			if (kenc->owned && kenc->encryptor->terminate != NULL)
 				WT_TRET(kenc->encryptor->terminate(
 				    kenc->encryptor, (WT_SESSION *)session));
 
-			/* Remove from the connection's list, free memory. */
-			TAILQ_REMOVE(&nenc->keyedqh, kenc, q);
 			__wt_free(session, kenc->keyid);
 			__wt_free(session, kenc);
 		}
@@ -553,8 +555,6 @@ __wt_conn_remove_encryptor(WT_SESSION_IMPL *session)
 			WT_TRET(nenc->encryptor->terminate(
 			    nenc->encryptor, (WT_SESSION *)session));
 
-		/* Remove from the connection's list, free memory. */
-		TAILQ_REMOVE(&conn->encryptqh, nenc, q);
 		__wt_free(session, nenc->name);
 		__wt_free(session, nenc);
 	}
@@ -680,13 +680,13 @@ __wt_conn_remove_extractor(WT_SESSION_IMPL *session)
 	conn = S2C(session);
 
 	while ((nextractor = TAILQ_FIRST(&conn->extractorqh)) != NULL) {
+		/* Remove from the connection's list, free memory. */
+		TAILQ_REMOVE(&conn->extractorqh, nextractor, q);
 		/* Call any termination method. */
 		if (nextractor->extractor->terminate != NULL)
 			WT_TRET(nextractor->extractor->terminate(
 			    nextractor->extractor, (WT_SESSION *)session));
 
-		/* Remove from the connection's list, free memory. */
-		TAILQ_REMOVE(&conn->extractorqh, nextractor, q);
 		__wt_free(session, nextractor->name);
 		__wt_free(session, nextractor);
 	}
diff --git a/src/conn/conn_dhandle.c b/src/conn/conn_dhandle.c
index 4b4f4b8bc3f..4a653dc4c8f 100644
--- a/src/conn/conn_dhandle.c
+++ b/src/conn/conn_dhandle.c
@@ -634,7 +634,7 @@ int
 __wt_conn_dhandle_discard(WT_SESSION_IMPL *session)
 {
 	WT_CONNECTION_IMPL *conn;
-	WT_DATA_HANDLE *dhandle;
+	WT_DATA_HANDLE *dhandle, *dhandle_tmp;
 	WT_DECL_RET;
 
 	conn = S2C(session);
@@ -680,10 +680,11 @@ restart:
 		WT_TRET(session->meta_cursor->close(session->meta_cursor));
 
 	/* Close the metadata file handle. */
-	while ((dhandle = TAILQ_FIRST(&conn->dhqh)) != NULL)
+	WT_TAILQ_SAFE_REMOVE_BEGIN(dhandle, &conn->dhqh, q, dhandle_tmp) {
 		WT_WITH_DHANDLE(session, dhandle,
 		    WT_TRET(__wt_conn_dhandle_discard_single(
 		    session, true, F_ISSET(conn, WT_CONN_IN_MEMORY))));
+	} WT_TAILQ_SAFE_REMOVE_END
 
 	return (ret);
 }
diff --git a/src/conn/conn_sweep.c b/src/conn/conn_sweep.c
index 4ad4050dd9e..fbedb938bd8 100644
--- a/src/conn/conn_sweep.c
+++ b/src/conn/conn_sweep.c
@@ -219,15 +219,12 @@ static int
 __sweep_remove_handles(WT_SESSION_IMPL *session)
 {
 	WT_CONNECTION_IMPL *conn;
-	WT_DATA_HANDLE *dhandle, *dhandle_next;
+	WT_DATA_HANDLE *dhandle, *dhandle_tmp;
 	WT_DECL_RET;
 
 	conn = S2C(session);
 
-	for (dhandle = TAILQ_FIRST(&conn->dhqh);
-	    dhandle != NULL;
-	    dhandle = dhandle_next) {
-		dhandle_next = TAILQ_NEXT(dhandle, q);
+	TAILQ_FOREACH_SAFE(dhandle, &conn->dhqh, q, dhandle_tmp) {
 		if (WT_IS_METADATA(dhandle))
 			continue;
 		if (!WT_DHANDLE_CAN_DISCARD(dhandle))
diff --git a/src/include/misc.h b/src/include/misc.h
index c982b74a858..c84368b235c 100644
--- a/src/include/misc.h
+++ b/src/include/misc.h
@@ -276,3 +276,22 @@ union __wt_rand_state {
 		uint32_t w, z;
 	} x;
 };
+
+/*
+ * WT_TAILQ_SAFE_REMOVE_BEGIN/END --
+ *	Macro to safely walk a TAILQ where we're expecting some underlying
+ * function to remove elements from the list, but we don't want to stop on
+ * error, nor do we want an error to turn into an infinite loop. Used during
+ * shutdown, when we're shutting down various lists. Unlike TAILQ_FOREACH_SAFE,
+ * this macro works even when the next element gets removed along with the
+ * current one.
+ */
+#define	WT_TAILQ_SAFE_REMOVE_BEGIN(var, head, field, tvar)		\
+	for ((tvar) = NULL; ((var) = TAILQ_FIRST(head)) != NULL;	\
+	    (tvar) = (var)) {						\
+		if ((tvar) == (var)) {					\
+			/* Leak the structure. */			\
+			TAILQ_REMOVE(head, (var), field);		\
+			continue;					\
+		}
+#define	WT_TAILQ_SAFE_REMOVE_END }
diff --git a/src/lsm/lsm_manager.c b/src/lsm/lsm_manager.c
index 82a6fc8f86c..88b3e1980be 100644
--- a/src/lsm/lsm_manager.c
+++ b/src/lsm/lsm_manager.c
@@ -500,7 +500,7 @@ void
 __wt_lsm_manager_clear_tree(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree)
 {
 	WT_LSM_MANAGER *manager;
-	WT_LSM_WORK_UNIT *current, *next;
+	WT_LSM_WORK_UNIT *current, *tmp;
 	uint64_t removed;
 
 	manager = &S2C(session)->lsm_manager;
@@ -508,11 +508,7 @@ __wt_lsm_manager_clear_tree(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree)
 
 	/* Clear out the tree from the switch queue */
 	__wt_spin_lock(session, &manager->switch_lock);
-
-	/* Structure the loop so that it's safe to free as we iterate */
-	for (current = TAILQ_FIRST(&manager->switchqh);
-	    current != NULL; current = next) {
-		next = TAILQ_NEXT(current, q);
+	TAILQ_FOREACH_SAFE(current, &manager->switchqh, q, tmp) {
 		if (current->lsm_tree != lsm_tree)
 			continue;
 		++removed;
@@ -522,9 +518,7 @@ __wt_lsm_manager_clear_tree(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree)
 	__wt_spin_unlock(session, &manager->switch_lock);
 	/* Clear out the tree from the application queue */
 	__wt_spin_lock(session, &manager->app_lock);
-	for (current = TAILQ_FIRST(&manager->appqh);
-	    current != NULL; current = next) {
-		next = TAILQ_NEXT(current, q);
+	TAILQ_FOREACH_SAFE(current, &manager->appqh, q, tmp) {
 		if (current->lsm_tree != lsm_tree)
 			continue;
 		++removed;
@@ -534,9 +528,7 @@ __wt_lsm_manager_clear_tree(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree)
 	__wt_spin_unlock(session, &manager->app_lock);
 	/* Clear out the tree from the manager queue */
 	__wt_spin_lock(session, &manager->manager_lock);
-	for (current = TAILQ_FIRST(&manager->managerqh);
-	    current != NULL; current = next) {
-		next = TAILQ_NEXT(current, q);
+	TAILQ_FOREACH_SAFE(current, &manager->managerqh, q, tmp) {
 		if (current->lsm_tree != lsm_tree)
 			continue;
 		++removed;
diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c
index d9c7a7d7284..fb8eb9d38a7 100644
--- a/src/lsm/lsm_tree.c
+++ b/src/lsm/lsm_tree.c
@@ -134,11 +134,12 @@ int
 __wt_lsm_tree_close_all(WT_SESSION_IMPL *session)
 {
 	WT_DECL_RET;
-	WT_LSM_TREE *lsm_tree;
+	WT_LSM_TREE *lsm_tree, *lsm_tree_tmp;
 
 	/* We are shutting down: the handle list lock isn't required. */
 
-	while ((lsm_tree = TAILQ_FIRST(&S2C(session)->lsmqh)) != NULL) {
+	WT_TAILQ_SAFE_REMOVE_BEGIN(lsm_tree,
+	    &S2C(session)->lsmqh, q, lsm_tree_tmp) {
 		/*
 		 * Tree close assumes that we have a reference to the tree
 		 * so it can tell when it's safe to do the close. We could
@@ -149,7 +150,7 @@ __wt_lsm_tree_close_all(WT_SESSION_IMPL *session)
 		(void)__wt_atomic_add32(&lsm_tree->refcnt, 1);
 		__lsm_tree_close(session, lsm_tree, true);
 		WT_TRET(__lsm_tree_discard(session, lsm_tree, true));
-	}
+	} WT_TAILQ_SAFE_REMOVE_END
 
 	return (ret);
 }
diff --git a/src/os_common/os_fhandle.c b/src/os_common/os_fhandle.c
index 2a67447f8d2..69a01b41d14 100644
--- a/src/os_common/os_fhandle.c
+++ b/src/os_common/os_fhandle.c
@@ -281,6 +281,41 @@ err:		if (open_called)
 }
 
 /*
+ * __handle_close --
+ *	Final close of a handle.
+ */
+static int
+__handle_close(WT_SESSION_IMPL *session, WT_FH *fh)
+{
+	WT_CONNECTION_IMPL *conn;
+	WT_DECL_RET;
+	uint64_t bucket;
+
+	conn = S2C(session);
+
+	if (fh->ref != 0) {
+		__wt_errx(session,
+		    "Closing a file handle with open references: %s", fh->name);
+		WT_TRET(EBUSY);
+	}
+
+	/* Remove from the list. */
+	bucket = fh->name_hash % WT_HASH_ARRAY_SIZE;
+	WT_FILE_HANDLE_REMOVE(conn, fh, bucket);
+	(void)__wt_atomic_sub32(&conn->open_file_count, 1);
+
+	__wt_spin_unlock(session, &conn->fh_lock);
+
+	/* Discard underlying resources. */
+	WT_TRET(fh->handle->close(fh->handle, (WT_SESSION *)session));
+
+	__wt_free(session, fh->name);
+	__wt_free(session, fh);
+
+	return (ret);
+}
+
+/*
  * __wt_close --
  *	Close a file handle.
  */
@@ -288,9 +323,7 @@ int
 __wt_close(WT_SESSION_IMPL *session, WT_FH **fhp)
 {
 	WT_CONNECTION_IMPL *conn;
-	WT_DECL_RET;
 	WT_FH *fh;
-	uint64_t bucket;
 
 	conn = S2C(session);
 
@@ -315,20 +348,7 @@ __wt_close(WT_SESSION_IMPL *session, WT_FH **fhp)
 		return (0);
 	}
 
-	/* Remove from the list. */
-	bucket = fh->name_hash % WT_HASH_ARRAY_SIZE;
-	WT_FILE_HANDLE_REMOVE(conn, fh, bucket);
-	(void)__wt_atomic_sub32(&conn->open_file_count, 1);
-
-	__wt_spin_unlock(session, &conn->fh_lock);
-
-	/* Discard underlying resources. */
-	ret = fh->handle->close(fh->handle, (WT_SESSION *)session);
-
-	__wt_free(session, fh->name);
-	__wt_free(session, fh);
-
-	return (ret);
+	return (__handle_close(session, fh));
 }
 
 /*
@@ -339,21 +359,10 @@ int
 __wt_close_connection_close(WT_SESSION_IMPL *session)
 {
 	WT_DECL_RET;
-	WT_FH *fh;
-	WT_CONNECTION_IMPL *conn;
+	WT_FH *fh, *fh_tmp;
 
-	conn = S2C(session);
-
-	while ((fh = TAILQ_FIRST(&conn->fhqh)) != NULL) {
-		if (fh->ref != 0) {
-			ret = EBUSY;
-			__wt_errx(session,
-			    "Connection has open file handles: %s", fh->name);
-		}
-
-		fh->ref = 1;
-
-		WT_TRET(__wt_close(session, &fh));
-	}
+	WT_TAILQ_SAFE_REMOVE_BEGIN(fh, &S2C(session)->fhqh, q, fh_tmp) {
+		WT_TRET(__handle_close(session, fh));
+	} WT_TAILQ_SAFE_REMOVE_END
 	return (ret);
 }
diff --git a/src/os_common/os_fs_inmemory.c b/src/os_common/os_fs_inmemory.c
index 3ea25530aef..e669ea2802d 100644
--- a/src/os_common/os_fs_inmemory.c
+++ b/src/os_common/os_fs_inmemory.c
@@ -52,7 +52,7 @@ __im_handle_search(WT_FILE_SYSTEM *file_system, const char *name)
  */
 static int
 __im_handle_remove(WT_SESSION_IMPL *session,
-    WT_FILE_SYSTEM *file_system, WT_FILE_HANDLE_INMEM *im_fh)
+    WT_FILE_SYSTEM *file_system, WT_FILE_HANDLE_INMEM *im_fh, bool force)
 {
 	WT_FILE_HANDLE *fhp;
 	WT_FILE_SYSTEM_INMEM *im_fs;
@@ -60,9 +60,11 @@ __im_handle_remove(WT_SESSION_IMPL *session,
 
 	im_fs = (WT_FILE_SYSTEM_INMEM *)file_system;
 
-	if (im_fh->ref != 0)
-		WT_RET_MSG(session, EBUSY,
-		    "%s: file-remove", im_fh->iface.name);
+	if (im_fh->ref != 0) {
+		__wt_err(session, EBUSY, "%s: file-remove", im_fh->iface.name);
+		if (!force)
+			return (EBUSY);
+	}
 
 	bucket = im_fh->name_hash % WT_HASH_ARRAY_SIZE;
 	WT_FILE_HANDLE_REMOVE(im_fs, im_fh, bucket);
@@ -205,7 +207,7 @@ __im_fs_remove(WT_FILE_SYSTEM *file_system,
 
 	ret = ENOENT;
 	if ((im_fh = __im_handle_search(file_system, name)) != NULL)
-		ret = __im_handle_remove(session, file_system, im_fh);
+		ret = __im_handle_remove(session, file_system, im_fh, false);
 
 	__wt_spin_unlock(session, &im_fs->lock);
 	return (ret);
@@ -511,15 +513,16 @@ static int
 __im_terminate(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session)
 {
 	WT_DECL_RET;
-	WT_FILE_HANDLE_INMEM *im_fh;
+	WT_FILE_HANDLE_INMEM *im_fh, *im_fh_tmp;
 	WT_FILE_SYSTEM_INMEM *im_fs;
 	WT_SESSION_IMPL *session;
 
 	session = (WT_SESSION_IMPL *)wt_session;
 	im_fs = (WT_FILE_SYSTEM_INMEM *)file_system;
 
-	while ((im_fh = TAILQ_FIRST(&im_fs->fhqh)) != NULL)
-		WT_TRET(__im_handle_remove(session, file_system, im_fh));
+	WT_TAILQ_SAFE_REMOVE_BEGIN(im_fh, &im_fs->fhqh, q, im_fh_tmp) {
+		WT_TRET(__im_handle_remove(session, file_system, im_fh, true));
+	} WT_TAILQ_SAFE_REMOVE_END
 
 	__wt_spin_destroy(session, &im_fs->lock);
 	__wt_free(session, im_fs);
diff --git a/src/schema/schema_list.c b/src/schema/schema_list.c
index bbdc3568af3..20e65d5acc9 100644
--- a/src/schema/schema_list.c
+++ b/src/schema/schema_list.c
@@ -244,9 +244,11 @@ int
 __wt_schema_close_tables(WT_SESSION_IMPL *session)
 {
 	WT_DECL_RET;
-	WT_TABLE *table;
+	WT_TABLE *table, *table_tmp;
 
-	while ((table = TAILQ_FIRST(&session->tables)) != NULL)
+	WT_TAILQ_SAFE_REMOVE_BEGIN(table, &session->tables, q, table_tmp) {
 		WT_TRET(__wt_schema_remove_table(session, table));
+	} WT_TAILQ_SAFE_REMOVE_END
+
 	return (ret);
 }
diff --git a/src/session/session_api.c b/src/session/session_api.c
index 21f03915e2a..3c01dadc48c 100644
--- a/src/session/session_api.c
+++ b/src/session/session_api.c
@@ -183,7 +183,7 @@ static int
 __session_close(WT_SESSION *wt_session, const char *config)
 {
 	WT_CONNECTION_IMPL *conn;
-	WT_CURSOR *cursor;
+	WT_CURSOR *cursor, *cursor_tmp;
 	WT_DECL_RET;
 	WT_SESSION_IMPL *session;
 
@@ -205,7 +205,7 @@ __session_close(WT_SESSION *wt_session, const char *config)
 		__wt_txn_release_snapshot(session);
 
 	/* Close all open cursors. */
-	while ((cursor = TAILQ_FIRST(&session->cursors)) != NULL) {
+	WT_TAILQ_SAFE_REMOVE_BEGIN(cursor, &session->cursors, q, cursor_tmp) {
 		/*
 		 * Notify the user that we are closing the cursor handle
 		 * via the registered close callback.
@@ -215,7 +215,7 @@ __session_close(WT_SESSION *wt_session, const char *config)
 			WT_TRET(session->event_handler->handle_close(
 			    session->event_handler, wt_session, cursor));
 		WT_TRET(cursor->close(cursor));
-	}
+	} WT_TAILQ_SAFE_REMOVE_END
 
 	WT_ASSERT(session, session->ncursors == 0);
 
diff --git a/src/session/session_dhandle.c b/src/session/session_dhandle.c
index 3cfbfcead36..2d0a2eeb2dc 100644
--- a/src/session/session_dhandle.c
+++ b/src/session/session_dhandle.c
@@ -369,10 +369,12 @@ retry:			WT_RET(__wt_meta_checkpoint_last_name(
 void
 __wt_session_close_cache(WT_SESSION_IMPL *session)
 {
-	WT_DATA_HANDLE_CACHE *dhandle_cache;
+	WT_DATA_HANDLE_CACHE *dhandle_cache, *dhandle_cache_tmp;
 
-	while ((dhandle_cache = TAILQ_FIRST(&session->dhandles)) != NULL)
+	WT_TAILQ_SAFE_REMOVE_BEGIN(dhandle_cache,
+	    &session->dhandles, q, dhandle_cache_tmp) {
 		__session_discard_dhandle(session, dhandle_cache);
+	} WT_TAILQ_SAFE_REMOVE_END
 }
 
 /*
@@ -384,7 +386,7 @@ __session_dhandle_sweep(WT_SESSION_IMPL *session)
 {
 	WT_CONNECTION_IMPL *conn;
 	WT_DATA_HANDLE *dhandle;
-	WT_DATA_HANDLE_CACHE *dhandle_cache, *dhandle_cache_next;
+	WT_DATA_HANDLE_CACHE *dhandle_cache, *dhandle_cache_tmp;
 	time_t now;
 
 	conn = S2C(session);
@@ -400,9 +402,8 @@ __session_dhandle_sweep(WT_SESSION_IMPL *session)
 
 	WT_STAT_CONN_INCR(session, dh_session_sweeps);
 
-	dhandle_cache = TAILQ_FIRST(&session->dhandles);
-	while (dhandle_cache != NULL) {
-		dhandle_cache_next = TAILQ_NEXT(dhandle_cache, q);
+	TAILQ_FOREACH_SAFE(dhandle_cache,
+	    &session->dhandles, q, dhandle_cache_tmp) {
 		dhandle = dhandle_cache->dhandle;
 		if (dhandle != session->dhandle &&
 		    dhandle->session_inuse == 0 &&
@@ -414,7 +415,6 @@ __session_dhandle_sweep(WT_SESSION_IMPL *session)
 			WT_ASSERT(session, !WT_IS_METADATA(dhandle));
 			__session_discard_dhandle(session, dhandle_cache);
 		}
-		dhandle_cache = dhandle_cache_next;
 	}
 }