Move begin_transaction from __wt_txn_checkpoint to __session_checkpoint.

This avoids a possible deadlock trying to reset cursors while holding the
schema lock.

2 files changed, 40 insertions, 20 deletions

diff --git a/src/session/session_api.c b/src/session/session_api.c
index 4d408f2aa37..acb87c16d9e 100644
--- a/src/session/session_api.c
+++ b/src/session/session_api.c
@@ -568,11 +568,40 @@ __session_checkpoint(WT_SESSION *wt_session, const char *config)
 {
 	WT_DECL_RET;
 	WT_SESSION_IMPL *session;
+	WT_TXN *txn;
 
 	session = (WT_SESSION_IMPL *)wt_session;
-	WT_CSTAT_INCR(session, checkpoint);
+	txn = &session->txn;
 
+	WT_CSTAT_INCR(session, checkpoint);
 	SESSION_API_CALL(session, checkpoint, config, cfg);
+
+	/*
+	 * Checkpoints require a snapshot to write a transactionally consistent
+	 * snapshot of the data.
+	 *
+	 * We can't use an application's transaction: if it has uncommitted
+	 * changes, they will be written in the checkpoint and may appear after
+	 * a crash.
+	 *
+	 * Use a real snapshot transaction: we don't want any chance of the
+	 * snapshot being updated during the checkpoint.  Eviction is prevented
+	 * from evicting anything newer than this because we track the oldest
+	 * transaction ID in the system that is not visible to all readers.
+	 */
+	if (F_ISSET(txn, TXN_RUNNING))
+		WT_ERR_MSG(session, EINVAL,
+		    "Checkpoint not permitted in a transaction");
+
+	/*
+	 * Reset open cursors.
+	 *
+	 * We do this here explicitly even though it will happen implicitly in
+	 * the call to begin_transaction for the checkpoint, in case some
+	 * implementation of WT_CURSOR::reset needs the schema lock.
+	 */
+	WT_ERR(__session_reset_cursors(session));
+
 	WT_WITH_SCHEMA_LOCK(session,
 	    ret = __wt_txn_checkpoint(session, cfg));
 
diff --git a/src/txn/txn_ckpt.c b/src/txn/txn_ckpt.c
index e873e1affd4..8d2a468bfef 100644
--- a/src/txn/txn_ckpt.c
+++ b/src/txn/txn_ckpt.c
@@ -30,27 +30,17 @@ __wt_txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[])
 	txn = &session->txn;
 
 	/* Only one checkpoint can be active at a time. */
-	WT_ASSERT(session, F_ISSET(session, WT_SESSION_SCHEMA_LOCKED));
+	WT_ASSERT(session, F_ISSET(session, WT_SESSION_SCHEMA_LOCKED) &&
+	    !F_ISSET(txn, TXN_RUNNING));
 
 	/*
-	 * Checkpoints require a snapshot to write a transactionally consistent
-	 * snapshot of the data.
-	 *
-	 * We can't use an application's transaction: if it has uncommitted
-	 * changes, they will be written in the checkpoint and may appear after
-	 * a crash.
-	 *
-	 * Use a real snapshot transaction: we don't want any chance of the
-	 * snapshot being updated during the checkpoint.  Eviction is prevented
-	 * from evicting anything newer than this because we track the oldest
-	 * transaction ID in the system that is not visible to all readers.
+	 * Begin a transaction for the checkpoint.  Checkpoints must run in
+	 * the same order as they update the metadata and we are using the
+	 * schema lock to determine that ordering, so we can't move this to
+	 * __session_checkpoint.
 	 */
-	if (F_ISSET(txn, TXN_RUNNING))
-		WT_RET_MSG(session, EINVAL,
-		    "Checkpoint not permitted in a transaction");
-
 	wt_session = &session->iface;
-	WT_RET(wt_session->begin_transaction(wt_session, "isolation=snapshot"));
+	WT_ERR(wt_session->begin_transaction(wt_session, "isolation=snapshot"));
 
 	WT_ERR(__wt_meta_track_on(session));
 	tracking = 1;
@@ -144,8 +134,8 @@ err:	/*
 	if (tracking)
 		WT_TRET(__wt_meta_track_off(session, ret != 0));
 
-	__wt_txn_release(session);
 	__wt_scr_free(&tmp);
+	__wt_txn_release(session);
 	return (ret);
 }
 
@@ -467,6 +457,8 @@ __wt_checkpoint(WT_SESSION_IMPL *session, const char *cfg[])
 			    "checkpoints cannot be dropped when in-use");
 		}
 
+	WT_ASSERT(session, txn->isolation = TXN_ISO_SNAPSHOT);
+
 	/*
 	 * Mark the root page dirty to ensure something gets written.
 	 *
@@ -503,7 +495,6 @@ __wt_checkpoint(WT_SESSION_IMPL *session, const char *cfg[])
 	    ckptbase, is_checkpoint ? WT_SYNC : WT_SYNC_DISCARD));
 
 	/* Update the object's metadata. */
-	txn = &session->txn;
 	txn->isolation = TXN_ISO_READ_UNCOMMITTED;
 	ret = __wt_meta_ckptlist_set(session, btree->name, ckptbase);
 	WT_ERR(ret);