WT-2969 Possible snapshot corruption during compaction (#3160)

* Change compaction to use database-wide checkpoints: rather than processing each file separately and checkpointing after each time we compact a file, do a compaction pass over all of the files then database-wide checkpoints.

* Save/restore the WT_SESSION_CAN_WAIT, WT_SESSION_NO_EVICTION flags set during checkpoint, the session handle may be used for other tasks in the future.

* There's no need to hold the checkpoint lock while opening the metadata cursor.

* When adc1cfb went in (WT-2394, gather handles for compaction first, before doing the actual compaction), we broke data-source support for compaction. Add back in data-source support and simplify the __wt_schema_worker() code, it no longer needs to know about LSM or data-source compaction.

* Disallow LSM compaction in an existing transaction (LSM didn't check, but there's no reason to special case LSM compaction so it can run in an existing transaction, and it's potentially confusing, or fragile if LSM compaction some day requires checkpoints).

* Do a checkpoint after removing the key/value pairs, otherwise we might not find anything to work with in our compaction pass.

* Replace WT_SESSION_LOCK_NO_WAIT with per-lock operation flags. If we set WT_SESSION_LOCK_NO_WAIT and the lock is acquired, then the underlying operation eventually needs to acquire its own locks, having the WT_SESSION_LOCK_NO_WAIT flag set in the session may not be correct.

6 files changed, 14 insertions, 20 deletions

diff --git a/src/lsm/lsm_cursor.c b/src/lsm/lsm_cursor.c
index 9c124885086..df209ce3b05 100644
--- a/src/lsm/lsm_cursor.c
+++ b/src/lsm/lsm_cursor.c
@@ -267,7 +267,7 @@ __clsm_enter(WT_CURSOR_LSM *clsm, bool reset, bool update)
 		    (!update && F_ISSET(clsm, WT_CLSM_OPEN_READ))))
 			break;
 
-open:		WT_WITH_SCHEMA_LOCK(session, ret,
+open:		WT_WITH_SCHEMA_LOCK(session,
 		    ret = __clsm_open_cursors(clsm, update, 0, 0));
 		WT_RET(ret);
 	}
@@ -763,7 +763,7 @@ __wt_clsm_init_merge(
 		F_SET(clsm, WT_CLSM_MINOR_MERGE);
 	clsm->nchunks = nchunks;
 
-	WT_WITH_SCHEMA_LOCK(session, ret,
+	WT_WITH_SCHEMA_LOCK(session,
 	    ret = __clsm_open_cursors(clsm, false, start_chunk, start_id));
 	return (ret);
 }
diff --git a/src/lsm/lsm_cursor_bulk.c b/src/lsm/lsm_cursor_bulk.c
index 319426de3f0..bae8206515e 100644
--- a/src/lsm/lsm_cursor_bulk.c
+++ b/src/lsm/lsm_cursor_bulk.c
@@ -113,7 +113,7 @@ __wt_clsm_open_bulk(WT_CURSOR_LSM *clsm, const char *cfg[])
 	 * switch inline, since switch needs a schema lock and online index
 	 * creation opens a bulk cursor while holding the schema lock.
 	 */
-	WT_WITH_SCHEMA_LOCK(session, ret,
+	WT_WITH_SCHEMA_LOCK(session,
 	    ret = __wt_lsm_tree_switch(session, lsm_tree));
 	WT_RET(ret);
 
diff --git a/src/lsm/lsm_merge.c b/src/lsm/lsm_merge.c
index dc47d0118a2..f05a9c4b2b7 100644
--- a/src/lsm/lsm_merge.c
+++ b/src/lsm/lsm_merge.c
@@ -434,7 +434,7 @@ __wt_lsm_merge(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, u_int id)
 	F_SET(src, WT_CURSTD_RAW);
 	WT_ERR(__wt_clsm_init_merge(src, start_chunk, start_id, nchunks));
 
-	WT_WITH_SCHEMA_LOCK(session, ret,
+	WT_WITH_SCHEMA_LOCK(session,
 	    ret = __wt_lsm_tree_setup_chunk(session, lsm_tree, chunk));
 	WT_ERR(ret);
 	if (create_bloom) {
@@ -604,13 +604,13 @@ err:	if (locked)
 	if (ret != 0 && created_chunk) {
 		/* Drop the newly-created files on error. */
 		if (chunk->uri != NULL) {
-			WT_WITH_SCHEMA_LOCK(session, tret,
+			WT_WITH_SCHEMA_LOCK(session,
 			    tret = __wt_schema_drop(
 			    session, chunk->uri, drop_cfg));
 			WT_TRET(tret);
 		}
 		if (create_bloom && chunk->bloom_uri != NULL) {
-			WT_WITH_SCHEMA_LOCK(session, tret,
+			WT_WITH_SCHEMA_LOCK(session,
 			    tret = __wt_schema_drop(
 			    session, chunk->bloom_uri, drop_cfg));
 			WT_TRET(tret);
diff --git a/src/lsm/lsm_stat.c b/src/lsm/lsm_stat.c
index 3fe3ca1ba81..150de968722 100644
--- a/src/lsm/lsm_stat.c
+++ b/src/lsm/lsm_stat.c
@@ -178,7 +178,7 @@ __wt_curstat_lsm_init(
 	 * Grab the schema lock because we will be locking the LSM tree and we
 	 * may need to open some files.
 	 */
-	WT_WITH_SCHEMA_LOCK(session, ret,
+	WT_WITH_SCHEMA_LOCK(session,
 	    ret = __curstat_lsm_init(session, uri, cst));
 
 	return (ret);
diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c
index 714007cda98..80d20ad24f8 100644
--- a/src/lsm/lsm_tree.c
+++ b/src/lsm/lsm_tree.c
@@ -251,7 +251,7 @@ __lsm_tree_cleanup_old(WT_SESSION_IMPL *session, const char *uri)
 
 	WT_RET(__wt_fs_exist(session, uri + strlen("file:"), &exists));
 	if (exists)
-		WT_WITH_SCHEMA_LOCK(session, ret,
+		WT_WITH_SCHEMA_LOCK(session,
 		    ret = __wt_schema_drop(session, uri, cfg));
 	return (ret);
 }
@@ -1102,7 +1102,6 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, bool *skipp)
 	WT_DECL_RET;
 	WT_LSM_CHUNK *chunk;
 	WT_LSM_TREE *lsm_tree;
-	time_t begin, end;
 	uint64_t progress;
 	uint32_t i;
 	bool compacting, flushing, locked, ref;
@@ -1139,8 +1138,6 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, bool *skipp)
 		return (0);
 	}
 
-	__wt_seconds(session, &begin);
-
 	/*
 	 * Compacting has two distinct phases.
 	 * 1.  All in-memory chunks up to and including the current
@@ -1266,12 +1263,9 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, bool *skipp)
 			} else
 				break;
 		}
+		WT_ERR(__wt_session_compact_check_timeout(session));
 		__wt_sleep(1, 0);
-		__wt_seconds(session, &end);
-		if (session->compact->max_time > 0 &&
-		    session->compact->max_time < (uint64_t)(end - begin)) {
-			WT_ERR(ETIMEDOUT);
-		}
+
 		/*
 		 * Push merge operations while they are still getting work
 		 * done. If we are pushing merges, make sure they are
diff --git a/src/lsm/lsm_work_unit.c b/src/lsm/lsm_work_unit.c
index 66a57f15875..f3414363e3e 100644
--- a/src/lsm/lsm_work_unit.c
+++ b/src/lsm/lsm_work_unit.c
@@ -172,7 +172,7 @@ __wt_lsm_work_switch(
 	*entryp = NULL;
 
 	if (entry->lsm_tree->need_switch) {
-		WT_WITH_SCHEMA_LOCK(session, ret,
+		WT_WITH_SCHEMA_LOCK(session,
 		    ret = __wt_lsm_tree_switch(session, entry->lsm_tree));
 		/* Failing to complete the switch is fine */
 		if (ret == EBUSY) {
@@ -346,8 +346,8 @@ __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session,
 	 * time, and our checkpoint operation should be very quick.
 	 */
 	WT_ERR(__wt_meta_track_on(session));
-	WT_WITH_CHECKPOINT_LOCK(session, ret,
-	    WT_WITH_SCHEMA_LOCK(session, ret,
+	WT_WITH_CHECKPOINT_LOCK(session,
+	    WT_WITH_SCHEMA_LOCK(session,
 		ret = __wt_schema_worker(
 		session, chunk->uri, __wt_checkpoint, NULL, NULL, 0)));
 	WT_TRET(__wt_meta_track_off(session, false, ret != 0));
@@ -526,7 +526,7 @@ __lsm_drop_file(WT_SESSION_IMPL *session, const char *uri)
 	 * results in the hot backup lock being taken when it updates the
 	 * metadata (which would be too late to prevent our drop).
 	 */
-	WT_WITH_SCHEMA_LOCK(session, ret,
+	WT_WITH_SCHEMA_LOCK(session,
 	    ret = __wt_schema_drop(session, uri, drop_cfg));
 
 	if (ret == 0)