Merge branch 'develop' into WT-2333

8 files changed, 129 insertions, 133 deletions

diff --git a/src/block/block_open.c b/src/block/block_open.c
index 981f1907489..dd0f3f0716a 100644
--- a/src/block/block_open.c
+++ b/src/block/block_open.c
@@ -11,39 +11,6 @@
 static int __desc_read(WT_SESSION_IMPL *, WT_BLOCK *);
 
 /*
- * __wt_block_manager_truncate --
- *	Truncate a file.
- */
-int
-__wt_block_manager_truncate(
-    WT_SESSION_IMPL *session, const char *filename, uint32_t allocsize)
-{
-	WT_DECL_RET;
-	WT_FH *fh;
-
-	/* Open the underlying file handle. */
-	WT_RET(__wt_open(
-	    session, filename, false, false, WT_FILE_TYPE_DATA, &fh));
-
-	/* Truncate the file. */
-	WT_ERR(__wt_block_truncate(session, fh, (wt_off_t)0));
-
-	/* Write out the file's meta-data. */
-	WT_ERR(__wt_desc_init(session, fh, allocsize));
-
-	/*
-	 * Ensure the truncated file has made it to disk, then the upper-level
-	 * is never surprised.
-	 */
-	WT_ERR(__wt_fsync(session, fh));
-
-	/* Close the file handle. */
-err:	WT_TRET(__wt_close(session, &fh));
-
-	return (ret);
-}
-
-/*
  * __wt_block_manager_drop --
  *	Drop a file.
  */
diff --git a/src/conn/conn_log.c b/src/conn/conn_log.c
index 3b5e8039fb4..ed226393fb0 100644
--- a/src/conn/conn_log.c
+++ b/src/conn/conn_log.c
@@ -705,12 +705,12 @@ __log_server(void *arg)
 	WT_LOG *log;
 	WT_SESSION_IMPL *session;
 	int freq_per_sec;
-	bool signalled;
+	bool locked, signalled;
 
 	session = arg;
 	conn = S2C(session);
 	log = conn->log;
-	signalled = false;
+	locked = signalled = false;
 
 	/*
 	 * Set this to the number of times per second we want to force out the
@@ -751,8 +751,22 @@ __log_server(void *arg)
 			/*
 			 * Perform log pre-allocation.
 			 */
-			if (conn->log_prealloc > 0)
-				WT_ERR(__log_prealloc_once(session));
+			if (conn->log_prealloc > 0) {
+				/*
+				 * Log file pre-allocation is disabled when a
+				 * hot backup cursor is open because we have
+				 * agreed not to rename or remove any files in
+				 * the database directory.
+				 */
+				WT_ERR(__wt_readlock(
+				    session, conn->hot_backup_lock));
+				locked = true;
+				if (!conn->hot_backup)
+					WT_ERR(__log_prealloc_once(session));
+				WT_ERR(__wt_readunlock(
+				    session, conn->hot_backup_lock));
+				locked = false;
+			}
 
 			/*
 			 * Perform the archive.
@@ -779,6 +793,9 @@ __log_server(void *arg)
 
 	if (0) {
 err:		__wt_err(session, ret, "log server error");
+		if (locked)
+			WT_TRET(__wt_readunlock(
+			    session, conn->hot_backup_lock));
 	}
 	return (WT_THREAD_RET_VALUE);
 }
diff --git a/src/docs/upgrading.dox b/src/docs/upgrading.dox
index e80c955a4a9..e0239919f0b 100644
--- a/src/docs/upgrading.dox
+++ b/src/docs/upgrading.dox
@@ -14,6 +14,18 @@ already-deleted rows.) To match the previous behavior, specify the
 \c append configuration string when opening the column-store bulk-load
 cursor; this causes the cursor's key to be ignored and each inserted row
 will be assigned the next record number.
+</dd>
+
+<dt>Change to WT_SESSION::truncate with URI</dt>
+<dd>
+If using the WT_SESSION::truncate API with a file: URI for a full table
+truncate, underlying algorithmic changes result in some visible differences.
+This call can now return WT_ROLLBACK.  Applications should be prepared to
+handle this error.  This method no longer requires exclusive access to the
+table.  Also the underlying disk space may not be immediately
+reclaimed when the call returns.  The performance of this API may differ
+from earlier releases.
+</dd>
 
 </dl><hr>
 @section version_270 Upgrading to Version 2.7.0
diff --git a/src/include/extern.h b/src/include/extern.h
index 0a2d1645b2f..b71f4b12486 100644
--- a/src/include/extern.h
+++ b/src/include/extern.h
@@ -44,7 +44,6 @@ extern void __wt_block_extlist_free(WT_SESSION_IMPL *session, WT_EXTLIST *el);
 extern int __wt_block_map( WT_SESSION_IMPL *session, WT_BLOCK *block, void *mapp, size_t *maplenp, void **mappingcookie);
 extern int __wt_block_unmap( WT_SESSION_IMPL *session, WT_BLOCK *block, void *map, size_t maplen, void **mappingcookie);
 extern int __wt_block_manager_open(WT_SESSION_IMPL *session, const char *filename, const char *cfg[], bool forced_salvage, bool readonly, uint32_t allocsize, WT_BM **bmp);
-extern int __wt_block_manager_truncate( WT_SESSION_IMPL *session, const char *filename, uint32_t allocsize);
 extern int __wt_block_manager_drop(WT_SESSION_IMPL *session, const char *filename);
 extern int __wt_block_manager_create( WT_SESSION_IMPL *session, const char *filename, uint32_t allocsize);
 extern void __wt_block_configure_first_fit(WT_BLOCK *block, bool on);
@@ -618,6 +617,7 @@ extern int __wt_session_release_resources(WT_SESSION_IMPL *session);
 extern int __wt_open_cursor(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp);
 extern int __wt_session_create( WT_SESSION_IMPL *session, const char *uri, const char *config);
 extern int __wt_session_drop(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]);
+extern int __wt_session_range_truncate(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *start, WT_CURSOR *stop);
 extern int __wt_open_session(WT_CONNECTION_IMPL *conn, WT_EVENT_HANDLER *event_handler, const char *config, bool open_metadata, WT_SESSION_IMPL **sessionp);
 extern int __wt_open_internal_session(WT_CONNECTION_IMPL *conn, const char *name, bool open_metadata, uint32_t session_flags, WT_SESSION_IMPL **sessionp);
 extern int __wt_compact_uri_analyze(WT_SESSION_IMPL *session, const char *uri, bool *skipp);
diff --git a/src/log/log.c b/src/log/log.c
index f239a8e7001..9c3269c474c 100644
--- a/src/log/log.c
+++ b/src/log/log.c
@@ -791,9 +791,10 @@ __log_newfile(WT_SESSION_IMPL *session, bool conn_open, bool *created)
 	WT_FULL_BARRIER();
 	/*
 	 * If we're pre-allocating log files, look for one.  If there aren't any
-	 * or we're not pre-allocating, then create one.
+	 * or we're not pre-allocating, or a backup cursor is open, then
+	 * create one.
 	 */
-	if (conn->log_prealloc > 0) {
+	if (conn->log_prealloc > 0 && !conn->hot_backup) {
 		ret = __log_alloc_prealloc(session, log->fileid);
 		/*
 		 * If ret is 0 it means we found a pre-allocated file.
diff --git a/src/schema/schema_truncate.c b/src/schema/schema_truncate.c
index 52a73b29fa4..e7752b60ca4 100644
--- a/src/schema/schema_truncate.c
+++ b/src/schema/schema_truncate.c
@@ -9,43 +9,6 @@
 #include "wt_internal.h"
 
 /*
- * __truncate_file --
- *	WT_SESSION::truncate for a file.
- */
-static int
-__truncate_file(WT_SESSION_IMPL *session, const char *uri)
-{
-	WT_DECL_RET;
-	const char *filename;
-	uint32_t allocsize;
-
-	filename = uri;
-	if (!WT_PREFIX_SKIP(filename, "file:"))
-		return (EINVAL);
-
-	/* Open and lock the file. */
-	WT_RET(__wt_session_get_btree(
-	    session, uri, NULL, NULL, WT_DHANDLE_EXCLUSIVE));
-	WT_STAT_FAST_DATA_INCR(session, cursor_truncate);
-
-	/* Get the allocation size. */
-	allocsize = S2BT(session)->allocsize;
-
-	WT_RET(__wt_session_release_btree(session));
-
-	/* Close any btree handles in the file. */
-	WT_WITH_HANDLE_LIST_LOCK(session, ret,
-	    ret = __wt_conn_dhandle_close_all(session, uri, false));
-	WT_RET(ret);
-
-	/* Delete the root address and truncate the file. */
-	WT_RET(__wt_meta_checkpoint_clear(session, uri));
-	WT_RET(__wt_block_manager_truncate(session, filename, allocsize));
-
-	return (0);
-}
-
-/*
  * __truncate_table --
  *	WT_SESSION::truncate for a table.
  */
@@ -112,9 +75,12 @@ __wt_schema_truncate(
 
 	tablename = uri;
 
-	if (WT_PREFIX_MATCH(uri, "file:")) {
-		ret = __truncate_file(session, uri);
-	} else if (WT_PREFIX_MATCH(uri, "lsm:"))
+	if (WT_PREFIX_MATCH(uri, "file:"))
+		/*
+		 * File truncate translates into a range truncate.
+		 */
+		ret = __wt_session_range_truncate(session, uri, NULL, NULL);
+	else if (WT_PREFIX_MATCH(uri, "lsm:"))
 		ret = __wt_lsm_tree_truncate(session, uri, cfg);
 	else if (WT_PREFIX_SKIP(tablename, "table:"))
 		ret = __truncate_table(session, tablename, cfg);
diff --git a/src/session/session_api.c b/src/session/session_api.c
index baf1f23419b..c03b5fdc044 100644
--- a/src/session/session_api.c
+++ b/src/session/session_api.c
@@ -823,62 +823,39 @@ err:	API_END_RET_NOTFOUND_MAP(session, ret);
 }
 
 /*
- * __session_truncate --
- *	WT_SESSION->truncate method.
+ * __wt_session_range_truncate --
+ *	Session handling of a range truncate.
  */
-static int
-__session_truncate(WT_SESSION *wt_session,
-    const char *uri, WT_CURSOR *start, WT_CURSOR *stop, const char *config)
+int
+__wt_session_range_truncate(WT_SESSION_IMPL *session,
+    const char *uri, WT_CURSOR *start, WT_CURSOR *stop)
 {
-	WT_DECL_RET;
-	WT_SESSION_IMPL *session;
 	WT_CURSOR *cursor;
+	WT_DECL_RET;
 	int cmp;
 	bool local_start;
 
 	local_start = false;
-
-	session = (WT_SESSION_IMPL *)wt_session;
-	SESSION_TXN_API_CALL(session, truncate, config, cfg);
-	WT_STAT_FAST_CONN_INCR(session, cursor_truncate);
-
-	/*
-	 * If the URI is specified, we don't need a start/stop, if start/stop
-	 * is specified, we don't need a URI.  One exception is the log URI
-	 * which may truncate (archive) log files for a backup cursor.
-	 *
-	 * If no URI is specified, and both cursors are specified, start/stop
-	 * must reference the same object.
-	 *
-	 * Any specified cursor must have been initialized.
-	 */
-	if ((uri == NULL && start == NULL && stop == NULL) ||
-	    (uri != NULL && !WT_PREFIX_MATCH(uri, "log:") &&
-	    (start != NULL || stop != NULL)))
-		WT_ERR_MSG(session, EINVAL,
-		    "the truncate method should be passed either a URI or "
-		    "start/stop cursors, but not both");
-
 	if (uri != NULL) {
-		/* Disallow objects in the WiredTiger name space. */
-		WT_ERR(__wt_str_name_check(session, uri));
-
-		if (WT_PREFIX_MATCH(uri, "log:")) {
+		WT_ASSERT(session, WT_PREFIX_MATCH(uri, "file:"));
+		/*
+		 * A URI file truncate becomes a range truncate where we
+		 * set a start cursor at the beginning.  We already
+		 * know the NULL stop goes to the end of the range.
+		 */
+		WT_ERR(__session_open_cursor(
+		    (WT_SESSION *)session, uri, NULL, NULL, &start));
+		local_start = true;
+		ret = start->next(start);
+		if (ret == WT_NOTFOUND) {
 			/*
-			 * Verify the user only gave the URI prefix and not
-			 * a specific target name after that.
+			 * If there are no elements, there is nothing
+			 * to do.
 			 */
-			if (!WT_STREQ(uri, "log:"))
-				WT_ERR_MSG(session, EINVAL,
-				    "the truncate method should not specify any"
-				    "target after the log: URI prefix.");
-			ret = __wt_log_truncate_files(session, start, cfg);
-		} else
-			/* Wait for checkpoints to avoid EBUSY errors. */
-			WT_WITH_CHECKPOINT_LOCK(session, ret,
-			    WT_WITH_SCHEMA_LOCK(session, ret,
-				ret = __wt_schema_truncate(session, uri, cfg)));
-		goto done;
+			ret = 0;
+			goto done;
+		}
+		WT_ERR(ret);
 	}
 
 	/*
@@ -936,7 +913,7 @@ __session_truncate(WT_SESSION *wt_session,
 	 */
 	if (start == NULL) {
 		WT_ERR(__session_open_cursor(
-		    wt_session, stop->uri, NULL, NULL, &start));
+		    (WT_SESSION *)session, stop->uri, NULL, NULL, &start));
 		local_start = true;
 		WT_ERR(start->next(start));
 	}
@@ -953,13 +930,72 @@ __session_truncate(WT_SESSION *wt_session,
 	WT_ERR(__wt_schema_range_truncate(session, start, stop));
 
 done:
-err:	TXN_API_END_RETRY(session, ret, 0);
-
-	/*
+err:	/*
 	 * Close any locally-opened start cursor.
 	 */
 	if (local_start)
 		WT_TRET(start->close(start));
+	return (ret);
+}
+
+/*
+ * __session_truncate --
+ *	WT_SESSION->truncate method.
+ */
+static int
+__session_truncate(WT_SESSION *wt_session,
+    const char *uri, WT_CURSOR *start, WT_CURSOR *stop, const char *config)
+{
+	WT_DECL_RET;
+	WT_SESSION_IMPL *session;
+
+	session = (WT_SESSION_IMPL *)wt_session;
+	SESSION_TXN_API_CALL(session, truncate, config, cfg);
+	WT_STAT_FAST_CONN_INCR(session, cursor_truncate);
+
+	/*
+	 * If the URI is specified, we don't need a start/stop, if start/stop
+	 * is specified, we don't need a URI.  One exception is the log URI
+	 * which may truncate (archive) log files for a backup cursor.
+	 *
+	 * If no URI is specified, and both cursors are specified, start/stop
+	 * must reference the same object.
+	 *
+	 * Any specified cursor must have been initialized.
+	 */
+	if ((uri == NULL && start == NULL && stop == NULL) ||
+	    (uri != NULL && !WT_PREFIX_MATCH(uri, "log:") &&
+	    (start != NULL || stop != NULL)))
+		WT_ERR_MSG(session, EINVAL,
+		    "the truncate method should be passed either a URI or "
+		    "start/stop cursors, but not both");
+
+	if (uri != NULL) {
+		/* Disallow objects in the WiredTiger name space. */
+		WT_ERR(__wt_str_name_check(session, uri));
+
+		if (WT_PREFIX_MATCH(uri, "log:")) {
+			/*
+			 * Verify the user only gave the URI prefix and not
+			 * a specific target name after that.
+			 */
+			if (!WT_STREQ(uri, "log:"))
+				WT_ERR_MSG(session, EINVAL,
+				    "the truncate method should not specify any"
+				    "target after the log: URI prefix.");
+			WT_ERR(__wt_log_truncate_files(session, start, cfg));
+		} else if (WT_PREFIX_MATCH(uri, "file:"))
+			WT_ERR(__wt_session_range_truncate(
+			    session, uri, start, stop));
+		else
+			/* Wait for checkpoints to avoid EBUSY errors. */
+			WT_WITH_CHECKPOINT_LOCK(session, ret,
+			    WT_WITH_SCHEMA_LOCK(session, ret,
+				ret = __wt_schema_truncate(session, uri, cfg)));
+	} else
+		WT_ERR(__wt_session_range_truncate(session, uri, start, stop));
+
+err:	TXN_API_END_RETRY(session, ret, 0);
 
 	/*
 	 * Only map WT_NOTFOUND to ENOENT if a URI was specified.
diff --git a/test/suite/test_bug006.py b/test/suite/test_bug006.py
index 12e254b8832..e522cdf96f7 100644
--- a/test/suite/test_bug006.py
+++ b/test/suite/test_bug006.py
@@ -58,9 +58,6 @@ class test_bug006(wttest.WiredTigerTestCase):
         self.assertRaises(
             wiredtiger.WiredTigerError, lambda: self.session.salvage(uri, None))
         self.assertRaises(
-            wiredtiger.WiredTigerError,
-            lambda: self.session.truncate(uri, None, None, None))
-        self.assertRaises(
             wiredtiger.WiredTigerError, lambda: self.session.upgrade(uri, None))
         self.assertRaises(
             wiredtiger.WiredTigerError, lambda: self.session.verify(uri, None))