WT-2151 Add zero_fill config to manually zero out log files.

6 files changed, 116 insertions, 35 deletions

diff --git a/dist/api_data.py b/dist/api_data.py
index 5652edc4ebe..6faee4ddd7f 100644
--- a/dist/api_data.py
+++ b/dist/api_data.py
@@ -642,6 +642,9 @@ common_wiredtiger_open = [
             run recovery or error if recovery needs to run after an
             unclean shutdown.''',
             choices=['error','on']),
+        Config('zero_fill', 'false', r'''
+            manually write zeroes into log files''',
+            type='boolean'),
         ]),
     Config('mmap', 'true', r'''
         Use memory mapping to access files when possible''',
diff --git a/src/config/config_def.c b/src/config/config_def.c
index a3dc24fafc4..990e604cf39 100644
--- a/src/config/config_def.c
+++ b/src/config/config_def.c
@@ -463,6 +463,7 @@ static const WT_CONFIG_CHECK
 	{ "recover", "string",
 	    NULL, "choices=[\"error\",\"on\"]",
 	    NULL, 0 },
+	{ "zero_fill", "boolean", NULL, NULL, NULL, 0 },
 	{ NULL, NULL, NULL, NULL, NULL, 0 }
 };
 
@@ -517,7 +518,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = {
 	{ "hazard_max", "int", NULL, "min=15", NULL, 0 },
 	{ "log", "category",
 	    NULL, NULL,
-	    confchk_wiredtiger_open_log_subconfigs, 7 },
+	    confchk_wiredtiger_open_log_subconfigs, 8 },
 	{ "lsm_manager", "category",
 	    NULL, NULL,
 	    confchk_wiredtiger_open_lsm_manager_subconfigs, 2 },
@@ -592,7 +593,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = {
 	{ "hazard_max", "int", NULL, "min=15", NULL, 0 },
 	{ "log", "category",
 	    NULL, NULL,
-	    confchk_wiredtiger_open_log_subconfigs, 7 },
+	    confchk_wiredtiger_open_log_subconfigs, 8 },
 	{ "lsm_manager", "category",
 	    NULL, NULL,
 	    confchk_wiredtiger_open_lsm_manager_subconfigs, 2 },
@@ -665,7 +666,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = {
 	{ "hazard_max", "int", NULL, "min=15", NULL, 0 },
 	{ "log", "category",
 	    NULL, NULL,
-	    confchk_wiredtiger_open_log_subconfigs, 7 },
+	    confchk_wiredtiger_open_log_subconfigs, 8 },
 	{ "lsm_manager", "category",
 	    NULL, NULL,
 	    confchk_wiredtiger_open_lsm_manager_subconfigs, 2 },
@@ -737,7 +738,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = {
 	{ "hazard_max", "int", NULL, "min=15", NULL, 0 },
 	{ "log", "category",
 	    NULL, NULL,
-	    confchk_wiredtiger_open_log_subconfigs, 7 },
+	    confchk_wiredtiger_open_log_subconfigs, 8 },
 	{ "lsm_manager", "category",
 	    NULL, NULL,
 	    confchk_wiredtiger_open_lsm_manager_subconfigs, 2 },
@@ -969,13 +970,14 @@ static const WT_CONFIG_ENTRY config_entries[] = {
 	  "file_extend=,file_manager=(close_handle_minimum=250,"
 	  "close_idle_time=30,close_scan_interval=10),hazard_max=1000,"
 	  "log=(archive=,compressor=,enabled=0,file_max=100MB,path=,"
-	  "prealloc=,recover=on),lsm_manager=(merge=,worker_thread_max=4),"
-	  "lsm_merge=,mmap=,multiprocess=0,session_max=100,"
-	  "session_scratch_max=2MB,shared_cache=(chunk=10MB,name=,quota=0,"
-	  "reserve=0,size=500MB),statistics=none,statistics_log=(on_close=0"
-	  ",path=\"WiredTigerStat.%d.%H\",sources=,"
-	  "timestamp=\"%b %d %H:%M:%S\",wait=0),transaction_sync=(enabled=0"
-	  ",method=fsync),use_environment_priv=0,verbose=",
+	  "prealloc=,recover=on,zero_fill=0),lsm_manager=(merge=,"
+	  "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0,"
+	  "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB"
+	  ",name=,quota=0,reserve=0,size=500MB),statistics=none,"
+	  "statistics_log=(on_close=0,path=\"WiredTigerStat.%d.%H\","
+	  "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0),"
+	  "transaction_sync=(enabled=0,method=fsync),use_environment_priv=0"
+	  ",verbose=",
 	  confchk_wiredtiger_open, 34
 	},
 	{ "wiredtiger_open_all",
@@ -989,14 +991,14 @@ static const WT_CONFIG_ENTRY config_entries[] = {
 	  "file_extend=,file_manager=(close_handle_minimum=250,"
 	  "close_idle_time=30,close_scan_interval=10),hazard_max=1000,"
 	  "log=(archive=,compressor=,enabled=0,file_max=100MB,path=,"
-	  "prealloc=,recover=on),lsm_manager=(merge=,worker_thread_max=4),"
-	  "lsm_merge=,mmap=,multiprocess=0,session_max=100,"
-	  "session_scratch_max=2MB,shared_cache=(chunk=10MB,name=,quota=0,"
-	  "reserve=0,size=500MB),statistics=none,statistics_log=(on_close=0"
-	  ",path=\"WiredTigerStat.%d.%H\",sources=,"
-	  "timestamp=\"%b %d %H:%M:%S\",wait=0),transaction_sync=(enabled=0"
-	  ",method=fsync),use_environment_priv=0,verbose=,version=(major=0,"
-	  "minor=0)",
+	  "prealloc=,recover=on,zero_fill=0),lsm_manager=(merge=,"
+	  "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0,"
+	  "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB"
+	  ",name=,quota=0,reserve=0,size=500MB),statistics=none,"
+	  "statistics_log=(on_close=0,path=\"WiredTigerStat.%d.%H\","
+	  "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0),"
+	  "transaction_sync=(enabled=0,method=fsync),use_environment_priv=0"
+	  ",verbose=,version=(major=0,minor=0)",
 	  confchk_wiredtiger_open_all, 35
 	},
 	{ "wiredtiger_open_basecfg",
@@ -1009,13 +1011,14 @@ static const WT_CONFIG_ENTRY config_entries[] = {
 	  ",extensions=,file_extend=,file_manager=(close_handle_minimum=250"
 	  ",close_idle_time=30,close_scan_interval=10),hazard_max=1000,"
 	  "log=(archive=,compressor=,enabled=0,file_max=100MB,path=,"
-	  "prealloc=,recover=on),lsm_manager=(merge=,worker_thread_max=4),"
-	  "lsm_merge=,mmap=,multiprocess=0,session_max=100,"
-	  "session_scratch_max=2MB,shared_cache=(chunk=10MB,name=,quota=0,"
-	  "reserve=0,size=500MB),statistics=none,statistics_log=(on_close=0"
-	  ",path=\"WiredTigerStat.%d.%H\",sources=,"
-	  "timestamp=\"%b %d %H:%M:%S\",wait=0),transaction_sync=(enabled=0"
-	  ",method=fsync),verbose=,version=(major=0,minor=0)",
+	  "prealloc=,recover=on,zero_fill=0),lsm_manager=(merge=,"
+	  "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0,"
+	  "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB"
+	  ",name=,quota=0,reserve=0,size=500MB),statistics=none,"
+	  "statistics_log=(on_close=0,path=\"WiredTigerStat.%d.%H\","
+	  "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0),"
+	  "transaction_sync=(enabled=0,method=fsync),verbose=,"
+	  "version=(major=0,minor=0)",
 	  confchk_wiredtiger_open_basecfg, 31
 	},
 	{ "wiredtiger_open_usercfg",
@@ -1028,13 +1031,13 @@ static const WT_CONFIG_ENTRY config_entries[] = {
 	  ",extensions=,file_extend=,file_manager=(close_handle_minimum=250"
 	  ",close_idle_time=30,close_scan_interval=10),hazard_max=1000,"
 	  "log=(archive=,compressor=,enabled=0,file_max=100MB,path=,"
-	  "prealloc=,recover=on),lsm_manager=(merge=,worker_thread_max=4),"
-	  "lsm_merge=,mmap=,multiprocess=0,session_max=100,"
-	  "session_scratch_max=2MB,shared_cache=(chunk=10MB,name=,quota=0,"
-	  "reserve=0,size=500MB),statistics=none,statistics_log=(on_close=0"
-	  ",path=\"WiredTigerStat.%d.%H\",sources=,"
-	  "timestamp=\"%b %d %H:%M:%S\",wait=0),transaction_sync=(enabled=0"
-	  ",method=fsync),verbose=",
+	  "prealloc=,recover=on,zero_fill=0),lsm_manager=(merge=,"
+	  "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0,"
+	  "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB"
+	  ",name=,quota=0,reserve=0,size=500MB),statistics=none,"
+	  "statistics_log=(on_close=0,path=\"WiredTigerStat.%d.%H\","
+	  "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0),"
+	  "transaction_sync=(enabled=0,method=fsync),verbose=",
 	  confchk_wiredtiger_open_usercfg, 30
 	},
 	{ NULL, NULL, NULL, 0 }
diff --git a/src/conn/conn_log.c b/src/conn/conn_log.c
index eba0a2769d6..336a195f8e9 100644
--- a/src/conn/conn_log.c
+++ b/src/conn/conn_log.c
@@ -74,6 +74,10 @@ __logmgr_config(WT_SESSION_IMPL *session, const char **cfg, bool *runp)
 	if (cval.val != 0)
 		FLD_SET(conn->log_flags, WT_CONN_LOG_ARCHIVE);
 
+	WT_RET(__wt_config_gets(session, cfg, "log.zero_fill", &cval));
+	if (cval.val != 0)
+		FLD_SET(conn->log_flags, WT_CONN_LOG_ZERO_FILL);
+
 	WT_RET(__wt_config_gets(session, cfg, "log.file_max", &cval));
 	conn->log_file_max = (wt_off_t)cval.val;
 	WT_STAT_FAST_CONN_SET(session, log_max_filesize, conn->log_file_max);
@@ -85,7 +89,7 @@ __logmgr_config(WT_SESSION_IMPL *session, const char **cfg, bool *runp)
 	 */
 	if (cval.val != 0) {
 		FLD_SET(conn->log_flags, WT_CONN_LOG_PREALLOC);
-		conn->log_prealloc = 1;
+		conn->log_prealloc = 5;
 	}
 	WT_RET(__wt_config_gets_def(session, cfg, "log.recover", 0, &cval));
 	if (cval.len != 0  && WT_STRING_MATCH("error", cval.str, cval.len))
diff --git a/src/include/connection.h b/src/include/connection.h
index 2c20c2f7936..0273414f42e 100644
--- a/src/include/connection.h
+++ b/src/include/connection.h
@@ -342,6 +342,7 @@ struct __wt_connection_impl {
 #define	WT_CONN_LOG_PREALLOC	0x08	/* Pre-allocation is enabled */
 #define	WT_CONN_LOG_RECOVER_DONE	0x10	/* Recovery completed */
 #define	WT_CONN_LOG_RECOVER_ERR	0x20	/* Error if recovery required */
+#define	WT_CONN_LOG_ZERO_FILL	0x40	/* Manually zero files */
 	uint32_t	 log_flags;	/* Global logging configuration */
 	WT_CONDVAR	*log_cond;	/* Log server wait mutex */
 	WT_SESSION_IMPL *log_session;	/* Log server session */
diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in
index 9078a0e2e99..57207a9f2d2 100644
--- a/src/include/wiredtiger.in
+++ b/src/include/wiredtiger.in
@@ -2212,6 +2212,8 @@ struct __wt_connection {
  * @config{    recover, run recovery
  * or error if recovery needs to run after an unclean shutdown., a string\,
  * chosen from the following options: \c "error"\, \c "on"; default \c on.}
+ * @config{    zero_fill, manually write zeroes into log
+ * files., a boolean flag; default \c false.}
  * @config{ ),,}
  * @config{lsm_manager = (, configure database wide options for LSM tree
  * management.  The LSM manager is started automatically the first time an LSM
diff --git a/src/log/log.c b/src/log/log.c
index ca0b81c4cf6..d78cc0c9134 100644
--- a/src/log/log.c
+++ b/src/log/log.c
@@ -357,6 +357,66 @@ __wt_log_extract_lognum(
 }
 
 /*
+ * __log_zero --
+ *	Zero a log file.
+ */
+static int
+__log_zero(WT_SESSION_IMPL *session,
+    WT_FH *fh, wt_off_t start_off, wt_off_t len)
+{
+	WT_CONNECTION_IMPL *conn;
+	WT_DECL_ITEM(zerobuf);
+	WT_DECL_RET;
+	WT_LOG *log;
+	wt_off_t off, partial;
+	uint32_t allocsize, bufsz, wrlen;
+
+	conn = S2C(session);
+	log = conn->log;
+	allocsize = log->allocsize;
+	zerobuf = NULL;
+	if (allocsize < WT_MEGABYTE)
+		bufsz = WT_MEGABYTE;
+	else
+		bufsz = allocsize;
+	/*
+	 * If they're using smaller log files, cap it at the file size.
+	 */
+	if (conn->log_file_max < bufsz)
+		bufsz = conn->log_file_max;
+	WT_RET(__wt_scr_alloc(session, bufsz, &zerobuf));
+	memset(zerobuf->mem, 0, zerobuf->size);
+
+	/*
+	 * Read in a chunk starting at the end of the file.  Keep going until
+	 * we reach the beginning or we find a chunk that contains any non-zero
+	 * bytes.  Compare against a known zero byte chunk.
+	 */
+	off = start_off;
+	while (off < len) {
+		/*
+		 * Typically we start to zero the file after the log header
+		 * and the bufsz is a sector-aligned size.  So we want to
+		 * align our writes when we can.
+		 */
+		partial = off % (wt_off_t)bufsz;
+		if (partial != 0)
+			wrlen = bufsz - partial;
+		else
+			wrlen = bufsz;
+		/*
+		 * Check if we're writing a partial amount at the end too.
+		 */
+		if (len - off < bufsz)
+			wrlen = len - off;
+		WT_ERR(__wt_write(session, fh, off, wrlen, zerobuf->mem));
+		off += wrlen;
+	}
+err:	__wt_scr_free(session, &zerobuf);
+	return (ret);
+}
+
+/*
  * __log_prealloc --
  *	Pre-allocate a log file.
  */
@@ -370,7 +430,15 @@ __log_prealloc(WT_SESSION_IMPL *session, WT_FH *fh)
 	conn = S2C(session);
 	log = conn->log;
 	ret = 0;
-	if (fh->fallocate_available == WT_FALLOCATE_NOT_AVAILABLE ||
+	/*
+	 * If the user configured zero filling, pre-allocate the log file
+	 * manually.  Otherwise use either fallocate or ftruncate to create
+	 * and zero the log file based on what is available.
+	 */
+	if (F_ISSET(conn, WT_CONN_LOG_ZERO_FILL))
+		ret = __log_zero(session, fh,
+		    WT_LOG_FIRST_RECORD, conn->log_file_max);
+	else if (fh->fallocate_available == WT_FALLOCATE_NOT_AVAILABLE ||
 	    (ret = __wt_fallocate(session, fh,
 	    WT_LOG_FIRST_RECORD, conn->log_file_max)) == ENOTSUP)
 		ret = __wt_ftruncate(session, fh,