Merge branch 'develop' into mongodb-3.4mongodb-3.3.8mongodb-3.3.7

10 files changed, 84 insertions, 28 deletions

diff --git a/src/btree/bt_sync.c b/src/btree/bt_sync.c
index 5d60c436a08..4404069e507 100644
--- a/src/btree/bt_sync.c
+++ b/src/btree/bt_sync.c
@@ -96,8 +96,10 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop)
 		 * snapshot now.
 		 *
 		 * All changes committed up to this point should be included.
-		 * We don't update the snapshot in between pages because (a)
-		 * the metadata shouldn't be that big, and (b) if we do ever
+		 * We don't update the snapshot in between pages because the
+		 * metadata shouldn't have many pages.  Instead, read-committed
+		 * isolation ensures that all metadata updates completed before
+		 * the checkpoint are included.
 		 */
 		if (txn->isolation == WT_ISO_READ_COMMITTED)
 			WT_ERR(__wt_txn_get_snapshot(session));
diff --git a/src/cache/cache_las.c b/src/cache/cache_las.c
index fd541458fa8..27c2900fa98 100644
--- a/src/cache/cache_las.c
+++ b/src/cache/cache_las.c
@@ -42,6 +42,17 @@ __wt_las_stats_update(WT_SESSION_IMPL *session)
 	WT_STAT_SET(session, cstats, cache_lookaside_insert, v);
 	v = WT_STAT_READ(dstats, cursor_remove);
 	WT_STAT_SET(session, cstats, cache_lookaside_remove, v);
+	/*
+	 * If we're clearing stats we need to clear the cursor values we just
+	 * read.  This does not clear the rest of the statistics in the
+	 * lookaside data source stat cursor, but we own that namespace so we
+	 * don't have to worry about users seeing inconsistent data source
+	 * information.
+	 */
+	if (FLD_ISSET(conn->stat_flags, WT_CONN_STAT_CLEAR)) {
+		WT_STAT_SET(session, dstats, cursor_insert, 0);
+		WT_STAT_SET(session, dstats, cursor_remove, 0);
+	}
 }
 
 /*
diff --git a/src/checksum/power8/crc32.S b/src/checksum/power8/crc32.S
index f990acb7b12..0b7870668b5 100644
--- a/src/checksum/power8/crc32.S
+++ b/src/checksum/power8/crc32.S
@@ -773,6 +773,6 @@ FUNC_END(__crc32_vpmsum)
 /*
  * Make sure the stack isn't executable with GCC (regardless of platform).
  */
-#ifndef __clang__
+#ifdef __ELF__
 .section	.note.GNU-stack,"",@progbits
 #endif
diff --git a/src/include/log.h b/src/include/log.h
index 387d0c6c154..7655cfbb3e9 100644
--- a/src/include/log.h
+++ b/src/include/log.h
@@ -257,6 +257,7 @@ struct __wt_log {
 	uint64_t	 write_calls;		/* Calls to log_write */
 #endif
 
+#define	WT_LOG_NOT_VERIFIED	0x1		/* Log just started */
 	uint32_t	 flags;
 };
 
diff --git a/src/include/os.h b/src/include/os.h
index 1e08683bc83..dd9b96f73a8 100644
--- a/src/include/os.h
+++ b/src/include/os.h
@@ -9,15 +9,26 @@
 #define	WT_SYSCALL_RETRY(call, ret) do {				\
 	int __retry;							\
 	for (__retry = 0; __retry < 10; ++__retry) {			\
-		if ((call) == 0) {					\
-			(ret) = 0;					\
-			break;						\
-		}							\
-		switch ((ret) = __wt_errno()) {				\
-		case 0:							\
-			/* The call failed but didn't set errno. */	\
-			(ret) = WT_ERROR;				\
+		/*							\
+		 * A call returning 0 indicates success; any call where	\
+		 * 0 is not the only successful return must provide an	\
+		 * expression evaluating to 0 in all successful	cases.	\
+		 */							\
+		if (((ret) = (call)) == 0)				\
 			break;						\
+		/*							\
+		 * The call's error was either returned by the call or	\
+		 * is in errno, and there are cases where it depends on	\
+		 * the software release as to which it is (for example,	\
+		 * posix_fadvise on FreeBSD and OS X). Failing calls	\
+		 * must either return a non-zero error value, or -1 if	\
+		 * the error value is in errno. (The WiredTiger errno	\
+		 * function returns WT_ERROR if errno is 0, which isn't	\
+		 * ideal but won't discard the failure.)		\
+		 */							\
+		if ((ret) == -1)					\
+			(ret) = __wt_errno();				\
+		switch (ret) {						\
 		case EAGAIN:						\
 		case EBUSY:						\
 		case EINTR:						\
diff --git a/src/log/log.c b/src/log/log.c
index 01bfb97718f..56e9f65f914 100644
--- a/src/log/log.c
+++ b/src/log/log.c
@@ -1090,6 +1090,7 @@ __wt_log_open(WT_SESSION_IMPL *session)
 	logcount = 0;
 	lastlog = 0;
 	firstlog = UINT32_MAX;
+	F_SET(log, WT_LOG_NOT_VERIFIED);
 
 	/*
 	 * Open up a file handle to the log directory if we haven't.
@@ -1744,6 +1745,22 @@ advance:
 		    &rd_lsn, WT_LOG_FILENAME, 0));
 
 err:	WT_STAT_FAST_CONN_INCR(session, log_scans);
+	/*
+	 * If the first attempt to read a log record results in
+	 * an error recovery is likely going to fail.  Try to provide
+	 * a helpful failure message.
+	 */
+	if (ret != 0 && F_ISSET(log, WT_LOG_NOT_VERIFIED)) {
+		__wt_errx(session,
+		    "WiredTiger is unable to read the recovery log.");
+		__wt_errx(session, "This may be due to the log"
+		    " files being encrypted, being from an older"
+		    " version or due to corruption on disk");
+		__wt_errx(session, "You should confirm that you have"
+		    " opened the database with the correct options including"
+		    " all encryption and compression options");
+	}
+	F_CLR(log, WT_LOG_NOT_VERIFIED);
 
 	WT_TRET(__wt_fs_directory_list_free(session, &logfiles, logcount));
 
diff --git a/src/os_posix/os_dir.c b/src/os_posix/os_dir.c
index a23051e5b93..ea0ca11fa54 100644
--- a/src/os_posix/os_dir.c
+++ b/src/os_posix/os_dir.c
@@ -38,7 +38,7 @@ __wt_posix_directory_list(WT_FILE_SYSTEM *file_system,
 	dirallocsz = 0;
 	entries = NULL;
 
-	WT_SYSCALL_RETRY(((dirp = opendir(directory)) == NULL ? 1 : 0), ret);
+	WT_SYSCALL_RETRY(((dirp = opendir(directory)) == NULL ? -1 : 0), ret);
 	if (ret != 0)
 		WT_RET_MSG(session, ret,
 		    "%s: directory-list: opendir", directory);
diff --git a/src/os_posix/os_fs.c b/src/os_posix/os_fs.c
index c05f75f2bd5..1cfa8fd2d2d 100644
--- a/src/os_posix/os_fs.c
+++ b/src/os_posix/os_fs.c
@@ -53,7 +53,7 @@ __posix_sync(
 	 * "This is currently implemented on HFS, MS-DOS (FAT), and Universal
 	 * Disk Format (UDF) file systems."
 	 */
-	WT_SYSCALL_RETRY(fcntl(fd, F_FULLFSYNC, 0), ret);
+	WT_SYSCALL_RETRY(fcntl(fd, F_FULLFSYNC, 0) == -1 ? -1 : 0, ret);
 	if (ret == 0)
 		return (0);
 	/*
@@ -92,7 +92,7 @@ __posix_directory_sync(
 	session = (WT_SESSION_IMPL *)wt_session;
 
 	WT_SYSCALL_RETRY((
-	    (fd = open(path, O_RDONLY, 0444)) == -1 ? 1 : 0), ret);
+	    (fd = open(path, O_RDONLY, 0444)) == -1 ? -1 : 0), ret);
 	if (ret != 0)
 		WT_RET_MSG(session, ret, "%s: directory-sync: open", path);
 
@@ -151,10 +151,17 @@ __posix_fs_remove(
 
 	session = (WT_SESSION_IMPL *)wt_session;
 
-	WT_SYSCALL_RETRY(remove(name), ret);
+	/*
+	 * ISO C doesn't require remove return -1 on failure or set errno (note
+	 * POSIX 1003.1 extends C with those requirements). Regardless, use the
+	 * unlink system call, instead of remove, to simplify error handling;
+	 * where we're not doing any special checking for standards compliance,
+	 * using unlink may be marginally safer.
+	 */
+	WT_SYSCALL_RETRY(unlink(name), ret);
 	if (ret == 0)
 		return (0);
-	WT_RET_MSG(session, ret, "%s: file-remove: remove", name);
+	WT_RET_MSG(session, ret, "%s: file-remove: unlink", name);
 }
 
 /*
@@ -172,7 +179,14 @@ __posix_fs_rename(WT_FILE_SYSTEM *file_system,
 
 	session = (WT_SESSION_IMPL *)wt_session;
 
-	WT_SYSCALL_RETRY(rename(from, to), ret);
+	/*
+	 * ISO C doesn't require rename return -1 on failure or set errno (note
+	 * POSIX 1003.1 extends C with those requirements). Be cautious, force
+	 * any non-zero return to -1 so we'll check errno. We can still end up
+	 * with the wrong errno (if errno is garbage), or the generic WT_ERROR
+	 * return (if errno is 0), but we've done the best we can.
+	 */
+	WT_SYSCALL_RETRY(rename(from, to) != 0 ? -1 : 0, ret);
 	if (ret == 0)
 		return (0);
 	WT_RET_MSG(session, ret, "%s to %s: file-rename: rename", from, to);
@@ -295,7 +309,7 @@ __posix_file_lock(
 	fl.l_type = lock ? F_WRLCK : F_UNLCK;
 	fl.l_whence = SEEK_SET;
 
-	WT_SYSCALL_RETRY(fcntl(pfh->fd, F_SETLK, &fl), ret);
+	WT_SYSCALL_RETRY(fcntl(pfh->fd, F_SETLK, &fl) == -1 ? -1 : 0, ret);
 	if (ret == 0)
 		return (0);
 	WT_RET_MSG(session, ret, "%s: handle-lock: fcntl", file_handle->name);
@@ -533,7 +547,7 @@ __posix_open_file(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session,
 		f |= O_CLOEXEC;
 #endif
 		WT_SYSCALL_RETRY((
-		    (pfh->fd = open(name, f, 0444)) == -1 ? 1 : 0), ret);
+		    (pfh->fd = open(name, f, 0444)) == -1 ? -1 : 0), ret);
 		if (ret != 0)
 			WT_ERR_MSG(session, ret, "%s: handle-open: open", name);
 		WT_ERR(__posix_open_file_cloexec(session, pfh->fd, name));
@@ -587,7 +601,7 @@ __posix_open_file(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session,
 #endif
 	}
 
-	WT_SYSCALL_RETRY(((pfh->fd = open(name, f, mode)) == -1 ? 1 : 0), ret);
+	WT_SYSCALL_RETRY(((pfh->fd = open(name, f, mode)) == -1 ? -1 : 0), ret);
 	if (ret != 0)
 		WT_ERR_MSG(session, ret,
 		    pfh->direct_io ?
diff --git a/src/os_posix/os_map.c b/src/os_posix/os_map.c
index 9cdb58b95c8..d89ba4d7c26 100644
--- a/src/os_posix/os_map.c
+++ b/src/os_posix/os_map.c
@@ -102,10 +102,13 @@ __wt_posix_map_preload(WT_FILE_HANDLE *fh,
 	 * size, so we will be conservative.
 	 */
 	length &= ~(size_t)(conn->page_size - 1);
+	if (length <= (size_t)conn->page_size)
+		return (0);
 
-	if (length <= (size_t)conn->page_size ||
-	    (ret = posix_madvise(blk, length, POSIX_MADV_WILLNEED)) == 0)
+	WT_SYSCALL_RETRY(posix_madvise(blk, length, POSIX_MADV_WILLNEED), ret);
+	if (ret == 0)
 		return (0);
+
 	WT_RET_MSG(session, ret,
 	    "%s: memory-map preload: posix_madvise: POSIX_MADV_WILLNEED",
 	    fh->name);
@@ -135,8 +138,10 @@ __wt_posix_map_discard(WT_FILE_HANDLE *fh,
 	blk = (void *)((uintptr_t)map & ~(uintptr_t)(conn->page_size - 1));
 	length += WT_PTRDIFF(map, blk);
 
-	if ((ret = posix_madvise(blk, length, POSIX_MADV_DONTNEED)) == 0)
+	WT_SYSCALL_RETRY(posix_madvise(blk, length, POSIX_MADV_DONTNEED), ret);
+	if (ret == 0)
 		return (0);
+
 	WT_RET_MSG(session, ret,
 	    "%s: memory-map discard: posix_madvise: POSIX_MADV_DONTNEED",
 	    fh->name);
diff --git a/src/os_win/os_fs.c b/src/os_win/os_fs.c
index c4a1235b61b..4da60d5ffb0 100644
--- a/src/os_win/os_fs.c
+++ b/src/os_win/os_fs.c
@@ -169,11 +169,6 @@ __win_file_lock(
 	 * WiredTiger requires this function be able to acquire locks past
 	 * the end of file.
 	 *
-	 * Note we're using fcntl(2) locking: all fcntl locks associated with a
-	 * file for a given process are removed when any file descriptor for the
-	 * file is closed by the process, even if a lock was never requested for
-	 * that file descriptor.
-	 *
 	 * http://msdn.microsoft.com/
 	 *    en-us/library/windows/desktop/aa365202%28v=vs.85%29.aspx
 	 *