Import wiredtiger-wiredtiger-2.7.0-650-g5cdd3e3.tar.gz from wiredtiger branch mongodb-3.2

ref: 07966a4..5cdd3e3

SERVER-22437  Coverity analysis defect 77704: Redundant test
SERVER-22438  Coverity analysis defect 77705: Dereference before null check
SERVER-22676  WiredTiger fails to open databases created by 3.0.0 or 3.0.1
WT-2130       Improve on-disk page utlilization with random workloads
WT-2215       WT_LSN needs to support atomic reads and updates
WT-2295       WT_SESSION.create does a full-scan of the main table
WT-2352       Allow build and test without requiring lz4
WT-2356       log scan advances to next log file on partially written record
WT-2363       Remove built in support for bzip2
WT-2368       row-store can pass garbage keys to collator functions
WT-2369       Use C compiler to detect headers instead of C++ compiler
WT-2371       parent split cannot access the page after page-index swap
WT-2372       WiredTiger windows builder fails with C4005 against the "inline" macro
WT-2377       WTPERF doesn't compile in Windows under MSVC
WT-2378       Tasks time out on LSM builder
WT-2397       Cursor traversal from end of the tree skips records.
WT-60         Big endian port

4 files changed, 45 insertions, 450 deletions

diff --git a/src/third_party/wiredtiger/ext/compressors/bzip2/Makefile.am b/src/third_party/wiredtiger/ext/compressors/bzip2/Makefile.am
deleted file mode 100644
index 0aedc2efd80..00000000000
--- a/src/third_party/wiredtiger/ext/compressors/bzip2/Makefile.am
+++ /dev/null
@@ -1,6 +0,0 @@
-AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include
-
-lib_LTLIBRARIES = libwiredtiger_bzip2.la
-libwiredtiger_bzip2_la_SOURCES = bzip2_compress.c
-libwiredtiger_bzip2_la_LDFLAGS = -avoid-version -module
-libwiredtiger_bzip2_la_LIBADD = -lbz2
diff --git a/src/third_party/wiredtiger/ext/compressors/bzip2/bzip2_compress.c b/src/third_party/wiredtiger/ext/compressors/bzip2/bzip2_compress.c
deleted file mode 100644
index 845107c609f..00000000000
--- a/src/third_party/wiredtiger/ext/compressors/bzip2/bzip2_compress.c
+++ /dev/null
@@ -1,415 +0,0 @@
-/*-
- * Public Domain 2014-2016 MongoDB, Inc.
- * Public Domain 2008-2014 WiredTiger, Inc.
- *
- * This is free and unencumbered software released into the public domain.
- *
- * Anyone is free to copy, modify, publish, use, compile, sell, or
- * distribute this software, either in source code form or as a compiled
- * binary, for any purpose, commercial or non-commercial, and by any
- * means.
- *
- * In jurisdictions that recognize copyright laws, the author or authors
- * of this software dedicate any and all copyright interest in the
- * software to the public domain. We make this dedication for the benefit
- * of the public at large and to the detriment of our heirs and
- * successors. We intend this dedication to be an overt act of
- * relinquishment in perpetuity of all present and future rights to this
- * software under copyright law.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
- * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- */
-
-#include <bzlib.h>
-#include <errno.h>
-#include <inttypes.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include <wiredtiger.h>
-#include <wiredtiger_ext.h>
-
-/* Local compressor structure. */
-typedef struct {
-	WT_COMPRESSOR compressor;		/* Must come first */
-
-	WT_EXTENSION_API *wt_api;		/* Extension API */
-
-	int bz_verbosity;			/* Configuration */
-	int bz_blocksize100k;
-	int bz_workfactor;
-	int bz_small;
-} BZIP_COMPRESSOR;
-
-/*
- * Bzip gives us a cookie to pass to the underlying allocation functions; we
- * we need two handles, package them up.
- */
-typedef struct {
-	WT_COMPRESSOR *compressor;
-	WT_SESSION *session;
-} BZIP_OPAQUE;
-
-/*
- * bzip2_error --
- *	Output an error message, and return a standard error code.
- */
-static int
-bzip2_error(
-    WT_COMPRESSOR *compressor, WT_SESSION *session, const char *call, int bzret)
-{
-	WT_EXTENSION_API *wt_api;
-	const char *msg;
-
-	wt_api = ((BZIP_COMPRESSOR *)compressor)->wt_api;
-
-	switch (bzret) {
-	case BZ_MEM_ERROR:
-		msg = "BZ_MEM_ERROR";
-		break;
-	case BZ_OUTBUFF_FULL:
-		msg = "BZ_OUTBUFF_FULL";
-		break;
-	case BZ_SEQUENCE_ERROR:
-		msg = "BZ_SEQUENCE_ERROR";
-		break;
-	case BZ_PARAM_ERROR:
-		msg = "BZ_PARAM_ERROR";
-		break;
-	case BZ_DATA_ERROR:
-		msg = "BZ_DATA_ERROR";
-		break;
-	case BZ_DATA_ERROR_MAGIC:
-		msg = "BZ_DATA_ERROR_MAGIC";
-		break;
-	case BZ_IO_ERROR:
-		msg = "BZ_IO_ERROR";
-		break;
-	case BZ_UNEXPECTED_EOF:
-		msg = "BZ_UNEXPECTED_EOF";
-		break;
-	case BZ_CONFIG_ERROR:
-		msg = "BZ_CONFIG_ERROR";
-		break;
-	default:
-		msg = "unknown error";
-		break;
-	}
-
-	(void)wt_api->err_printf(wt_api, session,
-	    "bzip2 error: %s: %s: %d", call, msg, bzret);
-	return (WT_ERROR);
-}
-
-/*
- * bzalloc --
- *	Allocate scratch buffers.
- */
-static void *
-bzalloc(void *cookie, int number, int size)
-{
-	BZIP_OPAQUE *opaque;
-	WT_EXTENSION_API *wt_api;
-
-	opaque = cookie;
-	wt_api = ((BZIP_COMPRESSOR *)opaque->compressor)->wt_api;
-	return (wt_api->scr_alloc(
-	    wt_api, opaque->session, (size_t)(number * size)));
-}
-
-/*
- * bzfree --
- *	Free scratch buffers.
- */
-static void
-bzfree(void *cookie, void *p)
-{
-	BZIP_OPAQUE *opaque;
-	WT_EXTENSION_API *wt_api;
-
-	opaque = cookie;
-	wt_api = ((BZIP_COMPRESSOR *)opaque->compressor)->wt_api;
-	wt_api->scr_free(wt_api, opaque->session, p);
-}
-
-/*
- * bzip2_compress --
- *	WiredTiger bzip2 compression.
- */
-static int
-bzip2_compress(WT_COMPRESSOR *compressor, WT_SESSION *session,
-    uint8_t *src, size_t src_len,
-    uint8_t *dst, size_t dst_len,
-    size_t *result_lenp, int *compression_failed)
-{
-	BZIP_COMPRESSOR *bzip_compressor;
-	BZIP_OPAQUE opaque;
-	bz_stream bz;
-	int ret;
-
-	bzip_compressor = (BZIP_COMPRESSOR *)compressor;
-
-	memset(&bz, 0, sizeof(bz));
-	bz.bzalloc = bzalloc;
-	bz.bzfree = bzfree;
-	opaque.compressor = compressor;
-	opaque.session = session;
-	bz.opaque = &opaque;
-
-	if ((ret = BZ2_bzCompressInit(&bz,
-	    bzip_compressor->bz_blocksize100k,
-	    bzip_compressor->bz_verbosity,
-	    bzip_compressor->bz_workfactor)) != BZ_OK)
-		return (bzip2_error(
-		    compressor, session, "BZ2_bzCompressInit", ret));
-
-	bz.next_in = (char *)src;
-	bz.avail_in = (uint32_t)src_len;
-	bz.next_out = (char *)dst;
-	bz.avail_out = (uint32_t)dst_len;
-	if ((ret = BZ2_bzCompress(&bz, BZ_FINISH)) == BZ_STREAM_END) {
-		*compression_failed = 0;
-		*result_lenp = dst_len - bz.avail_out;
-	} else
-		*compression_failed = 1;
-
-	if ((ret = BZ2_bzCompressEnd(&bz)) != BZ_OK)
-		return (
-		    bzip2_error(compressor, session, "BZ2_bzCompressEnd", ret));
-
-	return (0);
-}
-
-/*
- * __bzip2_compress_raw_random --
- *	Return a 32-bit pseudo-random number.
- *
- * This is an implementation of George Marsaglia's multiply-with-carry pseudo-
- * random number generator.  Computationally fast, with reasonable randomness
- * properties.
- */
-static uint32_t
-__bzip2_compress_raw_random(void)
-{
-	static uint32_t m_w = 521288629;
-	static uint32_t m_z = 362436069;
-
-	m_z = 36969 * (m_z & 65535) + (m_z >> 16);
-	m_w = 18000 * (m_w & 65535) + (m_w >> 16);
-	return (m_z << 16) + (m_w & 65535);
-}
-
-/*
- * bzip2_compress_raw --
- *	Test function for the test/format utility.
- */
-static int
-bzip2_compress_raw(WT_COMPRESSOR *compressor, WT_SESSION *session,
-    size_t page_max, int split_pct, size_t extra,
-    uint8_t *src, uint32_t *offsets, uint32_t slots,
-    uint8_t *dst, size_t dst_len, int final,
-    size_t *result_lenp, uint32_t *result_slotsp)
-{
-	uint32_t take, twenty_pct;
-	int compression_failed, ret;
-
-	(void)page_max;					/* Unused  parameters */
-	(void)split_pct;
-	(void)extra;
-	(void)final;
-
-	/*
-	 * This function is used by the test/format utility to test the
-	 * WT_COMPRESSOR::compress_raw functionality.
-	 *
-	 * I'm trying to mimic how a real application is likely to behave: if
-	 * it's a small number of slots, we're not going to take them because
-	 * they aren't worth compressing.  In all likelihood, that's going to
-	 * be because the btree is wrapping up a page, but that's OK, that is
-	 * going to happen a lot. In addition, add a 2% chance of not taking
-	 * anything at all just because we don't want to take it.  Otherwise,
-	 * select between 80 and 100% of the slots and compress them, stepping
-	 * down by 5 slots at a time until something works.
-	 */
-	take = slots;
-	if (take < 10 || __bzip2_compress_raw_random() % 100 < 2)
-		take = 0;
-	else {
-		twenty_pct = (slots / 10) * 2;
-		if (twenty_pct < slots)
-			take -= __bzip2_compress_raw_random() % twenty_pct;
-
-		for (;;) {
-			if ((ret = bzip2_compress(compressor, session,
-			    src, offsets[take],
-			    dst, dst_len,
-			    result_lenp, &compression_failed)) != 0)
-				return (ret);
-			if (!compression_failed)
-				break;
-			if (take < 10) {
-				take = 0;
-				break;
-			}
-			take -= 5;
-		}
-	}
-
-	*result_slotsp = take;
-	if (take == 0)
-		*result_lenp = 0;
-
-#if 0
-	fprintf(stderr,
-	    "bzip2_compress_raw (%s): page_max %" PRIuMAX
-	    ", split_pct %u, extra %" PRIuMAX
-	    ", slots %" PRIu32 ", take %" PRIu32 ": %" PRIu32 " -> %"
-	    PRIuMAX "\n",
-	    final ? "final" : "not final",
-	    (uintmax_t)page_max, split_pct, (uintmax_t)extra,
-	    slots, take, offsets[take], (uintmax_t)*result_lenp);
-#endif
-	return (take == 0 ? EAGAIN : 0);
-}
-
-/*
- * bzip2_decompress --
- *	WiredTiger bzip2 decompression.
- */
-static int
-bzip2_decompress(WT_COMPRESSOR *compressor, WT_SESSION *session,
-    uint8_t *src, size_t src_len,
-    uint8_t *dst, size_t dst_len,
-    size_t *result_lenp)
-{
-	BZIP_COMPRESSOR *bzip_compressor;
-	BZIP_OPAQUE opaque;
-	bz_stream bz;
-	int ret, tret;
-
-	bzip_compressor = (BZIP_COMPRESSOR *)compressor;
-
-	memset(&bz, 0, sizeof(bz));
-	bz.bzalloc = bzalloc;
-	bz.bzfree = bzfree;
-	opaque.compressor = compressor;
-	opaque.session = session;
-	bz.opaque = &opaque;
-
-	if ((ret = BZ2_bzDecompressInit(&bz,
-	    bzip_compressor->bz_small, bzip_compressor->bz_verbosity)) != BZ_OK)
-		return (bzip2_error(
-		    compressor, session, "BZ2_bzDecompressInit", ret));
-
-	bz.next_in = (char *)src;
-	bz.avail_in = (uint32_t)src_len;
-	bz.next_out = (char *)dst;
-	bz.avail_out = (uint32_t)dst_len;
-	if ((ret = BZ2_bzDecompress(&bz)) == BZ_STREAM_END) {
-		*result_lenp = dst_len - bz.avail_out;
-		ret = 0;
-	} else {
-		/*
-		 * If BZ2_bzDecompress returns 0, it expects there to be more
-		 * data available.  There isn't, so treat this as an error.
-		 */
-		if (ret == 0)
-			ret = BZ_DATA_ERROR;
-		(void)bzip2_error(compressor, session, "BZ2_bzDecompress", ret);
-	}
-
-	if ((tret = BZ2_bzDecompressEnd(&bz)) != BZ_OK)
-		return (bzip2_error(
-		    compressor, session, "BZ2_bzDecompressEnd", tret));
-
-	return (ret == 0 ?
-	    0 : bzip2_error(compressor, session, "BZ2_bzDecompressEnd", ret));
-}
-
-/*
- * bzip2_terminate --
- *	WiredTiger bzip2 compression termination.
- */
-static int
-bzip2_terminate(WT_COMPRESSOR *compressor, WT_SESSION *session)
-{
-	(void)session;					/* Unused parameters */
-
-	free(compressor);
-	return (0);
-}
-
-/*
- * bzip2_add_compressor --
- *	Add a bzip2 compressor.
- */
-static int
-bzip2_add_compressor(WT_CONNECTION *connection, int raw, const char *name)
-{
-	BZIP_COMPRESSOR *bzip_compressor;
-
-	/*
-	 * There are two almost identical bzip2 compressors: one supporting raw
-	 * compression (used by test/format to test raw compression), the other
-	 * without raw compression, that might be useful for real applications.
-	 */
-	if ((bzip_compressor = calloc(1, sizeof(BZIP_COMPRESSOR))) == NULL)
-		return (errno);
-
-	bzip_compressor->compressor.compress = bzip2_compress;
-	bzip_compressor->
-	    compressor.compress_raw = raw ? bzip2_compress_raw : NULL;
-	bzip_compressor->compressor.decompress = bzip2_decompress;
-	bzip_compressor->compressor.pre_size = NULL;
-	bzip_compressor->compressor.terminate = bzip2_terminate;
-
-	bzip_compressor->wt_api = connection->get_extension_api(connection);
-
-	/* between 0-4: set the amount of verbosity to stderr */
-	bzip_compressor->bz_verbosity = 0;
-
-	/*
-	 * between 1-9: set the block size to 100k x this number (compression
-	 * only)
-	 */
-	bzip_compressor->bz_blocksize100k = 1;
-
-	/*
-	 * between 0-250: workFactor: see bzip2 manual.  0 is a reasonable
-	 * default (compression only)
-	 */
-	bzip_compressor->bz_workfactor = 0;
-
-	/*
-	 * if nonzero, decompress using less memory, but slower (decompression
-	 * only)
-	 */
-	bzip_compressor->bz_small = 0;
-
-	return (connection->add_compressor(	/* Load the compressor */
-	    connection, name, (WT_COMPRESSOR *)bzip_compressor, NULL));
-}
-
-/*
- * wiredtiger_extension_init --
- *	WiredTiger bzip2 compression extension.
- */
-int
-wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config)
-{
-	int ret;
-
-	(void)config;				/* Unused parameters */
-
-	if ((ret = bzip2_add_compressor(connection, 0, "bzip2")) != 0)
-		return (ret);
-	if ((ret = bzip2_add_compressor(connection, 1, "bzip2-raw-test")) != 0)
-		return (ret);
-	return (0);
-}
diff --git a/src/third_party/wiredtiger/ext/compressors/lz4/lz4_compress.c b/src/third_party/wiredtiger/ext/compressors/lz4/lz4_compress.c
index d070dc3fb79..062307b721a 100644
--- a/src/third_party/wiredtiger/ext/compressors/lz4/lz4_compress.c
+++ b/src/third_party/wiredtiger/ext/compressors/lz4/lz4_compress.c
@@ -26,23 +26,13 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
+#include <wt_internal.h>
+
 #include <lz4.h>
 #include <errno.h>
 #include <stdlib.h>
 #include <string.h>
 
-#include <wiredtiger.h>
-#include <wiredtiger_ext.h>
-
-/*
- * We need to include the configuration file to detect whether this extension
- * is being built into the WiredTiger library.
- */
-#include "wiredtiger_config.h"
-#ifdef _MSC_VER
-#define	inline __inline
-#endif
-
 /* Local compressor structure. */
 typedef struct {
 	WT_COMPRESSOR compressor;		/* Must come first */
@@ -73,6 +63,24 @@ typedef struct {
 } LZ4_PREFIX;
 
 /*
+ * lz4_prefix_swap --
+ *	The additional information is written in little-endian format, handle
+ * the conversion.
+ */
+static inline void
+lz4_prefix_swap(LZ4_PREFIX *prefix)
+{
+#ifdef WORDS_BIGENDIAN
+	prefix->compressed_len = __wt_bswap32(prefix->compressed_len);
+	prefix->uncompressed_len = __wt_bswap32(prefix->uncompressed_len);
+	prefix->useful_len = __wt_bswap32(prefix->useful_len);
+	prefix->unused = __wt_bswap32(prefix->unused);
+#else
+	WT_UNUSED(prefix);
+#endif
+}
+
+/*
  * lz4_error --
  *	Output an error message, and return a standard error code.
  */
@@ -119,6 +127,7 @@ lz4_compress(WT_COMPRESSOR *compressor, WT_SESSION *session,
 		prefix.uncompressed_len = (uint32_t)src_len;
 		prefix.useful_len = (uint32_t)src_len;
 		prefix.unused = 0;
+		lz4_prefix_swap(&prefix);
 		memcpy(dst, &prefix, sizeof(LZ4_PREFIX));
 
 		*result_lenp = (size_t)lz4_len + sizeof(LZ4_PREFIX);
@@ -154,6 +163,7 @@ lz4_decompress(WT_COMPRESSOR *compressor, WT_SESSION *session,
 	 * decompressed bytes to return from the start of the source buffer.
 	 */
 	memcpy(&prefix, src, sizeof(LZ4_PREFIX));
+	lz4_prefix_swap(&prefix);
 
 	/*
 	 * Decompress, starting after the prefix bytes. Use safe decompression:
@@ -268,6 +278,7 @@ lz4_compress_raw(WT_COMPRESSOR *compressor, WT_SESSION *session,
 			prefix.uncompressed_len = (uint32_t)sourceSize;
 			prefix.useful_len = offsets[slot];
 			prefix.unused = 0;
+			lz4_prefix_swap(&prefix);
 			memcpy(dst, &prefix, sizeof(LZ4_PREFIX));
 
 			*result_slotsp = slot;
diff --git a/src/third_party/wiredtiger/ext/compressors/snappy/snappy_compress.c b/src/third_party/wiredtiger/ext/compressors/snappy/snappy_compress.c
index b5a347fce81..fcefb8bb575 100644
--- a/src/third_party/wiredtiger/ext/compressors/snappy/snappy_compress.c
+++ b/src/third_party/wiredtiger/ext/compressors/snappy/snappy_compress.c
@@ -26,20 +26,13 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
+#include <wt_internal.h>
+
 #include <snappy-c.h>
 #include <errno.h>
 #include <stdlib.h>
 #include <string.h>
 
-#include <wiredtiger.h>
-#include <wiredtiger_ext.h>
-
-/*
- * We need to include the configuration file to detect whether this extension
- * is being built into the WiredTiger library.
- */
-#include "wiredtiger_config.h"
-
 /* Local compressor structure. */
 typedef struct {
 	WT_COMPRESSOR compressor;		/* Must come first */
@@ -103,16 +96,22 @@ wt_snappy_compress(WT_COMPRESSOR *compressor, WT_SESSION *session,
 	snret = snappy_compress((char *)src, src_len, snapbuf, &snaplen);
 
 	if (snret == SNAPPY_OK) {
-		/*
-		 * On decompression, snappy requires the exact compressed byte
-		 * count (the current value of snaplen).  WiredTiger does not
-		 * preserve that value, so save snaplen at the beginning of the
-		 * destination buffer.
-		 */
 		if (snaplen + sizeof(size_t) < src_len) {
-			*(size_t *)dst = snaplen;
 			*result_lenp = snaplen + sizeof(size_t);
 			*compression_failed = 0;
+
+			/*
+			 * On decompression, snappy requires an exact compressed
+			 * byte count (the current value of snaplen). WiredTiger
+			 * does not preserve that value, so save snaplen at the
+			 * beginning of the destination buffer.
+			 *
+			 * Store the value in little-endian format.
+			 */
+#ifdef WORDS_BIGENDIAN
+			snaplen = __wt_bswap64(snaplen);
+#endif
+			*(size_t *)dst = snaplen;
 		} else
 			/* The compressor failed to produce a smaller result. */
 			*compression_failed = 1;
@@ -137,8 +136,14 @@ wt_snappy_decompress(WT_COMPRESSOR *compressor, WT_SESSION *session,
 
 	wt_api = ((SNAPPY_COMPRESSOR *)compressor)->wt_api;
 
-	/* retrieve the saved length */
+	/*
+	 * Retrieve the saved length, handling little- to big-endian conversion
+	 * as necessary.
+	 */
 	snaplen = *(size_t *)src;
+#ifdef WORDS_BIGENDIAN
+	snaplen = __wt_bswap64(snaplen);
+#endif
 	if (snaplen + sizeof(size_t) > src_len) {
 		(void)wt_api->err_printf(wt_api,
 		    session,