Import wiredtiger: e6c1b9724ed6ed2879a36d7e140f4fa9daceb261 from branch mongodb-4.2

ref: d5793d4dd5..e6c1b9724e
for: 4.1.7

WT-4366       Fix how test/format handles prepare conflict errors
WT-4426       Change WT data format to include timestamps in leaf page key/value cells
WT-4475       clang detected memory leak while executing csuite tests
WT-4499       Fix prepared transactions for cursor key order check failure
WT-4506       Bypass some csuite tests for valgrind

10 files changed, 224 insertions, 32 deletions

diff --git a/src/third_party/wiredtiger/test/checkpoint/smoke.sh b/src/third_party/wiredtiger/test/checkpoint/smoke.sh
index 2f1d4345ad7..8db6fc1ebc4 100755
--- a/src/third_party/wiredtiger/test/checkpoint/smoke.sh
+++ b/src/third_party/wiredtiger/test/checkpoint/smoke.sh
@@ -2,6 +2,9 @@
 
 set -e
 
+# Bypass this test for valgrind
+test "$TESTUTIL_BYPASS_VALGRIND" = "1" && exit 0
+
 # Smoke-test checkpoints as part of running "make check".
 echo "checkpoint: 3 mixed tables"
 $TEST_WRAPPER ./t -T 3 -t m
diff --git a/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c b/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c
index 2757f991c2a..6df68da932d 100644
--- a/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c
+++ b/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c
@@ -98,6 +98,10 @@ main(int argc, char *argv[])
 	uint64_t i, id;
 	char buf[100];
 
+	/* Bypass this test for valgrind */
+	if (testutil_is_flag_set("TESTUTIL_BYPASS_VALGRIND"))
+		return (EXIT_SUCCESS);
+
 	opts = &_opts;
 	memset(opts, 0, sizeof(*opts));
 	opts->table_type = TABLE_ROW;
diff --git a/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c b/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c
index 0b99df76cf3..6cd94ba7572 100644
--- a/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c
+++ b/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c
@@ -92,6 +92,10 @@ main(int argc, char *argv[])
 	TEST_OPTS *opts, _opts;
 	const char *tablename;
 
+	/* Bypass this test for valgrind */
+	if (testutil_is_flag_set("TESTUTIL_BYPASS_VALGRIND"))
+		return (EXIT_SUCCESS);
+
 	opts = &_opts;
 	sharedopts = &_sharedopts;
 	memset(opts, 0, sizeof(*opts));
diff --git a/src/third_party/wiredtiger/test/csuite/wt2834_join_bloom_fix/main.c b/src/third_party/wiredtiger/test/csuite/wt2834_join_bloom_fix/main.c
index a8d44bf3dab..521e67b2439 100644
--- a/src/third_party/wiredtiger/test/csuite/wt2834_join_bloom_fix/main.c
+++ b/src/third_party/wiredtiger/test/csuite/wt2834_join_bloom_fix/main.c
@@ -163,9 +163,7 @@ main(int argc, char *argv[])
 	testutil_assert(count == 0);
 
 	testutil_progress(opts, "cleanup starting");
-#if 0
 	testutil_cleanup(opts);
-#endif
 	return (EXIT_SUCCESS);
 }
 
diff --git a/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c b/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c
index 80911ddfd2d..2021ff1849e 100644
--- a/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c
+++ b/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c
@@ -186,10 +186,7 @@ main(int argc, char *argv[])
 
 	testutil_assert(nfail == 0);
 	testutil_progress(opts, "cleanup starting");
-#if 0
 	testutil_cleanup(opts);
-#endif
-
 	return (0);
 }
 
diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h
index e9063674476..4b8eadeea1d 100644
--- a/src/third_party/wiredtiger/test/format/format.h
+++ b/src/third_party/wiredtiger/test/format/format.h
@@ -377,6 +377,13 @@ mmrand(WT_RAND_STATE *rnd, u_int min, u_int max)
 	uint32_t v;
 	u_int range;
 
+	/*
+	 * Test runs with small row counts can easily pass a max of 0 (for
+	 * example, "g.rows / 20"). Avoid the problem.
+	 */
+	if (min <= max)
+		return (min);
+
 	v = rng(rnd);
 	range = (max - min) + 1;
 	v %= range;
diff --git a/src/third_party/wiredtiger/test/format/ops.c b/src/third_party/wiredtiger/test/format/ops.c
index f92f438a4f1..d5ed0320761 100644
--- a/src/third_party/wiredtiger/test/format/ops.c
+++ b/src/third_party/wiredtiger/test/format/ops.c
@@ -276,6 +276,44 @@ wts_ops(int lastrun)
 	free(tinfo_list);
 }
 
+typedef enum { NEXT, PREV, SEARCH, SEARCH_NEAR } read_operation;
+
+/*
+ * read_op --
+ *	Perform a read operation, waiting out prepare conflicts.
+ */
+static inline int
+read_op(WT_CURSOR *cursor, read_operation op, int *exactp)
+{
+	WT_DECL_RET;
+
+	/*
+	 * Read operations wait out prepare-conflicts. (As part of the snapshot
+	 * isolation checks, we repeat reads that succeeded before, they should
+	 * be repeatable.)
+	 */
+	switch (op) {
+	case NEXT:
+		while ((ret = cursor->next(cursor)) == WT_PREPARE_CONFLICT)
+			__wt_yield();
+		break;
+	case PREV:
+		while ((ret = cursor->prev(cursor)) == WT_PREPARE_CONFLICT)
+			__wt_yield();
+		break;
+	case SEARCH:
+		while ((ret = cursor->search(cursor)) == WT_PREPARE_CONFLICT)
+			__wt_yield();
+		break;
+	case SEARCH_NEAR:
+		while ((ret =
+		    cursor->search_near(cursor, exactp)) == WT_PREPARE_CONFLICT)
+			__wt_yield();
+		break;
+	}
+	return (ret);
+}
+
 typedef enum { INSERT, MODIFY, READ, REMOVE, TRUNCATE, UPDATE } thread_op;
 typedef struct {
 	thread_op op;			/* Operation */
@@ -401,7 +439,7 @@ snap_check(WT_CURSOR *cursor,
 			}
 		}
 
-		switch (ret = cursor->search(cursor)) {
+		switch (ret = read_op(cursor, SEARCH, NULL)) {
 		case 0:
 			if (g.type == FIX) {
 				testutil_check(
@@ -634,12 +672,22 @@ prepare_transaction(TINFO *tinfo, WT_SESSION *session)
  */
 #define	OP_FAILED(notfound_ok) do {					\
 	positioned = false;						\
-	if (intxn && (ret == WT_CACHE_FULL ||				\
-	    ret == WT_PREPARE_CONFLICT || ret == WT_ROLLBACK))		\
+	if (intxn && (ret == WT_CACHE_FULL || ret == WT_ROLLBACK))	\
 		goto rollback;						\
 	testutil_assert((notfound_ok && ret == WT_NOTFOUND) ||		\
-	    ret == WT_CACHE_FULL ||					\
-	    ret == WT_PREPARE_CONFLICT || ret == WT_ROLLBACK);		\
+	    ret == WT_CACHE_FULL || ret == WT_ROLLBACK);		\
+} while (0)
+
+/*
+ * Rollback updates returning prepare-conflict, they're unlikely to succeed
+ * unless the prepare aborts. Reads wait out the error, so it's unexpected.
+ */
+#define	READ_OP_FAILED(notfound_ok)					\
+	OP_FAILED(notfound_ok)
+#define	WRITE_OP_FAILED(notfound_ok) do {				\
+	if (ret == WT_PREPARE_CONFLICT)					\
+		ret = WT_ROLLBACK;					\
+	OP_FAILED(notfound_ok);						\
 } while (0)
 
 /*
@@ -826,7 +874,7 @@ ops(void *arg)
 				positioned = true;
 				SNAP_TRACK(READ, tinfo);
 			} else
-				OP_FAILED(true);
+				READ_OP_FAILED(true);
 		}
 
 		/* Optionally reserve a row. */
@@ -845,7 +893,7 @@ ops(void *arg)
 
 				__wt_yield();	/* Let other threads proceed. */
 			} else
-				OP_FAILED(true);
+				WRITE_OP_FAILED(true);
 		}
 
 		/* Perform the operation. */
@@ -875,7 +923,7 @@ ops(void *arg)
 				++tinfo->insert;
 				SNAP_TRACK(INSERT, tinfo);
 			} else
-				OP_FAILED(false);
+				WRITE_OP_FAILED(false);
 			break;
 		case MODIFY:
 			/*
@@ -899,7 +947,7 @@ ops(void *arg)
 				positioned = true;
 				SNAP_TRACK(MODIFY, tinfo);
 			} else
-				OP_FAILED(true);
+				WRITE_OP_FAILED(true);
 			break;
 		case READ:
 			++tinfo->search;
@@ -908,7 +956,7 @@ ops(void *arg)
 				positioned = true;
 				SNAP_TRACK(READ, tinfo);
 			} else
-				OP_FAILED(true);
+				READ_OP_FAILED(true);
 			break;
 		case REMOVE:
 remove_instead_of_truncate:
@@ -929,7 +977,7 @@ remove_instead_of_truncate:
 				 */
 				SNAP_TRACK(REMOVE, tinfo);
 			} else
-				OP_FAILED(true);
+				WRITE_OP_FAILED(true);
 			break;
 		case TRUNCATE:
 			/*
@@ -958,7 +1006,8 @@ remove_instead_of_truncate:
 			 * vice-versa).
 			 */
 			greater_than = mmrand(&tinfo->rnd, 0, 1) == 1;
-			range = mmrand(&tinfo->rnd, 1, (u_int)g.rows / 20);
+			range = g.rows < 20 ?
+			    1 : mmrand(&tinfo->rnd, 1, (u_int)g.rows / 20);
 			tinfo->last = tinfo->keyno;
 			if (greater_than) {
 				if (g.c_reverse) {
@@ -992,14 +1041,15 @@ remove_instead_of_truncate:
 				ret = col_truncate(tinfo, cursor);
 				break;
 			}
-			positioned = false;
 			(void)__wt_atomic_subv64(&g.truncate_cnt, 1);
 
+			/* Truncate never leaves the cursor positioned. */
+			positioned = false;
 			if (ret == 0) {
 				++tinfo->truncate;
 				SNAP_TRACK(TRUNCATE, tinfo);
 			} else
-				OP_FAILED(false);
+				WRITE_OP_FAILED(false);
 			break;
 		case UPDATE:
 update_instead_of_chosen_op:
@@ -1017,7 +1067,7 @@ update_instead_of_chosen_op:
 				positioned = true;
 				SNAP_TRACK(UPDATE, tinfo);
 			} else
-				OP_FAILED(false);
+				WRITE_OP_FAILED(false);
 			break;
 		}
 
@@ -1033,7 +1083,7 @@ update_instead_of_chosen_op:
 				if ((ret = nextprev(tinfo, cursor, next)) == 0)
 					continue;
 
-				OP_FAILED(true);
+				READ_OP_FAILED(true);
 				break;
 			}
 		}
@@ -1066,9 +1116,8 @@ update_instead_of_chosen_op:
 		 */
 		if (g.c_prepare && mmrand(&tinfo->rnd, 1, 10) == 1) {
 			ret = prepare_transaction(tinfo, session);
-			testutil_assert(ret == 0 || ret == WT_PREPARE_CONFLICT);
-			if (ret == WT_PREPARE_CONFLICT)
-				goto rollback;
+			if (ret != 0)
+				WRITE_OP_FAILED(false);
 
 			__wt_yield();		/* Let other threads proceed. */
 		}
@@ -1193,11 +1242,11 @@ read_row_worker(
 	}
 
 	if (sn) {
-		ret = cursor->search_near(cursor, &exact);
+		ret = read_op(cursor, SEARCH_NEAR, &exact);
 		if (ret == 0 && exact != 0)
 			ret = WT_NOTFOUND;
 	} else
-		ret = cursor->search(cursor);
+		ret = read_op(cursor, SEARCH, NULL);
 	switch (ret) {
 	case 0:
 		if (g.type == FIX) {
@@ -1288,7 +1337,7 @@ nextprev(TINFO *tinfo, WT_CURSOR *cursor, bool next)
 	keyno = 0;
 	which = next ? "WT_CURSOR.next" : "WT_CURSOR.prev";
 
-	switch (ret = (next ? cursor->next(cursor) : cursor->prev(cursor))) {
+	switch (ret = read_op(cursor, next ? NEXT : PREV, NULL)) {
 	case 0:
 		switch (g.type) {
 		case FIX:
@@ -2019,7 +2068,7 @@ row_remove(TINFO *tinfo, WT_CURSOR *cursor, bool positioned)
 	}
 
 	/* We use the cursor in overwrite mode, check for existence. */
-	if ((ret = cursor->search(cursor)) == 0)
+	if ((ret = read_op(cursor, SEARCH, NULL)) == 0)
 		ret = cursor->remove(cursor);
 
 	if (ret != 0 && ret != WT_NOTFOUND)
@@ -2053,7 +2102,7 @@ col_remove(TINFO *tinfo, WT_CURSOR *cursor, bool positioned)
 		cursor->set_key(cursor, tinfo->keyno);
 
 	/* We use the cursor in overwrite mode, check for existence. */
-	if ((ret = cursor->search(cursor)) == 0)
+	if ((ret = read_op(cursor, SEARCH, NULL)) == 0)
 		ret = cursor->remove(cursor);
 
 	if (ret != 0 && ret != WT_NOTFOUND)
diff --git a/src/third_party/wiredtiger/test/suite/test_dictionary.py b/src/third_party/wiredtiger/test/suite/test_dictionary.py
new file mode 100644
index 00000000000..f624e1ade35
--- /dev/null
+++ b/src/third_party/wiredtiger/test/suite/test_dictionary.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+#
+# Public Domain 2014-2018 MongoDB, Inc.
+# Public Domain 2008-2014 WiredTiger, Inc.
+#
+# This is free and unencumbered software released into the public domain.
+#
+# Anyone is free to copy, modify, publish, use, compile, sell, or
+# distribute this software, either in source code form or as a compiled
+# binary, for any purpose, commercial or non-commercial, and by any
+# means.
+#
+# In jurisdictions that recognize copyright laws, the author or authors
+# of this software dedicate any and all copyright interest in the
+# software to the public domain. We make this dedication for the benefit
+# of the public at large and to the detriment of our heirs and
+# successors. We intend this dedication to be an overt act of
+# relinquishment in perpetuity of all present and future rights to this
+# software under copyright law.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# test_dictionary.py
+#       Smoke test dictionary compression.
+
+from wtscenario import make_scenarios
+from wtdataset import simple_key
+from wiredtiger import stat
+import wiredtiger, wttest
+
+# Smoke test dictionary compression.
+class test_dictionary(wttest.WiredTigerTestCase):
+    conn_config = 'statistics=(all)'
+    scenarios = make_scenarios([
+        ('row', dict(key_format='S', value_format='S')),
+        ('var', dict(key_format='r', value_format='S')),
+    ])
+
+    # Smoke test dictionary compression.
+    def test_dictionary(self):
+        nentries = 25000
+        uri = 'file:test_dictionary'    # This is a btree layer test.
+
+        # Create the object, open the cursor, insert some records with identical values. Use
+        # alternating values, otherwise column-store will RLE compress them into a single item.
+        self.session.create(uri, 'dictionary=100,value_format=S,key_format=' + self.key_format)
+        cursor = self.session.open_cursor(uri, None)
+        i = 0
+        while i < nentries:
+            i = i + 1
+            cursor[simple_key(cursor, i)] = "the same value as the odd items"
+            i = i + 1
+            cursor[simple_key(cursor, i)] = "the same value as the even items"
+        cursor.close()
+
+        # Checkpoint to force the pages through reconciliation.
+        self.session.checkpoint()
+
+        # Confirm the dictionary was effective.
+        cursor = self.session.open_cursor('statistics:' + uri, None, None)
+        self.assertGreater(cursor[stat.dsrc.rec_dictionary][2], nentries - 100)
+
+if __name__ == '__main__':
+    wttest.run()
diff --git a/src/third_party/wiredtiger/test/suite/test_empty_value.py b/src/third_party/wiredtiger/test/suite/test_empty_value.py
new file mode 100644
index 00000000000..b40eaaef3d1
--- /dev/null
+++ b/src/third_party/wiredtiger/test/suite/test_empty_value.py
@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+#
+# Public Domain 2014-2018 MongoDB, Inc.
+# Public Domain 2008-2014 WiredTiger, Inc.
+#
+# This is free and unencumbered software released into the public domain.
+#
+# Anyone is free to copy, modify, publish, use, compile, sell, or
+# distribute this software, either in source code form or as a compiled
+# binary, for any purpose, commercial or non-commercial, and by any
+# means.
+#
+# In jurisdictions that recognize copyright laws, the author or authors
+# of this software dedicate any and all copyright interest in the
+# software to the public domain. We make this dedication for the benefit
+# of the public at large and to the detriment of our heirs and
+# successors. We intend this dedication to be an overt act of
+# relinquishment in perpetuity of all present and future rights to this
+# software under copyright law.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# test_dictionary.py
+#       Smoke test empty row-store values.
+
+from wtdataset import simple_key
+from wiredtiger import stat
+import wiredtiger, wttest
+
+# Smoke test empty row-store values.
+class test_row_store_empty_values(wttest.WiredTigerTestCase):
+    conn_config = 'statistics=(all)'
+
+    # Smoke test empty row-store values.
+    def test_row_store_empty_values(self):
+        nentries = 25000
+        uri = 'file:test_empty_values'          # This is a btree layer test.
+
+        # Create the object, open the cursor, insert some records with zero-length values.
+        self.session.create(uri, 'value_format=u,key_format=S')
+        cursor = self.session.open_cursor(uri, None)
+        for i in xrange(1, nentries + 1):
+            cursor[simple_key(cursor, i)] = ""
+        cursor.close()
+
+        # Reopen to force the object to disk.
+        self.reopen_conn()
+
+        # Confirm the values weren't stored..
+        cursor = self.session.open_cursor('statistics:' + uri, None, 'statistics=(tree_walk)')
+        self.assertEqual(cursor[stat.dsrc.btree_row_empty_values][2], nentries)
+
+if __name__ == '__main__':
+    wttest.run()
diff --git a/src/third_party/wiredtiger/test/utility/misc.c b/src/third_party/wiredtiger/test/utility/misc.c
index 8d5605208cf..b66ae497707 100644
--- a/src/third_party/wiredtiger/test/utility/misc.c
+++ b/src/third_party/wiredtiger/test/utility/misc.c
@@ -199,7 +199,7 @@ bool
 testutil_is_flag_set(const char *flag)
 {
 	const char *res;
-	bool enable_long_tests;
+	bool flag_being_set;
 
 	if (__wt_getenv(NULL, flag, &res) != 0 || res == NULL)
 		return (false);
@@ -208,11 +208,11 @@ testutil_is_flag_set(const char *flag)
 	 * This is a boolean test. So if the environment variable is set to any
 	 * value other than 0, we return success.
 	 */
-	enable_long_tests = res[0] != '0';
+	flag_being_set = res[0] != '0';
 
 	free((void *)res);
 
-	return (enable_long_tests);
+	return (flag_being_set);
 }
 
 /*