Import wiredtiger: e6c749653220cf701c23634cd704ae0c2d882dd9 from branch mongodb-4.0

ref: 7d3e691fd4..e6c7496532
for: 4.0.1

WT-3839       Document the undefined behavior when a range truncate overlaps with inserts
WT-3917       Enhance WT_CURSOR::reserve documentation around commit visibility
WT-4024       Fix a race between split and next/prev
WT-4048       Generalize timing_stress_for_test split functionality
WT-4067       Enhance LSM to not pin as much history in cache
WT-4101       Don't abort the eviction server during session verify when oldest_timestamp is held back
WT-4111       Improve checkpoint scrubbing algorithm
WT-4125       Ensure that subsequent checkpoints with stable timestamp don't read too much
WT-4133       Coverity 1393445, 1393446 Dereference before null check
WT-4136       Add a new timing stress flag that yields during tree search
WT-4138       Add an option to timeout waiting for space in the cache
WT-4139       rename the cursor restart statistic to match implementation
WT-4140       Cursor walk limits quick eviction page selection unnecessarily.
WT-4141       Enhance checkpoint with timestamps to unblock eviction sooner
WT-4143       Use WiredTiger.turtle.set if it exists but WiredTiger.turtle does not
WT-4145       Only include the checkpoint timestamp during checkpoints
WT-4146       Coverity 1393639, unused variable
WT-4152       Save return value for later comparison in transaction code
WT-4163       Lint

7 files changed, 200 insertions, 18 deletions

diff --git a/src/third_party/wiredtiger/test/format/config.h b/src/third_party/wiredtiger/test/format/config.h
index 51dc906465a..c398c1a96b2 100644
--- a/src/third_party/wiredtiger/test/format/config.h
+++ b/src/third_party/wiredtiger/test/format/config.h
@@ -331,41 +331,49 @@ static CONFIG c[] = {
 	  C_IGNORE, 0, 0, UINT_MAX, &g.c_timer, NULL },
 
 	{ "timing_stress_checkpoint",
-	  "configure slow checkpoints",				/* 2% */
+	  "stress checkpoints",					/* 2% */
 	  C_BOOL, 2, 0, 0, &g.c_timing_stress_checkpoint, NULL },
 
 	{ "timing_stress_lookaside_sweep",
-	  "configure slow lookaside sweep",			/* 2% */
+	  "stress lookaside sweep",				/* 2% */
 	  C_BOOL, 2, 0, 0, &g.c_timing_stress_lookaside_sweep, NULL },
 
 	{ "timing_stress_split_1",
-	  "configure slow splits (#1)",				/* 2% */
+	  "stress splits (#1)",					/* 2% */
 	  C_BOOL, 2, 0, 0, &g.c_timing_stress_split_1, NULL },
 
 	{ "timing_stress_split_2",
-	  "configure slow splits (#2)",				/* 2% */
+	  "stress splits (#2)",					/* 2% */
 	  C_BOOL, 2, 0, 0, &g.c_timing_stress_split_2, NULL },
 
 	{ "timing_stress_split_3",
-	  "configure slow splits (#3)",				/* 2% */
+	  "stress splits (#3)",					/* 2% */
 	  C_BOOL, 2, 0, 0, &g.c_timing_stress_split_3, NULL },
 
 	{ "timing_stress_split_4",
-	  "configure slow splits (#4)",				/* 2% */
+	  "stress splits (#4)",					/* 2% */
 	  C_BOOL, 2, 0, 0, &g.c_timing_stress_split_4, NULL },
 
 	{ "timing_stress_split_5",
-	  "configure slow splits (#5)",				/* 2% */
+	  "stress splits (#5)",					/* 2% */
 	  C_BOOL, 2, 0, 0, &g.c_timing_stress_split_5, NULL },
 
 	{ "timing_stress_split_6",
-	  "configure slow splits (#6)",				/* 2% */
+	  "stress splits (#6)",					/* 2% */
 	  C_BOOL, 2, 0, 0, &g.c_timing_stress_split_6, NULL },
 
 	{ "timing_stress_split_7",
-	  "configure slow splits (#7)",				/* 2% */
+	  "stress splits (#7)",					/* 2% */
 	  C_BOOL, 2, 0, 0, &g.c_timing_stress_split_7, NULL },
 
+	{ "timing_stress_split_8",
+	  "stress splits (#8)",					/* 2% */
+	  C_BOOL, 2, 0, 0, &g.c_timing_stress_split_8, NULL },
+
+	{ "timing_stress_split_9",
+	  "stress splits (#9)",					/* 2% */
+	  C_BOOL, 2, 0, 0, &g.c_timing_stress_split_9, NULL },
+
 	{ "transaction_timestamps",				/* 10% */
 	  "enable transaction timestamp support",
 	  C_BOOL, 10, 0, 0, &g.c_txn_timestamps, NULL },
diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h
index 0eca6657dd9..1406d2b3fb5 100644
--- a/src/third_party/wiredtiger/test/format/format.h
+++ b/src/third_party/wiredtiger/test/format/format.h
@@ -221,6 +221,8 @@ typedef struct {
 	uint32_t c_timing_stress_split_5;
 	uint32_t c_timing_stress_split_6;
 	uint32_t c_timing_stress_split_7;
+	uint32_t c_timing_stress_split_8;
+	uint32_t c_timing_stress_split_9;
 	uint32_t c_truncate;
 	uint32_t c_txn_freq;
 	uint32_t c_txn_timestamps;
diff --git a/src/third_party/wiredtiger/test/format/lrt.c b/src/third_party/wiredtiger/test/format/lrt.c
index 9d99933ef64..31c5de93870 100644
--- a/src/third_party/wiredtiger/test/format/lrt.c
+++ b/src/third_party/wiredtiger/test/format/lrt.c
@@ -110,8 +110,15 @@ lrt(void *arg)
 			 */
 			testutil_check(session->snapshot(session, "name=test"));
 			__wt_sleep(1, 0);
-			testutil_check(session->begin_transaction(
-			    session, "snapshot=test"));
+			/*
+			 * Keep trying to start a new transaction if it's
+			 * timing out - we know there aren't any resources
+			 * pinned so it should succeed eventually.
+			 */
+			while ((ret = session->begin_transaction(
+			    session, "snapshot=test")) == WT_CACHE_FULL)
+				;
+			testutil_check(ret);
 			testutil_check(session->snapshot(
 			    session, "drop=(all)"));
 			testutil_check(session->commit_transaction(
@@ -123,8 +130,10 @@ lrt(void *arg)
 			 * positioned. As soon as the cursor loses its position
 			 * a new snapshot will be allocated.
 			 */
-			testutil_check(session->begin_transaction(
-			    session, "isolation=snapshot"));
+			while ((ret = session->begin_transaction(
+			    session, "snapshot=snapshot")) == WT_CACHE_FULL)
+				;
+			testutil_check(ret);
 
 			/* Read a record at the end of the table. */
 			do {
diff --git a/src/third_party/wiredtiger/test/format/ops.c b/src/third_party/wiredtiger/test/format/ops.c
index 7d08dbd8bd8..27aa118a4ef 100644
--- a/src/third_party/wiredtiger/test/format/ops.c
+++ b/src/third_party/wiredtiger/test/format/ops.c
@@ -456,10 +456,10 @@ snap_check(WT_CURSOR *cursor,
 				print_item_data(
 				    "expected", start->vdata, start->vsize);
 			if (ret == WT_NOTFOUND)
-				fprintf(stderr, "\t   found {deleted}\n");
+				fprintf(stderr, "found {deleted}\n");
 			else
 				print_item_data(
-				    "   found", value->data, value->size);
+				    "found", value->data, value->size);
 
 			testutil_die(ret,
 			    "snapshot-isolation: %.*s search mismatch",
@@ -476,10 +476,10 @@ snap_check(WT_CURSOR *cursor,
 				print_item_data(
 				    "expected", start->vdata, start->vsize);
 			if (ret == WT_NOTFOUND)
-				fprintf(stderr, "\t   found {deleted}\n");
+				fprintf(stderr, "found {deleted}\n");
 			else
 				print_item_data(
-				    "   found", value->data, value->size);
+				    "found", value->data, value->size);
 
 			testutil_die(ret,
 			    "snapshot-isolation: %" PRIu64 " search mismatch",
@@ -498,6 +498,7 @@ static void
 begin_transaction(TINFO *tinfo, WT_SESSION *session, u_int *iso_configp)
 {
 	u_int v;
+	int ret;
 	const char *config;
 	char config_buf[64];
 	bool locked;
@@ -523,7 +524,15 @@ begin_transaction(TINFO *tinfo, WT_SESSION *session, u_int *iso_configp)
 	}
 	*iso_configp = v;
 
-	testutil_check(session->begin_transaction(session, config));
+	/*
+	 * Keep trying to start a new transaction if it's timing out - we
+	 * know there aren't any resources pinned so it should succeed
+	 * eventually.
+	 */
+	while ((ret =
+	    session->begin_transaction(session, config)) == WT_CACHE_FULL)
+		;
+	testutil_check(ret);
 
 	if (v == ISOLATION_SNAPSHOT && g.c_txn_timestamps) {
 		/* Avoid starting a new reader when a prepare is in progress. */
diff --git a/src/third_party/wiredtiger/test/format/wts.c b/src/third_party/wiredtiger/test/format/wts.c
index dd87adeae56..8040142aa19 100644
--- a/src/third_party/wiredtiger/test/format/wts.c
+++ b/src/third_party/wiredtiger/test/format/wts.c
@@ -262,6 +262,10 @@ wts_open(const char *home, bool set_api, WT_CONNECTION **connp)
 		CONFIG_APPEND(p, ",split_6");
 	if (g.c_timing_stress_split_7)
 		CONFIG_APPEND(p, ",split_7");
+	if (g.c_timing_stress_split_8)
+		CONFIG_APPEND(p, ",split_8");
+	if (g.c_timing_stress_split_9)
+		CONFIG_APPEND(p, ",split_9");
 	CONFIG_APPEND(p, "]");
 
 	/* Extensions. */
diff --git a/src/third_party/wiredtiger/test/suite/test_bug020.py b/src/third_party/wiredtiger/test/suite/test_bug020.py
new file mode 100644
index 00000000000..646f2505e83
--- /dev/null
+++ b/src/third_party/wiredtiger/test/suite/test_bug020.py
@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+#
+# Public Domain 2014-2018 MongoDB, Inc.
+# Public Domain 2008-2014 WiredTiger, Inc.
+#
+# This is free and unencumbered software released into the public domain.
+#
+# Anyone is free to copy, modify, publish, use, compile, sell, or
+# distribute this software, either in source code form or as a compiled
+# binary, for any purpose, commercial or non-commercial, and by any
+# means.
+#
+# In jurisdictions that recognize copyright laws, the author or authors
+# of this software dedicate any and all copyright interest in the
+# software to the public domain. We make this dedication for the benefit
+# of the public at large and to the detriment of our heirs and
+# successors. We intend this dedication to be an overt act of
+# relinquishment in perpetuity of all present and future rights to this
+# software under copyright law.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+
+import os
+import wiredtiger, wttest
+from wtdataset import SimpleDataSet
+
+# test_bug020.py
+#    Test that an existing set file will replace a missing turtle file.
+class test_bug020(wttest.WiredTigerTestCase):
+    def test_bug020(self):
+        SimpleDataSet(self, "table:bug020", 1000).populate()
+        self.close_conn()
+        os.rename("WiredTiger.turtle", "WiredTiger.turtle.set")
+        expectMessage = 'WiredTiger.turtle not found'
+        with self.expectedStdoutPattern(expectMessage):
+            self.open_conn()
+
+if __name__ == '__main__':
+    wttest.run()
diff --git a/src/third_party/wiredtiger/test/suite/test_las03.py b/src/third_party/wiredtiger/test/suite/test_las03.py
new file mode 100644
index 00000000000..6934bd9a741
--- /dev/null
+++ b/src/third_party/wiredtiger/test/suite/test_las03.py
@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+#
+# Public Domain 2014-2018 MongoDB, Inc.
+# Public Domain 2008-2014 WiredTiger, Inc.
+#
+# This is free and unencumbered software released into the public domain.
+#
+# Anyone is free to copy, modify, publish, use, compile, sell, or
+# distribute this software, either in source code form or as a compiled
+# binary, for any purpose, commercial or non-commercial, and by any
+# means.
+#
+# In jurisdictions that recognize copyright laws, the author or authors
+# of this software dedicate any and all copyright interest in the
+# software to the public domain. We make this dedication for the benefit
+# of the public at large and to the detriment of our heirs and
+# successors. We intend this dedication to be an overt act of
+# relinquishment in perpetuity of all present and future rights to this
+# software under copyright law.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+
+from helper import copy_wiredtiger_home
+import wiredtiger, wttest
+from wiredtiger import stat
+from wtdataset import SimpleDataSet
+
+def timestamp_str(t):
+    return '%x' % t
+
+# test_las03.py
+# Ensure checkpoints don't read too unnecessary lookaside entries.
+class test_las03(wttest.WiredTigerTestCase):
+    # Force a small cache.
+    def conn_config(self):
+        return 'cache_size=50MB,statistics=(fast)'
+
+    def get_stat(self, stat):
+        stat_cursor = self.session.open_cursor('statistics:')
+        val = stat_cursor[stat][2]
+        stat_cursor.close()
+        return val
+
+    def large_updates(self, session, uri, value, ds, nrows, nops):
+        # Update a large number of records, we'll hang if the lookaside table
+        # isn't doing its thing.
+        cursor = session.open_cursor(uri)
+        for i in range(nrows + 1, nrows + nops + 1):
+            session.begin_transaction()
+            cursor[ds.key(i)] = value
+            session.commit_transaction('commit_timestamp=' + timestamp_str(i))
+        cursor.close()
+
+    def test_checkpoint_las_reads(self):
+        if not wiredtiger.timestamp_build():
+            self.skipTest('requires a timestamp build')
+
+        # Create a small table.
+        uri = "table:test_las03"
+        nrows = 100
+        ds = SimpleDataSet(self, uri, nrows, key_format="S", value_format='u')
+        ds.populate()
+        bigvalue = "aaaaa" * 100
+
+        # Initially load huge data
+        cursor = self.session.open_cursor(uri)
+        for i in range(1, 10000):
+            cursor[ds.key(nrows + i)] = bigvalue
+        cursor.close()
+        self.session.checkpoint()
+
+        # Check to see LAS working with old timestamp
+        bigvalue2 = "ddddd" * 100
+        self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1))
+        las_writes_start = self.get_stat(stat.conn.cache_write_lookaside)
+        self.large_updates(self.session, uri, bigvalue2, ds, nrows, 10000)
+
+        # If the test sizing is correct, the history will overflow the cache
+        self.session.checkpoint()
+        las_writes = self.get_stat(stat.conn.cache_write_lookaside) - las_writes_start
+        self.assertGreaterEqual(las_writes, 0)
+
+        for ts in range(2, 4):
+            self.conn.set_timestamp('stable_timestamp=' + timestamp_str(ts))
+
+            # Now just update one record and checkpoint again
+            self.large_updates(self.session, uri, bigvalue2, ds, nrows, 1)
+
+            las_reads_start = self.get_stat(stat.conn.cache_read_lookaside)
+            self.session.checkpoint()
+            las_reads = self.get_stat(stat.conn.cache_read_lookaside) - las_reads_start
+
+            # Since we're dealing with eviction concurrent with checkpoints
+            # and skewing is controlled by a heuristic, we can't put too tight
+            # a bound on this.
+            self.assertLessEqual(las_reads, 100)
+
+if __name__ == '__main__':
+    wttest.run()