Import wiredtiger: 1d72f206c3078d002666cc6ea042ec9fa1fbe13c from branch mongodb-4.0

ref: 4f2ed4c4cb..1d72f206c3
for: 4.0.10

WT-4631       Always clear the read timestamp for transactions
WT-4687       Querying all_committed shouldn't get timestamp zero
WT-4693       WT_CONNECTION::reconfigure should not require quiescence when downgraded

5 files changed, 148 insertions, 26 deletions

diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index 8afc019a072..e6bd6967675 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -1,5 +1,5 @@
 {
-    "commit": "4f2ed4c4cb122342711ce82d553482124501fcd4", 
+    "commit": "1d72f206c3078d002666cc6ea042ec9fa1fbe13c", 
     "github": "wiredtiger/wiredtiger.git", 
     "vendor": "wiredtiger", 
     "branch": "mongodb-4.0"
diff --git a/src/third_party/wiredtiger/src/conn/conn_reconfig.c b/src/third_party/wiredtiger/src/conn/conn_reconfig.c
index c6d7203f08e..1cd589c32c9 100644
--- a/src/third_party/wiredtiger/src/conn/conn_reconfig.c
+++ b/src/third_party/wiredtiger/src/conn/conn_reconfig.c
@@ -55,7 +55,7 @@ __wt_conn_compat_config(
 	uint16_t max_major, max_minor, min_major, min_minor;
 	uint16_t rel_major, rel_minor;
 	char *value;
-	bool txn_active;
+	bool txn_active, unchg;
 
 	conn = S2C(session);
 	value = NULL;
@@ -63,6 +63,7 @@ __wt_conn_compat_config(
 	max_minor = WT_CONN_COMPAT_NONE;
 	min_major = WT_CONN_COMPAT_NONE;
 	min_minor = WT_CONN_COMPAT_NONE;
+	unchg = false;
 
 	WT_RET(__wt_config_gets(session, cfg, "compatibility.release", &cval));
 	if (cval.len == 0) {
@@ -74,21 +75,31 @@ __wt_conn_compat_config(
 		    session, &cval, &rel_major, &rel_minor));
 
 		/*
-		 * We're doing an upgrade or downgrade, check whether
-		 * transactions are active.
+		 * If the user is running downgraded, then the compatibility
+		 * string is part of the configuration string. Determine if
+		 * the user is actually changing the compatibility.
 		 */
-		WT_RET(__wt_txn_activity_check(session, &txn_active));
-		if (txn_active)
-			WT_RET_MSG(session, ENOTSUP,
-			    "system must be quiescent"
-			    " for upgrade or downgrade");
+		if (reconfig && rel_major == conn->compat_major &&
+		    rel_minor == conn->compat_minor)
+			unchg = true;
+		else {
+			/*
+			 * We're doing an upgrade or downgrade, check whether
+			 * transactions are active.
+			 */
+			WT_RET(__wt_txn_activity_check(session, &txn_active));
+			if (txn_active)
+				WT_RET_MSG(session, ENOTSUP,
+				    "system must be quiescent"
+				    " for upgrade or downgrade");
+		}
 		F_SET(conn, WT_CONN_COMPATIBILITY);
 	}
 	/*
-	 * If we're a reconfigure and the user did not set any compatibility,
-	 * we're done.
+	 * If we're a reconfigure and the user did not set any compatibility
+	 * or did not change the setting, we're done.
 	 */
-	if (reconfig && !F_ISSET(conn, WT_CONN_COMPATIBILITY))
+	if (reconfig && (!F_ISSET(conn, WT_CONN_COMPATIBILITY) || unchg))
 		goto done;
 
 	/*
diff --git a/src/third_party/wiredtiger/src/txn/txn_timestamp.c b/src/third_party/wiredtiger/src/txn/txn_timestamp.c
index 385817e89b9..b50da548f71 100644
--- a/src/third_party/wiredtiger/src/txn/txn_timestamp.c
+++ b/src/third_party/wiredtiger/src/txn/txn_timestamp.c
@@ -132,6 +132,20 @@ __wt_txn_parse_timestamp(WT_SESSION_IMPL *session, const char *name,
 }
 
 /*
+ * __txn_get_read_timestamp --
+ *	Get the read timestamp from the transaction. Additionally
+ *	return bool to specify whether the transaction has set
+ *	clear read queue flag.
+ */
+static bool
+__txn_get_read_timestamp(
+    WT_TXN *txn, wt_timestamp_t *read_timestampp)
+{
+	WT_ORDERED_READ(*read_timestampp, txn->read_timestamp);
+	return (!txn->clear_read_q);
+}
+
+/*
  * __txn_get_pinned_timestamp --
  *	Calculate the current pinned timestamp.
  */
@@ -142,7 +156,7 @@ __txn_get_pinned_timestamp(
 	WT_CONNECTION_IMPL *conn;
 	WT_TXN *txn;
 	WT_TXN_GLOBAL *txn_global;
-	wt_timestamp_t tmp_ts;
+	wt_timestamp_t tmp_read_ts, tmp_ts;
 	bool include_oldest, txn_has_write_lock;
 
 	conn = S2C(session);
@@ -171,15 +185,18 @@ __txn_get_pinned_timestamp(
 	TAILQ_FOREACH(txn, &txn_global->read_timestamph, read_timestampq) {
 		/*
 		 * Skip any transactions on the queue that are not active.
+		 * Copy out value of read timestamp to prevent possible
+		 * race where a transaction resets its read timestamp while
+		 * we traverse the queue.
 		 */
-		if (txn->clear_read_q)
+		if (!__txn_get_read_timestamp(txn, &tmp_read_ts))
 			continue;
 		/*
 		 * A zero timestamp is possible here only when the oldest
 		 * timestamp is not accounted for.
 		 */
-		if (tmp_ts == 0 || txn->read_timestamp < tmp_ts)
-			tmp_ts = txn->read_timestamp;
+		if (tmp_ts == 0 || tmp_read_ts < tmp_ts)
+			tmp_ts = tmp_read_ts;
 		/*
 		 * We break on the first active txn on the list.
 		 */
@@ -239,6 +256,13 @@ __txn_global_query_timestamp(
 			break;
 		}
 		__wt_readunlock(session, &txn_global->commit_timestamp_rwlock);
+
+		/*
+		 * If a transaction is committing with timestamp 1, we could
+		 * return zero here, which is unexpected.  Fail instead.
+		 */
+		if (ts == 0)
+			return (WT_NOTFOUND);
 	} else if (WT_STRING_MATCH("last_checkpoint", cval.str, cval.len))
 		/* Read-only value forever. No lock needed. */
 		ts = txn_global->last_ckpt_timestamp;
@@ -679,7 +703,7 @@ __wt_txn_parse_prepare_timestamp(
 	WT_CONFIG_ITEM cval;
 	WT_TXN *prev;
 	WT_TXN_GLOBAL *txn_global;
-	wt_timestamp_t oldest_ts;
+	wt_timestamp_t oldest_ts, tmp_timestamp;
 	char hex_timestamp[WT_TS_HEX_SIZE];
 
 	txn_global = &S2C(session)->txn_global;
@@ -706,12 +730,12 @@ __wt_txn_parse_prepare_timestamp(
 			/*
 			 * Skip any transactions that are not active.
 			 */
-			if (prev->clear_read_q) {
+			if (!__txn_get_read_timestamp(prev, &tmp_timestamp)) {
 				prev = TAILQ_PREV(
 				    prev, __wt_txn_rts_qh, read_timestampq);
 				continue;
 			}
-			if (prev->read_timestamp >= *timestamp) {
+			if (tmp_timestamp >= *timestamp) {
 				__wt_readunlock(session,
 				    &txn_global->read_timestamp_rwlock);
 				__wt_timestamp_to_hex_string(
@@ -972,6 +996,7 @@ __wt_txn_set_read_timestamp(WT_SESSION_IMPL *session)
 {
 	WT_TXN *qtxn, *txn, *txn_tmp;
 	WT_TXN_GLOBAL *txn_global;
+	wt_timestamp_t tmp_timestamp;
 	uint64_t walked;
 
 	txn = &session->txn;
@@ -1023,11 +1048,14 @@ __wt_txn_set_read_timestamp(WT_SESSION_IMPL *session)
 		 */
 		qtxn = TAILQ_LAST(
 		     &txn_global->read_timestamph, __wt_txn_rts_qh);
-		while (qtxn != NULL &&
-		    qtxn->read_timestamp > txn->read_timestamp) {
-			++walked;
-			qtxn = TAILQ_PREV(
-			    qtxn, __wt_txn_rts_qh, read_timestampq);
+		while (qtxn != NULL) {
+			if (!__txn_get_read_timestamp(qtxn, &tmp_timestamp) ||
+			    tmp_timestamp > txn->read_timestamp) {
+				++walked;
+				qtxn = TAILQ_PREV(qtxn,
+				    __wt_txn_rts_qh, read_timestampq);
+			} else
+				break;
 		}
 		if (qtxn == NULL) {
 			TAILQ_INSERT_HEAD(&txn_global->read_timestamph,
@@ -1061,9 +1089,10 @@ __wt_txn_clear_read_timestamp(WT_SESSION_IMPL *session)
 
 	txn = &session->txn;
 
-	if (!F_ISSET(txn, WT_TXN_PUBLIC_TS_READ))
+	if (!F_ISSET(txn, WT_TXN_PUBLIC_TS_READ)) {
+		txn->read_timestamp = WT_TS_NONE;
 		return;
-
+	}
 #ifdef HAVE_DIAGNOSTIC
 	{
 	WT_TXN_GLOBAL *txn_global;
@@ -1084,6 +1113,7 @@ __wt_txn_clear_read_timestamp(WT_SESSION_IMPL *session)
 	 */
 	WT_PUBLISH(txn->clear_read_q, true);
 	WT_PUBLISH(txn->flags, flags);
+	txn->read_timestamp = WT_TS_NONE;
 }
 
 /*
diff --git a/src/third_party/wiredtiger/test/suite/test_compat01.py b/src/third_party/wiredtiger/test/suite/test_compat01.py
index 9670a4cc137..faf5ed1673f 100644
--- a/src/third_party/wiredtiger/test/suite/test_compat01.py
+++ b/src/third_party/wiredtiger/test/suite/test_compat01.py
@@ -189,12 +189,20 @@ class test_reconfig_fail(wttest.WiredTigerTestCase):
         ds = SimpleDataSet(self, uri, 100, key_format='S')
         ds.populate()
 
+        # Reconfigure to an older version.
+        compat_str = 'compatibility=(release="2.6")'
+        self.conn.reconfigure(compat_str)
+
         self.session.begin_transaction("isolation=snapshot")
         c = self.session.open_cursor(uri, None)
         c.set_key(ds.key(20))
         c.set_value("abcde")
         self.assertEquals(c.update(), 0)
 
+        # Make sure we can reconfigure unrelated things while downgraded
+        # and we have an active transaction.
+        self.conn.reconfigure("cache_size=100M")
+
         compat_str = 'compatibility=(release="3.0.0")'
         msg = '/system must be quiescent/'
         self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp16.py b/src/third_party/wiredtiger/test/suite/test_timestamp16.py
new file mode 100644
index 00000000000..375150e4673
--- /dev/null
+++ b/src/third_party/wiredtiger/test/suite/test_timestamp16.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+#
+# Public Domain 2014-2019 MongoDB, Inc.
+# Public Domain 2008-2014 WiredTiger, Inc.
+#
+# This is free and unencumbered software released into the public domain.
+#
+# Anyone is free to copy, modify, publish, use, compile, sell, or
+# distribute this software, either in source code form or as a compiled
+# binary, for any purpose, commercial or non-commercial, and by any
+# means.
+#
+# In jurisdictions that recognize copyright laws, the author or authors
+# of this software dedicate any and all copyright interest in the
+# software to the public domain. We make this dedication for the benefit
+# of the public at large and to the detriment of our heirs and
+# successors. We intend this dedication to be an overt act of
+# relinquishment in perpetuity of all present and future rights to this
+# software under copyright law.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# test_timestamp16.py
+#   Test to ensure read timestamp is properly cleared at the
+#   end of a txn.
+#
+
+import random
+from suite_subprocess import suite_subprocess
+import wiredtiger, wttest
+from wtscenario import make_scenarios
+
+def timestamp_str(t):
+    return '%x' % t
+
+class test_timestamp16(wttest.WiredTigerTestCase, suite_subprocess):
+    tablename = 'test_timestamp16'
+    uri = 'table:' + tablename
+
+    def test_read_timestamp_cleared(self):
+        # Ensure that the read timestamp doesn't move our checkpoint.
+        self.session.create(self.uri, 'key_format=i,value_format=i')
+        self.session.begin_transaction('read_timestamp=100')
+        self.session.rollback_transaction()
+        self.session.checkpoint('use_timestamp=true')
+        self.assertTimestampsEqual('0',
+            self.conn.query_timestamp('get=last_checkpoint'))
+
+        # Set a stable and make sure that we still checkpoint at
+        # the stable.
+        self.conn.set_timestamp('stable_timestamp=1')
+        self.session.begin_transaction('read_timestamp=100')
+        self.session.rollback_transaction()
+        self.session.checkpoint('use_timestamp=true')
+        self.assertTimestampsEqual('1',
+            self.conn.query_timestamp('get=last_checkpoint'))
+
+        # Finally make sure that commit also resets the read timestamp.
+        self.session.create(self.uri, 'key_format=i,value_format=i')
+        self.session.begin_transaction('read_timestamp=150')
+        self.session.commit_transaction()
+        self.session.checkpoint('use_timestamp=true')
+        self.assertTimestampsEqual('1',
+            self.conn.query_timestamp('get=last_checkpoint'))
+
+if __name__ == '__main__':
+    wttest.run()