Import wiredtiger: 00c68a05cfaa95e4f9f8205e99ec0fbcc3cc8b3f from branch mongodb-4.4

ref: d247a62395..00c68a05cf
for: 4.4.16

WT-8366       API to set transaction commit timestamp from numerical timestamp

6 files changed, 331 insertions, 6 deletions

diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index fb6aa9017af..ad66791d466 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-4.4",
-    "commit": "d247a62395e1227f40909a1fd7ccec66057a47f4"
+    "commit": "00c68a05cfaa95e4f9f8205e99ec0fbcc3cc8b3f"
 }
diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h
index ffd9a61a4e1..f3c66f2dc0d 100644
--- a/src/third_party/wiredtiger/src/include/extern.h
+++ b/src/third_party/wiredtiger/src/include/extern.h
@@ -1604,6 +1604,8 @@ extern int __wt_txn_set_read_timestamp(WT_SESSION_IMPL *session, wt_timestamp_t
   WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern int __wt_txn_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[])
   WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
+extern int __wt_txn_set_timestamp_uint(WT_SESSION_IMPL *session, WT_TS_TXN_TYPE which,
+  wt_timestamp_t ts) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern int __wt_txn_truncate_log(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *start,
   WT_CURSOR_BTREE *stop) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern int __wt_txn_ts_log(WT_SESSION_IMPL *session)
diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in
index ec50d871f90..d7dd8ac993b 100644
--- a/src/third_party/wiredtiger/src/include/wiredtiger.in
+++ b/src/third_party/wiredtiger/src/include/wiredtiger.in
@@ -743,6 +743,14 @@ struct __wt_cursor {
 #endif
 };
 
+/*! WT_SESSION::timestamp_transaction_uint timestamp types */
+typedef enum {
+	WT_TS_TXN_TYPE_COMMIT, /*!< Commit timestamp. */
+	WT_TS_TXN_TYPE_DURABLE, /*!< Durable timestamp. */
+	WT_TS_TXN_TYPE_PREPARE, /*!< Prepare timestamp. */
+	WT_TS_TXN_TYPE_READ /*!< Read timestamp. */
+} WT_TS_TXN_TYPE;
+
 /*!
  * All data operations are performed in the context of a WT_SESSION.  This
  * encapsulates the thread and transactional context of the operation.
@@ -1848,6 +1856,26 @@ struct __wt_session {
 	int __F(timestamp_transaction)(WT_SESSION *session, const char *config);
 
 	/*!
+	 * Set a timestamp on a transaction numerically.  Prefer this over @ref
+	 * timestamp_transaction when the string parsing done in that method becomes a bottleneck.
+	 *
+	 * The WT_SESSION.timestamp_transaction_uint method can only be used at snapshot
+	 * isolation.
+	 *
+	 * @requires_transaction
+	 *
+	 * @param session the session handle
+	 * @param which the timestamp you wish to set. See @ref WT_TS_TXN_TYPE for available
+	 * options, and @ref timestamp_transaction for constraints on when timestamps can be
+	 * set.
+	 * @param ts the timestamp.
+	 * @errors
+	 */
+	int __F(timestamp_transaction_uint)(WT_SESSION *session, WT_TS_TXN_TYPE which,
+	        uint64_t ts);
+	/*! @} */
+
+	/*!
 	 * Query the session's transaction timestamp state.
 	 *
 	 * The WT_SESSION.query_timestamp method can only be used at snapshot isolation.
diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c
index 03827e3e76d..871b3945ec9 100644
--- a/src/third_party/wiredtiger/src/session/session_api.c
+++ b/src/third_party/wiredtiger/src/session/session_api.c
@@ -1748,7 +1748,8 @@ err:
 
 /*
  * __session_timestamp_transaction --
- *     WT_SESSION->timestamp_transaction method.
+ *     WT_SESSION->timestamp_transaction method. Also see __session_timestamp_transaction_uint if
+ *     config parsing is a performance issue.
  */
 static int
 __session_timestamp_transaction(WT_SESSION *wt_session, const char *config)
@@ -1770,6 +1771,24 @@ err:
 }
 
 /*
+ * __session_timestamp_transaction_uint --
+ *     WT_SESSION->timestamp_transaction_uint method.
+ */
+static int
+__session_timestamp_transaction_uint(WT_SESSION *wt_session, WT_TS_TXN_TYPE which, uint64_t ts)
+{
+    WT_DECL_RET;
+    WT_SESSION_IMPL *session;
+
+    session = (WT_SESSION_IMPL *)wt_session;
+    SESSION_API_CALL_PREPARE_ALLOWED_NOCONF(session, timestamp_transaction_uint);
+
+    ret = __wt_txn_set_timestamp_uint(session, which, (wt_timestamp_t)ts);
+err:
+    API_END_RET(session, ret);
+}
+
+/*
  * __session_query_timestamp --
  *     WT_SESSION->query_timestamp method.
  */
@@ -1962,8 +1981,9 @@ __open_session(WT_CONNECTION_IMPL *conn, WT_EVENT_HANDLER *event_handler, const
         __session_log_printf, __session_rename, __session_reset, __session_salvage,
         __session_truncate, __session_upgrade, __session_verify, __session_begin_transaction,
         __session_commit_transaction, __session_prepare_transaction, __session_reset_snapshot,
-        __session_rollback_transaction, __session_timestamp_transaction, __session_query_timestamp,
-        __session_checkpoint, __session_transaction_pinned_range, __wt_session_breakpoint},
+        __session_rollback_transaction, __session_timestamp_transaction,
+        __session_timestamp_transaction_uint, __session_query_timestamp, __session_checkpoint,
+        __session_transaction_pinned_range, __wt_session_breakpoint},
       stds_readonly = {NULL, NULL, __session_close, __session_reconfigure, __session_flush_tier,
         __wt_session_strerror, __session_open_cursor, __session_alter_readonly,
         __session_create_readonly, __wt_session_compact_readonly, __session_drop_readonly,
@@ -1972,7 +1992,8 @@ __open_session(WT_CONNECTION_IMPL *conn, WT_EVENT_HANDLER *event_handler, const
         __session_truncate_readonly, __session_upgrade_readonly, __session_verify,
         __session_begin_transaction, __session_commit_transaction,
         __session_prepare_transaction_readonly, __session_reset_snapshot,
-        __session_rollback_transaction, __session_timestamp_transaction, __session_query_timestamp,
+        __session_rollback_transaction, __session_timestamp_transaction,
+        __session_timestamp_transaction_uint, __session_query_timestamp,
         __session_checkpoint_readonly, __session_transaction_pinned_range, __wt_session_breakpoint};
     WT_DECL_RET;
     WT_SESSION_IMPL *session, *session_ret;
diff --git a/src/third_party/wiredtiger/src/txn/txn_timestamp.c b/src/third_party/wiredtiger/src/txn/txn_timestamp.c
index c138f23fe57..0e5413aa162 100644
--- a/src/third_party/wiredtiger/src/txn/txn_timestamp.c
+++ b/src/third_party/wiredtiger/src/txn/txn_timestamp.c
@@ -866,7 +866,7 @@ __wt_txn_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[])
     bool set_ts;
 
     set_ts = false;
-    WT_TRET(__wt_txn_context_check(session, true));
+    WT_RET(__wt_txn_context_check(session, true));
 
     /* Look for a commit timestamp. */
     ret = __wt_config_gets_def(session, cfg, "commit_timestamp", 0, &cval);
@@ -911,6 +911,44 @@ __wt_txn_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[])
 }
 
 /*
+ * __wt_txn_set_timestamp_uint --
+ *     Directly set the commit timestamp in a transaction, bypassing parsing logic. Prefer this to
+ *     __wt_txn_set_timestamp when string parsing is a performance bottleneck.
+ */
+int
+__wt_txn_set_timestamp_uint(WT_SESSION_IMPL *session, WT_TS_TXN_TYPE which, wt_timestamp_t ts)
+{
+    WT_CONNECTION_IMPL *conn;
+
+    WT_RET(__wt_txn_context_check(session, true));
+
+    conn = S2C(session);
+
+    switch (which) {
+    case WT_TS_TXN_TYPE_COMMIT:
+        WT_RET(__wt_txn_set_commit_timestamp(session, ts));
+        break;
+    case WT_TS_TXN_TYPE_DURABLE:
+        WT_RET(__wt_txn_set_durable_timestamp(session, ts));
+        break;
+    case WT_TS_TXN_TYPE_READ:
+        WT_RET(__wt_txn_set_read_timestamp(session, ts));
+        break;
+    case WT_TS_TXN_TYPE_PREPARE:
+        WT_RET(__wt_txn_set_prepare_timestamp(session, ts));
+        break;
+    }
+    __wt_txn_publish_durable_timestamp(session);
+
+    /* Timestamps are only logged in debugging mode. */
+    if (ts != WT_TS_NONE && FLD_ISSET(conn->log_flags, WT_CONN_LOG_DEBUG_MODE) &&
+      FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED) && !F_ISSET(conn, WT_CONN_RECOVERING))
+        WT_RET(__wt_txn_ts_log(session));
+
+    return (0);
+}
+
+/*
  * __wt_txn_publish_durable_timestamp --
  *     Publish a transaction's durable timestamp.
  */
diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp08.py b/src/third_party/wiredtiger/test/suite/test_timestamp08.py
new file mode 100644
index 00000000000..2c4f5d2d549
--- /dev/null
+++ b/src/third_party/wiredtiger/test/suite/test_timestamp08.py
@@ -0,0 +1,236 @@
+#!/usr/bin/env python
+#
+# Public Domain 2014-present MongoDB, Inc.
+# Public Domain 2008-2014 WiredTiger, Inc.
+#
+# This is free and unencumbered software released into the public domain.
+#
+# Anyone is free to copy, modify, publish, use, compile, sell, or
+# distribute this software, either in source code form or as a compiled
+# binary, for any purpose, commercial or non-commercial, and by any
+# means.
+#
+# In jurisdictions that recognize copyright laws, the author or authors
+# of this software dedicate any and all copyright interest in the
+# software to the public domain. We make this dedication for the benefit
+# of the public at large and to the detriment of our heirs and
+# successors. We intend this dedication to be an overt act of
+# relinquishment in perpetuity of all present and future rights to this
+# software under copyright law.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# test_timestamp08.py
+#   Timestamps: API
+#
+
+from suite_subprocess import suite_subprocess
+import wiredtiger, wttest
+
+class test_timestamp08(wttest.WiredTigerTestCase, suite_subprocess):
+    tablename = 'test_timestamp08'
+    uri = 'table:' + tablename
+
+    def test_timestamp_api(self):
+        self.session.create(self.uri, 'key_format=i,value_format=i')
+        c = self.session.open_cursor(self.uri)
+
+        # Begin by adding some data.
+        self.session.begin_transaction()
+        c[1] = 1
+        self.session.commit_transaction(
+            'commit_timestamp=' + self.timestamp_str(1))
+
+        # Can set a zero timestamp. These calls shouldn't raise any errors.
+        self.session.begin_transaction()
+        self.session.timestamp_transaction_uint(wiredtiger.WT_TS_TXN_TYPE_COMMIT, 0)
+        self.session.timestamp_transaction_uint(wiredtiger.WT_TS_TXN_TYPE_READ, 0)
+        self.session.rollback_transaction()
+
+        # In a single transaction it is illegal to set a commit timestamp
+        # older than the first commit timestamp used for this transaction.
+        # Check both timestamp_transaction_uint and commit_transaction APIs.
+        self.session.begin_transaction()
+        self.session.timestamp_transaction_uint(wiredtiger.WT_TS_TXN_TYPE_COMMIT, 3)
+        c[3] = 3
+        self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
+            lambda: self.session.timestamp_transaction_uint(wiredtiger.WT_TS_TXN_TYPE_COMMIT, 2),
+                '/older than the first commit timestamp/')
+        self.session.rollback_transaction()
+
+        # Commit timestamp > Oldest timestamp
+        self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(3))
+
+        self.session.begin_transaction()
+        self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
+            lambda: self.session.timestamp_transaction_uint(wiredtiger.WT_TS_TXN_TYPE_COMMIT, 2),
+                '/less than the oldest timestamp/')
+        self.session.rollback_transaction()
+
+        self.session.begin_transaction()
+        c[4] = 4
+        self.session.commit_transaction(
+            'commit_timestamp=' + self.timestamp_str(4))
+
+        # Commit timestamp > Stable timestamp.
+        # Check both timestamp_transaction and commit_transaction APIs.
+        # Oldest and stable timestamp are set to 5 at the moment.
+        self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(6))
+        self.session.begin_transaction()
+        self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
+            lambda: self.session.timestamp_transaction_uint(wiredtiger.WT_TS_TXN_TYPE_COMMIT, 5),
+                '/less than the stable timestamp/')
+        self.session.rollback_transaction()
+
+        # When explicitly set, commit timestamp for a transaction can be earlier
+        # than the commit timestamp of an earlier transaction.
+        self.session.begin_transaction()
+        c[6] = 6
+        self.session.timestamp_transaction_uint(wiredtiger.WT_TS_TXN_TYPE_COMMIT, 7)
+        self.session.commit_transaction()
+
+        self.session.begin_transaction()
+        c[8] = 8
+        self.session.timestamp_transaction_uint(wiredtiger.WT_TS_TXN_TYPE_COMMIT, 8)
+        self.session.commit_transaction()
+
+        self.session.begin_transaction()
+        c[7] = 7
+        self.session.timestamp_transaction_uint(wiredtiger.WT_TS_TXN_TYPE_COMMIT, 7)
+        self.session.commit_transaction()
+
+        # Read timestamp >= oldest timestamp
+        self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(7) +
+            ',stable_timestamp=' + self.timestamp_str(7))
+        with self.expectedStdoutPattern('less than the oldest timestamp'):
+            self.assertRaisesException(wiredtiger.WiredTigerError, lambda:
+                self.session.begin_transaction('read_timestamp=' + self.timestamp_str(6)))
+
+        # c[8] is not visible at read_timestamp < 8
+        self.session.begin_transaction()
+        self.session.timestamp_transaction_uint(wiredtiger.WT_TS_TXN_TYPE_READ, 7)
+        self.assertEqual(c[6], 6)
+        self.assertEqual(c[7], 7)
+        c.set_key(8)
+        self.assertEqual(c.search(), wiredtiger.WT_NOTFOUND)
+        self.session.commit_transaction()
+
+        self.session.begin_transaction()
+        self.session.timestamp_transaction_uint(wiredtiger.WT_TS_TXN_TYPE_READ, 8)
+        self.assertEqual(c[6], 6)
+        self.assertEqual(c[7], 7)
+        self.assertEqual(c[8], 8)
+        self.assertTimestampsEqual(
+            self.conn.query_timestamp('get=oldest_reader'), self.timestamp_str(8))
+        self.session.commit_transaction()
+
+        # We can move the oldest timestamp backwards with "force"
+        self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(5) + ',force')
+        self.session.begin_transaction()
+        with self.expectedStdoutPattern('less than the oldest timestamp'):
+            self.assertRaisesException(wiredtiger.WiredTigerError, lambda:
+                self.session.timestamp_transaction_uint(wiredtiger.WT_TS_TXN_TYPE_READ, 4))
+        self.session.rollback_transaction()
+
+        self.session.begin_transaction()
+        self.session.timestamp_transaction_uint(wiredtiger.WT_TS_TXN_TYPE_READ, 6)
+        self.assertTimestampsEqual(
+            self.conn.query_timestamp('get=oldest_reader'), self.timestamp_str(6))
+        self.session.commit_transaction()
+
+    def test_all_durable(self):
+        self.session.create(self.uri, 'key_format=i,value_format=i')
+        cur1 = self.session.open_cursor(self.uri)
+
+        # Since this is a non-prepared transaction, we'll be using the commit
+        # timestamp when calculating all_durable since it's implied that they're
+        # the same thing.
+        self.session.begin_transaction()
+        cur1[1] = 1
+        self.session.timestamp_transaction_uint(wiredtiger.WT_TS_TXN_TYPE_COMMIT, 3)
+        self.session.commit_transaction()
+        self.assertTimestampsEqual(
+            self.conn.query_timestamp('get=all_durable'), '3')
+
+        # We have a running transaction with a lower commit_timestamp than we've
+        # seen before. So all_durable should return (lowest commit timestamp - 1).
+        self.session.begin_transaction()
+        cur1[2] = 2
+        self.session.timestamp_transaction_uint(wiredtiger.WT_TS_TXN_TYPE_COMMIT, 2)
+        self.assertTimestampsEqual(
+            self.conn.query_timestamp('get=all_durable'), '1')
+        self.session.commit_transaction()
+
+        # After committing, go back to the value we saw previously.
+        self.assertTimestampsEqual(
+            self.conn.query_timestamp('get=all_durable'), '3')
+
+        # For prepared transactions, we take into account the durable timestamp
+        # when calculating all_durable.
+        self.session.begin_transaction()
+        cur1[3] = 3
+        self.session.timestamp_transaction_uint(wiredtiger.WT_TS_TXN_TYPE_PREPARE, 6)
+        self.session.prepare_transaction()
+
+        # If we have a commit timestamp for a prepared transaction, then we
+        # don't want that to be visible in the all_durable calculation.
+        self.session.timestamp_transaction_uint(wiredtiger.WT_TS_TXN_TYPE_COMMIT, 7)
+        self.assertTimestampsEqual(
+            self.conn.query_timestamp('get=all_durable'), '3')
+
+        # Now take into account the durable timestamp.
+        self.session.timestamp_transaction_uint(wiredtiger.WT_TS_TXN_TYPE_DURABLE, 8)
+        self.session.commit_transaction()
+        self.assertTimestampsEqual(
+            self.conn.query_timestamp('get=all_durable'), '8')
+
+        # All durable moves back when we have a running prepared transaction
+        # with a lower durable timestamp than has previously been committed.
+        self.session.begin_transaction()
+        cur1[4] = 4
+        self.session.timestamp_transaction_uint(wiredtiger.WT_TS_TXN_TYPE_PREPARE, 3)
+        self.session.prepare_transaction()
+
+        # If we have a commit timestamp for a prepared transaction, then we
+        # don't want that to be visible in the all_durable calculation.
+        self.session.timestamp_transaction_uint(wiredtiger.WT_TS_TXN_TYPE_COMMIT, 4)
+        self.assertTimestampsEqual(
+            self.conn.query_timestamp('get=all_durable'), '8')
+
+        # Now take into account the durable timestamp.
+        self.session.timestamp_transaction_uint(wiredtiger.WT_TS_TXN_TYPE_DURABLE, 5)
+        self.assertTimestampsEqual(
+            self.conn.query_timestamp('get=all_durable'), '4')
+        self.session.commit_transaction()
+
+        self.assertTimestampsEqual(
+            self.conn.query_timestamp('get=all_durable'), '8')
+
+        # Now test a scenario with multiple commit timestamps for a single txn.
+        self.session.begin_transaction()
+        cur1[5] = 5
+        self.session.timestamp_transaction_uint(wiredtiger.WT_TS_TXN_TYPE_COMMIT, 6)
+        self.assertTimestampsEqual(
+            self.conn.query_timestamp('get=all_durable'), '5')
+
+        # Make more changes and set a new commit timestamp.
+        # Our calculation should use the first commit timestamp so there should
+        # be no observable difference to the all_durable value.
+        cur1[6] = 6
+        self.session.timestamp_transaction_uint(wiredtiger.WT_TS_TXN_TYPE_COMMIT, 7)
+        self.assertTimestampsEqual(
+            self.conn.query_timestamp('get=all_durable'), '5')
+
+        # Once committed, we go back to 8.
+        self.session.commit_transaction()
+        self.assertTimestampsEqual(
+            self.conn.query_timestamp('get=all_durable'), '8')
+
+if __name__ == '__main__':
+    wttest.run()