Import wiredtiger: 58aba8adeaf57b85bdd869d55a519286628ecff8 from branch mongodb-master

ref: 06f6747bcb..58aba8adea
for: 5.3.0

WT-8543       Rollback-to-stable should detect a prepared transaction as an active transaction and fail

7 files changed, 146 insertions, 58 deletions

diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index a520ce24c7e..503cff44997 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-master",
-    "commit": "06f6747bcb94bc81936dd97f763483464247ec97"
+    "commit": "58aba8adeaf57b85bdd869d55a519286628ecff8"
 }
diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h
index 4cabe2ffacc..98e79f02fff 100644
--- a/src/third_party/wiredtiger/src/include/extern.h
+++ b/src/third_party/wiredtiger/src/include/extern.h
@@ -41,8 +41,6 @@ extern bool __wt_rwlock_islocked(WT_SESSION_IMPL *session, WT_RWLOCK *l)
   WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern bool __wt_txn_active(WT_SESSION_IMPL *session, uint64_t txnid)
   WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
-extern bool __wt_txn_user_active(WT_SESSION_IMPL *session)
-  WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern char *__wt_time_aggregate_to_string(WT_TIME_AGGREGATE *ta, char *ta_string)
   WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern char *__wt_time_point_to_string(wt_timestamp_t ts, wt_timestamp_t durable_ts,
diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c
index 74e2cdab612..0406ce7d1ce 100644
--- a/src/third_party/wiredtiger/src/session/session_api.c
+++ b/src/third_party/wiredtiger/src/session/session_api.c
@@ -321,9 +321,6 @@ __wt_session_close_internal(WT_SESSION_IMPL *session)
     /* Discard metadata tracking. */
     __wt_meta_track_discard(session);
 
-    /* Free transaction information. */
-    __wt_txn_destroy(session);
-
     /*
      * Close the file where we tracked long operations. Do this before releasing resources, as we do
      * scratch buffer management when we flush optrack buffers to disk.
@@ -344,6 +341,12 @@ __wt_session_close_internal(WT_SESSION_IMPL *session)
     /* The API lock protects opening and closing of sessions. */
     __wt_spin_lock(session, &conn->api_lock);
 
+    /*
+     * Free transaction information: inside the lock because we're freeing the WT_TXN structure and
+     * RTS looks at it.
+     */
+    __wt_txn_destroy(session);
+
     /* Decrement the count of open sessions. */
     WT_STAT_CONN_DECR(session, session_open);
 
diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c
index 5f136e22783..934b607b64a 100644
--- a/src/third_party/wiredtiger/src/txn/txn.c
+++ b/src/third_party/wiredtiger/src/txn/txn.c
@@ -139,48 +139,6 @@ __wt_txn_release_snapshot(WT_SESSION_IMPL *session)
 }
 
 /*
- * __wt_txn_user_active --
- *     Check whether there are any running user transactions. Note that a new transaction may start
- *     after we return from this call and therefore caller should be aware of this limitation.
- */
-bool
-__wt_txn_user_active(WT_SESSION_IMPL *session)
-{
-    WT_CONNECTION_IMPL *conn;
-    WT_SESSION_IMPL *session_in_list;
-    WT_TXN_GLOBAL *txn_global;
-    WT_TXN_SHARED *txn_shared;
-    uint32_t i, session_cnt;
-    bool txn_active;
-
-    conn = S2C(session);
-    txn_active = false;
-    txn_global = &conn->txn_global;
-
-    /* We're going to scan the table: wait for the lock. */
-    __wt_writelock(session, &txn_global->rwlock);
-
-    WT_ORDERED_READ(session_cnt, conn->session_cnt);
-    WT_STAT_CONN_INCR(session, txn_walk_sessions);
-    for (i = 0, session_in_list = conn->sessions, txn_shared = txn_global->txn_shared_list;
-         i < session_cnt; i++, session_in_list++, txn_shared++) {
-
-        /* Skip inactive or internal sessions. */
-        if (!session_in_list->active || F_ISSET(session_in_list, WT_SESSION_INTERNAL))
-            continue;
-
-        /* Check if a user session has a running transaction. */
-        if (txn_shared->id != WT_TXN_NONE || txn_shared->pinned_id != WT_TXN_NONE) {
-            txn_active = true;
-            break;
-        }
-    }
-
-    __wt_writeunlock(session, &txn_global->rwlock);
-    return (txn_active);
-}
-
-/*
  * __wt_txn_active --
  *     Check if a transaction is still active. If not, it is either committed, prepared, or rolled
  *     back. It is possible that we race with commit, prepare or rollback and a transaction is still
diff --git a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c
index f35fd4fe2e5..818eaf079c5 100644
--- a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c
+++ b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c
@@ -1319,6 +1319,52 @@ __rollback_to_stable_btree(WT_SESSION_IMPL *session, wt_timestamp_t rollback_tim
 }
 
 /*
+ * __txn_user_active --
+ *     Return if there are any running user transactions.
+ */
+static bool
+__txn_user_active(WT_SESSION_IMPL *session)
+{
+    WT_CONNECTION_IMPL *conn;
+    WT_SESSION_IMPL *session_in_list;
+    uint32_t i, session_cnt;
+    bool txn_active;
+
+    conn = S2C(session);
+    txn_active = false;
+
+    WT_STAT_CONN_INCR(session, txn_walk_sessions);
+
+    /*
+     * WT_TXN structures are allocated and freed as sessions are activated and closed. Lock the
+     * session open/close to ensure we don't race. This call is a rarely used RTS-only function,
+     * acquiring the lock shouldn't be an issue.
+     */
+    __wt_spin_lock(session, &conn->api_lock);
+
+    WT_ORDERED_READ(session_cnt, conn->session_cnt);
+    for (i = 0, session_in_list = conn->sessions; i < session_cnt; i++, session_in_list++) {
+
+        /* Skip inactive or internal sessions. */
+        if (!session_in_list->active || F_ISSET(session_in_list, WT_SESSION_INTERNAL))
+            continue;
+
+        /* Check if a user session has a running transaction. */
+        if (F_ISSET(session_in_list->txn, WT_TXN_RUNNING)) {
+            txn_active = true;
+            break;
+        }
+    }
+    __wt_spin_unlock(session, &conn->api_lock);
+
+    /*
+     * A new transaction may start after we return from this call and callers should be aware of
+     * this limitation.
+     */
+    return (txn_active);
+}
+
+/*
  * __rollback_to_stable_check --
  *     Ensure the rollback request is reasonable.
  */
@@ -1332,7 +1378,7 @@ __rollback_to_stable_check(WT_SESSION_IMPL *session)
      * Help the user comply with the requirement that there are no concurrent user operations. It is
      * okay to have a transaction in prepared state.
      */
-    txn_active = __wt_txn_user_active(session);
+    txn_active = __txn_user_active(session);
 #ifdef HAVE_DIAGNOSTIC
     if (txn_active)
         WT_TRET(__wt_verbose_dump_txn(session));
diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable19.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable19.py
index cbf7c839433..b88c4720426 100644
--- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable19.py
+++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable19.py
@@ -134,7 +134,6 @@ class test_rollback_to_stable19(test_rollback_to_stable_base):
                 # Close and reopen the connection
                 self.reopen_conn()
         else:
-            self.conn.rollback_to_stable()
             s.rollback_transaction()
 
         # Verify data is not visible.
@@ -148,7 +147,7 @@ class test_rollback_to_stable19(test_rollback_to_stable_base):
         # After restart (not crash) the stats for the aborted updates will be 0, as the updates
         # will be aborted during shutdown, and on startup there will be no updates to be aborted.
         # This is similar case with keys removed.
-        if not self.in_memory and not self.crash:
+        if self.in_memory or not self.crash:
             self.assertEqual(upd_aborted, 0)
             self.assertEqual(keys_removed, 0)
         else:
@@ -235,7 +234,6 @@ class test_rollback_to_stable19(test_rollback_to_stable_base):
                 # Close and reopen the connection
                 self.reopen_conn()
         else:
-            self.conn.rollback_to_stable()
             s.rollback_transaction()
 
         # Verify data.
@@ -250,11 +248,12 @@ class test_rollback_to_stable19(test_rollback_to_stable_base):
         # After restart (not crash) the stats for the aborted updates and history store removed will be 0,
         # as the updates aborted and history store removed will occur during shutdown, and on startup there
         # will be no updates to be removed.
-        if not self.in_memory:
-            if self.crash:
-                self.assertGreater(hs_removed, 0)
-            else:
-                self.assertEqual(hs_removed, 0)
-                self.assertEqual(upd_aborted, 0)
+        if self.in_memory or not self.crash:
+            self.assertEqual(hs_removed, 0)
+            self.assertEqual(upd_aborted, 0)
         else:
+            self.assertGreater(hs_removed, 0)
             self.assertGreater(upd_aborted, 0)
+
+if __name__ == '__main__':
+    wttest.run()
diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable30.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable30.py
new file mode 100644
index 00000000000..59379c00408
--- /dev/null
+++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable30.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+#
+# Public Domain 2014-present MongoDB, Inc.
+# Public Domain 2008-2014 WiredTiger, Inc.
+#
+# This is free and unencumbered software released into the public domain.
+#
+# Anyone is free to copy, modify, publish, use, compile, sell, or
+# distribute this software, either in source code form or as a compiled
+# binary, for any purpose, commercial or non-commercial, and by any
+# means.
+#
+# In jurisdictions that recognize copyright laws, the author or authors
+# of this software dedicate any and all copyright interest in the
+# software to the public domain. We make this dedication for the benefit
+# of the public at large and to the detriment of our heirs and
+# successors. We intend this dedication to be an overt act of
+# relinquishment in perpetuity of all present and future rights to this
+# software under copyright law.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+
+import wiredtiger, wttest
+from wtdataset import SimpleDataSet, ComplexDataSet
+from wtscenario import make_scenarios
+
+# test_rollback_to_stable30.py
+# Test RTS fails with active transactions and the subsequent transaction resolution succeeds.
+class test_rollback_to_stable30(wttest.WiredTigerTestCase):
+    scenarios = make_scenarios([
+        ('table-f', dict(keyfmt='r', valfmt='8t', dataset=SimpleDataSet)),
+        ('table-r', dict(keyfmt='r', valfmt='S', dataset=SimpleDataSet)),
+        ('table-S', dict(keyfmt='S', valfmt='S', dataset=SimpleDataSet)),
+        ('table-r-complex', dict(keyfmt='r', valfmt=None, dataset=ComplexDataSet)),
+        ('table-S-complex', dict(keyfmt='S', valfmt=None, dataset=ComplexDataSet)),
+    ])
+
+    def prepare_resolve(self, resolve):
+        ds = self.dataset(self, "table:rts30", 10, key_format=self.keyfmt, value_format=self.valfmt)
+        ds.populate()
+
+        # Pin oldest and stable timestamps to 1.
+        self.conn.set_timestamp(
+            'oldest_timestamp=' + self.timestamp_str(1) +
+            ',stable_timestamp=' + self.timestamp_str(1))
+
+        # Write some data and prepare it.
+        cursor = self.session.open_cursor(ds.uri)
+        self.session.begin_transaction()
+
+        # Modify a row.
+        cursor[ds.key(5)] = ds.value(20)
+
+        # Prepare at timestamp 10.
+        self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(10))
+
+        # Roll back to stable should fail because there's an active transaction.
+        with self.expectedStdoutPattern('transaction state dump'):
+            msg = '/rollback_to_stable.*active/'
+            self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
+                lambda:self.conn.rollback_to_stable(), msg)
+
+        # Set commit, durable timestamps to 20.
+        self.session.timestamp_transaction(
+            'commit_timestamp=' + self.timestamp_str(20) +
+            ',durable_timestamp=' + self.timestamp_str(20))
+
+        # Commit or abort the prepared transaction.
+        resolve()
+
+    def test_rts_prepare_commit(self):
+        self.prepare_resolve(self.session.commit_transaction)
+
+    def test_rts_prepare_rollback(self):
+        self.prepare_resolve(self.session.rollback_transaction)
+
+if __name__ == '__main__':
+    wttest.run()