Import wiredtiger: 279c544c010e88d67dd20c557cb8244ad8d2deae from branch mongodb-master

ref: 8264e0e17f..279c544c01
for: 6.1.0-rc0

WT-9186       Prevent transactions committing data at a timestamp prior to the stable using the timestamp_transaction API

10 files changed, 163 insertions, 116 deletions

diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index 677aaa5f5aa..abe7f3da597 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-master",
-    "commit": "8264e0e17fa236055672700926544a4cb9ccd138"
+    "commit": "279c544c010e88d67dd20c557cb8244ad8d2deae"
 }
diff --git a/src/third_party/wiredtiger/src/include/txn_inline.h b/src/third_party/wiredtiger/src/include/txn_inline.h
index d9441b7af69..9393875b7d6 100644
--- a/src/third_party/wiredtiger/src/include/txn_inline.h
+++ b/src/third_party/wiredtiger/src/include/txn_inline.h
@@ -1122,6 +1122,8 @@ __wt_txn_begin(WT_SESSION_IMPL *session, const char *cfg[])
     txn = session->txn;
     txn->isolation = session->isolation;
     txn->txn_logsync = S2C(session)->txn_logsync;
+    txn->commit_timestamp = WT_TS_NONE;
+    txn->first_commit_timestamp = WT_TS_NONE;
 
     WT_ASSERT(session, !F_ISSET(txn, WT_TXN_RUNNING));
 
diff --git a/src/third_party/wiredtiger/src/txn/txn_timestamp.c b/src/third_party/wiredtiger/src/txn/txn_timestamp.c
index 0f3141ee55a..f7acb69abf4 100644
--- a/src/third_party/wiredtiger/src/txn/txn_timestamp.c
+++ b/src/third_party/wiredtiger/src/txn/txn_timestamp.c
@@ -509,9 +509,20 @@ __wt_txn_validate_commit_timestamp(WT_SESSION_IMPL *session, wt_timestamp_t *com
         stable_ts = txn_global->stable_timestamp;
 
     if (!F_ISSET(txn, WT_TXN_HAS_TS_PREPARE)) {
+        /* Compare against the first commit timestamp of the current transaction. */
+        if (F_ISSET(txn, WT_TXN_HAS_TS_COMMIT)) {
+            if (commit_ts < txn->first_commit_timestamp)
+                WT_RET_MSG(session, EINVAL,
+                  "commit timestamp %s older than the first commit timestamp %s for this "
+                  "transaction",
+                  __wt_timestamp_to_string(commit_ts, ts_string[0]),
+                  __wt_timestamp_to_string(txn->first_commit_timestamp, ts_string[1]));
+            commit_ts = txn->first_commit_timestamp;
+        }
+
         /*
-         * For a non-prepared transactions the commit timestamp should not be less than the stable
-         * timestamp.
+         * For a non-prepared transactions the commit timestamp should not be less or equal to the
+         * stable timestamp.
          */
         if (has_oldest_ts && commit_ts < oldest_ts)
             WT_RET_MSG(session, EINVAL, "commit timestamp %s is less than the oldest timestamp %s",
@@ -523,16 +534,6 @@ __wt_txn_validate_commit_timestamp(WT_SESSION_IMPL *session, wt_timestamp_t *com
               __wt_timestamp_to_string(commit_ts, ts_string[0]),
               __wt_timestamp_to_string(stable_ts, ts_string[1]));
 
-        /*
-         * Compare against the commit timestamp of the current transaction. Return an error if the
-         * given timestamp is older than the first commit timestamp.
-         */
-        if (F_ISSET(txn, WT_TXN_HAS_TS_COMMIT) && commit_ts < txn->first_commit_timestamp)
-            WT_RET_MSG(session, EINVAL,
-              "commit timestamp %s older than the first commit timestamp %s for this transaction",
-              __wt_timestamp_to_string(commit_ts, ts_string[0]),
-              __wt_timestamp_to_string(txn->first_commit_timestamp, ts_string[1]));
-
         WT_RET(__txn_assert_after_reads(session, "commit", commit_ts));
     } else {
         /*
diff --git a/src/third_party/wiredtiger/test/cppsuite/configs/cache_resize_default.txt b/src/third_party/wiredtiger/test/cppsuite/configs/cache_resize_default.txt
index 83c76666733..ca6829b9cdb 100644
--- a/src/third_party/wiredtiger/test/cppsuite/configs/cache_resize_default.txt
+++ b/src/third_party/wiredtiger/test/cppsuite/configs/cache_resize_default.txt
@@ -7,6 +7,11 @@
 cache_max_wait_ms=1,
 cache_size_mb=500,
 duration_seconds=15,
+timestamp_manager=
+(
+    # We don't need the timestamp manager to update the timestamps in this test.
+    enabled=false,
+),
 workload_generator=
 (
     custom_config=
diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.cpp b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.cpp
index 57eece9c4fe..75f1c338f74 100644
--- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.cpp
+++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.cpp
@@ -148,14 +148,19 @@ transaction_context::try_rollback(const std::string &config)
         rollback(config);
 }
 
-void
+/*
+ * FIXME: WT-9198 We're concurrently doing a transaction that contains a bunch of operations while
+ * moving the stable timestamp. Eat the occasional EINVAL from the transaction's first commit
+ * timestamp being earlier than the stable timestamp.
+ */
+int
 transaction_context::set_commit_timestamp(wt_timestamp_t ts)
 {
     /* We don't want to set zero timestamps on transactions if we're not using timestamps. */
     if (!_timestamp_manager->enabled())
-        return;
+        return 0;
     const std::string config = COMMIT_TS + "=" + timestamp_manager::decimal_to_hex(ts);
-    testutil_check(_session->timestamp_transaction(_session, config.c_str()));
+    return _session->timestamp_transaction(_session, config.c_str());
 }
 
 void
@@ -220,7 +225,12 @@ thread_context::update(
     testutil_assert(cursor.get() != nullptr);
 
     wt_timestamp_t ts = tsm->get_next_ts();
-    transaction.set_commit_timestamp(ts);
+    ret = transaction.set_commit_timestamp(ts);
+    testutil_assert(ret == 0 || ret == EINVAL);
+    if (ret != 0) {
+        transaction.set_needs_rollback(true);
+        return (false);
+    }
 
     cursor->set_key(cursor.get(), key.c_str());
     cursor->set_value(cursor.get(), value.c_str());
@@ -257,7 +267,12 @@ thread_context::insert(
     testutil_assert(cursor.get() != nullptr);
 
     wt_timestamp_t ts = tsm->get_next_ts();
-    transaction.set_commit_timestamp(ts);
+    ret = transaction.set_commit_timestamp(ts);
+    testutil_assert(ret == 0 || ret == EINVAL);
+    if (ret != 0) {
+        transaction.set_needs_rollback(true);
+        return (false);
+    }
 
     cursor->set_key(cursor.get(), key.c_str());
     cursor->set_value(cursor.get(), value.c_str());
@@ -292,7 +307,12 @@ thread_context::remove(scoped_cursor &cursor, uint64_t collection_id, const std:
     testutil_assert(cursor.get() != nullptr);
 
     wt_timestamp_t ts = tsm->get_next_ts();
-    transaction.set_commit_timestamp(ts);
+    ret = transaction.set_commit_timestamp(ts);
+    testutil_assert(ret == 0 || ret == EINVAL);
+    if (ret != 0) {
+        transaction.set_needs_rollback(true);
+        return (false);
+    }
 
     cursor->set_key(cursor.get(), key.c_str());
     ret = cursor->remove(cursor.get());
diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h
index 290bd9d0d5d..e968db51ec9 100644
--- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h
+++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h
@@ -67,7 +67,7 @@ class transaction_context {
     /* Attempt to rollback the transaction given the requirements are met. */
     void try_rollback(const std::string &config = "");
     /* Set a commit timestamp. */
-    void set_commit_timestamp(wt_timestamp_t ts);
+    int set_commit_timestamp(wt_timestamp_t ts);
     /* Set that the transaction needs to be rolled back. */
     void set_needs_rollback(bool rollback);
     /*
diff --git a/src/third_party/wiredtiger/test/csuite/tiered_abort/main.c b/src/third_party/wiredtiger/test/csuite/tiered_abort/main.c
index 74e9f8b5789..76aa0dd7eb2 100644
--- a/src/third_party/wiredtiger/test/csuite/tiered_abort/main.c
+++ b/src/third_party/wiredtiger/test/csuite/tiered_abort/main.c
@@ -85,7 +85,7 @@ static const char *const uri_shadow = "shadow";
 static const char *const sentinel_file = "sentinel_ready";
 
 static bool use_ts;
-static volatile uint64_t global_ts = 1;
+static uint64_t global_ts = 1;
 static uint32_t flush_calls = 1;
 
 /*
@@ -136,8 +136,8 @@ typedef struct {
  * ticket is fixed. Flush_tier should be able to run with ongoing operations.
  */
 static pthread_rwlock_t flush_lock;
-/* Lock for transactional ops that set or query a timestamp. */
-static pthread_rwlock_t ts_lock;
+static uint32_t nth;                      /* Number of threads. */
+static wt_timestamp_t *active_timestamps; /* Oldest timestamps still in use. */
 
 static void handler(int) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn));
 static void usage(void) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn));
@@ -160,35 +160,31 @@ usage(void)
 static WT_THREAD_RET
 thread_ts_run(void *arg)
 {
-    WT_DECL_RET;
+    WT_CONNECTION *conn;
     WT_SESSION *session;
     THREAD_DATA *td;
-    char tscfg[64], ts_string[WT_TS_HEX_STRING_SIZE];
+    wt_timestamp_t last_ts, ts;
+    char tscfg[64];
 
     td = (THREAD_DATA *)arg;
+    conn = td->conn;
 
-    testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session));
-    /* Update the oldest timestamp every 1 millisecond. */
-    for (;; __wt_sleep(0, 1000)) {
-        /*
-         * We get the last committed timestamp periodically in order to update the oldest timestamp,
-         * that requires locking out transactional ops that set or query a timestamp. If there is no
-         * work to do, all-durable will be 0 and we just wait.
-         */
-        testutil_check(pthread_rwlock_wrlock(&ts_lock));
-        ret = td->conn->query_timestamp(td->conn, ts_string, "get=all_durable");
-        testutil_check(pthread_rwlock_unlock(&ts_lock));
-        testutil_assert(ret == 0);
-        if (testutil_timestamp_parse(ts_string) == 0)
+    testutil_check(conn->open_session(conn, NULL, NULL, &session));
+    /* Update the oldest/stable timestamps every 1 millisecond. */
+    for (last_ts = 0;; __wt_sleep(0, 1000)) {
+        /* Get the last committed timestamp periodically in order to update the oldest timestamp. */
+        ts = maximum_stable_ts(active_timestamps, nth);
+        if (ts == last_ts)
             continue;
+        last_ts = ts;
 
         /*
          * Set both the oldest and stable timestamp so that we don't need to maintain read
          * availability at older timestamps.
          */
         testutil_check(__wt_snprintf(
-          tscfg, sizeof(tscfg), "oldest_timestamp=%s,stable_timestamp=%s", ts_string, ts_string));
-        testutil_check(td->conn->set_timestamp(td->conn, tscfg));
+          tscfg, sizeof(tscfg), "oldest_timestamp=%" PRIx64 ",stable_timestamp=%" PRIx64, ts, ts));
+        testutil_check(conn->set_timestamp(conn, tscfg));
     }
     /* NOTREACHED */
 }
@@ -347,15 +343,13 @@ thread_run(void *arg)
         testutil_check(session->begin_transaction(session, NULL));
 
         if (use_ts) {
-            testutil_check(pthread_rwlock_rdlock(&ts_lock));
-            active_ts = __wt_atomic_addv64(&global_ts, 2);
+            active_ts = __wt_atomic_fetch_addv64(&global_ts, 2);
             testutil_check(
               __wt_snprintf(tscfg, sizeof(tscfg), "commit_timestamp=%" PRIx64, active_ts));
             /*
              * Set the transaction's timestamp now before performing the operation.
              */
             testutil_check(session->timestamp_transaction(session, tscfg));
-            testutil_check(pthread_rwlock_unlock(&ts_lock));
         }
 
         cur_coll->set_key(cur_coll, kname);
@@ -422,11 +416,15 @@ rollback:
                 locked = false;
             }
         }
+
+        /* We're done with the timestamps, allow oldest and stable to move forward. */
+        if (use_ts)
+            WT_PUBLISH(active_timestamps[td->info], active_ts);
     }
     /* NOTREACHED */
 }
 
-static void run_workload(uint32_t, const char *) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn));
+static void run_workload(const char *) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn));
 
 /*
  * run_workload --
@@ -434,7 +432,7 @@ static void run_workload(uint32_t, const char *) WT_GCC_FUNC_DECL_ATTRIBUTE((nor
  *     until it is killed by the parent.
  */
 static void
-run_workload(uint32_t nth, const char *build_dir)
+run_workload(const char *build_dir)
 {
     WT_CONNECTION *conn;
     WT_SESSION *session;
@@ -445,6 +443,7 @@ run_workload(uint32_t nth, const char *build_dir)
 
     thr = dcalloc(nth + NUM_INT_THREADS, sizeof(*thr));
     td = dcalloc(nth + NUM_INT_THREADS, sizeof(THREAD_DATA));
+    active_timestamps = dcalloc(nth, sizeof(wt_timestamp_t));
 
     /*
      * Size the cache appropriately for the number of threads. Each thread adds keys sequentially to
@@ -660,7 +659,7 @@ main(int argc, char *argv[])
     pid_t pid;
     uint64_t absent_coll, absent_local, absent_oplog, absent_shadow, count, key, last_key;
     uint64_t commit_fp, durable_fp, stable_val;
-    uint32_t i, nth, timeout;
+    uint32_t i, timeout;
     int ch, status, ret;
     const char *working_dir;
     char buf[512], bucket_dir[512], build_dir[512], fname[512], kname[64];
@@ -728,7 +727,6 @@ main(int argc, char *argv[])
     testutil_build_dir(opts, build_dir, 512);
 
     testutil_check(pthread_rwlock_init(&flush_lock, NULL));
-    testutil_check(pthread_rwlock_init(&ts_lock, NULL));
 
     testutil_work_dir_from_path(home, sizeof(home), working_dir);
     /*
@@ -771,7 +769,7 @@ main(int argc, char *argv[])
         testutil_assert_errno((pid = fork()) >= 0);
 
         if (pid == 0) { /* child */
-            run_workload(nth, build_dir);
+            run_workload(build_dir);
             /* NOTREACHED */
         }
 
@@ -1010,7 +1008,6 @@ main(int argc, char *argv[])
         fatal = true;
     }
     testutil_check(pthread_rwlock_destroy(&flush_lock));
-    testutil_check(pthread_rwlock_destroy(&ts_lock));
     if (fatal)
         return (EXIT_FAILURE);
     printf("%" PRIu64 " records verified\n", count);
diff --git a/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c b/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c
index 7ba89f2d72a..a20b1d23d81 100644
--- a/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c
+++ b/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c
@@ -80,7 +80,7 @@ static const char *const uri_shadow = "shadow";
 static const char *const ckpt_file = "checkpoint_done";
 
 static bool columns, compat, inmem, stress, use_ts;
-static volatile uint64_t global_ts = 1;
+static uint64_t global_ts = 1;
 
 /*
  * The configuration sets the eviction update and dirty targets at 20% so that on average, each
@@ -126,8 +126,8 @@ typedef struct {
     uint32_t info;
 } THREAD_DATA;
 
-/* Lock for transactional ops that set or query a timestamp. */
-static pthread_rwlock_t ts_lock;
+static uint32_t nth;                      /* Number of threads. */
+static wt_timestamp_t *active_timestamps; /* Oldest timestamps still in use. */
 
 static void handler(int) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn));
 static void usage(void) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn));
@@ -150,63 +150,38 @@ usage(void)
 static WT_THREAD_RET
 thread_ts_run(void *arg)
 {
-    WT_DECL_RET;
+    WT_CONNECTION *conn;
     WT_RAND_STATE rnd;
     WT_SESSION *session;
     THREAD_DATA *td;
-    wt_timestamp_t all_dur_ts, prev_all_dur_ts;
+    wt_timestamp_t last_ts, ts;
     uint32_t rand_op;
     int dbg;
-    char tscfg[64], ts_string[WT_TS_HEX_STRING_SIZE];
-    bool first;
-
-    prev_all_dur_ts = WT_TS_NONE;
+    char tscfg[64];
 
     td = (THREAD_DATA *)arg;
-    __wt_random_init(&rnd);
+    conn = td->conn;
 
-    testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session));
-    first = true;
-    /* Update the oldest timestamp every 1 millisecond. */
-    for (;; __wt_sleep(0, 1000)) {
-        /*
-         * We get the last committed timestamp periodically in order to update the oldest timestamp,
-         * that requires locking out transactional ops that set or query a timestamp. If there is no
-         * work to do, all-durable will be 0 and we just wait.
-         */
-        testutil_check(pthread_rwlock_wrlock(&ts_lock));
-        ret = td->conn->query_timestamp(td->conn, ts_string, "get=all_durable");
-        testutil_check(pthread_rwlock_unlock(&ts_lock));
-        testutil_assert(ret == 0);
-        /*
-         * All durable can intermittently move backwards, we do not want to set stable and the
-         * oldest timestamps backwards.
-         */
-        all_dur_ts = testutil_timestamp_parse(ts_string);
-        if (all_dur_ts == 0 || all_dur_ts < prev_all_dur_ts)
+    testutil_check(conn->open_session(conn, NULL, NULL, &session));
+    __wt_random_init_seed((WT_SESSION_IMPL *)session, &rnd);
+
+    /* Update the oldest/stable timestamps every 1 millisecond. */
+    for (last_ts = 0;; __wt_sleep(0, 1000)) {
+        /* Get the last committed timestamp periodically in order to update the oldest timestamp. */
+        ts = maximum_stable_ts(active_timestamps, nth);
+        if (ts == last_ts)
             continue;
-        prev_all_dur_ts = all_dur_ts;
+        last_ts = ts;
 
+        /* Let the oldest timestamp lag 25% of the time. */
         rand_op = __wt_random(&rnd) % 4;
-        /*
-         * Periodically let the oldest timestamp lag. Other times set the stable and oldest
-         * timestamps as separate API calls. The rest of the time set them both as one call.
-         */
-        if (rand_op == 0) {
-            testutil_check(__wt_snprintf(tscfg, sizeof(tscfg), "stable_timestamp=%s", ts_string));
-            testutil_check(td->conn->set_timestamp(td->conn, tscfg));
-            testutil_check(__wt_snprintf(tscfg, sizeof(tscfg), "oldest_timestamp=%s", ts_string));
-            testutil_check(td->conn->set_timestamp(td->conn, tscfg));
-        } else {
-            if (!first && rand_op == 1)
-                testutil_check(
-                  __wt_snprintf(tscfg, sizeof(tscfg), "stable_timestamp=%s", ts_string));
-            else
-                testutil_check(__wt_snprintf(tscfg, sizeof(tscfg),
-                  "oldest_timestamp=%s,stable_timestamp=%s", ts_string, ts_string));
-            testutil_check(td->conn->set_timestamp(td->conn, tscfg));
-        }
-        first = false;
+        if (rand_op == 1)
+            testutil_check(__wt_snprintf(tscfg, sizeof(tscfg), "stable_timestamp=%" PRIx64, ts));
+        else
+            testutil_check(__wt_snprintf(tscfg, sizeof(tscfg),
+              "oldest_timestamp=%" PRIx64 ",stable_timestamp=%" PRIx64, ts, ts));
+        testutil_check(conn->set_timestamp(conn, tscfg));
+
         /*
          * Set and reset the checkpoint retention setting on a regular basis. We want to test racing
          * with the internal log removal thread while we're here.
@@ -218,7 +193,7 @@ thread_ts_run(void *arg)
         else
             testutil_check(
               __wt_snprintf(tscfg, sizeof(tscfg), "debug_mode=(checkpoint_retention=5)"));
-        testutil_check(td->conn->reconfigure(td->conn, tscfg));
+        testutil_check(conn->reconfigure(conn, tscfg));
     }
     /* NOTREACHED */
 }
@@ -367,8 +342,8 @@ thread_run(void *arg)
             testutil_check(prepared_session->begin_transaction(prepared_session, NULL));
 
         if (use_ts) {
-            testutil_check(pthread_rwlock_rdlock(&ts_lock));
-            active_ts = __wt_atomic_addv64(&global_ts, 2);
+            /* Allocate two timestamps. */
+            active_ts = __wt_atomic_fetch_addv64(&global_ts, 2);
             testutil_check(
               __wt_snprintf(tscfg, sizeof(tscfg), "commit_timestamp=%" PRIx64, active_ts));
             /*
@@ -377,7 +352,6 @@ thread_run(void *arg)
              * collection session in that case would continue to use this timestamp.
              */
             testutil_check(session->timestamp_transaction(session, tscfg));
-            testutil_check(pthread_rwlock_unlock(&ts_lock));
         }
 
         if (columns) {
@@ -475,11 +449,15 @@ rollback:
             if (use_prep)
                 testutil_check(prepared_session->rollback_transaction(prepared_session, NULL));
         }
+
+        /* We're done with the timestamps, allow oldest and stable to move forward. */
+        if (use_ts)
+            WT_PUBLISH(active_timestamps[td->info], active_ts);
     }
     /* NOTREACHED */
 }
 
-static void run_workload(uint32_t) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn));
+static void run_workload(void) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn));
 
 /*
  * run_workload --
@@ -487,7 +465,7 @@ static void run_workload(uint32_t) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn));
  *     until it is killed by the parent.
  */
 static void
-run_workload(uint32_t nth)
+run_workload(void)
 {
     WT_CONNECTION *conn;
     WT_SESSION *session;
@@ -499,6 +477,7 @@ run_workload(uint32_t nth)
 
     thr = dcalloc(nth + 2, sizeof(*thr));
     td = dcalloc(nth + 2, sizeof(THREAD_DATA));
+    active_timestamps = dcalloc(nth, sizeof(wt_timestamp_t));
 
     /*
      * Size the cache appropriately for the number of threads. Each thread adds keys sequentially to
@@ -557,9 +536,7 @@ run_workload(uint32_t nth)
      */
     testutil_check(session->close(session, NULL));
 
-    /*
-     * The checkpoint thread and the timestamp threads are added at the end.
-     */
+    /* The checkpoint, timestamp and worker threads are added at the end. */
     ckpt_id = nth;
     td[ckpt_id].conn = conn;
     td[ckpt_id].info = nth;
@@ -657,7 +634,7 @@ main(int argc, char *argv[])
     pid_t pid;
     uint64_t absent_coll, absent_local, absent_oplog, absent_shadow, count, key, last_key;
     uint64_t commit_fp, durable_fp, stable_val;
-    uint32_t i, nth, timeout;
+    uint32_t i, timeout;
     int ch, status, ret;
     const char *working_dir;
     char buf[512], fname[64], kname[64], statname[1024];
@@ -726,7 +703,6 @@ main(int argc, char *argv[])
         usage();
 
     testutil_work_dir_from_path(home, sizeof(home), working_dir);
-    testutil_check(pthread_rwlock_init(&ts_lock, NULL));
 
     /*
      * If the user wants to verify they need to tell us how many threads there were so we can find
@@ -770,7 +746,7 @@ main(int argc, char *argv[])
         testutil_assert_errno((pid = fork()) >= 0);
 
         if (pid == 0) { /* child */
-            run_workload(nth);
+            run_workload();
             /* NOTREACHED */
         }
 
@@ -1015,7 +991,6 @@ main(int argc, char *argv[])
         printf("OPLOG: %" PRIu64 " record(s) absent from %" PRIu64 "\n", absent_oplog, count);
         fatal = true;
     }
-    testutil_check(pthread_rwlock_destroy(&ts_lock));
     if (fatal)
         return (EXIT_FAILURE);
     printf("%" PRIu64 " records verified\n", count);
diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp28.py b/src/third_party/wiredtiger/test/suite/test_timestamp28.py
index 259499d7182..74086a1b8bd 100644
--- a/src/third_party/wiredtiger/test/suite/test_timestamp28.py
+++ b/src/third_party/wiredtiger/test/suite/test_timestamp28.py
@@ -31,29 +31,54 @@
 
 import wiredtiger, wttest
 from wtdataset import SimpleDataSet
+from wtscenario import make_scenarios
 
 # Timestamps: smoke test that commit is tested at both commit and set time.
 class test_timestamp28(wttest.WiredTigerTestCase):
+    timestamps = [
+        ('stable', dict(timestamp='stable_timestamp')),
+        ('oldest', dict(timestamp='oldest_timestamp')),
+    ]
+    error_logs = {
+        'stable_timestamp': '/must be after/',
+        'oldest_timestamp': '/is less than/',
+    }
+    scenarios = make_scenarios(timestamps)
+
     def test_timestamp28(self):
+
         uri = 'table:timestamp28'
         ds = SimpleDataSet(self, uri, 50, key_format='i', value_format='S')
         ds.populate()
         c = self.session.open_cursor(uri)
 
-        self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(30))
+        self.conn.set_timestamp(self.timestamp + '=' + self.timestamp_str(30))
         self.session.begin_transaction()
         c[5] = 'xxx'
+
         self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
             lambda: self.session.commit_transaction(
-            'commit_timestamp=' + self.timestamp_str(20)), '/must be after/')
+            'commit_timestamp=' + self.timestamp_str(20)), self.error_logs[self.timestamp])
 
-        self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40))
+        self.conn.set_timestamp(self.timestamp + '=' + self.timestamp_str(40))
         self.session.begin_transaction()
         c[5] = 'xxx'
         self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(50))
-        self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(60))
+        self.conn.set_timestamp(self.timestamp + '=' + self.timestamp_str(60))
+        self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
+            lambda: self.session.commit_transaction(), self.error_logs[self.timestamp])
+
+        # Confirm the earliest commit time is tested.
+        self.session.begin_transaction()
+        c[5] = 'xxx'
+        self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(70))
+        c[6] = 'xxx'
+        self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(71))
+        c[7] = 'xxx'
+        self.conn.set_timestamp(self.timestamp + "=" + self.timestamp_str(75))
         self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
-            lambda: self.session.commit_transaction(), '/must be after/')
+            lambda: self.session.commit_transaction(
+            'commit_timestamp=' + self.timestamp_str(80)), self.error_logs[self.timestamp])
 
 if __name__ == '__main__':
     wttest.run()
diff --git a/src/third_party/wiredtiger/test/utility/test_util.h b/src/third_party/wiredtiger/test/utility/test_util.h
index 7c45587358a..1ae875865ce 100644
--- a/src/third_party/wiredtiger/test/utility/test_util.h
+++ b/src/third_party/wiredtiger/test/utility/test_util.h
@@ -310,6 +310,28 @@ testutil_timestamp_parse(const char *str)
     return (ts);
 }
 
+/*
+ * maximum_stable_ts --
+ *     Return the largest usable stable timestamp from a list of n committed timestamps.
+ */
+static inline wt_timestamp_t
+maximum_stable_ts(wt_timestamp_t *commit_timestamps, uint32_t n)
+{
+    wt_timestamp_t commit_ts, ts;
+    uint32_t i;
+
+    for (ts = WT_TS_MAX, i = 0; i < n; i++) {
+        commit_ts = commit_timestamps[i];
+        if (commit_ts == WT_TS_NONE)
+            return (WT_TS_NONE);
+        if (commit_ts < ts)
+            ts = commit_ts;
+    }
+
+    /* Return one less than the earliest in-use timestamp. */
+    return (ts == WT_TS_MAX ? WT_TS_NONE : ts - 1);
+}
+
 /* Allow tests to add their own death handling. */
 extern void (*custom_die)(void);