Import wiredtiger: b724e3ec5cd6e27bcfaaa53f6308519bda56c7e6 from branch mongodb-5.0

ref: c2f8ac069a..b724e3ec5c
for: 5.1.0

WT-7668       Overload the update method for the hs_cleanup test

10 files changed, 196 insertions, 71 deletions

diff --git a/src/third_party/wiredtiger/dist/test_data.py b/src/third_party/wiredtiger/dist/test_data.py
index b179ec92cc6..8194fd66bf3 100644
--- a/src/third_party/wiredtiger/dist/test_data.py
+++ b/src/third_party/wiredtiger/dist/test_data.py
@@ -112,8 +112,8 @@ transaction_config = [
 ]
 
 thread_count = [
-    Config('thread_count', 1, r'''
-        Specifies the number of threads that will be used to perform a certain function.''')
+    Config('thread_count', 0, r'''
+        Specifies the number of threads that will be used to perform a certain function.''', min=0)
 ]
 
 read_thread_config = thread_count + throttle_config + transaction_config
diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index b748fcaeebf..78611070478 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-5.0",
-    "commit": "c2f8ac069a38c5cbeb127560fb01a3e23548a912"
+    "commit": "b724e3ec5cd6e27bcfaaa53f6308519bda56c7e6"
 }
diff --git a/src/third_party/wiredtiger/src/config/test_config.c b/src/third_party/wiredtiger/src/config/test_config.c
index c1142a652e3..5811d5993e8 100644
--- a/src/third_party/wiredtiger/src/config/test_config.c
+++ b/src/third_party/wiredtiger/src/config/test_config.c
@@ -37,7 +37,7 @@ static const WT_CONFIG_CHECK confchk_ops_per_transaction_subconfigs[] = {
 static const WT_CONFIG_CHECK confchk_insert_config_subconfigs[] = {
   {"key_size", "int", NULL, "min=0,max=10000", NULL, 0}, {"op_rate", "string", NULL, NULL, NULL, 0},
   {"ops_per_transaction", "category", NULL, NULL, confchk_ops_per_transaction_subconfigs, 2},
-  {"thread_count", "string", NULL, NULL, NULL, 0},
+  {"thread_count", "int", NULL, "min=0", NULL, 0},
   {"value_size", "int", NULL, "min=0,max=1000000000", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}};
 
 static const WT_CONFIG_CHECK confchk_populate_config_subconfigs[] = {
@@ -50,12 +50,12 @@ static const WT_CONFIG_CHECK confchk_populate_config_subconfigs[] = {
 static const WT_CONFIG_CHECK confchk_read_config_subconfigs[] = {
   {"op_rate", "string", NULL, NULL, NULL, 0},
   {"ops_per_transaction", "category", NULL, NULL, confchk_ops_per_transaction_subconfigs, 2},
-  {"thread_count", "string", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}};
+  {"thread_count", "int", NULL, "min=0", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}};
 
 static const WT_CONFIG_CHECK confchk_update_config_subconfigs[] = {
   {"key_size", "int", NULL, "min=0,max=10000", NULL, 0}, {"op_rate", "string", NULL, NULL, NULL, 0},
   {"ops_per_transaction", "category", NULL, NULL, confchk_ops_per_transaction_subconfigs, 2},
-  {"thread_count", "string", NULL, NULL, NULL, 0},
+  {"thread_count", "int", NULL, "min=0", NULL, 0},
   {"value_size", "int", NULL, "min=0,max=1000000000", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}};
 
 static const WT_CONFIG_CHECK confchk_workload_generator_subconfigs[] = {
@@ -118,12 +118,12 @@ static const WT_CONFIG_ENTRY config_entries[] = {
     "timestamp_manager=(enabled=true,oldest_lag=1,op_rate=1s,"
     "stable_lag=1),workload_generator=(enabled=true,"
     "insert_config=(key_size=5,op_rate=1s,ops_per_transaction=(max=1,"
-    "min=0),thread_count=1,value_size=5),op_rate=1s,"
+    "min=0),thread_count=0,value_size=5),op_rate=1s,"
     "populate_config=(collection_count=1,key_count_per_collection=0,"
     "key_size=5,thread_count=1,value_size=5),read_config=(op_rate=1s,"
-    "ops_per_transaction=(max=1,min=0),thread_count=1),"
+    "ops_per_transaction=(max=1,min=0),thread_count=0),"
     "update_config=(key_size=5,op_rate=1s,ops_per_transaction=(max=1,"
-    "min=0),thread_count=1,value_size=5)),"
+    "min=0),thread_count=0,value_size=5)),"
     "workload_tracking=(enabled=true,op_rate=1s)",
     confchk_example_test, 9},
   {"hs_cleanup",
@@ -136,12 +136,12 @@ static const WT_CONFIG_ENTRY config_entries[] = {
     "timestamp_manager=(enabled=true,oldest_lag=1,op_rate=1s,"
     "stable_lag=1),workload_generator=(enabled=true,"
     "insert_config=(key_size=5,op_rate=1s,ops_per_transaction=(max=1,"
-    "min=0),thread_count=1,value_size=5),op_rate=1s,"
+    "min=0),thread_count=0,value_size=5),op_rate=1s,"
     "populate_config=(collection_count=1,key_count_per_collection=0,"
     "key_size=5,thread_count=1,value_size=5),read_config=(op_rate=1s,"
-    "ops_per_transaction=(max=1,min=0),thread_count=1),"
+    "ops_per_transaction=(max=1,min=0),thread_count=0),"
     "update_config=(key_size=5,op_rate=1s,ops_per_transaction=(max=1,"
-    "min=0),thread_count=1,value_size=5)),"
+    "min=0),thread_count=0,value_size=5)),"
     "workload_tracking=(enabled=true,op_rate=1s)",
     confchk_hs_cleanup, 9},
   {"poc_test",
@@ -154,12 +154,12 @@ static const WT_CONFIG_ENTRY config_entries[] = {
     "timestamp_manager=(enabled=true,oldest_lag=1,op_rate=1s,"
     "stable_lag=1),workload_generator=(enabled=true,"
     "insert_config=(key_size=5,op_rate=1s,ops_per_transaction=(max=1,"
-    "min=0),thread_count=1,value_size=5),op_rate=1s,"
+    "min=0),thread_count=0,value_size=5),op_rate=1s,"
     "populate_config=(collection_count=1,key_count_per_collection=0,"
     "key_size=5,thread_count=1,value_size=5),read_config=(op_rate=1s,"
-    "ops_per_transaction=(max=1,min=0),thread_count=1),"
+    "ops_per_transaction=(max=1,min=0),thread_count=0),"
     "update_config=(key_size=5,op_rate=1s,ops_per_transaction=(max=1,"
-    "min=0),thread_count=1,value_size=5)),"
+    "min=0),thread_count=0,value_size=5)),"
     "workload_tracking=(enabled=true,op_rate=1s)",
     confchk_poc_test, 9},
   {NULL, NULL, NULL, 0}};
diff --git a/src/third_party/wiredtiger/test/cppsuite/configs/config_hs_cleanup_default.txt b/src/third_party/wiredtiger/test/cppsuite/configs/config_hs_cleanup_default.txt
index f46526e554e..626cf9a2fa3 100644
--- a/src/third_party/wiredtiger/test/cppsuite/configs/config_hs_cleanup_default.txt
+++ b/src/third_party/wiredtiger/test/cppsuite/configs/config_hs_cleanup_default.txt
@@ -1,4 +1,59 @@
 # Configuration for hs_cleanup.
 # need to be defined.
-duration_seconds=5,
-cache_size_mb=250
+duration_seconds=240,
+cache_size_mb=400,
+statistics_config=
+(
+    type=all,
+    enable_logging=true
+),
+checkpoint_manager=
+(
+    enabled=true,
+    op_rate=20s
+),
+runtime_monitor=
+(
+    stat_cache_size=
+    (
+        enabled=true,
+        limit=100
+    )
+),
+timestamp_manager=
+(
+    enabled=true,
+    oldest_lag=5,
+    op_rate=1s,
+    stable_lag=5
+),
+workload_generator=
+(
+    populate_config=
+    (
+        collection_count=10,
+        key_count_per_collection=5000,
+        key_size=100,
+        thread_count=10,
+        value_size=10000
+    ),
+    read_config=
+    (
+        op_rate=5ms,
+        ops_per_transaction=(max=5000,min=1000),
+        thread_count=10
+    ),
+    update_config=
+    (
+        key_size=100,
+        op_rate=10ms,
+        ops_per_transaction=(max=100,min=20),
+        thread_count=10,
+        value_size=10000
+    )
+),
+# Tracking is currently disabled as verification can't handle rollbacks.
+workload_tracking=
+(
+    enabled=false
+)
diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/core/configuration.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/core/configuration.h
index 7eaa96214cb..0e33f7e059b 100644
--- a/src/third_party/wiredtiger/test/cppsuite/test_harness/core/configuration.h
+++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/core/configuration.h
@@ -50,7 +50,7 @@ class configuration {
         std::string default_config = std::string(config_entry->base);
         /* Merge in the default configuration. */
         _config = merge_default_config(default_config, config);
-        debug_print("Running with enriched config: " + _config, DEBUG_INFO);
+        debug_print("Full config: " + _config, DEBUG_INFO);
 
         int ret = wiredtiger_test_config_validate(
           nullptr, nullptr, test_config_name.c_str(), _config.c_str());
diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.h
index 274173a099f..f60f9164059 100644
--- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.h
+++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.h
@@ -144,12 +144,20 @@ class database_operation {
 
         while (tc->running()) {
             /* Walk each cursor. */
+            tc->transaction.try_begin(tc->session, "");
+
             for (const auto &it : cursors) {
                 if (it->next(it) != 0)
                     it->reset(it);
+                tc->transaction.op_count++;
             }
+
+            tc->transaction.try_rollback(tc->session, "");
             tc->sleep();
         }
+        /* Make sure the last operation is committed now the work is finished. */
+        if (tc->transaction.active())
+            tc->transaction.rollback(tc->session, "");
     }
 
     /*
@@ -218,9 +226,8 @@ class database_operation {
                   collection_id, collection_cursors{collection_name, random_cursor, update_cursor});
             }
 
-            /* Start a transaction. */
-            if (!tc->transaction.active())
-                tc->transaction.begin(tc->session, "");
+            /* Start a transaction if possible. */
+            tc->transaction.try_begin(tc->session, "");
 
             /* Get the random cursor associated with the collection. */
             auto collection = collections[collection_id];
@@ -275,8 +282,7 @@ class database_operation {
                 tc->transaction.rollback(tc->session, "");
 
             /* Commit the current transaction if we're able to. */
-            if (tc->transaction.can_commit())
-                tc->transaction.commit(tc->session, "");
+            tc->transaction.try_commit(tc->session, "");
         }
 
         /* Make sure the last operation is committed now the work is finished. */
@@ -284,7 +290,7 @@ class database_operation {
             tc->transaction.commit(tc->session, "");
     }
 
-    private:
+    protected:
     /* WiredTiger APIs wrappers for single operations. */
     template <typename K, typename V>
     static int
diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h
index 7f51cc90e47..b7beca55cfc 100644
--- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h
+++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h
@@ -46,42 +46,44 @@ class transaction_context {
         delete transaction_config;
     }
 
-    bool
-    active() const
+    /* Begin a transaction if we are not currently in one. */
+    void
+    try_begin(WT_SESSION *session, const std::string &config)
     {
-        return (_in_txn);
+        if (!_in_txn)
+            begin(session, config);
     }
 
-    /* Begin a transaction. */
     void
     begin(WT_SESSION *session, const std::string &config)
     {
-        if (!_in_txn) {
-            testutil_check(
-              session->begin_transaction(session, config.empty() ? nullptr : config.c_str()));
-            /* This randomizes the number of operations to be executed in one transaction. */
-            _target_op_count =
-              random_generator::instance().generate_integer<int64_t>(_min_op_count, _max_op_count);
-            op_count = 0;
-            _in_txn = true;
-        } else
-            testutil_die(EINVAL, "Begin called on a currently running transaction.");
+        testutil_assert(!_in_txn);
+        testutil_check(
+          session->begin_transaction(session, config.empty() ? nullptr : config.c_str()));
+        /* This randomizes the number of operations to be executed in one transaction. */
+        _target_op_count =
+          random_generator::instance().generate_integer<int64_t>(_min_op_count, _max_op_count);
+        op_count = 0;
+        _in_txn = true;
     }
 
-    /*
-     * The current transaction can be committed if: A transaction has started and the number of
-     * operations executed in the current transaction has exceeded the threshold.
-     */
     bool
-    can_commit() const
+    active() const
     {
-        return (_in_txn && op_count >= _target_op_count);
+        return (_in_txn);
+    }
+
+    /* Attempt to commit the transaction given the requirements are met. */
+    void
+    try_commit(WT_SESSION *session, const std::string &config)
+    {
+        if (can_commit_rollback())
+            commit(session, config);
     }
 
     void
     commit(WT_SESSION *session, const std::string &config)
     {
-        /* A transaction cannot be committed if not started. */
         testutil_assert(_in_txn);
         testutil_check(
           session->commit_transaction(session, config.empty() ? nullptr : config.c_str()));
@@ -89,6 +91,14 @@ class transaction_context {
         _in_txn = false;
     }
 
+    /* Attempt to rollback the transaction given the requirements are met. */
+    void
+    try_rollback(WT_SESSION *session, const std::string &config)
+    {
+        if (can_commit_rollback())
+            rollback(session, config);
+    }
+
     void
     rollback(WT_SESSION *session, const std::string &config)
     {
@@ -116,6 +126,12 @@ class transaction_context {
     int64_t op_count = 0;
 
     private:
+    bool
+    can_commit_rollback()
+    {
+        return (_in_txn && op_count >= _target_op_count);
+    }
+
     /*
      * _min_op_count and _max_op_count are the minimum and maximum number of operations within one
      * transaction. is the current maximum number of operations that can be executed in the current
@@ -150,15 +166,15 @@ class thread_context {
     thread_context(uint64_t id, thread_type type, configuration *config,
       timestamp_manager *timestamp_manager, workload_tracking *tracking, database &db)
         : id(id), type(type), database(db), timestamp_manager(timestamp_manager),
-          tracking(tracking), transaction(transaction_context(config))
+          tracking(tracking), transaction(transaction_context(config)),
+          /* These won't exist for read threads which is why we use optional here. */
+          key_size(config->get_optional_int(KEY_SIZE, 1)),
+          value_size(config->get_optional_int(VALUE_SIZE, 1)),
+          thread_count(config->get_int(THREAD_COUNT))
     {
         session = connection_manager::instance().create_session();
         _throttle = throttle(config);
 
-        /* These won't exist for read threads which is why we use optional here. */
-        key_size = config->get_optional_int(KEY_SIZE, 1);
-        value_size = config->get_optional_int(VALUE_SIZE, 1);
-
         testutil_assert(key_size > 0 && value_size > 0);
     }
 
@@ -185,8 +201,9 @@ class thread_context {
     test_harness::timestamp_manager *timestamp_manager;
     test_harness::workload_tracking *tracking;
     test_harness::database &database;
-    int64_t key_size = 0;
-    int64_t value_size = 0;
+    const int64_t key_size;
+    const int64_t value_size;
+    const int64_t thread_count;
     const uint64_t id;
     const thread_type type;
 
diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_generator.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_generator.h
index 10d6f4b9aa6..e6fb4db169c 100644
--- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_generator.h
+++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_generator.h
@@ -133,6 +133,12 @@ class workload_generator : public component {
              * into the thread context it is not saved, so we are safe to do this.
              */
             delete it.config;
+
+            /*
+             * Reset the thread_id counter to 0 as we're only interested in knowing per operation
+             * type which thread we are.
+             */
+            thread_id = 0;
         }
     }
 
diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/hs_cleanup.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/hs_cleanup.cxx
index 178ae35df09..d47b53d899b 100644
--- a/src/third_party/wiredtiger/test/cppsuite/tests/hs_cleanup.cxx
+++ b/src/third_party/wiredtiger/test/cppsuite/tests/hs_cleanup.cxx
@@ -28,31 +28,72 @@
 
 #include "test_harness/test.h"
 
+#include "test_harness/connection_manager.h"
+
+using namespace test_harness;
+
 /*
- * Class that defines operations that do nothing as an example.
- * This shows how database operations can be overriden and customized.
+ * Here we want to age out entire pages, i.e. the stop time pair on a page should be globally
+ * visible. To do so we'll update ranges of keys with increasing timestamps which will age out
+ * the pre-existing data. It may not trigger a cleanup on the data file but should result in
+ * data getting cleaned up from the history store.
+ *
+ * This is then tracked using the associated statistic which can be found in the runtime_monitor.
  */
-class hs_cleanup : public test_harness::test {
+class hs_cleanup : public test {
     public:
     hs_cleanup(const std::string &config, const std::string &name) : test(config, name) {}
 
     void
-    populate(test_harness::database &database, test_harness::timestamp_manager *_timestamp_manager,
-      test_harness::configuration *_config, test_harness::workload_tracking *tracking)
-       override final
+    update_operation(thread_context *tc) override final
     {
-        std::cout << "populate: nothing done." << std::endl;
-    }
+        WT_CURSOR *cursor = nullptr;
+        WT_DECL_RET;
+        const char *key_tmp;
+        WT_SESSION *session = connection_manager::instance().create_session();
+        std::string collection_name = tc->database.get_collection_name(tc->id);
+        wt_timestamp_t ts;
 
-    void
-    read_operation(test_harness::thread_context *context) override final
-    {
-        std::cout << "read_operation: nothing done." << std::endl;
-    }
+        /* In this test each thread gets a single collection. */
+        testutil_assert(tc->database.get_collection_count() == tc->thread_count);
+        testutil_check(session->open_cursor(session,  collection_name.c_str(), nullptr, nullptr, &cursor));
 
-    void
-    update_operation(test_harness::thread_context *context) override final
-    {
-        std::cout << "update_operation: nothing done." << std::endl;
+        /* We don't know the keyrange we're operating over here so we can't be much smarter here. */
+        while (tc->running()) {
+            tc->sleep();
+            ret = cursor->next(cursor);
+            if (ret != 0) {
+                if (ret == WT_NOTFOUND) {
+                    testutil_check(cursor->reset(cursor));
+                    continue;
+                } else
+                    testutil_die(ret, "cursor->next() failed unexpectedly.");
+            }
+            testutil_check(cursor->get_key(cursor, &key_tmp));
+
+            /* Start a transaction if possible. */
+            tc->transaction.try_begin(tc->session, "");
+
+            ts = tc->timestamp_manager->get_next_ts();
+            if (tc->timestamp_manager->enabled())
+                tc->transaction.set_commit_timestamp(
+                  tc->session, timestamp_manager::decimal_to_hex(ts));
+
+            /* Update the record but take care to handle WT_ROLLBACK. */
+            ret = update(tc->tracking, cursor, collection_name, key_value_t(key_tmp).c_str(),
+             random_generator::instance().generate_string(tc->value_size).c_str(), ts);
+            /* Increment the current op count for the current transaction. */
+            tc->transaction.op_count++;
+            if (ret == WT_ROLLBACK)
+                tc->transaction.rollback(tc->session, "");
+            else if (ret != 0)
+                testutil_die(ret, "failed to update a key.");
+
+            /* Commit our transaction. */
+            tc->transaction.try_commit(tc->session, "");
+        }
+        /* Ensure our last transaction is resolved. */
+        if (tc->transaction.active())
+            tc->transaction.commit(tc->session, "");
     }
 };
diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx
index 8a7acfee2bd..b877c23ab6b 100755
--- a/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx
+++ b/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx
@@ -110,7 +110,7 @@ run_test(const std::string &test_name, const std::string &config)
 {
     int error_code = 0;
 
-    test_harness::debug_print("Configuration\t:" + config, DEBUG_INFO);
+    test_harness::debug_print("Configuration\t:" + config, DEBUG_TRACE);
 
     if (test_name == "poc_test")
         poc_test(config, test_name).run();
@@ -190,7 +190,7 @@ main(int argc, char *argv[])
 
     if (error_code == 0) {
         test_harness::debug_print(
-          "Trace level\t:" + std::to_string(test_harness::_trace_level), DEBUG_INFO);
+          "Trace level: " + std::to_string(test_harness::_trace_level), DEBUG_INFO);
         if (test_name.empty()) {
             /* Run all tests. */
             test_harness::debug_print("Running all tests.", DEBUG_INFO);