diff options
10 files changed, 196 insertions, 71 deletions
diff --git a/src/third_party/wiredtiger/dist/test_data.py b/src/third_party/wiredtiger/dist/test_data.py index b179ec92cc6..8194fd66bf3 100644 --- a/src/third_party/wiredtiger/dist/test_data.py +++ b/src/third_party/wiredtiger/dist/test_data.py @@ -112,8 +112,8 @@ transaction_config = [ ] thread_count = [ - Config('thread_count', 1, r''' - Specifies the number of threads that will be used to perform a certain function.''') + Config('thread_count', 0, r''' + Specifies the number of threads that will be used to perform a certain function.''', min=0) ] read_thread_config = thread_count + throttle_config + transaction_config diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index b748fcaeebf..78611070478 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-5.0", - "commit": "c2f8ac069a38c5cbeb127560fb01a3e23548a912" + "commit": "b724e3ec5cd6e27bcfaaa53f6308519bda56c7e6" } diff --git a/src/third_party/wiredtiger/src/config/test_config.c b/src/third_party/wiredtiger/src/config/test_config.c index c1142a652e3..5811d5993e8 100644 --- a/src/third_party/wiredtiger/src/config/test_config.c +++ b/src/third_party/wiredtiger/src/config/test_config.c @@ -37,7 +37,7 @@ static const WT_CONFIG_CHECK confchk_ops_per_transaction_subconfigs[] = { static const WT_CONFIG_CHECK confchk_insert_config_subconfigs[] = { {"key_size", "int", NULL, "min=0,max=10000", NULL, 0}, {"op_rate", "string", NULL, NULL, NULL, 0}, {"ops_per_transaction", "category", NULL, NULL, confchk_ops_per_transaction_subconfigs, 2}, - {"thread_count", "string", NULL, NULL, NULL, 0}, + {"thread_count", "int", NULL, "min=0", NULL, 0}, {"value_size", "int", NULL, "min=0,max=1000000000", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_populate_config_subconfigs[] = { @@ -50,12 +50,12 @@ static const WT_CONFIG_CHECK confchk_populate_config_subconfigs[] = { static const WT_CONFIG_CHECK confchk_read_config_subconfigs[] = { {"op_rate", "string", NULL, NULL, NULL, 0}, {"ops_per_transaction", "category", NULL, NULL, confchk_ops_per_transaction_subconfigs, 2}, - {"thread_count", "string", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; + {"thread_count", "int", NULL, "min=0", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_update_config_subconfigs[] = { {"key_size", "int", NULL, "min=0,max=10000", NULL, 0}, {"op_rate", "string", NULL, NULL, NULL, 0}, {"ops_per_transaction", "category", NULL, NULL, confchk_ops_per_transaction_subconfigs, 2}, - {"thread_count", "string", NULL, NULL, NULL, 0}, + {"thread_count", "int", NULL, "min=0", NULL, 0}, {"value_size", "int", NULL, "min=0,max=1000000000", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_workload_generator_subconfigs[] = { @@ -118,12 +118,12 @@ static const WT_CONFIG_ENTRY config_entries[] = { "timestamp_manager=(enabled=true,oldest_lag=1,op_rate=1s," "stable_lag=1),workload_generator=(enabled=true," "insert_config=(key_size=5,op_rate=1s,ops_per_transaction=(max=1," - "min=0),thread_count=1,value_size=5),op_rate=1s," + "min=0),thread_count=0,value_size=5),op_rate=1s," "populate_config=(collection_count=1,key_count_per_collection=0," "key_size=5,thread_count=1,value_size=5),read_config=(op_rate=1s," - "ops_per_transaction=(max=1,min=0),thread_count=1)," + "ops_per_transaction=(max=1,min=0),thread_count=0)," "update_config=(key_size=5,op_rate=1s,ops_per_transaction=(max=1," - "min=0),thread_count=1,value_size=5))," + "min=0),thread_count=0,value_size=5))," "workload_tracking=(enabled=true,op_rate=1s)", confchk_example_test, 9}, {"hs_cleanup", @@ -136,12 +136,12 @@ static const WT_CONFIG_ENTRY config_entries[] = { "timestamp_manager=(enabled=true,oldest_lag=1,op_rate=1s," "stable_lag=1),workload_generator=(enabled=true," "insert_config=(key_size=5,op_rate=1s,ops_per_transaction=(max=1," - "min=0),thread_count=1,value_size=5),op_rate=1s," + "min=0),thread_count=0,value_size=5),op_rate=1s," "populate_config=(collection_count=1,key_count_per_collection=0," "key_size=5,thread_count=1,value_size=5),read_config=(op_rate=1s," - "ops_per_transaction=(max=1,min=0),thread_count=1)," + "ops_per_transaction=(max=1,min=0),thread_count=0)," "update_config=(key_size=5,op_rate=1s,ops_per_transaction=(max=1," - "min=0),thread_count=1,value_size=5))," + "min=0),thread_count=0,value_size=5))," "workload_tracking=(enabled=true,op_rate=1s)", confchk_hs_cleanup, 9}, {"poc_test", @@ -154,12 +154,12 @@ static const WT_CONFIG_ENTRY config_entries[] = { "timestamp_manager=(enabled=true,oldest_lag=1,op_rate=1s," "stable_lag=1),workload_generator=(enabled=true," "insert_config=(key_size=5,op_rate=1s,ops_per_transaction=(max=1," - "min=0),thread_count=1,value_size=5),op_rate=1s," + "min=0),thread_count=0,value_size=5),op_rate=1s," "populate_config=(collection_count=1,key_count_per_collection=0," "key_size=5,thread_count=1,value_size=5),read_config=(op_rate=1s," - "ops_per_transaction=(max=1,min=0),thread_count=1)," + "ops_per_transaction=(max=1,min=0),thread_count=0)," "update_config=(key_size=5,op_rate=1s,ops_per_transaction=(max=1," - "min=0),thread_count=1,value_size=5))," + "min=0),thread_count=0,value_size=5))," "workload_tracking=(enabled=true,op_rate=1s)", confchk_poc_test, 9}, {NULL, NULL, NULL, 0}}; diff --git a/src/third_party/wiredtiger/test/cppsuite/configs/config_hs_cleanup_default.txt b/src/third_party/wiredtiger/test/cppsuite/configs/config_hs_cleanup_default.txt index f46526e554e..626cf9a2fa3 100644 --- a/src/third_party/wiredtiger/test/cppsuite/configs/config_hs_cleanup_default.txt +++ b/src/third_party/wiredtiger/test/cppsuite/configs/config_hs_cleanup_default.txt @@ -1,4 +1,59 @@ # Configuration for hs_cleanup. # need to be defined. -duration_seconds=5, -cache_size_mb=250 +duration_seconds=240, +cache_size_mb=400, +statistics_config= +( + type=all, + enable_logging=true +), +checkpoint_manager= +( + enabled=true, + op_rate=20s +), +runtime_monitor= +( + stat_cache_size= + ( + enabled=true, + limit=100 + ) +), +timestamp_manager= +( + enabled=true, + oldest_lag=5, + op_rate=1s, + stable_lag=5 +), +workload_generator= +( + populate_config= + ( + collection_count=10, + key_count_per_collection=5000, + key_size=100, + thread_count=10, + value_size=10000 + ), + read_config= + ( + op_rate=5ms, + ops_per_transaction=(max=5000,min=1000), + thread_count=10 + ), + update_config= + ( + key_size=100, + op_rate=10ms, + ops_per_transaction=(max=100,min=20), + thread_count=10, + value_size=10000 + ) +), +# Tracking is currently disabled as verification can't handle rollbacks. +workload_tracking= +( + enabled=false +) diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/core/configuration.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/core/configuration.h index 7eaa96214cb..0e33f7e059b 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/core/configuration.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/core/configuration.h @@ -50,7 +50,7 @@ class configuration { std::string default_config = std::string(config_entry->base); /* Merge in the default configuration. */ _config = merge_default_config(default_config, config); - debug_print("Running with enriched config: " + _config, DEBUG_INFO); + debug_print("Full config: " + _config, DEBUG_INFO); int ret = wiredtiger_test_config_validate( nullptr, nullptr, test_config_name.c_str(), _config.c_str()); diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.h index 274173a099f..f60f9164059 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.h @@ -144,12 +144,20 @@ class database_operation { while (tc->running()) { /* Walk each cursor. */ + tc->transaction.try_begin(tc->session, ""); + for (const auto &it : cursors) { if (it->next(it) != 0) it->reset(it); + tc->transaction.op_count++; } + + tc->transaction.try_rollback(tc->session, ""); tc->sleep(); } + /* Make sure the last operation is committed now the work is finished. */ + if (tc->transaction.active()) + tc->transaction.rollback(tc->session, ""); } /* @@ -218,9 +226,8 @@ class database_operation { collection_id, collection_cursors{collection_name, random_cursor, update_cursor}); } - /* Start a transaction. */ - if (!tc->transaction.active()) - tc->transaction.begin(tc->session, ""); + /* Start a transaction if possible. */ + tc->transaction.try_begin(tc->session, ""); /* Get the random cursor associated with the collection. */ auto collection = collections[collection_id]; @@ -275,8 +282,7 @@ class database_operation { tc->transaction.rollback(tc->session, ""); /* Commit the current transaction if we're able to. */ - if (tc->transaction.can_commit()) - tc->transaction.commit(tc->session, ""); + tc->transaction.try_commit(tc->session, ""); } /* Make sure the last operation is committed now the work is finished. */ @@ -284,7 +290,7 @@ class database_operation { tc->transaction.commit(tc->session, ""); } - private: + protected: /* WiredTiger APIs wrappers for single operations. */ template <typename K, typename V> static int diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h index 7f51cc90e47..b7beca55cfc 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h @@ -46,42 +46,44 @@ class transaction_context { delete transaction_config; } - bool - active() const + /* Begin a transaction if we are not currently in one. */ + void + try_begin(WT_SESSION *session, const std::string &config) { - return (_in_txn); + if (!_in_txn) + begin(session, config); } - /* Begin a transaction. */ void begin(WT_SESSION *session, const std::string &config) { - if (!_in_txn) { - testutil_check( - session->begin_transaction(session, config.empty() ? nullptr : config.c_str())); - /* This randomizes the number of operations to be executed in one transaction. */ - _target_op_count = - random_generator::instance().generate_integer<int64_t>(_min_op_count, _max_op_count); - op_count = 0; - _in_txn = true; - } else - testutil_die(EINVAL, "Begin called on a currently running transaction."); + testutil_assert(!_in_txn); + testutil_check( + session->begin_transaction(session, config.empty() ? nullptr : config.c_str())); + /* This randomizes the number of operations to be executed in one transaction. */ + _target_op_count = + random_generator::instance().generate_integer<int64_t>(_min_op_count, _max_op_count); + op_count = 0; + _in_txn = true; } - /* - * The current transaction can be committed if: A transaction has started and the number of - * operations executed in the current transaction has exceeded the threshold. - */ bool - can_commit() const + active() const { - return (_in_txn && op_count >= _target_op_count); + return (_in_txn); + } + + /* Attempt to commit the transaction given the requirements are met. */ + void + try_commit(WT_SESSION *session, const std::string &config) + { + if (can_commit_rollback()) + commit(session, config); } void commit(WT_SESSION *session, const std::string &config) { - /* A transaction cannot be committed if not started. */ testutil_assert(_in_txn); testutil_check( session->commit_transaction(session, config.empty() ? nullptr : config.c_str())); @@ -89,6 +91,14 @@ class transaction_context { _in_txn = false; } + /* Attempt to rollback the transaction given the requirements are met. */ + void + try_rollback(WT_SESSION *session, const std::string &config) + { + if (can_commit_rollback()) + rollback(session, config); + } + void rollback(WT_SESSION *session, const std::string &config) { @@ -116,6 +126,12 @@ class transaction_context { int64_t op_count = 0; private: + bool + can_commit_rollback() + { + return (_in_txn && op_count >= _target_op_count); + } + /* * _min_op_count and _max_op_count are the minimum and maximum number of operations within one * transaction. is the current maximum number of operations that can be executed in the current @@ -150,15 +166,15 @@ class thread_context { thread_context(uint64_t id, thread_type type, configuration *config, timestamp_manager *timestamp_manager, workload_tracking *tracking, database &db) : id(id), type(type), database(db), timestamp_manager(timestamp_manager), - tracking(tracking), transaction(transaction_context(config)) + tracking(tracking), transaction(transaction_context(config)), + /* These won't exist for read threads which is why we use optional here. */ + key_size(config->get_optional_int(KEY_SIZE, 1)), + value_size(config->get_optional_int(VALUE_SIZE, 1)), + thread_count(config->get_int(THREAD_COUNT)) { session = connection_manager::instance().create_session(); _throttle = throttle(config); - /* These won't exist for read threads which is why we use optional here. */ - key_size = config->get_optional_int(KEY_SIZE, 1); - value_size = config->get_optional_int(VALUE_SIZE, 1); - testutil_assert(key_size > 0 && value_size > 0); } @@ -185,8 +201,9 @@ class thread_context { test_harness::timestamp_manager *timestamp_manager; test_harness::workload_tracking *tracking; test_harness::database &database; - int64_t key_size = 0; - int64_t value_size = 0; + const int64_t key_size; + const int64_t value_size; + const int64_t thread_count; const uint64_t id; const thread_type type; diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_generator.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_generator.h index 10d6f4b9aa6..e6fb4db169c 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_generator.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_generator.h @@ -133,6 +133,12 @@ class workload_generator : public component { * into the thread context it is not saved, so we are safe to do this. */ delete it.config; + + /* + * Reset the thread_id counter to 0 as we're only interested in knowing per operation + * type which thread we are. + */ + thread_id = 0; } } diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/hs_cleanup.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/hs_cleanup.cxx index 178ae35df09..d47b53d899b 100644 --- a/src/third_party/wiredtiger/test/cppsuite/tests/hs_cleanup.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/tests/hs_cleanup.cxx @@ -28,31 +28,72 @@ #include "test_harness/test.h" +#include "test_harness/connection_manager.h" + +using namespace test_harness; + /* - * Class that defines operations that do nothing as an example. - * This shows how database operations can be overriden and customized. + * Here we want to age out entire pages, i.e. the stop time pair on a page should be globally + * visible. To do so we'll update ranges of keys with increasing timestamps which will age out + * the pre-existing data. It may not trigger a cleanup on the data file but should result in + * data getting cleaned up from the history store. + * + * This is then tracked using the associated statistic which can be found in the runtime_monitor. */ -class hs_cleanup : public test_harness::test { +class hs_cleanup : public test { public: hs_cleanup(const std::string &config, const std::string &name) : test(config, name) {} void - populate(test_harness::database &database, test_harness::timestamp_manager *_timestamp_manager, - test_harness::configuration *_config, test_harness::workload_tracking *tracking) - override final + update_operation(thread_context *tc) override final { - std::cout << "populate: nothing done." << std::endl; - } + WT_CURSOR *cursor = nullptr; + WT_DECL_RET; + const char *key_tmp; + WT_SESSION *session = connection_manager::instance().create_session(); + std::string collection_name = tc->database.get_collection_name(tc->id); + wt_timestamp_t ts; - void - read_operation(test_harness::thread_context *context) override final - { - std::cout << "read_operation: nothing done." << std::endl; - } + /* In this test each thread gets a single collection. */ + testutil_assert(tc->database.get_collection_count() == tc->thread_count); + testutil_check(session->open_cursor(session, collection_name.c_str(), nullptr, nullptr, &cursor)); - void - update_operation(test_harness::thread_context *context) override final - { - std::cout << "update_operation: nothing done." << std::endl; + /* We don't know the keyrange we're operating over here so we can't be much smarter here. */ + while (tc->running()) { + tc->sleep(); + ret = cursor->next(cursor); + if (ret != 0) { + if (ret == WT_NOTFOUND) { + testutil_check(cursor->reset(cursor)); + continue; + } else + testutil_die(ret, "cursor->next() failed unexpectedly."); + } + testutil_check(cursor->get_key(cursor, &key_tmp)); + + /* Start a transaction if possible. */ + tc->transaction.try_begin(tc->session, ""); + + ts = tc->timestamp_manager->get_next_ts(); + if (tc->timestamp_manager->enabled()) + tc->transaction.set_commit_timestamp( + tc->session, timestamp_manager::decimal_to_hex(ts)); + + /* Update the record but take care to handle WT_ROLLBACK. */ + ret = update(tc->tracking, cursor, collection_name, key_value_t(key_tmp).c_str(), + random_generator::instance().generate_string(tc->value_size).c_str(), ts); + /* Increment the current op count for the current transaction. */ + tc->transaction.op_count++; + if (ret == WT_ROLLBACK) + tc->transaction.rollback(tc->session, ""); + else if (ret != 0) + testutil_die(ret, "failed to update a key."); + + /* Commit our transaction. */ + tc->transaction.try_commit(tc->session, ""); + } + /* Ensure our last transaction is resolved. */ + if (tc->transaction.active()) + tc->transaction.commit(tc->session, ""); } }; diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx index 8a7acfee2bd..b877c23ab6b 100755 --- a/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx @@ -110,7 +110,7 @@ run_test(const std::string &test_name, const std::string &config) { int error_code = 0; - test_harness::debug_print("Configuration\t:" + config, DEBUG_INFO); + test_harness::debug_print("Configuration\t:" + config, DEBUG_TRACE); if (test_name == "poc_test") poc_test(config, test_name).run(); @@ -190,7 +190,7 @@ main(int argc, char *argv[]) if (error_code == 0) { test_harness::debug_print( - "Trace level\t:" + std::to_string(test_harness::_trace_level), DEBUG_INFO); + "Trace level: " + std::to_string(test_harness::_trace_level), DEBUG_INFO); if (test_name.empty()) { /* Run all tests. */ test_harness::debug_print("Running all tests.", DEBUG_INFO); |