diff options
author | Etienne Petrel <etienne.petrel@mongodb.com> | 2022-12-05 19:30:56 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2022-12-05 20:32:57 +0000 |
commit | b0fe06c9685f80eb5146fff48c70e6942c6e27b4 (patch) | |
tree | 548c03cc3312d22e3ab8643b9ea63ac16aef14dd | |
parent | 41e4ac37149fef7fc62647c0f3653f7e3ccfc682 (diff) | |
download | mongo-b0fe06c9685f80eb5146fff48c70e6942c6e27b4.tar.gz |
Import wiredtiger: 2cd06f459e46317e22e5af91ea466099c36bc134 from branch mongodb-master
ref: babc338960..2cd06f459e
for: 6.3.0-rc0
WT-10241 Creating a CppSuite workload that stresses the reverse split code path
20 files changed, 452 insertions, 20 deletions
diff --git a/src/third_party/wiredtiger/dist/test_data.py b/src/third_party/wiredtiger/dist/test_data.py index 41c8a625aa2..eca69ccfd2f 100644 --- a/src/third_party/wiredtiger/dist/test_data.py +++ b/src/third_party/wiredtiger/dist/test_data.py @@ -240,20 +240,24 @@ test_config = [ ]), ] +# +# Test and their respective configuration sorted alphabetically. +# methods = { 'bounded_cursor_perf' : Method(test_config), - 'burst_inserts' : Method(test_config + [ - Config("burst_duration", 90, r''' - How long the insertions will occur for.''')]), - 'cache_resize' : Method(test_config), 'bounded_cursor_prefix_indices' : Method(test_config), 'bounded_cursor_prefix_search_near' : Method(test_config), 'bounded_cursor_prefix_stat' : Method(test_config + [ Config("search_near_threads", 10, r''' Number of threads that execute search near calls.''')]), 'bounded_cursor_stress' : Method(test_config), + 'burst_inserts' : Method(test_config + [ + Config("burst_duration", 90, r''' + How long the insertions will occur for.''')]), + 'cache_resize' : Method(test_config), 'hs_cleanup' : Method(test_config), 'operations_test' : Method(test_config), + 'reverse_split' : Method(test_config), 'search_near_01' : Method(test_config + [ Config("search_near_threads", 10, r''' Number of threads that execute search near calls.''')]), diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 41d4e445285..8ee5c405eef 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-master", - "commit": "babc338960decf4e495bb74a5de50e6340d73b00" + "commit": "2cd06f459e46317e22e5af91ea466099c36bc134" } diff --git a/src/third_party/wiredtiger/src/config/test_config.c b/src/third_party/wiredtiger/src/config/test_config.c index fe7167d5f61..5120724705f 100644 --- a/src/third_party/wiredtiger/src/config/test_config.c +++ b/src/third_party/wiredtiger/src/config/test_config.c @@ -228,6 +228,20 @@ static const WT_CONFIG_CHECK confchk_operations_test[] = { {"workload_manager", "category", NULL, NULL, confchk_workload_manager_subconfigs, 9}, {NULL, NULL, NULL, NULL, NULL, 0}}; +static const WT_CONFIG_CHECK confchk_reverse_split[] = { + {"cache_max_wait_ms", "int", NULL, "min=0", NULL, 0}, + {"cache_size_mb", "int", NULL, "min=0,max=100000000000", NULL, 0}, + {"compression_enabled", "boolean", NULL, NULL, NULL, 0}, + {"duration_seconds", "int", NULL, "min=0,max=1000000", NULL, 0}, + {"enable_logging", "boolean", NULL, NULL, NULL, 0}, + {"metrics_monitor", "category", NULL, NULL, confchk_metrics_monitor_subconfigs, 6}, + {"operation_tracker", "category", NULL, NULL, confchk_operation_tracker_subconfigs, 4}, + {"reverse_collator", "boolean", NULL, NULL, NULL, 0}, + {"statistics_config", "category", NULL, NULL, confchk_statistics_config_subconfigs, 2}, + {"timestamp_manager", "category", NULL, NULL, confchk_timestamp_manager_subconfigs, 4}, + {"workload_manager", "category", NULL, NULL, confchk_workload_manager_subconfigs, 9}, + {NULL, NULL, NULL, NULL, NULL, 0}}; + static const WT_CONFIG_CHECK confchk_search_near_01[] = { {"cache_max_wait_ms", "int", NULL, "min=0", NULL, 0}, {"cache_size_mb", "int", NULL, "min=0,max=100000000000", NULL, 0}, @@ -521,6 +535,32 @@ static const WT_CONFIG_ENTRY config_entries[] = { "update_config=(key_size=5,op_rate=1s,ops_per_transaction=(max=1," "min=0),thread_count=0,value_size=5))", confchk_operations_test, 11}, + {"reverse_split", + "cache_max_wait_ms=0,cache_size_mb=0,compression_enabled=false," + "duration_seconds=0,enable_logging=false," + "metrics_monitor=(cache_hs_insert=(max=1,min=0,postrun=false," + "runtime=false,save=false),cc_pages_removed=(max=1,min=0," + "postrun=false,runtime=false,save=false),enabled=true,op_rate=1s," + "stat_cache_size=(max=1,min=0,postrun=false,runtime=false," + "save=false),stat_db_size=(max=1,min=0,postrun=false," + "runtime=false,save=false)),operation_tracker=(enabled=true," + "op_rate=1s,tracking_key_format=QSQ,tracking_value_format=iS)," + "reverse_collator=false,statistics_config=(enable_logging=true," + "type=all),timestamp_manager=(enabled=true,oldest_lag=1," + "op_rate=1s,stable_lag=1)," + "workload_manager=(checkpoint_config=(op_rate=60s,thread_count=1)" + ",custom_config=(key_size=5,op_rate=1s,ops_per_transaction=(max=1" + ",min=0),thread_count=0,value_size=5),enabled=true," + "insert_config=(key_size=5,op_rate=1s,ops_per_transaction=(max=1," + "min=0),thread_count=0,value_size=5),op_rate=1s," + "populate_config=(collection_count=1,key_count_per_collection=0," + "key_size=5,thread_count=1,value_size=5),read_config=(key_size=5," + "op_rate=1s,ops_per_transaction=(max=1,min=0),thread_count=0," + "value_size=5),remove_config=(op_rate=1s," + "ops_per_transaction=(max=1,min=0),thread_count=0)," + "update_config=(key_size=5,op_rate=1s,ops_per_transaction=(max=1," + "min=0),thread_count=0,value_size=5))", + confchk_reverse_split, 11}, {"search_near_01", "cache_max_wait_ms=0,cache_size_mb=0,compression_enabled=false," "duration_seconds=0,enable_logging=false," diff --git a/src/third_party/wiredtiger/test/cppsuite/configs/reverse_split_default.txt b/src/third_party/wiredtiger/test/cppsuite/configs/reverse_split_default.txt new file mode 100644 index 00000000000..17785017cf1 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/configs/reverse_split_default.txt @@ -0,0 +1,46 @@ +# This workload inserts content continuously and truncates off the start of the file. +# The truncations truncate up to 83% of the file each time. +# The default config intends to be a shorter configuration that tests the functionality of the test. +# It is is largely copied from the stress config. +duration_seconds=30, +cache_size_mb=256, +compression_enabled=true, +timestamp_manager= +( + enabled=true, + oldest_lag=30, + stable_lag=30 +), +workload_manager= +( + populate_config= + ( + collection_count=5, + key_count_per_collection=100, + key_size=50, + thread_count=5, + value_size=100000 + ), + insert_config= + ( + key_size=50, + op_rate=2ms, + ops_per_transaction=(max=30,min=0), + thread_count=5, + value_size=100000 + ), + remove_config= + ( + op_rate=5s, + ops_per_transaction=(max=1,min=0), + thread_count=5 + ), + checkpoint_config= + ( + op_rate=10s, + ) +), +operation_tracker= +( + enabled=false, +) diff --git a/src/third_party/wiredtiger/test/cppsuite/configs/reverse_split_stress.txt b/src/third_party/wiredtiger/test/cppsuite/configs/reverse_split_stress.txt new file mode 100644 index 00000000000..80431886066 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/configs/reverse_split_stress.txt @@ -0,0 +1,44 @@ +# This workload is inserts content continuously and truncates off the start of the file. +# The truncations truncate up to 83% of the file each time. +duration_seconds=7200, +cache_size_mb=256, +compression_enabled=true, +timestamp_manager= +( + enabled=true, + oldest_lag=30, + stable_lag=30 +), +workload_manager= +( + populate_config= + ( + collection_count=5, + key_count_per_collection=10000, + key_size=50, + thread_count=5, + value_size=100000 + ), + insert_config= + ( + key_size=50, + op_rate= 1ms, + ops_per_transaction=(max=30,min=0), + thread_count=5, + value_size=100000 + ), + remove_config= + ( + op_rate=45s, + ops_per_transaction=(max=1,min=0), + thread_count=5 + ), + checkpoint_config= + ( + op_rate=60s, + ) +), +operation_tracker= +( + enabled=false, +) diff --git a/src/third_party/wiredtiger/test/cppsuite/src/common/random_generator.cpp b/src/third_party/wiredtiger/test/cppsuite/src/common/random_generator.cpp index 2c481be0692..211c8f067d3 100644 --- a/src/third_party/wiredtiger/test/cppsuite/src/common/random_generator.cpp +++ b/src/third_party/wiredtiger/test/cppsuite/src/common/random_generator.cpp @@ -73,6 +73,12 @@ random_generator::generate_pseudo_random_string(std::size_t length, characters_t return (random_string); } +bool +random_generator::generate_bool() +{ + return generate_integer<int>(0, 1) == 1; +} + random_generator::random_generator() { _generator = std::mt19937(std::random_device{}()); diff --git a/src/third_party/wiredtiger/test/cppsuite/src/common/random_generator.h b/src/third_party/wiredtiger/test/cppsuite/src/common/random_generator.h index afdebbb3aef..b11fc745a81 100644 --- a/src/third_party/wiredtiger/test/cppsuite/src/common/random_generator.h +++ b/src/third_party/wiredtiger/test/cppsuite/src/common/random_generator.h @@ -64,6 +64,11 @@ class random_generator { std::string generate_pseudo_random_string( std::size_t length, characters_type type = characters_type::PSEUDO_ALPHANUMERIC); + /* + * Generate a boolean with 50/50 probability. + */ + bool generate_bool(); + /* Generate a random integer between min and max. */ template <typename T> T diff --git a/src/third_party/wiredtiger/test/cppsuite/src/component/workload_manager.cpp b/src/third_party/wiredtiger/test/cppsuite/src/component/workload_manager.cpp index e54175bfc4e..de7e3804f76 100644 --- a/src/third_party/wiredtiger/test/cppsuite/src/component/workload_manager.cpp +++ b/src/third_party/wiredtiger/test/cppsuite/src/component/workload_manager.cpp @@ -25,6 +25,7 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ +#include <memory> #include "workload_manager.h" @@ -32,6 +33,7 @@ #include "src/common/logger.h" #include "src/main/operation_configuration.h" #include "src/storage/connection_manager.h" +#include "src/util/barrier.h" namespace test_harness { workload_manager::workload_manager(configuration *configuration, database_operation *db_operation, @@ -88,10 +90,12 @@ workload_manager::run() logger::log_msg(LOG_INFO, "workload_manager: Creating " + std::to_string(it.thread_count) + " " + type_string(it.type) + " threads."); + /* Create a synchronisation object to provide to the thread workers. */ + std::shared_ptr<barrier> barrier_ptr = std::make_shared<barrier>(it.thread_count); for (size_t i = 0; i < it.thread_count && _running; ++i) { thread_worker *tc = new thread_worker(thread_id++, it.type, it.config, connection_manager::instance().create_session(), _timestamp_manager, - _operation_tracker, _database); + _operation_tracker, _database, barrier_ptr); _workers.push_back(tc); _thread_manager.add_thread(it.get_func(_database_operation), tc); } diff --git a/src/third_party/wiredtiger/test/cppsuite/src/main/test.cpp b/src/third_party/wiredtiger/test/cppsuite/src/main/test.cpp index a42b176f491..579b9834531 100644 --- a/src/third_party/wiredtiger/test/cppsuite/src/main/test.cpp +++ b/src/third_party/wiredtiger/test/cppsuite/src/main/test.cpp @@ -128,7 +128,7 @@ test::run() db_create_config += ",cache_max_wait_ms=" + std::to_string(cache_max_wait_ms); /* Add the user supplied wiredtiger open config. */ - db_create_config += _args.wt_open_config; + db_create_config += "," + _args.wt_open_config; /* Create connection. */ connection_manager::instance().create( diff --git a/src/third_party/wiredtiger/test/cppsuite/src/main/test.h b/src/third_party/wiredtiger/test/cppsuite/src/main/test.h index 5258c98a426..dd919860add 100644 --- a/src/third_party/wiredtiger/test/cppsuite/src/main/test.h +++ b/src/third_party/wiredtiger/test/cppsuite/src/main/test.h @@ -40,7 +40,7 @@ namespace test_harness { struct test_args { const std::string test_config; const std::string test_name; - const std::string wt_open_config; + std::string wt_open_config; const std::string home; }; diff --git a/src/third_party/wiredtiger/test/cppsuite/src/main/thread_worker.cpp b/src/third_party/wiredtiger/test/cppsuite/src/main/thread_worker.cpp index d88bd718a30..534ae7d95f8 100644 --- a/src/third_party/wiredtiger/test/cppsuite/src/main/thread_worker.cpp +++ b/src/third_party/wiredtiger/test/cppsuite/src/main/thread_worker.cpp @@ -60,6 +60,14 @@ type_string(thread_type type) thread_worker::thread_worker(uint64_t id, thread_type type, configuration *config, scoped_session &&created_session, timestamp_manager *timestamp_manager, operation_tracker *op_tracker, database &dbase) + : thread_worker( + id, type, config, std::move(created_session), timestamp_manager, op_tracker, dbase, nullptr) +{ +} + +thread_worker::thread_worker(uint64_t id, thread_type type, configuration *config, + scoped_session &&created_session, timestamp_manager *timestamp_manager, + operation_tracker *op_tracker, database &dbase, std::shared_ptr<barrier> barrier_ptr) : /* These won't exist for certain threads which is why we use optional here. */ collection_count(config->get_optional_int(COLLECTION_COUNT, 1)), key_count(config->get_optional_int(KEY_COUNT_PER_COLLECTION, 1)), @@ -68,7 +76,7 @@ thread_worker::thread_worker(uint64_t id, thread_type type, configuration *confi thread_count(config->get_int(THREAD_COUNT)), type(type), id(id), db(dbase), session(std::move(created_session)), tsm(timestamp_manager), txn(transaction(config, timestamp_manager, session.get())), op_tracker(op_tracker), - _sleep_time_ms(config->get_throttle_ms()) + _sleep_time_ms(config->get_throttle_ms()), _barrier(barrier_ptr) { if (op_tracker->enabled()) op_track_cursor = session.open_scoped_cursor(op_tracker->get_operation_table_name()); @@ -260,6 +268,12 @@ thread_worker::sleep() std::this_thread::sleep_for(std::chrono::milliseconds(_sleep_time_ms)); } +void +thread_worker::sync() +{ + _barrier->wait(); +} + bool thread_worker::running() const { diff --git a/src/third_party/wiredtiger/test/cppsuite/src/main/thread_worker.h b/src/third_party/wiredtiger/test/cppsuite/src/main/thread_worker.h index 707e2a658d9..b533deb5695 100644 --- a/src/third_party/wiredtiger/test/cppsuite/src/main/thread_worker.h +++ b/src/third_party/wiredtiger/test/cppsuite/src/main/thread_worker.h @@ -30,6 +30,7 @@ #define THREAD_WORKER_H #include <optional> +#include <memory> #include <string> #include "database.h" @@ -39,6 +40,7 @@ #include "src/storage/scoped_cursor.h" #include "src/storage/scoped_session.h" #include "transaction.h" +#include "src/util/barrier.h" namespace test_harness { enum class thread_type { CHECKPOINT, CUSTOM, INSERT, READ, REMOVE, UPDATE }; @@ -52,6 +54,10 @@ class thread_worker { scoped_session &&created_session, timestamp_manager *timestamp_manager, operation_tracker *op_tracker, database &dbase); + thread_worker(uint64_t id, thread_type type, configuration *config, + scoped_session &&created_session, timestamp_manager *timestamp_manager, + operation_tracker *op_tracker, database &dbase, std::shared_ptr<barrier> barrier_ptr); + virtual ~thread_worker() = default; void finish(); @@ -95,6 +101,7 @@ class thread_worker { std::optional<std::string> stop_key, const std::string &config); void sleep(); bool running() const; + void sync(); public: const int64_t collection_count; @@ -113,6 +120,7 @@ class thread_worker { operation_tracker *op_tracker; private: + std::shared_ptr<barrier> _barrier = nullptr; bool _running = true; uint64_t _sleep_time_ms = 1000; }; diff --git a/src/third_party/wiredtiger/test/cppsuite/src/util/barrier.cpp b/src/third_party/wiredtiger/test/cppsuite/src/util/barrier.cpp new file mode 100644 index 00000000000..eeee0c43183 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/util/barrier.cpp @@ -0,0 +1,50 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "barrier.h" + +namespace test_harness { +barrier::barrier(std::size_t thread_count) + : _threshold(thread_count), _count(thread_count), _generation(0) +{ +} + +void +barrier::wait() +{ + std::unique_lock<std::mutex> lock{_mutex}; + auto lock_gen = _generation; + if (!--_count) { + _generation++; + _count = _threshold; + _cond.notify_all(); + } else { + _cond.wait(lock, [this, lock_gen] { return lock_gen != _generation; }); + } +} +} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/src/util/barrier.h b/src/third_party/wiredtiger/test/cppsuite/src/util/barrier.h new file mode 100644 index 00000000000..95e24344886 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/util/barrier.h @@ -0,0 +1,58 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#pragma once + +#include <condition_variable> +#include <mutex> + +namespace test_harness { + +/* + * A barrier is a sychronization class that allows thread_count threads to perform an action at the + * same time. They are introduced in C++20 so we need a basic implementation in the meantime. + * + * In order to synchronize across threads call wait() on each thread, once thread_count threads have + * called wait() they will exit the wait() and continue. + */ +class barrier { + public: + /* Mutexes have a deleted copy constructor so we need to as well. */ + barrier(barrier const &) = delete; + ~barrier() = default; + explicit barrier(std::size_t thread_count); + void wait(); + + private: + std::mutex _mutex; + std::condition_variable _cond; + std::size_t _threshold; + std::size_t _count; + std::size_t _generation; +}; +} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/src/component/execution_timer.cpp b/src/third_party/wiredtiger/test/cppsuite/src/util/execution_timer.cpp index 66136aa7a88..5cfdef60118 100644 --- a/src/third_party/wiredtiger/test/cppsuite/src/component/execution_timer.cpp +++ b/src/third_party/wiredtiger/test/cppsuite/src/util/execution_timer.cpp @@ -28,7 +28,7 @@ #include "execution_timer.h" -#include "metrics_writer.h" +#include "src/component/metrics_writer.h" namespace test_harness { execution_timer::execution_timer(const std::string id, const std::string &test_name) diff --git a/src/third_party/wiredtiger/test/cppsuite/src/component/execution_timer.h b/src/third_party/wiredtiger/test/cppsuite/src/util/execution_timer.h index 1a3f498908e..1a3f498908e 100644 --- a/src/third_party/wiredtiger/test/cppsuite/src/component/execution_timer.h +++ b/src/third_party/wiredtiger/test/cppsuite/src/util/execution_timer.h diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/bounded_cursor_perf.cpp b/src/third_party/wiredtiger/test/cppsuite/tests/bounded_cursor_perf.cpp index efefcbef70e..cdd90b06c5d 100644 --- a/src/third_party/wiredtiger/test/cppsuite/tests/bounded_cursor_perf.cpp +++ b/src/third_party/wiredtiger/test/cppsuite/tests/bounded_cursor_perf.cpp @@ -26,7 +26,7 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include "src/component/execution_timer.cpp" +#include "src/util/execution_timer.cpp" #include "src/main/test.h" using namespace test_harness; diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/reverse_split.cpp b/src/third_party/wiredtiger/test/cppsuite/tests/reverse_split.cpp new file mode 100644 index 00000000000..ce9b458a14b --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/tests/reverse_split.cpp @@ -0,0 +1,113 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "src/common/constants.h" +#include "src/common/logger.h" +#include "src/main/test.h" + +namespace test_harness { +/* + * This test inserts data at the end of the collection and truncates off from the start of the + * collection. In doing so pages at the start of the tree are gradually emptied while pages are + * added at the end of the tree. This means the test frequently executes the reverse split path. + */ +class reverse_split : public test { + public: + reverse_split(test_args &args) : test(args) + { + /* + * Add split timing stresses to the conn_open config if the user supplied configuration is + * empty. It's easier doing that than figuring out whether they already specified the same + * configuration and then adding ours on the end. + */ + if (args.wt_open_config.empty()) { + std::string stress; + if (random_generator::instance().generate_bool()) + stress = "timing_stress_for_test=[split_3]"; + else + stress = "timing_stress_for_test=[split_4]"; + logger::log_msg(LOG_WARN, "Adding config to WiredTiger open: " + stress); + args.wt_open_config = stress; + } + init_operation_tracker(); + } + + /* Remove operation simulates burst truncates. */ + void + remove_operation(thread_worker *tc) override + { + logger::log_msg( + LOG_INFO, type_string(tc->type) + " thread {" + std::to_string(tc->id) + "} commencing."); + /* Must have unique collection for each thread. */ + testutil_assert(tc->db.get_collection_count() == tc->thread_count); + collection &coll = tc->db.get_collection(tc->id); + scoped_cursor write_cursor = std::move(tc->session.open_scoped_cursor(coll.name)); + + while (tc->running()) { + /* + * Synchronize across our truncate threads so they don't diverge over time. This results + * in a more bursty truncation workload. All threads truncate at roughly the same time + * which means in theory more eviction, or reverse splits happen at the same time. + */ + tc->sync(); + testutil_check(write_cursor->reset(write_cursor.get())); + tc->txn.begin(); + int ret = write_cursor->next(write_cursor.get()); + if (ret != 0) { + tc->txn.rollback(); + continue; + } + const char *key; + testutil_check(write_cursor->get_key(write_cursor.get(), &key)); + const std::string key_str(key); + uint64_t min_key_id = std::stoi(key_str); + uint64_t key_count = coll.get_key_count(); + /* Truncate up to 83% of the range. */ + uint64_t end_key_id = random_generator::instance().generate_integer<uint64_t>( + min_key_id, min_key_id + ((key_count - min_key_id) / 1.2)); + std::string end_key = tc->pad_string(std::to_string(end_key_id), tc->key_size); + /* If we generate an invalid range or our truncate fails rollback the transaction. */ + if (min_key_id == end_key_id || !tc->truncate(coll.id, key_str, end_key, "")) { + tc->txn.rollback(); + continue; + } + if (tc->txn.commit()) + logger::log_msg(LOG_TRACE, + "thread {" + std::to_string(tc->id) + "} committed truncation of " + + std::to_string(end_key_id - min_key_id) + " records."); + else + logger::log_msg(LOG_WARN, + "thread {" + std::to_string(tc->id) + "} failed to commit truncation of " + + std::to_string(end_key_id - min_key_id) + " records."); + tc->sleep(); + } + /* Make sure the last transaction is rolled back now the work is finished. */ + tc->txn.try_rollback(); + } +}; +} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/run.cpp b/src/third_party/wiredtiger/test/cppsuite/tests/run.cpp index 4306bd8722e..6ca0360deaa 100755 --- a/src/third_party/wiredtiger/test/cppsuite/tests/run.cpp +++ b/src/third_party/wiredtiger/test/cppsuite/tests/run.cpp @@ -34,14 +34,15 @@ #include "src/main/test.h" #include "bounded_cursor_perf.cpp" +#include "bounded_cursor_prefix_indices.cpp" +#include "bounded_cursor_prefix_search_near.cpp" +#include "bounded_cursor_prefix_stat.cpp" +#include "bounded_cursor_stress.cpp" #include "burst_inserts.cpp" #include "cache_resize.cpp" -#include "bounded_cursor_stress.cpp" -#include "bounded_cursor_prefix_stat.cpp" -#include "bounded_cursor_prefix_search_near.cpp" -#include "bounded_cursor_prefix_indices.cpp" #include "hs_cleanup.cpp" #include "operations_test.cpp" +#include "reverse_split.cpp" #include "search_near_01.cpp" #include "search_near_02.cpp" #include "search_near_03.cpp" @@ -139,10 +140,6 @@ run_test(const std::string &test_name, const std::string &config, const std::str if (test_name == "bounded_cursor_perf") bounded_cursor_perf(args).run(); - else if (test_name == "burst_inserts") - burst_inserts(args).run(); - else if (test_name == "cache_resize") - cache_resize(args).run(); else if (test_name == "bounded_cursor_prefix_indices") bounded_cursor_prefix_indices(args).run(); else if (test_name == "bounded_cursor_prefix_search_near") @@ -151,10 +148,16 @@ run_test(const std::string &test_name, const std::string &config, const std::str bounded_cursor_prefix_stat(args).run(); else if (test_name == "bounded_cursor_stress") bounded_cursor_stress(args).run(); + else if (test_name == "burst_inserts") + burst_inserts(args).run(); + else if (test_name == "cache_resize") + cache_resize(args).run(); else if (test_name == "hs_cleanup") hs_cleanup(args).run(); else if (test_name == "operations_test") operations_test(args).run(); + else if (test_name == "reverse_split") + reverse_split(args).run(); else if (test_name == "search_near_01") search_near_01(args).run(); else if (test_name == "search_near_02") @@ -186,7 +189,7 @@ main(int argc, char *argv[]) std::string cfg, config_filename, current_cfg, current_test_name, home, test_name, wt_open_config; int64_t error_code = 0; - const std::vector<std::string> all_tests = {"bounded_cursor_perf", + const std::vector<std::string> all_tests = {"reverse_split", "bounded_cursor_perf", "bounded_cursor_prefix_indices", "bounded_cursor_prefix_search_near", "bounded_cursor_prefix_stat", "bounded_cursor_stress", "burst_inserts", "cache_resize", "hs_cleanup", "operations_test", "search_near_01", "search_near_02", "search_near_03", diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index 07c2d2653b7..af976979d8e 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -1333,6 +1333,18 @@ tasks: test_config_filename: configs/bounded_cursor_prefix_indices_default.txt test_name: bounded_cursor_prefix_indices + - name: cppsuite-reverse-split-default + tags: ["pull_request"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "cppsuite test" + vars: + test_config: debug_mode=(cursor_copy=true) + test_config_filename: configs/reverse_split_default.txt + test_name: reverse_split + - name: cppsuite-operations-test-stress depends_on: - name: compile @@ -1454,6 +1466,7 @@ tasks: test_config_filename: configs/search_near_03_stress.txt test_name: search_near_03 + # This is a perf test and as such doesn't run under the stress test tag. - name: cppsuite-bounded-cursor-perf-stress depends_on: - name: compile @@ -1464,6 +1477,17 @@ tasks: test_config_filename: configs/bounded_cursor_perf_stress.txt test_name: bounded_cursor_perf + - name: cppsuite-reverse-split-stress + tags: ["cppsuite-stress-test"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "cppsuite test" + vars: + test_config_filename: configs/reverse_split_stress.txt + test_name: reverse_split + - name: cppsuite-operations-test-stress-nonstandalone depends_on: - name: compile-nonstandalone @@ -1607,6 +1631,19 @@ tasks: test_config_filename: configs/search_near_03_stress.txt test_name: search_near_03 + - name: cppsuite-reverse-split-stress-nonstandalone + tags: ["cppsuite-stress-test-nonstandalone"] + depends_on: + - name: compile-nonstandalone + commands: + - func: "fetch artifacts" + vars: + dependent_task: compile-nonstandalone + - func: "cppsuite test" + vars: + test_config_filename: configs/reverse_split_stress.txt + test_name: reverse_split + # End of cppsuite test tasks. # Start of csuite test tasks |