diff options
Diffstat (limited to 'src/third_party/wiredtiger/test/cppsuite/tests')
5 files changed, 887 insertions, 10 deletions
diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/csuite_style_example_test.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/csuite_style_example_test.cxx new file mode 100644 index 00000000000..d0059880446 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/tests/csuite_style_example_test.cxx @@ -0,0 +1,169 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/* + * This file provides an example of how to create a test in C++ using a few features from the + * framework if any. This file can be used as a template for quick testing and/or when stress + * testing is not required. For any stress testing, it is encouraged to use the framework, see + * example_test.cxx and create_script.sh. + */ + +#include "test_harness/connection_manager.h" +#include "test_harness/thread_manager.h" +#include "test_harness/util/api_const.h" +#include "test_harness/util/logger.h" +#include "test_harness/workload/random_generator.h" + +extern "C" { +#include "wiredtiger.h" +} + +using namespace test_harness; + +bool do_inserts = false; +bool do_reads = false; + +void +insert_op(WT_CURSOR *cursor, int key_size, int value_size) +{ + logger::log_msg(LOG_INFO, "called insert_op"); + + /* Insert random data. */ + std::string key, value; + while (do_inserts) { + key = random_generator::instance().generate_random_string(key_size); + value = random_generator::instance().generate_random_string(value_size); + cursor->set_key(cursor, key.c_str()); + cursor->set_value(cursor, value.c_str()); + testutil_check(cursor->insert(cursor)); + } +} + +void +read_op(WT_CURSOR *cursor, int key_size) +{ + logger::log_msg(LOG_INFO, "called read_op"); + + /* Read random data. */ + std::string key; + while (do_reads) { + key = random_generator::instance().generate_random_string(key_size); + cursor->set_key(cursor, key.c_str()); + cursor->search(cursor); + } +} + +int +main(int argc, char *argv[]) +{ + /* Set the program name for error messages. */ + const std::string progname = testutil_set_progname(argv); + + /* Set the tracing level for the logger component. */ + logger::trace_level = LOG_INFO; + + /* Printing some messages. */ + logger::log_msg(LOG_INFO, "Starting " + progname); + logger::log_msg(LOG_ERROR, "This could be an error."); + + /* Create a connection, set the cache size and specify the home directory. */ + const std::string conn_config = std::string(CONNECTION_CREATE) + ",cache_size=500MB"; + const std::string home_dir = std::string(DEFAULT_DIR) + '_' + progname; + connection_manager::instance().create(conn_config, home_dir); + WT_CONNECTION *conn = connection_manager::instance().get_connection(); + + /* Open different sessions. */ + WT_SESSION *insert_session, *read_session; + testutil_check(conn->open_session(conn, nullptr, nullptr, &insert_session)); + testutil_check(conn->open_session(conn, nullptr, nullptr, &read_session)); + + /* Create a collection. */ + const std::string collection_name = "table:my_collection"; + testutil_check( + insert_session->create(insert_session, collection_name.c_str(), DEFAULT_FRAMEWORK_SCHEMA)); + + /* Open different cursors. */ + WT_CURSOR *insert_cursor, *read_cursor; + const std::string cursor_config = ""; + testutil_check(insert_session->open_cursor( + insert_session, collection_name.c_str(), nullptr, cursor_config.c_str(), &insert_cursor)); + testutil_check(read_session->open_cursor( + read_session, collection_name.c_str(), nullptr, cursor_config.c_str(), &read_cursor)); + + /* Store cursors. */ + std::vector<WT_CURSOR *> cursors; + cursors.push_back(insert_cursor); + cursors.push_back(read_cursor); + + /* Insert some data. */ + std::string key = "a"; + const std::string value = "b"; + insert_cursor->set_key(insert_cursor, key.c_str()); + insert_cursor->set_value(insert_cursor, value.c_str()); + testutil_check(insert_cursor->insert(insert_cursor)); + + /* Read some data. */ + key = "b"; + read_cursor->set_key(read_cursor, key.c_str()); + testutil_assert(read_cursor->search(read_cursor) == WT_NOTFOUND); + + key = "a"; + read_cursor->set_key(read_cursor, key.c_str()); + testutil_check(read_cursor->search(read_cursor)); + + /* Create a thread manager and spawn some threads that will work. */ + thread_manager t; + int key_size = 1, value_size = 2; + + do_inserts = true; + t.add_thread(insert_op, insert_cursor, key_size, value_size); + + do_reads = true; + t.add_thread(read_op, read_cursor, key_size); + + /* Sleep for the test duration. */ + int test_duration_s = 5; + std::this_thread::sleep_for(std::chrono::seconds(test_duration_s)); + + /* Stop the threads. */ + do_reads = false; + do_inserts = false; + t.join(); + + /* Close cursors. */ + for (auto c : cursors) + testutil_check(c->close(c)); + + /* Close the connection. */ + connection_manager::instance().close(); + + /* Another message. */ + logger::log_msg(LOG_INFO, "End of test."); + + return (0); +} diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/example_test.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/example_test.cxx index 5a1996b45a1..4b49ad2b148 100644 --- a/src/third_party/wiredtiger/test/cppsuite/tests/example_test.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/tests/example_test.cxx @@ -37,6 +37,13 @@ class example_test : public test_harness::test { example_test(const test_harness::test_args &args) : test(args) {} void + run() + { + /* You can remove the call to the base class to fully customized your test. */ + test::run(); + } + + void populate(test_harness::database &, test_harness::timestamp_manager *, test_harness::configuration *, test_harness::workload_tracking *) override final { diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx index 0c231e6568d..627921a9aa4 100755 --- a/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx @@ -34,9 +34,11 @@ #include "test_harness/test.h" #include "base_test.cxx" -#include "example_test.cxx" #include "burst_inserts.cxx" +#include "example_test.cxx" #include "hs_cleanup.cxx" +#include "search_near_01.cxx" +#include "search_near_02.cxx" std::string parse_configuration_from_file(const std::string &filename) @@ -72,7 +74,7 @@ print_help() std::cout << "\trun -C [WIREDTIGER_OPEN_CONFIGURATION]" << std::endl; std::cout << "\trun -c [TEST_FRAMEWORK_CONFIGURATION]" << std::endl; std::cout << "\trun -f [FILE]" << std::endl; - std::cout << "\trun -l [TRACEL_LEVEL]" << std::endl; + std::cout << "\trun -l [TRACE_LEVEL]" << std::endl; std::cout << "\trun -t [TEST_NAME]" << std::endl; std::cout << std::endl; std::cout << "DESCRIPTION" << std::endl; @@ -113,10 +115,14 @@ run_test(const std::string &test_name, const std::string &config, const std::str base_test(test_harness::test_args{config, test_name, wt_open_config}).run(); else if (test_name == "example_test") example_test(test_harness::test_args{config, test_name, wt_open_config}).run(); + else if (test_name == "search_near_02") + search_near_02(test_harness::test_args{config, test_name, wt_open_config}).run(); else if (test_name == "hs_cleanup") hs_cleanup(test_harness::test_args{config, test_name, wt_open_config}).run(); else if (test_name == "burst_inserts") burst_inserts(test_harness::test_args{config, test_name, wt_open_config}).run(); + else if (test_name == "search_near_01") + search_near_01(test_harness::test_args{config, test_name, wt_open_config}).run(); else { test_harness::logger::log_msg(LOG_ERROR, "Test not found: " + test_name); error_code = -1; @@ -139,8 +145,8 @@ main(int argc, char *argv[]) { std::string cfg, config_filename, current_cfg, current_test_name, test_name, wt_open_config; int64_t error_code = 0; - const std::vector<std::string> all_tests = { - "example_test", "burst_inserts", "hs_cleanup", "base_test"}; + const std::vector<std::string> all_tests = {"base_test", "burst_inserts", "example_test", + "hs_cleanup", "search_near_01", "search_near_02"}; /* Set the program name for error messages. */ (void)testutil_set_progname(argv); @@ -219,12 +225,20 @@ main(int argc, char *argv[]) } } else { current_test_name = test_name; - /* Configuration parsing. */ - if (!config_filename.empty()) - cfg = parse_configuration_from_file(config_filename); - else if (cfg.empty()) - cfg = parse_configuration_from_file(get_default_config_path(current_test_name)); - error_code = run_test(current_test_name, cfg, wt_open_config); + /* Check the test exists. */ + if (std::find(all_tests.begin(), all_tests.end(), current_test_name) == + all_tests.end()) { + test_harness::logger::log_msg( + LOG_ERROR, "The test " + current_test_name + " was not found."); + error_code = -1; + } else { + /* Configuration parsing. */ + if (!config_filename.empty()) + cfg = parse_configuration_from_file(config_filename); + else if (cfg.empty()) + cfg = parse_configuration_from_file(get_default_config_path(current_test_name)); + error_code = run_test(current_test_name, cfg, wt_open_config); + } } if (error_code != 0) diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/search_near_01.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/search_near_01.cxx new file mode 100644 index 00000000000..abacf3dc196 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/tests/search_near_01.cxx @@ -0,0 +1,262 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "test_harness/util/api_const.h" +#include "test_harness/workload/random_generator.h" +#include "test_harness/workload/thread_context.h" +#include "test_harness/test.h" +#include "test_harness/thread_manager.h" + +using namespace test_harness; +/* + * In this test, we want to verify that search_near with prefix enabled only traverses the portion + * of the tree that follows the prefix portion of the search key. The test is composed of a populate + * phase followed by a read phase. The populate phase will insert a set of random generated keys + * with a prefix of aaa -> zzz. The read phase will continuously perform prefix search near calls, + * and validate that the number of entries traversed is within bounds of the search key. + */ +class search_near_01 : public test_harness::test { + uint64_t keys_per_prefix = 0; + uint64_t srchkey_len = 0; + const std::string ALPHABET{"abcdefghijklmnopqrstuvwxyz"}; + const uint64_t PREFIX_KEY_LEN = 3; + + static void + populate_worker(thread_context *tc, const std::string &ALPHABET, uint64_t PREFIX_KEY_LEN) + { + logger::log_msg(LOG_INFO, "Populate with thread id: " + std::to_string(tc->id)); + + std::string prefix_key; + uint64_t collections_per_thread = tc->collection_count; + const uint64_t MAX_ROLLBACKS = 100; + uint32_t rollback_retries = 0; + int cmpp; + + /* + * Generate a table of data with prefix keys aaa -> zzz. We have 26 threads from ids + * starting from 0 to 26. Each populate thread will insert separate prefix keys based on the + * id. + */ + for (int64_t i = 0; i < collections_per_thread; ++i) { + collection &coll = tc->db.get_collection(i); + scoped_cursor cursor = tc->session.open_scoped_cursor(coll.name.c_str()); + for (uint64_t j = 0; j < ALPHABET.size(); ++j) { + for (uint64_t k = 0; k < ALPHABET.size(); ++k) { + for (uint64_t count = 0; count < tc->key_count; ++count) { + tc->transaction.begin(); + /* + * Generate the prefix key, and append a random generated key string based + * on the key size configuration. + */ + prefix_key = {ALPHABET.at(tc->id), ALPHABET.at(j), ALPHABET.at(k)}; + prefix_key += random_generator::instance().generate_random_string( + tc->key_size - PREFIX_KEY_LEN); + if (!tc->insert(cursor, coll.id, prefix_key)) { + testutil_assert(rollback_retries < MAX_ROLLBACKS); + /* We failed to insert, rollback our transaction and retry. */ + tc->transaction.rollback(); + ++rollback_retries; + --count; + } else { + /* Commit txn at commit timestamp 100. */ + tc->transaction.commit( + "commit_timestamp=" + tc->tsm->decimal_to_hex(100)); + rollback_retries = 0; + } + } + } + } + } + } + + public: + search_near_01(const test_harness::test_args &args) : test(args) {} + + void + populate(test_harness::database &database, test_harness::timestamp_manager *tsm, + test_harness::configuration *config, test_harness::workload_tracking *tracking) override final + { + uint64_t collection_count, key_size; + std::vector<thread_context *> workers; + thread_manager tm; + + /* Validate our config. */ + collection_count = config->get_int(COLLECTION_COUNT); + keys_per_prefix = config->get_int(KEY_COUNT_PER_COLLECTION); + key_size = config->get_int(KEY_SIZE); + testutil_assert(collection_count > 0); + testutil_assert(keys_per_prefix > 0); + /* Check the prefix length is not greater than the key size. */ + testutil_assert(key_size >= PREFIX_KEY_LEN); + + logger::log_msg(LOG_INFO, + "Populate configuration with key size: " + std::to_string(key_size) + + " key count: " + std::to_string(keys_per_prefix) + + " number of collections: " + std::to_string(collection_count)); + + /* Create n collections as per the configuration. */ + for (uint64_t i = 0; i < collection_count; ++i) + /* + * The database model will call into the API and create the collection, with its own + * session. + */ + database.add_collection(); + + /* Spawn 26 threads to populate the database. */ + for (uint64_t i = 0; i < ALPHABET.size(); ++i) { + thread_context *tc = new thread_context(i, thread_type::INSERT, config, + connection_manager::instance().create_session(), tsm, tracking, database); + workers.push_back(tc); + tm.add_thread(populate_worker, tc, ALPHABET, PREFIX_KEY_LEN); + } + + /* Wait for our populate threads to finish and then join them. */ + logger::log_msg(LOG_INFO, "Populate: waiting for threads to complete."); + tm.join(); + + /* Cleanup our workers. */ + for (auto &it : workers) { + delete it; + it = nullptr; + } + + /* Force evict all the populated keys in all of the collections. */ + int cmpp; + scoped_session session = connection_manager::instance().create_session(); + for (uint64_t count = 0; count < collection_count; ++count) { + collection &coll = database.get_collection(count); + scoped_cursor evict_cursor = + session.open_scoped_cursor(coll.name.c_str(), "debug=(release_evict=true)"); + + for (uint64_t i = 0; i < ALPHABET.size(); ++i) { + for (uint64_t j = 0; j < ALPHABET.size(); ++j) { + for (uint64_t k = 0; k < ALPHABET.size(); ++k) { + std::string key = {ALPHABET.at(i), ALPHABET.at(j), ALPHABET.at(k)}; + evict_cursor->set_key(evict_cursor.get(), key.c_str()); + evict_cursor->search_near(evict_cursor.get(), &cmpp); + testutil_check(evict_cursor->reset(evict_cursor.get())); + } + } + } + } + srchkey_len = + random_generator::instance().generate_integer(static_cast<uint64_t>(1), PREFIX_KEY_LEN); + logger::log_msg(LOG_INFO, "Populate: finished."); + } + + void + read_operation(test_harness::thread_context *tc) override final + { + /* Make sure that thread statistics cursor is null before we open it. */ + testutil_assert(tc->stat_cursor.get() == nullptr); + logger::log_msg( + LOG_INFO, type_string(tc->type) + " thread {" + std::to_string(tc->id) + "} commencing."); + std::map<uint64_t, scoped_cursor> cursors; + tc->stat_cursor = tc->session.open_scoped_cursor(STATISTICS_URI); + std::string srch_key; + int64_t entries_stat, prefix_stat, prev_entries_stat, prev_prefix_stat, expected_entries; + int cmpp; + + cmpp = 0; + prev_entries_stat = 0; + prev_prefix_stat = 0; + + /* + * The number of expected entries is calculated to account for the maximum allowed entries + * per search near function call. The key we search near can be different in length, which + * will increase the number of entries search by a factor of 26. + */ + expected_entries = tc->thread_count * keys_per_prefix * 2 * + pow(ALPHABET.size(), PREFIX_KEY_LEN - srchkey_len); + + /* + * Read at timestamp 10, so that no keys are visible to this transaction. This allows prefix + * search near to early exit out of it's prefix range when it's trying to search for a + * visible key in the tree. + */ + tc->transaction.begin("read_timestamp=" + tc->tsm->decimal_to_hex(10)); + while (tc->running()) { + + /* Get a collection and find a cached cursor. */ + collection &coll = tc->db.get_random_collection(); + if (cursors.find(coll.id) == cursors.end()) { + scoped_cursor cursor = tc->session.open_scoped_cursor(coll.name.c_str()); + cursor->reconfigure(cursor.get(), "prefix_search=true"); + cursors.emplace(coll.id, std::move(cursor)); + } + + /* Generate search prefix key of random length between a -> zzz. */ + srch_key = random_generator::instance().generate_random_string( + srchkey_len, characters_type::ALPHABET); + logger::log_msg(LOG_INFO, + "Read thread {" + std::to_string(tc->id) + + "} performing prefix search near with key: " + srch_key); + + /* Do a second lookup now that we know it exists. */ + auto &cursor = cursors[coll.id]; + if (tc->transaction.active()) { + runtime_monitor::get_stat( + tc->stat_cursor, WT_STAT_CONN_CURSOR_NEXT_SKIP_LT_100, &prev_entries_stat); + runtime_monitor::get_stat(tc->stat_cursor, + WT_STAT_CONN_CURSOR_SEARCH_NEAR_PREFIX_FAST_PATHS, &prev_prefix_stat); + + cursor->set_key(cursor.get(), srch_key.c_str()); + testutil_assert(cursor->search_near(cursor.get(), &cmpp) == WT_NOTFOUND); + + runtime_monitor::get_stat( + tc->stat_cursor, WT_STAT_CONN_CURSOR_NEXT_SKIP_LT_100, &entries_stat); + runtime_monitor::get_stat( + tc->stat_cursor, WT_STAT_CONN_CURSOR_SEARCH_NEAR_PREFIX_FAST_PATHS, &prefix_stat); + logger::log_msg(LOG_INFO, + "Read thread {" + std::to_string(tc->id) + + "} skipped entries: " + std::to_string(entries_stat - prev_entries_stat) + + " prefix fash path: " + std::to_string(prefix_stat - prev_prefix_stat)); + + /* + * It is possible that WiredTiger increments the entries skipped stat irrelevant to + * prefix search near. This is dependent on how many read threads are present in the + * test. Account for this by creating a small buffer using thread count. Assert that + * the number of expected entries is the upper limit which the prefix search near + * can traverse and the prefix fast path is incremented. + */ + testutil_assert( + (expected_entries + (2 * tc->thread_count)) >= entries_stat - prev_entries_stat); + testutil_assert(prefix_stat > prev_prefix_stat); + + tc->transaction.add_op(); + tc->sleep(); + } + /* Reset our cursor to avoid pinning content. */ + testutil_check(cursor->reset(cursor.get())); + } + tc->transaction.commit(); + /* Make sure the last transaction is rolled back now the work is finished. */ + if (tc->transaction.active()) + tc->transaction.rollback(); + } +}; diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/search_near_02.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/search_near_02.cxx new file mode 100644 index 00000000000..1df75a83bc0 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/tests/search_near_02.cxx @@ -0,0 +1,425 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "test_harness/test.h" +#include "test_harness/util/api_const.h" +#include "test_harness/workload/random_generator.h" + +using namespace test_harness; + +/* + * In this test, we want to verify search_near with prefix enabled returns the correct key. + * During the test duration: + * - N threads will keep inserting new random keys + * - M threads will execute search_near calls with prefix enabled using random prefixes as well. + * Each search_near call with prefix enabled is verified using the default search_near. + */ +class search_near_02 : public test_harness::test { + public: + search_near_02(const test_harness::test_args &args) : test(args) {} + + void + populate(test_harness::database &database, test_harness::timestamp_manager *, + test_harness::configuration *config, test_harness::workload_tracking *) override final + { + /* + * The populate phase only creates empty collections. The number of collections is defined + * in the configuration. + */ + int64_t collection_count = config->get_int(COLLECTION_COUNT); + + logger::log_msg( + LOG_INFO, "Populate: " + std::to_string(collection_count) + " creating collections."); + + for (uint64_t i = 0; i < collection_count; ++i) + database.add_collection(); + + logger::log_msg(LOG_INFO, "Populate: finished."); + } + + void + insert_operation(test_harness::thread_context *tc) override final + { + /* Each insert operation will insert new keys in the collections. */ + logger::log_msg( + LOG_INFO, type_string(tc->type) + " thread {" + std::to_string(tc->id) + "} commencing."); + + /* Helper struct which stores a pointer to a collection and a cursor associated with it. */ + struct collection_cursor { + collection_cursor(collection &coll, scoped_cursor &&cursor) + : coll(coll), cursor(std::move(cursor)) + { + } + collection &coll; + scoped_cursor cursor; + }; + + /* Collection cursor vector. */ + std::vector<collection_cursor> ccv; + int64_t collection_count = tc->db.get_collection_count(); + int64_t collections_per_thread = collection_count / tc->thread_count; + + /* Must have unique collections for each thread. */ + testutil_assert(collection_count % tc->thread_count == 0); + const uint64_t thread_offset = tc->id * collections_per_thread; + for (uint64_t i = thread_offset; + i < thread_offset + collections_per_thread && tc->running(); ++i) { + collection &coll = tc->db.get_collection(i); + scoped_cursor cursor = tc->session.open_scoped_cursor(coll.name.c_str()); + ccv.push_back({coll, std::move(cursor)}); + } + + std::string key; + uint64_t counter = 0; + + while (tc->running()) { + + auto &cc = ccv[counter]; + tc->transaction.begin(); + + while (tc->transaction.active() && tc->running()) { + + /* Generate a random key. */ + key = random_generator::instance().generate_random_string(tc->key_size); + + /* Insert a key value pair. */ + if (tc->insert(cc.cursor, cc.coll.id, key)) { + if (tc->transaction.can_commit()) + /* We are not checking the result of commit as it is not necessary. */ + tc->transaction.commit(); + } else { + tc->transaction.rollback(); + } + + /* Sleep the duration defined by the configuration. */ + tc->sleep(); + } + + /* Rollback any transaction that could not commit before the end of the test. */ + if (tc->transaction.active()) + tc->transaction.rollback(); + + /* Reset our cursor to avoid pinning content. */ + testutil_check(cc.cursor->reset(cc.cursor.get())); + if (++counter == ccv.size()) + counter = 0; + testutil_assert(counter < collections_per_thread); + } + } + + void + read_operation(test_harness::thread_context *tc) override final + { + /* + * Each read operation performs search_near calls with and without prefix enabled on random + * collections. Each prefix is randomly generated. The result of the seach_near call with + * prefix enabled is then validated using the search_near call without prefix enabled. + */ + logger::log_msg( + LOG_INFO, type_string(tc->type) + " thread {" + std::to_string(tc->id) + "} commencing."); + + const char *key_prefix; + int exact_prefix, ret; + int64_t prefix_size; + std::map<uint64_t, scoped_cursor> cursors; + std::string generated_prefix, key_prefix_str; + + while (tc->running()) { + /* Get a random collection to work on. */ + collection &coll = tc->db.get_random_collection(); + + /* Find a cached cursor or create one if none exists. */ + if (cursors.find(coll.id) == cursors.end()) { + cursors.emplace( + coll.id, std::move(tc->session.open_scoped_cursor(coll.name.c_str()))); + auto &cursor_prefix = cursors[coll.id]; + /* The cached cursors have the prefix configuration enabled. */ + testutil_check( + cursor_prefix.get()->reconfigure(cursor_prefix.get(), "prefix_search=true")); + } + + auto &cursor_prefix = cursors[coll.id]; + + /* + * Pick a random timestamp between the oldest and now. Get rid of the last 32 bits as + * they represent an increment for uniqueness. + */ + wt_timestamp_t ts = random_generator::instance().generate_integer( + (tc->tsm->get_oldest_ts() >> 32), (tc->tsm->get_next_ts() >> 32)); + /* Put back the timestamp in the correct format. */ + ts <<= 32; + + /* + * The oldest timestamp might move ahead and the reading timestamp might become invalid. + * To tackle this issue, we round the timestamp to the oldest timestamp value. + */ + tc->transaction.begin( + "roundup_timestamps=(read=true),read_timestamp=" + tc->tsm->decimal_to_hex(ts)); + + while (tc->transaction.active() && tc->running()) { + /* + * Generate a random prefix. For this, we start by generating a random size and then + * its value. + */ + prefix_size = random_generator::instance().generate_integer( + static_cast<int64_t>(1), tc->key_size); + generated_prefix = random_generator::instance().generate_random_string( + prefix_size, characters_type::ALPHABET); + + /* Call search near with the prefix cursor. */ + cursor_prefix->set_key(cursor_prefix.get(), generated_prefix.c_str()); + ret = cursor_prefix->search_near(cursor_prefix.get(), &exact_prefix); + testutil_assert(ret == 0 || ret == WT_NOTFOUND); + if (ret == 0) { + testutil_check(cursor_prefix->get_key(cursor_prefix.get(), &key_prefix)); + key_prefix_str = key_prefix; + } else { + key_prefix_str = ""; + } + + /* Open a cursor with the default configuration on the selected collection. */ + scoped_cursor cursor_default(tc->session.open_scoped_cursor(coll.name.c_str())); + + /* Verify the prefix search_near output using the default cursor. */ + validate_prefix_search_near( + ret, exact_prefix, key_prefix_str, cursor_default, generated_prefix); + + tc->transaction.add_op(); + tc->transaction.try_rollback(); + tc->sleep(); + } + testutil_check(cursor_prefix->reset(cursor_prefix.get())); + } + /* Roll back the last transaction if still active now the work is finished. */ + if (tc->transaction.active()) + tc->transaction.rollback(); + } + + private: + /* Validate prefix search_near call outputs using a cursor without prefix key enabled. */ + void + validate_prefix_search_near(int ret_prefix, int exact_prefix, const std::string &key_prefix, + scoped_cursor &cursor_default, const std::string &prefix) + { + /* Call search near with the default cursor using the given prefix. */ + int exact_default; + cursor_default->set_key(cursor_default.get(), prefix.c_str()); + int ret_default = cursor_default->search_near(cursor_default.get(), &exact_default); + + /* + * It is not possible to have a prefix search near call successful and the default search + * near call unsuccessful. + */ + testutil_assert( + ret_default == ret_prefix || (ret_default == 0 && ret_prefix == WT_NOTFOUND)); + + /* We only have to perform validation when the default search near call is successful. */ + if (ret_default == 0) { + /* Both calls are successful. */ + if (ret_prefix == 0) + validate_successful_calls( + ret_prefix, exact_prefix, key_prefix, cursor_default, exact_default, prefix); + /* The prefix search near call failed. */ + else + validate_unsuccessful_prefix_call(cursor_default, prefix, exact_default); + } + } + + /* + * Validate a successful prefix enabled search near call using a successful default search near + * call. + * The exact value set by the prefix search near call has to be either 0 or 1. Indeed, it cannot + * be -1 as the key needs to contain the prefix. + * - If it is 0, both search near calls should return the same outputs and both cursors should + * be positioned on the prefix we are looking for. + * - If it is 1, it will depend on the exact value set by the default search near call which can + * be -1 or 1. If it is -1, calling next on the default cursor should get us ti the key found by + * the prefix search near call. If it is 1, it means both search near calls have found the same + * key that is lexicographically greater than the prefix but still contains the prefix. + */ + void + validate_successful_calls(int ret_prefix, int exact_prefix, const std::string &key_prefix, + scoped_cursor &cursor_default, int exact_default, const std::string &prefix) + { + const char *k; + std::string k_str; + int ret; + + /* + * The prefix search near call cannot retrieve a key with a smaller value than the prefix we + * searched. + */ + testutil_assert(exact_prefix >= 0); + + /* The key at the prefix cursor should contain the prefix. */ + testutil_assert(key_prefix.substr(0, prefix.size()) == prefix); + + /* Retrieve the key the default cursor is pointing at. */ + const char *key_default; + testutil_check(cursor_default->get_key(cursor_default.get(), &key_default)); + std::string key_default_str = key_default; + + logger::log_msg(LOG_TRACE, + "search_near (normal) exact " + std::to_string(exact_default) + " key " + key_default); + logger::log_msg(LOG_TRACE, + "search_near (prefix) exact " + std::to_string(exact_prefix) + " key " + key_prefix); + + /* Example: */ + /* keys: a, bb, bba. */ + /* Only bb is not visible. */ + /* Default search_near(bb) returns a, exact < 0. */ + /* Prefix search_near(bb) returns bba, exact > 0. */ + if (exact_default < 0) { + /* The key at the default cursor should not contain the prefix. */ + testutil_assert((key_default_str.substr(0, prefix.size()) != prefix)); + + /* + * The prefix cursor should be positioned at a key lexicographically greater than the + * prefix. + */ + testutil_assert(exact_prefix > 0); + + /* + * The next key of the default cursor should be equal to the key pointed by the prefix + * cursor. + */ + testutil_assert(cursor_default->next(cursor_default.get()) == 0); + testutil_check(cursor_default->get_key(cursor_default.get(), &k)); + testutil_assert(k == key_prefix); + } + /* Example: */ + /* keys: a, bb, bba */ + /* Case 1: all keys are visible. */ + /* Default search_near(bb) returns bb, exact = 0 */ + /* Prefix search_near(bb) returns bb, exact = 0 */ + /* Case 2: only bb is not visible. */ + /* Default search_near(bb) returns bba, exact > 0. */ + /* Prefix search_near(bb) returns bba, exact > 0. */ + else { + /* Both cursors should be pointing at the same key. */ + testutil_assert(exact_prefix == exact_default); + testutil_assert(key_default_str == key_prefix); + /* Both cursors should have found the exact key. */ + if (exact_default == 0) + testutil_assert(key_default_str == prefix); + /* Both cursors have found a key that is lexicographically greater than the prefix. */ + else + testutil_assert(key_default_str != prefix); + } + } + + /* + * Validate that no keys with the prefix used for the search have been found. + * To validate this, we can use the exact value set by the default search near. Since the prefix + * search near failed, the exact value set by the default search near call has to be either -1 + * or 1: + * - If it is -1, we need to check the next key, if it exists, is lexicographically greater than + * the prefix we looked for. + * - If it is 1, we need to check the previous keys, if it exists, if lexicographically smaller + * than the prefix we looked for. + */ + void + validate_unsuccessful_prefix_call( + scoped_cursor &cursor_default, const std::string &prefix, int exact_default) + { + int ret; + const char *k; + std::string k_str; + + /* + * The exact value from the default search near call cannot be 0, otherwise the prefix + * search near should be successful too. + */ + testutil_assert(exact_default != 0); + + /* Retrieve the key at the default cursor. */ + const char *key_default; + testutil_check(cursor_default->get_key(cursor_default.get(), &key_default)); + std::string key_default_str = key_default; + + /* The key at the default cursor should not contain the prefix. */ + testutil_assert(key_default_str.substr(0, prefix.size()) != prefix); + + /* Example: */ + /* keys: a, bb, bbb. */ + /* All keys are visible. */ + /* Default search_near(bba) returns bb, exact < 0. */ + /* Prefix search_near(bba) returns WT_NOTFOUND. */ + if (exact_default < 0) { + /* + * The current key of the default cursor should be lexicographically smaller than the + * prefix. + */ + testutil_assert(std::lexicographical_compare( + key_default_str.begin(), key_default_str.end(), prefix.begin(), prefix.end())); + + /* + * The next key of the default cursor should be lexicographically greater than the + * prefix if it exists. + */ + ret = cursor_default->next(cursor_default.get()); + if (ret == 0) { + testutil_check(cursor_default->get_key(cursor_default.get(), &k)); + k_str = k; + testutil_assert(!std::lexicographical_compare( + k_str.begin(), k_str.end(), prefix.begin(), prefix.end())); + } else { + /* End of the table. */ + testutil_assert(ret == WT_NOTFOUND); + } + } + /* Example: */ + /* keys: a, bb, bbb. */ + /* All keys are visible. */ + /* Default search_near(bba) returns bbb, exact > 0. */ + /* Prefix search_near(bba) returns WT_NOTFOUND. */ + else { + /* + * The current key of the default cursor should be lexicographically greater than the + * prefix. + */ + testutil_assert(!std::lexicographical_compare( + key_default_str.begin(), key_default_str.end(), prefix.begin(), prefix.end())); + + /* + * The next key of the default cursor should be lexicographically smaller than the + * prefix if it exists. + */ + ret = cursor_default->prev(cursor_default.get()); + if (ret == 0) { + testutil_check(cursor_default->get_key(cursor_default.get(), &k)); + k_str = k; + testutil_assert(std::lexicographical_compare( + k_str.begin(), k_str.end(), prefix.begin(), prefix.end())); + } else { + /* End of the table. */ + testutil_assert(ret == WT_NOTFOUND); + } + } + } +}; |