diff options
Diffstat (limited to 'src/third_party/wiredtiger/test')
49 files changed, 1945 insertions, 287 deletions
diff --git a/src/third_party/wiredtiger/test/checkpoint/workers.c b/src/third_party/wiredtiger/test/checkpoint/workers.c index de2798413ee..3c9313c4c99 100644 --- a/src/third_party/wiredtiger/test/checkpoint/workers.c +++ b/src/third_party/wiredtiger/test/checkpoint/workers.c @@ -332,7 +332,7 @@ real_worker(void) /* If we have specified to run with mix mode deletes we need to do it in it's own txn. */ if (g.use_timestamps && g.mixed_mode_deletes && new_txn && __wt_random(&rnd) % 72 == 0) { new_txn = false; - for (j = 0; ret == 0 && j < g.ntables; j++) { + for (j = 0; j < g.ntables; j++) { ret = worker_mm_delete(cursors[j], keyno); if (ret == WT_ROLLBACK || ret == WT_PREPARE_CONFLICT) break; diff --git a/src/third_party/wiredtiger/test/cppsuite/Makefile.am b/src/third_party/wiredtiger/test/cppsuite/Makefile.am index 82c30442482..5e896ce2e04 100644 --- a/src/third_party/wiredtiger/test/cppsuite/Makefile.am +++ b/src/third_party/wiredtiger/test/cppsuite/Makefile.am @@ -11,7 +11,7 @@ all: all_TESTS= noinst_PROGRAMS= -run_SOURCES = test_harness/core/component.cxx \ +test_harness = test_harness/core/component.cxx \ test_harness/core/configuration.cxx \ test_harness/core/throttle.cxx \ test_harness/util/logger.cxx \ @@ -28,9 +28,15 @@ run_SOURCES = test_harness/core/component.cxx \ test_harness/test.cxx \ test_harness/thread_manager.cxx \ test_harness/timestamp_manager.cxx \ - test_harness/workload_generator.cxx \ - tests/run.cxx + test_harness/workload_generator.cxx +# If you prefer to not use the run binary you can add a test via this +# mechanism but it is generally frowned upon. +csuite_style_example_test_SOURCES = $(test_harness) tests/csuite_style_example_test.cxx +noinst_PROGRAMS += csuite_style_example_test +all_TESTS += csuite_style_example_test + +run_SOURCES = $(test_harness) tests/run.cxx noinst_PROGRAMS += run all_TESTS += run diff --git a/src/third_party/wiredtiger/test/cppsuite/configs/search_near_01_default.txt b/src/third_party/wiredtiger/test/cppsuite/configs/search_near_01_default.txt new file mode 100644 index 00000000000..b3b3d6c668a --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/configs/search_near_01_default.txt @@ -0,0 +1,22 @@ +# Configuration for search_near_01. +# The test will generate key_count_per_collection number of keys for each prefix in aaa -> zzz. +# This config will have a 3 minute duration, with 3 tables of an entry for each prefix. +duration_seconds=180, +cache_size_mb=1000, +timestamp_manager= +( + enabled=false, +), +workload_generator= +( + populate_config= + ( + collection_count=3, + key_count_per_collection=1, + key_size=5, + ), + read_config= + ( + thread_count=10 + ) +),
\ No newline at end of file diff --git a/src/third_party/wiredtiger/test/cppsuite/configs/search_near_02_default.txt b/src/third_party/wiredtiger/test/cppsuite/configs/search_near_02_default.txt new file mode 100644 index 00000000000..34ab79e1980 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/configs/search_near_02_default.txt @@ -0,0 +1,31 @@ +# Configuration for search_near_02. +# The configuration creates: +# - threads that continuously insert random keys and values. +# - threads that continuously perform prefix search_near calls on random keys. +duration_seconds=180, +cache_size_mb=500, +timestamp_manager= +( + # This will let us randomly pick a read timestamp in a bigger range to trigger visibility + # checks. + oldest_lag=10, +), +workload_generator= +( + populate_config= + ( + collection_count=10, + ), + insert_config= + ( + key_size=5, + op_rate=100ms, + thread_count=10 + ), + read_config= + ( + key_size=5, + op_rate=250ms, + thread_count=10 + ) +) diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/connection_manager.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/connection_manager.cxx index 6ff134c7c96..8a8b75b7b8f 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/connection_manager.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/connection_manager.cxx @@ -80,6 +80,12 @@ connection_manager::create_session() return (session); } +WT_CONNECTION * +connection_manager::get_connection() +{ + return (_conn); +} + /* * set_timestamp calls into the connection API in a thread safe manner to set global timestamps. */ diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/connection_manager.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/connection_manager.h index c6245160df1..a5d44903717 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/connection_manager.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/connection_manager.h @@ -30,8 +30,12 @@ #define CONN_API_H /* Following definitions are required in order to use printing format specifiers in C++. */ +#ifndef __STDC_LIMIT_MACROS #define __STDC_LIMIT_MACROS +#endif +#ifndef __STDC_FORMAT_MACROS #define __STDC_FORMAT_MACROS +#endif #include <mutex> @@ -60,6 +64,8 @@ class connection_manager { void create(const std::string &config, const std::string &home = DEFAULT_DIR); scoped_session create_session(); + WT_CONNECTION *get_connection(); + /* * set_timestamp calls into the connection API in a thread safe manner to set global timestamps. */ diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.cxx index ebb6520b465..0e454a4f4f0 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.cxx @@ -36,16 +36,6 @@ namespace test_harness { /* Static methods implementation. */ -static void -get_stat(scoped_cursor &cursor, int stat_field, int64_t *valuep) -{ - const char *desc, *pvalue; - cursor->set_key(cursor.get(), stat_field); - testutil_check(cursor->search(cursor.get())); - testutil_check(cursor->get_value(cursor.get(), &desc, &pvalue, valuep)); - testutil_check(cursor->reset(cursor.get())); -} - static std::string collection_name_to_file_name(const std::string &collection_name) { @@ -100,9 +90,9 @@ cache_limit_statistic::check(scoped_cursor &cursor) int64_t cache_bytes_image, cache_bytes_other, cache_bytes_max; double use_percent; /* Three statistics are required to compute cache use percentage. */ - get_stat(cursor, WT_STAT_CONN_CACHE_BYTES_IMAGE, &cache_bytes_image); - get_stat(cursor, WT_STAT_CONN_CACHE_BYTES_OTHER, &cache_bytes_other); - get_stat(cursor, WT_STAT_CONN_CACHE_BYTES_MAX, &cache_bytes_max); + runtime_monitor::get_stat(cursor, WT_STAT_CONN_CACHE_BYTES_IMAGE, &cache_bytes_image); + runtime_monitor::get_stat(cursor, WT_STAT_CONN_CACHE_BYTES_OTHER, &cache_bytes_other); + runtime_monitor::get_stat(cursor, WT_STAT_CONN_CACHE_BYTES_MAX, &cache_bytes_max); /* * Assert that we never exceed our configured limit for cache usage. Add 0.0 to avoid floating * point conversion errors. @@ -222,7 +212,7 @@ postrun_statistic_check::check_stat(scoped_cursor &cursor, const postrun_statist int64_t stat_value; testutil_assert(cursor.get() != nullptr); - get_stat(cursor, stat.field, &stat_value); + runtime_monitor::get_stat(cursor, stat.field, &stat_value); if (stat_value < stat.min_limit || stat_value > stat.max_limit) { const std::string error_string = "runtime_monitor: Postrun stat \"" + stat.name + "\" was outside of the specified limits. Min=" + std::to_string(stat.min_limit) + @@ -236,6 +226,16 @@ postrun_statistic_check::check_stat(scoped_cursor &cursor, const postrun_statist } /* runtime_monitor class implementation */ +void +runtime_monitor::get_stat(scoped_cursor &cursor, int stat_field, int64_t *valuep) +{ + const char *desc, *pvalue; + cursor->set_key(cursor.get(), stat_field); + testutil_check(cursor->search(cursor.get())); + testutil_check(cursor->get_value(cursor.get(), &desc, &pvalue, valuep)); + testutil_check(cursor->reset(cursor.get())); +} + runtime_monitor::runtime_monitor(configuration *config, database &database) : component("runtime_monitor", config), _postrun_stats(config), _database(database) { diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.h index e7e69302d1d..0f63585290d 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.h @@ -116,6 +116,9 @@ class postrun_statistic_check { */ class runtime_monitor : public component { public: + static void get_stat(scoped_cursor &, int, int64_t *); + + public: runtime_monitor(configuration *config, database &database); ~runtime_monitor(); diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/logger.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/logger.h index 8d67bfd7e27..2c35ef0d162 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/logger.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/logger.h @@ -30,8 +30,12 @@ #define DEBUG_UTILS_H /* Following definitions are required in order to use printing format specifiers in C++. */ +#ifndef __STDC_LIMIT_MACROS #define __STDC_LIMIT_MACROS +#endif +#ifndef __STDC_FORMAT_MACROS #define __STDC_FORMAT_MACROS +#endif #include <chrono> #include <iostream> diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_types.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_types.h index 47b8592ede0..edb38e3e22c 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_types.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_types.h @@ -30,8 +30,12 @@ #define SCOPED_TYPES_H /* Following definitions are required in order to use printing format specifiers in C++. */ +#ifndef __STDC_LIMIT_MACROS #define __STDC_LIMIT_MACROS +#endif +#ifndef __STDC_FORMAT_MACROS #define __STDC_FORMAT_MACROS +#endif extern "C" { #include "test_util.h" diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/random_generator.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/random_generator.cxx index c539fbc34fc..3973af7242c 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/random_generator.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/random_generator.cxx @@ -44,10 +44,11 @@ random_generator::instance() std::string random_generator::generate_random_string(std::size_t length, characters_type type) { + const std::string characters = get_characters(type); std::string str; while (str.size() < length) - str += get_characters(type); + str += characters; std::shuffle(str.begin(), str.end(), _generator); return (str.substr(0, length)); diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/random_generator.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/random_generator.h index 31f44bbe98e..967d5566ce1 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/random_generator.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/random_generator.h @@ -30,8 +30,12 @@ #define RANDOM_GENERATOR_H /* Following definitions are required in order to use printing format specifiers in C++. */ +#ifndef __STDC_LIMIT_MACROS #define __STDC_LIMIT_MACROS +#endif +#ifndef __STDC_FORMAT_MACROS #define __STDC_FORMAT_MACROS +#endif #include <random> #include <string> diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h index 61e1b99f28a..28c1ee4265b 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h @@ -159,6 +159,7 @@ class thread_context { public: scoped_session session; scoped_cursor op_track_cursor; + scoped_cursor stat_cursor; transaction_context transaction; timestamp_manager *tsm; workload_tracking *tracking; diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_tracking.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_tracking.cxx index 1211749ec28..e0e7738590d 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_tracking.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_tracking.cxx @@ -99,7 +99,8 @@ workload_tracking::do_work() /* Take a copy of the oldest so that we sweep with a consistent timestamp. */ oldest_ts = _tsm.get_oldest_ts(); - while ((ret = _sweep_cursor->prev(_sweep_cursor.get())) == 0) { + /* We need to check if the component is still running to avoid unecessary iterations. */ + while (_running && (ret = _sweep_cursor->prev(_sweep_cursor.get())) == 0) { testutil_check(_sweep_cursor->get_key(_sweep_cursor.get(), &collection_id, &key, &ts)); testutil_check(_sweep_cursor->get_value(_sweep_cursor.get(), &op_type, &value)); /* @@ -137,7 +138,13 @@ workload_tracking::do_work() free(sweep_key); - if (ret != WT_NOTFOUND) + /* + * If we get here and the test is still running, it means we must have reached the end of the + * table. We can also get here because the test is no longer running. In this case, the cursor + * can either be at the end of the table or still on a valid entry since we interrupted the + * work. + */ + if (ret != 0 && ret != WT_NOTFOUND) testutil_die(LOG_ERROR, "Tracking table sweep failed: cursor->next() returned an unexpected error %d.", ret); diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/csuite_style_example_test.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/csuite_style_example_test.cxx new file mode 100644 index 00000000000..d0059880446 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/tests/csuite_style_example_test.cxx @@ -0,0 +1,169 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/* + * This file provides an example of how to create a test in C++ using a few features from the + * framework if any. This file can be used as a template for quick testing and/or when stress + * testing is not required. For any stress testing, it is encouraged to use the framework, see + * example_test.cxx and create_script.sh. + */ + +#include "test_harness/connection_manager.h" +#include "test_harness/thread_manager.h" +#include "test_harness/util/api_const.h" +#include "test_harness/util/logger.h" +#include "test_harness/workload/random_generator.h" + +extern "C" { +#include "wiredtiger.h" +} + +using namespace test_harness; + +bool do_inserts = false; +bool do_reads = false; + +void +insert_op(WT_CURSOR *cursor, int key_size, int value_size) +{ + logger::log_msg(LOG_INFO, "called insert_op"); + + /* Insert random data. */ + std::string key, value; + while (do_inserts) { + key = random_generator::instance().generate_random_string(key_size); + value = random_generator::instance().generate_random_string(value_size); + cursor->set_key(cursor, key.c_str()); + cursor->set_value(cursor, value.c_str()); + testutil_check(cursor->insert(cursor)); + } +} + +void +read_op(WT_CURSOR *cursor, int key_size) +{ + logger::log_msg(LOG_INFO, "called read_op"); + + /* Read random data. */ + std::string key; + while (do_reads) { + key = random_generator::instance().generate_random_string(key_size); + cursor->set_key(cursor, key.c_str()); + cursor->search(cursor); + } +} + +int +main(int argc, char *argv[]) +{ + /* Set the program name for error messages. */ + const std::string progname = testutil_set_progname(argv); + + /* Set the tracing level for the logger component. */ + logger::trace_level = LOG_INFO; + + /* Printing some messages. */ + logger::log_msg(LOG_INFO, "Starting " + progname); + logger::log_msg(LOG_ERROR, "This could be an error."); + + /* Create a connection, set the cache size and specify the home directory. */ + const std::string conn_config = std::string(CONNECTION_CREATE) + ",cache_size=500MB"; + const std::string home_dir = std::string(DEFAULT_DIR) + '_' + progname; + connection_manager::instance().create(conn_config, home_dir); + WT_CONNECTION *conn = connection_manager::instance().get_connection(); + + /* Open different sessions. */ + WT_SESSION *insert_session, *read_session; + testutil_check(conn->open_session(conn, nullptr, nullptr, &insert_session)); + testutil_check(conn->open_session(conn, nullptr, nullptr, &read_session)); + + /* Create a collection. */ + const std::string collection_name = "table:my_collection"; + testutil_check( + insert_session->create(insert_session, collection_name.c_str(), DEFAULT_FRAMEWORK_SCHEMA)); + + /* Open different cursors. */ + WT_CURSOR *insert_cursor, *read_cursor; + const std::string cursor_config = ""; + testutil_check(insert_session->open_cursor( + insert_session, collection_name.c_str(), nullptr, cursor_config.c_str(), &insert_cursor)); + testutil_check(read_session->open_cursor( + read_session, collection_name.c_str(), nullptr, cursor_config.c_str(), &read_cursor)); + + /* Store cursors. */ + std::vector<WT_CURSOR *> cursors; + cursors.push_back(insert_cursor); + cursors.push_back(read_cursor); + + /* Insert some data. */ + std::string key = "a"; + const std::string value = "b"; + insert_cursor->set_key(insert_cursor, key.c_str()); + insert_cursor->set_value(insert_cursor, value.c_str()); + testutil_check(insert_cursor->insert(insert_cursor)); + + /* Read some data. */ + key = "b"; + read_cursor->set_key(read_cursor, key.c_str()); + testutil_assert(read_cursor->search(read_cursor) == WT_NOTFOUND); + + key = "a"; + read_cursor->set_key(read_cursor, key.c_str()); + testutil_check(read_cursor->search(read_cursor)); + + /* Create a thread manager and spawn some threads that will work. */ + thread_manager t; + int key_size = 1, value_size = 2; + + do_inserts = true; + t.add_thread(insert_op, insert_cursor, key_size, value_size); + + do_reads = true; + t.add_thread(read_op, read_cursor, key_size); + + /* Sleep for the test duration. */ + int test_duration_s = 5; + std::this_thread::sleep_for(std::chrono::seconds(test_duration_s)); + + /* Stop the threads. */ + do_reads = false; + do_inserts = false; + t.join(); + + /* Close cursors. */ + for (auto c : cursors) + testutil_check(c->close(c)); + + /* Close the connection. */ + connection_manager::instance().close(); + + /* Another message. */ + logger::log_msg(LOG_INFO, "End of test."); + + return (0); +} diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/example_test.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/example_test.cxx index 5a1996b45a1..4b49ad2b148 100644 --- a/src/third_party/wiredtiger/test/cppsuite/tests/example_test.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/tests/example_test.cxx @@ -37,6 +37,13 @@ class example_test : public test_harness::test { example_test(const test_harness::test_args &args) : test(args) {} void + run() + { + /* You can remove the call to the base class to fully customized your test. */ + test::run(); + } + + void populate(test_harness::database &, test_harness::timestamp_manager *, test_harness::configuration *, test_harness::workload_tracking *) override final { diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx index 0c231e6568d..627921a9aa4 100755 --- a/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx @@ -34,9 +34,11 @@ #include "test_harness/test.h" #include "base_test.cxx" -#include "example_test.cxx" #include "burst_inserts.cxx" +#include "example_test.cxx" #include "hs_cleanup.cxx" +#include "search_near_01.cxx" +#include "search_near_02.cxx" std::string parse_configuration_from_file(const std::string &filename) @@ -72,7 +74,7 @@ print_help() std::cout << "\trun -C [WIREDTIGER_OPEN_CONFIGURATION]" << std::endl; std::cout << "\trun -c [TEST_FRAMEWORK_CONFIGURATION]" << std::endl; std::cout << "\trun -f [FILE]" << std::endl; - std::cout << "\trun -l [TRACEL_LEVEL]" << std::endl; + std::cout << "\trun -l [TRACE_LEVEL]" << std::endl; std::cout << "\trun -t [TEST_NAME]" << std::endl; std::cout << std::endl; std::cout << "DESCRIPTION" << std::endl; @@ -113,10 +115,14 @@ run_test(const std::string &test_name, const std::string &config, const std::str base_test(test_harness::test_args{config, test_name, wt_open_config}).run(); else if (test_name == "example_test") example_test(test_harness::test_args{config, test_name, wt_open_config}).run(); + else if (test_name == "search_near_02") + search_near_02(test_harness::test_args{config, test_name, wt_open_config}).run(); else if (test_name == "hs_cleanup") hs_cleanup(test_harness::test_args{config, test_name, wt_open_config}).run(); else if (test_name == "burst_inserts") burst_inserts(test_harness::test_args{config, test_name, wt_open_config}).run(); + else if (test_name == "search_near_01") + search_near_01(test_harness::test_args{config, test_name, wt_open_config}).run(); else { test_harness::logger::log_msg(LOG_ERROR, "Test not found: " + test_name); error_code = -1; @@ -139,8 +145,8 @@ main(int argc, char *argv[]) { std::string cfg, config_filename, current_cfg, current_test_name, test_name, wt_open_config; int64_t error_code = 0; - const std::vector<std::string> all_tests = { - "example_test", "burst_inserts", "hs_cleanup", "base_test"}; + const std::vector<std::string> all_tests = {"base_test", "burst_inserts", "example_test", + "hs_cleanup", "search_near_01", "search_near_02"}; /* Set the program name for error messages. */ (void)testutil_set_progname(argv); @@ -219,12 +225,20 @@ main(int argc, char *argv[]) } } else { current_test_name = test_name; - /* Configuration parsing. */ - if (!config_filename.empty()) - cfg = parse_configuration_from_file(config_filename); - else if (cfg.empty()) - cfg = parse_configuration_from_file(get_default_config_path(current_test_name)); - error_code = run_test(current_test_name, cfg, wt_open_config); + /* Check the test exists. */ + if (std::find(all_tests.begin(), all_tests.end(), current_test_name) == + all_tests.end()) { + test_harness::logger::log_msg( + LOG_ERROR, "The test " + current_test_name + " was not found."); + error_code = -1; + } else { + /* Configuration parsing. */ + if (!config_filename.empty()) + cfg = parse_configuration_from_file(config_filename); + else if (cfg.empty()) + cfg = parse_configuration_from_file(get_default_config_path(current_test_name)); + error_code = run_test(current_test_name, cfg, wt_open_config); + } } if (error_code != 0) diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/search_near_01.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/search_near_01.cxx new file mode 100644 index 00000000000..abacf3dc196 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/tests/search_near_01.cxx @@ -0,0 +1,262 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "test_harness/util/api_const.h" +#include "test_harness/workload/random_generator.h" +#include "test_harness/workload/thread_context.h" +#include "test_harness/test.h" +#include "test_harness/thread_manager.h" + +using namespace test_harness; +/* + * In this test, we want to verify that search_near with prefix enabled only traverses the portion + * of the tree that follows the prefix portion of the search key. The test is composed of a populate + * phase followed by a read phase. The populate phase will insert a set of random generated keys + * with a prefix of aaa -> zzz. The read phase will continuously perform prefix search near calls, + * and validate that the number of entries traversed is within bounds of the search key. + */ +class search_near_01 : public test_harness::test { + uint64_t keys_per_prefix = 0; + uint64_t srchkey_len = 0; + const std::string ALPHABET{"abcdefghijklmnopqrstuvwxyz"}; + const uint64_t PREFIX_KEY_LEN = 3; + + static void + populate_worker(thread_context *tc, const std::string &ALPHABET, uint64_t PREFIX_KEY_LEN) + { + logger::log_msg(LOG_INFO, "Populate with thread id: " + std::to_string(tc->id)); + + std::string prefix_key; + uint64_t collections_per_thread = tc->collection_count; + const uint64_t MAX_ROLLBACKS = 100; + uint32_t rollback_retries = 0; + int cmpp; + + /* + * Generate a table of data with prefix keys aaa -> zzz. We have 26 threads from ids + * starting from 0 to 26. Each populate thread will insert separate prefix keys based on the + * id. + */ + for (int64_t i = 0; i < collections_per_thread; ++i) { + collection &coll = tc->db.get_collection(i); + scoped_cursor cursor = tc->session.open_scoped_cursor(coll.name.c_str()); + for (uint64_t j = 0; j < ALPHABET.size(); ++j) { + for (uint64_t k = 0; k < ALPHABET.size(); ++k) { + for (uint64_t count = 0; count < tc->key_count; ++count) { + tc->transaction.begin(); + /* + * Generate the prefix key, and append a random generated key string based + * on the key size configuration. + */ + prefix_key = {ALPHABET.at(tc->id), ALPHABET.at(j), ALPHABET.at(k)}; + prefix_key += random_generator::instance().generate_random_string( + tc->key_size - PREFIX_KEY_LEN); + if (!tc->insert(cursor, coll.id, prefix_key)) { + testutil_assert(rollback_retries < MAX_ROLLBACKS); + /* We failed to insert, rollback our transaction and retry. */ + tc->transaction.rollback(); + ++rollback_retries; + --count; + } else { + /* Commit txn at commit timestamp 100. */ + tc->transaction.commit( + "commit_timestamp=" + tc->tsm->decimal_to_hex(100)); + rollback_retries = 0; + } + } + } + } + } + } + + public: + search_near_01(const test_harness::test_args &args) : test(args) {} + + void + populate(test_harness::database &database, test_harness::timestamp_manager *tsm, + test_harness::configuration *config, test_harness::workload_tracking *tracking) override final + { + uint64_t collection_count, key_size; + std::vector<thread_context *> workers; + thread_manager tm; + + /* Validate our config. */ + collection_count = config->get_int(COLLECTION_COUNT); + keys_per_prefix = config->get_int(KEY_COUNT_PER_COLLECTION); + key_size = config->get_int(KEY_SIZE); + testutil_assert(collection_count > 0); + testutil_assert(keys_per_prefix > 0); + /* Check the prefix length is not greater than the key size. */ + testutil_assert(key_size >= PREFIX_KEY_LEN); + + logger::log_msg(LOG_INFO, + "Populate configuration with key size: " + std::to_string(key_size) + + " key count: " + std::to_string(keys_per_prefix) + + " number of collections: " + std::to_string(collection_count)); + + /* Create n collections as per the configuration. */ + for (uint64_t i = 0; i < collection_count; ++i) + /* + * The database model will call into the API and create the collection, with its own + * session. + */ + database.add_collection(); + + /* Spawn 26 threads to populate the database. */ + for (uint64_t i = 0; i < ALPHABET.size(); ++i) { + thread_context *tc = new thread_context(i, thread_type::INSERT, config, + connection_manager::instance().create_session(), tsm, tracking, database); + workers.push_back(tc); + tm.add_thread(populate_worker, tc, ALPHABET, PREFIX_KEY_LEN); + } + + /* Wait for our populate threads to finish and then join them. */ + logger::log_msg(LOG_INFO, "Populate: waiting for threads to complete."); + tm.join(); + + /* Cleanup our workers. */ + for (auto &it : workers) { + delete it; + it = nullptr; + } + + /* Force evict all the populated keys in all of the collections. */ + int cmpp; + scoped_session session = connection_manager::instance().create_session(); + for (uint64_t count = 0; count < collection_count; ++count) { + collection &coll = database.get_collection(count); + scoped_cursor evict_cursor = + session.open_scoped_cursor(coll.name.c_str(), "debug=(release_evict=true)"); + + for (uint64_t i = 0; i < ALPHABET.size(); ++i) { + for (uint64_t j = 0; j < ALPHABET.size(); ++j) { + for (uint64_t k = 0; k < ALPHABET.size(); ++k) { + std::string key = {ALPHABET.at(i), ALPHABET.at(j), ALPHABET.at(k)}; + evict_cursor->set_key(evict_cursor.get(), key.c_str()); + evict_cursor->search_near(evict_cursor.get(), &cmpp); + testutil_check(evict_cursor->reset(evict_cursor.get())); + } + } + } + } + srchkey_len = + random_generator::instance().generate_integer(static_cast<uint64_t>(1), PREFIX_KEY_LEN); + logger::log_msg(LOG_INFO, "Populate: finished."); + } + + void + read_operation(test_harness::thread_context *tc) override final + { + /* Make sure that thread statistics cursor is null before we open it. */ + testutil_assert(tc->stat_cursor.get() == nullptr); + logger::log_msg( + LOG_INFO, type_string(tc->type) + " thread {" + std::to_string(tc->id) + "} commencing."); + std::map<uint64_t, scoped_cursor> cursors; + tc->stat_cursor = tc->session.open_scoped_cursor(STATISTICS_URI); + std::string srch_key; + int64_t entries_stat, prefix_stat, prev_entries_stat, prev_prefix_stat, expected_entries; + int cmpp; + + cmpp = 0; + prev_entries_stat = 0; + prev_prefix_stat = 0; + + /* + * The number of expected entries is calculated to account for the maximum allowed entries + * per search near function call. The key we search near can be different in length, which + * will increase the number of entries search by a factor of 26. + */ + expected_entries = tc->thread_count * keys_per_prefix * 2 * + pow(ALPHABET.size(), PREFIX_KEY_LEN - srchkey_len); + + /* + * Read at timestamp 10, so that no keys are visible to this transaction. This allows prefix + * search near to early exit out of it's prefix range when it's trying to search for a + * visible key in the tree. + */ + tc->transaction.begin("read_timestamp=" + tc->tsm->decimal_to_hex(10)); + while (tc->running()) { + + /* Get a collection and find a cached cursor. */ + collection &coll = tc->db.get_random_collection(); + if (cursors.find(coll.id) == cursors.end()) { + scoped_cursor cursor = tc->session.open_scoped_cursor(coll.name.c_str()); + cursor->reconfigure(cursor.get(), "prefix_search=true"); + cursors.emplace(coll.id, std::move(cursor)); + } + + /* Generate search prefix key of random length between a -> zzz. */ + srch_key = random_generator::instance().generate_random_string( + srchkey_len, characters_type::ALPHABET); + logger::log_msg(LOG_INFO, + "Read thread {" + std::to_string(tc->id) + + "} performing prefix search near with key: " + srch_key); + + /* Do a second lookup now that we know it exists. */ + auto &cursor = cursors[coll.id]; + if (tc->transaction.active()) { + runtime_monitor::get_stat( + tc->stat_cursor, WT_STAT_CONN_CURSOR_NEXT_SKIP_LT_100, &prev_entries_stat); + runtime_monitor::get_stat(tc->stat_cursor, + WT_STAT_CONN_CURSOR_SEARCH_NEAR_PREFIX_FAST_PATHS, &prev_prefix_stat); + + cursor->set_key(cursor.get(), srch_key.c_str()); + testutil_assert(cursor->search_near(cursor.get(), &cmpp) == WT_NOTFOUND); + + runtime_monitor::get_stat( + tc->stat_cursor, WT_STAT_CONN_CURSOR_NEXT_SKIP_LT_100, &entries_stat); + runtime_monitor::get_stat( + tc->stat_cursor, WT_STAT_CONN_CURSOR_SEARCH_NEAR_PREFIX_FAST_PATHS, &prefix_stat); + logger::log_msg(LOG_INFO, + "Read thread {" + std::to_string(tc->id) + + "} skipped entries: " + std::to_string(entries_stat - prev_entries_stat) + + " prefix fash path: " + std::to_string(prefix_stat - prev_prefix_stat)); + + /* + * It is possible that WiredTiger increments the entries skipped stat irrelevant to + * prefix search near. This is dependent on how many read threads are present in the + * test. Account for this by creating a small buffer using thread count. Assert that + * the number of expected entries is the upper limit which the prefix search near + * can traverse and the prefix fast path is incremented. + */ + testutil_assert( + (expected_entries + (2 * tc->thread_count)) >= entries_stat - prev_entries_stat); + testutil_assert(prefix_stat > prev_prefix_stat); + + tc->transaction.add_op(); + tc->sleep(); + } + /* Reset our cursor to avoid pinning content. */ + testutil_check(cursor->reset(cursor.get())); + } + tc->transaction.commit(); + /* Make sure the last transaction is rolled back now the work is finished. */ + if (tc->transaction.active()) + tc->transaction.rollback(); + } +}; diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/search_near_02.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/search_near_02.cxx new file mode 100644 index 00000000000..1df75a83bc0 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/tests/search_near_02.cxx @@ -0,0 +1,425 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "test_harness/test.h" +#include "test_harness/util/api_const.h" +#include "test_harness/workload/random_generator.h" + +using namespace test_harness; + +/* + * In this test, we want to verify search_near with prefix enabled returns the correct key. + * During the test duration: + * - N threads will keep inserting new random keys + * - M threads will execute search_near calls with prefix enabled using random prefixes as well. + * Each search_near call with prefix enabled is verified using the default search_near. + */ +class search_near_02 : public test_harness::test { + public: + search_near_02(const test_harness::test_args &args) : test(args) {} + + void + populate(test_harness::database &database, test_harness::timestamp_manager *, + test_harness::configuration *config, test_harness::workload_tracking *) override final + { + /* + * The populate phase only creates empty collections. The number of collections is defined + * in the configuration. + */ + int64_t collection_count = config->get_int(COLLECTION_COUNT); + + logger::log_msg( + LOG_INFO, "Populate: " + std::to_string(collection_count) + " creating collections."); + + for (uint64_t i = 0; i < collection_count; ++i) + database.add_collection(); + + logger::log_msg(LOG_INFO, "Populate: finished."); + } + + void + insert_operation(test_harness::thread_context *tc) override final + { + /* Each insert operation will insert new keys in the collections. */ + logger::log_msg( + LOG_INFO, type_string(tc->type) + " thread {" + std::to_string(tc->id) + "} commencing."); + + /* Helper struct which stores a pointer to a collection and a cursor associated with it. */ + struct collection_cursor { + collection_cursor(collection &coll, scoped_cursor &&cursor) + : coll(coll), cursor(std::move(cursor)) + { + } + collection &coll; + scoped_cursor cursor; + }; + + /* Collection cursor vector. */ + std::vector<collection_cursor> ccv; + int64_t collection_count = tc->db.get_collection_count(); + int64_t collections_per_thread = collection_count / tc->thread_count; + + /* Must have unique collections for each thread. */ + testutil_assert(collection_count % tc->thread_count == 0); + const uint64_t thread_offset = tc->id * collections_per_thread; + for (uint64_t i = thread_offset; + i < thread_offset + collections_per_thread && tc->running(); ++i) { + collection &coll = tc->db.get_collection(i); + scoped_cursor cursor = tc->session.open_scoped_cursor(coll.name.c_str()); + ccv.push_back({coll, std::move(cursor)}); + } + + std::string key; + uint64_t counter = 0; + + while (tc->running()) { + + auto &cc = ccv[counter]; + tc->transaction.begin(); + + while (tc->transaction.active() && tc->running()) { + + /* Generate a random key. */ + key = random_generator::instance().generate_random_string(tc->key_size); + + /* Insert a key value pair. */ + if (tc->insert(cc.cursor, cc.coll.id, key)) { + if (tc->transaction.can_commit()) + /* We are not checking the result of commit as it is not necessary. */ + tc->transaction.commit(); + } else { + tc->transaction.rollback(); + } + + /* Sleep the duration defined by the configuration. */ + tc->sleep(); + } + + /* Rollback any transaction that could not commit before the end of the test. */ + if (tc->transaction.active()) + tc->transaction.rollback(); + + /* Reset our cursor to avoid pinning content. */ + testutil_check(cc.cursor->reset(cc.cursor.get())); + if (++counter == ccv.size()) + counter = 0; + testutil_assert(counter < collections_per_thread); + } + } + + void + read_operation(test_harness::thread_context *tc) override final + { + /* + * Each read operation performs search_near calls with and without prefix enabled on random + * collections. Each prefix is randomly generated. The result of the seach_near call with + * prefix enabled is then validated using the search_near call without prefix enabled. + */ + logger::log_msg( + LOG_INFO, type_string(tc->type) + " thread {" + std::to_string(tc->id) + "} commencing."); + + const char *key_prefix; + int exact_prefix, ret; + int64_t prefix_size; + std::map<uint64_t, scoped_cursor> cursors; + std::string generated_prefix, key_prefix_str; + + while (tc->running()) { + /* Get a random collection to work on. */ + collection &coll = tc->db.get_random_collection(); + + /* Find a cached cursor or create one if none exists. */ + if (cursors.find(coll.id) == cursors.end()) { + cursors.emplace( + coll.id, std::move(tc->session.open_scoped_cursor(coll.name.c_str()))); + auto &cursor_prefix = cursors[coll.id]; + /* The cached cursors have the prefix configuration enabled. */ + testutil_check( + cursor_prefix.get()->reconfigure(cursor_prefix.get(), "prefix_search=true")); + } + + auto &cursor_prefix = cursors[coll.id]; + + /* + * Pick a random timestamp between the oldest and now. Get rid of the last 32 bits as + * they represent an increment for uniqueness. + */ + wt_timestamp_t ts = random_generator::instance().generate_integer( + (tc->tsm->get_oldest_ts() >> 32), (tc->tsm->get_next_ts() >> 32)); + /* Put back the timestamp in the correct format. */ + ts <<= 32; + + /* + * The oldest timestamp might move ahead and the reading timestamp might become invalid. + * To tackle this issue, we round the timestamp to the oldest timestamp value. + */ + tc->transaction.begin( + "roundup_timestamps=(read=true),read_timestamp=" + tc->tsm->decimal_to_hex(ts)); + + while (tc->transaction.active() && tc->running()) { + /* + * Generate a random prefix. For this, we start by generating a random size and then + * its value. + */ + prefix_size = random_generator::instance().generate_integer( + static_cast<int64_t>(1), tc->key_size); + generated_prefix = random_generator::instance().generate_random_string( + prefix_size, characters_type::ALPHABET); + + /* Call search near with the prefix cursor. */ + cursor_prefix->set_key(cursor_prefix.get(), generated_prefix.c_str()); + ret = cursor_prefix->search_near(cursor_prefix.get(), &exact_prefix); + testutil_assert(ret == 0 || ret == WT_NOTFOUND); + if (ret == 0) { + testutil_check(cursor_prefix->get_key(cursor_prefix.get(), &key_prefix)); + key_prefix_str = key_prefix; + } else { + key_prefix_str = ""; + } + + /* Open a cursor with the default configuration on the selected collection. */ + scoped_cursor cursor_default(tc->session.open_scoped_cursor(coll.name.c_str())); + + /* Verify the prefix search_near output using the default cursor. */ + validate_prefix_search_near( + ret, exact_prefix, key_prefix_str, cursor_default, generated_prefix); + + tc->transaction.add_op(); + tc->transaction.try_rollback(); + tc->sleep(); + } + testutil_check(cursor_prefix->reset(cursor_prefix.get())); + } + /* Roll back the last transaction if still active now the work is finished. */ + if (tc->transaction.active()) + tc->transaction.rollback(); + } + + private: + /* Validate prefix search_near call outputs using a cursor without prefix key enabled. */ + void + validate_prefix_search_near(int ret_prefix, int exact_prefix, const std::string &key_prefix, + scoped_cursor &cursor_default, const std::string &prefix) + { + /* Call search near with the default cursor using the given prefix. */ + int exact_default; + cursor_default->set_key(cursor_default.get(), prefix.c_str()); + int ret_default = cursor_default->search_near(cursor_default.get(), &exact_default); + + /* + * It is not possible to have a prefix search near call successful and the default search + * near call unsuccessful. + */ + testutil_assert( + ret_default == ret_prefix || (ret_default == 0 && ret_prefix == WT_NOTFOUND)); + + /* We only have to perform validation when the default search near call is successful. */ + if (ret_default == 0) { + /* Both calls are successful. */ + if (ret_prefix == 0) + validate_successful_calls( + ret_prefix, exact_prefix, key_prefix, cursor_default, exact_default, prefix); + /* The prefix search near call failed. */ + else + validate_unsuccessful_prefix_call(cursor_default, prefix, exact_default); + } + } + + /* + * Validate a successful prefix enabled search near call using a successful default search near + * call. + * The exact value set by the prefix search near call has to be either 0 or 1. Indeed, it cannot + * be -1 as the key needs to contain the prefix. + * - If it is 0, both search near calls should return the same outputs and both cursors should + * be positioned on the prefix we are looking for. + * - If it is 1, it will depend on the exact value set by the default search near call which can + * be -1 or 1. If it is -1, calling next on the default cursor should get us ti the key found by + * the prefix search near call. If it is 1, it means both search near calls have found the same + * key that is lexicographically greater than the prefix but still contains the prefix. + */ + void + validate_successful_calls(int ret_prefix, int exact_prefix, const std::string &key_prefix, + scoped_cursor &cursor_default, int exact_default, const std::string &prefix) + { + const char *k; + std::string k_str; + int ret; + + /* + * The prefix search near call cannot retrieve a key with a smaller value than the prefix we + * searched. + */ + testutil_assert(exact_prefix >= 0); + + /* The key at the prefix cursor should contain the prefix. */ + testutil_assert(key_prefix.substr(0, prefix.size()) == prefix); + + /* Retrieve the key the default cursor is pointing at. */ + const char *key_default; + testutil_check(cursor_default->get_key(cursor_default.get(), &key_default)); + std::string key_default_str = key_default; + + logger::log_msg(LOG_TRACE, + "search_near (normal) exact " + std::to_string(exact_default) + " key " + key_default); + logger::log_msg(LOG_TRACE, + "search_near (prefix) exact " + std::to_string(exact_prefix) + " key " + key_prefix); + + /* Example: */ + /* keys: a, bb, bba. */ + /* Only bb is not visible. */ + /* Default search_near(bb) returns a, exact < 0. */ + /* Prefix search_near(bb) returns bba, exact > 0. */ + if (exact_default < 0) { + /* The key at the default cursor should not contain the prefix. */ + testutil_assert((key_default_str.substr(0, prefix.size()) != prefix)); + + /* + * The prefix cursor should be positioned at a key lexicographically greater than the + * prefix. + */ + testutil_assert(exact_prefix > 0); + + /* + * The next key of the default cursor should be equal to the key pointed by the prefix + * cursor. + */ + testutil_assert(cursor_default->next(cursor_default.get()) == 0); + testutil_check(cursor_default->get_key(cursor_default.get(), &k)); + testutil_assert(k == key_prefix); + } + /* Example: */ + /* keys: a, bb, bba */ + /* Case 1: all keys are visible. */ + /* Default search_near(bb) returns bb, exact = 0 */ + /* Prefix search_near(bb) returns bb, exact = 0 */ + /* Case 2: only bb is not visible. */ + /* Default search_near(bb) returns bba, exact > 0. */ + /* Prefix search_near(bb) returns bba, exact > 0. */ + else { + /* Both cursors should be pointing at the same key. */ + testutil_assert(exact_prefix == exact_default); + testutil_assert(key_default_str == key_prefix); + /* Both cursors should have found the exact key. */ + if (exact_default == 0) + testutil_assert(key_default_str == prefix); + /* Both cursors have found a key that is lexicographically greater than the prefix. */ + else + testutil_assert(key_default_str != prefix); + } + } + + /* + * Validate that no keys with the prefix used for the search have been found. + * To validate this, we can use the exact value set by the default search near. Since the prefix + * search near failed, the exact value set by the default search near call has to be either -1 + * or 1: + * - If it is -1, we need to check the next key, if it exists, is lexicographically greater than + * the prefix we looked for. + * - If it is 1, we need to check the previous keys, if it exists, if lexicographically smaller + * than the prefix we looked for. + */ + void + validate_unsuccessful_prefix_call( + scoped_cursor &cursor_default, const std::string &prefix, int exact_default) + { + int ret; + const char *k; + std::string k_str; + + /* + * The exact value from the default search near call cannot be 0, otherwise the prefix + * search near should be successful too. + */ + testutil_assert(exact_default != 0); + + /* Retrieve the key at the default cursor. */ + const char *key_default; + testutil_check(cursor_default->get_key(cursor_default.get(), &key_default)); + std::string key_default_str = key_default; + + /* The key at the default cursor should not contain the prefix. */ + testutil_assert(key_default_str.substr(0, prefix.size()) != prefix); + + /* Example: */ + /* keys: a, bb, bbb. */ + /* All keys are visible. */ + /* Default search_near(bba) returns bb, exact < 0. */ + /* Prefix search_near(bba) returns WT_NOTFOUND. */ + if (exact_default < 0) { + /* + * The current key of the default cursor should be lexicographically smaller than the + * prefix. + */ + testutil_assert(std::lexicographical_compare( + key_default_str.begin(), key_default_str.end(), prefix.begin(), prefix.end())); + + /* + * The next key of the default cursor should be lexicographically greater than the + * prefix if it exists. + */ + ret = cursor_default->next(cursor_default.get()); + if (ret == 0) { + testutil_check(cursor_default->get_key(cursor_default.get(), &k)); + k_str = k; + testutil_assert(!std::lexicographical_compare( + k_str.begin(), k_str.end(), prefix.begin(), prefix.end())); + } else { + /* End of the table. */ + testutil_assert(ret == WT_NOTFOUND); + } + } + /* Example: */ + /* keys: a, bb, bbb. */ + /* All keys are visible. */ + /* Default search_near(bba) returns bbb, exact > 0. */ + /* Prefix search_near(bba) returns WT_NOTFOUND. */ + else { + /* + * The current key of the default cursor should be lexicographically greater than the + * prefix. + */ + testutil_assert(!std::lexicographical_compare( + key_default_str.begin(), key_default_str.end(), prefix.begin(), prefix.end())); + + /* + * The next key of the default cursor should be lexicographically smaller than the + * prefix if it exists. + */ + ret = cursor_default->prev(cursor_default.get()); + if (ret == 0) { + testutil_check(cursor_default->get_key(cursor_default.get(), &k)); + k_str = k; + testutil_assert(std::lexicographical_compare( + k_str.begin(), k_str.end(), prefix.begin(), prefix.end())); + } else { + /* End of the table. */ + testutil_assert(ret == WT_NOTFOUND); + } + } + } +}; diff --git a/src/third_party/wiredtiger/test/csuite/random_abort/main.c b/src/third_party/wiredtiger/test/csuite/random_abort/main.c index 26789a1ac9b..ce935d26e3f 100644 --- a/src/third_party/wiredtiger/test/csuite/random_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/random_abort/main.c @@ -38,6 +38,7 @@ static char home[1024]; /* Program working dir */ */ static const char *const col_uri = "table:col_main"; static const char *const uri = "table:main"; +static bool compaction; static bool compat; static bool inmem; @@ -218,6 +219,17 @@ thread_run(void *arg) testutil_die(errno, "fprintf"); /* + * If configured, run compaction on database after each epoch of 100000 operations. + */ + if (compaction && i >= 100000 && i % 100000 == 0) { + printf("Running compaction in Thread %" PRIu32 "\n", td->id); + if (columnar_table) + testutil_check(session->compact(session, col_uri, NULL)); + else + testutil_check(session->compact(session, uri, NULL)); + } + + /* * Decide what kind of operation can be performed on the already inserted data. */ if (i % MAX_NUM_OPS == OP_TYPE_DELETE) { @@ -607,18 +619,21 @@ main(int argc, char *argv[]) (void)testutil_set_progname(argv); - compat = inmem = false; + compaction = compat = inmem = false; nth = MIN_TH; rand_th = rand_time = true; timeout = MIN_TIME; verify_only = false; working_dir = "WT_TEST.random-abort"; - while ((ch = __wt_getopt(progname, argc, argv, "Ch:mT:t:v")) != EOF) + while ((ch = __wt_getopt(progname, argc, argv, "Cch:mT:t:v")) != EOF) switch (ch) { case 'C': compat = true; break; + case 'c': + compaction = true; + break; case 'h': working_dir = __wt_optarg; break; @@ -669,8 +684,9 @@ main(int argc, char *argv[]) printf("Parent: Compatibility %s in-mem log %s\n", compat ? "true" : "false", inmem ? "true" : "false"); printf("Parent: Create %" PRIu32 " threads; sleep %" PRIu32 " seconds\n", nth, timeout); - printf("CONFIG: %s%s%s -h %s -T %" PRIu32 " -t %" PRIu32 "\n", progname, - compat ? " -C" : "", inmem ? " -m" : "", working_dir, nth, timeout); + printf("CONFIG: %s%s%s%s -h %s -T %" PRIu32 " -t %" PRIu32 "\n", progname, + compat ? " -C" : "", compaction ? " -c" : "", inmem ? " -m" : "", working_dir, nth, + timeout); /* * Fork a child to insert as many items. We will then randomly kill the child, run recovery * and make sure all items we wrote exist after recovery runs. diff --git a/src/third_party/wiredtiger/test/csuite/tiered_abort/main.c b/src/third_party/wiredtiger/test/csuite/tiered_abort/main.c index c29460b8f46..8111cd108ac 100644 --- a/src/third_party/wiredtiger/test/csuite/tiered_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/tiered_abort/main.c @@ -55,15 +55,17 @@ static char home[1024]; /* Program working dir */ * Also each worker thread creates its own textual records file that records the data it * inserted and it records the timestamp that was used for that insertion. */ +#define LOCAL_RETENTION 2 /* Local retention time */ +#define MIN_TIME LOCAL_RETENTION * 8 /* Make sure checkpoint and flush_tier run enough */ +#define MAX_TIME MIN_TIME * 4 + #define BUCKET "bucket" #define INVALID_KEY UINT64_MAX -#define MAX_CKPT_INVL 5 /* Maximum interval between checkpoints */ -#define MAX_FLUSH_INVL 5 /* Maximum interval between flush_tier calls */ -#define MAX_TH 20 /* Maximum configurable threads */ -#define MAX_TIME 40 +#define MAX_CKPT_INVL LOCAL_RETENTION * 3 /* Maximum interval between checkpoints */ +#define MAX_FLUSH_INVL LOCAL_RETENTION * 2 /* Maximum interval between flush_tier calls */ +#define MAX_TH 20 /* Maximum configurable threads */ #define MAX_VAL 1024 #define MIN_TH 5 -#define MIN_TIME 10 #define NUM_INT_THREADS 3 #define RECORDS_FILE "records-%" PRIu32 /* Include worker threads and extra sessions */ @@ -96,7 +98,7 @@ static uint32_t flush_calls = 1; "eviction_updates_target=20,eviction_updates_trigger=90," \ "log=(archive=true,file_max=10M,enabled),session_max=%d," \ "statistics=(fast),statistics_log=(wait=1,json=true)," \ - "tiered_storage=(bucket=%s,bucket_prefix=pfx,name=local_store)" + "tiered_storage=(bucket=%s,bucket_prefix=pfx,local_retention=%d,name=local_store)" #define ENV_CONFIG_TXNSYNC \ ENV_CONFIG_DEF \ ",eviction_dirty_target=20,eviction_dirty_trigger=90" \ @@ -440,8 +442,8 @@ run_workload(uint32_t nth, const char *build_dir) if (chdir(home) != 0) testutil_die(errno, "Child chdir: %s", home); - testutil_check( - __wt_snprintf(envconf, sizeof(envconf), ENV_CONFIG_TXNSYNC, cache_mb, SESSION_MAX, BUCKET)); + testutil_check(__wt_snprintf(envconf, sizeof(envconf), ENV_CONFIG_TXNSYNC, cache_mb, + SESSION_MAX, BUCKET, LOCAL_RETENTION)); testutil_check(__wt_snprintf(extconf, sizeof(extconf), ",extensions=(%s/%s=(early_load=true))", build_dir, WT_STORAGE_LIB)); @@ -490,7 +492,6 @@ run_workload(uint32_t nth, const char *build_dir) testutil_check(__wt_thread_create(NULL, &thr[ts_id], thread_ts_run, &td[ts_id])); } printf("Create %" PRIu32 " writer threads\n", nth); - printf("Create %" PRIu32 " writer threads\n", nth); for (i = 0; i < nth; ++i) { td[i].conn = conn; td[i].start = WT_BILLION * (uint64_t)i; diff --git a/src/third_party/wiredtiger/test/csuite/wt3338_partial_update/main.c b/src/third_party/wiredtiger/test/csuite/wt3338_partial_update/main.c index 8b6b9c5f841..569339eb4af 100644 --- a/src/third_party/wiredtiger/test/csuite/wt3338_partial_update/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt3338_partial_update/main.c @@ -111,94 +111,6 @@ modify_build(void) } /* - * slow_apply_api -- - * Apply a set of modification changes using a different algorithm. - */ -static void -slow_apply_api(WT_ITEM *orig) -{ - static WT_ITEM _tb; - WT_ITEM *ta, *tb, *tmp, _tmp; - size_t len, size; - int i; - - ta = orig; - tb = &_tb; - - /* Mess up anything not initialized in the buffers. */ - if ((ta->memsize - ta->size) > 0) - memset((uint8_t *)ta->mem + ta->size, 0xff, ta->memsize - ta->size); - - if (tb->memsize > 0) - memset((uint8_t *)tb->mem, 0xff, tb->memsize); - - /* - * Process the entries to figure out how large a buffer we need. This is a bit pessimistic - * because we're ignoring replacement bytes, but it's a simpler calculation. - */ - for (size = ta->size, i = 0; i < nentries; ++i) { - if (entries[i].offset >= size) - size = entries[i].offset; - size += entries[i].data.size; - } - - testutil_check(__wt_buf_grow(NULL, ta, size)); - testutil_check(__wt_buf_grow(NULL, tb, size)); - -#if DEBUG - show(ta, "slow-apply start"); -#endif - /* - * From the starting buffer, create a new buffer b based on changes in the entries array. We're - * doing a brute force solution here to test the faster solution implemented in the library. - */ - for (i = 0; i < nentries; ++i) { - /* Take leading bytes from the original, plus any gap bytes. */ - if (entries[i].offset >= ta->size) { - memcpy(tb->mem, ta->mem, ta->size); - if (entries[i].offset > ta->size) - memset((uint8_t *)tb->mem + ta->size, '\0', entries[i].offset - ta->size); - } else if (entries[i].offset > 0) - memcpy(tb->mem, ta->mem, entries[i].offset); - tb->size = entries[i].offset; - - /* Take replacement bytes. */ - if (entries[i].data.size > 0) { - memcpy((uint8_t *)tb->mem + tb->size, entries[i].data.data, entries[i].data.size); - tb->size += entries[i].data.size; - } - - /* Take trailing bytes from the original. */ - len = entries[i].offset + entries[i].size; - if (ta->size > len) { - memcpy((uint8_t *)tb->mem + tb->size, (uint8_t *)ta->mem + len, ta->size - len); - tb->size += ta->size - len; - } - testutil_assert(tb->size <= size); - - /* Swap the buffers and do it again. */ - tmp = ta; - ta = tb; - tb = tmp; - } - ta->data = ta->mem; - tb->data = tb->mem; - - /* - * The final results may not be in the original buffer, in which case we swap them back around. - */ - if (ta != orig) { - _tmp = *ta; - *ta = *tb; - *tb = _tmp; - } - -#if DEBUG - show(ta, "slow-apply finish"); -#endif -} - -/* * compare -- * Compare two results. */ @@ -249,6 +161,7 @@ modify_run(TEST_OPTS *opts) WT_CURSOR *cursor, _cursor; WT_DECL_RET; WT_ITEM *localA, _localA, *localB, _localB; + WT_ITEM modtmp; WT_SESSION_IMPL *session; size_t len; int i, j; @@ -264,7 +177,6 @@ modify_run(TEST_OPTS *opts) /* Set up replacement information. */ modify_repl_init(); - /* We need three WT_ITEMs, one of them part of a fake cursor. */ localA = &_localA; memset(&_localA, 0, sizeof(_localA)); localB = &_localB; @@ -273,6 +185,7 @@ modify_run(TEST_OPTS *opts) memset(&_cursor, 0, sizeof(_cursor)); cursor->session = (WT_SESSION *)session; cursor->value_format = "u"; + memset(&modtmp, 0, sizeof(modtmp)); #define NRUNS 10000 for (i = 0; i < NRUNS; ++i) { @@ -295,7 +208,7 @@ modify_run(TEST_OPTS *opts) modify_build(); testutil_check(__wt_buf_set(session, &cursor->value, localA->data, localA->size)); testutil_check(__wt_modify_apply_api(cursor, entries, nentries)); - slow_apply_api(localA); + testutil_modify_apply(localA, &modtmp, entries, nentries); compare(localB, localA, &cursor->value); /* @@ -324,6 +237,7 @@ modify_run(TEST_OPTS *opts) __wt_buf_free(session, localA); __wt_buf_free(session, localB); __wt_buf_free(session, &cursor->value); + __wt_buf_free(session, &modtmp); } int diff --git a/src/third_party/wiredtiger/test/csuite/wt7989_compact_checkpoint/main.c b/src/third_party/wiredtiger/test/csuite/wt7989_compact_checkpoint/main.c index 6e388b54535..a93d83b0985 100644 --- a/src/third_party/wiredtiger/test/csuite/wt7989_compact_checkpoint/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt7989_compact_checkpoint/main.c @@ -37,6 +37,7 @@ */ #define NUM_RECORDS 1000000 +#define CHECKPOINT_NUM 3 /* Constants and variables declaration. */ /* @@ -78,11 +79,13 @@ main(int argc, char *argv[]) /* * First run test with WT_TIMING_STRESS_CHECKPOINT_SLOW. */ + printf("Running stress test...\n"); run_test(true, opts->home, opts->uri); /* * Now run test where compact and checkpoint threads are synchronized using condition variable. */ + printf("Running normal test...\n"); testutil_assert(sizeof(home_cv) > strlen(opts->home) + 3); sprintf(home_cv, "%s.CV", opts->home); run_test(false, home_cv, opts->uri); @@ -110,8 +113,8 @@ run_test(bool stress_test, const char *home, const char *uri) if (stress_test) { /* - * Set WT_TIMING_STRESS_CHECKPOINT_SLOW flag. It adds 10 seconds sleep before each - * checkpoint. + * Set WT_TIMING_STRESS_CHECKPOINT_SLOW flag for stress test. It adds 10 seconds sleep + * before each checkpoint. */ set_timing_stress_checkpoint(conn); } @@ -161,17 +164,17 @@ run_test(bool stress_test, const char *home, const char *uri) } testutil_check(session->close(session, NULL)); + session = NULL; + + testutil_check(conn->close(conn, NULL)); + conn = NULL; /* Check if there's at least 10% compaction. */ printf(" - Compressed file size MB: %f\n - Original file size MB: %f\n", file_sz_after / (1024.0 * 1024), file_sz_before / (1024.0 * 1024)); - /* - * FIXME-WT-8055 At the moment the assert below is commented out to prevent evergreen from going - * red. Please enable the assert as soon as the underlying defect is fixed and compact does its - * job well. - */ - /*testutil_assert(file_sz_before * 0.9 > file_sz_after);*/ + /* Make sure the compact operation has reduced the file size by at least 20%. */ + testutil_assert((file_sz_before / 100) * 80 > file_sz_after); } static void * @@ -198,7 +201,9 @@ thread_func_compact(void *arg) /* Perform compact operation. */ testutil_check(session->compact(session, td->uri, NULL)); + testutil_check(session->close(session, NULL)); + session = NULL; return (NULL); } @@ -219,13 +224,18 @@ static void * thread_func_checkpoint(void *arg) { struct thread_data *td; + WT_RAND_STATE rnd; WT_SESSION *session; + uint64_t sleep_sec; + int i; bool signalled; td = (struct thread_data *)arg; testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + __wt_random_init_seed((WT_SESSION_IMPL *)session, &rnd); + if (td->cond != NULL) { printf("Waiting for the signal...\n"); /* @@ -238,8 +248,22 @@ thread_func_checkpoint(void *arg) printf("Signal received!\n"); } - testutil_check(session->checkpoint(session, NULL)); + /* + * Run several checkpoints. First one without any delay. Others will have a random delay before + * start. + */ + for (i = 0; i < CHECKPOINT_NUM; i++) { + testutil_check(session->checkpoint(session, NULL)); + + if (i < CHECKPOINT_NUM - 1) { + sleep_sec = (uint64_t)__wt_random(&rnd) % 15 + 1; + printf("Sleep %" PRIu64 " sec before next checkpoint.\n", sleep_sec); + __wt_sleep(sleep_sec, 0); + } + } + testutil_check(session->close(session, NULL)); + session = NULL; return (NULL); } @@ -248,27 +272,28 @@ static void populate(WT_SESSION *session, const char *uri) { WT_CURSOR *cursor; - time_t t; + WT_RAND_STATE rnd; uint64_t val; int i, str_len; - srand((u_int)time(&t)); + __wt_random_init_seed((WT_SESSION_IMPL *)session, &rnd); str_len = sizeof(data_str) / sizeof(data_str[0]); for (i = 0; i < str_len - 1; i++) - data_str[i] = 'a' + rand() % 26; + data_str[i] = 'a' + __wt_random(&rnd) % 26; data_str[str_len - 1] = '\0'; testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); for (i = 0; i < NUM_RECORDS; i++) { cursor->set_key(cursor, i); - val = (uint64_t)rand(); + val = (uint64_t)__wt_random(&rnd); cursor->set_value(cursor, val, val, val, data_str); testutil_check(cursor->insert(cursor)); } testutil_check(cursor->close(cursor)); + cursor = NULL; } static void @@ -286,6 +311,7 @@ remove_records(WT_SESSION *session, const char *uri) } testutil_check(cursor->close(cursor)); + cursor = NULL; } static uint64_t @@ -293,8 +319,7 @@ get_file_size(WT_SESSION *session, const char *uri) { WT_CURSOR *cur_stat; uint64_t val; - char *descr, *str_val; - char stat_uri[128]; + char *descr, *str_val, stat_uri[128]; sprintf(stat_uri, "statistics:%s", uri); testutil_check(session->open_cursor(session, stat_uri, NULL, "statistics=(all)", &cur_stat)); @@ -302,6 +327,7 @@ get_file_size(WT_SESSION *session, const char *uri) testutil_check(cur_stat->search(cur_stat)); testutil_check(cur_stat->get_value(cur_stat, &descr, &str_val, &val)); testutil_check(cur_stat->close(cur_stat)); + cur_stat = NULL; return (val); } diff --git a/src/third_party/wiredtiger/test/ctest_helpers.cmake b/src/third_party/wiredtiger/test/ctest_helpers.cmake index 1a8a7b4152e..a6afb901f71 100644 --- a/src/third_party/wiredtiger/test/ctest_helpers.cmake +++ b/src/third_party/wiredtiger/test/ctest_helpers.cmake @@ -153,16 +153,23 @@ function(define_test_variants target) 1 "DEFINE_TEST" "" - "" - "VARIANTS;LABELS" + "DIR_NAME" + "VARIANTS;LABELS;CMDS" ) if (NOT "${DEFINE_TEST_UNPARSED_ARGUMENTS}" STREQUAL "") - message(FATAL_ERROR "Unknown arguments to define_test_variants: ${DEFINE_TEST_VARIANTS_UNPARSED_ARGUMENTS}") + message(FATAL_ERROR "Unknown arguments to define_test_variants: ${DEFINE_TEST_UNPARSED_ARGUMENTS}") endif() if ("${DEFINE_TEST_VARIANTS}" STREQUAL "") message(FATAL_ERROR "Need at least one variant for define_test_variants") endif() + set(dir_prefix) + if(DEFINE_TEST_DIR_NAME) + set(dir_prefix ${CMAKE_CURRENT_BINARY_DIR}/${DEFINE_TEST_DIR_NAME}) + else() + set(dir_prefix ${CMAKE_CURRENT_BINARY_DIR}) + endif() + set(defined_tests) foreach(variant ${DEFINE_TEST_VARIANTS}) list(LENGTH variant variant_length) @@ -182,17 +189,23 @@ function(define_test_variants target) endif() # Create a variant directory to run the test in. add_custom_command(OUTPUT ${curr_variant_name}_test_dir - COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/${curr_variant_name}_test_dir + COMMAND ${CMAKE_COMMAND} -E make_directory ${dir_prefix}/${curr_variant_name}_test_dir ) add_custom_target(create_dir_${curr_variant_name} DEPENDS ${curr_variant_name}_test_dir) # Ensure the variant target is created prior to building the test. add_dependencies(${target} create_dir_${curr_variant_name}) + set(test_cmd) + if(DEFINE_TEST_CMDS) + set(test_cmd ${DEFINE_TEST_CMDS}) + else() + set(test_cmd $<TARGET_FILE:${target}>) + endif() add_test( NAME ${curr_variant_name} - COMMAND $<TARGET_FILE:${target}> ${variant_args} + COMMAND ${test_cmd} ${variant_args} # Run each variant in its own subdirectory, allowing us to execute variants in # parallel. - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${curr_variant_name}_test_dir + WORKING_DIRECTORY ${dir_prefix}/${curr_variant_name}_test_dir ) list(APPEND defined_tests ${curr_variant_name}) endforeach() @@ -201,13 +214,14 @@ function(define_test_variants target) endif() endfunction() -macro(define_c_test) +function(define_c_test) cmake_parse_arguments( + PARSE_ARGV + 0 "C_TEST" - "SMOKE" - "TARGET;DIR_NAME;DEPENDS" - "SOURCES;FLAGS;ARGUMENTS" - ${ARGN} + "" + "TARGET;DIR_NAME;DEPENDS;EXEC_SCRIPT" + "SOURCES;FLAGS;ARGUMENTS;VARIANTS" ) if (NOT "${C_TEST_UNPARSED_ARGUMENTS}" STREQUAL "") message(FATAL_ERROR "Unknown arguments to define_c_test: ${C_TEST_UNPARSED_ARGUMENTS}") @@ -222,6 +236,10 @@ macro(define_c_test) message(FATAL_ERROR "No directory given to define_c_test") endif() + if("${C_TEST_ARGUMENTS}" AND "${C_TEST_VARIANTS}") + message(FATAL_ERROR "Can't pass both ARGUMENTS and VARIANTS, use only one") + endif() + # Check that the csuite dependencies are enabled before compiling and creating the test. eval_dependency("${C_TEST_DEPENDS}" enabled) if(enabled) @@ -236,35 +254,40 @@ macro(define_c_test) # Which while technically valid breaks assumptions in our testing utilities. Wrap the execution in powershell to avoid this. set(exec_wrapper "powershell.exe") endif() - if (C_TEST_SMOKE) - # csuite test comes with a smoke execution wrapper. + set(test_cmd) + if (C_TEST_EXEC_SCRIPT) + # Define the c test to be executed with a script, rather than invoking the binary directly. create_test_executable(${C_TEST_TARGET} SOURCES ${C_TEST_SOURCES} - ADDITIONAL_FILES ${CMAKE_CURRENT_SOURCE_DIR}/${C_TEST_DIR_NAME}/smoke.sh + ADDITIONAL_FILES ${C_TEST_EXEC_SCRIPT} BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/${C_TEST_DIR_NAME} ${additional_executable_args} ) - add_test(NAME ${C_TEST_TARGET} - COMMAND ${exec_wrapper} ${CMAKE_CURRENT_BINARY_DIR}/${C_TEST_DIR_NAME}/smoke.sh ${C_TEST_ARGUMENTS} $<TARGET_FILE:${C_TEST_TARGET}> - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${C_TEST_DIR_NAME} - ) + get_filename_component(exec_script_basename ${C_TEST_EXEC_SCRIPT} NAME) + set(test_cmd ${exec_wrapper} ${exec_script_basename}) else() create_test_executable(${C_TEST_TARGET} SOURCES ${C_TEST_SOURCES} BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/${C_TEST_DIR_NAME} ${additional_executable_args} ) - # Take a CMake-based path and convert it to a platform-specfic path (/ for Unix, \ for Windows). - set(wt_test_home_dir ${CMAKE_CURRENT_BINARY_DIR}/${C_TEST_DIR_NAME}/WT_HOME_${C_TEST_TARGET}) - file(TO_NATIVE_PATH "${wt_test_home_dir}" wt_test_home_dir) - # Ensure each DB home directory is run under the tests working directory. - set(command_args -h ${wt_test_home_dir}) - list(APPEND command_args ${C_TEST_ARGUMENTS}) + set(test_cmd ${exec_wrapper} $<TARGET_FILE:${C_TEST_TARGET}>) + endif() + # Define the ctest target. + if(C_TEST_VARIANTS) + # If we want to define multiple variant executions of the test script/binary. + define_test_variants(${C_TEST_TARGET} + VARIANTS ${C_TEST_VARIANTS} + CMDS ${test_cmd} + DIR_NAME ${C_TEST_DIR_NAME} + LABELS "check;csuite" + ) + else() add_test(NAME ${C_TEST_TARGET} - COMMAND ${exec_wrapper} $<TARGET_FILE:${C_TEST_TARGET}> ${command_args} + COMMAND ${test_cmd} ${C_TEST_ARGUMENTS} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${C_TEST_DIR_NAME} ) + set_tests_properties(${C_TEST_TARGET} PROPERTIES LABELS "check;csuite") endif() - list(APPEND c_tests ${C_TEST_TARGET}) endif() -endmacro(define_c_test) +endfunction(define_c_test) diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index ba9f33806d5..a775a9aec7a 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -317,7 +317,7 @@ functions: if [ ${is_cmake_build|false} = true ]; then . test/evergreen/find_cmake.sh cd cmake_build - ${test_env_vars|} $CTEST -L check ${smp_command|} -VV 2>&1 + ${test_env_vars|} $CTEST -L check ${smp_command|} --output-on-failure 2>&1 else cd build_posix ${test_env_vars|} ${make_command|make} VERBOSE=1 check ${smp_command|} 2>&1 @@ -554,6 +554,35 @@ functions: done done + "generic-perf-test": + # Run a performance test + # Parameterised using the 'perf-test-name' and 'maxruns' variables + - command: shell.exec + params: + working_dir: "wiredtiger/bench/wtperf/wtperf_run_py" + shell: bash + script: | + set -o errexit + set -o verbose + ${virtualenv_binary} -p ${python_binary} venv + source venv/bin/activate + ${pip3_binary} install psutil + ${python_binary} wtperf_run.py -p ../../../cmake_build/bench/wtperf/wtperf -t ../runners/${perf-test-name}.wtperf -v -ho WT_TEST -m ${maxruns} -o out.json + + "generic-perf-test-push-results": + # Push the json results to the 'Files' tab of the task in Evergreen + # Parameterised using the 'perf-test-name' variable + - command: s3.put + params: + aws_secret: ${aws_secret} + aws_key: ${aws_key} + local_file: wiredtiger/bench/wtperf/wtperf_run_py/out.json + bucket: build_external + permissions: public-read + content_type: text/html + display_name: "Test results (JSON)" + remote_file: wiredtiger/${build_variant}/${revision}/perf-test-${perf-test-name}-${build_id}-${execution}/test-results.json + ######################################################################################### # VARIABLES # @@ -968,6 +997,22 @@ tasks: ${test_env_vars|} $(pwd)/test/cppsuite/run -t hs_cleanup -C 'debug_mode=(cursor_copy=true)' -f test/cppsuite/configs/hs_cleanup_default.txt -l 2 + - name: cppsuite-search-near-default + tags: ["pull_request"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix/" + script: | + set -o errexit + set -o verbose + + ${test_env_vars|} $(pwd)/test/cppsuite/run -t search_near_01 -f test/cppsuite/configs/search_near_01_default.txt -l 2 + ${test_env_vars|} $(pwd)/test/cppsuite/run -t search_near_02 -f test/cppsuite/configs/search_near_02_default.txt -l 2 + - name: cppsuite-base-test-stress depends_on: - name: compile @@ -2366,6 +2411,10 @@ tasks: - func: "random abort test" vars: random_abort_args: -t 40 + # random-abort - run compaction + - func: "random abort test" + vars: + random_abort_args: -c -t 60 # truncated-log - func: "truncated log test" @@ -2378,6 +2427,12 @@ tasks: vars: extra_args: file_type=row + # format test for stressing compaction code path + - func: "format test" + vars: + times: 3 + extra_args: file_type=row compaction=1 verify=1 runs.timer=3 ops.pct.delete=30 + #FIXME-WT-5270: Add wtperf testing from Jenkin "wiredtiger-test-check-long" after fixing WT-5270 - name: time-shift-sensitivity-test @@ -2847,6 +2902,111 @@ tasks: set -o errexit python "../metrixplusplus/metrix++.py" limit --max-limit=std.code.complexity:cyclomatic:95 + ############################# + # Performance Tests for lsm # + ############################# + + - name: perf-test-small-lsm + commands: + - func: "get project" + - func: "compile wiredtiger" + - func: "generic-perf-test" + vars: + perf-test-name: small-lsm + maxruns: 3 + - func: "generic-perf-test-push-results" + vars: + perf-test-name: small-lsm + + - name: perf-test-medium-lsm + commands: + - func: "get project" + - func: "compile wiredtiger" + - func: "generic-perf-test" + vars: + perf-test-name: medium-lsm + maxruns: 1 + - func: "generic-perf-test-push-results" + vars: + perf-test-name: medium-lsm + + - name: perf-test-medium-lsm-compact + commands: + - func: "get project" + - func: "compile wiredtiger" + - func: "generic-perf-test" + vars: + perf-test-name: medium-lsm-compact + maxruns: 1 + - func: "generic-perf-test-push-results" + vars: + perf-test-name: medium-lsm-compact + + - name: perf-test-medium-multi-lsm + commands: + - func: "get project" + - func: "compile wiredtiger" + - func: "generic-perf-test" + vars: + perf-test-name: medium-multi-lsm + maxruns: 1 + - func: "generic-perf-test-push-results" + vars: + perf-test-name: medium-multi-lsm + + ############################### + # Performance Tests for btree # + ############################### + + - name: perf-test-small-btree + commands: + - func: "get project" + - func: "compile wiredtiger" + - func: "generic-perf-test" + vars: + perf-test-name: small-btree + maxruns: 1 + - func: "generic-perf-test-push-results" + vars: + perf-test-name: small-btree + + - name: perf-test-small-btree-backup + commands: + - func: "get project" + - func: "compile wiredtiger" + - func: "generic-perf-test" + vars: + perf-test-name: small-btree-backup + maxruns: 1 + - func: "generic-perf-test-push-results" + vars: + perf-test-name: small-btree-backup + + - name: perf-test-medium-btree + commands: + - func: "get project" + - func: "compile wiredtiger" + - func: "generic-perf-test" + vars: + perf-test-name: medium-btree + maxruns: 3 + - func: "generic-perf-test-push-results" + vars: + perf-test-name: medium-btree + + - name: perf-test-medium-btree-backup + commands: + - func: "get project" + - func: "compile wiredtiger" + - func: "generic-perf-test" + vars: + perf-test-name: medium-btree-backup + maxruns: 3 + - func: "generic-perf-test-push-results" + vars: + perf-test-name: medium-btree-backup + + buildvariants: - name: ubuntu2004 @@ -2990,6 +3150,7 @@ buildvariants: - name: make-check-test - name: cppsuite-base-test-default - name: cppsuite-hs-cleanup-default + - name: cppsuite-search-near-default - name: ubuntu2004-compilers display_name: "! Ubuntu 20.04 Compilers" @@ -3024,6 +3185,33 @@ buildvariants: - name: ".stress-test-4" - name: format-abort-recovery-stress-test +- name: ubuntu2004-perf-tests + display_name: Ubuntu 20.04 Performance tests + run_on: + - ubuntu2004-test + expansions: + test_env_vars: LD_LIBRARY_PATH=$(pwd) WT_BUILDDIR=$(pwd) + posix_configure_flags: -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/mongodbtoolchain_v3_gcc.cmake -DCMAKE_C_FLAGS="-ggdb" -DHAVE_DIAGNOSTIC=1 -DENABLE_ZLIB=1 -DENABLE_SNAPPY=1 -DENABLE_STRICT=1 -DCMAKE_INSTALL_PREFIX=$(pwd)/LOCAL_INSTALL + python_binary: '/opt/mongodbtoolchain/v3/bin/python3' + pip3_binary: '/opt/mongodbtoolchain/v3/bin/pip3' + virtualenv_binary: '/opt/mongodbtoolchain/v3/bin/virtualenv' + smp_command: -j $(echo "`grep -c ^processor /proc/cpuinfo` * 2" | bc) + cmake_generator: Ninja + make_command: ninja + is_cmake_build: true + tasks: + # btree tests + - name: perf-test-small-btree + - name: perf-test-small-btree-backup + - name: perf-test-medium-btree + - name: perf-test-medium-btree-backup + # lsm tests + - name: perf-test-small-lsm + - name: perf-test-medium-lsm + - name: perf-test-medium-lsm-compact + - name: perf-test-medium-multi-lsm + + - name: large-scale-tests display_name: "Large scale tests" batchtime: 480 # 3 times a day @@ -3052,8 +3240,8 @@ buildvariants: test_env_vars: LD_LIBRARY_PATH=$(pwd)/../../.libs PATH=/opt/mongodbtoolchain/v3/bin:$PATH make_command: PATH=/opt/mongodbtoolchain/v3/bin:$PATH make posix_configure_flags: - --enable-silent-rules --enable-python --enable-zlib --enable-snappy - --enable-strict --enable-static + --enable-diagnostic --enable-python --enable-silent-rules --enable-snappy --enable-static + --enable-strict --enable-zlib tasks: - name: compile - name: cppsuite-hs-cleanup-stress diff --git a/src/third_party/wiredtiger/test/format/config.h b/src/third_party/wiredtiger/test/format/config.h index cfb4f78ba27..d12c30ed756 100644 --- a/src/third_party/wiredtiger/test/format/config.h +++ b/src/third_party/wiredtiger/test/format/config.h @@ -353,9 +353,6 @@ static CONFIG c[] = { /* 2% */ {"stress.split_7", "stress splits (#7)", C_BOOL, 2, 0, 0, &g.c_timing_stress_split_7, NULL}, - /* 2% */ - {"stress.split_8", "stress splits (#8)", C_BOOL, 2, 0, 0, &g.c_timing_stress_split_8, NULL}, - {"transaction.implicit", "implicit, without timestamps, transactions (percentage)", 0x0, 0, 100, 100, &g.c_txn_implicit, NULL}, diff --git a/src/third_party/wiredtiger/test/format/config_compat.c b/src/third_party/wiredtiger/test/format/config_compat.c index 2926d54ca4b..935cbea5eab 100644 --- a/src/third_party/wiredtiger/test/format/config_compat.c +++ b/src/third_party/wiredtiger/test/format/config_compat.c @@ -181,8 +181,6 @@ static const char *list[] = { "stress.split_6", "timing_stress_split_7=", "stress.split_7", - "timing_stress_split_8=", - "stress.split_8", "transaction-frequency=", "transaction.frequency", "transaction_timestamps=", diff --git a/src/third_party/wiredtiger/test/format/config_compat.sed b/src/third_party/wiredtiger/test/format/config_compat.sed index b90b21332e8..81d7a58f025 100644 --- a/src/third_party/wiredtiger/test/format/config_compat.sed +++ b/src/third_party/wiredtiger/test/format/config_compat.sed @@ -78,7 +78,6 @@ s/^stress.split_4=/timing_stress_split_4=/ s/^stress.split_5=/timing_stress_split_5=/ s/^stress.split_6=/timing_stress_split_6=/ s/^stress.split_7=/timing_stress_split_7=/ -s/^stress.split_8=/timing_stress_split_8=/ s/^transaction.frequency=/transaction-frequency=/ s/^transaction.isolation=/isolation=/ s/^transaction.timestamps=/transaction_timestamps=/ diff --git a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-5637 b/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-5637 index 68338d050ae..8e27f5b5967 100644 --- a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-5637 +++ b/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-5637 @@ -91,7 +91,6 @@ stress.split_4=0 stress.split_5=0 stress.split_6=0 stress.split_7=0 -stress.split_8=0 transaction.frequency=100 transaction.isolation=snapshot transaction.rollback_to_stable=0 diff --git a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-6568 b/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-6568 index 4e695a8d87e..3f3192a5ef2 100644 --- a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-6568 +++ b/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-6568 @@ -90,7 +90,6 @@ stress.split_4=0 stress.split_5=0 stress.split_6=1 stress.split_7=0 -stress.split_8=0 transaction.frequency=100 transaction.isolation=snapshot transaction.rollback_to_stable=0 diff --git a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-6725 b/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-6725 index 144d9aaadd4..54aafd08f0b 100644 --- a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-6725 +++ b/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-6725 @@ -90,7 +90,6 @@ stress.split_5=0 stress.split_6=0 stress.split_7=0 - stress.split_8=0 transaction.frequency=100 transaction.isolation=snapshot transaction.rollback_to_stable=0 diff --git a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-6727 b/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-6727 index 62b3f7bba5f..587a8e5b978 100644 --- a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-6727 +++ b/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-6727 @@ -87,7 +87,6 @@ stress.split_4=0 stress.split_5=0 stress.split_6=0 stress.split_7=0 -stress.split_8=0 transaction.frequency=100 transaction.isolation=snapshot transaction.timestamps=1 diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h index ecfb83a37ce..cce28406706 100644 --- a/src/third_party/wiredtiger/test/format/format.h +++ b/src/third_party/wiredtiger/test/format/format.h @@ -264,7 +264,6 @@ typedef struct { uint32_t c_timing_stress_split_5; uint32_t c_timing_stress_split_6; uint32_t c_timing_stress_split_7; - uint32_t c_timing_stress_split_8; uint32_t c_truncate; uint32_t c_txn_implicit; uint32_t c_txn_timestamps; @@ -402,7 +401,8 @@ typedef struct { uint64_t insert_list[256]; /* column-store inserted records */ u_int insert_list_cnt; - WT_ITEM vprint; /* Temporary buffer for printable values */ + WT_ITEM vprint; /* Temporary buffer for printable values */ + WT_ITEM moda, modb; /* Temporary buffer for modify operations */ #define TINFO_RUNNING 1 /* Running */ #define TINFO_COMPLETE 2 /* Finished */ diff --git a/src/third_party/wiredtiger/test/format/ops.c b/src/third_party/wiredtiger/test/format/ops.c index 81d21111ec3..4c24ba9fd45 100644 --- a/src/third_party/wiredtiger/test/format/ops.c +++ b/src/third_party/wiredtiger/test/format/ops.c @@ -171,6 +171,8 @@ tinfo_teardown(void) tinfo = tinfo_list[i]; __wt_buf_free(NULL, &tinfo->vprint); + __wt_buf_free(NULL, &tinfo->moda); + __wt_buf_free(NULL, &tinfo->modb); /* * Assert records were not removed unless configured to do so, otherwise subsequent runs can @@ -1454,26 +1456,50 @@ modify_build(TINFO *tinfo, WT_MODIFY *entries, int *nentriesp) } /* + * modify -- + * Cursor modify worker function. + */ +static int +modify(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) +{ + WT_MODIFY entries[MAX_MODIFY_ENTRIES]; + int nentries; + bool modify_check; + + /* Periodically verify the WT_CURSOR.modify return. */ + modify_check = positioned && mmrand(&tinfo->rnd, 1, 10) == 1; + if (modify_check) { + testutil_check(cursor->get_value(cursor, &tinfo->moda)); + testutil_check( + __wt_buf_set(CUR2S(cursor), &tinfo->moda, tinfo->moda.data, tinfo->moda.size)); + } + + modify_build(tinfo, entries, &nentries); + WT_RET(cursor->modify(cursor, entries, nentries)); + + testutil_check(cursor->get_value(cursor, tinfo->value)); + if (modify_check) { + testutil_modify_apply(&tinfo->moda, &tinfo->modb, entries, nentries); + testutil_assert(tinfo->moda.size == tinfo->value->size && + memcmp(tinfo->moda.data, tinfo->value->data, tinfo->moda.size) == 0); + } + return (0); +} + +/* * row_modify -- * Modify a row in a row-store file. */ static int row_modify(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) { - WT_DECL_RET; - WT_MODIFY entries[MAX_MODIFY_ENTRIES]; - int nentries; if (!positioned) { key_gen(tinfo->key, tinfo->keyno); cursor->set_key(cursor, tinfo->key); } - modify_build(tinfo, entries, &nentries); - if ((ret = cursor->modify(cursor, entries, nentries)) != 0) - return (ret); - - testutil_check(cursor->get_value(cursor, tinfo->value)); + WT_RET(modify(tinfo, cursor, positioned)); trace_op(tinfo, "modify %" PRIu64 " {%.*s}, {%s}", tinfo->keyno, (int)tinfo->key->size, (char *)tinfo->key->data, trace_item(tinfo, tinfo->value)); @@ -1488,18 +1514,10 @@ row_modify(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) static int col_modify(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) { - WT_DECL_RET; - WT_MODIFY entries[MAX_MODIFY_ENTRIES]; - int nentries; - if (!positioned) cursor->set_key(cursor, tinfo->keyno); - modify_build(tinfo, entries, &nentries); - if ((ret = cursor->modify(cursor, entries, nentries)) != 0) - return (ret); - - testutil_check(cursor->get_value(cursor, tinfo->value)); + WT_RET(modify(tinfo, cursor, positioned)); trace_op(tinfo, "modify %" PRIu64 ", {%s}", tinfo->keyno, trace_item(tinfo, tinfo->value)); diff --git a/src/third_party/wiredtiger/test/format/wts.c b/src/third_party/wiredtiger/test/format/wts.c index d9f0c81a34c..74027748246 100644 --- a/src/third_party/wiredtiger/test/format/wts.c +++ b/src/third_party/wiredtiger/test/format/wts.c @@ -292,8 +292,6 @@ create_database(const char *home, WT_CONNECTION **connp) CONFIG_APPEND(p, ",split_6"); if (g.c_timing_stress_split_7) CONFIG_APPEND(p, ",split_7"); - if (g.c_timing_stress_split_8) - CONFIG_APPEND(p, ",split_8"); CONFIG_APPEND(p, "]"); /* Extensions. */ @@ -523,8 +521,6 @@ wts_open(const char *home, WT_CONNECTION **connp, WT_SESSION **sessionp, bool al CONFIG_APPEND(p, ",split_6"); if (g.c_timing_stress_split_7) CONFIG_APPEND(p, ",split_7"); - if (g.c_timing_stress_split_8) - CONFIG_APPEND(p, ",split_8"); CONFIG_APPEND(p, "]"); /* If in-memory, there's only a single, shared WT_CONNECTION handle. */ diff --git a/src/third_party/wiredtiger/test/suite/test_backup10.py b/src/third_party/wiredtiger/test/suite/test_backup10.py index 36593c205e2..988b8467e87 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup10.py +++ b/src/third_party/wiredtiger/test/suite/test_backup10.py @@ -56,6 +56,7 @@ class test_backup10(backup_base): def test_backup10(self): log2 = "WiredTigerLog.0000000002" log3 = "WiredTigerLog.0000000003" + log4 = "WiredTigerLog.0000000004" self.session.create(self.uri, "key_format=S,value_format=S") @@ -84,7 +85,8 @@ class test_backup10(backup_base): # We expect that the duplicate logs are a superset of the # original logs. And we expect the difference to be the - # addition of log file 3 only. + # addition of two log files, one switch when opening the backup + # cursor and a switch when opening the duplicate cursor. orig_set = set(orig_logs) dup_set = set(dup_logs) self.assertTrue(dup_set.issuperset(orig_set)) @@ -92,6 +94,7 @@ class test_backup10(backup_base): self.assertEqual(len(diff), 1) self.assertTrue(log3 in dup_set) self.assertFalse(log3 in orig_set) + self.assertFalse(log4 in dup_set) # Test a few error cases now. # - We cannot make multiple duplcate backup cursors. @@ -114,20 +117,6 @@ class test_backup10(backup_base): lambda:self.assertEquals(self.session.open_cursor(None, bkup_c, None), 0), msg) - # Open duplicate backup cursor again now that the first - # one is closed. Test every log file returned is the same - # as the first time. - dupc = self.session.open_cursor(None, bkup_c, config) - while True: - ret = dupc.next() - if ret != 0: - break - newfile = dupc.get_key() - self.assertTrue("WiredTigerLog" in newfile) - self.assertTrue(newfile in dup_logs) - self.assertEqual(ret, wiredtiger.WT_NOTFOUND) - - dupc.close() bkup_c.close() # After the full backup, open and recover the backup database. diff --git a/src/third_party/wiredtiger/test/suite/test_cursor17.py b/src/third_party/wiredtiger/test/suite/test_cursor17.py new file mode 100644 index 00000000000..322b82a65f8 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_cursor17.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_cursor17.py +# Test the largest_key interface under various scenarios. +# +import wttest +import wiredtiger +from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet +from wtscenario import make_scenarios + +class test_cursor17(wttest.WiredTigerTestCase): + tablename = 'test_cursor17' + + # Enable the lsm tests once it is supported. + types = [ + ('file-row', dict(type='file:', keyformat='i', valueformat='i', dataset=SimpleDataSet)), + ('table-row', dict(type='table:', keyformat='i', valueformat='i', dataset=SimpleDataSet)), + ('file-var', dict(type='file:', keyformat='r', valueformat='i', dataset=SimpleDataSet)), + ('table-var', dict(type='table:', keyformat='r', valueformat='i', dataset=SimpleDataSet)), + ('file-fix', dict(type='file:', keyformat='r', valueformat='8t', dataset=SimpleDataSet)), + # ('lsm', dict(type='lsm:', keyformat='i', valueformat='i', dataset=SimpleDataSet)), + ('table-r-complex', dict(type='table:', keyformat='r', valueformat=None, + dataset=ComplexDataSet)), + # ('table-i-complex-lsm', dict(type='table:', keyformat='i', valueformat=None, + # dataset=ComplexLSMDataSet)), + ] + + scenarios = make_scenarios(types) + + def populate(self, rownum): + if self.valueformat != None: + self.ds = self.dataset(self, self.type + self.tablename, rownum, key_format=self.keyformat, value_format=self.valueformat) + else: + self.ds = self.dataset(self, self.type + self.tablename, rownum, key_format=self.keyformat) + self.ds.populate() + + def test_globally_deleted_key(self): + self.populate(100) + + # Delete the largest key. + cursor = self.session.open_cursor(self.type + self.tablename, None) + self.session.begin_transaction() + cursor.set_key(100) + self.assertEqual(cursor.remove(), 0) + self.session.commit_transaction() + + # Verify the key is not visible. + self.session.begin_transaction() + cursor.set_key(100) + if self.valueformat != '8t': + self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) + else: + self.assertEqual(cursor.search(), 0) + self.session.rollback_transaction() + + # Verify the largest key. + self.session.begin_transaction() + self.assertEqual(cursor.largest_key(), 0) + self.assertEqual(cursor.get_key(), 100) + self.session.rollback_transaction() + + # Verify the key is still not visible after the largest call. + self.session.begin_transaction() + cursor.set_key(100) + if self.valueformat != '8t': + self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) + else: + self.assertEqual(cursor.search(), 0) + self.session.rollback_transaction() + + # Use evict cursor to evict the key from memory. + evict_cursor = self.session.open_cursor(self.type + self.tablename, None, "debug=(release_evict)") + evict_cursor.set_key(100) + if self.valueformat != '8t': + self.assertEquals(evict_cursor.search(), wiredtiger.WT_NOTFOUND) + else: + self.assertEquals(evict_cursor.search(), 0) + evict_cursor.close() + + # Verify the largest key changed. + self.session.begin_transaction() + self.assertEqual(cursor.largest_key(), 0) + if self.valueformat != '8t': + self.assertEqual(cursor.get_key(), 99) + else: + self.assertEquals(cursor.get_key(), 100) + self.session.rollback_transaction() + + def test_uncommitted_insert(self): + self.populate(100) + + session2 = self.setUpSessionOpen(self.conn) + cursor2 = session2.open_cursor(self.type + self.tablename, None) + session2.begin_transaction() + cursor2[101] = self.ds.value(101) + + cursor = self.session.open_cursor(self.type + self.tablename, None) + + # Verify the largest key. + self.session.begin_transaction() + self.assertEqual(cursor.largest_key(), 0) + self.assertEqual(cursor.get_key(), 101) + self.session.rollback_transaction() + + session2.rollback_transaction() + + def test_read_timestamp(self): + self.populate(100) + + cursor = self.session.open_cursor(self.type + self.tablename, None) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(5)) + # Expect the largest key to throw. + with self.expectedStderrPattern("largest key cannot be called with a read timestamp"): + try: + cursor.largest_key() + except wiredtiger.WiredTigerError as e: + gotException = True + self.pr('got expected exception: ' + str(e)) + self.assertTrue(str(e).find('nvalid argument') >= 0) + self.assertTrue(gotException, msg = 'expected exception') + self.session.rollback_transaction() + + def test_not_positioned(self): + self.populate(100) + + cursor = self.session.open_cursor(self.type + self.tablename, None) + # Verify the largest key. + self.session.begin_transaction() + self.assertEqual(cursor.largest_key(), 0) + self.assertEqual(cursor.get_key(), 100) + + # Call prev + self.assertEqual(cursor.prev(), 0) + self.assertEqual(cursor.get_key(), 100) + + # Verify the largest key again. + self.assertEqual(cursor.largest_key(), 0) + self.assertEqual(cursor.get_key(), 100) + + self.assertEqual(cursor.next(), 0) + self.assertEqual(cursor.get_key(), 1) + self.session.rollback_transaction() + + def test_get_value(self): + self.populate(100) + + cursor = self.session.open_cursor(self.type + self.tablename, None) + # Verify the largest key. + self.session.begin_transaction() + self.assertEqual(cursor.largest_key(), 0) + self.assertEqual(cursor.get_key(), 100) + with self.expectedStderrPattern("requires value be set"): + try: + cursor.get_value() + except wiredtiger.WiredTigerError as e: + gotException = True + self.pr('got expected exception: ' + str(e)) + self.assertTrue(str(e).find('nvalid argument') >= 0) + self.assertTrue(gotException, msg = 'expected exception') + self.session.rollback_transaction() + + def test_empty_table(self): + self.populate(0) + + cursor = self.session.open_cursor(self.type + self.tablename, None) + # Verify the largest key. + self.session.begin_transaction() + self.assertEquals(cursor.largest_key(), wiredtiger.WT_NOTFOUND) + self.session.rollback_transaction() + + def test_fast_truncate(self): + self.populate(100) + + # evict all the pages + evict_cursor = self.session.open_cursor(self.type + self.tablename, None, "debug=(release_evict)") + self.session.begin_transaction() + for i in range(1, 101): + evict_cursor.set_key(i) + self.assertEquals(evict_cursor.search(), 0) + self.session.rollback_transaction() + evict_cursor.close() + + # truncate + cursor = self.session.open_cursor(self.type + self.tablename, None) + self.session.begin_transaction() + cursor.set_key(1) + self.session.truncate(None, cursor, None, None) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) + + # verify the largest key + self.session.begin_transaction() + self.assertEqual(cursor.largest_key(), 0) + self.assertEqual(cursor.get_key(), 100) + self.session.rollback_transaction() + + def test_slow_truncate(self): + self.populate(100) + + # truncate + cursor = self.session.open_cursor(self.type + self.tablename, None) + self.session.begin_transaction() + cursor.set_key(100) + self.session.truncate(None, cursor, None, None) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) + + # verify the largest key + self.session.begin_transaction() + self.assertEqual(cursor.largest_key(), 0) + self.assertEqual(cursor.get_key(), 100) + self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py index 25aaae7ff67..63989155146 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py @@ -173,6 +173,10 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): def test_rollback_to_stable_prepare(self): nrows = 1000 + # FIXME-WT-7250 This test fails because of cache stuck on Windows. + if os.name == "nt": + self.skipTest('rollback_to_stable10 prepare test skipped on Windows') + # Create a table without logging. self.pr("create/populate tables") uri_1 = "table:rollback_to_stable10_1" diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable27.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable27.py new file mode 100644 index 00000000000..ee0499e72da --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable27.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +from test_rollback_to_stable01 import test_rollback_to_stable_base +from wiredtiger import stat, Modify, WT_NOTFOUND +from wtdataset import SimpleDataSet +from wtscenario import make_scenarios + +# test_rollback_to_stable27.py +# +# Test mixing timestamped and non-timestamped updates on the same VLCS RLE cell. +class test_rollback_to_stable27(test_rollback_to_stable_base): + session_config = 'isolation=snapshot' + + # Run it all on row-store as well as a control group: if something odd arises from the + # RLE cell handling it won't happen in row-store. + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + in_memory_values = [ + ('no_inmem', dict(in_memory=False)), + ('inmem', dict(in_memory=True)) + ] + + scenarios = make_scenarios(key_format_values, in_memory_values) + + def conn_config(self): + if self.in_memory: + return 'in_memory=true' + else: + return 'in_memory=false' + + # Evict the page to force reconciliation. + def evict(self, uri, key, check_value): + evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") + self.session.begin_transaction() + v = evict_cursor[1] + self.assertEqual(v, check_value) + self.assertEqual(evict_cursor.reset(), 0) + self.session.rollback_transaction() + evict_cursor.close() + + def test_rollback_to_stable(self): + nrows = 10 + + # Create a table without logging. + uri = "table:rollback_to_stable27" + ds = SimpleDataSet( + self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') + ds.populate() + + value_a = "aaaaa" * 10 + value_b = "bbbbb" * 10 + + # Pin oldest and stable to timestamp 10. + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) + + # Write aaaaaa to all the keys at time 20. + self.large_updates(uri, value_a, ds, nrows, False, 20) + + # Evict the page to force reconciliation. + self.evict(uri, 1, value_a) + + # Ideally here we'd check to make sure we actually have a single RLE cell, because + # if not the rest of the work isn't going to do much good. Maybe via stats...? + + cursor = self.session.open_cursor(uri) + self.session.begin_transaction() + cursor[7] = value_b + self.session.commit_transaction() + cursor.close() + + # Now roll back. + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(15)) + self.conn.rollback_to_stable() + + # The only thing we should see (at any time) is value_b at key 7. + cursor = self.session.open_cursor(uri) + for ts in [10, 20, 30]: + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(ts)) + for k, v in cursor: + self.assertEqual(k, 7) + self.assertEqual(v, value_b) + self.session.rollback_transaction() + cursor.close() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_search_near01.py b/src/third_party/wiredtiger/test/suite/test_search_near01.py index 412cbd894f0..6df71b438ff 100755 --- a/src/third_party/wiredtiger/test/suite/test_search_near01.py +++ b/src/third_party/wiredtiger/test/suite/test_search_near01.py @@ -98,7 +98,7 @@ class test_search_near01(wttest.WiredTigerTestCase): # range forward, and then the whole range backwards. self.assertGreater(skip_count, key_count * 2) - cursor2.reconfigure("prefix_key=true") + cursor2.reconfigure("prefix_search=true") cursor2.set_key('aa') cursor2.search_near() @@ -196,7 +196,7 @@ class test_search_near01(wttest.WiredTigerTestCase): # range forward, and then the whole range backwards. self.assertGreater(skip_count, key_count * 2) - cursor2.reconfigure("prefix_key=true") + cursor2.reconfigure("prefix_search=true") cursor2.set_key('cc') cursor2.search_near() self.assertEqual(self.get_stat(stat.conn.cursor_search_near_prefix_fast_paths), 2) @@ -304,7 +304,7 @@ class test_search_near01(wttest.WiredTigerTestCase): # range forward, and then the whole range backwards. self.assertGreater(skip_count, key_count) - cursor2.reconfigure("prefix_key=true") + cursor2.reconfigure("prefix_search=true") cursor2.set_key('c') cursor2.search_near() @@ -317,14 +317,14 @@ class test_search_near01(wttest.WiredTigerTestCase): session2.rollback_transaction() session2.begin_transaction('ignore_prepare=true') cursor4 = session2.open_cursor(uri) - cursor4.reconfigure("prefix_key=true") + cursor4.reconfigure("prefix_search=true") cursor4.set_key('c') cursor4.search_near() prefix_skip_count = self.get_stat(stat.conn.cursor_next_skip_lt_100, session2) self.assertEqual(prefix_skip_count - skip_count, 2) skip_count = prefix_skip_count - cursor4.reconfigure("prefix_key=false") + cursor4.reconfigure("prefix_search=false") cursor4.set_key('c') cursor4.search_near() self.assertEqual(self.get_stat(stat.conn.cursor_next_skip_lt_100, session2) - skip_count, 2) diff --git a/src/third_party/wiredtiger/test/suite/test_search_near02.py b/src/third_party/wiredtiger/test/suite/test_search_near02.py index a8377aac078..0f981f84af4 100644 --- a/src/third_party/wiredtiger/test/suite/test_search_near02.py +++ b/src/third_party/wiredtiger/test/suite/test_search_near02.py @@ -44,7 +44,12 @@ class test_search_near02(wttest.WiredTigerTestCase): ('byte_array', dict(key_format='u')), ] - scenarios = make_scenarios(key_format_values) + eviction = [ + ('eviction', dict(eviction=True)), + ('no eviction', dict(eviction=False)), + ] + + scenarios = make_scenarios(key_format_values, eviction) def check_key(self, key): if self.key_format == 'u': @@ -80,16 +85,16 @@ class test_search_near02(wttest.WiredTigerTestCase): cursor[prefix + "zab"] = prefix + "zab" self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(250)) - # Evict the whole range. - for k in range (0, 26): - cursor2.set_key(prefix + l[k]) + if self.eviction: + # Evict the whole range. + for k in range (0, 26): + cursor2.set_key(prefix + l[k]) + self.assertEqual(cursor2.search(), 0) + self.assertEqual(cursor2.reset(), 0) + cursor2.set_key(prefix + "zab") self.assertEqual(cursor2.search(), 0) self.assertEqual(cursor2.reset(), 0) - cursor2.set_key(prefix + "zab") - self.assertEqual(cursor2.search(), 0) - self.assertEqual(cursor2.reset(), 0) - # Start a transaction at timestamp 100, aaz should be the only key that is visible. self.session.begin_transaction('read_timestamp=' + self.timestamp_str(100)) cursor3 = self.session.open_cursor(uri) @@ -113,7 +118,7 @@ class test_search_near02(wttest.WiredTigerTestCase): self.assertEqual(cursor3.get_key(), self.check_key("aaz")) # Enable prefix search. - cursor3.reconfigure("prefix_key=true") + cursor3.reconfigure("prefix_search=true") # The only visible key is aaz. As long we are looking for a key that starts with either "a", # "aa" or "aaz", search near should return back aaz. Otherwise, search near should return @@ -148,7 +153,7 @@ class test_search_near02(wttest.WiredTigerTestCase): self.assertEqual(cursor3.search_near(), wiredtiger.WT_NOTFOUND) # Enable prefix search. - cursor3.reconfigure("prefix_key=true") + cursor3.reconfigure("prefix_search=true") cursor3.set_key("aaz") self.assertEqual(cursor3.search_near(), wiredtiger.WT_NOTFOUND) @@ -176,7 +181,7 @@ class test_search_near02(wttest.WiredTigerTestCase): self.assertEqual(cursor3.get_key(), self.check_key("aazab")) # Enable prefix search. - cursor3.reconfigure("prefix_key=true") + cursor3.reconfigure("prefix_search=true") # Search near for a, should return the closest visible key with a matching prefix: aaa. cursor3.set_key("a") diff --git a/src/third_party/wiredtiger/test/suite/test_search_near03.py b/src/third_party/wiredtiger/test/suite/test_search_near03.py index 073e5c44a13..748ef746cba 100644 --- a/src/third_party/wiredtiger/test/suite/test_search_near03.py +++ b/src/third_party/wiredtiger/test/suite/test_search_near03.py @@ -64,9 +64,9 @@ class test_search_near03(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(uri) # Check if the format is valid for prefix configuration. if self.valid_key_format(): - self.assertEqual(cursor.reconfigure("prefix_key=true"), 0) + self.assertEqual(cursor.reconfigure("prefix_search=true"), 0) else: msg = '/Invalid argument/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: cursor.reconfigure("prefix_key=true"), msg) + lambda: cursor.reconfigure("prefix_search=true"), msg) cursor.close() diff --git a/src/third_party/wiredtiger/test/suite/test_tiered02.py b/src/third_party/wiredtiger/test/suite/test_tiered02.py index bed8e57ef54..2041bcdf767 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered02.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered02.py @@ -83,9 +83,7 @@ class test_tiered02(wttest.WiredTigerTestCase): self.assertEqual(len(got), self.flushed_objects) self.flushed_objects = len(got) - # Test tiered storage with the old prototype way of signaling flushing to the shared - # tier via checkpoints. When flush_tier is working, the checkpoint calls can be - # replaced with flush_tier. + # Test tiered storage with checkpoints and flush_tier calls. def test_tiered(self): self.flushed_objects = 0 args = 'key_format=S' @@ -110,7 +108,7 @@ class test_tiered02(wttest.WiredTigerTestCase): self.close_conn() self.progress('reopen_conn') self.reopen_conn() - # Check what was there before + # Check what was there before. ds = SimpleDataSet(self, self.uri, 10, config=args) ds.check() @@ -152,7 +150,7 @@ class test_tiered02(wttest.WiredTigerTestCase): self.progress('reopen_conn') self.reopen_conn() - # Check what was there before + # Check what was there before. ds = SimpleDataSet(self, self.uri, 200, config=args) ds.check() diff --git a/src/third_party/wiredtiger/test/suite/test_tiered04.py b/src/third_party/wiredtiger/test/suite/test_tiered04.py index efb5630eda6..05fbbc44a5e 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered04.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered04.py @@ -26,7 +26,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import os, wiredtiger, wttest +import os, time, wiredtiger, wttest from wiredtiger import stat StorageSource = wiredtiger.StorageSource # easy access to constants @@ -35,8 +35,11 @@ StorageSource = wiredtiger.StorageSource # easy access to constants class test_tiered04(wttest.WiredTigerTestCase): # If the 'uri' changes all the other names must change with it. - fileuri_base = 'file:test_tiered04-000000000' - objuri = 'object:test_tiered04-0000000001.wtobj' + base = 'test_tiered04-000000000' + fileuri_base = 'file:' + base + obj1file = base + '1.wtobj' + obj2file = base + '2.wtobj' + objuri = 'object:' + base + '1.wtobj' tiereduri = "tiered:test_tiered04" uri = "table:test_tiered04" @@ -53,8 +56,8 @@ class test_tiered04(wttest.WiredTigerTestCase): object_sys_val = 9 * 1024 * 1024 object_uri = "15M" object_uri_val = 15 * 1024 * 1024 - retention = 600 - retention1 = 350 + retention = 3 + retention1 = 600 def conn_config(self): os.mkdir(self.bucket) os.mkdir(self.bucket1) @@ -120,19 +123,50 @@ class test_tiered04(wttest.WiredTigerTestCase): self.pr("flush tier") c = self.session.open_cursor(self.uri) + c1 = self.session.open_cursor(self.uri1) + cn = self.session.open_cursor(self.uri_none) c["0"] = "0" + c1["0"] = "0" + cn["0"] = "0" self.check(c, 1) + self.check(c1, 1) + self.check(cn, 1) c.close() + + # Check the local retention. After a flush_tier call the object file should exist in + # the local database. Then after sleeping long enough it should be removed. + self.session.checkpoint() + self.session.flush_tier(None) + self.pr("Check for ") + self.pr(self.obj1file) + self.assertTrue(os.path.exists(self.obj1file)) + self.assertTrue(os.path.exists(self.obj2file)) + self.pr("Sleep") + time.sleep(self.retention + 1) + # We call flush_tier here because otherwise the internal thread that + # processes the work units won't run for a while. This call will signal + # the internal thread to process the work units. self.session.flush_tier(None) + time.sleep(1) + self.pr("Check removal of ") + self.pr(self.obj1file) + self.assertFalse(os.path.exists(self.obj1file)) c = self.session.open_cursor(self.uri) c["1"] = "1" + c1["1"] = "1" + cn["1"] = "1" self.check(c, 2) c.close() c = self.session.open_cursor(self.uri) c["2"] = "2" + c1["2"] = "2" + cn["2"] = "2" self.check(c, 3) + c1.close() + cn.close() + self.session.checkpoint() self.pr("flush tier again, holding open cursor") self.session.flush_tier(None) @@ -142,7 +176,7 @@ class test_tiered04(wttest.WiredTigerTestCase): c.close() calls = self.get_stat(stat.conn.flush_tier, None) - flush = 2 + flush = 3 self.assertEqual(calls, flush) obj = self.get_stat(stat.conn.tiered_object_size, None) self.assertEqual(obj, self.object_sys_val) @@ -174,26 +208,27 @@ class test_tiered04(wttest.WiredTigerTestCase): self.assertEqual(retain, self.retention) self.session.flush_tier(None) self.session.flush_tier('force=true') + flush += 2 calls = self.get_stat(stat.conn.flush_tier, None) - self.assertEqual(calls, 4) + self.assertEqual(calls, flush) # Test reconfiguration. - new = self.retention * 2 - config = 'tiered_storage=(local_retention=%d)' % new + config = 'tiered_storage=(local_retention=%d)' % self.retention1 self.pr("reconfigure") self.conn.reconfigure(config) retain = self.get_stat(stat.conn.tiered_retention, None) - self.assertEqual(retain, new) - self.pr("reconfigure flush_tier") + self.assertEqual(retain, self.retention1) + # Call flush_tier with its various configuration arguments. It is difficult # to force a timeout or lock contention with a unit test. So just test the # call for now. self.session.flush_tier('timeout=10') self.session.flush_tier('lock_wait=false') self.session.flush_tier('sync=off') + flush += 3 self.pr("reconfigure get stat") calls = self.get_stat(stat.conn.flush_tier, None) - self.assertEqual(calls, 7) + self.assertEqual(calls, flush) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_tiered06.py b/src/third_party/wiredtiger/test/suite/test_tiered06.py index 614619fdef2..f129bafd0c0 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered06.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered06.py @@ -81,7 +81,7 @@ class test_tiered06(wttest.WiredTigerTestCase): # Nothing is in the directory list until a flush. self.assertEquals(fs.fs_directory_list(session, '', ''), []) - # Flushing moves the file into the file system + # Flushing copies the file into the file system. local.ss_flush(session, fs, 'foobar', 'foobar', None) local.ss_flush_finish(session, fs, 'foobar', 'foobar', None) @@ -91,7 +91,7 @@ class test_tiered06(wttest.WiredTigerTestCase): fh = fs.fs_open_file(session, 'foobar', FileSystem.open_file_type_data, FileSystem.open_readonly) inbytes = bytes(1000000) # An empty buffer with a million zero bytes. - fh.fh_read(session, 0, inbytes) # read into the buffer + fh.fh_read(session, 0, inbytes) # Read into the buffer. self.assertEquals(outbytes[0:1000000], inbytes) self.assertEquals(fs.fs_size(session, 'foobar'), len(outbytes)) self.assertEquals(fh.fh_size(session), len(outbytes)) @@ -136,29 +136,29 @@ class test_tiered06(wttest.WiredTigerTestCase): block_size = 4096 f = open('abc', 'wb') - # blocks filled with 'a', etc. + # Create some blocks filled with 'a', etc. a_block = ('a' * block_size).encode() b_block = ('b' * block_size).encode() c_block = ('c' * block_size).encode() file_size = nblocks * block_size - # write all blocks as 'a', but in reverse order + # Write all blocks as 'a', but in reverse order. for pos in range(file_size - block_size, 0, -block_size): f.seek(pos) f.write(a_block) - # write the even blocks as 'b', forwards + # Write the even blocks as 'b', forwards. for pos in range(0, file_size, block_size * 2): f.seek(pos) f.write(b_block) - # write every third block as 'c', backwards + # Write every third block as 'c', backwards. for pos in range(file_size - block_size, 0, -block_size * 3): f.seek(pos) f.write(c_block) f.close() - # Flushing moves the file into the file system + # Flushing copies the file into the file system. local.ss_flush(session, fs, 'abc', 'abc', None) local.ss_flush_finish(session, fs, 'abc', 'abc', None) @@ -172,7 +172,7 @@ class test_tiered06(wttest.WiredTigerTestCase): in_block = bytes(block_size) fh = fs.fs_open_file(session, 'abc', FileSystem.open_file_type_data, FileSystem.open_readonly) - # Do some spot checks, reading non-sequentially + # Do some spot checks, reading non-sequentially. fh.fh_read(session, 500 * block_size, in_block) # divisible by 2, not 3 self.assertEquals(in_block, b_block) fh.fh_read(session, 333 * block_size, in_block) # divisible by 3, not 2 @@ -208,7 +208,7 @@ class test_tiered06(wttest.WiredTigerTestCase): cachedir1 = "./cache1" cachedir2 = "./cache2" - # Add a suffix to each in a list + # Add a suffix to each in a list. def suffix(self, lst, sfx): return [x + '.' + sfx for x in lst] @@ -221,7 +221,7 @@ class test_tiered06(wttest.WiredTigerTestCase): # Check for data files in the WiredTiger home directory. def check_home(self, expect): - # Get list of all .wt files in home, prune out the WiredTiger produced ones + # Get list of all .wt files in home, prune out the WiredTiger produced ones. got = sorted(list(os.listdir(self.home))) got = [x for x in got if not x.startswith('WiredTiger') and x.endswith('.wt')] expect = sorted(self.suffix(expect, 'wt')) @@ -229,7 +229,7 @@ class test_tiered06(wttest.WiredTigerTestCase): # Check that objects are "in the cloud" after a flush. # Using the local storage module, they are actually going to be in either - # objectdir1 or objectdir2 + # objectdir1 or objectdir2. def check_objects(self, expect1, expect2): got = sorted(list(os.listdir(self.objectdir1))) expect = sorted(self.suffix(expect1, 'wtobj')) @@ -252,7 +252,7 @@ class test_tiered06(wttest.WiredTigerTestCase): f.write('hello') def test_local_file_systems(self): - # Test using various buckets, hosts + # Test using various buckets, hosts. session = self.session local = self.conn.get_storage_source('local_store') @@ -310,12 +310,12 @@ class test_tiered06(wttest.WiredTigerTestCase): self.check_caches([], []) self.check_objects(['beagle'], []) - # Bad file to flush + # Bad file to flush. errmsg = '/No such file/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: local.ss_flush(session, fs1, 'bad.wt', 'bad.wtobj'), errmsg) - # It's okay to flush again, nothing changes + # It's okay to flush again, nothing changes. local.ss_flush(session, fs1, 'beagle.wt', 'beagle.wtobj') self.check_home(['beagle', 'bird', 'bison', 'bat', 'cat', 'cougar', 'coyote', 'cub']) self.check_dirlist(fs1, '', ['beagle']) @@ -323,15 +323,15 @@ class test_tiered06(wttest.WiredTigerTestCase): self.check_caches([], []) self.check_objects(['beagle'], []) - # When we flush_finish, the local file will move to the cache directory + # When we flush_finish, the local file will be in both the local and cache directory. local.ss_flush_finish(session, fs1, 'beagle.wt', 'beagle.wtobj') - self.check_home(['bird', 'bison', 'bat', 'cat', 'cougar', 'coyote', 'cub']) + self.check_home(['beagle', 'bird', 'bison', 'bat', 'cat', 'cougar', 'coyote', 'cub']) self.check_dirlist(fs1, '', ['beagle']) self.check_dirlist(fs2, '', []) self.check_caches(['beagle'], []) self.check_objects(['beagle'], []) - # Do a some more in each file ssytem + # Do a some more in each file system. local.ss_flush(session, fs1, 'bison.wt', 'bison.wtobj') local.ss_flush(session, fs2, 'cat.wt', 'cat.wtobj') local.ss_flush(session, fs1, 'bat.wt', 'bat.wtobj') @@ -339,13 +339,13 @@ class test_tiered06(wttest.WiredTigerTestCase): local.ss_flush(session, fs2, 'cub.wt', 'cub.wtobj') local.ss_flush_finish(session, fs1, 'bat.wt', 'bat.wtobj') - self.check_home(['bird', 'bison', 'cougar', 'coyote', 'cub']) + self.check_home(['beagle', 'bird', 'bison', 'bat', 'cat', 'cougar', 'coyote', 'cub']) self.check_dirlist(fs1, '', ['beagle', 'bat', 'bison']) self.check_dirlist(fs2, '', ['cat', 'cub']) self.check_caches(['beagle', 'bat'], ['cat']) self.check_objects(['beagle', 'bat', 'bison'], ['cat', 'cub']) - # Test directory listing prefixes + # Test directory listing prefixes. self.check_dirlist(fs1, '', ['beagle', 'bat', 'bison']) self.check_dirlist(fs1, 'ba', ['bat']) self.check_dirlist(fs1, 'be', ['beagle']) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp22.py b/src/third_party/wiredtiger/test/suite/test_timestamp22.py index 55d6329af2f..c45eea97ac6 100755 --- a/src/third_party/wiredtiger/test/suite/test_timestamp22.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp22.py @@ -44,6 +44,7 @@ class test_timestamp22(wttest.WiredTigerTestCase): rand = suite_random.suite_random() oldest_ts = 0 stable_ts = 0 + last_durable = 0 SUCCESS = 'success' FAILURE = 'failure' @@ -147,6 +148,12 @@ class test_timestamp22(wttest.WiredTigerTestCase): else: # It's possible this will succeed, we'll check below. this_commit_ts = self.gen_ts(commit_ts) + + # OOD does not work with prepared updates. Hence, the commit ts should always be + # greater than the last durable ts. + if this_commit_ts <= self.last_durable: + this_commit_ts = self.last_durable + 1 + config += ',commit_timestamp=' + self.timestamp_str(this_commit_ts) if this_commit_ts >= 0: @@ -288,6 +295,8 @@ class test_timestamp22(wttest.WiredTigerTestCase): with self.expect(ok_commit, 'commit'): session.commit_transaction(commit_config) self.commit_value = value + if do_prepare: + self.last_durable = durable_ts if needs_rollback: # Rollback this one transaction, and continue the loop self.report('rollback_transaction') @@ -422,6 +431,12 @@ class test_timestamp22(wttest.WiredTigerTestCase): read_ts = self.gen_ts(iternum) else: read_ts = -1 # no read_timestamp used in txn + + # OOD does not work with prepared updates. Hence, the commit ts should always be + # greater than the last durable ts. + if commit_ts <= self.last_durable: + commit_ts = self.last_durable + 1 + if do_prepare: # If we doing a prepare, we must abide by some additional rules. # If we don't we'll immediately panic diff --git a/src/third_party/wiredtiger/test/utility/Makefile.am b/src/third_party/wiredtiger/test/utility/Makefile.am index a2923eb41a8..4bdbc775a59 100644 --- a/src/third_party/wiredtiger/test/utility/Makefile.am +++ b/src/third_party/wiredtiger/test/utility/Makefile.am @@ -1,4 +1,4 @@ AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include -libtest_util_la_SOURCES = misc.c parse_opts.c thread.c +libtest_util_la_SOURCES = misc.c modify.c parse_opts.c thread.c noinst_LTLIBRARIES = libtest_util.la diff --git a/src/third_party/wiredtiger/test/utility/modify.c b/src/third_party/wiredtiger/test/utility/modify.c new file mode 100644 index 00000000000..254dd58e3b4 --- /dev/null +++ b/src/third_party/wiredtiger/test/utility/modify.c @@ -0,0 +1,115 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "test_util.h" + +/* + * testutil_modify_apply -- + * Implement a modify using a completely separate algorithm as a check on the internal library + * algorithms. + */ +void +testutil_modify_apply(WT_ITEM *value, WT_ITEM *workspace, WT_MODIFY *entries, int nentries) +{ + WT_ITEM *ta, *tb, *tmp, _tmp; + size_t len, size; + int i; + + /* + * Passed a value and array of modifications, plus a temporary buffer for an additional work + * space. + * + * Process the entries to figure out the largest possible buffer we need. This is pessimistic + * because we're ignoring replacement bytes, but it's a simpler calculation. + */ + for (size = value->size, i = 0; i < nentries; ++i) { + if (entries[i].offset >= size) + size = entries[i].offset; + size += entries[i].data.size; + } + + /* Grow the buffers. */ + testutil_check(__wt_buf_grow(NULL, value, size)); + testutil_check(__wt_buf_grow(NULL, workspace, size)); + + /* + * Overwrite anything not initialized in the original buffer, and overwrite the entire workspace + * buffer. + */ + if ((value->memsize - value->size) > 0) + memset((uint8_t *)value->mem + value->size, 0xff, value->memsize - value->size); + if (workspace->memsize > 0) + memset((uint8_t *)workspace->mem, 0xff, workspace->memsize); + + ta = value; + tb = workspace; + + /* + * From the starting buffer, create a new buffer b based on changes in the entries array. We're + * doing a brute force solution here to test the faster solution implemented in the library. + */ + for (i = 0; i < nentries; ++i) { + /* Take leading bytes from the original, plus any gap bytes. */ + if (entries[i].offset >= ta->size) { + memcpy(tb->mem, ta->mem, ta->size); + if (entries[i].offset > ta->size) + memset((uint8_t *)tb->mem + ta->size, '\0', entries[i].offset - ta->size); + } else if (entries[i].offset > 0) + memcpy(tb->mem, ta->mem, entries[i].offset); + tb->size = entries[i].offset; + + /* Take replacement bytes. */ + if (entries[i].data.size > 0) { + memcpy((uint8_t *)tb->mem + tb->size, entries[i].data.data, entries[i].data.size); + tb->size += entries[i].data.size; + } + + /* Take trailing bytes from the original. */ + len = entries[i].offset + entries[i].size; + if (ta->size > len) { + memcpy((uint8_t *)tb->mem + tb->size, (uint8_t *)ta->mem + len, ta->size - len); + tb->size += ta->size - len; + } + testutil_assert(tb->size <= size); + + /* Swap the buffers and do it again. */ + tmp = ta; + ta = tb; + tb = tmp; + } + ta->data = ta->mem; + tb->data = tb->mem; + + /* + * The final results may not be in the original buffer, in which case we swap them back around. + */ + if (ta != value) { + _tmp = *ta; + *ta = *tb; + *tb = _tmp; + } +} diff --git a/src/third_party/wiredtiger/test/utility/test_util.h b/src/third_party/wiredtiger/test/utility/test_util.h index 24e37e5acb9..54444684ddf 100644 --- a/src/third_party/wiredtiger/test/utility/test_util.h +++ b/src/third_party/wiredtiger/test/utility/test_util.h @@ -266,21 +266,22 @@ void op_create(void *); void op_create_unique(void *); void op_cursor(void *); void op_drop(void *); +bool testutil_is_flag_set(const char *); +void testutil_build_dir(TEST_OPTS *, char *, int); void testutil_clean_work_dir(const char *); void testutil_cleanup(TEST_OPTS *); void testutil_copy_data(const char *); -bool testutil_is_flag_set(const char *); -void testutil_build_dir(TEST_OPTS *, char *, int); +void testutil_copy_file(WT_SESSION *, const char *); +void testutil_create_backup_directory(const char *); void testutil_make_work_dir(const char *); +void testutil_modify_apply(WT_ITEM *, WT_ITEM *, WT_MODIFY *, int); int testutil_parse_opts(int, char *const *, TEST_OPTS *); void testutil_print_command_line(int argc, char *const *argv); void testutil_progress(TEST_OPTS *, const char *); -void testutil_timestamp_parse(const char *, uint64_t *); -void testutil_create_backup_directory(const char *); -void testutil_copy_file(WT_SESSION *, const char *); #ifndef _WIN32 void testutil_sleep_wait(uint32_t, pid_t); #endif +void testutil_timestamp_parse(const char *, uint64_t *); void testutil_work_dir_from_path(char *, size_t, const char *); WT_THREAD_RET thread_append(void *); |