diff options
Diffstat (limited to 'src/third_party/wiredtiger/test')
59 files changed, 3652 insertions, 1046 deletions
diff --git a/src/third_party/wiredtiger/test/cppsuite/configs/config_example_test_default.txt b/src/third_party/wiredtiger/test/cppsuite/configs/config_example_test_default.txt new file mode 100644 index 00000000000..b46fed225eb --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/configs/config_example_test_default.txt @@ -0,0 +1,48 @@ +# Same parameters as config_poc_test_default +duration_seconds=10, +cache_size_mb=1000, +enable_logging=true, +runtime_monitor= +( + op_count=3, + interval=s, + stat_cache_size= + ( + enabled=true, + limit=100 + ) +), +timestamp_manager= +( + enabled=true, + oldest_lag=1, + stable_lag=1 +), +workload_generator= +( + collection_count=2, + key_count=5, + key_size=1, + ops_per_transaction= + ( + min=5, + max=50 + ), + read_threads=1, + update_threads=1, + value_size=10, + update_config= + ( + op_count=1, + interval=s + ), + insert_config= + ( + op_count=1, + interval=s + ) +), +workload_tracking= +( + enabled=true +) diff --git a/src/third_party/wiredtiger/test/cppsuite/configs/config_poc_test_default.txt b/src/third_party/wiredtiger/test/cppsuite/configs/config_poc_test_default.txt index 52f4f536876..c677142234d 100644 --- a/src/third_party/wiredtiger/test/cppsuite/configs/config_poc_test_default.txt +++ b/src/third_party/wiredtiger/test/cppsuite/configs/config_poc_test_default.txt @@ -1,40 +1,50 @@ # Sets up a basic database with 2 collections and 5 keys and run thread for 10 seconds. # All components are enabled. # Used as a basic test for the framework. -duration_seconds=10 -cache_size_mb=1000 -enable_logging=true +duration_seconds=10, +cache_size_mb=1000, +enable_logging=true, runtime_monitor= -{ - rate_per_second=3 +( + op_count=3, + interval=s, stat_cache_size= - { - enabled=true + ( + enabled=true, limit=100 - } -} + ) +), timestamp_manager= -{ - enabled=true - oldest_lag=1 +( + enabled=true, + oldest_lag=1, stable_lag=1 -} +), workload_generator= -{ - collection_count=2 - key_count=5 - key_format=i - key_size=1 +( + collection_count=2, + key_count=5, + key_size=1, ops_per_transaction= - { - min=5 + ( + min=5, max=50 - } - read_threads=1 + ), + read_threads=1, + update_threads=1, + update_config= + ( + op_count=1, + interval=s + ), + insert_config= + ( + op_count=1, + interval=s + ), value_size=10 - value_format=S -} +), workload_tracking= -{ +( enabled=true -} +) diff --git a/src/third_party/wiredtiger/test/cppsuite/configs/config_poc_test_stress.txt b/src/third_party/wiredtiger/test/cppsuite/configs/config_poc_test_stress.txt new file mode 100644 index 00000000000..6067bea3983 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/configs/config_poc_test_stress.txt @@ -0,0 +1,39 @@ +# Sets up a basic database with 2 collections and 50000 keys and run thread for 10 seconds. +# All components are enabled. +# Used as a stress test for the framework. +duration_seconds=10, +cache_size_mb=5000, +enable_logging=true, +runtime_monitor= +( + rate_per_second=3, + stat_cache_size= + ( + enabled=true, + limit=100 + ) +), +timestamp_manager= +( + enabled=true, + oldest_lag=1, + stable_lag=1 +), +workload_generator= +( + collection_count=2, + key_count=50000, + key_size=10, + ops_per_transaction= + ( + min=5, + max=50 + ), + read_threads=1, + update_threads=1, + value_size=2000 +), +workload_tracking= +( + enabled=true +) diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/connection_manager.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/connection_manager.h index 23df4dfc001..29c76b59a2b 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/connection_manager.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/connection_manager.h @@ -36,8 +36,8 @@ extern "C" { #include "wiredtiger.h" } -#include "api_const.h" -#include "debug_utils.h" +#include "util/api_const.h" +#include "util/debug_utils.h" namespace test_harness { /* diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/component.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/core/component.h index 466fdbe890e..91b165d8f29 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/component.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/core/component.h @@ -30,6 +30,7 @@ #define COMPONENT_H #include "configuration.h" +#include "throttle.h" namespace test_harness { /* @@ -38,7 +39,7 @@ namespace test_harness { */ class component { public: - component(configuration *config) : _enabled(true), _running(false), _config(config) {} + component(const std::string &name, configuration *config) : _name(name), _config(config) {} ~component() { @@ -56,13 +57,35 @@ class component { virtual void load() { - _running = true; + debug_print("Loading component: " + _name, DEBUG_INFO); + _enabled = _config->get_optional_bool(ENABLED, true); + _throttle = throttle(_config); + /* If we're not enabled we shouldn't be running. */ + _running = _enabled; } /* - * The run phase encompasses all operations that occur during the primary phase of the workload. + * The run function provides a top level loop that calls the do_work function every X seconds as + * defined by the throttle. Each run() method defined by the components is called in its own + * thread by the top level test class. + * + * If a component does not wish to use the standard run function, it can be overloaded. */ - virtual void run() = 0; + virtual void + run() + { + debug_print("Running component: " + _name, DEBUG_INFO); + while (_enabled && _running) { + do_work(); + _throttle.sleep(); + } + } + + virtual void + do_work() + { + /* Not implemented. */ + } bool is_enabled() const @@ -78,15 +101,18 @@ class component { virtual void finish() { + debug_print("Finishing component: " + _name, DEBUG_INFO); _running = false; } - static const std::string name; - protected: - bool _enabled; - volatile bool _running; + bool _enabled = false; + volatile bool _running = false; + throttle _throttle; configuration *_config; + + private: + std::string _name; }; } // namespace test_harness #endif diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/configuration.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/core/configuration.h index adae5b1b8c5..c2b9494487f 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/configuration.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/core/configuration.h @@ -35,6 +35,8 @@ extern "C" { #include "test_util.h" } +enum class types { BOOL, INT, STRING, STRUCT }; + namespace test_harness { class configuration { public: @@ -74,70 +76,105 @@ class configuration { } /* - * Wrapper functions for retrieving basic configuration values. Ideally the tests can avoid - * using the config item struct provided by wiredtiger. However if they still wish to use it the - * get and next functions can be used. + * Wrapper functions for retrieving basic configuration values. Ideally tests can avoid using + * the config item struct provided by wiredtiger. + * + * When getting a configuration value that may not exist for that configuration string or + * component, the optional forms of the functions can be used. In this case a default value must + * be passed and it will be set to that value. */ - int - get_string(const std::string &key, std::string &value) const + std::string + get_string(const std::string &key) { - WT_CONFIG_ITEM temp_value; - testutil_check(_config_parser->get(_config_parser, key.c_str(), &temp_value)); - if (temp_value.type != WT_CONFIG_ITEM::WT_CONFIG_ITEM_STRING || - temp_value.type != WT_CONFIG_ITEM::WT_CONFIG_ITEM_ID) - return (-1); - value = std::string(temp_value.str, temp_value.len); - return (0); + return get<std::string>(key, false, types::STRING, "", config_item_to_string); } - int - get_bool(const std::string &key, bool &value) const + std::string + get_optional_string(const std::string &key, const std::string &def) { - WT_CONFIG_ITEM temp_value; - testutil_check(_config_parser->get(_config_parser, key.c_str(), &temp_value)); - if (temp_value.type != WT_CONFIG_ITEM::WT_CONFIG_ITEM_BOOL) - return (-1); - value = (temp_value.val != 0); - return (0); + return get<std::string>(key, true, types::STRING, def, config_item_to_string); } - int - get_int(const std::string &key, int64_t &value) const + bool + get_bool(const std::string &key) { - WT_CONFIG_ITEM temp_value; - testutil_check(_config_parser->get(_config_parser, key.c_str(), &temp_value)); - if (temp_value.type != WT_CONFIG_ITEM::WT_CONFIG_ITEM_NUM) - return (-1); - value = temp_value.val; - return (0); + return get<bool>(key, false, types::BOOL, false, config_item_to_bool); } - configuration * - get_subconfig(const std::string &key) const + bool + get_optional_bool(const std::string &key, const bool def) { - WT_CONFIG_ITEM subconfig; - testutil_check(get(key, &subconfig)); - return new configuration(subconfig); + return get<bool>(key, true, types::BOOL, def, config_item_to_bool); } - /* - * Basic configuration parsing helper functions. - */ - int - next(WT_CONFIG_ITEM *key, WT_CONFIG_ITEM *value) const + int64_t + get_int(const std::string &key) + { + return get<int64_t>(key, false, types::INT, 0, config_item_to_int); + } + + int64_t + get_optional_int(const std::string &key, const int64_t def) { - return (_config_parser->next(_config_parser, key, value)); + return get<int64_t>(key, true, types::INT, def, config_item_to_int); } - int - get(const std::string &key, WT_CONFIG_ITEM *value) const + configuration * + get_subconfig(const std::string &key) { - return (_config_parser->get(_config_parser, key.c_str(), value)); + return get<configuration *>(key, false, types::STRUCT, nullptr, + [](WT_CONFIG_ITEM item) { return new configuration(item); }); } private: + static bool + config_item_to_bool(const WT_CONFIG_ITEM item) + { + return (item.val != 0); + } + + static int64_t + config_item_to_int(const WT_CONFIG_ITEM item) + { + return (item.val); + } + + static std::string + config_item_to_string(const WT_CONFIG_ITEM item) + { + return std::string(item.str, item.len); + } + + template <typename T> + T + get(const std::string &key, bool optional, types type, T def, T (*func)(WT_CONFIG_ITEM item)) + { + WT_DECL_RET; + WT_CONFIG_ITEM value = {"", 0, 1, WT_CONFIG_ITEM::WT_CONFIG_ITEM_BOOL}; + const char *error_msg = "Configuration value doesn't match requested type"; + + ret = _config_parser->get(_config_parser, key.c_str(), &value); + if (ret == WT_NOTFOUND && optional) + return (def); + else if (ret != 0) + testutil_die(ret, "Error while finding config"); + + if (type == types::STRING && + (value.type != WT_CONFIG_ITEM::WT_CONFIG_ITEM_STRING && + value.type != WT_CONFIG_ITEM::WT_CONFIG_ITEM_ID)) + testutil_die(-1, error_msg); + else if (type == types::BOOL && value.type != WT_CONFIG_ITEM::WT_CONFIG_ITEM_BOOL) + testutil_die(-1, error_msg); + else if (type == types::INT && value.type != WT_CONFIG_ITEM::WT_CONFIG_ITEM_NUM) + testutil_die(-1, error_msg); + else if (type == types::STRUCT && value.type != WT_CONFIG_ITEM::WT_CONFIG_ITEM_STRUCT) + testutil_die(-1, error_msg); + + return func(value); + } + std::string _config; - WT_CONFIG_PARSER *_config_parser; + WT_CONFIG_PARSER *_config_parser = nullptr; }; } // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/core/throttle.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/core/throttle.h new file mode 100644 index 00000000000..bfe5816c70e --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/core/throttle.h @@ -0,0 +1,73 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef THROTTLE_H +#define THROTTLE_H + +#include <thread> + +#include "configuration.h" + +namespace test_harness { +class throttle { + public: + throttle(const int64_t op_count, const char interval) + { + testutil_assert(op_count > 0); + /* Lazily compute the ms for every type. */ + if (interval == 's') + _ms = 1000 / op_count; + else if (interval == 'm') + _ms = (60 * 1000) / op_count; + else if (interval == 'h') + _ms = (60 * 60 * 1000) / op_count; + else + testutil_die(-1, "Specified throttle interval not supported."); + } + + throttle(configuration *config) + : throttle( + config->get_optional_int(OP_COUNT, 1), config->get_optional_string(INTERVAL, "s")[0]) + { + } + + /* Default to a second per operation. */ + throttle() : throttle(1, 's') {} + + void + sleep() + { + std::this_thread::sleep_for(std::chrono::milliseconds(_ms)); + } + + private: + uint64_t _ms = 1000; +}; +} // namespace test_harness + +#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.h index e81a8bfe47b..b7897eb39f1 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.h @@ -29,16 +29,15 @@ #ifndef RUNTIME_MONITOR_H #define RUNTIME_MONITOR_H -#include <thread> - extern "C" { #include "wiredtiger.h" } -#include "api_const.h" -#include "component.h" +#include "util/debug_utils.h" +#include "util/api_const.h" +#include "core/component.h" +#include "core/throttle.h" #include "connection_manager.h" -#include "debug_utils.h" namespace test_harness { /* Static statistic get function. */ @@ -55,8 +54,7 @@ class statistic { public: statistic(configuration *config) { - testutil_assert(config != nullptr); - testutil_check(config->get_bool(ENABLED, _enabled)); + _enabled = config->get_bool(ENABLED); } /* Check that the given statistic is within bounds. */ @@ -79,7 +77,7 @@ class cache_limit_statistic : public statistic { public: cache_limit_statistic(configuration *config) : statistic(config) { - testutil_check(config->get_int(LIMIT, limit)); + limit = config->get_int(LIMIT); } void @@ -104,7 +102,7 @@ class cache_limit_statistic : public statistic { debug_print(error_string, DEBUG_ERROR); testutil_assert(use_percent < limit); } else - debug_print("Usage: " + std::to_string(use_percent), DEBUG_TRACE); + debug_print("Cache usage: " + std::to_string(use_percent), DEBUG_TRACE); } private: @@ -117,7 +115,7 @@ class cache_limit_statistic : public statistic { */ class runtime_monitor : public component { public: - runtime_monitor(configuration *config) : component(config), _ops(1) {} + runtime_monitor(configuration *config) : component("runtime_monitor", config) {} ~runtime_monitor() { @@ -135,37 +133,35 @@ class runtime_monitor : public component { { configuration *sub_config; std::string statistic_list; - /* Parse the configuration for the runtime monitor. */ - testutil_check(_config->get_int(RATE_PER_SECOND, _ops)); - /* Load known statistics. */ - sub_config = _config->get_subconfig(STAT_CACHE_SIZE); - _stats.push_back(new cache_limit_statistic(sub_config)); - delete sub_config; + /* Load the general component things. */ component::load(); + + if (_enabled) { + _session = connection_manager::instance().create_session(); + + /* Open our statistic cursor. */ + _session->open_cursor(_session, STATISTICS_URI, nullptr, nullptr, &_cursor); + + /* Load known statistics. */ + sub_config = _config->get_subconfig(STAT_CACHE_SIZE); + _stats.push_back(new cache_limit_statistic(sub_config)); + delete sub_config; + } } void - run() + do_work() { - WT_SESSION *session = connection_manager::instance().create_session(); - WT_CURSOR *cursor = nullptr; - - /* Open a statistics cursor. */ - testutil_check(session->open_cursor(session, STATISTICS_URI, nullptr, nullptr, &cursor)); - - while (_running) { - /* Sleep so that we do x operations per second. To be replaced by throttles. */ - std::this_thread::sleep_for(std::chrono::milliseconds(1000 / _ops)); - for (const auto &it : _stats) { - if (it->is_enabled()) - it->check(cursor); - } + for (const auto &it : _stats) { + if (it->is_enabled()) + it->check(_cursor); } } private: - int64_t _ops; + WT_CURSOR *_cursor = nullptr; + WT_SESSION *_session = nullptr; std::vector<statistic *> _stats; }; } // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/test.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/test.h index e11d17ab51b..a753e131f0f 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/test.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/test.h @@ -38,34 +38,32 @@ extern "C" { #include "wiredtiger.h" } -#include "api_const.h" -#include "component.h" -#include "configuration.h" +#include "util/api_const.h" +#include "core/component.h" +#include "core/configuration.h" #include "connection_manager.h" #include "runtime_monitor.h" #include "timestamp_manager.h" #include "thread_manager.h" #include "workload_generator.h" -#include "workload_validation.h" +#include "workload/workload_validation.h" namespace test_harness { /* * The base class for a test, the standard usage pattern is to just call run(). */ -class test { +class test : public database_operation { public: test(const std::string &config, const std::string &name) { - _configuration = new configuration(name, config); - _runtime_monitor = new runtime_monitor(_configuration->get_subconfig(RUNTIME_MONITOR)); - _timestamp_manager = - new timestamp_manager(_configuration->get_subconfig(TIMESTAMP_MANAGER)); - _workload_tracking = new workload_tracking(_configuration->get_subconfig(WORKLOAD_TRACKING), + _config = new configuration(name, config); + _runtime_monitor = new runtime_monitor(_config->get_subconfig(RUNTIME_MONITOR)); + _timestamp_manager = new timestamp_manager(_config->get_subconfig(TIMESTAMP_MANAGER)); + _workload_tracking = new workload_tracking(_config->get_subconfig(WORKLOAD_TRACKING), OPERATION_TRACKING_TABLE_CONFIG, TABLE_OPERATION_TRACKING, SCHEMA_TRACKING_TABLE_CONFIG, TABLE_SCHEMA_TRACKING); - _workload_generator = - new workload_generator(_configuration->get_subconfig(WORKLOAD_GENERATOR), - _timestamp_manager, _workload_tracking); + _workload_generator = new workload_generator( + _config->get_subconfig(WORKLOAD_GENERATOR), this, _timestamp_manager, _workload_tracking); _thread_manager = new thread_manager(); /* * Ordering is not important here, any dependencies between components should be resolved @@ -77,13 +75,13 @@ class test { ~test() { - delete _configuration; + delete _config; delete _runtime_monitor; delete _timestamp_manager; delete _thread_manager; delete _workload_generator; delete _workload_tracking; - _configuration = nullptr; + _config = nullptr; _runtime_monitor = nullptr; _timestamp_manager = nullptr; _thread_manager = nullptr; @@ -103,15 +101,16 @@ class test { virtual void run() { - int64_t cache_size_mb = 100, duration_seconds = 0; + int64_t cache_size_mb, duration_seconds; bool enable_logging, is_success = true; /* Build the database creation config string. */ std::string db_create_config = CONNECTION_CREATE; - testutil_check(_configuration->get_int(CACHE_SIZE_MB, cache_size_mb)); + /* Get the cache size, and turn logging on or off. */ + cache_size_mb = _config->get_int(CACHE_SIZE_MB); db_create_config += ",statistics=(fast),cache_size=" + std::to_string(cache_size_mb) + "MB"; - testutil_check(_configuration->get_bool(ENABLE_LOGGING, enable_logging)); + enable_logging = _config->get_bool(ENABLE_LOGGING); db_create_config += ",log=(enabled=" + std::string(enable_logging ? "true" : "false") + ")"; /* Set up the test environment. */ @@ -125,12 +124,12 @@ class test { for (const auto &it : _components) _thread_manager->add_thread(&component::run, it); - /* Sleep duration seconds. */ - testutil_check(_configuration->get_int(DURATION_SECONDS, duration_seconds)); + /* The test will run for the duration as defined in the config. */ + duration_seconds = _config->get_int(DURATION_SECONDS); testutil_assert(duration_seconds >= 0); std::this_thread::sleep_for(std::chrono::seconds(duration_seconds)); - /* End the test. */ + /* End the test by calling finish on all known components. */ for (const auto &it : _components) it->finish(); _thread_manager->join(); @@ -139,7 +138,7 @@ class test { if (_workload_tracking->is_enabled()) { workload_validation wv; is_success = wv.validate(_workload_tracking->get_operation_table_name(), - _workload_tracking->get_schema_table_name()); + _workload_tracking->get_schema_table_name(), _workload_generator->get_database()); } debug_print(is_success ? "SUCCESS" : "FAILED", DEBUG_INFO); @@ -177,12 +176,12 @@ class test { private: std::string _name; std::vector<component *> _components; - configuration *_configuration; - runtime_monitor *_runtime_monitor; - thread_manager *_thread_manager; - timestamp_manager *_timestamp_manager; - workload_generator *_workload_generator; - workload_tracking *_workload_tracking; + configuration *_config; + runtime_monitor *_runtime_monitor = nullptr; + thread_manager *_thread_manager = nullptr; + timestamp_manager *_timestamp_manager = nullptr; + workload_generator *_workload_generator = nullptr; + workload_tracking *_workload_tracking = nullptr; }; } // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/thread_manager.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/thread_manager.h index 749f5c1d8f3..b7f736c169d 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/thread_manager.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/thread_manager.h @@ -31,7 +31,8 @@ #include <thread> -#include "thread_context.h" +#include "workload/database_operation.h" +#include "workload/thread_context.h" namespace test_harness { /* Class that handles threads, from their initialization to their deletion. */ @@ -56,10 +57,10 @@ class thread_manager { */ template <typename Callable> void - add_thread(thread_context *tc, Callable &&fct) + add_thread(thread_context *tc, database_operation *db_operation, Callable &&fct) { tc->set_running(true); - std::thread *t = new std::thread(fct, std::ref(*tc)); + std::thread *t = new std::thread(fct, std::ref(*tc), std::ref(*db_operation)); _workers.push_back(t); } diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/timestamp_manager.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/timestamp_manager.h index 8a5940c7637..96b5f6bc69c 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/timestamp_manager.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/timestamp_manager.h @@ -34,7 +34,7 @@ #include <sstream> #include <thread> -#include "component.h" +#include "core/component.h" namespace test_harness { /* @@ -44,63 +44,54 @@ namespace test_harness { */ class timestamp_manager : public component { public: - timestamp_manager(configuration *config) - : /* _periodic_update_s is hardcoded to 1 second for now. */ - component(config), _increment_ts(0U), _latest_ts(0U), _oldest_lag(0), _oldest_ts(0U), - _periodic_update_s(1), _stable_lag(0), _stable_ts(0U) - { - } + timestamp_manager(configuration *config) : component("timestamp_manager", config) {} void load() { - testutil_assert(_config != nullptr); - testutil_check(_config->get_int(OLDEST_LAG, _oldest_lag)); + component::load(); + + _oldest_lag = _config->get_int(OLDEST_LAG); testutil_assert(_oldest_lag >= 0); - testutil_check(_config->get_int(STABLE_LAG, _stable_lag)); + _stable_lag = _config->get_int(STABLE_LAG); testutil_assert(_stable_lag >= 0); - testutil_check(_config->get_bool(ENABLED, _enabled)); - component::load(); } void - run() + do_work() { std::string config; /* latest_ts_s represents the time component of the latest timestamp provided. */ wt_timestamp_t latest_ts_s; - while (_enabled && _running) { - /* Timestamps are checked periodically. */ - std::this_thread::sleep_for(std::chrono::seconds(_periodic_update_s)); - latest_ts_s = (_latest_ts >> 32); - /* - * Keep a time window between the latest and stable ts less than the max defined in the - * configuration. - */ - testutil_assert(latest_ts_s >= _stable_ts); - if ((latest_ts_s - _stable_ts) > _stable_lag) { - _stable_ts = latest_ts_s - _stable_lag; - config += std::string(STABLE_TS) + "=" + decimal_to_hex(_stable_ts); - } - - /* - * Keep a time window between the stable and oldest ts less than the max defined in the - * configuration. - */ - testutil_assert(_stable_ts > _oldest_ts); - if ((_stable_ts - _oldest_ts) > _oldest_lag) { - _oldest_ts = _stable_ts - _oldest_lag; - if (!config.empty()) - config += ","; - config += std::string(OLDEST_TS) + "=" + decimal_to_hex(_oldest_ts); - } - - /* Save the new timestamps. */ - if (!config.empty()) { - connection_manager::instance().set_timestamp(config); - config = ""; - } + /* Timestamps are checked periodically. */ + latest_ts_s = (_latest_ts >> 32); + /* + * Keep a time window between the latest and stable ts less than the max defined in the + * configuration. + */ + testutil_assert(latest_ts_s >= _stable_ts); + if ((latest_ts_s - _stable_ts) > _stable_lag) { + _stable_ts = latest_ts_s - _stable_lag; + config += std::string(STABLE_TS) + "=" + decimal_to_hex(_stable_ts); + } + + /* + * Keep a time window between the stable and oldest ts less than the max defined in the + * configuration. + */ + testutil_assert(_stable_ts >= _oldest_ts); + if ((_stable_ts - _oldest_ts) > _oldest_lag) { + _oldest_ts = _stable_ts - _oldest_lag; + if (!config.empty()) + config += ","; + config += std::string(OLDEST_TS) + "=" + decimal_to_hex(_oldest_ts); + } + + /* Save the new timestamps. */ + if (!config.empty()) { + connection_manager::instance().set_timestamp(config); + config = ""; } } @@ -140,14 +131,13 @@ class timestamp_manager : public component { } private: - const wt_timestamp_t _periodic_update_s; std::atomic<wt_timestamp_t> _increment_ts; - wt_timestamp_t _latest_ts, _oldest_ts, _stable_ts; + wt_timestamp_t _latest_ts = 0U, _oldest_ts = 0U, _stable_ts = 0U; /* * _oldest_lag is the time window between the stable and oldest timestamps. * _stable_lag is the time window between the latest and stable timestamps. */ - int64_t _oldest_lag, _stable_lag; + int64_t _oldest_lag = 0, _stable_lag = 0; }; } // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/api_const.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/api_const.h index 46a6a775677..2ea702b4848 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/api_const.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/api_const.h @@ -44,16 +44,21 @@ static const char *COLLECTION_COUNT = "collection_count"; static const char *DURATION_SECONDS = "duration_seconds"; static const char *ENABLED = "enabled"; static const char *ENABLE_LOGGING = "enable_logging"; +static const char *INTERVAL = "interval"; +static const char *INSERT_CONFIG = "insert_config"; static const char *KEY_COUNT = "key_count"; +static const char *KEY_SIZE = "key_size"; static const char *LIMIT = "limit"; static const char *MAX = "max"; static const char *MIN = "min"; static const char *OLDEST_LAG = "oldest_lag"; +static const char *OP_COUNT = "op_count"; static const char *OPS_PER_TRANSACTION = "ops_per_transaction"; -static const char *RATE_PER_SECOND = "rate_per_second"; static const char *READ_THREADS = "read_threads"; static const char *STABLE_LAG = "stable_lag"; static const char *STAT_CACHE_SIZE = "stat_cache_size"; +static const char *UPDATE_THREADS = "update_threads"; +static const char *UPDATE_CONFIG = "update_config"; static const char *VALUE_SIZE = "value_size"; /* WiredTiger API consts. */ @@ -63,6 +68,7 @@ static const char *OLDEST_TS = "oldest_timestamp"; static const char *STABLE_TS = "stable_timestamp"; /* Test harness consts. */ +static const char *DEFAULT_FRAMEWORK_SCHEMA = "key_format=S,value_format=S"; static const char *TABLE_OPERATION_TRACKING = "table:operation_tracking"; static const char *TABLE_SCHEMA_TRACKING = "table:schema_tracking"; static const char *STATISTICS_URI = "statistics:"; diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/debug_utils.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/debug_utils.h index da09a08c9d8..da09a08c9d8 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/debug_utils.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/debug_utils.h diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_model.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_model.h new file mode 100644 index 00000000000..07e7c007ea7 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_model.h @@ -0,0 +1,89 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef DATABASE_MODEL_H +#define DATABASE_MODEL_H + +#include <map> +#include <string> + +namespace test_harness { + +/* Key/Value type. */ +typedef std::string key_value_t; + +/* Representation of key states. */ +struct key_t { + bool exists; +}; + +/* Iterator type used to iterate over keys that are stored in the data model. */ +typedef std::map<test_harness::key_value_t, test_harness::key_t>::const_iterator keys_iterator_t; + +/* Representation of a value. */ +struct value_t { + key_value_t value; +}; + +/* A collection is made of mapped Key objects. */ +struct collection_t { + std::map<key_value_t, key_t> keys; + std::map<key_value_t, value_t> *values = {nullptr}; +}; + +/* Representation of the collections in memory. */ +class database { + public: + const keys_iterator_t + get_collection_keys_begin(const std::string &collection_name) const + { + return (collections.at(collection_name).keys.begin()); + } + + const keys_iterator_t + get_collection_keys_end(const std::string &collection_name) const + { + return (collections.at(collection_name).keys.end()); + } + + const std::vector<std::string> + get_collection_names() const + { + std::vector<std::string> collection_names; + + for (auto const &it : collections) + collection_names.push_back(it.first); + + return (collection_names); + } + + std::map<std::string, collection_t> collections; +}; +} // namespace test_harness + +#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.h new file mode 100644 index 00000000000..7a88ed9b662 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.h @@ -0,0 +1,274 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef DATABASE_OPERATION_H +#define DATABASE_OPERATION_H + +#include "database_model.h" +#include "workload_tracking.h" +#include "thread_context.h" + +namespace test_harness { +class database_operation { + public: + /* + * Function that performs the following steps using the configuration that is defined by the + * test: + * - Create the working dir. + * - Open a connection. + * - Open a session. + * - Create n collections as per the configuration. + * - Open a cursor on each collection. + * - Insert m key/value pairs in each collection. Values are random strings which size is + * defined by the configuration. + * - Store in memory the created collections and the generated keys that were inserted. + */ + virtual void + populate(database &database, timestamp_manager *timestamp_manager, configuration *config, + workload_tracking *tracking) + { + WT_CURSOR *cursor; + WT_SESSION *session; + wt_timestamp_t ts; + int64_t collection_count, key_count, key_cpt, key_size, value_size; + std::string collection_name, cfg, home; + key_value_t generated_key, generated_value; + bool ts_enabled = timestamp_manager->is_enabled(); + + cursor = nullptr; + collection_count = key_count = key_size = value_size = 0; + + /* Get a session. */ + session = connection_manager::instance().create_session(); + /* Create n collections as per the configuration and store each collection name. */ + collection_count = config->get_int(COLLECTION_COUNT); + for (int i = 0; i < collection_count; ++i) { + collection_name = "table:collection" + std::to_string(i); + database.collections[collection_name] = {}; + testutil_check( + session->create(session, collection_name.c_str(), DEFAULT_FRAMEWORK_SCHEMA)); + ts = timestamp_manager->get_next_ts(); + testutil_check(tracking->save(tracking_operation::CREATE, collection_name, 0, "", ts)); + } + debug_print(std::to_string(collection_count) + " collections created", DEBUG_TRACE); + + /* Open a cursor on each collection and use the configuration to insert key/value pairs. */ + key_count = config->get_int(KEY_COUNT); + value_size = config->get_int(VALUE_SIZE); + testutil_assert(value_size > 0); + key_size = config->get_int(KEY_SIZE); + testutil_assert(key_size > 0); + /* Keys must be unique. */ + testutil_assert(key_count <= pow(10, key_size)); + + for (const auto &it_collections : database.collections) { + collection_name = it_collections.first; + key_cpt = 0; + /* WiredTiger lets you open a cursor on a collection using the same pointer. When a + * session is closed, WiredTiger APIs close the cursors too. */ + testutil_check( + session->open_cursor(session, collection_name.c_str(), NULL, NULL, &cursor)); + for (size_t j = 0; j < key_count; ++j) { + /* Generation of a unique key. */ + generated_key = number_to_string(key_size, key_cpt); + ++key_cpt; + /* + * Generation of a random string value using the size defined in the test + * configuration. + */ + generated_value = + random_generator::random_generator::instance().generate_string(value_size); + ts = timestamp_manager->get_next_ts(); + if (ts_enabled) + testutil_check(session->begin_transaction(session, "")); + testutil_check(insert(cursor, tracking, collection_name, generated_key.c_str(), + generated_value.c_str(), ts)); + if (ts_enabled) { + cfg = std::string(COMMIT_TS) + "=" + timestamp_manager->decimal_to_hex(ts); + testutil_check(session->commit_transaction(session, cfg.c_str())); + } + /* Update the memory representation of the collections. */ + database.collections[collection_name].keys[generated_key].exists = true; + /* Values are not stored here. */ + database.collections[collection_name].values = nullptr; + } + } + debug_print("Populate stage done", DEBUG_TRACE); + } + + /* Basic read operation that walks a cursors across all collections. */ + virtual void + read_operation(thread_context &context, WT_SESSION *session) + { + WT_CURSOR *cursor; + std::vector<WT_CURSOR *> cursors; + + testutil_assert(session != nullptr); + /* Get a cursor for each collection in collection_names. */ + for (const auto &it : context.get_collection_names()) { + testutil_check(session->open_cursor(session, it.c_str(), NULL, NULL, &cursor)); + cursors.push_back(cursor); + } + + while (!cursors.empty() && context.is_running()) { + /* Walk each cursor. */ + for (const auto &it : cursors) { + if (it->next(it) != 0) + it->reset(it); + } + } + } + + /* + * Basic update operation that updates all the keys to a random value in each collection. + */ + virtual void + update_operation(thread_context &context, WT_SESSION *session) + { + WT_CURSOR *cursor; + wt_timestamp_t ts; + std::vector<WT_CURSOR *> cursors; + std::string collection_name; + std::vector<std::string> collection_names = context.get_collection_names(); + key_value_t generated_value, key; + int64_t cpt, value_size = context.get_value_size(); + + testutil_assert(session != nullptr); + /* Get a cursor for each collection in collection_names. */ + for (const auto &it : collection_names) { + testutil_check(session->open_cursor(session, it.c_str(), NULL, NULL, &cursor)); + cursors.push_back(cursor); + } + + cpt = 0; + /* Walk each cursor. */ + for (const auto &it : cursors) { + collection_name = collection_names[cpt]; + /* Walk each key. */ + for (keys_iterator_t iter_key = context.get_collection_keys_begin(collection_name); + iter_key != context.get_collection_keys_end(collection_name); ++iter_key) { + /* Do not process removed keys. */ + if (!iter_key->second.exists) + continue; + + ts = context.get_timestamp_manager()->get_next_ts(); + + /* Start a transaction if possible. */ + if (!context.is_in_transaction()) { + context.begin_transaction(session, ""); + context.set_commit_timestamp(session, ts); + } + generated_value = + random_generator::random_generator::instance().generate_string(value_size); + testutil_check(update(context.get_tracking(), it, collection_name, + iter_key->first.c_str(), generated_value.c_str(), ts)); + + /* Commit the current transaction if possible. */ + context.increment_operation_count(); + if (context.can_commit_transaction()) + context.commit_transaction(session, ""); + } + ++cpt; + } + + /* + * The update operations will be later on inside a loop that will be managed through + * throttle management. + */ + while (context.is_running()) + context.sleep(); + + /* Make sure the last operation is committed now the work is finished. */ + if (context.is_in_transaction()) + context.commit_transaction(session, ""); + } + + private: + /* WiredTiger APIs wrappers for single operations. */ + template <typename K, typename V> + int + insert(WT_CURSOR *cursor, workload_tracking *tracking, const std::string &collection_name, + const K &key, const V &value, wt_timestamp_t ts) + { + int error_code; + + testutil_assert(cursor != nullptr); + cursor->set_key(cursor, key); + cursor->set_value(cursor, value); + error_code = cursor->insert(cursor); + + if (error_code == 0) { + debug_print("key/value inserted", DEBUG_TRACE); + error_code = + tracking->save(tracking_operation::INSERT, collection_name, key, value, ts); + } else + debug_print("key/value insertion failed", DEBUG_ERROR); + + return (error_code); + } + + template <typename K, typename V> + static int + update(workload_tracking *tracking, WT_CURSOR *cursor, const std::string &collection_name, + K key, V value, wt_timestamp_t ts) + { + int error_code; + + testutil_assert(tracking != nullptr); + testutil_assert(cursor != nullptr); + cursor->set_key(cursor, key); + cursor->set_value(cursor, value); + error_code = cursor->update(cursor); + + if (error_code == 0) { + debug_print("key/value update", DEBUG_TRACE); + error_code = + tracking->save(tracking_operation::UPDATE, collection_name, key, value, ts); + } else + debug_print("key/value update failed", DEBUG_ERROR); + + return (error_code); + } + + /* + * Convert a number to a string. If the resulting string is less than the given length, padding + * of '0' is added. + */ + static std::string + number_to_string(uint64_t size, uint64_t value) + { + std::string str, value_str = std::to_string(value); + testutil_assert(size >= value_str.size()); + uint64_t diff = size - value_str.size(); + std::string s(diff, '0'); + str = s.append(value_str); + return (str); + } +}; +} // namespace test_harness +#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/random_generator.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/random_generator.h index 7df4d7da3fb..7df4d7da3fb 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/random_generator.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/random_generator.h diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/thread_context.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h index 61ee7e01a88..e5275bc7819 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/thread_context.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h @@ -29,6 +29,8 @@ #ifndef THREAD_CONTEXT_H #define THREAD_CONTEXT_H +#include "../core/throttle.h" +#include "database_model.h" #include "random_generator.h" #include "workload_tracking.h" @@ -48,13 +50,10 @@ enum class thread_operation { /* Container class for a thread and any data types it may need to interact with the database. */ class thread_context { public: - thread_context(timestamp_manager *timestamp_manager, workload_tracking *tracking, - std::vector<std::string> collection_names, thread_operation type, int64_t max_op, - int64_t min_op, int64_t value_size) - : _collection_names(collection_names), _current_op_count(0U), _in_txn(false), - _running(false), _min_op(min_op), _max_op(max_op), _max_op_count(0), - _timestamp_manager(timestamp_manager), _type(type), _tracking(tracking), - _value_size(value_size) + thread_context(timestamp_manager *timestamp_manager, workload_tracking *tracking, database &db, + thread_operation type, int64_t max_op, int64_t min_op, int64_t value_size, throttle throttle) + : _database(db), _min_op(min_op), _max_op(max_op), _timestamp_manager(timestamp_manager), + _type(type), _tracking(tracking), _value_size(value_size), _throttle(throttle) { } @@ -64,10 +63,22 @@ class thread_context { _running = false; } - const std::vector<std::string> & + const std::vector<std::string> get_collection_names() const { - return (_collection_names); + return (_database.get_collection_names()); + } + + const keys_iterator_t + get_collection_keys_begin(const std::string &collection_name) const + { + return (_database.get_collection_keys_begin(collection_name)); + } + + const keys_iterator_t + get_collection_keys_end(const std::string &collection_name) const + { + return (_database.get_collection_keys_end(collection_name)); } thread_operation @@ -76,6 +87,12 @@ class thread_context { return (_type); } + timestamp_manager * + get_timestamp_manager() const + { + return (_timestamp_manager); + } + workload_tracking * get_tracking() const { @@ -94,6 +111,18 @@ class thread_context { return (_running); } + bool + is_in_transaction() const + { + return (_in_txn); + } + + void + sleep() + { + _throttle.sleep(); + } + void set_running(bool running) { @@ -105,7 +134,7 @@ class thread_context { { if (!_in_txn && _timestamp_manager->is_enabled()) { testutil_check( - session->begin_transaction(session, config.empty() ? NULL : config.c_str())); + session->begin_transaction(session, config.empty() ? nullptr : config.c_str())); /* This randomizes the number of operations to be executed in one transaction. */ _max_op_count = random_generator::instance().generate_integer(_min_op, _max_op); _current_op_count = 0; @@ -113,65 +142,69 @@ class thread_context { } } - /* Returns true if the current transaction has been committed. */ + /* + * The current transaction can be committed if: + * - The timestamp manager is enabled and + * - A transaction has started and + * - The thread is done working. This is useful when the test is ended and the thread has + * not reached the maximum number of operations per transaction or + * - The number of operations executed in the current transaction has exceeded the + * threshold. + */ bool - commit_transaction(WT_SESSION *session, const std::string &config) + can_commit_transaction() const { - if (!_timestamp_manager->is_enabled()) - return (true); + return (_timestamp_manager->is_enabled() && _in_txn && + (!_running || (_current_op_count > _max_op_count))); + } + void + commit_transaction(WT_SESSION *session, const std::string &config) + { /* A transaction cannot be committed if not started. */ testutil_assert(_in_txn); - /* The current transaction should be committed if: - * - The thread is done working. This is useful when the test is ended and the thread has - * not reached the maximum number of operations per transaction. - * - The number of operations executed in the current transaction has exceeded the - * threshold. - */ - if (!_running || (++_current_op_count > _max_op_count)) { - testutil_check( - session->commit_transaction(session, config.empty() ? nullptr : config.c_str())); - _in_txn = false; - } + testutil_check( + session->commit_transaction(session, config.empty() ? nullptr : config.c_str())); + _in_txn = false; + } - return (!_in_txn); + void + increment_operation_count(uint64_t inc = 1) + { + _current_op_count += inc; } /* - * Set a commit timestamp if the timestamp manager is enabled and always return the timestamp - * that should have been used for the commit. + * Set a commit timestamp if the timestamp manager is enabled. */ - wt_timestamp_t - set_commit_timestamp(WT_SESSION *session) + void + set_commit_timestamp(WT_SESSION *session, wt_timestamp_t ts) { + if (!_timestamp_manager->is_enabled()) + return; - wt_timestamp_t ts = _timestamp_manager->get_next_ts(); - std::string config; - - if (_timestamp_manager->is_enabled()) { - config = std::string(COMMIT_TS) + "=" + _timestamp_manager->decimal_to_hex(ts); - testutil_check(session->timestamp_transaction(session, config.c_str())); - } - - return (ts); + std::string config = std::string(COMMIT_TS) + "=" + _timestamp_manager->decimal_to_hex(ts); + testutil_check(session->timestamp_transaction(session, config.c_str())); } private: - const std::vector<std::string> _collection_names; + /* Representation of the collections and their key/value pairs in memory. */ + database _database; /* * _current_op_count is the current number of operations that have been executed in the current * transaction. */ - uint64_t _current_op_count; - bool _in_txn, _running; + uint64_t _current_op_count = 0U; + bool _in_txn = false, _running = false; /* * _min_op and _max_op are the minimum and maximum number of operations within one transaction. * _max_op_count is the current maximum number of operations that can be executed in the current * transaction. _max_op_count will always be <= _max_op. */ - int64_t _min_op, _max_op, _max_op_count; + int64_t _min_op, _max_op, _max_op_count = 0; timestamp_manager *_timestamp_manager; const thread_operation _type; + throttle _throttle; workload_tracking *_tracking; /* Temporary member that comes from the test configuration. */ int64_t _value_size; diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_tracking.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_tracking.h index d1464e60970..4d1b2d755a8 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_tracking.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_tracking.h @@ -33,7 +33,7 @@ * Default schema for tracking operations on collections (key_format: Collection name / Key / * Timestamp, value_format: Operation type / Value) */ -#define OPERATION_TRACKING_KEY_FORMAT WT_UNCHECKED_STRING(Sii) +#define OPERATION_TRACKING_KEY_FORMAT WT_UNCHECKED_STRING(SSQ) #define OPERATION_TRACKING_VALUE_FORMAT WT_UNCHECKED_STRING(iS) #define OPERATION_TRACKING_TABLE_CONFIG \ "key_format=" OPERATION_TRACKING_KEY_FORMAT ",value_format=" OPERATION_TRACKING_VALUE_FORMAT @@ -42,7 +42,7 @@ * Default schema for tracking schema operations on collections (key_format: Collection name / * Timestamp, value_format: Operation type) */ -#define SCHEMA_TRACKING_KEY_FORMAT WT_UNCHECKED_STRING(Si) +#define SCHEMA_TRACKING_KEY_FORMAT WT_UNCHECKED_STRING(SQ) #define SCHEMA_TRACKING_VALUE_FORMAT WT_UNCHECKED_STRING(i) #define SCHEMA_TRACKING_TABLE_CONFIG \ "key_format=" SCHEMA_TRACKING_KEY_FORMAT ",value_format=" SCHEMA_TRACKING_VALUE_FORMAT @@ -57,8 +57,7 @@ class workload_tracking : public component { workload_tracking(configuration *_config, const std::string &operation_table_config, const std::string &operation_table_name, const std::string &schema_table_config, const std::string &schema_table_name) - : component(_config), _cursor_operations(nullptr), _cursor_schema(nullptr), - _operation_table_config(operation_table_config), + : component("workload_tracking", _config), _operation_table_config(operation_table_config), _operation_table_name(operation_table_name), _schema_table_config(schema_table_config), _schema_table_name(schema_table_name) { @@ -81,7 +80,8 @@ class workload_tracking : public component { { WT_SESSION *session; - testutil_check(_config->get_bool(ENABLED, _enabled)); + component::load(); + if (!_enabled) return; @@ -145,8 +145,8 @@ class workload_tracking : public component { } private: - WT_CURSOR *_cursor_operations; - WT_CURSOR *_cursor_schema; + WT_CURSOR *_cursor_operations = nullptr; + WT_CURSOR *_cursor_schema = nullptr; const std::string _operation_table_config; const std::string _operation_table_name; const std::string _schema_table_config; diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_validation.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_validation.h index 86ff567bcc2..5ef7992e773 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_validation.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_validation.h @@ -35,112 +35,114 @@ extern "C" { #include "wiredtiger.h" } +#include "database_model.h" + namespace test_harness { + /* * Class that can validate database state and collection data. */ class workload_validation { public: /* - * Validate the on disk data against what has been tracked during the test. The first step is to - * replay the tracked operations so a representation in memory of the collections is created. - * This representation is then compared to what is on disk. The second step is to go through - * what has been saved on disk and make sure the memory representation has the same data. + * Validate the on disk data against what has been tracked during the test. + * - The first step is to replay the tracked operations so a representation in memory of the + * collections is created. This representation is then compared to what is on disk. + * - The second step is to go through what has been saved on disk and make sure the memory + * representation has the same data. * operation_table_name is the collection that contains all the operations about the key/value * pairs in the different collections used during the test. schema_table_name is the collection * that contains all the operations about the creation or deletion of collections during the * test. */ bool - validate(const std::string &operation_table_name, const std::string &schema_table_name) + validate(const std::string &operation_table_name, const std::string &schema_table_name, + database &database) { WT_SESSION *session; std::string collection_name; - /* - * Representation in memory of the collections at the end of the test. The first level is a - * map that contains collection names as keys. The second level is another map that contains - * the different key/value pairs within a given collection. If a collection yields to a null - * map of key/value pairs, this means the collection should not be present on disk. If a - * value from a key/value pair is null, this means the key should not be present in the - * collection on disk. - */ - std::map<std::string, std::map<int, std::string *> *> collections; /* Existing collections after the test. */ - std::vector<std::string> created_collections; - bool is_valid; + std::vector<std::string> created_collections, deleted_collections; + bool is_valid = true; session = connection_manager::instance().create_session(); - /* Retrieve the created collections that need to be checked. */ + /* Retrieve the collections that were created and deleted during the test. */ collection_name = schema_table_name; - created_collections = parse_schema_tracking_table(session, collection_name); + parse_schema_tracking_table( + session, collection_name, created_collections, deleted_collections); - /* Allocate memory to the operations performed on the created collections. */ + /* Make sure they exist in memory. */ for (auto const &it : created_collections) { - std::map<int, std::string *> *map = new std::map<int, std::string *>(); - collections[it] = map; + if (database.collections.count(it) == 0) { + debug_print("Collection missing in memory: " + it, DEBUG_ERROR); + is_valid = false; + break; + } } - /* - * Build in memory the final state of each created collection according to the tracked - * operations. - */ - collection_name = operation_table_name; - for (auto const &active_collection : created_collections) - parse_operation_tracking_table( - session, collection_name, active_collection, collections); - - /* Check all tracked operations in memory against the database on disk. */ - is_valid = check_reference(session, collections); + if (!is_valid) + return (is_valid); - /* Check what has been saved on disk against what has been tracked. */ - if (is_valid) { - for (auto const &collection : created_collections) { - is_valid = check_disk_state(session, collection, collections); - if (!is_valid) { - debug_print( - "check_disk_state failed for collection " + collection, DEBUG_ERROR); - break; - } + /* Make sure they don't exist in memory nor on disk. */ + for (auto const &it : deleted_collections) { + if (database.collections.count(it) > 0) { + debug_print( + "Collection present in memory while it has been tracked as deleted: " + it, + DEBUG_ERROR); + is_valid = false; + break; } + if (!verify_collection_state(session, it, false)) { + debug_print( + "Collection present on disk while it has been tracked as deleted: " + it, + DEBUG_ERROR); + is_valid = false; + break; + } + } - } else - debug_print("check_reference failed!", DEBUG_ERROR); - - /* Clean the allocated memory. */ - clean_memory(collections); - - return (is_valid); - } - - /* Clean the memory used to represent the collections after the test. */ - void - clean_memory(std::map<std::string, std::map<int, std::string *> *> &collections) - { - for (auto &it_collections : collections) { - if (it_collections.second == nullptr) - continue; + for (auto const &collection_name : database.get_collection_names()) { + if (!is_valid) + break; - for (auto &it_operations : *it_collections.second) { - delete it_operations.second; - it_operations.second = nullptr; + /* Get the values associated to the different keys in the current collection. */ + parse_operation_tracking_table( + session, operation_table_name, collection_name, database); + /* Check all tracked operations in memory against the database on disk. */ + if (!check_reference(session, collection_name, database)) { + debug_print( + "check_reference failed for collection " + collection_name, DEBUG_ERROR); + is_valid = false; + } + /* Check what has been saved on disk against what has been tracked. */ + else if (!check_disk_state(session, collection_name, database)) { + debug_print( + "check_disk_state failed for collection " + collection_name, DEBUG_ERROR); + is_valid = false; } - delete it_collections.second; - it_collections.second = nullptr; + /* Clear memory. */ + delete database.collections[collection_name].values; + database.collections[collection_name].values = nullptr; } + + return (is_valid); } + private: /* + * Read the tracking table to retrieve the created and deleted collections during the test. * collection_name is the collection that contains the operations on the different collections * during the test. */ - const std::vector<std::string> - parse_schema_tracking_table(WT_SESSION *session, const std::string &collection_name) + void + parse_schema_tracking_table(WT_SESSION *session, const std::string &collection_name, + std::vector<std::string> &created_collections, std::vector<std::string> &deleted_collections) { WT_CURSOR *cursor; + wt_timestamp_t key_timestamp; const char *key_collection_name; - int key_timestamp, value_operation_type; - std::vector<std::string> created_collections; + int value_operation_type; testutil_check(session->open_cursor(session, collection_name.c_str(), NULL, NULL, &cursor)); @@ -154,16 +156,18 @@ class workload_validation { if (static_cast<tracking_operation>(value_operation_type) == tracking_operation::CREATE) { + deleted_collections.erase(std::remove(deleted_collections.begin(), + deleted_collections.end(), key_collection_name), + deleted_collections.end()); created_collections.push_back(key_collection_name); } else if (static_cast<tracking_operation>(value_operation_type) == tracking_operation::DELETE_COLLECTION) { created_collections.erase(std::remove(created_collections.begin(), created_collections.end(), key_collection_name), created_collections.end()); + deleted_collections.push_back(key_collection_name); } } - - return (created_collections); } /* @@ -174,32 +178,44 @@ class workload_validation { */ void parse_operation_tracking_table(WT_SESSION *session, const std::string &tracking_collection_name, - const std::string &collection_name, - std::map<std::string, std::map<int, std::string *> *> &collections) + const std::string &collection_name, database &database) { WT_CURSOR *cursor; - int error_code, exact, key, key_timestamp, value_operation_type; - const char *key_collection_name, *value; + wt_timestamp_t key_timestamp; + int exact, value_operation_type; + const char *key, *key_collection_name, *value; + std::vector<key_value_t> collection_keys; + std::string key_str; + + /* Retrieve all keys from the given collection. */ + for (auto const &it : database.collections.at(collection_name).keys) + collection_keys.push_back(it.first); + /* There must be at least a key. */ + testutil_assert(!collection_keys.empty()); + /* Sort keys. */ + std::sort(collection_keys.begin(), collection_keys.end()); + /* Use the first key as a parameter for search_near. */ + key_str = collection_keys[0]; testutil_check( session->open_cursor(session, tracking_collection_name.c_str(), NULL, NULL, &cursor)); - /* Our keys start at 0. */ - cursor->set_key(cursor, collection_name.c_str(), 0); - error_code = cursor->search_near(cursor, &exact); - + cursor->set_key(cursor, collection_name.c_str(), key_str.c_str()); + testutil_check(cursor->search_near(cursor, &exact)); /* - * As we don't support deletion, the searched collection is expected to be found. Since the - * timestamp which is part of the key is not provided, exact is expected to be > 0. + * Since the timestamp which is part of the key is not provided, exact cannot be 0. If it is + * -1, we need to go to the next key. */ - testutil_check(exact < 1); + testutil_assert(exact != 0); + if (exact < 0) + testutil_check(cursor->next(cursor)); - while (error_code == 0) { + do { testutil_check(cursor->get_key(cursor, &key_collection_name, &key, &key_timestamp)); testutil_check(cursor->get_value(cursor, &value_operation_type, &value)); debug_print("Collection name is " + std::string(key_collection_name), DEBUG_TRACE); - debug_print("Key is " + std::to_string(key), DEBUG_TRACE); + debug_print("Key is " + std::string(key), DEBUG_TRACE); debug_print("Timestamp is " + std::to_string(key_timestamp), DEBUG_TRACE); debug_print("Operation type is " + std::to_string(value_operation_type), DEBUG_TRACE); debug_print("Value is " + std::string(value), DEBUG_TRACE); @@ -217,141 +233,138 @@ class workload_validation { /* * Operations are parsed from the oldest to the most recent one. It is safe to * assume the key has been inserted previously in an existing collection and can be - * deleted safely. + * safely deleted. */ - delete collections.at(key_collection_name)->at(key); - collections.at(key_collection_name)->at(key) = nullptr; + database.collections.at(key_collection_name).keys.at(std::string(key)).exists = + false; + delete database.collections.at(key_collection_name).values; + database.collections.at(key_collection_name).values = nullptr; break; case tracking_operation::INSERT: { /* Keys are unique, it is safe to assume the key has not been encountered before. */ - std::pair<int, std::string *> pair(key, new std::string(value)); - collections.at(key_collection_name)->insert(pair); + database.collections[key_collection_name].keys[std::string(key)].exists = true; + if (database.collections[key_collection_name].values == nullptr) { + database.collections[key_collection_name].values = + new std::map<key_value_t, value_t>(); + } + value_t v; + v.value = key_value_t(value); + std::pair<key_value_t, value_t> pair(key_value_t(key), v); + database.collections[key_collection_name].values->insert(pair); break; } - case tracking_operation::CREATE: - case tracking_operation::DELETE_COLLECTION: - testutil_die(DEBUG_ABORT, "Unexpected operation in the tracking table: %d", - static_cast<tracking_operation>(value_operation_type)); + case tracking_operation::UPDATE: + database.collections[key_collection_name].values->at(key).value = + key_value_t(value); + break; default: - testutil_die( - DEBUG_ABORT, "tracking operation is unknown : %d", value_operation_type); + testutil_die(DEBUG_ABORT, "Unexpected operation in the tracking table: %d", + value_operation_type); break; } - error_code = cursor->next(cursor); - } + } while (cursor->next(cursor) == 0); if (cursor->reset(cursor) != 0) debug_print("Cursor could not be reset !", DEBUG_ERROR); } /* - * Compare the tracked operations against what has been saved on disk. collections is the + * Compare the tracked operations against what has been saved on disk. database is the * representation in memory of the collections after the test according to the tracking table. */ bool check_reference( - WT_SESSION *session, std::map<std::string, std::map<int, std::string *> *> &collections) + WT_SESSION *session, const std::string &collection_name, const database &database) { + bool is_valid; + collection_t collection; + key_t key; + key_value_t key_str; + + /* Check the collection exists on disk. */ + is_valid = verify_collection_state(session, collection_name, true); - bool collection_exists, is_valid = true; - std::map<int, std::string *> *collection; - workload_validation wv; - std::string *value; - - for (const auto &it_collections : collections) { - /* Check the collection is in the correct state. */ - collection_exists = (it_collections.second != nullptr); - is_valid = wv.verify_database_state(session, it_collections.first, collection_exists); - - if (is_valid && collection_exists) { - collection = it_collections.second; - for (const auto &it_operations : *collection) { - value = (*collection)[it_operations.first]; - /* The key/value pair exists. */ - if (value != nullptr) - is_valid = (wv.is_key_present( - session, it_collections.first, it_operations.first) == true); - /* The key has been deleted. */ - else - is_valid = (wv.is_key_present( - session, it_collections.first, it_operations.first) == false); - - /* Check the associated value is valid. */ - if (is_valid && (value != nullptr)) { - is_valid = (wv.verify_value( - session, it_collections.first, it_operations.first, *value)); - } - - if (!is_valid) { - debug_print( - "check_reference failed for key " + std::to_string(it_operations.first), - DEBUG_ERROR); - break; - } + if (is_valid) { + collection = database.collections.at(collection_name); + /* Walk through each key/value pair of the current collection. */ + for (const auto &keys : collection.keys) { + key_str = keys.first; + key = keys.second; + /* The key/value pair exists. */ + if (key.exists) + is_valid = (is_key_present(session, collection_name, key_str.c_str()) == true); + /* The key has been deleted. */ + else + is_valid = (is_key_present(session, collection_name, key_str.c_str()) == false); + + /* Check the associated value is valid. */ + if (is_valid && key.exists) { + testutil_assert(collection.values != nullptr); + is_valid = verify_value(session, collection_name, key_str.c_str(), + collection.values->at(key_str).value); } - } - if (!is_valid) { - debug_print( - "check_reference failed for collection " + it_collections.first, DEBUG_ERROR); - break; + if (!is_valid) { + debug_print("check_reference failed for key " + key_str, DEBUG_ERROR); + break; + } } } + if (!is_valid) + debug_print("check_reference failed for collection " + collection_name, DEBUG_ERROR); + return (is_valid); } /* Check what is present on disk against what has been tracked. */ bool - check_disk_state(WT_SESSION *session, const std::string &collection_name, - std::map<std::string, std::map<int, std::string *> *> &collections) + check_disk_state( + WT_SESSION *session, const std::string &collection_name, const database &database) { WT_CURSOR *cursor; - int key; - const char *value; - bool is_valid; - std::string *value_str; - std::map<int, std::string *> *collection; + collection_t collection; + bool is_valid = true; + /* Key/value pairs on disk. */ + const char *key_on_disk, *value_on_disk; + key_value_t key_str, value_str; testutil_check(session->open_cursor(session, collection_name.c_str(), NULL, NULL, &cursor)); - /* Check the collection has been tracked and contains data. */ - is_valid = - ((collections.count(collection_name) > 0) && (collections[collection_name] != nullptr)); - - if (!is_valid) - debug_print( - "Collection " + collection_name + " has not been tracked or has been deleted", - DEBUG_ERROR); - else - collection = collections[collection_name]; + collection = database.collections.at(collection_name); /* Read the collection on disk. */ while (is_valid && (cursor->next(cursor) == 0)) { - testutil_check(cursor->get_key(cursor, &key)); - testutil_check(cursor->get_value(cursor, &value)); + testutil_check(cursor->get_key(cursor, &key_on_disk)); + testutil_check(cursor->get_value(cursor, &value_on_disk)); - debug_print("Key is " + std::to_string(key), DEBUG_TRACE); - debug_print("Value is " + std::string(value), DEBUG_TRACE); + key_str = std::string(key_on_disk); + + debug_print("Key on disk is " + key_str, DEBUG_TRACE); + debug_print("Value on disk is " + std::string(value_on_disk), DEBUG_TRACE); - if (collection->count(key) > 0) { - value_str = collection->at(key); + /* Check the key on disk has been saved in memory too. */ + if ((collection.keys.count(key_str) > 0) && collection.keys.at(key_str).exists) { + /* Memory should be allocated for values. */ + testutil_assert(collection.values != nullptr); + value_str = collection.values->at(key_str).value; /* * Check the key/value pair on disk matches the one in memory from the tracked * operations. */ - is_valid = (value_str != nullptr) && (*value_str == std::string(value)); + is_valid = (value_str == key_value_t(value_on_disk)); if (!is_valid) - debug_print(" Key/Value pair mismatch.\n Disk key: " + std::to_string(key) + - "\n Disk value: " + std ::string(value) + - "\n Tracking table key: " + std::to_string(key) + - "\n Tracking table value: " + (value_str == nullptr ? "NULL" : *value_str), + debug_print(" Key/Value pair mismatch.\n Disk key: " + key_str + + "\n Disk value: " + std ::string(value_on_disk) + + "\n Tracking table key: " + key_str + "\n Tracking table value exists: " + + std::to_string(collection.keys.at(key_str).exists) + + "\n Tracking table value: " + value_str, DEBUG_ERROR); } else { is_valid = false; debug_print( - "The key " + std::to_string(key) + " present on disk has not been tracked", + "The key " + std::string(key_on_disk) + " present on disk has not been tracked", DEBUG_ERROR); } } @@ -364,7 +377,7 @@ class workload_validation { * needs to be set to true if the collection is expected to be existing, false otherwise. */ bool - verify_database_state( + verify_collection_state( WT_SESSION *session, const std::string &collection_name, bool exists) const { WT_CURSOR *cursor; @@ -396,10 +409,8 @@ class workload_validation { testutil_check(cursor->search(cursor)); testutil_check(cursor->get_value(cursor, &value)); - return (value == expected_value); + return (key_value_t(value) == expected_value); } - - private: }; } // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_generator.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_generator.h index f9445cd892a..9413834ba31 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_generator.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_generator.h @@ -33,8 +33,11 @@ #include <atomic> #include <map> -#include "random_generator.h" -#include "workload_tracking.h" +#include "core/throttle.h" +#include "workload/database_model.h" +#include "workload/database_operation.h" +#include "workload/random_generator.h" +#include "workload/workload_tracking.h" namespace test_harness { /* @@ -42,9 +45,10 @@ namespace test_harness { */ class workload_generator : public component { public: - workload_generator(configuration *configuration, timestamp_manager *timestamp_manager, - workload_tracking *tracking) - : component(configuration), _timestamp_manager(timestamp_manager), _tracking(tracking) + workload_generator(configuration *configuration, database_operation *db_operation, + timestamp_manager *timestamp_manager, workload_tracking *tracking) + : component("workload_generator", configuration), _database_operation(db_operation), + _timestamp_manager(timestamp_manager), _tracking(tracking) { } @@ -58,118 +62,73 @@ class workload_generator : public component { workload_generator(const workload_generator &) = delete; workload_generator &operator=(const workload_generator &) = delete; - /* - * Function that performs the following steps using the configuration that is defined by the - * test: - * - Create the working dir. - * - Open a connection. - * - Open a session. - * - Create n collections as per the configuration. - * - Open a cursor on each collection. - * - Insert m key/value pairs in each collection. Values are random strings which size is - * defined by the configuration. - */ - void - populate() - { - WT_CURSOR *cursor; - WT_SESSION *session; - wt_timestamp_t ts; - int64_t collection_count, key_count, value_size; - std::string collection_name, config, generated_value, home; - bool ts_enabled = _timestamp_manager->is_enabled(); - - cursor = nullptr; - collection_count = key_count = value_size = 0; - collection_name = ""; - - /* Get a session. */ - session = connection_manager::instance().create_session(); - /* Create n collections as per the configuration and store each collection name. */ - testutil_check(_config->get_int(COLLECTION_COUNT, collection_count)); - for (int i = 0; i < collection_count; ++i) { - collection_name = "table:collection" + std::to_string(i); - testutil_check(session->create(session, collection_name.c_str(), DEFAULT_TABLE_SCHEMA)); - ts = _timestamp_manager->get_next_ts(); - testutil_check(_tracking->save(tracking_operation::CREATE, collection_name, 0, "", ts)); - _collection_names.push_back(collection_name); - } - debug_print(std::to_string(collection_count) + " collections created", DEBUG_TRACE); - - /* Open a cursor on each collection and use the configuration to insert key/value pairs. */ - testutil_check(_config->get_int(KEY_COUNT, key_count)); - testutil_check(_config->get_int(VALUE_SIZE, value_size)); - testutil_assert(value_size >= 0); - for (const auto &collection_name : _collection_names) { - /* WiredTiger lets you open a cursor on a collection using the same pointer. When a - * session is closed, WiredTiger APIs close the cursors too. */ - testutil_check( - session->open_cursor(session, collection_name.c_str(), NULL, NULL, &cursor)); - for (size_t j = 0; j < key_count; ++j) { - /* - * Generation of a random string value using the size defined in the test - * configuration. - */ - generated_value = - random_generator::random_generator::instance().generate_string(value_size); - ts = _timestamp_manager->get_next_ts(); - if (ts_enabled) - testutil_check(session->begin_transaction(session, "")); - testutil_check(insert(cursor, collection_name, j + 1, generated_value.c_str(), ts)); - if (ts_enabled) { - config = std::string(COMMIT_TS) + "=" + _timestamp_manager->decimal_to_hex(ts); - testutil_check(session->commit_transaction(session, config.c_str())); - } - } - } - debug_print("Populate stage done", DEBUG_TRACE); - } - /* Do the work of the main part of the workload. */ void run() { - configuration *sub_config; - int64_t read_threads, min_operation_per_transaction, max_operation_per_transaction, - value_size; + configuration *transaction_config, *update_config, *insert_config; + int64_t min_operation_per_transaction, max_operation_per_transaction, read_threads, + update_threads, value_size; /* Populate the database. */ - populate(); + _database_operation->populate(_database, _timestamp_manager, _config, _tracking); /* Retrieve useful parameters from the test configuration. */ - testutil_check(_config->get_int(READ_THREADS, read_threads)); - sub_config = _config->get_subconfig(OPS_PER_TRANSACTION); - testutil_check(sub_config->get_int(MIN, min_operation_per_transaction)); - testutil_check(sub_config->get_int(MAX, max_operation_per_transaction)); + transaction_config = _config->get_subconfig(OPS_PER_TRANSACTION); + update_config = _config->get_subconfig(UPDATE_CONFIG); + insert_config = _config->get_subconfig(INSERT_CONFIG); + read_threads = _config->get_int(READ_THREADS); + update_threads = _config->get_int(UPDATE_THREADS); + + min_operation_per_transaction = transaction_config->get_int(MIN); + max_operation_per_transaction = transaction_config->get_int(MAX); testutil_assert(max_operation_per_transaction >= min_operation_per_transaction); - testutil_check(_config->get_int(VALUE_SIZE, value_size)); + value_size = _config->get_int(VALUE_SIZE); testutil_assert(value_size >= 0); - delete sub_config; - /* Generate threads to execute read operations on the collections. */ for (int i = 0; i < read_threads; ++i) { - thread_context *tc = new thread_context(_timestamp_manager, _tracking, - _collection_names, thread_operation::READ, max_operation_per_transaction, - min_operation_per_transaction, value_size); + thread_context *tc = new thread_context(_timestamp_manager, _tracking, _database, + thread_operation::READ, max_operation_per_transaction, min_operation_per_transaction, + value_size, throttle()); _workers.push_back(tc); - _thread_manager.add_thread(tc, &execute_operation); + _thread_manager.add_thread(tc, _database_operation, &execute_operation); } + + /* Generate threads to execute update operations on the collections. */ + for (int i = 0; i < update_threads; ++i) { + thread_context *tc = new thread_context(_timestamp_manager, _tracking, _database, + thread_operation::UPDATE, max_operation_per_transaction, + min_operation_per_transaction, value_size, throttle(update_config)); + _workers.push_back(tc); + _thread_manager.add_thread(tc, _database_operation, &execute_operation); + } + + delete transaction_config; + delete update_config; + delete insert_config; } void finish() { - for (const auto &it : _workers) { + component::finish(); + + for (const auto &it : _workers) it->finish(); - } _thread_manager.join(); debug_print("Workload generator: run stage done", DEBUG_TRACE); } + database & + get_database() + { + return _database; + } + /* Workload threaded operations. */ static void - execute_operation(thread_context &context) + execute_operation(thread_context &context, database_operation &db_operation) { WT_SESSION *session; @@ -177,7 +136,7 @@ class workload_generator : public component { switch (context.get_thread_operation()) { case thread_operation::READ: - read_operation(context, session); + db_operation.read_operation(context, session); break; case thread_operation::REMOVE: case thread_operation::INSERT: @@ -186,7 +145,7 @@ class workload_generator : public component { std::this_thread::sleep_for(std::chrono::seconds(1)); break; case thread_operation::UPDATE: - update_operation(context, session); + db_operation.update_operation(context, session); break; default: testutil_die(DEBUG_ABORT, "system: thread_operation is unknown : %d", @@ -195,134 +154,9 @@ class workload_generator : public component { } } - /* - * Basic update operation that currently update the same key with a random value in each - * collection. - */ - static void - update_operation(thread_context &context, WT_SESSION *session) - { - WT_CURSOR *cursor; - wt_timestamp_t ts; - std::vector<WT_CURSOR *> cursors; - std::vector<std::string> collection_names; - std::string generated_value; - bool has_committed = true; - int64_t cpt, value_size = context.get_value_size(); - - testutil_assert(session != nullptr); - /* Get a cursor for each collection in collection_names. */ - for (const auto &it : context.get_collection_names()) { - testutil_check(session->open_cursor(session, it.c_str(), NULL, NULL, &cursor)); - cursors.push_back(cursor); - collection_names.push_back(it); - } - - while (context.is_running()) { - /* Walk each cursor. */ - context.begin_transaction(session, ""); - ts = context.set_commit_timestamp(session); - cpt = 0; - for (const auto &it : cursors) { - generated_value = - random_generator::random_generator::instance().generate_string(value_size); - /* Key is hard coded for now. */ - testutil_check(update(context.get_tracking(), it, collection_names[cpt], 1, - generated_value.c_str(), ts)); - ++cpt; - } - has_committed = context.commit_transaction(session, ""); - } - - /* Make sure the last operation is committed now the work is finished. */ - if (!has_committed) - context.commit_transaction(session, ""); - } - - /* Basic read operation that walks a cursors across all collections. */ - static void - read_operation(thread_context &context, WT_SESSION *session) - { - WT_CURSOR *cursor; - std::vector<WT_CURSOR *> cursors; - - testutil_assert(session != nullptr); - /* Get a cursor for each collection in collection_names. */ - for (const auto &it : context.get_collection_names()) { - testutil_check(session->open_cursor(session, it.c_str(), NULL, NULL, &cursor)); - cursors.push_back(cursor); - } - - while (context.is_running()) { - /* Walk each cursor. */ - for (const auto &it : cursors) { - if (it->next(it) != 0) - it->reset(it); - } - } - } - - /* WiredTiger APIs wrappers for single operations. */ - template <typename K, typename V> - int - insert(WT_CURSOR *cursor, const std::string &collection_name, K key, V value, wt_timestamp_t ts) - { - int error_code; - - testutil_assert(cursor != nullptr); - cursor->set_key(cursor, key); - cursor->set_value(cursor, value); - error_code = cursor->insert(cursor); - - if (error_code == 0) { - debug_print("key/value inserted", DEBUG_TRACE); - error_code = - _tracking->save(tracking_operation::INSERT, collection_name, key, value, ts); - } else - debug_print("key/value insertion failed", DEBUG_ERROR); - - return (error_code); - } - - static int - search(WT_CURSOR *cursor) - { - testutil_assert(cursor != nullptr); - return (cursor->search(cursor)); - } - - static int - search_near(WT_CURSOR *cursor, int *exact) - { - testutil_assert(cursor != nullptr); - return (cursor->search_near(cursor, exact)); - } - - template <typename K, typename V> - static int - update(workload_tracking *tracking, WT_CURSOR *cursor, const std::string &collection_name, - K key, V value, wt_timestamp_t ts) - { - int error_code; - - testutil_assert(tracking != nullptr); - testutil_assert(cursor != nullptr); - cursor->set_key(cursor, key); - cursor->set_value(cursor, value); - error_code = cursor->update(cursor); - - if (error_code == 0) { - debug_print("key/value update", DEBUG_TRACE); - error_code = - tracking->save(tracking_operation::UPDATE, collection_name, key, value, ts); - } else - debug_print("key/value update failed", DEBUG_ERROR); - - return (error_code); - } - private: - std::vector<std::string> _collection_names; + database _database; + database_operation *_database_operation; thread_manager _thread_manager; timestamp_manager *_timestamp_manager; workload_tracking *_tracking; diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/example_test.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/example_test.cxx new file mode 100644 index 00000000000..cc08d3d003a --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/tests/example_test.cxx @@ -0,0 +1,55 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "test_harness/test.h" + +/* + * Class that defines operations that do nothing as an example. + * This shows how database operations can be overriden and customized. + */ +class example_test : public test_harness::test { + public: + example_test(const std::string &config, const std::string &name) : test(config, name) {} + + virtual void + populate(test_harness::database &database, test_harness::timestamp_manager *_timestamp_manager, + test_harness::configuration *_config, test_harness::workload_tracking *tracking) + { + std::cout << "populate: nothing done." << std::endl; + } + virtual void + read_operation(test_harness::thread_context &context, WT_SESSION *session) + { + std::cout << "read_operation: nothing done." << std::endl; + } + virtual void + update_operation(test_harness::thread_context &context, WT_SESSION *session) + { + std::cout << "update_operation: nothing done." << std::endl; + } +}; diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx index f9d5902b7ff..5fe6641cc3b 100755 --- a/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx @@ -30,51 +30,25 @@ #include <iostream> #include <string> -#include "test_harness/debug_utils.h" +#include "test_harness/util/debug_utils.h" #include "test_harness/test.h" +#include "example_test.cxx" #include "poc_test.cxx" std::string parse_configuration_from_file(const std::string &filename) { - std::string cfg, line, prev_line, error; + std::string cfg, line, error; std::ifstream cFile(filename); if (cFile.is_open()) { while (getline(cFile, line)) { - - if (line[0] == '#' || line.empty()) - continue; - /* Whitespaces are only for readability, they can be removed safely. */ line.erase(std::remove_if(line.begin(), line.end(), isspace), line.end()); - - if (prev_line == line && line != "}") { - error = - "Error when parsing configuration. Two consecutive lines are equal to " + line; - testutil_die(EINVAL, error.c_str()); - break; - } - - /* Start of a sub config. */ - if (line == "{") - cfg += "("; - /* End of a sub config. */ - else if (line == "}") - cfg += ")"; - else { - /* First line. */ - if (cfg.empty()) - cfg += line; - /* No comma needed at the start of a subconfig. */ - else if (prev_line == "{") - cfg += line; - else - cfg += "," + line; - } - - prev_line = line; + if (line[0] == '#' || line.empty()) + continue; + cfg += line; } } else { @@ -86,31 +60,60 @@ parse_configuration_from_file(const std::string &filename) } void +print_help() +{ + std::cout << "NAME" << std::endl; + std::cout << "\trun" << std::endl; + std::cout << std::endl; + std::cout << "SYNOPSIS" << std::endl; + std::cout << "\trun [OPTIONS]" << std::endl; + std::cout << "\trun -C [CONFIGURATION]" << std::endl; + std::cout << "\trun -f [FILE]" << std::endl; + std::cout << "\trun -l [TRACEL_LEVEL]" << std::endl; + std::cout << "\trun -t [TEST_NAME]" << std::endl; + std::cout << std::endl; + std::cout << "DESCRIPTION" << std::endl; + std::cout << "\trun executes the test framework." << std::endl; + std::cout << "\tIf no test is indicated, all tests are executed." << std::endl; + std::cout + << "\tIf no configuration is indicated, the default configuration for each test will be used." + << std::endl; + std::cout + << "\tIf a configuration is indicated, the given configuration will be used either for " + "all tests or the test indicated." + << std::endl; + std::cout << std::endl; + std::cout << "OPTIONS" << std::endl; + std::cout << "\t-h Output a usage message and exit." << std::endl; + std::cout << "\t-C Configuration. Cannot be used with -f." << std::endl; + std::cout << "\t-f File that contains the configuration. Cannot be used with -C." << std::endl; + std::cout << "\t-l Trace level from 0 (default) to 2." << std::endl; + std::cout << "\t-t Test name to be executed." << std::endl; +} + +void value_missing_error(const std::string &str) { - test_harness::debug_print("Value missing for option " + str, DEBUG_ERROR); + test_harness::debug_print( + "Value missing for option " + str + ".\nTry './run -h' for more information.", DEBUG_ERROR); } /* * Run a specific test. - * config_name is the configuration name. The default configuration is used if it is left empty. + * test_name: specifies which test to run. + * config: defines the configuration used for the test. */ int64_t -run_test(const std::string &test_name, const std::string &config_name = "") +run_test(const std::string &test_name, const std::string &config) { - std::string cfg, cfg_path; int error_code = 0; - if (config_name.empty()) - cfg_path = "configs/config_" + test_name + "_default.txt"; - else - cfg_path = config_name; - cfg = parse_configuration_from_file(cfg_path); - - test_harness::debug_print("Configuration\t: " + cfg, DEBUG_INFO); + test_harness::debug_print("Configuration\t:" + config, DEBUG_INFO); if (test_name == "poc_test") - poc_test(cfg, test_name).run(); + poc_test(config, test_name).run(); + else if (test_name == "example_test") + example_test(config, test_name).run(); else { test_harness::debug_print("Test not found: " + test_name, DEBUG_ERROR); error_code = -1; @@ -125,19 +128,24 @@ run_test(const std::string &test_name, const std::string &config_name = "") int main(int argc, char *argv[]) { - std::string cfg, config_name, test_name; + std::string cfg, config_filename, test_name, current_test_name; int64_t error_code = 0; - const std::vector<std::string> all_tests = {"poc_test"}; + const std::vector<std::string> all_tests = {"example_test", "poc_test"}; /* Parse args - * -C : Configuration. Cannot be used with -f. - * -f : Filename that contains the configuration. Cannot be used with -C. + * -C : Configuration. Cannot be used with -f. If no specific test is specified to be run, the + * same coniguration will be used for all existing tests. + * -f : Filename that contains the configuration. Cannot be used with -C. If no specific test + * is specified to be run, the same coniguration will be used for all existing tests. * -l : Trace level. * -t : Test to run. All tests are run if not specified. */ for (int i = 1; (i < argc) && (error_code == 0); ++i) { - if (std::string(argv[i]) == "-C") { - if (!config_name.empty()) { + if (std::string(argv[i]) == "-h") { + print_help(); + return 0; + } else if (std::string(argv[i]) == "-C") { + if (!config_filename.empty()) { test_harness::debug_print("Option -C cannot be used with -f", DEBUG_ERROR); error_code = -1; } else if ((i + 1) < argc) @@ -151,7 +159,7 @@ main(int argc, char *argv[]) test_harness::debug_print("Option -f cannot be used with -C", DEBUG_ERROR); error_code = -1; } else if ((i + 1) < argc) - config_name = argv[++i]; + config_filename = argv[++i]; else { value_missing_error(argv[i]); error_code = -1; @@ -180,14 +188,33 @@ main(int argc, char *argv[]) /* Run all tests. */ test_harness::debug_print("Running all tests.", DEBUG_INFO); for (auto const &it : all_tests) { - error_code = run_test(it); - if (error_code != 0) { - test_harness::debug_print("Test " + it + " failed.", DEBUG_ERROR); - break; + current_test_name = it; + /* Configuration parsing. */ + if (!config_filename.empty()) + cfg = parse_configuration_from_file(config_filename); + else if (cfg.empty()) { + config_filename = "configs/config_" + current_test_name + "_default.txt"; + cfg = parse_configuration_from_file(config_filename); } + + error_code = run_test(current_test_name, cfg); + if (error_code != 0) + break; + } + } else { + current_test_name = test_name; + /* Configuration parsing. */ + if (!config_filename.empty()) + cfg = parse_configuration_from_file(config_filename); + else if (cfg.empty()) { + config_filename = "configs/config_" + test_name + "_default.txt"; + cfg = parse_configuration_from_file(config_filename); } - } else - error_code = run_test(test_name, config_name); + error_code = run_test(current_test_name, cfg); + } + + if (error_code != 0) + test_harness::debug_print("Test " + current_test_name + " failed.", DEBUG_ERROR); } return (error_code); diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index 1fc83cfe862..b4ab0507399 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -205,13 +205,19 @@ functions: # Use separate shell.exec with "silent" directive to avoid exposing credentail in task log. - command: shell.exec params: - working_dir: "wiredtiger/wiredtiger.github.com" + working_dir: "wiredtiger" shell: bash silent: true script: | set -o errexit set -o verbose + if [[ "${branch_name}" != "develop" ]]; then + echo "We only run the documentation update task on the WiredTiger (develop) Evergreen project." + exit 0 + fi + + cd wiredtiger.github.com git push https://${doc-update-github-token}@github.com/wiredtiger/wiredtiger.github.com "make check directory": command: shell.exec @@ -300,11 +306,11 @@ functions: "truncated log test": command: shell.exec params: - working_dir: "wiredtiger/build_posix/test/csuite" + working_dir: "wiredtiger/build_posix/" script: | set -o errexit set -o verbose - ${test_env_vars|} ./test_truncated_log ${truncated_log_args|} 2>&1 + ${test_env_vars|} ./test/csuite/test_truncated_log ${truncated_log_args|} 2>&1 "recovery stress test script": command: shell.exec params: @@ -554,17 +560,6 @@ tasks: - func: "upload artifact" - func: "cleanup" - - name: compile-asan - tags: ["pull_request"] - commands: - - func: "get project" - - func: "compile wiredtiger" - vars: - configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/clang CXX=/opt/mongodbtoolchain/v3/bin/clang++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-fsanitize=address -fno-omit-frame-pointer -ggdb" CXXFLAGS="-fsanitize=address -fno-omit-frame-pointer -ggdb" - posix_configure_flags: --enable-silent-rules --enable-strict --enable-diagnostic --disable-static - - func: "upload artifact" - - func: "cleanup" - - name: compile-msan commands: - func: "get project" @@ -700,22 +695,6 @@ tasks: smp_command: -j 1 test_env_vars: MSAN_OPTIONS=abort_on_error=1:disable_coredump=0 MSAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer TESTUTIL_SLOW_MACHINE=1 - - name: make-check-asan-test - depends_on: - - name: compile-asan - commands: - - func: "fetch artifacts" - vars: - dependent_task: compile-asan - - func: "compile wiredtiger" - vars: - configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/clang CXX=/opt/mongodbtoolchain/v3/bin/clang++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-fsanitize=address -fno-omit-frame-pointer -ggdb" CXXFLAGS="-fsanitize=address -fno-omit-frame-pointer -ggdb" - posix_configure_flags: --enable-silent-rules --enable-strict --enable-diagnostic --disable-static - - func: "make check all" - vars: - smp_command: -j 1 - test_env_vars: ASAN_OPTIONS=detect_leaks=1:abort_on_error=1:disable_coredump=0 ASAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer TESTUTIL_SLOW_MACHINE=1 TESTUTIL_BYPASS_ASAN=1 - - name: make-check-linux-no-ftruncate-test depends_on: - name: compile-linux-no-ftruncate @@ -731,7 +710,7 @@ tasks: # Start of normal make check test tasks - name: lang-python-test - tags: ["pull_request"] + tags: ["pull_request", "python"] depends_on: - name: compile commands: @@ -752,23 +731,6 @@ tasks: vars: directory: examples/c - - name: examples-c-asan-test - tags: ["pull_request"] - depends_on: - - name: compile-asan - commands: - - func: "fetch artifacts" - vars: - dependent_task: compile-asan - - func: "compile wiredtiger" - vars: - configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/clang CXX=/opt/mongodbtoolchain/v3/bin/clang++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-fsanitize=address -ggdb" CXXFLAGS="-fsanitize=address -ggdb" - posix_configure_flags: --enable-silent-rules --enable-strict --enable-diagnostic --disable-static - - func: "make check directory" - vars: - test_env_vars: ASAN_OPTIONS=detect_leaks=1:abort_on_error=1:disable_coredump=0 ASAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer - directory: examples/c - - name: examples-c-production-disable-shared-test tags: ["pull_request"] depends_on: @@ -952,6 +914,24 @@ tasks: set -o verbose UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1:abort_on_error=1:disable_coredump=0 ./ex_access + # Start of cppsuite test tasks. + + - name: poc-test-cpp + tags: ["pull_request"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix/" + script: | + set -o errexit + set -o verbose + + ${test_env_vars|} $(pwd)/test/cppsuite/run -t poc_test -f test/cppsuite/configs/config_poc_test_default.txt -l 1 + + # End of cppsuite test tasks. # Start of csuite test tasks - name: csuite-incr-backup-test @@ -1508,6 +1488,7 @@ tasks: # Start of Python unit test tasks - name: unit-test + tags: ["python"] depends_on: - name: compile commands: @@ -1515,12 +1496,14 @@ tasks: - func: "unit test" - name: unit-test-with-compile + tags: ["python"] commands: - func: "get project" - func: "compile wiredtiger" - func: "unit test" - name: unit-test-long + tags: ["python"] depends_on: - name: compile commands: @@ -1530,6 +1513,7 @@ tasks: unit_test_args: -v 2 --long - name: unit-linux-no-ftruncate-test + tags: ["python"] depends_on: - name: compile-linux-no-ftruncate commands: @@ -1540,6 +1524,7 @@ tasks: # Run the tests that uses suite_random with a random starting seed - name: unit-test-random-seed + tags: ["python"] depends_on: - name: compile commands: @@ -1551,7 +1536,7 @@ tasks: # and we use the -b option of the test/suite/run.py script to split up the tests. - name: unit-test-bucket00 - tags: ["pull_request", "unit_test"] + tags: ["pull_request", "python", "unit_test"] depends_on: - name: compile commands: @@ -1561,7 +1546,7 @@ tasks: unit_test_args: -v 2 -b 0/11 - name: unit-test-bucket01 - tags: ["pull_request", "unit_test"] + tags: ["pull_request", "python", "unit_test"] depends_on: - name: compile commands: @@ -1571,7 +1556,7 @@ tasks: unit_test_args: -v 2 -b 1/11 - name: unit-test-bucket02 - tags: ["pull_request", "unit_test"] + tags: ["pull_request", "python", "unit_test"] depends_on: - name: compile commands: @@ -1581,7 +1566,7 @@ tasks: unit_test_args: -v 2 -b 2/11 - name: unit-test-bucket03 - tags: ["pull_request", "unit_test"] + tags: ["pull_request", "python", "unit_test"] depends_on: - name: compile commands: @@ -1591,7 +1576,7 @@ tasks: unit_test_args: -v 2 -b 3/11 - name: unit-test-bucket04 - tags: ["pull_request", "unit_test"] + tags: ["pull_request", "python", "unit_test"] depends_on: - name: compile commands: @@ -1601,7 +1586,7 @@ tasks: unit_test_args: -v 2 -b 4/11 - name: unit-test-bucket05 - tags: ["pull_request", "unit_test"] + tags: ["pull_request", "python", "unit_test"] depends_on: - name: compile commands: @@ -1611,7 +1596,7 @@ tasks: unit_test_args: -v 2 -b 5/11 - name: unit-test-bucket06 - tags: ["pull_request", "unit_test"] + tags: ["pull_request", "python", "unit_test"] depends_on: - name: compile commands: @@ -1621,7 +1606,7 @@ tasks: unit_test_args: -v 2 -b 6/11 - name: unit-test-bucket07 - tags: ["pull_request", "unit_test"] + tags: ["pull_request", "python", "unit_test"] depends_on: - name: compile commands: @@ -1631,7 +1616,7 @@ tasks: unit_test_args: -v 2 -b 7/11 - name: unit-test-bucket08 - tags: ["pull_request", "unit_test"] + tags: ["pull_request", "python", "unit_test"] depends_on: - name: compile commands: @@ -1641,7 +1626,7 @@ tasks: unit_test_args: -v 2 -b 8/11 - name: unit-test-bucket09 - tags: ["pull_request", "unit_test"] + tags: ["pull_request", "python", "unit_test"] depends_on: - name: compile commands: @@ -1651,7 +1636,7 @@ tasks: unit_test_args: -v 2 -b 9/11 - name: unit-test-bucket10 - tags: ["pull_request", "unit_test"] + tags: ["pull_request", "python", "unit_test"] depends_on: - name: compile commands: @@ -1676,7 +1661,7 @@ tasks: sh s_all -A -E 2>&1 - name: conf-dump-test - tags: ["pull_request"] + tags: ["pull_request", "python"] depends_on: - name: compile commands: @@ -1735,14 +1720,15 @@ tasks: - func: "fetch artifacts" - command: shell.exec params: - working_dir: "wiredtiger" + working_dir: "wiredtiger/build_posix" script: | set -o errexit set -o verbose if [ "Windows_NT" = "$OS" ]; then + cd .. cmd.exe /c t_fops.exe else - build_posix/test/fops/t + ${test_env_vars|} test/fops/t fi - name: million-collection-test @@ -2561,7 +2547,6 @@ buildvariants: - name: linux-directio distros: ubuntu1804-build - name: syscall-linux - - name: make-check-asan-test - name: configure-combinations - name: checkpoint-filetypes-test - name: unit-test-long @@ -2575,6 +2560,29 @@ buildvariants: - name: static-wt-build-test - name: format-failure-configs-test +- name: ubuntu1804-asan + display_name: "! Ubuntu 18.04 ASAN" + run_on: + - ubuntu1804-test + expansions: + configure_env_vars: + CC=/opt/mongodbtoolchain/v3/bin/clang + CXX=/opt/mongodbtoolchain/v3/bin/clang++ + PATH=/opt/mongodbtoolchain/v3/bin:$PATH + CFLAGS="-fsanitize=address -fno-omit-frame-pointer -ggdb" + CXXFLAGS="-fsanitize=address -fno-omit-frame-pointer -ggdb" + posix_configure_flags: --enable-silent-rules --enable-strict --enable-diagnostic --disable-static --prefix=$(pwd)/LOCAL_INSTALL + smp_command: -j $(grep -c ^processor /proc/cpuinfo) + make_command: PATH=/opt/mongodbtoolchain/v3/bin:$PATH make + test_env_vars: + ASAN_OPTIONS="detect_leaks=1:abort_on_error=1:disable_coredump=0" + ASAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer + TESTUTIL_BYPASS_ASAN=1 + LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libeatmydata.so PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs top_srcdir=$(pwd)/.. top_builddir=$(pwd) + tasks: + - name: ".pull_request !.windows_only !.pull_request_compilers !.python" + - examples-c-test + - name: ubuntu1804-compilers display_name: "! Ubuntu 18.04 Compilers" run_on: @@ -2663,8 +2671,6 @@ buildvariants: - name: linux-directio distros: rhel80-build - name: syscall-linux - - name: compile-asan - - name: make-check-asan-test - name: checkpoint-filetypes-test - name: unit-test-long - name: spinlock-gcc-test diff --git a/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh b/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh index 4d5a04bf0e7..4c62d379d38 100755 --- a/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh +++ b/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh @@ -72,6 +72,7 @@ run_format() args+="checkpoints=1 " # Force periodic writes args+="compression=snappy " # We only built with snappy, force the choice args+="data_source=table " + args+="huffman_key=0 " # Not supoprted by newer releases args+="in_memory=0 " # Interested in the on-disk format args+="leak_memory=1 " # Faster runs args+="logging=1 " # Test log compatibility diff --git a/src/third_party/wiredtiger/test/format/Makefile.am b/src/third_party/wiredtiger/test/format/Makefile.am index aa49dc6d732..771e41fd662 100644 --- a/src/third_party/wiredtiger/test/format/Makefile.am +++ b/src/third_party/wiredtiger/test/format/Makefile.am @@ -4,7 +4,7 @@ AM_CPPFLAGS +=-I$(top_srcdir)/test/utility noinst_PROGRAMS = t t_SOURCES =\ - alter.c backup.c bulk.c checkpoint.c compact.c config.c config_compat.c hs.c kv.c ops.c \ + alter.c backup.c bulk.c checkpoint.c compact.c config.c config_compat.c hs.c import.c kv.c ops.c \ random.c salvage.c snap.c t.c trace.c util.c wts.c t_LDADD = $(top_builddir)/test/utility/libtest_util.la diff --git a/src/third_party/wiredtiger/test/format/backup.c b/src/third_party/wiredtiger/test/format/backup.c index afeed3b247a..8ec113dd6d8 100644 --- a/src/third_party/wiredtiger/test/format/backup.c +++ b/src/third_party/wiredtiger/test/format/backup.c @@ -266,21 +266,21 @@ copy_blocks(WT_SESSION *session, WT_CURSOR *bkup_c, const char *name) len = strlen(g.home) + strlen(name) + 10; tmp = dmalloc(len); testutil_check(__wt_snprintf(tmp, len, "%s/%s", g.home, name)); - error_sys_check(rfd = open(tmp, O_RDONLY, 0)); + error_sys_check(rfd = open(tmp, O_RDONLY, 0644)); free(tmp); tmp = NULL; len = strlen(g.home) + strlen("BACKUP") + strlen(name) + 10; tmp = dmalloc(len); testutil_check(__wt_snprintf(tmp, len, "%s/BACKUP/%s", g.home, name)); - error_sys_check(wfd1 = open(tmp, O_WRONLY | O_CREAT, 0)); + error_sys_check(wfd1 = open(tmp, O_WRONLY | O_CREAT, 0644)); free(tmp); tmp = NULL; len = strlen(g.home) + strlen("BACKUP.copy") + strlen(name) + 10; tmp = dmalloc(len); testutil_check(__wt_snprintf(tmp, len, "%s/BACKUP.copy/%s", g.home, name)); - error_sys_check(wfd2 = open(tmp, O_WRONLY | O_CREAT, 0)); + error_sys_check(wfd2 = open(tmp, O_WRONLY | O_CREAT, 0644)); free(tmp); tmp = NULL; @@ -349,39 +349,6 @@ copy_blocks(WT_SESSION *session, WT_CURSOR *bkup_c, const char *name) free(tmp); } -/* - * copy_file -- - * Copy a single file into the backup directories. - */ -static void -copy_file(WT_SESSION *session, const char *name) -{ - size_t len; - char *first, *second; - - len = strlen("BACKUP") + strlen(name) + 10; - first = dmalloc(len); - testutil_check(__wt_snprintf(first, len, "BACKUP/%s", name)); - testutil_check(__wt_copy_and_sync(session, name, first)); - - /* - * Save another copy of the original file to make debugging recovery errors easier. - */ - len = strlen("BACKUP.copy") + strlen(name) + 10; - second = dmalloc(len); - testutil_check(__wt_snprintf(second, len, "BACKUP.copy/%s", name)); - testutil_check(__wt_copy_and_sync(session, first, second)); - - free(first); - free(second); -} - -/* - * Backup directory initialize command, remove and re-create the primary backup directory, plus a - * copy we maintain for recovery testing. - */ -#define HOME_BACKUP_INIT_CMD "rm -rf %s/BACKUP %s/BACKUP.copy && mkdir %s/BACKUP %s/BACKUP.copy" - #define RESTORE_SKIP 1 #define RESTORE_SUCCESS 0 /* @@ -507,11 +474,10 @@ backup(void *arg) WT_CURSOR *backup_cursor; WT_DECL_RET; WT_SESSION *session; - size_t len; u_int incremental, period; uint64_t src_id, this_id; const char *config, *key; - char cfg[512], *cmd; + char cfg[512]; bool full, incr_full; (void)(arg); @@ -615,12 +581,7 @@ backup(void *arg) /* If we're taking a full backup, create the backup directories. */ if (full || incremental == 0) { - len = strlen(g.home) * 4 + strlen(HOME_BACKUP_INIT_CMD) + 1; - cmd = dmalloc(len); - testutil_check( - __wt_snprintf(cmd, len, HOME_BACKUP_INIT_CMD, g.home, g.home, g.home, g.home)); - testutil_checkfmt(system(cmd), "%s", "backup directory creation failed"); - free(cmd); + testutil_create_backup_directory(g.home); } /* @@ -636,12 +597,12 @@ backup(void *arg) testutil_check(backup_cursor->get_key(backup_cursor, &key)); if (g.c_backup_incr_flag == INCREMENTAL_BLOCK) { if (full) - copy_file(session, key); + testutil_copy_file(session, key); else copy_blocks(session, backup_cursor, key); } else - copy_file(session, key); + testutil_copy_file(session, key); active_files_add(active_now, key); } if (ret != WT_NOTFOUND) diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c index 0094cec7c88..20431b3f1ab 100644 --- a/src/third_party/wiredtiger/test/format/config.c +++ b/src/third_party/wiredtiger/test/format/config.c @@ -695,11 +695,11 @@ config_in_memory(void) */ if (config_is_perm("backup")) return; - if (config_is_perm("checkpoint")) - return; if (config_is_perm("btree.compression")) return; - if (config_is_perm("runs.source") && DATASOURCE("lsm")) + if (config_is_perm("checkpoint")) + return; + if (config_is_perm("import")) return; if (config_is_perm("logging")) return; @@ -709,6 +709,8 @@ config_in_memory(void) return; if (config_is_perm("ops.verify")) return; + if (config_is_perm("runs.source") && DATASOURCE("lsm")) + return; if (!config_is_perm("runs.in_memory") && mmrand(NULL, 1, 20) == 1) g.c_in_memory = 1; @@ -724,18 +726,20 @@ config_in_memory_reset(void) uint32_t cache; /* Turn off a lot of stuff. */ - if (!config_is_perm("ops.alter")) - config_single("ops.alter=off", false); if (!config_is_perm("backup")) config_single("backup=off", false); - if (!config_is_perm("checkpoint")) - config_single("checkpoint=off", false); if (!config_is_perm("btree.compression")) config_single("btree.compression=none", false); - if (!config_is_perm("ops.hs_cursor")) - config_single("ops.hs_cursor=off", false); + if (!config_is_perm("checkpoint")) + config_single("checkpoint=off", false); + if (!config_is_perm("import")) + config_single("import=off", false); if (!config_is_perm("logging")) config_single("logging=off", false); + if (!config_is_perm("ops.alter")) + config_single("ops.alter=off", false); + if (!config_is_perm("ops.hs_cursor")) + config_single("ops.hs_cursor=off", false); if (!config_is_perm("ops.salvage")) config_single("ops.salvage=off", false); if (!config_is_perm("ops.verify")) diff --git a/src/third_party/wiredtiger/test/format/config.h b/src/third_party/wiredtiger/test/format/config.h index 44906866aaa..a06509b0dba 100644 --- a/src/third_party/wiredtiger/test/format/config.h +++ b/src/third_party/wiredtiger/test/format/config.h @@ -182,6 +182,13 @@ static CONFIG c[] = { {"format.major_timeout", "long-running operations timeout (minutes)", C_IGNORE, 0, 0, 1000, &g.c_major_timeout, NULL}, + /* + * 0% + * FIXME-WT-7418 and FIXME-WT-7416: Temporarily disable import until WT_ROLLBACK error and + * interaction with backup thread is fixed. Should be 20% + */ + {"import", "import table from newly created database", C_BOOL, 0, 0, 0, &g.c_import, NULL}, + /* 50% */ {"logging", "configure logging", C_BOOL, 50, 0, 0, &g.c_logging, NULL}, diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h index 81eec697d4e..7aefc071396 100644 --- a/src/third_party/wiredtiger/test/format/format.h +++ b/src/third_party/wiredtiger/test/format/format.h @@ -172,6 +172,7 @@ typedef struct { uint32_t c_firstfit; uint32_t c_hs_cursor; uint32_t c_huffman_value; + uint32_t c_import; uint32_t c_in_memory; uint32_t c_independent_thread_rng; uint32_t c_insert_pct; @@ -384,6 +385,7 @@ WT_THREAD_RET backup(void *); WT_THREAD_RET checkpoint(void *); WT_THREAD_RET compact(void *); WT_THREAD_RET hs_cursor(void *); +WT_THREAD_RET import(void *); WT_THREAD_RET random_kv(void *); WT_THREAD_RET timestamp(void *); @@ -395,6 +397,7 @@ void config_final(void); void config_print(bool); void config_run(void); void config_single(const char *, bool); +void create_database(const char *home, WT_CONNECTION **connp); void fclose_and_clear(FILE **); bool fp_readv(FILE *, char *, uint32_t *); void key_gen_common(WT_ITEM *, uint64_t, const char *); diff --git a/src/third_party/wiredtiger/test/format/hs.c b/src/third_party/wiredtiger/test/format/hs.c index e226fe1b3c0..d338a714cf5 100644 --- a/src/third_party/wiredtiger/test/format/hs.c +++ b/src/third_party/wiredtiger/test/format/hs.c @@ -48,7 +48,7 @@ hs_cursor(void *arg) uint32_t hs_btree_id, i; u_int period; int exact; - bool restart; + bool next, restart; (void)(arg); /* Unused parameter */ @@ -69,23 +69,12 @@ hs_cursor(void *arg) hs_counter = 0; /* [-Wconditional-uninitialized] */ hs_btree_id = 0; /* [-Wconditional-uninitialized] */ for (restart = true;;) { - /* - * open_cursor can return EBUSY if concurrent with a metadata operation, retry in that case. - */ - while ((ret = session->open_cursor(session, WT_HS_URI, NULL, NULL, &cursor)) == EBUSY) - __wt_yield(); - testutil_check(ret); - - /* - * The history file has mostly tombstones, ignore them and retrieve the underlying values. - * We don't care about tombstones, but we do want to hit every key rather than skip over - * them. This is a rollback-to-stable flag we're using for our own purposes. - */ - F_SET(cursor, WT_CURSTD_IGNORE_TOMBSTONE); + testutil_check(__wt_curhs_open((WT_SESSION_IMPL *)session, NULL, &cursor)); + F_SET(cursor, WT_CURSTD_HS_READ_COMMITTED); /* Search to the last-known location. */ if (!restart) { - cursor->set_key(cursor, hs_btree_id, &key, hs_start_ts, hs_counter); + cursor->set_key(cursor, 4, hs_btree_id, &key, hs_start_ts, hs_counter); /* * Limit expected errors because this is a diagnostic check (the WiredTiger API allows @@ -99,8 +88,9 @@ hs_cursor(void *arg) * Get some more key/value pairs. Always retrieve at least one key, that ensures we have a * valid key when we copy it to start the next run. */ + next = mmrand(NULL, 0, 1) == 1; for (i = mmrand(NULL, 1, 1000); i > 0; --i) { - if ((ret = cursor->next(cursor)) == 0) { + if ((ret = (next ? cursor->next(cursor) : cursor->prev(cursor))) == 0) { testutil_check( cursor->get_key(cursor, &hs_btree_id, &hs_key, &hs_start_ts, &hs_counter)); testutil_check(cursor->get_value( @@ -116,7 +106,8 @@ hs_cursor(void *arg) * Otherwise, reset so we'll start over. */ if (ret == 0) { - testutil_check(__wt_buf_set(CUR2S(cursor), &key, hs_key.data, hs_key.size)); + testutil_check( + __wt_buf_set((WT_SESSION_IMPL *)session, &key, hs_key.data, hs_key.size)); restart = false; } else restart = true; @@ -130,7 +121,7 @@ hs_cursor(void *arg) break; } - __wt_buf_free(CUR2S(cursor), &key); + __wt_buf_free((WT_SESSION_IMPL *)session, &key); testutil_check(session->close(session, NULL)); #endif diff --git a/src/third_party/wiredtiger/test/format/import.c b/src/third_party/wiredtiger/test/format/import.c new file mode 100644 index 00000000000..0b85515b806 --- /dev/null +++ b/src/third_party/wiredtiger/test/format/import.c @@ -0,0 +1,223 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "format.h" + +static void copy_file_into_directory(WT_SESSION *, const char *); +static void get_file_metadata(WT_SESSION *, const char **, const char **); +static void populate_table(WT_SESSION *); +static void verify_import(WT_SESSION *); + +/* + * Import directory initialize command, remove and create import directory, to place new database + * connection. + */ +#define HOME_IMPORT_INIT_CMD "rm -rf %s/" IMPORT_DIR "&& mkdir %s/" IMPORT_DIR +#define IMPORT_DIR "IMPORT" +/* + * The number of entries in the import table, primary use for validating contents after import. + * There is no benefit to varying the number of entries in the import table. + */ +#define IMPORT_ENTRIES 1000 +#define IMPORT_TABLE_CONFIG "key_format=i,value_format=i" +#define IMPORT_URI "table:import" +#define IMPORT_URI_FILE "file:import.wt" + +/* + * import -- + * Periodically import table. + */ +WT_THREAD_RET +import(void *arg) +{ + WT_CONNECTION *conn, *import_conn; + WT_DECL_RET; + WT_SESSION *import_session, *session; + size_t cmd_len; + uint32_t import_value; + u_int period; + char buf[2048], *cmd; + const char *file_config, *table_config; + + WT_UNUSED(arg); + conn = g.wts_conn; + file_config = table_config = NULL; + import_value = 0; + + /* + * Create a new database, primarily used for testing import. + */ + cmd_len = strlen(g.home) * 2 + strlen(HOME_IMPORT_INIT_CMD) + 1; + cmd = dmalloc(cmd_len); + testutil_check(__wt_snprintf(cmd, cmd_len, HOME_IMPORT_INIT_CMD, g.home, g.home)); + testutil_checkfmt(system(cmd), "%s", "import directory creation failed"); + free(cmd); + + cmd_len = strlen(g.home) + strlen(IMPORT_DIR) + 10; + cmd = dmalloc(cmd_len); + testutil_check(__wt_snprintf(cmd, cmd_len, "%s/%s", g.home, IMPORT_DIR)); + /* Open a connection to the database, creating it if necessary. */ + create_database(cmd, &import_conn); + free(cmd); + + /* + * Open two sessions, one for test/format database and one for the import database. + */ + testutil_check(import_conn->open_session(import_conn, NULL, NULL, &import_session)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + + /* Create new table and populate with data in import database. */ + testutil_checkfmt( + import_session->create(import_session, IMPORT_URI, IMPORT_TABLE_CONFIG), "%s", IMPORT_URI); + populate_table(import_session); + + /* Grab metadata information for table from import database connection. */ + get_file_metadata(import_session, &file_config, &table_config); + + while (!g.workers_finished) { + /* Copy table into test/format database directory. */ + copy_file_into_directory(import_session, "import.wt"); + + /* Perform import with either repair or file metadata. */ + memset(buf, 0, sizeof(buf)); + import_value = mmrand(NULL, 0, 1); + if (import_value == 0) { + testutil_check(__wt_snprintf(buf, sizeof(buf), "import=(enabled,repair=true)")); + if ((ret = session->create(session, IMPORT_URI, buf)) != 0) + testutil_die(ret, "session.import", ret); + } else { + testutil_check(__wt_snprintf(buf, sizeof(buf), + "%s,import=(enabled,repair=false,file_metadata=(%s))", table_config, file_config)); + if ((ret = session->create(session, IMPORT_URI, buf)) != 0) + testutil_die(ret, "session.import", ret); + } + + verify_import(session); + + /* Perform checkpoint, to make sure we perform drop */ + session->checkpoint(session, NULL); + + /* Drop import table, so we can import the table again */ + while ((ret = session->drop(session, IMPORT_URI, NULL)) == EBUSY) { + __wt_yield(); + } + testutil_check(ret); + + period = mmrand(NULL, 1, 10); + while (period > 0 && !g.workers_finished) { + --period; + __wt_sleep(1, 0); + } + } + wts_close(&import_conn, &import_session); + testutil_check(session->close(session, NULL)); + return (WT_THREAD_RET_VALUE); +} + +/* + * verify_import -- + * Verify all the values inside the imported table. + */ +static void +verify_import(WT_SESSION *session) +{ + WT_CURSOR *cursor; + WT_DECL_RET; + int iteration, key, value; + + iteration = 0; + testutil_check(session->open_cursor(session, IMPORT_URI, NULL, NULL, &cursor)); + + while ((ret = cursor->next(cursor)) == 0) { + error_check(cursor->get_key(cursor, &key)); + testutil_assert(key == iteration); + error_check(cursor->get_value(cursor, &value)); + testutil_assert(value == iteration); + iteration++; + } + testutil_assert(iteration == IMPORT_ENTRIES); + scan_end_check(ret == WT_NOTFOUND); + testutil_check(cursor->close(cursor)); +} + +/* + * populate_table -- + * Populate the import table with simple data. + */ +static void +populate_table(WT_SESSION *session) +{ + WT_CURSOR *cursor; + int i; + + testutil_check(session->open_cursor(session, IMPORT_URI, NULL, NULL, &cursor)); + + for (i = 0; i < IMPORT_ENTRIES; ++i) { + cursor->set_key(cursor, i); + cursor->set_value(cursor, i); + testutil_check(cursor->insert(cursor)); + } + testutil_check(cursor->close(cursor)); + testutil_check(session->checkpoint(session, NULL)); +} + +/* + * get_file_metadata -- + * Get import file and table metadata information from import database connection. + */ +static void +get_file_metadata(WT_SESSION *session, const char **file_config, const char **table_config) +{ + WT_CURSOR *metadata_cursor; + + testutil_check(session->open_cursor(session, "metadata:", NULL, NULL, &metadata_cursor)); + metadata_cursor->set_key(metadata_cursor, IMPORT_URI); + testutil_check(metadata_cursor->search(metadata_cursor)); + metadata_cursor->get_value(metadata_cursor, table_config); + + metadata_cursor->set_key(metadata_cursor, IMPORT_URI_FILE); + testutil_check(metadata_cursor->search(metadata_cursor)); + metadata_cursor->get_value(metadata_cursor, file_config); + + testutil_check(metadata_cursor->close(metadata_cursor)); +} + +/* + * copy_file_into_directory -- + * Copy a single file into the test/format directory. + */ +static void +copy_file_into_directory(WT_SESSION *session, const char *name) +{ + size_t buf_len; + char to[64]; + + buf_len = strlen(name) + 10; + testutil_check(__wt_snprintf(to, buf_len, "../%s", name)); + testutil_check(__wt_copy_and_sync(session, name, to)); +} diff --git a/src/third_party/wiredtiger/test/format/ops.c b/src/third_party/wiredtiger/test/format/ops.c index 7e8a12c5434..0e5f8a30422 100644 --- a/src/third_party/wiredtiger/test/format/ops.c +++ b/src/third_party/wiredtiger/test/format/ops.c @@ -235,7 +235,7 @@ operations(u_int ops_seconds, bool lastrun) TINFO *tinfo, total; WT_CONNECTION *conn; WT_SESSION *session; - wt_thread_t alter_tid, backup_tid, checkpoint_tid, compact_tid, hs_tid, random_tid; + wt_thread_t alter_tid, backup_tid, checkpoint_tid, compact_tid, hs_tid, import_tid, random_tid; wt_thread_t timestamp_tid; int64_t fourths, quit_fourths, thread_ops; uint32_t i; @@ -249,6 +249,7 @@ operations(u_int ops_seconds, bool lastrun) memset(&checkpoint_tid, 0, sizeof(checkpoint_tid)); memset(&compact_tid, 0, sizeof(compact_tid)); memset(&hs_tid, 0, sizeof(hs_tid)); + memset(&import_tid, 0, sizeof(import_tid)); memset(&random_tid, 0, sizeof(random_tid)); memset(×tamp_tid, 0, sizeof(timestamp_tid)); @@ -302,6 +303,8 @@ operations(u_int ops_seconds, bool lastrun) testutil_check(__wt_thread_create(NULL, &compact_tid, compact, NULL)); if (g.c_hs_cursor) testutil_check(__wt_thread_create(NULL, &hs_tid, hs_cursor, NULL)); + if (g.c_import) + testutil_check(__wt_thread_create(NULL, &import_tid, import, NULL)); if (g.c_random_cursor) testutil_check(__wt_thread_create(NULL, &random_tid, random_kv, NULL)); if (g.c_txn_timestamps) @@ -386,6 +389,8 @@ operations(u_int ops_seconds, bool lastrun) testutil_check(__wt_thread_join(NULL, &compact_tid)); if (g.c_hs_cursor) testutil_check(__wt_thread_join(NULL, &hs_tid)); + if (g.c_import) + testutil_check(__wt_thread_join(NULL, &import_tid)); if (g.c_random_cursor) testutil_check(__wt_thread_join(NULL, &random_tid)); if (g.c_txn_timestamps) diff --git a/src/third_party/wiredtiger/test/format/wts.c b/src/third_party/wiredtiger/test/format/wts.c index cc489c52623..3b37b3a43d1 100644 --- a/src/third_party/wiredtiger/test/format/wts.c +++ b/src/third_party/wiredtiger/test/format/wts.c @@ -148,7 +148,7 @@ static WT_EVENT_HANDLER event_handler = { * create_database -- * Create a WiredTiger database. */ -static void +void create_database(const char *home, WT_CONNECTION **connp) { WT_CONNECTION *conn; diff --git a/src/third_party/wiredtiger/test/suite/hook_demo.py b/src/third_party/wiredtiger/test/suite/hook_demo.py new file mode 100755 index 00000000000..113c427c8b7 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/hook_demo.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# [TEST_TAGS] +# ignored_file +# [END_TAGS] + +# hook_demo.py +# Demonstration of hooks. Run via: +# python run.py --hook demo=N base01 +# +# These hooks are set up: +# - alter wiredtiger_open arguments (in a benign way) +# - report after wiredtiger_open is called. +# - notify on session.open_cursor +# - intercept the session.create call +# +# With N == 0, the session.create call reports its arguments and calls original session.create. +# with N == 1, it does an additional session.drop call (which should cause tests to fail); +# with N == 2, it does an additional session.create after the drop call (which should work). +# +# Note that notify hooks don't have to simply report, they can call other methods, +# set attributes on objects, etc. For example, one can save the open_cursor +# config string as an attribute on the cursor object, and examine it in another +# hooked method. +from __future__ import print_function + +import os, sys, wthooks +from wttest import WiredTigerTestCase + +# Print to /dev/tty for debugging, since anything extraneous to stdout/stderr will +# cause a test error. +def tty(s): + WiredTigerTestCase.tty(s) + +# These are the hook functions that are run when particular APIs are called. + +# Called to manipulate args for wiredtiger_open +def wiredtiger_open_args(ignored_self, args): + tty('>>> wiredtiger_open, adding cache_size') + args = list(args) # convert from a readonly tuple to a writeable list + args[-1] += ',,,cache_size=500M,,,' # modify the last arg + return args + +# Called to notify after successful wiredtiger_open +def wiredtiger_open_notify(ignored_self, ret, *args): + tty('>>> wiredtiger_open({}) returned {}'.format(args, ret)) + +# Called to notify after successful Session.open_cursor +def session_open_cursor_notify(self, ret, *args): + tty('>>> session.open_cursor({}) returned {}, session is {}'.format(args, ret, self)) + +# Called to replace Session.create +# We do different things (described above) as indicated by our command line argument. +def session_create_replace(arg, orig_session_create, session_self, uri, config): + tty('>>> session.create({},{}), session is {}'.format(uri, config, session_self)) + if arg == 0: + # Just do a regular create + return orig_session_create(session_self, uri, config) + elif arg == 1: + # Do a regular create, followed by a drop. This will cause test failures. + ret = orig_session_create(session_self, uri, config) + # We didn't replace drop, so we can call it as a method + tty('>>> session.drop({})'.format(uri)) + session_self.drop(uri) + return ret + elif arg == 2: + # Do a regular create, followed by a drop, then another create. Should work. + ret = orig_session_create(session_self, uri, config) + # We didn't replace drop, so we can call it as a method + tty('>>> session.drop({})'.format(uri)) + session_self.drop(uri) + tty('>>> session.create({},{})'.format(uri, config)) + orig_session_create(session_self, uri, config) + return ret + +# Every hook file must have one or more classes descended from WiredTigerHook +# This is where the hook functions are 'hooked' to API methods. +class DemoHookCreator(wthooks.WiredTigerHookCreator): + def __init__(self, arg=0): + # An argument may alter the test + if arg == None: + self.arg = 0 + else: + self.arg = int(arg) + + # We have an opportunity to filter the list of tests to be run. + # For this demo, we don't filter. + def filter_tests(self, tests): + print('Filtering: ' + str(tests)) + return tests + + def setup_hooks(self): + tty('>> SETUP HOOKS RUN') + orig_session_create = self.Session['create'] # gets original function + self.wiredtiger['wiredtiger_open'] = (wthooks.HOOK_ARGS, wiredtiger_open_args) + self.wiredtiger['wiredtiger_open'] = (wthooks.HOOK_NOTIFY, wiredtiger_open_notify) + self.Session['create'] = (wthooks.HOOK_REPLACE, lambda s, uri, config: + session_create_replace(self.arg, orig_session_create, s, uri, config)) + self.Session['open_cursor'] = (wthooks.HOOK_NOTIFY, session_open_cursor_notify) + +# Every hook file must have a top level initialize function, +# returning a list of WiredTigerHook objects. +def initialize(arg): + return [DemoHookCreator(arg)] diff --git a/src/third_party/wiredtiger/test/suite/hook_tiered.py b/src/third_party/wiredtiger/test/suite/hook_tiered.py new file mode 100755 index 00000000000..5bb97ea399b --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/hook_tiered.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# [TEST_TAGS] +# ignored_file +# [END_TAGS] + +# hook_tiered.py +# +# Substitute tiered tables for regular (row-store) tables in Python tests. +# +# These hooks can be used to run the existing cursor tests on tiered tables. +# They identify tests that create row-store tables and create tiered tables +# instead. The hook takes an optional argument to specify how many tiers +# to create. The default is 2. +# +# To run with 3 tiers per table: +# ../test/suite/run.py --hooks tiered=3 cursor +# +# The hooks work with may other tests in the python suite but also encounter +# a variety of failures that I haven't tried to sort out. +from __future__ import print_function + +import os, sys, wthooks +import unittest +from wttest import WiredTigerTestCase + +# These are the hook functions that are run when particular APIs are called. + +# Called to replace Session.create +def session_create_replace(ntiers, orig_session_create, session_self, uri, config): + if config == None: + base_config = "" + else: + base_config = config + + # If the test is creating a table (not colstore or lsm), create a tiered table instead, + # using arg to determine number of tiers. Otherwise just do the create as normal. + # + # NOTE: The following code uses the old API for creating tiered tables. As of WT-7173 + # this no longer works. It will be updated and fixed in WT-7440. + if (uri.startswith("table:") and "key_format=r" not in base_config and + "type=lsm" not in base_config): + tier_string = "" + for i in range(ntiers): + new_uri = uri.replace('table:', 'file:tier' + str(i) + '_') + orig_session_create(session_self, new_uri, config) + tier_string = tier_string + '"' + new_uri + '", ' + tier_config = 'type=tiered,tiered=(tiers=(' + tier_string[0:-2] + ')),' + base_config + WiredTigerTestCase.verbose(None, 3, + 'Creating tiered table {} with config = \'{}\''.format(uri, tier_config)) + ret = orig_session_create(session_self, uri, tier_config) + else: + ret = orig_session_create(session_self, uri, config) + return ret + +# Called to replace Session.drop +def session_drop_replace(ntiers, orig_session_drop, session_self, uri, config): + # Drop isn't implemented for tiered tables. Only do the delete if this could be a + # uri we created a tiered table for. Note this isn't a precise match for when we + # did/didn't create a tiered table, but we don't have the create config around to check. + ret = 0 + if not uri.startswith("table:"): + ret = orig_session_drop(session_self, uri, config) + return ret + +# Called to replace Session.verify +def session_verify_replace(ntiers, orig_session_verify, session_self, uri): + return 0 + +# Every hook file must have one or more classes descended from WiredTigerHook +# This is where the hook functions are 'hooked' to API methods. +class TieredHookCreator(wthooks.WiredTigerHookCreator): + def __init__(self, ntiers=0): + # Argument specifies the number of tiers to test. The default is 2. + if ntiers == None: + self.ntiers = 2 + else: + self.ntiers = int(ntiers) + + # Is this test one we should skip? We skip tests of features supported on standard + # tables but not tiered tables, specififically cursor caching and checkpoint cursors. + def skip_test(self, test): + skip = ["bulk_backup", + "checkpoint", + "test_cursor13_big", + "test_cursor13_drops", + "test_cursor13_dup", + "test_cursor13_reopens"] + for item in skip: + if item in str(test): + return True + return False + + # Remove tests that won't work on tiered cursors + def filter_tests(self, tests): + new_tests = unittest.TestSuite() + new_tests.addTests([t for t in tests if not self.skip_test(t)]) + return new_tests + + def setup_hooks(self): + orig_session_create = self.Session['create'] + self.Session['create'] = (wthooks.HOOK_REPLACE, lambda s, uri, config: + session_create_replace(self.ntiers, orig_session_create, s, uri, config)) + + orig_session_drop = self.Session['drop'] + self.Session['drop'] = (wthooks.HOOK_REPLACE, lambda s, uri, config: + session_drop_replace(self.ntiers, orig_session_drop, s, uri, config)) + + orig_session_verify = self.Session['verify'] + self.Session['verify'] = (wthooks.HOOK_REPLACE, lambda s, uri: + session_verify_replace(self.ntiers, orig_session_verify, s, uri)) + +# Every hook file must have a top level initialize function, +# returning a list of WiredTigerHook objects. +def initialize(arg): + return [TieredHookCreator(arg)] diff --git a/src/third_party/wiredtiger/test/suite/run.py b/src/third_party/wiredtiger/test/suite/run.py index a5ae88fa966..8d74b84259d 100755 --- a/src/third_party/wiredtiger/test/suite/run.py +++ b/src/third_party/wiredtiger/test/suite/run.py @@ -119,8 +119,10 @@ Options:\n\ be run without executing any.\n\ -g | --gdb all subprocesses (like calls to wt) use gdb\n\ -h | --help show this message\n\ + | --hook name[=arg] set up hooks from hook_<name>.py, with optional arg\n\ -j N | --parallel N run all tests in parallel using N processes\n\ -l | --long run the entire test suite\n\ + | --noremove do not remove WT_TEST or -D target before run\n\ -p | --preserve preserve output files in WT_TEST/<testname>\n\ -r N | --random-sample N randomly sort scenarios to be run, then\n\ execute every Nth (2<=N<=1000) scenario.\n\ @@ -306,6 +308,7 @@ def error(exitval, prefix, msg): if __name__ == '__main__': # Turn numbers and ranges into test module names preserve = timestamp = debug = dryRun = gdbSub = lldbSub = longtest = ignoreStdout = False + removeAtStart = True asan = False parallel = 0 random_sample = 0 @@ -318,6 +321,7 @@ if __name__ == '__main__': verbose = 1 args = sys.argv[1:] testargs = [] + hook_names = [] while len(args) > 0: arg = args.pop(0) from unittest import defaultTestLoader as loader @@ -367,9 +371,18 @@ if __name__ == '__main__': if option == '-help' or option == 'h': usage() sys.exit(0) + if option == '-hook': + if len(args) == 0: + usage() + sys.exit(2) + hook_names.append(args.pop(0)) + continue if option == '-long' or option == 'l': longtest = True continue + if option == '-noremove': + removeAtStart = False + continue if option == '-random-sample' or option == 'r': if len(args) == 0: usage() @@ -519,11 +532,13 @@ if __name__ == '__main__': tests = unittest.TestSuite() from testscenarios.scenarios import generate_scenarios + import wthooks + hookmgr = wthooks.WiredTigerHookManager(hook_names) # All global variables should be set before any test classes are loaded. # That way, verbose printing can be done at the class definition level. - wttest.WiredTigerTestCase.globalSetup(preserve, timestamp, gdbSub, lldbSub, - verbose, wt_builddir, dirarg, - longtest, ignoreStdout, seedw, seedz) + wttest.WiredTigerTestCase.globalSetup(preserve, removeAtStart, timestamp, gdbSub, lldbSub, + verbose, wt_builddir, dirarg, longtest, + ignoreStdout, seedw, seedz, hookmgr) # Without any tests listed as arguments, do discovery if len(testargs) == 0: @@ -542,6 +557,7 @@ if __name__ == '__main__': for arg in testargs: testsFromArg(tests, loader, arg, scenario) + tests = hookmgr.filter_tests(tests) # Shuffle the tests and create a new suite containing every Nth test from # the original suite if random_sample > 0: diff --git a/src/third_party/wiredtiger/test/suite/test_backup21.py b/src/third_party/wiredtiger/test/suite/test_backup21.py new file mode 100644 index 00000000000..42e2405c22c --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_backup21.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import queue, threading, wiredtiger, wttest +from wtbackup import backup_base +from wtscenario import make_scenarios +from wtthread import op_thread + +# test_backup21.py +# Run create/drop operations while backup is ongoing. +class test_backup21(backup_base): + # Backup directory name. + dir = 'backup.dir' + uri = 'test_backup21' + ops = 50 + key_fmt = "S" + + def test_concurrent_operations_with_backup(self): + done = threading.Event() + table_uri = 'table:' + self.uri + + # Create and populate the table. + self.session.create(table_uri, "key_format=S,value_format=S") + self.add_data(table_uri, 'key', 'value', True) + + work_queue = queue.Queue() + t = op_thread(self.conn, [table_uri], self.key_fmt, work_queue, done) + try: + t.start() + # Place create or drop operation into work queue. + iteration = 0 + op = 't' + for _ in range(0, self.ops): + # Open backup cursor. + bkup_c = self.session.open_cursor('backup:', None, None) + work_queue.put_nowait((op, str(iteration), 'value')) + + all_files = self.take_full_backup(self.dir, bkup_c) + if op == 't': + # Newly created table shouldn't be present in backup. + self.assertTrue(self.uri + str(iteration) + ".wt" not in all_files) + iteration = iteration + 1 + else: + # Dropped table should still be present in backup. + self.assertTrue(self.uri + str(iteration) + ".wt" in all_files) + iteration = iteration + 1 + bkup_c.close() + # Once we reach midway point, start drop operations. + if iteration == self.ops/2: + iteration = 0 + op = 'd' + except: + # Deplete the work queue if there's an error. + while not work_queue.empty(): + work_queue.get() + work_queue.task_done() + raise + finally: + work_queue.join() + done.set() + t.join() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_hs21.py b/src/third_party/wiredtiger/test/suite/test_hs21.py new file mode 100644 index 00000000000..e2c8885661f --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_hs21.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import time, re +import wiredtiger, wttest +from wtdataset import SimpleDataSet +from wiredtiger import stat + +def timestamp_str(t): + return '%x' % t + +# test_hs21.py +# Test we don't lose any data when idle files with an active history are closed/sweeped. +# Files with active history, ie content newer than the oldest timestamp can be closed when idle. +# We want to ensure that when an active history file is idle closed we can continue reading the +# correct version of data and their base write generation hasn't changed (since we haven't +# restarted the system). +class test_hs21(wttest.WiredTigerTestCase): + # Configure handle sweeping to occur within a specific amount of time. + conn_config = 'file_manager=(close_handle_minimum=0,close_idle_time=2,close_scan_interval=1),' + \ + 'statistics=(all),operation_tracking=(enabled=false)' + session_config = 'isolation=snapshot' + file_name = 'test_hs21' + numfiles = 10 + nrows = 10000 + + def large_updates(self, uri, value, ds, nrows, commit_ts): + # Update a large number of records, we'll hang if the history store table isn't working. + session = self.session + cursor = session.open_cursor(uri) + session.begin_transaction() + for i in range(1, nrows + 1): + cursor[ds.key(i)] = value + session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + cursor.close() + + def check(self, session, check_value, uri, nrows, read_ts=-1): + # Validate we read an expected value (optionally at a given read timestamp). + if read_ts != -1: + session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + cursor = session.open_cursor(uri) + count = 0 + for k, v in cursor: + self.assertEqual(v, check_value) + count += 1 + if read_ts != -1: + session.rollback_transaction() + self.assertEqual(count, nrows) + cursor.close() + + def parse_run_write_gen(self, uri): + meta_cursor = self.session.open_cursor('metadata:') + config = meta_cursor[uri] + meta_cursor.close() + # The search string will look like: 'run_write_gen=<num>'. + # Just reverse the string and take the digits from the back until we hit '='. + write_gen = re.search('run_write_gen=\d+', config) + self.assertTrue(write_gen is not None) + write_gen_str = str() + for c in reversed(write_gen.group(0)): + if not c.isdigit(): + self.assertEqual(c, '=') + break + write_gen_str = c + write_gen_str + return int(write_gen_str) + + def test_hs(self): + active_files = [] + value1 = 'a' * 500 + value2 = 'd' * 500 + + # Set up 'numfiles' with 'numrows' entries. We want to create a number of files that + # contain active history (content newer than the oldest timestamp). + for f in range(self.numfiles): + table_uri = 'table:%s.%d' % (self.file_name, f) + file_uri = 'file:%s.%d.wt' % (self.file_name, f) + # Create a small table. + ds = SimpleDataSet( + self, table_uri, 0, key_format='S', value_format='S', config='log=(enabled=false)') + ds.populate() + # Checkpoint to ensure we write the files metadata checkpoint value. + self.session.checkpoint() + # Get the base write gen of the file so we can compare after the handles get closed. + base_write_gen = self.parse_run_write_gen(file_uri) + active_files.append((base_write_gen, ds)) + + # Pin oldest and stable to timestamp 1. + self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + + ',stable_timestamp=' + timestamp_str(1)) + + # Perform a series of updates over our files at timestamp 2. This being data we can later assert + # to ensure the history store is working as intended. + for (_, ds) in active_files: + # Load data at timestamp 2. + self.large_updates(ds.uri, value1, ds, self.nrows // 2 , 2) + + # We want to create a long running read transaction in a seperate session which we will persist over the closing and + # re-opening of handles. We want to ensure the correct data gets read throughout this time period. + session_read = self.conn.open_session() + session_read.begin_transaction('read_timestamp=' + timestamp_str(2)) + # Check our inital set of updates are seen at the read timestamp. + for (_, ds) in active_files: + # Check that all updates at timestamp 2 are seen. + self.check(session_read, value1, ds.uri, self.nrows // 2) + + # Perform a series of updates over over files at a later timestamp. Checking the history store data is consistent + # with old and new timestamps. + for (_, ds) in active_files: + # Load more data with a later timestamp. + self.large_updates(ds.uri, value2, ds, self.nrows, 100) + # Check that the new updates are only seen after the update timestamp. + self.check(self.session, value1, ds.uri, self.nrows // 2, 2) + self.check(self.session, value2, ds.uri, self.nrows, 100) + + # Our sweep scan interval is every 1 second and the amount of idle time needed for a handle to be closed is 2 seconds. + # It should take roughly 3 seconds for the sweep server to close our file handles. Lets wait at least double + # that to be safe. + max = 6 + sleep = 0 + # After waiting for the sweep server to remove our idle handles, the only open + # handles that should be the metadata file, history store file and lock file. + final_numfiles = 3 + # Open the stats cursor to collect the dhandle sweep status. + stat_cursor = self.session.open_cursor('statistics:', None, None) + while sleep < max: + # We continue doing checkpoints which as a side effect runs the session handle sweep. This encouraging the idle + # handles get removed. + # Note, though checkpointing blocks sweeping, the checkpoint should be fast and not add too much extra time to the + # overall test time. + self.session.checkpoint() + sleep += 0.5 + time.sleep(0.5) + stat_cursor.reset() + curr_files_open = stat_cursor[stat.conn.file_open][2] + curr_dhandles_removed = stat_cursor[stat.conn.dh_sweep_remove][2] + curr_dhandle_sweep_closes = stat_cursor[stat.conn.dh_sweep_close][2] + + self.printVerbose(3, "==== loop " + str(sleep)) + self.printVerbose(3, "Number of files open: " + str(curr_files_open)) + self.printVerbose(3, "Number of connection sweep dhandles closed: " + str(curr_dhandle_sweep_closes)) + self.printVerbose(3, "Number of connection sweep dhandles removed from hashlist: " + str(curr_dhandles_removed)) + + # We've sweeped all the handles we can if we are left with the number of final dhandles + # that we expect to be always open. + if curr_files_open == final_numfiles and curr_dhandle_sweep_closes >= self.numfiles: + break + + stat_cursor.reset() + final_dhandle_sweep_closes = stat_cursor[stat.conn.dh_sweep_close][2] + stat_cursor.close() + # We want to assert our active history files have all been closed. + self.assertGreaterEqual(final_dhandle_sweep_closes, self.numfiles) + + # Using our long running read transaction, we want to now check the correct data can still be read after the + # handles have been closed. + for (_, ds) in active_files: + # Check that all updates at timestamp 2 are seen. + self.check(session_read, value1, ds.uri, self.nrows // 2) + session_read.rollback_transaction() + + # Perform a series of checks over our files to ensure that our transactions have been written + # before the dhandles were closed/sweeped. + # Also despite the dhandle is being re-opened, we don't expect the base write generation + # to have changed since we haven't actually restarted the system. + for idx, (initial_base_write_gen, ds) in enumerate(active_files): + # Check that the most recent transaction has the correct data. + self.check(self.session, value2, ds.uri, self.nrows, 100) + file_uri = 'file:%s.%d.wt' % (self.file_name, idx) + # Get the current base_write_gen and ensure it hasn't changed since being + # closed. + base_write_gen = self.parse_run_write_gen(file_uri) + self.assertEqual(initial_base_write_gen, base_write_gen) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_hs22.py b/src/third_party/wiredtiger/test/suite/test_hs22.py new file mode 100644 index 00000000000..cf30767b8bc --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_hs22.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest + +def timestamp_str(t): + return '%x' % t + +# test_hs22.py +# Test the case that out of order timestamp +# update is followed by a tombstone. +class test_hs22(wttest.WiredTigerTestCase): + conn_config = 'cache_size=50MB' + session_config = 'isolation=snapshot' + + def test_onpage_out_of_order_timestamp_update(self): + uri = 'table:test_hs22' + self.session.create(uri, 'key_format=S,value_format=S') + cursor = self.session.open_cursor(uri) + self.conn.set_timestamp( + 'oldest_timestamp=' + timestamp_str(1) + ',stable_timestamp=' + timestamp_str(1)) + + value1 = 'a' + value2 = 'b' + + # Insert a key. + self.session.begin_transaction() + cursor[str(0)] = value1 + self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + + # Remove the key. + self.session.begin_transaction() + cursor.set_key(str(0)) + self.assertEqual(cursor.remove(), 0) + self.session.commit_transaction('commit_timestamp=' + timestamp_str(20)) + + # Do an out of order timestamp + # update and write it to the data + # store later. + self.session.begin_transaction() + cursor[str(0)] = value2 + self.session.commit_transaction('commit_timestamp=' + timestamp_str(15)) + + # Insert another key. + self.session.begin_transaction() + cursor[str(1)] = value1 + self.session.commit_transaction('commit_timestamp=' + timestamp_str(20)) + + # Update the key. + self.session.begin_transaction() + cursor[str(1)] = value2 + self.session.commit_transaction('commit_timestamp=' + timestamp_str(30)) + + # Do a checkpoint to trigger + # history store reconciliation. + self.session.checkpoint() + + evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") + + # Search the key to evict it. + self.session.begin_transaction("read_timestamp=" + timestamp_str(15)) + self.assertEqual(evict_cursor[str(0)], value2) + self.assertEqual(evict_cursor.reset(), 0) + self.session.rollback_transaction() + + # Search the key again to verify the data is still as expected. + self.session.begin_transaction("read_timestamp=" + timestamp_str(15)) + self.assertEqual(cursor[str(0)], value2) + self.session.rollback_transaction() + + def test_out_of_order_timestamp_update_newer_than_tombstone(self): + uri = 'table:test_hs22' + self.session.create(uri, 'key_format=S,value_format=S') + cursor = self.session.open_cursor(uri) + self.conn.set_timestamp( + 'oldest_timestamp=' + timestamp_str(1) + ',stable_timestamp=' + timestamp_str(1)) + + value1 = 'a' + value2 = 'b' + + # Insert a key. + self.session.begin_transaction() + cursor[str(0)] = value1 + self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + + # Remove a key. + self.session.begin_transaction() + cursor.set_key(str(0)) + self.assertEqual(cursor.remove(), 0) + self.session.commit_transaction('commit_timestamp=' + timestamp_str(20)) + + # Do an out of order timestamp + # update and write it to the + # history store later. + self.session.begin_transaction() + cursor[str(0)] = value2 + self.session.commit_transaction('commit_timestamp=' + timestamp_str(15)) + + # Add another update. + self.session.begin_transaction() + cursor[str(0)] = value1 + self.session.commit_transaction('commit_timestamp=' + timestamp_str(20)) + + # Insert another key. + self.session.begin_transaction() + cursor[str(1)] = value1 + self.session.commit_transaction('commit_timestamp=' + timestamp_str(20)) + + # Update the key. + self.session.begin_transaction() + cursor[str(1)] = value2 + self.session.commit_transaction('commit_timestamp=' + timestamp_str(30)) + + # Do a checkpoint to trigger + # history store reconciliation. + self.session.checkpoint() + + evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") + + # Search the key to evict it. + self.session.begin_transaction("read_timestamp=" + timestamp_str(15)) + self.assertEqual(evict_cursor[str(0)], value2) + self.assertEqual(evict_cursor.reset(), 0) + self.session.rollback_transaction() + + # Search the key again to verify the data is still as expected. + self.session.begin_transaction("read_timestamp=" + timestamp_str(15)) + self.assertEqual(cursor[str(0)], value2) + self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_import10.py b/src/third_party/wiredtiger/test/suite/test_import10.py index 0d56f799291..7715a2754ad 100644 --- a/src/third_party/wiredtiger/test/suite/test_import10.py +++ b/src/third_party/wiredtiger/test/suite/test_import10.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2021 MongoDB, Inc. +# Public Domain 2014-present MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. @@ -87,7 +87,7 @@ class test_import10(backup_base): cursor.close() all_files = self.take_full_backup(self.dir, bkup_c) - self.assertTrue(self.uri + "wt" not in all_files) + self.assertTrue(self.uri + ".wt" not in all_files) bkup_c.close() if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_prepare14.py b/src/third_party/wiredtiger/test/suite/test_prepare14.py new file mode 100644 index 00000000000..fb32aefc713 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_prepare14.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wttest +from wiredtiger import WT_NOTFOUND +from wtscenario import make_scenarios + +def timestamp_str(t): + return '%x' % t + +# test_prepare14.py +# Test that the transaction visibility of an on-disk update +# that has both the start and the stop time points from the +# same uncommitted prepared transaction. +class test_prepare14(wttest.WiredTigerTestCase): + session_config = 'isolation=snapshot' + + in_memory_values = [ + ('no_inmem', dict(in_memory=False)), + ('inmem', dict(in_memory=True)) + ] + + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + scenarios = make_scenarios(in_memory_values, key_format_values) + + def conn_config(self): + config = 'cache_size=50MB' + if self.in_memory: + config += ',in_memory=true' + else: + config += ',in_memory=false' + return config + + def test_prepare14(self): + # Prepare transactions for column store table is not yet supported. + if self.key_format == 'r': + self.skipTest('Prepare transactions for column store table is not yet supported') + + # Create a table without logging. + uri = "table:prepare14" + create_config = 'allocation_size=512,key_format=S,value_format=S' + self.session.create(uri, create_config) + + # Pin oldest and stable timestamps to 10. + self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + + ',stable_timestamp=' + timestamp_str(10)) + + value = 'a' + + # Perform several updates and removes. + s = self.conn.open_session() + cursor = s.open_cursor(uri) + s.begin_transaction() + cursor[str(0)] = value + cursor.set_key(str(0)) + cursor.remove() + cursor.close() + s.prepare_transaction('prepare_timestamp=' + timestamp_str(20)) + + # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. + evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") + + # Search for the key so we position our cursor on the page that we want to evict. + self.session.begin_transaction("ignore_prepare = true") + evict_cursor.set_key(str(0)) + self.assertEquals(evict_cursor.search(), WT_NOTFOUND) + evict_cursor.reset() + evict_cursor.close() + self.session.commit_transaction() + + self.session.begin_transaction("ignore_prepare = true") + cursor2 = self.session.open_cursor(uri) + cursor2.set_key(str(0)) + self.assertEquals(cursor2.search(), WT_NOTFOUND) + self.session.commit_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py index b3bf62f03ef..da5b6d1ca91 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py @@ -110,6 +110,7 @@ class test_rollback_to_stable_base(wttest.WiredTigerTestCase): count += 1 session.commit_transaction() self.assertEqual(count, nrows) + cursor.close() # Test that rollback to stable clears the remove operation. class test_rollback_to_stable01(test_rollback_to_stable_base): diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py index 50a31085c38..8aefee48b16 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py @@ -207,6 +207,10 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): def test_rollback_to_stable_prepare(self): nrows = 1000 + # Prepare transactions for column store table is not yet supported. + if self.prepare and self.key_format == 'r': + self.skipTest('Prepare transactions for column store table is not yet supported') + # Create a table without logging. self.pr("create/populate tables") uri_1 = "table:rollback_to_stable10_1" diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable14.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable14.py index 20d0f9f7744..66614938d3c 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable14.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable14.py @@ -179,7 +179,10 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): self.assertEqual(keys_removed, 0) self.assertEqual(hs_restore_updates, nrows) self.assertEqual(keys_restored, 0) - self.assertEqual(upd_aborted, 0) + if self.prepare: + self.assertGreaterEqual(upd_aborted, 0) + else: + self.assertEqual(upd_aborted, 0) self.assertGreater(pages_visited, 0) self.assertGreaterEqual(hs_removed, nrows) self.assertGreaterEqual(hs_sweep, 0) @@ -196,6 +199,10 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): def test_rollback_to_stable_same_ts(self): nrows = 1500 + # Prepare transactions for column store table is not yet supported. + if self.prepare and self.key_format == 'r': + self.skipTest('Prepare transactions for column store table is not yet supported') + # Create a table without logging. self.pr("create/populate table") uri = "table:rollback_to_stable14" @@ -277,7 +284,10 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): self.assertEqual(keys_removed, 0) self.assertEqual(hs_restore_updates, nrows) self.assertEqual(keys_restored, 0) - self.assertEqual(upd_aborted, 0) + if self.prepare: + self.assertGreaterEqual(upd_aborted, 0) + else: + self.assertEqual(upd_aborted, 0) self.assertGreater(pages_visited, 0) self.assertGreaterEqual(hs_removed, nrows * 3) self.assertGreaterEqual(hs_sweep, 0) @@ -292,6 +302,10 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): def test_rollback_to_stable_same_ts_append(self): nrows = 1500 + # Prepare transactions for column store table is not yet supported. + if self.prepare and self.key_format == 'r': + self.skipTest('Prepare transactions for column store table is not yet supported') + # Create a table without logging. self.pr("create/populate table") uri = "table:rollback_to_stable14" @@ -373,7 +387,10 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): self.assertEqual(keys_removed, 0) self.assertEqual(hs_restore_updates, nrows) self.assertEqual(keys_restored, 0) - self.assertEqual(upd_aborted, 0) + if self.prepare: + self.assertGreaterEqual(upd_aborted, 0) + else: + self.assertEqual(upd_aborted, 0) self.assertGreater(pages_visited, 0) self.assertGreaterEqual(hs_removed, nrows * 3) self.assertGreaterEqual(hs_sweep, 0) diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable16.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable16.py index 2539a4b88d0..0c0a3235e94 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable16.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable16.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python # # Public Domain 2014-present MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable18.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable18.py new file mode 100644 index 00000000000..68c2e8d0205 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable18.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import fnmatch, os, shutil, time +from helper import simulate_crash_restart +from test_rollback_to_stable01 import test_rollback_to_stable_base +from wiredtiger import stat +from wtdataset import SimpleDataSet +from wtscenario import make_scenarios + +def timestamp_str(t): + return '%x' % t + +# test_rollback_to_stable18.py +# Test the rollback to stable shouldn't skip any pages that don't have aggregated time window. +class test_rollback_to_stable18(test_rollback_to_stable_base): + session_config = 'isolation=snapshot' + + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + prepare_values = [ + ('no_prepare', dict(prepare=False)), + ('prepare', dict(prepare=True)) + ] + + scenarios = make_scenarios(key_format_values, prepare_values) + + def conn_config(self): + config = 'cache_size=50MB,in_memory=true,statistics=(all),log=(enabled=false),eviction_dirty_trigger=5,eviction_updates_trigger=5' + return config + + def test_rollback_to_stable(self): + nrows = 10000 + + # Prepare transactions for column store table is not yet supported. + if self.prepare and self.key_format == 'r': + self.skipTest('Prepare transactions for column store table is not yet supported') + + # Create a table without logging. + uri = "table:rollback_to_stable18" + ds = SimpleDataSet( + self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') + ds.populate() + + # Pin oldest and stable to timestamp 10. + self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + + ',stable_timestamp=' + timestamp_str(10)) + + value_a = "aaaaa" * 100 + + # Perform several updates. + self.large_updates(uri, value_a, ds, nrows, self.prepare, 20) + + # Perform several removes. + self.large_removes(uri, ds, nrows, self.prepare, 30) + + # Verify data is visible and correct. + self.check(value_a, uri, nrows, 20) + self.check(None, uri, 0, 30) + + # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. + evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") + + # Search for the key so we position our cursor on the page that we want to evict. + evict_cursor.set_key(1) + evict_cursor.search() + evict_cursor.reset() + evict_cursor.close() + + # Pin stable and oldest to timestamp 30 if prepare otherwise 20. + if self.prepare: + self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(30) + + ',stable_timestamp=' + timestamp_str(30)) + else: + self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(20) + + ',stable_timestamp=' + timestamp_str(20)) + + # Perform rollback to stable. + self.conn.rollback_to_stable() + + # Verify data is not visible. + self.check(value_a, uri, nrows, 30) + + stat_cursor = self.session.open_cursor('statistics:', None, None) + calls = stat_cursor[stat.conn.txn_rts][2] + upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2] + self.assertEqual(calls, 1) + self.assertEqual(upd_aborted, nrows) diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable19.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable19.py index 3a24113fa32..284499dae64 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable19.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable19.py @@ -29,7 +29,7 @@ import fnmatch, os, shutil, time from helper import simulate_crash_restart from test_rollback_to_stable01 import test_rollback_to_stable_base -from wiredtiger import stat +from wiredtiger import stat, WT_NOTFOUND from wtdataset import SimpleDataSet from wtscenario import make_scenarios @@ -102,11 +102,18 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): # Search for the key so we position our cursor on the page that we want to evict. self.session.begin_transaction("ignore_prepare = true") evict_cursor.set_key(1) - evict_cursor.search() + self.assertEquals(evict_cursor.search(), WT_NOTFOUND) evict_cursor.reset() evict_cursor.close() self.session.commit_transaction() + # Search to make sure the data is not visible + self.session.begin_transaction("ignore_prepare = true") + cursor2 = self.session.open_cursor(uri) + cursor2.set_key(1) + self.assertEquals(cursor2.search(), WT_NOTFOUND) + self.session.commit_transaction() + # Pin stable timestamp to 20. self.conn.set_timestamp('stable_timestamp=' + timestamp_str(20)) if not self.in_memory: @@ -175,11 +182,18 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): # Search for the key so we position our cursor on the page that we want to evict. self.session.begin_transaction("ignore_prepare = true") evict_cursor.set_key(1) - evict_cursor.search() + self.assertEquals(evict_cursor.search(), WT_NOTFOUND) evict_cursor.reset() evict_cursor.close() self.session.commit_transaction() + # Search to make sure the data is not visible + self.session.begin_transaction("ignore_prepare = true") + cursor2 = self.session.open_cursor(uri) + cursor2.set_key(1) + self.assertEquals(cursor2.search(), WT_NOTFOUND) + self.session.commit_transaction() + # Pin stable timestamp to 40. self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) if not self.in_memory: diff --git a/src/third_party/wiredtiger/test/suite/test_search_near01.py b/src/third_party/wiredtiger/test/suite/test_search_near01.py new file mode 100644 index 00000000000..2e54671c06c --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_search_near01.py @@ -0,0 +1,330 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +import time, wiredtiger, wttest, unittest +from wiredtiger import stat + +def timestamp_str(t): + return '%x' % t + +# test_search_near01.py +# Test various prefix search near scenarios. +class test_search_near01(wttest.WiredTigerTestCase): + conn_config = 'statistics=(all)' + session_config = 'isolation=snapshot' + + def get_stat(self, stat, local_session = None): + if (local_session != None): + stat_cursor = local_session.open_cursor('statistics:') + else: + stat_cursor = self.session.open_cursor('statistics:') + val = stat_cursor[stat][2] + stat_cursor.close() + return val + + def unique_insert(self, cursor, prefix, id, keys): + key = prefix + ',' + str(id) + keys.append(key) + cursor.set_key(prefix) + cursor.set_value(prefix) + self.assertEqual(cursor.insert(), 0) + cursor.set_key(prefix) + self.assertEqual(cursor.remove(), 0) + cursor.set_key(prefix) + cursor.search_near() + cursor.set_key(key) + cursor.set_value(key) + self.assertEqual(cursor.insert(), 0) + + def test_base_scenario(self): + uri = 'table:test_base_scenario' + self.session.create(uri, 'key_format=u,value_format=u') + cursor = self.session.open_cursor(uri) + session2 = self.conn.open_session() + cursor3 = self.session.open_cursor(uri, None, "debug=(release_evict=true)") + + # Basic character array. + l = "abcdefghijklmnopqrstuvwxyz" + + # Start our older reader. + session2.begin_transaction() + + key_count = 26*26*26 + # Insert keys aaa -> zzz. + self.session.begin_transaction() + for i in range (0, 26): + for j in range (0, 26): + for k in range (0, 26): + cursor[l[i] + l[j] + l[k]] = l[i] + l[j] + l[k] + self.session.commit_transaction() + + # Evict the whole range. + for i in range (0, 26): + for j in range(0, 26): + cursor3.set_key(l[i] + l[j] + 'a') + cursor3.search() + cursor3.reset() + + # Search near for the "aa" part of the range. + cursor2 = session2.open_cursor(uri) + cursor2.set_key('aa') + cursor2.search_near() + + skip_count = self.get_stat(stat.conn.cursor_next_skip_lt_100) + # This should be equal to roughly key_count * 2 as we're going to traverse the whole + # range forward, and then the whole range backwards. + self.assertGreater(skip_count, key_count * 2) + + cursor2.reconfigure("prefix_key=true") + cursor2.set_key('aa') + cursor2.search_near() + + prefix_skip_count = self.get_stat(stat.conn.cursor_next_skip_lt_100) + # We should've skipped ~26*2 here as we're only looking at the "aa" range * 2. + self.assertGreaterEqual(prefix_skip_count - skip_count, 26*2) + skip_count = prefix_skip_count + + # The prefix code will have come into play at once as we walked to "aba". The prev + # traversal will go off the end of the file and as such we don't expect it to increment + # this statistic again. + self.assertEqual(self.get_stat(stat.conn.cursor_search_near_prefix_fast_paths), 1) + + # Search for a key not at the start. + cursor2.set_key('bb') + cursor2.search_near() + + # Assert it to have only incremented the skipped statistic ~26*2 times. + prefix_skip_count = self.get_stat(stat.conn.cursor_next_skip_lt_100) + self.assertGreaterEqual(prefix_skip_count - skip_count, 26*2) + skip_count = prefix_skip_count + + # Here we should've hit the prefix fast path code twice. Plus the time we already did. + self.assertEqual(self.get_stat(stat.conn.cursor_search_near_prefix_fast_paths), 2+1) + + cursor2.close() + cursor2 = session2.open_cursor(uri) + cursor2.set_key('bb') + cursor2.search_near() + # Assert that we've incremented the stat key_count times, as we closed the cursor and + # reopened it. + # + # This validates cursor caching logic, as if we don't clear the flag correctly this will + # fail. + # + # It should be closer to key_count * 2 but this an approximation. + prefix_skip_count = self.get_stat(stat.conn.cursor_next_skip_lt_100) + self.assertGreaterEqual(prefix_skip_count - skip_count, key_count) + + # This test aims to simulate a unique index insertion. + def test_unique_index_case(self): + uri = 'table:test_unique_index_case' + self.session.create(uri, 'key_format=u,value_format=u') + cursor = self.session.open_cursor(uri) + session2 = self.conn.open_session() + cursor3 = self.session.open_cursor(uri, None, "debug=(release_evict=true)") + l = "abcdefghijklmnopqrstuvwxyz" + + # A unique index has the following insertion method: + # 1. Insert the prefix + # 2. Remove the prefix + # 3. Search near for the prefix + # 4. Insert the full value + # All of these operations are wrapped in the same txn, this test attempts to test scenarios + # that could arise from this insertion method. + + # A unique index key has the format (prefix, _id), we'll insert keys that look similar. + + # Start our old reader txn. + session2.begin_transaction() + + key_count = 26*26 + id = 0 + cc_id = 0 + keys = [] + + # Insert keys aa,1 -> zz,N + for i in range (0, 26): + for j in range (0, 26): + # Skip inserting 'c'. + if (i == 2 and j == 2): + cc_id = id + id = id + 1 + continue + self.session.begin_transaction() + prefix = l[i] + l[j] + self.unique_insert(cursor, prefix, id, keys) + id = id + 1 + self.session.commit_transaction() + + # Evict the whole range. + for i in keys: + cursor3.set_key(i) + cursor3.search() + cursor3.reset() + + # Using our older reader attempt to find a value. + # Search near for the "cc" prefix. + cursor2 = session2.open_cursor(uri) + cursor2.set_key('cc') + cursor2.search_near() + + skip_count = self.get_stat(stat.conn.cursor_next_skip_lt_100) + # This should be equal to roughly key_count * 2 as we're going to traverse most of the + # range forward, and then the whole range backwards. + self.assertGreater(skip_count, key_count * 2) + + cursor2.reconfigure("prefix_key=true") + cursor2.set_key('cc') + cursor2.search_near() + self.assertEqual(self.get_stat(stat.conn.cursor_search_near_prefix_fast_paths), 2) + + # This still isn't visible to our older reader and as such we expect this statistic to + # increment twice. + self.unique_insert(cursor2, 'cc', cc_id, keys) + self.assertEqual(self.get_stat(stat.conn.cursor_search_near_prefix_fast_paths), 4) + + # In order for prefix key fast pathing to work we rely on some guarantees provided by row + # search. Test some of the guarantees. + def test_row_search(self): + uri = 'table:test_row_search' + self.session.create(uri, 'key_format=u,value_format=u') + cursor = self.session.open_cursor(uri) + session2 = self.conn.open_session() + l = "abcdefghijklmnopqrstuvwxyz" + # Insert keys a -> z, except c + self.session.begin_transaction() + for i in range (0, 26): + if (i == 2): + continue + cursor[l[i]] = l[i] + self.session.commit_transaction() + # Start our older reader transaction. + session2.begin_transaction() + # Insert a few keys in the 'c' range + self.session.begin_transaction() + cursor['c'] = 'c' + cursor['cc'] = 'cc' + cursor['ccc'] = 'ccc' + self.session.commit_transaction() + # Search_near for 'c' and assert we skip 3 entries. Internally the row search is landing on + # 'c'. + cursor2 = session2.open_cursor(uri) + cursor2.set_key('c') + cursor2.search_near() + + skip_count = self.get_stat(stat.conn.cursor_next_skip_lt_100) + self.assertEqual(skip_count, 3) + session2.commit_transaction() + + # Perform an insertion and removal of a key next to another key, then search for the + # removed key. + self.session.begin_transaction() + cursor.set_key('dd') + cursor.set_value('dd') + cursor.insert() + cursor.set_key('dd') + cursor.remove() + cursor.set_key('ddd') + cursor.set_value('ddd') + cursor.insert() + cursor.set_key('dd') + cursor.search_near() + self.session.commit_transaction() + skip_count = self.get_stat(stat.conn.cursor_next_skip_lt_100) + self.assertEqual(skip_count, 4) + + # Test a basic prepared scenario. + def test_prepared(self): + uri = 'table:test_base_scenario' + self.session.create(uri, 'key_format=u,value_format=u') + cursor = self.session.open_cursor(uri) + session2 = self.conn.open_session() + cursor3 = session2.open_cursor(uri, None, "debug=(release_evict=true)") + # Insert an update without timestamp + l = "abcdefghijklmnopqrstuvwxyz" + session2.begin_transaction() + + key_count = 26*26 + + # Insert 'cc' + self.session.begin_transaction() + cursor['cc'] = 'cc' + self.session.commit_transaction() + + # Prepare keys aa -> zz + self.session.begin_transaction() + for i in range (0, 26): + if (i == 2): + continue + for j in range (0, 26): + cursor[l[i] + l[j]] = l[i] + l[j] + + self.session.prepare_transaction('prepare_timestamp=2') + + # Evict the whole range. + for i in range (0, 26): + for j in range(0, 26): + cursor3.set_key(l[i] + l[j]) + cursor3.search() + cursor3.reset() + + # Search near for the "aa" part of the range. + cursor2 = session2.open_cursor(uri) + cursor2.set_key('c') + cursor2.search_near() + + skip_count = self.get_stat(stat.conn.cursor_next_skip_lt_100, session2) + # This should be equal to roughly key_count * 2 as we're going to traverse the whole + # range forward, and then the whole range backwards. + self.assertGreater(skip_count, key_count) + + cursor2.reconfigure("prefix_key=true") + cursor2.set_key('c') + cursor2.search_near() + + prefix_skip_count = self.get_stat(stat.conn.cursor_next_skip_lt_100, session2) + self.assertEqual(prefix_skip_count - skip_count, 3) + skip_count = prefix_skip_count + + self.assertEqual(self.get_stat(stat.conn.cursor_search_near_prefix_fast_paths, session2), 2) + + session2.rollback_transaction() + session2.begin_transaction('ignore_prepare=true') + cursor4 = session2.open_cursor(uri) + cursor4.reconfigure("prefix_key=true") + cursor4.set_key('c') + cursor4.search_near() + prefix_skip_count = self.get_stat(stat.conn.cursor_next_skip_lt_100, session2) + self.assertEqual(prefix_skip_count - skip_count, 2) + skip_count = prefix_skip_count + + cursor4.reconfigure("prefix_key=false") + cursor4.set_key('c') + cursor4.search_near() + self.assertEqual(self.get_stat(stat.conn.cursor_next_skip_lt_100, session2) - skip_count, 2) diff --git a/src/third_party/wiredtiger/test/suite/test_tiered01.py b/src/third_party/wiredtiger/test/suite/test_tiered01.py deleted file mode 100644 index 8356f066d81..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_tiered01.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -import wiredtiger, wtscenario, wttest -from wtdataset import SimpleDataSet - -# test_tiered01.py -# Basic tiered tree test -class test_tiered01(wttest.WiredTigerTestCase): - K = 1024 - M = 1024 * K - G = 1024 * M - uri = "table:test_tiered01" - - # Occasionally add a lot of records. - record_count_scenarios = wtscenario.quick_scenarios( - 'nrecs', [10, 10000], [0.9, 0.1]) - - config_vars = [] - - scenarios = wtscenario.make_scenarios(record_count_scenarios, prune=100, prunelong=500) - - # Test create of an object. - def test_tiered(self): - self.session.create('file:first.wt', 'key_format=S') - self.session.create('file:second.wt', 'key_format=S') - args = 'type=tiered,key_format=S' - args += ',tiered=(' # Start the tiered configuration options. - args += 'tiers=("file:first.wt", "file:second.wt"),' - # add names to args, e.g. args += ',session_max=30' - for var in self.config_vars: - value = getattr(self, 's_' + var) - if value != None: - if var == 'verbose': - value = '[' + str(value) + ']' - value = {True : 'true', False : 'false'}.get(value, value) - args += ',' + var + '=' + str(value) - args += ')' # Close the tiered configuration option group - self.verbose(3, - 'Test tiered with config: ' + args + ' count: ' + str(self.nrecs)) - SimpleDataSet(self, self.uri, self.nrecs, config=args).populate() - - # self.session.drop(self.uri) - - # It is an error to configure a tiered table with no tiers - def test_no_tiers(self): - msg = '/tiered table must specify at least one tier/' - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.create(self.uri, 'type=tiered,key_format=S,tiered=(tiers=())'), - msg) - -if __name__ == '__main__': - wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_tiered02.py b/src/third_party/wiredtiger/test/suite/test_tiered02.py index 3317ecdb5b6..4b638a4015f 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered02.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered02.py @@ -26,7 +26,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import wiredtiger, wtscenario, wttest +import os, wiredtiger, wtscenario, wttest from wtdataset import SimpleDataSet # test_tiered02.py @@ -37,27 +37,83 @@ class test_tiered02(wttest.WiredTigerTestCase): G = 1024 * M uri = "file:test_tiered02" - # Occasionally add a lot of records, so that merges (and bloom) happen. - record_count_scenarios = wtscenario.quick_scenarios( - 'nrecs', [10, 10000], [0.9, 0.1]) + auth_token = "test_token" + bucket = "mybucket" + bucket_prefix = "pfx_" + extension_name = "local_store" - scenarios = wtscenario.make_scenarios(record_count_scenarios, prune=100, prunelong=500) + def conn_config(self): + os.makedirs(self.bucket, exist_ok=True) + return \ + 'tiered_storage=(auth_token={},bucket={},bucket_prefix={},name={})'.format( \ + self.auth_token, self.bucket, self.bucket_prefix, self.extension_name) - # Test drop of an object. + # Load the local store extension, but skip the test if it is missing. + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('storage_sources', self.extension_name) + + def confirm_flush(self, increase=True): + # TODO: tiered: flush tests disabled, as the interface + # for flushing will be changed. + return + + self.flushed_objects + got = sorted(list(os.listdir(self.bucket))) + self.pr('Flushed objects: ' + str(got)) + if increase: + self.assertGreater(len(got), self.flushed_objects) + else: + self.assertEqual(len(got), self.flushed_objects) + self.flushed_objects = len(got) + + # Test tiered storage with the old prototype way of signaling flushing to the shared + # tier via checkpoints. When flush_tier is working, the checkpoint calls can be + # replaced with flush_tier. def test_tiered(self): + self.flushed_objects = 0 args = 'key_format=S,block_allocation=log-structured' - self.verbose(3, - 'Test log-structured allocation with config: ' + args + ' count: ' + str(self.nrecs)) - #ds = SimpleDataSet(self, self.uri, self.nrecs, config=args) + self.verbose(3, 'Test log-structured allocation with config: ' + args) + ds = SimpleDataSet(self, self.uri, 10, config=args) ds.populate() + ds.check() self.session.checkpoint() - ds = SimpleDataSet(self, self.uri, 10000, config=args) + # For some reason, every checkpoint does not cause a flush. + # As we're about to move to a new model of flushing, we're not going to chase this error. + #self.confirm_flush() + + ds = SimpleDataSet(self, self.uri, 50, config=args) ds.populate() + ds.check() + self.session.checkpoint() + self.confirm_flush() + + ds = SimpleDataSet(self, self.uri, 100, config=args) + ds.populate() + ds.check() + self.session.checkpoint() + self.confirm_flush() + + ds = SimpleDataSet(self, self.uri, 200, config=args) + ds.populate() + ds.check() + self.close_conn() + self.confirm_flush() # closing the connection does a checkpoint self.reopen_conn() - ds = SimpleDataSet(self, self.uri, 1000, config=args) + # Check what was there before + ds = SimpleDataSet(self, self.uri, 200, config=args) + ds.check() + + # Now add some more. + ds = SimpleDataSet(self, self.uri, 300, config=args) ds.populate() + ds.check() + + # We haven't done a checkpoint/flush so there should be + # nothing extra on the shared tier. + self.confirm_flush(increase=False) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_tiered04.py b/src/third_party/wiredtiger/test/suite/test_tiered04.py index f4007f4df49..0347647031f 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered04.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered04.py @@ -33,14 +33,22 @@ StorageSource = wiredtiger.StorageSource # easy access to constants # test_tiered04.py # Basic tiered storage API test. class test_tiered04(wttest.WiredTigerTestCase): - uri = "table:test_tiered04_sys" - uri1 = "table:test_tiered04" + + # If the 'uri' changes all the other names must change with it. + fileuri = 'file:test_tiered04-0000000001.wt' + objuri = 'object:test_tiered04-0000000001.wtobj' + tiereduri = "tiered:test_tiered04" + uri = "table:test_tiered04" + + uri1 = "table:test_other_tiered04" uri_none = "table:test_local04" auth_token = "test_token" bucket = "mybucket" bucket1 = "otherbucket" extension_name = "local_store" + prefix = "this_pfx" + prefix1 = "other_pfx" object_sys = "9M" object_sys_val = 9 * 1024 * 1024 object_uri = "15M" @@ -48,10 +56,13 @@ class test_tiered04(wttest.WiredTigerTestCase): retention = 600 retention1 = 350 def conn_config(self): + os.mkdir(self.bucket) + os.mkdir(self.bucket1) return \ 'statistics=(all),' + \ 'tiered_storage=(auth_token=%s,' % self.auth_token + \ 'bucket=%s,' % self.bucket + \ + 'bucket_prefix=%s,' % self.prefix + \ 'local_retention=%d,' % self.retention + \ 'name=%s,' % self.extension_name + \ 'object_target_size=%s)' % self.object_sys @@ -61,8 +72,18 @@ class test_tiered04(wttest.WiredTigerTestCase): extlist.skip_if_missing = True extlist.extension('storage_sources', self.extension_name) + # Check for a specific string as part of the uri's metadata. + def check_metadata(self, uri, val_str): + c = self.session.open_cursor('metadata:') + val = c[uri] + c.close() + self.assertTrue(val_str in val) + def get_stat(self, stat, uri): - stat_cursor = self.session.open_cursor('statistics:' + uri) + if uri == None: + stat_cursor = self.session.open_cursor('statistics:') + else: + stat_cursor = self.session.open_cursor('statistics:' + uri) val = stat_cursor[stat][2] stat_cursor.close() return val @@ -72,48 +93,73 @@ class test_tiered04(wttest.WiredTigerTestCase): # Create three tables. One using the system tiered storage, one # specifying its own bucket and object size and one using no # tiered storage. Use stats to verify correct setup. - base_create = 'key_format=S' + intl_page = 'internal_page_max=16K' + base_create = 'key_format=S,' + intl_page + self.pr("create sys") self.session.create(self.uri, base_create) conf = \ ',tiered_storage=(auth_token=%s,' % self.auth_token + \ 'bucket=%s,' % self.bucket1 + \ + 'bucket_prefix=%s,' % self.prefix1 + \ 'local_retention=%d,' % self.retention1 + \ 'name=%s,' % self.extension_name + \ 'object_target_size=%s)' % self.object_uri + self.pr("create non-sys tiered") self.session.create(self.uri1, base_create + conf) conf = ',tiered_storage=(name=none)' + self.pr("create non tiered/local") self.session.create(self.uri_none, base_create + conf) - # Verify the table settings. - obj = self.get_stat(stat.dsrc.tiered_object_size, self.uri) + #self.pr("open cursor") + #c = self.session.open_cursor(self.uri) + self.pr("flush tier") + self.session.flush_tier(None) + + self.pr("flush tier again") + self.session.flush_tier(None) + calls = self.get_stat(stat.conn.flush_tier, None) + self.assertEqual(calls, 2) + obj = self.get_stat(stat.conn.tiered_object_size, None) self.assertEqual(obj, self.object_sys_val) - obj = self.get_stat(stat.dsrc.tiered_object_size, self.uri1) - self.assertEqual(obj, self.object_uri_val) - obj = self.get_stat(stat.dsrc.tiered_object_size, self.uri_none) - self.assertEqual(obj, 0) - retain = self.get_stat(stat.dsrc.tiered_retention, self.uri) - self.assertEqual(retain, self.retention) - retain = self.get_stat(stat.dsrc.tiered_retention, self.uri1) - self.assertEqual(retain, self.retention1) - retain = self.get_stat(stat.dsrc.tiered_retention, self.uri_none) - self.assertEqual(retain, 0) + self.check_metadata(self.tiereduri, intl_page) + self.check_metadata(self.fileuri, intl_page) + self.check_metadata(self.objuri, intl_page) + + #self.pr("verify stats") + # Verify the table settings. + #obj = self.get_stat(stat.dsrc.tiered_object_size, self.uri) + #self.assertEqual(obj, self.object_sys_val) + #obj = self.get_stat(stat.dsrc.tiered_object_size, self.uri1) + #self.assertEqual(obj, self.object_uri_val) + #obj = self.get_stat(stat.dsrc.tiered_object_size, self.uri_none) + #self.assertEqual(obj, 0) + + #retain = self.get_stat(stat.dsrc.tiered_retention, self.uri) + #self.assertEqual(retain, self.retention) + #retain = self.get_stat(stat.dsrc.tiered_retention, self.uri1) + #self.assertEqual(retain, self.retention1) + #retain = self.get_stat(stat.dsrc.tiered_retention, self.uri_none) + #self.assertEqual(retain, 0) # Now test some connection statistics with operations. - retain = self.get_stat(stat.conn.tiered_retention, '') + retain = self.get_stat(stat.conn.tiered_retention, None) self.assertEqual(retain, self.retention) self.session.flush_tier(None) self.session.flush_tier('force=true') - calls = self.get_stat(stat.conn.flush_tier, '') - self.assertEqual(calls, 2) + calls = self.get_stat(stat.conn.flush_tier, None) + self.assertEqual(calls, 4) + + # Test reconfiguration. new = self.retention * 2 config = 'tiered_storage=(local_retention=%d)' % new + self.pr("reconfigure") self.conn.reconfigure(config) self.session.flush_tier(None) - retain = self.get_stat(stat.conn.tiered_retention, '') - calls = self.get_stat(stat.conn.flush_tier, '') + retain = self.get_stat(stat.conn.tiered_retention, None) + calls = self.get_stat(stat.conn.flush_tier, None) self.assertEqual(retain, new) - self.assertEqual(calls, 3) + self.assertEqual(calls, 5) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_tiered05.py b/src/third_party/wiredtiger/test/suite/test_tiered05.py index 098c079fb3d..5cbfe4366c7 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered05.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered05.py @@ -36,17 +36,23 @@ class test_tiered05(wttest.WiredTigerTestCase): uri = "table:test_tiered05" auth_token = "test_token" + bucket = "my_bucket" + bucket_prefix = "my_prefix" extension_name = "local_store" + bucket = "./objects" def conn_extensions(self, extlist): extlist.skip_if_missing = True extlist.extension('storage_sources', self.extension_name) def conn_config(self): + os.mkdir(self.bucket) return \ 'statistics=(fast),' + \ 'tiered_manager=(wait=10),' + \ 'tiered_storage=(auth_token=%s,' % self.auth_token + \ + 'bucket=%s,' % self.bucket + \ + 'bucket_prefix=%s,' % self.bucket_prefix + \ 'name=%s,' % self.extension_name + \ 'object_target_size=20M)' diff --git a/src/third_party/wiredtiger/test/suite/test_tiered06.py b/src/third_party/wiredtiger/test/suite/test_tiered06.py index aba6b8a81b2..e0614cd8c1b 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered06.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered06.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2020 MongoDB, Inc. +# Public Domain 2014-present MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. @@ -27,7 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import os, wiredtiger, wttest -StorageSource = wiredtiger.StorageSource # easy access to constants +FileSystem = wiredtiger.FileSystem # easy access to constants # test_tiered06.py # Test the local storage source. @@ -64,58 +64,79 @@ class test_tiered06(wttest.WiredTigerTestCase): local = self.get_local_storage_source() os.mkdir("objects") - location = local.ss_location_handle(session, - 'cluster="cluster1",bucket="./objects",auth_token="Secret"') + fs = local.ss_customize_file_system(session, "./objects", "cluster1-", "Secret", None) # The object doesn't exist yet. - self.assertFalse(local.ss_exist(session, location, 'foobar')) + self.assertFalse(fs.fs_exist(session, 'foobar')) - fh = local.ss_open_object(session, location, 'foobar', StorageSource.open_create) + fh = fs.fs_open_file(session, 'foobar', FileSystem.open_file_type_data, FileSystem.open_create) + + # Just like a regular file system, the object exists now. + self.assertTrue(fs.fs_exist(session, 'foobar')) outbytes = ('MORE THAN ENOUGH DATA\n'*100000).encode() fh.fh_write(session, 0, outbytes) - # The object doesn't even exist now. - self.assertFalse(local.ss_exist(session, location, 'foobar')) - # The object exists after close fh.close(session) - self.assertTrue(local.ss_exist(session, location, 'foobar')) + self.assertTrue(fs.fs_exist(session, 'foobar')) - fh = local.ss_open_object(session, location, 'foobar', StorageSource.open_readonly) + fh = fs.fs_open_file(session, 'foobar', FileSystem.open_file_type_data, FileSystem.open_readonly) inbytes = bytes(1000000) # An empty buffer with a million zero bytes. fh.fh_read(session, 0, inbytes) # read into the buffer self.assertEquals(outbytes[0:1000000], inbytes) - self.assertEquals(local.ss_size(session, location, 'foobar'), len(outbytes)) + self.assertEquals(fs.fs_size(session, 'foobar'), len(outbytes)) self.assertEquals(fh.fh_size(session), len(outbytes)) fh.close(session) # The fh_lock call doesn't do anything in the local store implementation. - fh = local.ss_open_object(session, location, 'foobar', StorageSource.open_readonly) + fh = fs.fs_open_file(session, 'foobar', FileSystem.open_file_type_data, FileSystem.open_readonly) fh.fh_lock(session, True) fh.fh_lock(session, False) fh.close(session) - self.assertEquals(local.ss_location_list(session, location, '', 0), ['foobar']) + self.assertEquals(fs.fs_directory_list(session, '', ''), ['foobar']) + + # Newly created objects are in the list. + fh = fs.fs_open_file(session, 'zzz', FileSystem.open_file_type_data, FileSystem.open_create) + + # TODO: tiered: the newly created file should be visible, but it is not yet. + # self.assertEquals(sorted(fs.fs_directory_list(session, '', '')), ['foobar', 'zzz' ]) - # Make sure any new object is not in the list until it is closed. - fh = local.ss_open_object(session, location, 'zzz', StorageSource.open_create) - self.assertEquals(local.ss_location_list(session, location, '', 0), ['foobar']) # Sync merely syncs to the local disk. fh.fh_sync(session) fh.close(session) # zero length - self.assertEquals(sorted(local.ss_location_list(session, location, '', 0)), - ['foobar', 'zzz']) + self.assertEquals(sorted(fs.fs_directory_list(session, '', '')), ['foobar', 'zzz' ]) + + # See that we can rename objects. + fs.fs_rename(session, 'zzz', 'yyy', 0) + self.assertEquals(sorted(fs.fs_directory_list(session, '', '')), ['foobar', 'yyy' ]) # See that we can remove objects. - local.ss_remove(session, location, 'zzz', 0) - self.assertEquals(local.ss_location_list(session, location, '', 0), ['foobar']) + fs.fs_remove(session, 'yyy', 0) + self.assertEquals(fs.fs_directory_list(session, '', ''), ['foobar']) + + # TODO: tiered: flush tests disabled, as the interface + # for flushing will be changed. + return # Flushing doesn't do anything that's visible. - local.ss_flush(session, location, None, '') - self.assertEquals(local.ss_location_list(session, location, '', 0), ['foobar']) + local.ss_flush(session, fs, None, '') + self.assertEquals(fs.fs_directory_list(session, '', ''), ['foobar']) + + # Files that have been flushed cannot be manipulated. + with self.expectedStderrPattern('foobar: rename of flushed file not allowed'): + self.assertRaisesException(wiredtiger.WiredTigerError, + lambda: fs.fs_rename(session, 'foobar', 'barfoo', 0)) + self.assertEquals(fs.fs_directory_list(session, '', ''), ['foobar']) - location.close(session) + # Files that have been flushed cannot be manipulated through the custom file system. + with self.expectedStderrPattern('foobar: remove of flushed file not allowed'): + self.assertRaisesException(wiredtiger.WiredTigerError, + lambda: fs.fs_remove(session, 'foobar', 0)) + self.assertEquals(fs.fs_directory_list(session, '', ''), ['foobar']) + + fs.terminate(session) def test_local_write_read(self): # Write and read to a file non-sequentially. @@ -124,14 +145,13 @@ class test_tiered06(wttest.WiredTigerTestCase): local = self.get_local_storage_source() os.mkdir("objects") - location = local.ss_location_handle(session, - 'cluster="cluster1",bucket="./objects",auth_token="Secret"') + fs = local.ss_customize_file_system(session, "./objects", "cluster1-", "Secret", None) # We call these 4K chunks of data "blocks" for this test, but that doesn't # necessarily relate to WT block sizing. nblocks = 1000 block_size = 4096 - fh = local.ss_open_object(session, location, 'abc', StorageSource.open_create) + fh = fs.fs_open_file(session, 'abc', FileSystem.open_file_type_data, FileSystem.open_create) # blocks filled with 'a', etc. a_block = ('a' * block_size).encode() @@ -153,7 +173,7 @@ class test_tiered06(wttest.WiredTigerTestCase): fh.close(session) in_block = bytes(block_size) - fh = local.ss_open_object(session, location, 'abc', StorageSource.open_readonly) + fh = fs.fs_open_file(session, 'abc', FileSystem.open_file_type_data, FileSystem.open_readonly) # Do some spot checks, reading non-sequentially fh.fh_read(session, 500 * block_size, in_block) # divisible by 2, not 3 @@ -176,90 +196,203 @@ class test_tiered06(wttest.WiredTigerTestCase): self.assertEquals(in_block, a_block) fh.close(session) - def create_in_loc(self, loc, objname): + def create_with_fs(self, fs, fname): session = self.session - fh = self.local.ss_open_object(session, loc, objname, StorageSource.open_create) + fh = fs.fs_open_file(session, fname, FileSystem.open_file_type_data, FileSystem.open_create) fh.fh_write(session, 0, 'some stuff'.encode()) fh.close(session) - def check(self, loc, prefix, limit, expect): - # We don't require any sorted output for location lists, + objectdir1 = "./objects1" + objectdir2 = "./objects2" + + cachedir1 = "./cache1" + cachedir2 = "./cache2" + + def check(self, fs, prefix, expect): + # We don't require any sorted output for directory lists, # so we'll sort before comparing.' - got = sorted(self.local.ss_location_list(self.session, loc, prefix, limit)) + got = sorted(fs.fs_directory_list(self.session, '', prefix)) expect = sorted(expect) self.assertEquals(got, expect) - def test_local_locations(self): - # Test using various buckets, clusters + # Check that objects are "in the cloud" after a flush. + # Using the local storage module, they are actually going to be in either + # objectdir1 or objectdir2 + def check_objects(self, expect1, expect2): + got = sorted(list(os.listdir(self.objectdir1))) + expect = sorted(expect1) + self.assertEquals(got, expect) + got = sorted(list(os.listdir(self.objectdir2))) + expect = sorted(expect2) + self.assertEquals(got, expect) + + def test_local_file_systems(self): + # Test using various buckets, hosts session = self.session local = self.conn.get_storage_source('local_store') self.local = local - os.mkdir("objects1") - os.mkdir("objects2") - - # Any of the activity that happens in the various locations - # should be independent. - location1 = local.ss_location_handle(session, - 'cluster="cluster1",bucket="./objects1",auth_token="k1"') - location2 = local.ss_location_handle(session, - 'cluster="cluster1",bucket="./objects2",auth_token="k2"') - location3 = local.ss_location_handle(session, - 'cluster="cluster2",bucket="./objects1",auth_token="k3"') - location4 = local.ss_location_handle(session, - 'cluster="cluster2",bucket="./objects2",auth_token="k4"') - - # Create files in the locations with some name overlap - self.create_in_loc(location1, 'alpaca') - self.create_in_loc(location2, 'bear') - self.create_in_loc(location3, 'crab') - self.create_in_loc(location4, 'deer') + os.mkdir(self.objectdir1) + os.mkdir(self.objectdir2) + os.mkdir(self.cachedir1) + os.mkdir(self.cachedir2) + config1 = "cache_directory=" + self.cachedir1 + config2 = "cache_directory=" + self.cachedir2 + bad_config = "cache_directory=BAD" + + # Create file system objects. First try some error cases. + errmsg = '/No such file or directory/' + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: local.ss_customize_file_system( + session, "./objects1", "pre1-", "k1", bad_config), errmsg) + + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: local.ss_customize_file_system( + session, "./objects_BAD", "pre1-", "k1", config1), errmsg) + + # Create an empty file, try to use it as a directory. + with open("some_file", "w"): + pass + errmsg = '/Invalid argument/' + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: local.ss_customize_file_system( + session, "some_file", "pre1-", "k1", config1), errmsg) + + # Now create some file systems that should succeed. + # Use either different bucket directories or different prefixes, + # so activity that happens in the various file systems should be independent. + fs1 = local.ss_customize_file_system(session, "./objects1", "pre1-", "k1", config1) + fs2 = local.ss_customize_file_system(session, "./objects2", "pre1-", "k2", config2) + fs3 = local.ss_customize_file_system(session, "./objects1", "pre2-", "k3", config1) + fs4 = local.ss_customize_file_system(session, "./objects2", "pre2-", "k4", config2) + + # Create files in the file systems with some name overlap + self.create_with_fs(fs1, 'alpaca') + self.create_with_fs(fs2, 'bear') + self.create_with_fs(fs3, 'crab') + self.create_with_fs(fs4, 'deer') for a in ['beagle', 'bird', 'bison', 'bat']: - self.create_in_loc(location1, a) + self.create_with_fs(fs1, a) for a in ['bird', 'bison', 'bat', 'badger']: - self.create_in_loc(location2, a) + self.create_with_fs(fs2, a) for a in ['bison', 'bat', 'badger', 'baboon']: - self.create_in_loc(location3, a) + self.create_with_fs(fs3, a) for a in ['bat', 'badger', 'baboon', 'beagle']: - self.create_in_loc(location4, a) + self.create_with_fs(fs4, a) # Make sure we see the expected file names - self.check(location1, '', 0, ['alpaca', 'beagle', 'bird', 'bison', 'bat']) - self.check(location1, 'a', 0, ['alpaca']) - self.check(location1, 'b', 0, ['beagle', 'bird', 'bison', 'bat']) - self.check(location1, 'c', 0, []) - self.check(location1, 'd', 0, []) - - self.check(location2, '', 0, ['bear', 'bird', 'bison', 'bat', 'badger']) - self.check(location2, 'a', 0, []) - self.check(location2, 'b', 0, ['bear', 'bird', 'bison', 'bat', 'badger']) - self.check(location2, 'c', 0, []) - self.check(location2, 'd', 0, []) - - self.check(location3, '', 0, ['crab', 'bison', 'bat', 'badger', 'baboon']) - self.check(location3, 'a', 0, []) - self.check(location3, 'b', 0, ['bison', 'bat', 'badger', 'baboon']) - self.check(location3, 'c', 0, ['crab']) - self.check(location3, 'd', 0, []) - - self.check(location4, '', 0, ['deer', 'bat', 'badger', 'baboon', 'beagle']) - self.check(location4, 'a', 0, []) - self.check(location4, 'b', 0, ['bat', 'badger', 'baboon', 'beagle']) - self.check(location4, 'c', 0, []) - self.check(location4, 'd', 0, ['deer']) - - # Flushing doesn't do anything that's visible, but calling it still exercises code paths. - # At some point, we'll have statistics we can check. + self.check(fs1, '', ['alpaca', 'beagle', 'bird', 'bison', 'bat']) + self.check(fs1, 'a', ['alpaca']) + self.check(fs1, 'b', ['beagle', 'bird', 'bison', 'bat']) + self.check(fs1, 'c', []) + self.check(fs1, 'd', []) + + self.check(fs2, '', ['bear', 'bird', 'bison', 'bat', 'badger']) + self.check(fs2, 'a', []) + self.check(fs2, 'b', ['bear', 'bird', 'bison', 'bat', 'badger']) + self.check(fs2, 'c', []) + self.check(fs2, 'd', []) + + self.check(fs3, '', ['crab', 'bison', 'bat', 'badger', 'baboon']) + self.check(fs3, 'a', []) + self.check(fs3, 'b', ['bison', 'bat', 'badger', 'baboon']) + self.check(fs3, 'c', ['crab']) + self.check(fs3, 'd', []) + + self.check(fs4, '', ['deer', 'bat', 'badger', 'baboon', 'beagle']) + self.check(fs4, 'a', []) + self.check(fs4, 'b', ['bat', 'badger', 'baboon', 'beagle']) + self.check(fs4, 'c', []) + self.check(fs4, 'd', ['deer']) + + # Flushing copies files to one of the subdirectories: + # "./objects1" (for fs1 and fs3) + # "./objects2" (for fs2 and fs4) # - # For now, we can turn on the verbose config option for the local_store extension to verify. - local.ss_flush(session, location4, None, '') - local.ss_flush(session, location3, 'badger', '') - local.ss_flush(session, location3, 'c', '') # make sure we don't flush prefixes - local.ss_flush(session, location3, 'b', '') # or suffixes - local.ss_flush(session, location3, 'crab', '') - local.ss_flush(session, location3, 'crab', '') # should do nothing - local.ss_flush(session, None, None, '') # flush everything else - local.ss_flush(session, None, None, '') # should do nothing + # After every flush, we'll check that the right objects appear in the right directory. + # check_objects takes two lists: objects expected to be in ./objects1, + # and objects expected to be in ./objects2 . + self.check_objects([], []) + + # TODO: tiered: flush tests disabled, as the interface + # for flushing will be changed. + enable_fs_flush_tests = False + if enable_fs_flush_tests: + local.ss_flush(session, fs4, None, '') + self.check_objects([], ['pre2-deer', 'pre2-bat', 'pre2-badger', 'pre2-baboon', 'pre2-beagle']) + + local.ss_flush(session, fs3, 'badger', '') + self.check_objects(['pre2-badger'], + ['pre2-deer', 'pre2-bat', 'pre2-badger', 'pre2-baboon', 'pre2-beagle']) + + #local.ss_flush(session, fs3, 'c', '') # make sure we don't flush prefixes + self.check_objects(['pre2-badger'], + ['pre2-deer', 'pre2-bat', 'pre2-badger', 'pre2-baboon', 'pre2-beagle']) + + local.ss_flush(session, fs3, 'b', '') # or suffixes + self.check_objects(['pre2-badger'], + ['pre2-deer', 'pre2-bat', 'pre2-badger', 'pre2-baboon', 'pre2-beagle']) + + local.ss_flush(session, fs3, 'crab', '') + self.check_objects(['pre2-crab', 'pre2-badger'], + ['pre2-deer', 'pre2-bat', 'pre2-badger', 'pre2-baboon', 'pre2-beagle']) + + local.ss_flush(session, fs3, 'crab', '') # should do nothing + self.check_objects(['pre2-crab', 'pre2-badger'], + ['pre2-deer', 'pre2-bat', 'pre2-badger', 'pre2-baboon', 'pre2-beagle']) + + local.ss_flush(session, None, None, '') # flush everything else + self.check_objects(['pre1-alpaca', 'pre1-beagle', 'pre1-bird', 'pre1-bison', 'pre1-bat', + 'pre2-crab', 'pre2-bison', 'pre2-bat', 'pre2-badger', 'pre2-baboon'], + ['pre1-bear', 'pre1-bird', 'pre1-bison', 'pre1-bat', 'pre1-badger', + 'pre2-deer', 'pre2-bat', 'pre2-badger', 'pre2-baboon', 'pre2-beagle']) + + local.ss_flush(session, None, None, '') # should do nothing + self.check_objects(['pre1-alpaca', 'pre1-beagle', 'pre1-bird', 'pre1-bison', 'pre1-bat', + 'pre2-crab', 'pre2-bison', 'pre2-bat', 'pre2-badger', 'pre2-baboon'], + ['pre1-bear', 'pre1-bird', 'pre1-bison', 'pre1-bat', 'pre1-badger', + 'pre2-deer', 'pre2-bat', 'pre2-badger', 'pre2-baboon', 'pre2-beagle']) + + self.create_with_fs(fs4, 'zebra') # should do nothing in the objects directories + self.create_with_fs(fs4, 'yeti') # should do nothing in the objects directories + self.check_objects(['pre1-alpaca', 'pre1-beagle', 'pre1-bird', 'pre1-bison', 'pre1-bat', + 'pre2-crab', 'pre2-bison', 'pre2-bat', 'pre2-badger', 'pre2-baboon'], + ['pre1-bear', 'pre1-bird', 'pre1-bison', 'pre1-bat', 'pre1-badger', + 'pre2-deer', 'pre2-bat', 'pre2-badger', 'pre2-baboon', 'pre2-beagle']) + + # Try remove and rename, should be possible until we flush + self.check(fs4, '', ['deer', 'bat', 'badger', 'baboon', 'beagle', 'yeti', 'zebra']) + fs4.fs_remove(session, 'yeti', 0) + self.check(fs4, '', ['deer', 'bat', 'badger', 'baboon', 'beagle', 'zebra']) + fs4.fs_rename(session, 'zebra', 'okapi', 0) + self.check(fs4, '', ['deer', 'bat', 'badger', 'baboon', 'beagle', 'okapi']) + local.ss_flush(session, None, None, '') + self.check(fs4, '', ['deer', 'bat', 'badger', 'baboon', 'beagle', 'okapi']) + self.check_objects(['pre1-alpaca', 'pre1-beagle', 'pre1-bird', 'pre1-bison', 'pre1-bat', + 'pre2-crab', 'pre2-bison', 'pre2-bat', 'pre2-badger', 'pre2-baboon'], + ['pre1-bear', 'pre1-bird', 'pre1-bison', 'pre1-bat', 'pre1-badger', + 'pre2-deer', 'pre2-bat', 'pre2-badger', 'pre2-baboon', 'pre2-beagle', + 'pre2-okapi']) + + errmsg = '/rename of flushed file not allowed/' + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: fs4.fs_rename(session, 'okapi', 'zebra', 0), errmsg) + + # XXX + # At the moment, removal of flushed files is not allowed - as flushed files are immutable. + # We may need to explicitly evict flushed files from cache directory via the API, if so, + # the API to do that might be on the local store object, not the file system. + errmsg = '/remove of flushed file not allowed/' + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: fs4.fs_remove(session, 'okapi', 0), errmsg) + + # No change since last time. + self.check(fs4, '', ['deer', 'bat', 'badger', 'baboon', 'beagle', 'okapi']) + self.check_objects(['pre1-alpaca', 'pre1-beagle', 'pre1-bird', 'pre1-bison', 'pre1-bat', + 'pre2-crab', 'pre2-bison', 'pre2-bat', 'pre2-badger', 'pre2-baboon'], + ['pre1-bear', 'pre1-bird', 'pre1-bison', 'pre1-bat', 'pre1-badger', + 'pre2-deer', 'pre2-bat', 'pre2-badger', 'pre2-baboon', 'pre2-beagle', + 'pre2-okapi']) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_txn26.py b/src/third_party/wiredtiger/test/suite/test_txn26.py new file mode 100644 index 00000000000..75633b275e3 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_txn26.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest + +# test_txn26.py +# Test that commit should fail if commit timestamp is smaller or equal to the active timestamp. Our handling of out of order timestamp relies on this to ensure repeated reads are working as expected. +def timestamp_str(t): + return '%x' % t +class test_txn26(wttest.WiredTigerTestCase): + conn_config = 'cache_size=50MB' + session_config = 'isolation=snapshot' + + def test_commit_larger_than_active_timestamp(self): + if not wiredtiger.diagnostic_build(): + self.skipTest('requires a diagnostic build') + + uri = 'table:test_txn26' + self.session.create(uri, 'key_format=S,value_format=S') + cursor = self.session.open_cursor(uri) + self.conn.set_timestamp( + 'oldest_timestamp=' + timestamp_str(1) + ',stable_timestamp=' + timestamp_str(1)) + + value = 'a' + + # Start a session with timestamp 10 + session2 = self.conn.open_session(self.session_config) + session2.begin_transaction('read_timestamp=' + timestamp_str(10)) + + # Try to commit at timestamp 10 + self.session.begin_transaction() + cursor[str(0)] = value + with self.expectedStderrPattern("must be greater than the latest active read timestamp"): + try: + self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + except wiredtiger.WiredTigerError as e: + gotException = True + self.pr('got expected exception: ' + str(e)) + self.assertTrue(str(e).find('nvalid argument') >= 0) + self.assertTrue(gotException, msg = 'expected exception') diff --git a/src/third_party/wiredtiger/test/suite/test_util21.py b/src/third_party/wiredtiger/test/suite/test_util21.py index cdd117649db..2271ad8b312 100644 --- a/src/third_party/wiredtiger/test/suite/test_util21.py +++ b/src/third_party/wiredtiger/test/suite/test_util21.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2021 MongoDB, Inc. +# Public Domain 2014-present MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/src/third_party/wiredtiger/test/suite/wthooks.py b/src/third_party/wiredtiger/test/suite/wthooks.py new file mode 100755 index 00000000000..56827350e29 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/wthooks.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# [TEST_TAGS] +# ignored_file +# [END_TAGS] +# +# WiredTigerHookManager +# Manage running of hooks +# +from __future__ import print_function + +import os, sys +from importlib import import_module +from abc import ABC, abstractmethod +import wiredtiger + +# Three kinds of hooks available: +HOOK_REPLACE = 1 # replace the call with the hook function +HOOK_NOTIFY = 2 # call the hook function after the function +HOOK_ARGS = 3 # transform the arg list before the call + +# Print to /dev/tty for debugging, since anything extraneous to stdout/stderr will +# cause a test error. +def tty(message): + from wttest import WiredTigerTestCase + WiredTigerTestCase.tty(message) + +################ +# Hooks Overview +# +# Here are some useful terms to know, with some commentary for each. +# +# API functions +# potentially any WiredTiger API functions that a hook creator wishes to modify (like +# Session.rename). In Python most everything is an object. Of course an instance of +# "Session" is an object, but also the "Session" class itself is an object. The Session.rename +# function is also an object (of a certain form that can be called). Also in Python, +# attributes on an object don't have to be "pre-declared", they can be created at any time. +# So it's easy to imagine assigning Session._rename_orig to be (the original value of) +# Session.rename, and then assigning Session.rename to be some other function object, that +# knows how to do something and then perhaps calls Session._rename_orig . This is the +# essence of the hook concept. +# +# Hook Creator: +# A way to attach a set of "behavior modifications" to various API functions. More precisely, +# a hook creator derives from WiredTigerHookCreator and sets up a number of "hook functions", +# that are actions that are done either just before, after, or instead of, an API function. +# A XxxxHookCreator lives in a hook_xxxx.py file. When a HookCreator is loaded, it may be +# given an optional argument. This argument comes from the original python command line. +# For example, "python run.py --hook abc" loads hook_abc.py (where it expects to find a hook). +# "python run.py --hook abc=123" loads hook_abc.py with an argument "123". +# +# Hook Function: +# One function that will be called before, after or instead of, an API function. A hook +# function will be bound to an API function. It is the job of the HookCreator to set up that +# binding. It is possible to have multiple hook functions bound to the same API function. +# A hook function that replaces an API function will have the same args as the function +# it replaces (but there is a trick to give it additional context if needed - +# see session_create_replace in hook_demo.py). + +# For every API function altered, there is one of these objects +# stashed in the <class>._<api_name>_hooks attribute. +class WiredTigerHookInfo(object): + def __init__(self): + self.arg_funcs = [] # The set of hook functions for manipulating arguments + self.notify_funcs = [] # The set of hook functions for manipulating arguments + # At the moment, we can only replace a method once. + # If needed, we can think about removing this restriction. + self.replace_func = None + +# hooked_function - +# A helper function for the hook manager. +def hooked_function(self, orig_func, hook_info_name, *args): + hook_info = getattr(self, hook_info_name) + + notifies = [] + replace_func = None + + # The three kinds of hooks are acted upon at different times. + # Before we call the function, we modify the args as indicated + # by hooks. Then we call the function, possibly with a replacement. + # Finally, we'll call any notify hooks. + # + # We only walk through the hook list once, and process the config + # hooks while we're doing that, and copy any other hooks needed. + for hook_func in hook_info.arg_funcs: + args = hook_func(self, args) + call_func = hook_info.replace_func + if call_func == None: + call_func = orig_func + if self == wiredtiger: + ret = call_func(*args) + else: + ret = call_func(self, *args) + for hook_func in hook_info.notify_funcs: + hook_func(ret, self, *args) + return ret + +# WiredTigerHookManager - +# The hook manager class. There is only one hook manager. It is responsible for finding all the +# HookCreators at the beginning of the run, and calling setup_hooks() for each one, to have it bind +# hook functions to API functions. The hook manager is initialized with a list of hook names. Each +# name is expanded, for example, "demo" causes the hook manager to load hook_demo.py, and to call +# the "initialize" global function in that file. We expect "initialize" to return a list of objects +# (hooks) derived from WiredTigerHook (class defined below). Generally, "initialize" returns a +# single object (setting up some number of "hook functions") but to allow flexibility for different +# sorts of packaging, we allow any number of hooks to be returned. +# +# A hook can set up any number of "hook functions". See hook_demo.py for a sample hook class. +class WiredTigerHookManager(object): + def __init__(self, hooknames = []): + self.hooks = [] + names_seen = [] + for name in hooknames: + # The hooks are indicated as "somename=arg" or simply "somename". + # hook_somename.py will be imported, and initialized with the arg. + # Names must be unique, as we stash some info into extra fields + # on the connection/session/cursor, these are named using the + # unique name of the hook. + if '=' in name: + name,arg = name.split('=', 1) + else: + arg = None + if name in names_seen: + raise Exception(name + ': hook name cannot be used multiple times') + names_seen.append(name) + + modname = 'hook_' + name + try: + imported = import_module(modname) + for hook in imported.initialize(arg): + hook._initialize(name, self) + self.hooks.append(hook) + except: + print('Cannot import hook: ' + name + ', check file ' + modname + '.py') + raise + for hook in self.hooks: + hook.setup_hooks() + + def add_hook(self, clazz, method_name, hook_type, hook_func): + if not hasattr(clazz, method_name): + raise Exception('Cannot find method ' + method_name + ' on class ' + str(clazz)) + + # We need to set up some extra attributes on the Connection class. + # Given that the method name is XXXX, and class is Connection, here's what we're doing: + # orig = wiredtiger.Connection.XXXX + # wiredtiger.Connection._XXXX_hooks = WiredTigerHookInfo() + # wiredtiger.Connection._XXXX_orig = wiredtiger.Connection.XXXX + # wiredtiger.Connection.XXXX = lambda self, *args: + # hooked_function(self, orig, '_XXXX_hooks', *args) + hook_info_name = '_' + method_name + '_hooks' + orig_name = '_' + method_name + '_orig' + if not hasattr(clazz, hook_info_name): + #tty('Setting up hook on ' + str(clazz) + '.' + method_name) + orig_func = getattr(clazz, method_name) + if orig_func == None: + raise Exception('method ' + method_name + ' hook setup: method does not exist') + setattr(clazz, hook_info_name, WiredTigerHookInfo()) + + # If we're using the wiredtiger module and not a class, we need a slightly different + # style of hooked_function, since there is no self. What would be the "self" argument + # is in fact the class. + if clazz == wiredtiger: + f = lambda *args: hooked_function(wiredtiger, orig_func, hook_info_name, *args) + else: + f = lambda self, *args: hooked_function(self, orig_func, hook_info_name, *args) + setattr(clazz, method_name, f) + setattr(clazz, orig_name, orig_func) + + # Now add to the list of hook functions + # If it's a replace hook, we only allow one of them for a given method name + hook_info = getattr(clazz, hook_info_name) + if hook_type == HOOK_ARGS: + hook_info.arg_funcs.append(hook_func) + elif hook_type == HOOK_NOTIFY: + hook_info.notify_funcs.append(hook_func) + elif hook_type == HOOK_REPLACE: + if hook_info.replace_func == None: + hook_info.replace_func = hook_func + else: + raise Exception('method ' + method_name + ' hook setup: trying to replace the same method with two hooks') + #tty('Setting up hooks list in ' + str(clazz) + '.' + hook_info_name) + + def get_function(self, clazz, method_name): + orig_name = '_' + method_name + '_orig' + if hasattr(clazz, orig_name): + orig_func = getattr(clazz, orig_name) + else: + orig_func = getattr(clazz, method_name) + return orig_func + + def filter_tests(self, tests): + for hook in self.hooks: + tests = hook.filter_tests(tests) + return tests + +class HookCreatorProxy(object): + def __init__(self, hookmgr, clazz): + self.hookmgr = hookmgr + self.clazz = clazz + + # Get the original function/method before any hooks applied + def __getitem__(self, name): + return self.hookmgr.get_function(self.clazz, name) + + # Get the original function/method before any hooks applied + def __setitem__(self, name, value): + try: + hooktype = int(value[0]) + fcn = value[1] + except: + raise ValueError('value must be (HOOK_xxxx, function)') + self.hookmgr.add_hook(self.clazz, name, hooktype, fcn) + +# Hooks must derive from this class +class WiredTigerHookCreator(ABC): + # This is called right after creation and should not be overridden. + def _initialize(self, name, hookmgr): + self.name = name + self.hookmgr = hookmgr + self.wiredtiger = HookCreatorProxy(self.hookmgr, wiredtiger) + self.Connection = HookCreatorProxy(self.hookmgr, wiredtiger.Connection) + self.Session = HookCreatorProxy(self.hookmgr, wiredtiger.Session) + self.Cursor = HookCreatorProxy(self.hookmgr, wiredtiger.Cursor) + + # default version of filter_tests, can be overridden + def filter_tests(self, tests): + return tests + + @abstractmethod + def setup_hooks(self): + """Set up all hooks using add_*_hook methods.""" + return diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py index be38c1a748a..87db069fefe 100755 --- a/src/third_party/wiredtiger/test/suite/wttest.py +++ b/src/third_party/wiredtiger/test/suite/wttest.py @@ -43,7 +43,7 @@ except ImportError: from contextlib import contextmanager import errno, glob, os, re, shutil, sys, time, traceback -import wiredtiger, wtscenario +import wiredtiger, wtscenario, wthooks def shortenWithEllipsis(s, maxlen): if len(s) > maxlen: @@ -183,6 +183,7 @@ class ExtensionList(list): class WiredTigerTestCase(unittest.TestCase): _globalSetup = False _printOnceSeen = {} + _ttyDescriptor = None # set this early, to allow tty() to be called any time. # conn_config can be overridden to add to basic connection configuration. # Can be a string or a callable function or lambda expression. @@ -200,14 +201,15 @@ class WiredTigerTestCase(unittest.TestCase): conn_extensions = () @staticmethod - def globalSetup(preserveFiles = False, useTimestamp = False, + def globalSetup(preserveFiles = False, removeAtStart = True, useTimestamp = False, gdbSub = False, lldbSub = False, verbose = 1, builddir = None, dirarg = None, - longtest = False, ignoreStdout = False, seedw = 0, seedz = 0): + longtest = False, ignoreStdout = False, seedw = 0, seedz = 0, hookmgr = None): WiredTigerTestCase._preserveFiles = preserveFiles d = 'WT_TEST' if dirarg == None else dirarg if useTimestamp: d += '.' + time.strftime('%Y%m%d-%H%M%S', time.localtime()) - shutil.rmtree(d, ignore_errors=True) + if removeAtStart: + shutil.rmtree(d, ignore_errors=True) os.makedirs(d) wtscenario.set_long_run(longtest) WiredTigerTestCase._parentTestdir = d @@ -224,9 +226,11 @@ class WiredTigerTestCase(unittest.TestCase): WiredTigerTestCase._stderr = sys.stderr WiredTigerTestCase._concurrent = False WiredTigerTestCase._globalSetup = True - WiredTigerTestCase._ttyDescriptor = None WiredTigerTestCase._seeds = [521288629, 362436069] WiredTigerTestCase._randomseed = False + if hookmgr == None: + hookmgr = wthooks.WiredTigerHookManager() + WiredTigerTestCase._hookmgr = hookmgr if seedw != 0 and seedz != 0: WiredTigerTestCase._randomseed = True WiredTigerTestCase._seeds = [seedw, seedz] diff --git a/src/third_party/wiredtiger/test/utility/misc.c b/src/third_party/wiredtiger/test/utility/misc.c index fc0896aced6..674a868c35e 100644 --- a/src/third_party/wiredtiger/test/utility/misc.c +++ b/src/third_party/wiredtiger/test/utility/misc.c @@ -35,6 +35,12 @@ void (*custom_die)(void) = NULL; const char *progname = "program name not set"; /* + * Backup directory initialize command, remove and re-create the primary backup directory, plus a + * copy we maintain for recovery testing. + */ +#define HOME_BACKUP_INIT_CMD "rm -rf %s/BACKUP %s/BACKUP.copy && mkdir %s/BACKUP %s/BACKUP.copy" + +/* * testutil_die -- * Report an error and abort. */ @@ -256,6 +262,46 @@ testutil_timestamp_parse(const char *str, uint64_t *tsp) testutil_assert(p - str <= 16); } +void +testutil_create_backup_directory(const char *home) +{ + size_t len; + char *cmd; + + len = strlen(home) * 4 + strlen(HOME_BACKUP_INIT_CMD) + 1; + cmd = dmalloc(len); + testutil_check(__wt_snprintf(cmd, len, HOME_BACKUP_INIT_CMD, home, home, home, home)); + testutil_checkfmt(system(cmd), "%s", "backup directory creation failed"); + free(cmd); +} + +/* + * copy_file -- + * Copy a single file into the backup directories. + */ +void +testutil_copy_file(WT_SESSION *session, const char *name) +{ + size_t len; + char *first, *second; + + len = strlen("BACKUP") + strlen(name) + 10; + first = dmalloc(len); + testutil_check(__wt_snprintf(first, len, "BACKUP/%s", name)); + testutil_check(__wt_copy_and_sync(session, name, first)); + + /* + * Save another copy of the original file to make debugging recovery errors easier. + */ + len = strlen("BACKUP.copy") + strlen(name) + 10; + second = dmalloc(len); + testutil_check(__wt_snprintf(second, len, "BACKUP.copy/%s", name)); + testutil_check(__wt_copy_and_sync(session, first, second)); + + free(first); + free(second); +} + /* * testutil_is_flag_set -- * Return if an environment variable flag is set. diff --git a/src/third_party/wiredtiger/test/utility/test_util.h b/src/third_party/wiredtiger/test/utility/test_util.h index be32e7c0206..b657d5717a7 100644 --- a/src/third_party/wiredtiger/test/utility/test_util.h +++ b/src/third_party/wiredtiger/test/utility/test_util.h @@ -274,6 +274,8 @@ int testutil_parse_opts(int, char *const *, TEST_OPTS *); void testutil_print_command_line(int argc, char *const *argv); void testutil_progress(TEST_OPTS *, const char *); void testutil_timestamp_parse(const char *, uint64_t *); +void testutil_create_backup_directory(const char *); +void testutil_copy_file(WT_SESSION *, const char *); #ifndef _WIN32 void testutil_sleep_wait(uint32_t, pid_t); #endif |