diff options
author | Luke Chen <luke.chen@mongodb.com> | 2021-06-08 16:45:31 +1000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-06-08 07:12:32 +0000 |
commit | 7c2ed6ff9adfbec5928a35f94ac7f070007bc934 (patch) | |
tree | e728ff686e3ab63a6a14157ced1bf363ccf2a617 /src/third_party | |
parent | 24fa251076f24279a3f762a061f4c22486016766 (diff) | |
download | mongo-7c2ed6ff9adfbec5928a35f94ac7f070007bc934.tar.gz |
Import wiredtiger: a901a0e9d7c74f6687f4bb542331a5090fd7a7b2 from branch mongodb-5.0
ref: 99311e58f7..a901a0e9d7
for: 5.0.0-rc1
WT-7320 Add the ability to check database size to the test framework
Diffstat (limited to 'src/third_party')
11 files changed, 186 insertions, 56 deletions
diff --git a/src/third_party/wiredtiger/dist/test_data.py b/src/third_party/wiredtiger/dist/test_data.py index b13906c22ed..4afefc61497 100644 --- a/src/third_party/wiredtiger/dist/test_data.py +++ b/src/third_party/wiredtiger/dist/test_data.py @@ -134,6 +134,9 @@ checkpoint_manager = enabled_config_false + component_config runtime_monitor = enabled_config_true + component_config + [ Config('stat_cache_size', '', ''' The maximum cache percentage that can be hit while running.''', + type='category', subconfig=limit_stat), + Config('stat_db_size', '', ''' + The maximum on-disk database size in bytes that can be hit while running.''', type='category', subconfig=limit_stat) ] diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 1fec843d0d4..0af4dc02711 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-5.0", - "commit": "99311e58f76242da720219d8bd35127397400b8a" + "commit": "a901a0e9d7c74f6687f4bb542331a5090fd7a7b2" } diff --git a/src/third_party/wiredtiger/src/config/test_config.c b/src/third_party/wiredtiger/src/config/test_config.c index ed314e7e1b3..777b6902a1c 100644 --- a/src/third_party/wiredtiger/src/config/test_config.c +++ b/src/third_party/wiredtiger/src/config/test_config.c @@ -11,11 +11,16 @@ static const WT_CONFIG_CHECK confchk_stat_cache_size_subconfigs[] = { {"enabled", "boolean", NULL, NULL, NULL, 0}, {"limit", "int", NULL, "min=0", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; +static const WT_CONFIG_CHECK confchk_stat_db_size_subconfigs[] = { + {"enabled", "boolean", NULL, NULL, NULL, 0}, {"limit", "int", NULL, "min=0", NULL, 0}, + {NULL, NULL, NULL, NULL, NULL, 0}}; + static const WT_CONFIG_CHECK confchk_runtime_monitor_subconfigs[] = { {"enabled", "boolean", NULL, NULL, NULL, 0}, {"interval", "string", NULL, "choices=[\"s\",\"m\",\"h\"]", NULL, 0}, {"op_count", "int", NULL, "min=1,max=10000", NULL, 0}, {"stat_cache_size", "category", NULL, NULL, confchk_stat_cache_size_subconfigs, 2}, + {"stat_db_size", "category", NULL, NULL, confchk_stat_db_size_subconfigs, 2}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_timestamp_manager_subconfigs[] = { @@ -73,7 +78,7 @@ static const WT_CONFIG_CHECK confchk_example_test[] = { {"checkpoint_manager", "category", NULL, NULL, confchk_checkpoint_manager_subconfigs, 3}, {"duration_seconds", "int", NULL, "min=0,max=1000000", NULL, 0}, {"enable_logging", "boolean", NULL, NULL, NULL, 0}, - {"runtime_monitor", "category", NULL, NULL, confchk_runtime_monitor_subconfigs, 4}, + {"runtime_monitor", "category", NULL, NULL, confchk_runtime_monitor_subconfigs, 5}, {"timestamp_manager", "category", NULL, NULL, confchk_timestamp_manager_subconfigs, 5}, {"workload_generator", "category", NULL, NULL, confchk_workload_generator_subconfigs, 10}, {"workload_tracking", "category", NULL, NULL, confchk_workload_tracking_subconfigs, 3}, @@ -84,7 +89,7 @@ static const WT_CONFIG_CHECK confchk_poc_test[] = { {"checkpoint_manager", "category", NULL, NULL, confchk_checkpoint_manager_subconfigs, 3}, {"duration_seconds", "int", NULL, "min=0,max=1000000", NULL, 0}, {"enable_logging", "boolean", NULL, NULL, NULL, 0}, - {"runtime_monitor", "category", NULL, NULL, confchk_runtime_monitor_subconfigs, 4}, + {"runtime_monitor", "category", NULL, NULL, confchk_runtime_monitor_subconfigs, 5}, {"timestamp_manager", "category", NULL, NULL, confchk_timestamp_manager_subconfigs, 5}, {"workload_generator", "category", NULL, NULL, confchk_workload_generator_subconfigs, 10}, {"workload_tracking", "category", NULL, NULL, confchk_workload_tracking_subconfigs, 3}, @@ -95,7 +100,8 @@ static const WT_CONFIG_ENTRY config_entries[] = { "cache_size_mb=0,checkpoint_manager=(enabled=false,interval=s," "op_count=1),duration_seconds=0,enable_logging=false," "runtime_monitor=(enabled=true,interval=s,op_count=1," - "stat_cache_size=(enabled=false,limit=0))," + "stat_cache_size=(enabled=false,limit=0)," + "stat_db_size=(enabled=false,limit=0))," "timestamp_manager=(enabled=true,interval=s,oldest_lag=1," "op_count=1,stable_lag=1),workload_generator=(collection_count=1," "enabled=true,insert_config=(interval=s,key_size=5,op_count=1," @@ -111,7 +117,8 @@ static const WT_CONFIG_ENTRY config_entries[] = { "cache_size_mb=0,checkpoint_manager=(enabled=false,interval=s," "op_count=1),duration_seconds=0,enable_logging=false," "runtime_monitor=(enabled=true,interval=s,op_count=1," - "stat_cache_size=(enabled=false,limit=0))," + "stat_cache_size=(enabled=false,limit=0)," + "stat_db_size=(enabled=false,limit=0))," "timestamp_manager=(enabled=true,interval=s,oldest_lag=1," "op_count=1,stable_lag=1),workload_generator=(collection_count=1," "enabled=true,insert_config=(interval=s,key_size=5,op_count=1," diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.h index d24317b7352..90aecc9aa59 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.h @@ -38,6 +38,7 @@ extern "C" { #include "core/component.h" #include "core/throttle.h" #include "connection_manager.h" +#include "workload/database_operation.h" namespace test_harness { /* Static statistic get function. */ @@ -52,7 +53,7 @@ get_stat(WT_CURSOR *cursor, int stat_field, int64_t *valuep) class statistic { public: - statistic(configuration *config) + explicit statistic(configuration *config) { _enabled = config->get_bool(ENABLED); } @@ -75,13 +76,13 @@ class statistic { class cache_limit_statistic : public statistic { public: - cache_limit_statistic(configuration *config) : statistic(config) + explicit cache_limit_statistic(configuration *config) : statistic(config) { limit = config->get_int(LIMIT); } void - check(WT_CURSOR *cursor) + check(WT_CURSOR *cursor) override final { testutil_assert(cursor != nullptr); int64_t cache_bytes_image, cache_bytes_other, cache_bytes_max; @@ -96,11 +97,10 @@ class cache_limit_statistic : public statistic { */ use_percent = ((cache_bytes_image + cache_bytes_other + 0.0) / cache_bytes_max) * 100; if (use_percent > limit) { - std::string error_string = + const std::string error_string = "runtime_monitor: Cache usage exceeded during test! Limit: " + std::to_string(limit) + " usage: " + std::to_string(use_percent); - debug_print(error_string, DEBUG_ERROR); - testutil_assert(use_percent < limit); + testutil_die(-1, error_string.c_str()); } else debug_print("Cache usage: " + std::to_string(use_percent), DEBUG_TRACE); } @@ -109,13 +109,89 @@ class cache_limit_statistic : public statistic { int64_t limit; }; +static std::string +collection_name_to_file_name(const std::string &collection_name) +{ + /* Strip out the URI prefix. */ + const size_t colon_pos = collection_name.find(':'); + testutil_assert(colon_pos != std::string::npos); + const auto stripped_name = collection_name.substr(colon_pos + 1); + + /* Now add the directory and file extension. */ + return std::string(DEFAULT_DIR) + "/" + stripped_name + ".wt"; +} + +class db_size_statistic : public statistic { + public: + db_size_statistic(configuration *config, database &database) + : statistic(config), _database(database) + { + _limit = config->get_int(LIMIT); +#ifdef _WIN32 + debug_print("Database size checking is not implemented on Windows", DEBUG_ERROR); +#endif + } + virtual ~db_size_statistic() = default; + + /* Don't need the stat cursor for this. */ + void + check(WT_CURSOR *) override final + { + const auto file_names = get_file_names(); +#ifndef _WIN32 + size_t db_size = 0; + for (const auto &name : file_names) { + struct stat sb; + if (stat(name.c_str(), &sb) == 0) { + db_size += sb.st_size; + debug_print(name + " was " + std::to_string(sb.st_size) + " bytes", DEBUG_TRACE); + } else + /* The only good reason for this to fail is if the file hasn't been created yet. */ + testutil_assert(errno == ENOENT); + } + debug_print("Current database size is " + std::to_string(db_size) + " bytes", DEBUG_TRACE); + if (db_size > _limit) { + const std::string error_string = + "runtime_monitor: Database size limit exceeded during test! Limit: " + + std::to_string(_limit) + " db size: " + std::to_string(db_size); + testutil_die(-1, error_string.c_str()); + } +#else + static_cast<void>(file_names); + static_cast<void>(_database); + static_cast<void>(_limit); +#endif + } + + private: + std::vector<std::string> + get_file_names() + { + std::vector<std::string> file_names; + for (const auto &name : _database.get_collection_names()) + file_names.push_back(collection_name_to_file_name(name)); + + /* Add WiredTiger internal tables. */ + file_names.push_back(std::string(DEFAULT_DIR) + "/" + WT_HS_FILE); + file_names.push_back(std::string(DEFAULT_DIR) + "/" + WT_METAFILE); + + return file_names; + } + + database &_database; + int64_t _limit; +}; + /* * The runtime monitor class is designed to track various statistics or other runtime signals * relevant to the given workload. */ class runtime_monitor : public component { public: - runtime_monitor(configuration *config) : component("runtime_monitor", config) {} + runtime_monitor(configuration *config, database &database) + : component("runtime_monitor", config), _database(database) + { + } ~runtime_monitor() { @@ -147,6 +223,10 @@ class runtime_monitor : public component { sub_config = _config->get_subconfig(STAT_CACHE_SIZE); _stats.push_back(new cache_limit_statistic(sub_config)); delete sub_config; + + sub_config = _config->get_subconfig(STAT_DB_SIZE); + _stats.push_back(new db_size_statistic(sub_config, _database)); + delete sub_config; } } @@ -163,6 +243,7 @@ class runtime_monitor : public component { WT_CURSOR *_cursor = nullptr; WT_SESSION *_session = nullptr; std::vector<statistic *> _stats; + database &_database; }; } // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/test.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/test.h index 06337d2a000..2434704f6f9 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/test.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/test.h @@ -59,13 +59,13 @@ class test : public database_operation { { _config = new configuration(name, config); _checkpoint_manager = new checkpoint_manager(_config->get_subconfig(CHECKPOINT_MANAGER)); - _runtime_monitor = new runtime_monitor(_config->get_subconfig(RUNTIME_MONITOR)); + _runtime_monitor = new runtime_monitor(_config->get_subconfig(RUNTIME_MONITOR), _database); _timestamp_manager = new timestamp_manager(_config->get_subconfig(TIMESTAMP_MANAGER)); _workload_tracking = new workload_tracking(_config->get_subconfig(WORKLOAD_TRACKING), OPERATION_TRACKING_TABLE_CONFIG, TABLE_OPERATION_TRACKING, SCHEMA_TRACKING_TABLE_CONFIG, TABLE_SCHEMA_TRACKING); - _workload_generator = new workload_generator( - _config->get_subconfig(WORKLOAD_GENERATOR), this, _timestamp_manager, _workload_tracking); + _workload_generator = new workload_generator(_config->get_subconfig(WORKLOAD_GENERATOR), + this, _timestamp_manager, _workload_tracking, _database); _thread_manager = new thread_manager(); /* * Ordering is not important here, any dependencies between components should be resolved @@ -191,6 +191,7 @@ class test : public database_operation { timestamp_manager *_timestamp_manager = nullptr; workload_generator *_workload_generator = nullptr; workload_tracking *_workload_tracking = nullptr; + database _database; }; } // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/api_const.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/api_const.h index 495989407c1..51200d389c9 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/api_const.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/api_const.h @@ -58,6 +58,7 @@ static const char *OPS_PER_TRANSACTION = "ops_per_transaction"; static const char *READ_CONFIG = "read_config"; static const char *STABLE_LAG = "stable_lag"; static const char *STAT_CACHE_SIZE = "stat_cache_size"; +static const char *STAT_DB_SIZE = "stat_db_size"; static const char *THREAD_COUNT = "thread_count"; static const char *UPDATE_CONFIG = "update_config"; static const char *VALUE_SIZE = "value_size"; diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_model.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_model.h index c2a7ed9f6a6..c9562954bfd 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_model.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_model.h @@ -50,24 +50,77 @@ struct value_t { /* A collection is made of mapped Key objects. */ struct collection_t { std::map<key_value_t, key_t> keys; - std::map<key_value_t, value_t> *values = {nullptr}; + std::map<key_value_t, value_t> values; }; /* Representation of the collections in memory. */ class database { public: - const std::vector<std::string> - get_collection_names() const + std::vector<std::string> + get_collection_names() { + std::lock_guard<std::mutex> lg(_mtx); std::vector<std::string> collection_names; - for (auto const &it : collections) + for (auto const &it : _collections) collection_names.push_back(it.first); return (collection_names); } - std::map<std::string, collection_t> collections; + std::map<key_value_t, key_t> + get_keys(const std::string &collection_name) + { + std::lock_guard<std::mutex> lg(_mtx); + return (_collections.at(collection_name).keys); + } + + void + add_collection(const std::string &collection_name) + { + std::lock_guard<std::mutex> lg(_mtx); + testutil_assert(_collections.find(collection_name) == _collections.end()); + _collections[collection_name] = {}; + } + + value_t + get_record(const std::string &collection_name, const char *key) + { + std::lock_guard<std::mutex> lg(_mtx); + return (_collections.at(collection_name).values.at(key)); + } + + void + insert_record(const std::string &collection_name, const char *key, const char *value) + { + std::lock_guard<std::mutex> lg(_mtx); + auto &c = _collections.at(collection_name); + c.keys[key].exists = true; + value_t v; + v.value = key_value_t(value); + c.values.emplace(key_value_t(key), v); + } + + void + update_record(const std::string &collection_name, const char *key, const char *value) + { + std::lock_guard<std::mutex> lg(_mtx); + auto &c = _collections.at(collection_name); + c.values.at(key).value = key_value_t(value); + } + + void + delete_record(const std::string &collection_name, const char *key) + { + std::lock_guard<std::mutex> lg(_mtx); + auto &c = _collections.at(collection_name); + c.keys.at(key).exists = false; + c.values.erase(key); + } + + private: + std::map<std::string, collection_t> _collections; + std::mutex _mtx; }; } // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.h index ab192f798b1..007d39da345 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.h @@ -69,7 +69,7 @@ class database_operation { collection_count = config->get_int(COLLECTION_COUNT); for (size_t i = 0; i < collection_count; ++i) { collection_name = "table:collection" + std::to_string(i); - database.collections[collection_name] = {}; + database.add_collection(collection_name); testutil_check( session->create(session, collection_name.c_str(), DEFAULT_FRAMEWORK_SCHEMA)); ts = timestamp_manager->get_next_ts(); @@ -87,8 +87,7 @@ class database_operation { /* Keys must be unique. */ testutil_assert(key_count <= pow(10, key_size)); - for (const auto &it_collections : database.collections) { - collection_name = it_collections.first; + for (const auto &collection_name : database.get_collection_names()) { key_cpt = 0; /* * WiredTiger lets you open a cursor on a collection using the same pointer. When a diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h index 47f0453512d..63291ac9756 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h @@ -30,6 +30,7 @@ #define THREAD_CONTEXT_H #include "../core/throttle.h" +#include "../timestamp_manager.h" #include "database_model.h" #include "random_generator.h" #include "workload_tracking.h" @@ -153,7 +154,7 @@ class thread_context { transaction_context transaction; test_harness::timestamp_manager *timestamp_manager; test_harness::workload_tracking *tracking; - test_harness::database database; + test_harness::database &database; int64_t key_size = 0; int64_t value_size = 0; diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_validation.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_validation.h index aaab9ad25a9..7f583a66888 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_validation.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_validation.h @@ -120,11 +120,7 @@ class workload_validation { * The data model is now fully updated for the last read collection. It can be * checked. */ - check_reference(session, collection_name, database.collections.at(collection_name)); - /* Clear memory. */ - delete database.collections[collection_name].values; - database.collections[collection_name].values = nullptr; - + check_reference(session, collection_name, database); collection_name = key_collection_name; } }; @@ -139,12 +135,8 @@ class workload_validation { * empty if there is no collections to check after the end of the test (no collections * created or all deleted). */ - if (!collection_name.empty()) { - check_reference(session, collection_name, database.collections.at(collection_name)); - /* Clear memory. */ - delete database.collections[collection_name].values; - database.collections[collection_name].values = nullptr; - } + if (!collection_name.empty()) + check_reference(session, collection_name, database); } private: @@ -200,25 +192,17 @@ class workload_validation { * the key has been inserted previously in an existing collection and can be safely * deleted. */ - database.collections.at(collection_name).keys.at(key).exists = false; - delete database.collections.at(collection_name).values; - database.collections.at(collection_name).values = nullptr; + database.delete_record(collection_name, key); break; case tracking_operation::INSERT: { /* * Keys are unique, it is safe to assume the key has not been encountered before. */ - database.collections[collection_name].keys[key].exists = true; - if (database.collections[collection_name].values == nullptr) - database.collections[collection_name].values = new std::map<key_value_t, value_t>(); - value_t v; - v.value = key_value_t(value); - std::pair<key_value_t, value_t> pair(key_value_t(key), v); - database.collections[collection_name].values->insert(pair); + database.insert_record(collection_name, key, value); break; } case tracking_operation::UPDATE: - database.collections[collection_name].values->at(key).value = key_value_t(value); + database.update_record(collection_name, key, value); break; default: testutil_die(DEBUG_ERROR, "Unexpected operation in the tracking table: %d", @@ -232,8 +216,7 @@ class workload_validation { * representation in memory of the collection values and keys according to the tracking table. */ void - check_reference( - WT_SESSION *session, const std::string &collection_name, const collection_t &collection) + check_reference(WT_SESSION *session, const std::string &collection_name, database &database) { bool is_valid; key_t key; @@ -247,7 +230,7 @@ class workload_validation { collection_name.c_str()); /* Walk through each key/value pair of the current collection. */ - for (const auto &keys : collection.keys) { + for (const auto &keys : database.get_keys(collection_name)) { key_str = keys.first; key = keys.second; /* The key/value pair exists. */ @@ -263,12 +246,12 @@ class workload_validation { /* Check the associated value is valid. */ if (key.exists) { - testutil_assert(collection.values != nullptr); if (!verify_value(session, collection_name, key_str.c_str(), - collection.values->at(key_str).value)) + database.get_record(collection_name, key_str.c_str()).value)) testutil_die(DEBUG_ERROR, "check_reference: failed for key %s / value %s in collection %s.", - key_str.c_str(), collection.values->at(key_str).value.c_str(), + key_str.c_str(), + database.get_record(collection_name, key_str.c_str()).value.c_str(), collection_name.c_str()); } } diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_generator.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_generator.h index bc7b0c45c03..1980cf6ac6c 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_generator.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_generator.h @@ -46,9 +46,10 @@ namespace test_harness { class workload_generator : public component { public: workload_generator(configuration *configuration, database_operation *db_operation, - timestamp_manager *timestamp_manager, workload_tracking *tracking) - : component("workload_generator", configuration), _database_operation(db_operation), - _timestamp_manager(timestamp_manager), _tracking(tracking) + timestamp_manager *timestamp_manager, workload_tracking *tracking, database &database) + : component("workload_generator", configuration), _database(database), + _database_operation(db_operation), _timestamp_manager(timestamp_manager), + _tracking(tracking) { } @@ -123,7 +124,7 @@ class workload_generator : public component { } private: - database _database; + database &_database; database_operation *_database_operation; thread_manager _thread_manager; timestamp_manager *_timestamp_manager; |