From 49a1b239f122661789b8a0468cba33349f8aece3 Mon Sep 17 00:00:00 2001 From: Luke Chen Date: Mon, 6 Jun 2022 10:08:02 +1000 Subject: Import wiredtiger: b017f238da1ffd71a0623f6bc2e9eb2425887d44 from branch mongodb-master ref: dd171a05d5..b017f238da for: 6.1.0-rc0 WT-9400 Read at or before the stable timestamp in the CppSuite. --- src/third_party/wiredtiger/test/cppsuite/README.md | 1 + .../test/cppsuite/src/common/api_const.cpp | 92 +++++ .../test/cppsuite/src/common/api_const.h | 116 ++++++ .../wiredtiger/test/cppsuite/src/common/logger.cpp | 98 +++++ .../wiredtiger/test/cppsuite/src/common/logger.h | 74 ++++ .../test/cppsuite/src/common/random_generator.cpp | 115 ++++++ .../test/cppsuite/src/common/random_generator.h | 89 +++++ .../test/cppsuite/src/common/thread_manager.cpp | 60 ++++ .../test/cppsuite/src/common/thread_manager.h | 62 ++++ .../cppsuite/src/component/checkpoint_manager.cpp | 63 ++++ .../cppsuite/src/component/checkpoint_manager.h | 54 +++ .../test/cppsuite/src/component/component.cpp | 87 +++++ .../test/cppsuite/src/component/component.h | 95 +++++ .../test/cppsuite/src/component/op_tracker.cpp | 65 ++++ .../test/cppsuite/src/component/op_tracker.h | 61 ++++ .../test/cppsuite/src/component/perf_plotter.cpp | 66 ++++ .../test/cppsuite/src/component/perf_plotter.h | 62 ++++ .../cppsuite/src/component/runtime_monitor.cpp | 372 +++++++++++++++++++ .../test/cppsuite/src/component/runtime_monitor.h | 133 +++++++ .../cppsuite/src/component/timestamp_manager.cpp | 166 +++++++++ .../cppsuite/src/component/timestamp_manager.h | 94 +++++ .../cppsuite/src/component/workload_generator.cpp | 156 ++++++++ .../cppsuite/src/component/workload_generator.h | 91 +++++ .../cppsuite/src/component/workload_tracking.cpp | 229 ++++++++++++ .../cppsuite/src/component/workload_tracking.h | 99 +++++ .../cppsuite/src/component/workload_validation.cpp | 292 +++++++++++++++ .../cppsuite/src/component/workload_validation.h | 95 +++++ .../test/cppsuite/src/main/configuration.cpp | 311 ++++++++++++++++ .../test/cppsuite/src/main/configuration.h | 112 ++++++ .../test/cppsuite/src/main/database_model.cpp | 152 ++++++++ .../test/cppsuite/src/main/database_model.h | 118 ++++++ .../test/cppsuite/src/main/database_operation.cpp | 395 ++++++++++++++++++++ .../test/cppsuite/src/main/database_operation.h | 71 ++++ .../wiredtiger/test/cppsuite/src/main/test.cpp | 190 ++++++++++ .../wiredtiger/test/cppsuite/src/main/test.h | 99 +++++ .../test/cppsuite/src/main/thread_context.cpp | 349 ++++++++++++++++++ .../test/cppsuite/src/main/thread_context.h | 164 +++++++++ .../wiredtiger/test/cppsuite/src/main/throttle.cpp | 79 ++++ .../wiredtiger/test/cppsuite/src/main/throttle.h | 54 +++ .../cppsuite/src/storage/connection_manager.cpp | 103 ++++++ .../test/cppsuite/src/storage/connection_manager.h | 78 ++++ .../cppsuite/src/storage/scoped_connection.cpp | 45 +++ .../test/cppsuite/src/storage/scoped_connection.h | 56 +++ .../test/cppsuite/src/storage/scoped_types.cpp | 170 +++++++++ .../test/cppsuite/src/storage/scoped_types.h | 105 ++++++ .../cppsuite/test_harness/checkpoint_manager.cpp | 57 --- .../cppsuite/test_harness/checkpoint_manager.h | 53 --- .../cppsuite/test_harness/connection_manager.cpp | 100 ------ .../cppsuite/test_harness/connection_manager.h | 83 ----- .../test/cppsuite/test_harness/core/component.cpp | 85 ----- .../test/cppsuite/test_harness/core/component.h | 95 ----- .../cppsuite/test_harness/core/configuration.cpp | 305 ---------------- .../cppsuite/test_harness/core/configuration.h | 114 ------ .../test/cppsuite/test_harness/core/op_tracker.cpp | 37 -- .../test/cppsuite/test_harness/core/op_tracker.h | 58 --- .../test/cppsuite/test_harness/core/throttle.cpp | 75 ---- .../test/cppsuite/test_harness/core/throttle.h | 55 --- .../test/cppsuite/test_harness/runtime_monitor.cpp | 370 ------------------- .../test/cppsuite/test_harness/runtime_monitor.h | 141 -------- .../wiredtiger/test/cppsuite/test_harness/test.cpp | 198 ---------- .../wiredtiger/test/cppsuite/test_harness/test.h | 105 ------ .../test/cppsuite/test_harness/thread_manager.cpp | 59 --- .../test/cppsuite/test_harness/thread_manager.h | 62 ---- .../cppsuite/test_harness/timestamp_manager.cpp | 151 -------- .../test/cppsuite/test_harness/timestamp_manager.h | 85 ----- .../test/cppsuite/test_harness/util/api_const.cpp | 92 ----- .../test/cppsuite/test_harness/util/api_const.h | 116 ------ .../test/cppsuite/test_harness/util/logger.cpp | 96 ----- .../test/cppsuite/test_harness/util/logger.h | 82 ----- .../cppsuite/test_harness/util/perf_plotter.cpp | 65 ---- .../test/cppsuite/test_harness/util/perf_plotter.h | 58 --- .../test_harness/util/scoped_connection.cpp | 46 --- .../cppsuite/test_harness/util/scoped_connection.h | 56 --- .../cppsuite/test_harness/util/scoped_types.cpp | 169 --------- .../test/cppsuite/test_harness/util/scoped_types.h | 103 ------ .../test_harness/workload/database_model.cpp | 156 -------- .../test_harness/workload/database_model.h | 121 ------- .../test_harness/workload/database_operation.cpp | 397 --------------------- .../test_harness/workload/database_operation.h | 71 ---- .../test_harness/workload/random_generator.cpp | 114 ------ .../test_harness/workload/random_generator.h | 89 ----- .../test_harness/workload/thread_context.cpp | 351 ------------------ .../test_harness/workload/thread_context.h | 170 --------- .../test_harness/workload/workload_tracking.cpp | 229 ------------ .../test_harness/workload/workload_tracking.h | 99 ----- .../test_harness/workload/workload_validation.cpp | 291 --------------- .../test_harness/workload/workload_validation.h | 99 ----- .../cppsuite/test_harness/workload_generator.cpp | 163 --------- .../cppsuite/test_harness/workload_generator.h | 97 ----- .../test/cppsuite/tests/bounded_cursor_perf.cpp | 17 +- .../test/cppsuite/tests/burst_inserts.cpp | 5 +- .../test/cppsuite/tests/cache_resize.cpp | 20 +- .../cppsuite/tests/csuite_style_example_test.cpp | 13 +- .../test/cppsuite/tests/cursor_bound_01.cpp | 16 +- .../wiredtiger/test/cppsuite/tests/hs_cleanup.cpp | 5 +- .../test/cppsuite/tests/operations_test.cpp | 8 +- .../wiredtiger/test/cppsuite/tests/run.cpp | 10 +- .../test/cppsuite/tests/search_near_01.cpp | 25 +- .../test/cppsuite/tests/search_near_02.cpp | 20 +- .../test/cppsuite/tests/search_near_03.cpp | 18 +- .../test/cppsuite/tests/test_template.cpp | 24 +- src/third_party/wiredtiger/test/evergreen.yml | 3 + .../wiredtiger/test/evergreen/build_windows.ps1 | 3 +- .../wiredtiger/test/suite/test_cursor_bound01.py | 25 +- .../wiredtiger/test/suite/test_cursor_bound02.py | 294 +++++++++++++++ .../tests/test_reconciliation_tracking.cpp | 2 +- .../unittest/tests/wrappers/connection_wrapper.cpp | 15 +- .../unittest/tests/wrappers/connection_wrapper.h | 4 + .../wiredtiger/test/windows/windows_shim.h | 1 + 109 files changed, 6067 insertions(+), 5717 deletions(-) create mode 100644 src/third_party/wiredtiger/test/cppsuite/README.md create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/common/api_const.cpp create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/common/api_const.h create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/common/logger.cpp create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/common/logger.h create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/common/random_generator.cpp create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/common/random_generator.h create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/common/thread_manager.cpp create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/common/thread_manager.h create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/component/checkpoint_manager.cpp create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/component/checkpoint_manager.h create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/component/component.cpp create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/component/component.h create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/component/op_tracker.cpp create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/component/op_tracker.h create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/component/perf_plotter.cpp create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/component/perf_plotter.h create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/component/runtime_monitor.cpp create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/component/runtime_monitor.h create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/component/timestamp_manager.cpp create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/component/timestamp_manager.h create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/component/workload_generator.cpp create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/component/workload_generator.h create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/component/workload_tracking.cpp create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/component/workload_tracking.h create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/component/workload_validation.cpp create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/component/workload_validation.h create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/main/configuration.cpp create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/main/configuration.h create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/main/database_model.cpp create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/main/database_model.h create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/main/database_operation.cpp create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/main/database_operation.h create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/main/test.cpp create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/main/test.h create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/main/thread_context.cpp create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/main/thread_context.h create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/main/throttle.cpp create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/main/throttle.h create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/storage/connection_manager.cpp create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/storage/connection_manager.h create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/storage/scoped_connection.cpp create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/storage/scoped_connection.h create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/storage/scoped_types.cpp create mode 100644 src/third_party/wiredtiger/test/cppsuite/src/storage/scoped_types.h delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/checkpoint_manager.cpp delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/checkpoint_manager.h delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/connection_manager.cpp delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/connection_manager.h delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/core/component.cpp delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/core/component.h delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/core/configuration.cpp delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/core/configuration.h delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/core/op_tracker.cpp delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/core/op_tracker.h delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/core/throttle.cpp delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/core/throttle.h delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.cpp delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.h delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/test.cpp delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/test.h delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/thread_manager.cpp delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/thread_manager.h delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/timestamp_manager.cpp delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/timestamp_manager.h delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/util/api_const.cpp delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/util/api_const.h delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/util/logger.cpp delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/util/logger.h delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/util/perf_plotter.cpp delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/util/perf_plotter.h delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_connection.cpp delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_connection.h delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_types.cpp delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_types.h delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_model.cpp delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_model.h delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.cpp delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.h delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/workload/random_generator.cpp delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/workload/random_generator.h delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.cpp delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_tracking.cpp delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_tracking.h delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_validation.cpp delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_validation.h delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/workload_generator.cpp delete mode 100644 src/third_party/wiredtiger/test/cppsuite/test_harness/workload_generator.h create mode 100644 src/third_party/wiredtiger/test/suite/test_cursor_bound02.py (limited to 'src/third_party/wiredtiger/test') diff --git a/src/third_party/wiredtiger/test/cppsuite/README.md b/src/third_party/wiredtiger/test/cppsuite/README.md new file mode 100644 index 00000000000..26c95664ab3 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/README.md @@ -0,0 +1 @@ +# WiredTiger cppsuite diff --git a/src/third_party/wiredtiger/test/cppsuite/src/common/api_const.cpp b/src/third_party/wiredtiger/test/cppsuite/src/common/api_const.cpp new file mode 100644 index 00000000000..db313b93ebc --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/common/api_const.cpp @@ -0,0 +1,92 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "api_const.h" + +/* Define all constants related to WiredTiger APIs and testing. */ +namespace test_harness { + +/* Component names. */ +const std::string CHECKPOINT_MANAGER = "checkpoint_manager"; +const std::string RUNTIME_MONITOR = "runtime_monitor"; +const std::string TIMESTAMP_MANAGER = "timestamp_manager"; +const std::string WORKLOAD_GENERATOR = "workload_generator"; +const std::string WORKLOAD_TRACKING = "workload_tracking"; + +/* Configuration API consts. */ +const std::string CACHE_HS_INSERT = "cache_hs_insert"; +const std::string CACHE_MAX_WAIT_MS = "cache_max_wait_ms"; +const std::string CACHE_SIZE_MB = "cache_size_mb"; +const std::string CC_PAGES_REMOVED = "cc_pages_removed"; +const std::string COLLECTION_COUNT = "collection_count"; +const std::string COMPRESSION_ENABLED = "compression_enabled"; +const std::string CUSTOM_OP_CONFIG = "custom_config"; +const std::string DURATION_SECONDS = "duration_seconds"; +const std::string ENABLED = "enabled"; +const std::string ENABLE_LOGGING = "enable_logging"; +const std::string INSERT_OP_CONFIG = "insert_config"; +const std::string KEY_COUNT_PER_COLLECTION = "key_count_per_collection"; +const std::string KEY_SIZE = "key_size"; +const std::string LIMIT = "limit"; +const std::string MAX = "max"; +const std::string MIN = "min"; +const std::string OLDEST_LAG = "oldest_lag"; +const std::string OP_RATE = "op_rate"; +const std::string OPS_PER_TRANSACTION = "ops_per_transaction"; +const std::string POPULATE_CONFIG = "populate_config"; +const std::string POSTRUN_STATISTICS = "postrun"; +const std::string READ_OP_CONFIG = "read_config"; +const std::string REMOVE_OP_CONFIG = "remove_config"; +const std::string REVERSE_COLLATOR = "reverse_collator"; +const std::string RUNTIME_STATISTICS = "runtime"; +const std::string SAVE = "save"; +const std::string STABLE_LAG = "stable_lag"; +const std::string STAT_CACHE_SIZE = "stat_cache_size"; +const std::string STAT_DB_SIZE = "stat_db_size"; +const std::string STATISTICS_CONFIG = "statistics_config"; +const std::string THREAD_COUNT = "thread_count"; +const std::string TRACKING_KEY_FORMAT = "tracking_key_format"; +const std::string TRACKING_VALUE_FORMAT = "tracking_value_format"; +const std::string TYPE = "type"; +const std::string UPDATE_OP_CONFIG = "update_config"; +const std::string VALUE_SIZE = "value_size"; + +/* WiredTiger API consts. */ +const std::string COMMIT_TS = "commit_timestamp"; +const std::string CONNECTION_CREATE = "create"; +const std::string OLDEST_TS = "oldest_timestamp"; +const std::string STABLE_TS = "stable_timestamp"; +const std::string STATISTICS_LOG = "statistics_log=(json,wait=1)"; + +/* Test harness consts. */ +const std::string DEFAULT_FRAMEWORK_SCHEMA = "key_format=S,value_format=S,"; +const std::string TABLE_OPERATION_TRACKING = "table:operation_tracking"; +const std::string TABLE_SCHEMA_TRACKING = "table:schema_tracking"; +const std::string STATISTICS_URI = "statistics:"; + +} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/src/common/api_const.h b/src/third_party/wiredtiger/test/cppsuite/src/common/api_const.h new file mode 100644 index 00000000000..8c0038fdee5 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/common/api_const.h @@ -0,0 +1,116 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef API_CONST_H +#define API_CONST_H + +#include + +/* Define all constants related to WiredTiger APIs and testing. */ +namespace test_harness { + +/* Component names. */ +extern const std::string CHECKPOINT_MANAGER; +extern const std::string RUNTIME_MONITOR; +extern const std::string TIMESTAMP_MANAGER; +extern const std::string WORKLOAD_GENERATOR; +extern const std::string WORKLOAD_TRACKING; + +/* Configuration API consts. */ +extern const std::string CACHE_HS_INSERT; +extern const std::string CACHE_MAX_WAIT_MS; +extern const std::string CACHE_SIZE_MB; +extern const std::string CC_PAGES_REMOVED; +extern const std::string COLLECTION_COUNT; +extern const std::string COMPRESSION_ENABLED; +extern const std::string CUSTOM_OP_CONFIG; +extern const std::string DURATION_SECONDS; +extern const std::string ENABLED; +extern const std::string ENABLE_LOGGING; +extern const std::string INSERT_OP_CONFIG; +extern const std::string KEY_COUNT_PER_COLLECTION; +extern const std::string KEY_SIZE; +extern const std::string LIMIT; +extern const std::string MAX; +extern const std::string MIN; +extern const std::string OLDEST_LAG; +extern const std::string OP_RATE; +extern const std::string OPS_PER_TRANSACTION; +extern const std::string POPULATE_CONFIG; +extern const std::string POSTRUN_STATISTICS; +extern const std::string READ_OP_CONFIG; +extern const std::string REMOVE_OP_CONFIG; +extern const std::string REVERSE_COLLATOR; +extern const std::string RUNTIME_STATISTICS; +extern const std::string SAVE; +extern const std::string STABLE_LAG; +extern const std::string STAT_CACHE_SIZE; +extern const std::string STAT_DB_SIZE; +extern const std::string STATISTICS_CONFIG; +extern const std::string THREAD_COUNT; +extern const std::string TRACKING_KEY_FORMAT; +extern const std::string TRACKING_VALUE_FORMAT; +extern const std::string TYPE; +extern const std::string UPDATE_OP_CONFIG; +extern const std::string VALUE_SIZE; + +/* WiredTiger API consts. */ +extern const std::string COMMIT_TS; +extern const std::string CONNECTION_CREATE; +extern const std::string OLDEST_TS; +extern const std::string STABLE_TS; +extern const std::string STATISTICS_LOG; + +/* + * Use the Snappy compressor for stress testing to avoid excessive disk space usage. Our builds can + * pre-specify 'EXTSUBPATH' to indicate any special sub-directories the module is located. If unset + * we fallback to the '.libs' sub-directory used by autoconf. + */ +#define BLKCMP_PFX "block_compressor=" +#define SNAPPY_BLK BLKCMP_PFX "snappy" +#define EXTPATH "../../ext/" +#ifndef EXTSUBPATH +#define EXTSUBPATH ".libs/" +#endif + +/* Use reverse collator to test changes that deal different table sorting orders. */ +#define REVERSE_COL_CFG "collator=reverse" + +#define SNAPPY_PATH EXTPATH "compressors/snappy/" EXTSUBPATH "libwiredtiger_snappy.so" +#define REVERSE_COLLATOR_PATH \ + EXTPATH "collators/reverse/" EXTSUBPATH "libwiredtiger_reverse_collator.so" + +/* Test harness consts. */ +extern const std::string DEFAULT_FRAMEWORK_SCHEMA; +extern const std::string TABLE_OPERATION_TRACKING; +extern const std::string TABLE_SCHEMA_TRACKING; +extern const std::string STATISTICS_URI; + +} // namespace test_harness + +#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/src/common/logger.cpp b/src/third_party/wiredtiger/test/cppsuite/src/common/logger.cpp new file mode 100644 index 00000000000..4bb0e982a31 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/common/logger.cpp @@ -0,0 +1,98 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "logger.h" + +#include +#include +#include + +extern "C" { +#include "test_util.h" +} + +/* Define helpful functions related to debugging. */ +namespace test_harness { +/* Order of elements in this array corresponds to the definitions above. */ +const char *const LOG_LEVELS[] = {"ERROR", "WARN", "INFO", "TRACE"}; + +/* Mutex used by logger to synchronize printing. */ +static std::mutex _logger_mtx; + +/* Set default log level. */ +int64_t logger::trace_level = LOG_WARN; + +/* Include date in the logs by default. */ +bool logger::include_date = true; + +void +get_time(char *time_buf, size_t buf_size) +{ + size_t alloc_size; + struct tm *tm, _tm; + + /* Get time since epoch in nanoseconds. */ + uint64_t epoch_nanosec = std::chrono::high_resolution_clock::now().time_since_epoch().count(); + + /* Calculate time since epoch in seconds. */ + time_t time_epoch_sec = epoch_nanosec / WT_BILLION; + + tm = localtime_r(&time_epoch_sec, &_tm); + testutil_assert(tm != nullptr); + + alloc_size = + strftime(time_buf, buf_size, logger::include_date ? "[%Y-%m-%dT%H:%M:%S" : "[%H:%M:%S", tm); + + testutil_assert(alloc_size <= buf_size); + WT_IGNORE_RET(__wt_snprintf(&time_buf[alloc_size], buf_size - alloc_size, ".%" PRIu64 "Z]", + (uint64_t)epoch_nanosec % WT_BILLION)); +} + +/* Used to print out traces for debugging purpose. */ +void +logger::log_msg(int64_t trace_type, const std::string &str) +{ + if (logger::trace_level >= trace_type) { + testutil_assert( + trace_type >= LOG_ERROR && trace_type < sizeof(LOG_LEVELS) / sizeof(LOG_LEVELS[0])); + + char time_buf[64]; + get_time(time_buf, sizeof(time_buf)); + + std::ostringstream ss; + ss << time_buf << "[TID:" << std::this_thread::get_id() << "][" << LOG_LEVELS[trace_type] + << "]: " << str << std::endl; + + std::lock_guard lg(_logger_mtx); + if (trace_type == LOG_ERROR) + std::cerr << ss.str(); + else + std::cout << ss.str(); + } +} +} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/src/common/logger.h b/src/third_party/wiredtiger/test/cppsuite/src/common/logger.h new file mode 100644 index 00000000000..3284d82ee00 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/common/logger.h @@ -0,0 +1,74 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef LOGGER_H +#define LOGGER_H + +/* Following definitions are required in order to use printing format specifiers in C++. */ +#ifndef __STDC_LIMIT_MACROS +#define __STDC_LIMIT_MACROS +#endif +#ifndef __STDC_FORMAT_MACROS +#define __STDC_FORMAT_MACROS +#endif + +#include + +/* Define helpful functions related to debugging. */ +namespace test_harness { + +/* Possible log levels. If you change anything here make sure you update LOG_LEVELS accordingly. */ +#define LOG_ERROR 0 +#define LOG_WARN 1 +#define LOG_INFO 2 +/* + * The trace log level can incur a performance overhead since the current logging implementation + * requires per-line locking. + */ +#define LOG_TRACE 3 + +void get_time(char *time_buf, size_t buf_size); + +class logger { + public: + /* Current log level. Default is LOG_WARN. */ + static int64_t trace_level; + + /* Include date in the logs if enabled. Default is true. */ + static bool include_date; + + public: + /* Used to print out traces for debugging purpose. */ + static void log_msg(int64_t trace_type, const std::string &str); + + /* Make sure the class will not be instantiated. */ + logger() = delete; +}; +} // namespace test_harness + +#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/src/common/random_generator.cpp b/src/third_party/wiredtiger/test/cppsuite/src/common/random_generator.cpp new file mode 100644 index 00000000000..f391db0d180 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/common/random_generator.cpp @@ -0,0 +1,115 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "random_generator.h" + +#include + +extern "C" { +#include "test_util.h" +} + +namespace test_harness { +random_generator & +random_generator::instance() +{ + thread_local random_generator _instance; + return (_instance); +} + +std::string +random_generator::generate_random_string(std::size_t length, characters_type type) +{ + const std::string characters = get_characters(type); + std::string str; + + while (str.size() < length) + str += characters; + + std::shuffle(str.begin(), str.end(), _generator); + return (str.substr(0, length)); +} + +std::string +random_generator::generate_pseudo_random_string(std::size_t length, characters_type type) +{ + std::string random_string; + std::uniform_int_distribution<> &distribution = get_distribution(type); + std::size_t start_location = distribution(_generator); + const std::string &characters = get_characters(type); + + for (std::size_t i = 0; i < length; ++i) { + random_string += characters[start_location]; + if (start_location == characters.size() - 1) + start_location = 0; + else + start_location++; + } + return (random_string); +} + +random_generator::random_generator() +{ + _generator = std::mt19937(std::random_device{}()); + _alphanum_distrib = std::uniform_int_distribution<>(0, _pseudo_alphanum.size() - 1); + _alpha_distrib = std::uniform_int_distribution<>(0, _alphabet.size() - 1); +} + +std::uniform_int_distribution<> & +random_generator::get_distribution(characters_type type) +{ + switch (type) { + case characters_type::ALPHABET: + return (_alpha_distrib); + break; + case characters_type::PSEUDO_ALPHANUMERIC: + return (_alphanum_distrib); + break; + default: + testutil_die(type, "Unexpected characters_type"); + break; + } +} + +const std::string & +random_generator::get_characters(characters_type type) +{ + switch (type) { + case characters_type::ALPHABET: + return (_alphabet); + break; + case characters_type::PSEUDO_ALPHANUMERIC: + return (_pseudo_alphanum); + break; + default: + testutil_die(type, "Unexpected characters_type"); + break; + } +} + +} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/src/common/random_generator.h b/src/third_party/wiredtiger/test/cppsuite/src/common/random_generator.h new file mode 100644 index 00000000000..967d5566ce1 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/common/random_generator.h @@ -0,0 +1,89 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef RANDOM_GENERATOR_H +#define RANDOM_GENERATOR_H + +/* Following definitions are required in order to use printing format specifiers in C++. */ +#ifndef __STDC_LIMIT_MACROS +#define __STDC_LIMIT_MACROS +#endif +#ifndef __STDC_FORMAT_MACROS +#define __STDC_FORMAT_MACROS +#endif + +#include +#include + +namespace test_harness { +/* Helper class to generate random values using uniform distributions. */ + +enum characters_type { PSEUDO_ALPHANUMERIC, ALPHABET }; + +class random_generator { + public: + static random_generator &instance(); + + /* No copies of the singleton allowed. */ + random_generator(random_generator const &) = delete; + random_generator &operator=(random_generator const &) = delete; + + /* Generate a random string of a given length. */ + std::string generate_random_string( + std::size_t length, characters_type type = PSEUDO_ALPHANUMERIC); + + /* + * Generate a pseudo random string which compresses better. It should not be used to generate + * keys due to the limited randomness. + */ + std::string generate_pseudo_random_string( + std::size_t length, characters_type type = PSEUDO_ALPHANUMERIC); + + /* Generate a random integer between min and max. */ + template + T + generate_integer(T min, T max) + { + std::uniform_int_distribution dis(min, max); + return dis(_generator); + } + + private: + random_generator(); + std::uniform_int_distribution<> &get_distribution(characters_type type); + const std::string &get_characters(characters_type type); + + std::mt19937 _generator; + std::uniform_int_distribution<> _alphanum_distrib, _alpha_distrib; + const std::string _alphabet = "abcdefghijklmnopqrstuvwxyz"; + const std::string _pseudo_alphanum = + "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; +}; +} // namespace test_harness + +#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/src/common/thread_manager.cpp b/src/third_party/wiredtiger/test/cppsuite/src/common/thread_manager.cpp new file mode 100644 index 00000000000..e41516c196e --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/common/thread_manager.cpp @@ -0,0 +1,60 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "thread_manager.h" + +#include "logger.h" + +namespace test_harness { +thread_manager::~thread_manager() +{ + for (auto &it : _workers) { + if (it != nullptr && it->joinable()) { + logger::log_msg(LOG_ERROR, "You should've called join on the thread manager"); + it->join(); + } + delete it; + it = nullptr; + } + _workers.clear(); +} + +void +thread_manager::join() +{ + for (const auto &it : _workers) { + while (!it->joinable()) { + /* Helpful for diagnosing hangs. */ + logger::log_msg(LOG_TRACE, "Thread manager: Waiting to join."); + /* Check every so often to avoid spamming the log. */ + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + it->join(); + } +} +} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/src/common/thread_manager.h b/src/third_party/wiredtiger/test/cppsuite/src/common/thread_manager.h new file mode 100644 index 00000000000..0fce6bfeb45 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/common/thread_manager.h @@ -0,0 +1,62 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef THREAD_MANAGER_H +#define THREAD_MANAGER_H + +#include +#include + +namespace test_harness { +/* Class that handles threads, from their initialization to their deletion. */ +class thread_manager { + public: + ~thread_manager(); + + /* + * Generic function to create threads that call member function of classes. + */ + template + void + add_thread(Callable &&fct, Args &&... args) + { + std::thread *t = new std::thread(fct, std::forward(args)...); + _workers.push_back(t); + } + + /* + * Complete the operations for all threads. + */ + void join(); + + private: + std::vector _workers; +}; +} // namespace test_harness + +#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/src/component/checkpoint_manager.cpp b/src/third_party/wiredtiger/test/cppsuite/src/component/checkpoint_manager.cpp new file mode 100644 index 00000000000..ba8de32a03f --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/component/checkpoint_manager.cpp @@ -0,0 +1,63 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "checkpoint_manager.h" + +#include "src/common/api_const.h" +#include "src/common/logger.h" +#include "src/main/configuration.h" +#include "src/storage/connection_manager.h" + +extern "C" { +#include "test_util.h" +} + +namespace test_harness { +checkpoint_manager::checkpoint_manager(configuration *configuration) + : component(CHECKPOINT_MANAGER, configuration) +{ +} + +void +checkpoint_manager::load() +{ + /* Load the general component things. */ + component::load(); + + /* Create session that we'll use for checkpointing. */ + if (_enabled) + _session = connection_manager::instance().create_session(); +} + +void +checkpoint_manager::do_work() +{ + logger::log_msg(LOG_INFO, "Running checkpoint"); + testutil_check(_session->checkpoint(_session.get(), nullptr)); +} +} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/src/component/checkpoint_manager.h b/src/third_party/wiredtiger/test/cppsuite/src/component/checkpoint_manager.h new file mode 100644 index 00000000000..a570c8d215e --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/component/checkpoint_manager.h @@ -0,0 +1,54 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef CHECKPOINT_MANAGER_H +#define CHECKPOINT_MANAGER_H + +#include "component.h" + +#include "src/storage/scoped_types.h" + +namespace test_harness { +class checkpoint_manager : public component { + public: + explicit checkpoint_manager(configuration *configuration); + virtual ~checkpoint_manager() = default; + + /* Delete the copy constructor and the assignment operator. */ + checkpoint_manager(const checkpoint_manager &) = delete; + checkpoint_manager &operator=(const checkpoint_manager &) = delete; + + void load() override final; + void do_work() override final; + + private: + scoped_session _session; +}; +} // namespace test_harness + +#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/src/component/component.cpp b/src/third_party/wiredtiger/test/cppsuite/src/component/component.cpp new file mode 100644 index 00000000000..f2966db8d22 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/component/component.cpp @@ -0,0 +1,87 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "component.h" + +#include "src/common/api_const.h" +#include "src/common/logger.h" + +namespace test_harness { +component::component(const std::string &name, configuration *config) : _config(config), _name(name) +{ +} + +component::~component() +{ + delete _config; +} + +void +component::load() +{ + logger::log_msg(LOG_INFO, "Loading component: " + _name); + _enabled = _config->get_optional_bool(ENABLED, true); + _throttle = throttle(_config); + /* If we're not enabled we shouldn't be running. */ + _running = _enabled; +} + +void +component::run() +{ + logger::log_msg(LOG_INFO, "Running component: " + _name); + while (_enabled && _running) { + do_work(); + _throttle.sleep(); + } +} + +void +component::do_work() +{ + /* Not implemented. */ +} + +bool +component::enabled() const +{ + return (_enabled); +} + +void +component::end_run() +{ + _running = false; +} + +void +component::finish() +{ + logger::log_msg(LOG_INFO, "Running finish stage of component: " + _name); +} +} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/src/component/component.h b/src/third_party/wiredtiger/test/cppsuite/src/component/component.h new file mode 100644 index 00000000000..effbd259310 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/component/component.h @@ -0,0 +1,95 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef COMPONENT_H +#define COMPONENT_H + +#include + +#include "src/main/configuration.h" +#include "src/main/throttle.h" + +namespace test_harness { +/* + * A component is a class that defines 4 unique stages in its life-cycle, the stages must be run in + * the following order: load, run, end_run, finish. + */ +class component { + public: + explicit component(const std::string &name, configuration *config); + virtual ~component(); + + /* Delete the copy constructor and the assignment operator. */ + component(const component &) = delete; + component &operator=(const component &) = delete; + + /* + * The load function should perform all tasks required to setup the component for the main phase + * of the test. An example operation performed in the load phase would be populating a database. + */ + virtual void load(); + + /* + * The run function provides a top level loop that calls the do_work function every X seconds as + * defined by the throttle. Each run() method defined by the components is called in its own + * thread by the top level test class. + * + * If a component does not wish to use the standard run function, it can be overloaded. + */ + virtual void run(); + + /* end_run informs the component that is no longer running which closes out its run loop. */ + void end_run(); + + /* + * do_work is called every X seconds as defined by the throttle. Generally most components + * should do their "operation" in the do_work function. + */ + virtual void do_work(); + + /* Gets the value of the _enabled variable. */ + bool enabled() const; + + /* + * The finish phase is a cleanup phase. Created objects are destroyed here and any final testing + * requirements can be performed in this phase. An example could be the verification of the + * database, or checking some relevant statistics. + */ + virtual void finish(); + + protected: + bool _enabled = false; + volatile bool _running = false; + throttle _throttle; + configuration *_config; + + private: + std::string _name; +}; +} // namespace test_harness +#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/src/component/op_tracker.cpp b/src/third_party/wiredtiger/test/cppsuite/src/component/op_tracker.cpp new file mode 100644 index 00000000000..f40cfc22479 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/component/op_tracker.cpp @@ -0,0 +1,65 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "op_tracker.h" + +#include "perf_plotter.h" + +namespace test_harness { +op_tracker::op_tracker(const std::string id, const std::string &test_name) + : _id(id), _test_name(test_name), _it_count(0), _total_time_taken(0) +{ +} + +void +op_tracker::append_stats() +{ + uint64_t avg = (uint64_t)_total_time_taken / _it_count; + std::string stat = "{\"name\":\"" + _id + "\",\"value\":" + std::to_string(avg) + "}"; + perf_plotter::instance().add_stat(stat); +} + +template +auto +op_tracker::track(T lambda) +{ + auto _start_time = std::chrono::steady_clock::now(); + int ret = lambda(); + auto _end_time = std::chrono::steady_clock::now(); + _total_time_taken += (_end_time - _start_time).count(); + _it_count += 1; + + return ret; +} + +op_tracker::~op_tracker() +{ + if (_it_count != 0) + append_stats(); +} +}; // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/src/component/op_tracker.h b/src/third_party/wiredtiger/test/cppsuite/src/component/op_tracker.h new file mode 100644 index 00000000000..f9a3c3c5698 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/component/op_tracker.h @@ -0,0 +1,61 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef OP_TRACKER_H +#define OP_TRACKER_H + +#include + +namespace test_harness { + +/* + * Class that tracks the performance of given operations and appends the stats to the perf file. + */ +class op_tracker { + public: + explicit op_tracker(const std::string id, const std::string &test_name); + virtual ~op_tracker(); + + /* Calculates the average time and appends the stat to the perf file. */ + void append_stats(); + + /* + * Does timing for a given operation and keeps track of how many operations have been executed + * as well as total time taken. + */ + template auto track(T lambda); + + private: + std::string _id; + std::string _test_name; + int _it_count; + uint64_t _total_time_taken; +}; +} // namespace test_harness + +#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/src/component/perf_plotter.cpp b/src/third_party/wiredtiger/test/cppsuite/src/component/perf_plotter.cpp new file mode 100644 index 00000000000..780dcb4fff9 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/component/perf_plotter.cpp @@ -0,0 +1,66 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "perf_plotter.h" + +namespace test_harness { +void +perf_plotter::add_stat(const std::string &stat_string) +{ + std::lock_guard lg(_stat_mutex); + _stats.push_back(stat_string); +} + +void +perf_plotter::output_perf_file(const std::string &test_name) +{ + std::ofstream perf_file; + std::string stat_info = "[{\"info\":{\"test_name\": \"" + test_name + "\"},\"metrics\": ["; + + perf_file.open(test_name + ".json"); + + for (const auto &stat : _stats) + stat_info += stat + ","; + + /* Remove last extra comma. */ + if (stat_info.back() == ',') + stat_info.pop_back(); + + perf_file << stat_info << "]}]"; + perf_file.close(); +} + +perf_plotter & +perf_plotter::instance() +{ + static perf_plotter _instance; + return (_instance); +} + +perf_plotter::perf_plotter() {} +} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/src/component/perf_plotter.h b/src/third_party/wiredtiger/test/cppsuite/src/component/perf_plotter.h new file mode 100644 index 00000000000..3710f0d1ddf --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/component/perf_plotter.h @@ -0,0 +1,62 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef PERF_PLOTTER_H +#define PERF_PLOTTER_H + +#include +#include +#include +#include + +namespace test_harness { +/* + * Singleton class owning the perf plot json, provides access to add statistics, and a central call + * point to output the file. + */ +class perf_plotter { + public: + static perf_plotter &instance(); + + public: + /* No copies of the singleton allowed. */ + perf_plotter(perf_plotter const &) = delete; + perf_plotter &operator=(perf_plotter const &) = delete; + + /* Anyone can get the perf_plotter and add a statistic to it. */ + void add_stat(const std::string &stat_string); + void output_perf_file(const std::string &test_name); + + private: + perf_plotter(); + std::vector _stats; + std::mutex _stat_mutex; +}; +} // namespace test_harness + +#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/src/component/runtime_monitor.cpp b/src/third_party/wiredtiger/test/cppsuite/src/component/runtime_monitor.cpp new file mode 100644 index 00000000000..1dce6df6c16 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/component/runtime_monitor.cpp @@ -0,0 +1,372 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "runtime_monitor.h" + +#include + +#include "src/common/api_const.h" +#include "src/common/logger.h" +#include "src/component/perf_plotter.h" +#include "src/storage/connection_manager.h" + +extern "C" { +#include "test_util.h" +} + +namespace test_harness { + +/* Static methods implementation. */ +static std::string +collection_name_to_file_name(const std::string &collection_name) +{ + /* Strip out the URI prefix. */ + const size_t colon_pos = collection_name.find(':'); + testutil_assert(colon_pos != std::string::npos); + const auto stripped_name = collection_name.substr(colon_pos + 1); + + /* Now add the directory and file extension. */ + return (std::string(DEFAULT_DIR) + "/" + stripped_name + ".wt"); +} + +/* Inline methods implementation. */ + +/* + * The WiredTiger configuration API doesn't accept string statistic names when retrieving statistic + * values. This function provides the required mapping to statistic id. We should consider + * generating it programmatically in `stat.py` to avoid having to manually add a condition every + * time we want to observe a new postrun statistic. + */ +inline int +get_stat_field(const std::string &name) +{ + if (name == CACHE_HS_INSERT) + return (WT_STAT_CONN_CACHE_HS_INSERT); + else if (name == CC_PAGES_REMOVED) + return (WT_STAT_CONN_CC_PAGES_REMOVED); + testutil_die(EINVAL, "get_stat_field: Stat \"%s\" is unrecognized", name.c_str()); +} + +/* statistics class implementation */ +statistics::statistics(configuration &config, const std::string &stat_name, int stat_field) + : field(stat_field), max(config.get_int(MAX)), min(config.get_int(MIN)), name(stat_name), + postrun(config.get_bool(POSTRUN_STATISTICS)), runtime(config.get_bool(RUNTIME_STATISTICS)), + save(config.get_bool(SAVE)) +{ +} + +void +statistics::check(scoped_cursor &cursor) +{ + int64_t stat_value; + runtime_monitor::get_stat(cursor, field, &stat_value); + if (stat_value < min || stat_value > max) { + const std::string error_string = "runtime_monitor: Postrun stat \"" + name + + "\" was outside of the specified limits. Min=" + std::to_string(min) + + " Max=" + std::to_string(max) + " Actual=" + std::to_string(stat_value); + testutil_die(-1, error_string.c_str()); + } else + logger::log_msg(LOG_TRACE, name + " usage: " + std::to_string(stat_value)); +} + +std::string +statistics::get_value_str(scoped_cursor &cursor) +{ + int64_t stat_value; + runtime_monitor::get_stat(cursor, field, &stat_value); + return std::to_string(stat_value); +} + +int +statistics::get_field() const +{ + return field; +} + +int64_t +statistics::get_max() const +{ + return max; +} + +int64_t +statistics::get_min() const +{ + return min; +} + +const std::string & +statistics::get_name() const +{ + return name; +} + +bool +statistics::get_postrun() const +{ + return postrun; +} + +bool +statistics::get_runtime() const +{ + return runtime; +} + +bool +statistics::get_save() const +{ + return save; +} + +/* cache_limit_statistic class implementation */ +cache_limit_statistic::cache_limit_statistic(configuration &config, const std::string &name) + : statistics(config, name, -1) +{ +} + +void +cache_limit_statistic::check(scoped_cursor &cursor) +{ + double use_percent = get_cache_value(cursor); + if (use_percent > max) { + const std::string error_string = + "runtime_monitor: Cache usage exceeded during test! Limit: " + std::to_string(max) + + " usage: " + std::to_string(use_percent); + testutil_die(-1, error_string.c_str()); + } else + logger::log_msg(LOG_TRACE, name + " usage: " + std::to_string(use_percent)); +} + +std::string +cache_limit_statistic::get_value_str(scoped_cursor &cursor) +{ + return std::to_string(get_cache_value(cursor)); +} + +double +cache_limit_statistic::get_cache_value(scoped_cursor &cursor) +{ + int64_t cache_bytes_image, cache_bytes_other, cache_bytes_max; + double use_percent; + /* Three statistics are required to compute cache use percentage. */ + runtime_monitor::get_stat(cursor, WT_STAT_CONN_CACHE_BYTES_IMAGE, &cache_bytes_image); + runtime_monitor::get_stat(cursor, WT_STAT_CONN_CACHE_BYTES_OTHER, &cache_bytes_other); + runtime_monitor::get_stat(cursor, WT_STAT_CONN_CACHE_BYTES_MAX, &cache_bytes_max); + /* + * Assert that we never exceed our configured limit for cache usage. Add 0.0 to avoid floating + * point conversion errors. + */ + testutil_assert(cache_bytes_max > 0); + use_percent = ((cache_bytes_image + cache_bytes_other + 0.0) / cache_bytes_max) * 100; + return use_percent; +} + +/* db_size_statistic class implementation */ +db_size_statistic::db_size_statistic( + configuration &config, const std::string &name, database &database) + : statistics(config, name, -1), _database(database) +{ +#ifdef _WIN32 + Logger::log_msg("Database size checking is not implemented on Windows", LOG_ERROR); +#endif +} + +void +db_size_statistic::check(scoped_cursor &) +{ +#ifndef _WIN32 + const auto file_names = get_file_names(); + size_t db_size = get_db_size(); + logger::log_msg(LOG_TRACE, "Current database size is " + std::to_string(db_size) + " bytes"); + + if (db_size > max) { + const std::string error_string = + "runtime_monitor: Database size limit exceeded during test! Limit: " + + std::to_string(max) + " db size: " + std::to_string(db_size); + testutil_die(-1, error_string.c_str()); + } +#endif +} + +std::string +db_size_statistic::get_value_str(scoped_cursor &) +{ + return std::to_string(get_db_size()); +} + +size_t +db_size_statistic::get_db_size() const +{ + const auto file_names = get_file_names(); + size_t db_size = 0; + + for (const auto &name : file_names) { + struct stat sb; + if (stat(name.c_str(), &sb) == 0) { + db_size += sb.st_size; + logger::log_msg(LOG_TRACE, name + " was " + std::to_string(sb.st_size) + " bytes"); + } else + /* The only good reason for this to fail is if the file hasn't been created yet. */ + testutil_assert(errno == ENOENT); + } + + return db_size; +} + +const std::vector +db_size_statistic::get_file_names() const +{ + std::vector file_names; + for (const auto &name : _database.get_collection_names()) + file_names.push_back(collection_name_to_file_name(name)); + + /* Add WiredTiger internal tables. */ + file_names.push_back(std::string(DEFAULT_DIR) + "/" + WT_HS_FILE); + file_names.push_back(std::string(DEFAULT_DIR) + "/" + WT_METAFILE); + + return (file_names); +} + +/* runtime_monitor class implementation */ +void +runtime_monitor::get_stat(scoped_cursor &cursor, int stat_field, int64_t *valuep) +{ + const char *desc, *pvalue; + cursor->set_key(cursor.get(), stat_field); + testutil_check(cursor->search(cursor.get())); + testutil_check(cursor->get_value(cursor.get(), &desc, &pvalue, valuep)); + testutil_check(cursor->reset(cursor.get())); +} + +runtime_monitor::runtime_monitor( + const std::string &test_name, configuration *config, database &database) + : component(RUNTIME_MONITOR, config), _test_name(test_name), _database(database) +{ +} + +void +runtime_monitor::load() +{ + /* Load the general component things. */ + component::load(); + + /* If the component is enabled, load all the known statistics. */ + if (_enabled) { + + std::unique_ptr stat_config(_config->get_subconfig(STAT_CACHE_SIZE)); + _stats.push_back(std::unique_ptr( + new cache_limit_statistic(*stat_config, STAT_CACHE_SIZE))); + + stat_config.reset(_config->get_subconfig(STAT_DB_SIZE)); + _stats.push_back(std::unique_ptr( + new db_size_statistic(*stat_config, STAT_DB_SIZE, _database))); + + stat_config.reset(_config->get_subconfig(CACHE_HS_INSERT)); + _stats.push_back(std::unique_ptr( + new statistics(*stat_config, CACHE_HS_INSERT, get_stat_field(CACHE_HS_INSERT)))); + + stat_config.reset(_config->get_subconfig(CC_PAGES_REMOVED)); + _stats.push_back(std::unique_ptr( + new statistics(*stat_config, CC_PAGES_REMOVED, get_stat_field(CC_PAGES_REMOVED)))); + + /* Open our statistic cursor. */ + _session = connection_manager::instance().create_session(); + _cursor = _session.open_scoped_cursor(STATISTICS_URI); + } +} + +void +runtime_monitor::do_work() +{ + /* Check runtime statistics. */ + for (const auto &stat : _stats) { + if (stat->get_runtime()) + stat->check(_cursor); + } +} + +void +runtime_monitor::finish() +{ + component::finish(); + + /* Append stats to perf plotter. */ + append_stats(); + + /* Check the post run statistics now. */ + bool success = true; + int64_t stat_max, stat_min, stat_value; + std::string stat_name; + + for (const auto &stat : _stats) { + + if (!stat->get_postrun()) + continue; + + stat_max = stat->get_max(); + stat_min = stat->get_min(); + stat_name = stat->get_name(); + + stat_value = std::stoi(stat->get_value_str(_cursor)); + + if (stat_value < stat_min || stat_value > stat_max) { + const std::string error_string = "runtime_monitor: Postrun stat \"" + stat_name + + "\" was outside of the specified limits. Min=" + std::to_string(stat_min) + + " Max=" + std::to_string(stat_max) + " Actual=" + std::to_string(stat_value); + logger::log_msg(LOG_ERROR, error_string); + success = false; + } + + logger::log_msg(LOG_INFO, + "runtime_monitor: Final value of stat " + stat_name + + " is: " + std::to_string(stat_value)); + } + + if (!success) + testutil_die(-1, + "runtime_monitor: One or more postrun statistics were outside of their specified " + "limits."); +} + +/* + * This function appends the values of the different statistics that need to be saved as indicated + * by the configuration file to the perf plotter. + */ +void +runtime_monitor::append_stats() +{ + for (const auto &stat : _stats) { + if (stat->get_save()) { + auto stat_str = "{\"name\":\"" + stat->get_name() + + "\",\"value\":" + stat->get_value_str(_cursor) + "}"; + perf_plotter::instance().add_stat(stat_str); + } + } +} +} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/src/component/runtime_monitor.h b/src/third_party/wiredtiger/test/cppsuite/src/component/runtime_monitor.h new file mode 100644 index 00000000000..383401aefeb --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/component/runtime_monitor.h @@ -0,0 +1,133 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef RUNTIME_MONITOR_H +#define RUNTIME_MONITOR_H + +#include + +#include "src/main/configuration.h" +#include "src/main/database_model.h" +#include "src/storage/scoped_types.h" + +namespace test_harness { + +class statistics { + public: + statistics() = default; + explicit statistics(configuration &config, const std::string &stat_name, int stat_field); + virtual ~statistics() = default; + + /* Check that the statistics are within bounds. */ + virtual void check(scoped_cursor &cursor); + + /* Retrieve the value associated to the stat in a string format. */ + virtual std::string get_value_str(scoped_cursor &cursor); + + /* Getters. */ + int get_field() const; + int64_t get_max() const; + int64_t get_min() const; + const std::string &get_name() const; + bool get_postrun() const; + bool get_runtime() const; + bool get_save() const; + + protected: + int field; + int64_t max; + int64_t min; + std::string name; + bool postrun; + bool runtime; + bool save; +}; + +class cache_limit_statistic : public statistics { + public: + explicit cache_limit_statistic(configuration &config, const std::string &name); + virtual ~cache_limit_statistic() = default; + + void check(scoped_cursor &cursor) override final; + std::string get_value_str(scoped_cursor &cursor) override final; + + private: + double get_cache_value(scoped_cursor &cursor); +}; + +class db_size_statistic : public statistics { + public: + explicit db_size_statistic(configuration &config, const std::string &name, database &database); + virtual ~db_size_statistic() = default; + + /* Don't need the stat cursor for these. */ + void check(scoped_cursor &) override final; + std::string get_value_str(scoped_cursor &) override final; + + private: + size_t get_db_size() const; + const std::vector get_file_names() const; + + private: + database &_database; +}; + +/* + * The runtime monitor class is designed to track various statistics or other runtime signals + * relevant to the given workload. + */ +class runtime_monitor : public component { + public: + static void get_stat(scoped_cursor &, int, int64_t *); + + public: + explicit runtime_monitor( + const std::string &test_name, configuration *config, database &database); + virtual ~runtime_monitor() = default; + + /* Delete the copy constructor and the assignment operator. */ + runtime_monitor(const runtime_monitor &) = delete; + runtime_monitor &operator=(const runtime_monitor &) = delete; + + void load() override final; + void do_work() override final; + void finish() override final; + + private: + void append_stats(); + + private: + scoped_session _session; + scoped_cursor _cursor; + const std::string _test_name; + std::vector> _stats; + database &_database; +}; +} // namespace test_harness + +#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/src/component/timestamp_manager.cpp b/src/third_party/wiredtiger/test/cppsuite/src/component/timestamp_manager.cpp new file mode 100644 index 00000000000..865dad819fa --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/component/timestamp_manager.cpp @@ -0,0 +1,166 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "timestamp_manager.h" + +#include + +#include "src/common/api_const.h" +#include "src/common/logger.h" +#include "src/common/random_generator.h" +#include "src/storage/connection_manager.h" + +namespace test_harness { +const std::string +timestamp_manager::decimal_to_hex(uint64_t value) +{ + std::stringstream ss; + ss << std::hex << value; + std::string res(ss.str()); + return (res); +} + +timestamp_manager::timestamp_manager(configuration *config) : component("timestamp_manager", config) +{ +} + +void +timestamp_manager::load() +{ + component::load(); + int64_t oldest_lag = _config->get_int(OLDEST_LAG); + testutil_assert(oldest_lag >= 0); + /* Cast and then shift left to match the seconds position. */ + _oldest_lag = oldest_lag; + _oldest_lag <<= 32; + + int64_t stable_lag = _config->get_int(STABLE_LAG); + testutil_assert(stable_lag >= 0); + /* Cast and then shift left to match the seconds position. */ + _stable_lag = stable_lag; + _stable_lag <<= 32; +} + +void +timestamp_manager::do_work() +{ + std::string config, log_msg; + /* latest_ts_s represents the time component of the latest timestamp provided. */ + wt_timestamp_t latest_ts_s = get_time_now_s(); + + /* + * Keep a time window between the latest and stable ts less than the max defined in the + * configuration. + */ + wt_timestamp_t new_stable_ts = _stable_ts; + testutil_assert(latest_ts_s >= _stable_ts); + if ((latest_ts_s - _stable_ts) > _stable_lag) { + log_msg = "Timestamp_manager: Stable timestamp expired."; + new_stable_ts = latest_ts_s - _stable_lag; + config += STABLE_TS + "=" + decimal_to_hex(new_stable_ts); + } + + /* + * Keep a time window between the stable and oldest ts less than the max defined in the + * configuration. + */ + wt_timestamp_t new_oldest_ts = _oldest_ts; + testutil_assert(_stable_ts >= _oldest_ts); + if ((new_stable_ts - _oldest_ts) > _oldest_lag) { + if (log_msg.empty()) + log_msg = "Timestamp_manager: Oldest timestamp expired."; + else + log_msg += " Oldest timestamp expired."; + new_oldest_ts = new_stable_ts - _oldest_lag; + if (!config.empty()) + config += ","; + config += OLDEST_TS + "=" + decimal_to_hex(new_oldest_ts); + } + + if (!log_msg.empty()) + logger::log_msg(LOG_TRACE, log_msg); + + /* + * Save the new timestamps. Any timestamps that we're viewing from another thread should be set + * AFTER we've saved the new timestamps to avoid races where we sweep data that is not yet + * obsolete. + */ + if (!config.empty()) { + connection_manager::instance().set_timestamp(config); + _oldest_ts = new_oldest_ts; + _stable_ts = new_stable_ts; + } +} + +wt_timestamp_t +timestamp_manager::get_next_ts() +{ + uint64_t current_time = get_time_now_s(); + uint64_t increment = _increment_ts.fetch_add(1); + current_time |= (increment & 0x00000000FFFFFFFF); + return (current_time); +} + +wt_timestamp_t +timestamp_manager::get_oldest_ts() const +{ + return (_oldest_ts); +} + +wt_timestamp_t +timestamp_manager::get_valid_read_ts() const +{ + /* Use get_oldest_ts here to convert from atomic to wt_timestamp_t. */ + wt_timestamp_t current_oldest = get_oldest_ts(); + wt_timestamp_t current_stable = _stable_ts; + if (current_stable > current_oldest) { + --current_stable; + } + /* + * Assert that our stable and oldest match if 0 or that the stable is greater than or equal to + * the oldest. Ensuring that the oldest is never greater than the stable. + */ + testutil_assert( + (current_stable == 0 && current_oldest == 0) || current_stable >= current_oldest); + /* + * Its okay to return a timestamp less than a concurrently updated oldest timestamp as all + * readers should be reading with timestamp rounding. + */ + return random_generator::instance().generate_integer( + current_oldest, current_stable); +} + +uint64_t +timestamp_manager::get_time_now_s() const +{ + auto now = std::chrono::system_clock::now().time_since_epoch(); + uint64_t current_time_s = + static_cast(std::chrono::duration_cast(now).count()) << 32; + return (current_time_s); +} +} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/src/component/timestamp_manager.h b/src/third_party/wiredtiger/test/cppsuite/src/component/timestamp_manager.h new file mode 100644 index 00000000000..55fa4a3cc41 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/component/timestamp_manager.h @@ -0,0 +1,94 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef TIMESTAMP_MANAGER_H +#define TIMESTAMP_MANAGER_H + +#include + +#include "component.h" + +extern "C" { +#include "test_util.h" +} + +namespace test_harness { +/* + * The timestamp monitor class manages global timestamp state for all components in the test + * harness. It also manages the global timestamps within WiredTiger. + * + * The format of a timestamp is as follows, the first 32 bits represent the epoch time in seconds. + * The last 32 bits represent an increment for uniqueness. + */ +class timestamp_manager : public component { + public: + static const std::string decimal_to_hex(uint64_t value); + + public: + explicit timestamp_manager(configuration *config); + virtual ~timestamp_manager() = default; + + void load() override final; + void do_work() override final; + + /* Get a unique timestamp. */ + wt_timestamp_t get_next_ts(); + + /* Get oldest timestamp. */ + wt_timestamp_t get_oldest_ts() const; + + /* + * Generate a timestamp between the oldest timestamp and the stable timestamp. + * + * WiredTiger will abort commit transactions that attempt to commit behind an active read + * timestamp in order to preserve repeatable reads. Currently the cppsuite doesn't handle that + * well, so to avoid this issue we will read behind the stable timestamp. + * + * This timestamp isn't guaranteed to provide a repeatable read as the oldest could move + * concurrently removing the previously seen data. + */ + wt_timestamp_t get_valid_read_ts() const; + + private: + /* Get the current time in seconds, bit shifted to the expected location. */ + uint64_t get_time_now_s() const; + + private: + std::atomic _increment_ts{0}; + /* The tracking table sweep needs to read the oldest timestamp. */ + std::atomic _oldest_ts{0U}; + wt_timestamp_t _stable_ts = 0U; + /* + * _oldest_lag is the time window between the stable and oldest timestamps. + * _stable_lag is the time window between the latest and stable timestamps. + */ + uint64_t _oldest_lag = 0U, _stable_lag = 0U; +}; +} // namespace test_harness + +#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/src/component/workload_generator.cpp b/src/third_party/wiredtiger/test/cppsuite/src/component/workload_generator.cpp new file mode 100644 index 00000000000..e643a576a55 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/component/workload_generator.cpp @@ -0,0 +1,156 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "workload_generator.h" + +#include "src/common/api_const.h" +#include "src/common/logger.h" +#include "src/storage/connection_manager.h" + +namespace test_harness { +/* operation_config class implementation */ +operation_config::operation_config(configuration *config, thread_type type) + : config(config), type(type), thread_count(config->get_int(THREAD_COUNT)) +{ +} + +std::function +operation_config::get_func(database_operation *dbo) +{ + switch (type) { + case thread_type::CUSTOM: + return (std::bind(&database_operation::custom_operation, dbo, std::placeholders::_1)); + case thread_type::INSERT: + return (std::bind(&database_operation::insert_operation, dbo, std::placeholders::_1)); + case thread_type::READ: + return (std::bind(&database_operation::read_operation, dbo, std::placeholders::_1)); + case thread_type::REMOVE: + return (std::bind(&database_operation::remove_operation, dbo, std::placeholders::_1)); + case thread_type::UPDATE: + return (std::bind(&database_operation::update_operation, dbo, std::placeholders::_1)); + default: + /* This may cause a separate testutil_die in type_string but that should be okay. */ + testutil_die(EINVAL, "unexpected thread_type: %s", type_string(type).c_str()); + } +} + +/* workload_generator class implementation */ +workload_generator::workload_generator(configuration *configuration, + database_operation *db_operation, timestamp_manager *timestamp_manager, database &database) + : component(WORKLOAD_GENERATOR, configuration), _database(database), + _database_operation(db_operation), _timestamp_manager(timestamp_manager) +{ +} + +workload_generator::~workload_generator() +{ + for (auto &it : _workers) + delete it; +} + +void +workload_generator::set_workload_tracking(workload_tracking *tracking) +{ + testutil_assert(_tracking == nullptr); + _tracking = tracking; +} + +void +workload_generator::run() +{ + configuration *populate_config; + std::vector operation_configs; + uint64_t thread_id = 0; + + /* Retrieve useful parameters from the test configuration. */ + operation_configs.push_back( + operation_config(_config->get_subconfig(CUSTOM_OP_CONFIG), thread_type::CUSTOM)); + operation_configs.push_back( + operation_config(_config->get_subconfig(INSERT_OP_CONFIG), thread_type::INSERT)); + operation_configs.push_back( + operation_config(_config->get_subconfig(READ_OP_CONFIG), thread_type::READ)); + operation_configs.push_back( + operation_config(_config->get_subconfig(REMOVE_OP_CONFIG), thread_type::REMOVE)); + operation_configs.push_back( + operation_config(_config->get_subconfig(UPDATE_OP_CONFIG), thread_type::UPDATE)); + populate_config = _config->get_subconfig(POPULATE_CONFIG); + + /* Populate the database. */ + _database_operation->populate(_database, _timestamp_manager, populate_config, _tracking); + _db_populated = true; + delete populate_config; + + /* Generate threads to execute read operations on the collections. */ + for (auto &it : operation_configs) { + if (it.thread_count != 0) + logger::log_msg(LOG_INFO, + "Workload_generator: Creating " + std::to_string(it.thread_count) + " " + + type_string(it.type) + " threads."); + for (size_t i = 0; i < it.thread_count && _running; ++i) { + thread_context *tc = new thread_context(thread_id++, it.type, it.config, + connection_manager::instance().create_session(), _timestamp_manager, _tracking, + _database); + _workers.push_back(tc); + _thread_manager.add_thread(it.get_func(_database_operation), tc); + } + /* + * Don't forget to delete the config we created earlier. While we do pass the config into + * the thread context it is not saved, so we are safe to do this. + */ + delete it.config; + + /* + * Reset the thread_id counter to 0 as we're only interested in knowing per operation type + * which thread we are. + */ + thread_id = 0; + } +} + +void +workload_generator::finish() +{ + component::finish(); + for (const auto &it : _workers) + it->finish(); + _thread_manager.join(); + logger::log_msg(LOG_TRACE, "Workload generator: run stage done"); +} + +database & +workload_generator::get_database() +{ + return (_database); +} + +bool +workload_generator::db_populated() const +{ + return (_db_populated); +} +} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/src/component/workload_generator.h b/src/third_party/wiredtiger/test/cppsuite/src/component/workload_generator.h new file mode 100644 index 00000000000..53a33311bfc --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/component/workload_generator.h @@ -0,0 +1,91 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef WORKLOAD_GENERATOR_H +#define WORKLOAD_GENERATOR_H + +#include + +#include "src/common/thread_manager.h" +#include "src/main/configuration.h" +#include "src/main/database_operation.h" +#include "src/main/thread_context.h" + +namespace test_harness { +/* + * Helper class to enable scalable operation types in the database_operation. + */ +class operation_config { + public: + explicit operation_config(configuration *config, thread_type type); + + /* Returns a function pointer to the member function of the supplied database operation. */ + std::function get_func(database_operation *dbo); + + public: + configuration *config; + const thread_type type; + const int64_t thread_count; +}; + +/* + * Class that can execute operations based on a given configuration. + */ +class workload_generator : public component { + public: + explicit workload_generator(configuration *configuration, database_operation *db_operation, + timestamp_manager *timestamp_manager, database &database); + + ~workload_generator(); + + /* Delete the copy constructor and the assignment operator. */ + workload_generator(const workload_generator &) = delete; + workload_generator &operator=(const workload_generator &) = delete; + + /* Do the work of the main part of the workload. */ + void run() override final; + void finish() override final; + + database &get_database(); + bool db_populated() const; + + /* Set the tracking component. */ + void set_workload_tracking(workload_tracking *tracking); + + private: + database &_database; + database_operation *_database_operation = nullptr; + thread_manager _thread_manager; + timestamp_manager *_timestamp_manager = nullptr; + workload_tracking *_tracking = nullptr; + std::vector _workers; + bool _db_populated = false; +}; +} // namespace test_harness + +#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/src/component/workload_tracking.cpp b/src/third_party/wiredtiger/test/cppsuite/src/component/workload_tracking.cpp new file mode 100644 index 00000000000..668ce176291 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/component/workload_tracking.cpp @@ -0,0 +1,229 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "workload_tracking.h" + +#include "src/common/api_const.h" +#include "src/common/logger.h" +#include "src/storage/connection_manager.h" + +namespace test_harness { +workload_tracking::workload_tracking( + configuration *_config, const bool use_compression, timestamp_manager &tsm) + : component(WORKLOAD_TRACKING, _config), _operation_table_name(TABLE_OPERATION_TRACKING), + _schema_table_config(SCHEMA_TRACKING_TABLE_CONFIG), _schema_table_name(TABLE_SCHEMA_TRACKING), + _use_compression(use_compression), _tsm(tsm) +{ + _operation_table_config = "key_format=" + _config->get_string(TRACKING_KEY_FORMAT) + + ",value_format=" + _config->get_string(TRACKING_VALUE_FORMAT) + ",log=(enabled=true)"; +} + +const std::string & +workload_tracking::get_schema_table_name() const +{ + return (_schema_table_name); +} + +const std::string & +workload_tracking::get_operation_table_name() const +{ + return (_operation_table_name); +} + +void +workload_tracking::load() +{ + component::load(); + + if (!_enabled) + return; + + /* Initiate schema tracking. */ + _session = connection_manager::instance().create_session(); + testutil_check( + _session->create(_session.get(), _schema_table_name.c_str(), _schema_table_config.c_str())); + _schema_track_cursor = _session.open_scoped_cursor(_schema_table_name); + logger::log_msg(LOG_TRACE, "Schema tracking initiated"); + + /* Initiate operations tracking. */ + testutil_check(_session->create( + _session.get(), _operation_table_name.c_str(), _operation_table_config.c_str())); + logger::log_msg(LOG_TRACE, "Operations tracking created"); + + /* + * Open sweep cursor in a dedicated sweep session. This cursor will be used to clear out + * obsolete data from the tracking table. + */ + _sweep_session = connection_manager::instance().create_session(); + _sweep_cursor = _sweep_session.open_scoped_cursor(_operation_table_name); + logger::log_msg(LOG_TRACE, "Tracking table sweep initialized"); +} + +void +workload_tracking::do_work() +{ + WT_DECL_RET; + wt_timestamp_t ts, oldest_ts; + uint64_t collection_id, sweep_collection_id; + int op_type; + const char *key, *value; + char *sweep_key; + bool globally_visible_update_found; + + /* + * This function prunes old data from the tracking table as the default validation logic doesn't + * use it. User-defined validation may need this data, so don't allow it to be removed. + */ + const std::string key_format(_sweep_cursor->key_format); + const std::string value_format(_sweep_cursor->value_format); + if (key_format != OPERATION_TRACKING_KEY_FORMAT || + value_format != OPERATION_TRACKING_VALUE_FORMAT) + return; + + key = sweep_key = nullptr; + globally_visible_update_found = false; + + /* Take a copy of the oldest so that we sweep with a consistent timestamp. */ + oldest_ts = _tsm.get_oldest_ts(); + + /* We need to check if the component is still running to avoid unnecessary iterations. */ + while (_running && (ret = _sweep_cursor->prev(_sweep_cursor.get())) == 0) { + testutil_check(_sweep_cursor->get_key(_sweep_cursor.get(), &collection_id, &key, &ts)); + testutil_check(_sweep_cursor->get_value(_sweep_cursor.get(), &op_type, &value)); + /* + * If we're on a new key, reset the check. We want to track whether we have a globally + * visible update for the current key. + */ + if (sweep_key == nullptr || sweep_collection_id != collection_id || + strcmp(sweep_key, key) != 0) { + globally_visible_update_found = false; + if (sweep_key != nullptr) + free(sweep_key); + sweep_key = static_cast(dstrdup(key)); + sweep_collection_id = collection_id; + } + if (ts <= oldest_ts) { + if (globally_visible_update_found) { + if (logger::trace_level == LOG_TRACE) + logger::log_msg(LOG_TRACE, + std::string("workload tracking: Obsoleted update, key=") + sweep_key + + ", collection_id=" + std::to_string(collection_id) + + ", timestamp=" + std::to_string(ts) + + ", oldest_timestamp=" + std::to_string(oldest_ts) + ", value=" + value); + /* + * Wrap the removal in a transaction as we need to specify we aren't using a + * timestamp on purpose. + */ + testutil_check( + _sweep_session->begin_transaction(_sweep_session.get(), "no_timestamp=true")); + testutil_check(_sweep_cursor->remove(_sweep_cursor.get())); + testutil_check(_sweep_session->commit_transaction(_sweep_session.get(), nullptr)); + } else if (static_cast(op_type) == tracking_operation::INSERT) { + if (logger::trace_level == LOG_TRACE) + logger::log_msg(LOG_TRACE, + std::string("workload tracking: Found globally visible update, key=") + + sweep_key + ", collection_id=" + std::to_string(collection_id) + + ", timestamp=" + std::to_string(ts) + + ", oldest_timestamp=" + std::to_string(oldest_ts) + ", value=" + value); + globally_visible_update_found = true; + } + } + } + + free(sweep_key); + + /* + * If we get here and the test is still running, it means we must have reached the end of the + * table. We can also get here because the test is no longer running. In this case, the cursor + * can either be at the end of the table or still on a valid entry since we interrupted the + * work. + */ + if (ret != 0 && ret != WT_NOTFOUND) + testutil_die(LOG_ERROR, + "Tracking table sweep failed: cursor->next() returned an unexpected error %d.", ret); + + /* If we have a position, give it up. */ + testutil_check(_sweep_cursor->reset(_sweep_cursor.get())); +} + +void +workload_tracking::save_schema_operation( + const tracking_operation &operation, const uint64_t &collection_id, wt_timestamp_t ts) +{ + std::string error_message; + + if (!_enabled) + return; + + if (operation == tracking_operation::CREATE_COLLECTION || + operation == tracking_operation::DELETE_COLLECTION) { + _schema_track_cursor->set_key(_schema_track_cursor.get(), collection_id, ts); + _schema_track_cursor->set_value(_schema_track_cursor.get(), static_cast(operation)); + testutil_check(_schema_track_cursor->insert(_schema_track_cursor.get())); + } else { + error_message = + "save_schema_operation: invalid operation " + std::to_string(static_cast(operation)); + testutil_die(EINVAL, error_message.c_str()); + } +} + +int +workload_tracking::save_operation(const uint64_t txn_id, const tracking_operation &operation, + const uint64_t &collection_id, const std::string &key, const std::string &value, + wt_timestamp_t ts, scoped_cursor &op_track_cursor) +{ + WT_DECL_RET; + + if (!_enabled) + return (0); + + testutil_assert(op_track_cursor.get() != nullptr); + + if (operation == tracking_operation::CREATE_COLLECTION || + operation == tracking_operation::DELETE_COLLECTION) { + const std::string error_message = + "save_operation: invalid operation " + std::to_string(static_cast(operation)); + testutil_die(EINVAL, error_message.c_str()); + } else { + set_tracking_cursor(txn_id, operation, collection_id, key, value, ts, op_track_cursor); + ret = op_track_cursor->insert(op_track_cursor.get()); + } + return (ret); +} + +/* Note that the transaction id is not used in the default implementation of the tracking table. */ +void +workload_tracking::set_tracking_cursor(const uint64_t txn_id, const tracking_operation &operation, + const uint64_t &collection_id, const std::string &key, const std::string &value, + wt_timestamp_t ts, scoped_cursor &op_track_cursor) +{ + op_track_cursor->set_key(op_track_cursor.get(), collection_id, key.c_str(), ts); + op_track_cursor->set_value(op_track_cursor.get(), static_cast(operation), value.c_str()); +} + +} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/src/component/workload_tracking.h b/src/third_party/wiredtiger/test/cppsuite/src/component/workload_tracking.h new file mode 100644 index 00000000000..640cb294f0c --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/component/workload_tracking.h @@ -0,0 +1,99 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef WORKLOAD_TRACKING_H +#define WORKLOAD_TRACKING_H + +#include "component.h" +#include "src/storage/scoped_types.h" +#include "timestamp_manager.h" + +/* + * Default schema for tracking operations on collections (key_format: Collection id / Key / + * Timestamp, value_format: Operation type / Value) + */ +#define OPERATION_TRACKING_KEY_FORMAT WT_UNCHECKED_STRING(QSQ) +#define OPERATION_TRACKING_VALUE_FORMAT WT_UNCHECKED_STRING(iS) + +/* + * Default schema for tracking schema operations on collections (key_format: Collection id / + * Timestamp, value_format: Operation type) + */ +#define SCHEMA_TRACKING_KEY_FORMAT WT_UNCHECKED_STRING(QQ) +#define SCHEMA_TRACKING_VALUE_FORMAT WT_UNCHECKED_STRING(i) +#define SCHEMA_TRACKING_TABLE_CONFIG \ + "key_format=" SCHEMA_TRACKING_KEY_FORMAT ",value_format=" SCHEMA_TRACKING_VALUE_FORMAT \ + ",log=(enabled=true)" + +namespace test_harness { +/* Tracking operations. */ +enum class tracking_operation { CREATE_COLLECTION, CUSTOM, DELETE_COLLECTION, DELETE_KEY, INSERT }; + +/* Class used to track operations performed on collections */ +class workload_tracking : public component { + public: + workload_tracking(configuration *_config, const bool use_compression, timestamp_manager &tsm); + virtual ~workload_tracking() = default; + + const std::string &get_schema_table_name() const; + const std::string &get_operation_table_name() const; + void load() override final; + + /* + * As every operation is tracked in the tracking table we need to clear out obsolete operations + * otherwise the file size grow continuously, as such we cleanup operations that are no longer + * relevant, i.e. older than the oldest timestamp. + */ + void do_work() override final; + + void save_schema_operation( + const tracking_operation &operation, const uint64_t &collection_id, wt_timestamp_t ts); + + virtual void set_tracking_cursor(const uint64_t txn_id, const tracking_operation &operation, + const uint64_t &collection_id, const std::string &key, const std::string &value, + wt_timestamp_t ts, scoped_cursor &op_track_cursor); + + int save_operation(const uint64_t txn_id, const tracking_operation &operation, + const uint64_t &collection_id, const std::string &key, const std::string &value, + wt_timestamp_t ts, scoped_cursor &op_track_cursor); + + private: + scoped_session _session; + scoped_session _sweep_session; + scoped_cursor _schema_track_cursor; + scoped_cursor _sweep_cursor; + std::string _operation_table_config; + const std::string _operation_table_name; + const std::string _schema_table_config; + const std::string _schema_table_name; + const bool _use_compression; + timestamp_manager &_tsm; +}; +} // namespace test_harness + +#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/src/component/workload_validation.cpp b/src/third_party/wiredtiger/test/cppsuite/src/component/workload_validation.cpp new file mode 100644 index 00000000000..b10958be297 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/component/workload_validation.cpp @@ -0,0 +1,292 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "workload_validation.h" + +#include + +#include "src/common/logger.h" +#include "src/storage/connection_manager.h" + +namespace test_harness { +void +workload_validation::validate(const std::string &operation_table_name, + const std::string &schema_table_name, const std::vector &known_collection_ids) +{ + WT_DECL_RET; + wt_timestamp_t tracked_timestamp; + std::vector created_collections, deleted_collections; + uint64_t tracked_collection_id; + const char *tracked_key, *tracked_value; + int tracked_op_type; + uint64_t current_collection_id = 0; + + logger::log_msg(LOG_INFO, "Beginning validation."); + + scoped_session session = connection_manager::instance().create_session(); + scoped_cursor cursor = session.open_scoped_cursor(operation_table_name); + + /* + * Default validation depends on specific fields being present in the tracking table. If the + * tracking table schema has been modified the user must define their own validation. + */ + const std::string key_format(cursor->key_format); + const std::string value_format(cursor->value_format); + if (key_format != OPERATION_TRACKING_KEY_FORMAT || + value_format != OPERATION_TRACKING_VALUE_FORMAT) { + testutil_die(EINVAL, + "Attempting to perform default validation on a test with a user-defined tracking " + "table. Please define validation for your test"); + } + + /* Retrieve the collections that were created and deleted during the test. */ + parse_schema_tracking_table( + session, schema_table_name, created_collections, deleted_collections); + + /* + * Make sure the deleted collections do not exist on disk. The created collections are checked + * in check_reference. + */ + for (auto const &it : deleted_collections) { + if (!verify_collection_file_state(session, it, false)) + testutil_die(LOG_ERROR, + "Validation failed: collection %s present on disk while it has been tracked as " + "deleted.", + database::build_collection_name(it).c_str()); + } + + /* + * All collections in memory should match those created in the schema tracking table. Dropping + * is currently not supported. + */ + std::sort(created_collections.begin(), created_collections.end()); + auto on_disk_collection_id = created_collections.begin(); + if (created_collections.size() != known_collection_ids.size()) + testutil_die(LOG_ERROR, + "Validation failed: collection state mismatch, expected %lu" + " collections to exist but have %lu on disk", + created_collections.size(), known_collection_ids.size()); + for (const auto id : known_collection_ids) { + if (id != *on_disk_collection_id) + testutil_die(LOG_ERROR, + "Validation failed: collection state mismatch expected " + "collection id %lu but got %lu.", + id, *on_disk_collection_id); + on_disk_collection_id++; + } + + /* Parse the tracking table. */ + validation_collection current_collection_records; + while ((ret = cursor->next(cursor.get())) == 0) { + testutil_check( + cursor->get_key(cursor.get(), &tracked_collection_id, &tracked_key, &tracked_timestamp)); + testutil_check(cursor->get_value(cursor.get(), &tracked_op_type, &tracked_value)); + + logger::log_msg(LOG_TRACE, + "Retrieved tracked values. \n Collection id: " + std::to_string(tracked_collection_id) + + "\n Key: " + std::string(tracked_key) + + "\n Timestamp: " + std::to_string(tracked_timestamp) + "\n Operation type: " + + std::to_string(tracked_op_type) + "\n Value: " + std::string(tracked_value)); + + /* + * Check if we've stepped over to the next collection. The tracking table is sorted by + * collection_id so this is correct. + */ + if (tracked_collection_id != current_collection_id) { + if (std::find(known_collection_ids.begin(), known_collection_ids.end(), + tracked_collection_id) == known_collection_ids.end()) + testutil_die(LOG_ERROR, + "Validation failed: The collection id %lu is not part of the known " + "collection set.", + tracked_collection_id); + if (tracked_collection_id < current_collection_id) + testutil_die(LOG_ERROR, "Validation failed: The collection id %lu is out of order.", + tracked_collection_id); + + /* + * Given that we've stepped over to the next collection we've built a full picture of + * the current collection and can now validate it. + */ + verify_collection(session, current_collection_id, current_collection_records); + + /* Begin processing the next collection. */ + current_collection_id = tracked_collection_id; + current_collection_records.clear(); + } + + /* + * Add the values from the tracking table to the current collection model. + */ + update_data_model(static_cast(tracked_op_type), + current_collection_records, current_collection_id, tracked_key, tracked_value); + }; + + /* The value of ret should be WT_NOTFOUND once the cursor has read all rows. */ + if (ret != WT_NOTFOUND) + testutil_die( + LOG_ERROR, "Validation failed: cursor->next() return an unexpected error %d.", ret); + + /* + * We still need to validate the last collection. But we can also end up here if there aren't + * any collections, check for that. + */ + if (known_collection_ids.size() != 0) + verify_collection(session, current_collection_id, current_collection_records); +} + +void +workload_validation::parse_schema_tracking_table(scoped_session &session, + const std::string &tracking_table_name, std::vector &created_collections, + std::vector &deleted_collections) +{ + wt_timestamp_t key_timestamp; + uint64_t key_collection_id; + int value_operation_type; + + scoped_cursor cursor = session.open_scoped_cursor(tracking_table_name); + + while (cursor->next(cursor.get()) == 0) { + testutil_check(cursor->get_key(cursor.get(), &key_collection_id, &key_timestamp)); + testutil_check(cursor->get_value(cursor.get(), &value_operation_type)); + + logger::log_msg(LOG_TRACE, "Collection id is " + std::to_string(key_collection_id)); + logger::log_msg(LOG_TRACE, "Timestamp is " + std::to_string(key_timestamp)); + logger::log_msg(LOG_TRACE, "Operation type is " + std::to_string(value_operation_type)); + + if (static_cast(value_operation_type) == + tracking_operation::CREATE_COLLECTION) { + deleted_collections.erase(std::remove(deleted_collections.begin(), + deleted_collections.end(), key_collection_id), + deleted_collections.end()); + created_collections.push_back(key_collection_id); + } else if (static_cast(value_operation_type) == + tracking_operation::DELETE_COLLECTION) { + created_collections.erase(std::remove(created_collections.begin(), + created_collections.end(), key_collection_id), + created_collections.end()); + deleted_collections.push_back(key_collection_id); + } + } +} + +void +workload_validation::update_data_model(const tracking_operation &operation, + validation_collection &collection, const uint64_t collection_id, const char *key, + const char *value) +{ + if (operation == tracking_operation::DELETE_KEY) { + /* Search for the key validating that it exists. */ + const auto it = collection.find(key); + if (it == collection.end()) + testutil_die(LOG_ERROR, + "Validation failed: key deleted that doesn't exist. Collection id: %lu Key: %s", + collection_id, key); + else if (it->second.exists == false) + /* The key has been deleted twice. */ + testutil_die(LOG_ERROR, + "Validation failed: deleted key deleted again. Collection id: %lu Key: %s", + collection_id, it->first.c_str()); + + /* Update the key_state to deleted. */ + it->second.exists = false; + } else if (operation == tracking_operation::INSERT) + collection[key_value_t(key)] = key_state{true, key_value_t(value)}; + else + testutil_die(LOG_ERROR, "Validation failed: unexpected operation in the tracking table: %d", + static_cast(operation)); +} + +void +workload_validation::verify_collection( + scoped_session &session, const uint64_t collection_id, validation_collection &collection) +{ + /* Check the collection exists on disk. */ + if (!verify_collection_file_state(session, collection_id, true)) + testutil_die(LOG_ERROR, + "Validation failed: collection %lu not present on disk while it has been tracked as " + "created.", + collection_id); + + /* Walk through each key/value pair of the current collection. */ + for (const auto &record : collection) + verify_key_value(session, collection_id, record.first, record.second); +} + +bool +workload_validation::verify_collection_file_state( + scoped_session &session, const uint64_t collection_id, bool exists) const +{ + /* + * We don't necessarily expect to successfully open the cursor so don't create a scoped cursor. + */ + WT_CURSOR *cursor; + int ret = session->open_cursor(session.get(), + database::build_collection_name(collection_id).c_str(), nullptr, nullptr, &cursor); + if (ret == 0) + testutil_check(cursor->close(cursor)); + return (exists ? (ret == 0) : (ret != 0)); +} + +void +workload_validation::verify_key_value(scoped_session &session, const uint64_t collection_id, + const std::string &key, const key_state &key_state) +{ + WT_DECL_RET; + const char *retrieved_value; + + scoped_cursor cursor = + session.open_scoped_cursor(database::build_collection_name(collection_id)); + cursor->set_key(cursor.get(), key.c_str()); + ret = cursor->search(cursor.get()); + testutil_assertfmt(ret == 0 || ret == WT_NOTFOUND, + "Validation failed: Unexpected error returned %d while searching for a key. Key: %s, " + "Collection_id: %lu", + ret, key.c_str(), collection_id); + if (ret == WT_NOTFOUND && key_state.exists) + testutil_die(LOG_ERROR, + "Validation failed: Search failed to find key that should exist. Key: %s, " + "Collection_id: %lu", + key.c_str(), collection_id); + else if (ret == 0 && key_state.exists == false) { + testutil_die(LOG_ERROR, + "Validation failed: Key exists when it is expected to be deleted. Key: %s, " + "Collection_id: %lu", + key.c_str(), collection_id); + } + + if (key_state.exists == false) + return; + + testutil_check(cursor->get_value(cursor.get(), &retrieved_value)); + if (key_state.value != key_value_t(retrieved_value)) + testutil_die(LOG_ERROR, + "Validation failed: Value mismatch for key. Key: %s, Collection_id: %lu, Expected " + "value: %s, Found value: %s", + key.c_str(), collection_id, key_state.value.c_str(), retrieved_value); +} +} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/src/component/workload_validation.h b/src/third_party/wiredtiger/test/cppsuite/src/component/workload_validation.h new file mode 100644 index 00000000000..ecb8f660bc3 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/component/workload_validation.h @@ -0,0 +1,95 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef WORKLOAD_VALIDATION_H +#define WORKLOAD_VALIDATION_H + +#include +#include +#include + +#include "src/main/database_model.h" + +namespace test_harness { +struct key_state { + key_state() = default; + key_state(bool exists, const key_value_t &value) : exists(exists), value(value) {} + bool exists = false; + key_value_t value; +}; +typedef std::map validation_collection; +/* + * Class that can validate database state and collection data. + */ +class workload_validation { + public: + /* + * Validate the on disk data against what has been tracked during the test. This is done by + * replaying the tracked operations so a representation in memory of the collections is created. + * This representation is then compared to what is on disk. + * + * - operation_table_name: Table that contains all the operations performed on keys. + * - schema_table_name: Table that contains all the schema operations performed. + */ + void validate(const std::string &operation_table_name, const std::string &schema_table_name, + const std::vector &known_collection_ids); + + private: + /* + * Read the tracking table to retrieve the created and deleted collections during the test. + * collection_name: collection that contains the operations on the different collections during + * the test. + */ + void parse_schema_tracking_table(scoped_session &session, + const std::string &tracking_table_name, std::vector &created_collections, + std::vector &deleted_collections); + + /* Update the data model. */ + void update_data_model(const tracking_operation &operation, validation_collection &collection, + const uint64_t collection_id, const char *key, const char *value); + + /* + * Compare the tracked operations against what has been saved on disk. + */ + void verify_collection( + scoped_session &session, const uint64_t collection_id, validation_collection &collection); + + /* + * Check whether a collection exists on disk. exists: needs to be set to true if the collection + * is expected to be existing, false otherwise. + */ + bool verify_collection_file_state( + scoped_session &session, const uint64_t collection_id, bool exists) const; + + /* Verify the given expected value is the same on disk. */ + void verify_key_value(scoped_session &session, const uint64_t collection_id, + const std::string &key, const key_state &key_state); +}; +} // namespace test_harness + +#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/src/main/configuration.cpp b/src/third_party/wiredtiger/test/cppsuite/src/main/configuration.cpp new file mode 100644 index 00000000000..3e00cb7450b --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/main/configuration.cpp @@ -0,0 +1,311 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "configuration.h" + +#include +#include + +#include "src/common/logger.h" + +extern "C" { +#include "test_util.h" +} + +namespace test_harness { +/* Static methods implementation. */ +static bool +config_item_to_bool(const WT_CONFIG_ITEM item) +{ + return (item.val != 0); +} + +static int64_t +config_item_to_int(const WT_CONFIG_ITEM item) +{ + return (item.val); +} + +static std::string +config_item_to_string(const WT_CONFIG_ITEM item) +{ + return std::string(item.str, item.len); +} + +static std::vector +config_item_to_list(const WT_CONFIG_ITEM item) +{ + auto str = config_item_to_string(item); + + /* Get rid of the brackets. */ + testutil_assert(!str.empty() && str.front() == '[' && str.back() == ']'); + str.pop_back(); + str.erase(0, 1); + + return (split_string(str, ',')); +} + +/* configuration class implementation. */ +configuration::configuration(const std::string &test_config_name, const std::string &config) +{ + const auto *config_entry = __wt_test_config_match(test_config_name.c_str()); + if (config_entry == nullptr) + testutil_die(EINVAL, "failed to match test config name"); + std::string default_config = std::string(config_entry->base); + /* Merge in the default configuration. */ + _config = merge_default_config(default_config, config); + logger::log_msg(LOG_INFO, "Full config: " + _config); + + int ret = + wiredtiger_test_config_validate(nullptr, nullptr, test_config_name.c_str(), _config.c_str()); + if (ret != 0) + testutil_die(EINVAL, "failed to validate given config, ensure test config exists"); + ret = wiredtiger_config_parser_open(nullptr, _config.c_str(), _config.size(), &_config_parser); + if (ret != 0) + testutil_die(EINVAL, "failed to create configuration parser for provided config"); +} + +configuration::configuration(const WT_CONFIG_ITEM &nested) +{ + if (nested.type != WT_CONFIG_ITEM::WT_CONFIG_ITEM_STRUCT) + testutil_die(EINVAL, "provided config item isn't a structure"); + int ret = wiredtiger_config_parser_open(nullptr, nested.str, nested.len, &_config_parser); + if (ret != 0) + testutil_die(EINVAL, "failed to create configuration parser for provided sub config"); +} + +configuration::~configuration() +{ + if (_config_parser != nullptr) { + _config_parser->close(_config_parser); + _config_parser = nullptr; + } +} + +std::string +configuration::get_string(const std::string &key) +{ + return get(key, false, types::STRING, "", config_item_to_string); +} + +std::string +configuration::get_optional_string(const std::string &key, const std::string &def) +{ + return get(key, true, types::STRING, def, config_item_to_string); +} + +bool +configuration::get_bool(const std::string &key) +{ + return get(key, false, types::BOOL, false, config_item_to_bool); +} + +bool +configuration::get_optional_bool(const std::string &key, const bool def) +{ + return get(key, true, types::BOOL, def, config_item_to_bool); +} + +int64_t +configuration::get_int(const std::string &key) +{ + return get(key, false, types::INT, 0, config_item_to_int); +} + +int64_t +configuration::get_optional_int(const std::string &key, const int64_t def) +{ + return get(key, true, types::INT, def, config_item_to_int); +} + +configuration * +configuration::get_subconfig(const std::string &key) +{ + return get(key, false, types::STRUCT, nullptr, + [](WT_CONFIG_ITEM item) { return new configuration(item); }); +} + +configuration * +configuration::get_optional_subconfig(const std::string &key) +{ + return get(key, true, types::STRUCT, nullptr, + [](WT_CONFIG_ITEM item) { return new configuration(item); }); +} + +std::vector +configuration::get_list(const std::string &key) +{ + return get>(key, false, types::LIST, {}, config_item_to_list); +} + +template +T +configuration::get( + const std::string &key, bool optional, types type, T def, T (*func)(WT_CONFIG_ITEM item)) +{ + WT_DECL_RET; + WT_CONFIG_ITEM value = {"", 0, 1, WT_CONFIG_ITEM::WT_CONFIG_ITEM_BOOL}; + + ret = _config_parser->get(_config_parser, key.c_str(), &value); + if (ret == WT_NOTFOUND && optional) + return (def); + else if (ret != 0) + testutil_die(ret, ("Error while finding config with key \"" + key + "\"").c_str()); + + const char *error_msg = "Configuration value doesn't match requested type"; + if (type == types::STRING && + (value.type != WT_CONFIG_ITEM::WT_CONFIG_ITEM_STRING && + value.type != WT_CONFIG_ITEM::WT_CONFIG_ITEM_ID)) + testutil_die(-1, error_msg); + else if (type == types::BOOL && value.type != WT_CONFIG_ITEM::WT_CONFIG_ITEM_BOOL) + testutil_die(-1, error_msg); + else if (type == types::INT && value.type != WT_CONFIG_ITEM::WT_CONFIG_ITEM_NUM) + testutil_die(-1, error_msg); + else if (type == types::STRUCT && value.type != WT_CONFIG_ITEM::WT_CONFIG_ITEM_STRUCT) + testutil_die(-1, error_msg); + else if (type == types::LIST && value.type != WT_CONFIG_ITEM::WT_CONFIG_ITEM_STRUCT) + testutil_die(-1, error_msg); + + return func(value); +} + +std::string +configuration::merge_default_config( + const std::string &default_config, const std::string &user_config) +{ + std::string merged_config; + auto split_default_config = split_config(default_config); + auto split_user_config = split_config(user_config); + auto user_it = split_user_config.begin(); + for (auto default_it = split_default_config.begin(); default_it != split_default_config.end(); + ++default_it) { + if (user_it == split_user_config.end() || user_it->first != default_it->first) + /* The default does not exist in the user configuration, add it. */ + merged_config += default_it->first + "=" + default_it->second; + else { + /* If we have a sub config merge it in. */ + if (user_it->second[0] == '(') + merged_config += default_it->first + "=(" + + merge_default_config(default_it->second, user_it->second) + ')'; + else + /* Add the user configuration as it exists. */ + merged_config += user_it->first + "=" + user_it->second; + ++user_it; + } + /* Add a comma after every item we add except the last one. */ + if (split_default_config.end() - default_it != 1) + merged_config += ","; + } + /* Add any remaining user config items. */ + while (user_it != split_user_config.end()) { + merged_config += "," + user_it->first + "=" + user_it->second; + ++user_it; + } + return (merged_config); +} + +std::vector> +configuration::split_config(const std::string &config) +{ + std::string cut_config = config; + std::vector> split_config; + std::string key = "", value = ""; + bool in_subconfig = false; + bool expect_value = false; + std::stack parens; + + /* All configuration strings must be at least 2 characters. */ + testutil_assert(config.size() > 1); + + /* Remove prefix and trailing "()". */ + if (config[0] == '(') + cut_config = config.substr(1, config.size() - 2); + + size_t start = 0, len = 0; + for (size_t i = 0; i < cut_config.size(); ++i) { + if (cut_config[i] == '(' || cut_config[i] == '[') { + parens.push(cut_config[i]); + in_subconfig = true; + } + if (cut_config[i] == ')' || cut_config[i] == ']') { + parens.pop(); + in_subconfig = !parens.empty(); + } + if (cut_config[i] == '=' && !in_subconfig) { + if (len == 0) { + testutil_die(EINVAL, "error parsing config: detected empty key"); + } + if (expect_value) { + testutil_die(EINVAL, + "error parsing config: syntax error parsing value for key ['%s']: '%s'", + key.c_str(), cut_config.substr(start, len).c_str()); + } + expect_value = true; + key = cut_config.substr(start, len); + start += len + 1; + len = 0; + continue; + } + if (cut_config[i] == ',' && !in_subconfig) { + if (len == 0) { + testutil_die( + EINVAL, "error parsing config: detected empty value for key:'%s'", key.c_str()); + } + if (!expect_value) { + testutil_die(EINVAL, + "error parsing config: syntax error parsing key value pair: '%s'", + cut_config.substr(start, len).c_str()); + } + expect_value = false; + if (start + len >= cut_config.size()) + break; + value = cut_config.substr(start, len); + start += len + 1; + len = 0; + split_config.push_back(std::make_pair(key, value)); + continue; + } + ++len; + } + if (expect_value) { + value = cut_config.substr(start, len); + split_config.push_back(std::make_pair(key, value)); + } + + /* We have to sort the config here otherwise we will match incorrectly while merging. */ + std::sort(split_config.begin(), split_config.end(), comparator); + return (split_config); +} + +bool +configuration::comparator( + std::pair a, std::pair b) +{ + return (a.first < b.first); +} +} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/src/main/configuration.h b/src/third_party/wiredtiger/test/cppsuite/src/main/configuration.h new file mode 100644 index 00000000000..e30291f199b --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/main/configuration.h @@ -0,0 +1,112 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef CONFIGURATION_H +#define CONFIGURATION_H + +#include +#include + +extern "C" { +#include "wiredtiger.h" +} + +namespace test_harness { +inline std::vector +split_string(const std::string &str, const char delim) +{ + std::vector splits; + std::string current_str; + for (const auto c : str) { + if (c == delim) { + if (!current_str.empty()) { + splits.push_back(current_str); + current_str.clear(); + } + } else + current_str.push_back(c); + } + if (!current_str.empty()) + splits.push_back(std::move(current_str)); + return (splits); +} + +class configuration { + public: + explicit configuration(const std::string &test_config_name, const std::string &config); + explicit configuration(const WT_CONFIG_ITEM &nested); + + ~configuration(); + + /* + * Wrapper functions for retrieving basic configuration values. Ideally tests can avoid using + * the config item struct provided by wiredtiger. + * + * When getting a configuration value that may not exist for that configuration string or + * component, the optional forms of the functions can be used. In this case a default value must + * be passed and it will be set to that value. + */ + std::string get_string(const std::string &key); + std::string get_optional_string(const std::string &key, const std::string &def); + bool get_bool(const std::string &key); + bool get_optional_bool(const std::string &key, const bool def); + int64_t get_int(const std::string &key); + int64_t get_optional_int(const std::string &key, const int64_t def); + configuration *get_subconfig(const std::string &key); + configuration *get_optional_subconfig(const std::string &key); + std::vector get_list(const std::string &key); + std::vector get_optional_list(const std::string &key); + + private: + enum class types { BOOL, INT, LIST, STRING, STRUCT }; + + template + T get(const std::string &key, bool optional, types type, T def, T (*func)(WT_CONFIG_ITEM item)); + + /* + * Merge together two configuration strings, the user one and the default one. + */ + static std::string merge_default_config( + const std::string &default_config, const std::string &user_config); + + /* + * Split a config string into keys and values, taking care to not split incorrectly when we have + * a sub config or array. + */ + static std::vector> split_config(const std::string &config); + + static bool comparator( + std::pair a, std::pair b); + + private: + std::string _config; + WT_CONFIG_PARSER *_config_parser = nullptr; +}; +} // namespace test_harness + +#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/src/main/database_model.cpp b/src/third_party/wiredtiger/test/cppsuite/src/main/database_model.cpp new file mode 100644 index 00000000000..848eff3863d --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/main/database_model.cpp @@ -0,0 +1,152 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "database_model.h" + +#include "src/common/api_const.h" +#include "src/common/random_generator.h" +#include "src/storage/connection_manager.h" + +namespace test_harness { +/* collection class implementation */ +collection::collection(const uint64_t id, const uint64_t key_count, const std::string &name) + : name(name), id(id), _key_count(key_count) +{ +} + +uint64_t +collection::get_key_count() const +{ + return (_key_count); +} + +void +collection::increase_key_count(uint64_t increment) +{ + _key_count += increment; +} + +/* database class implementation */ +std::string +database::build_collection_name(const uint64_t id) +{ + return (std::string("table:collection_" + std::to_string(id))); +} + +void +database::add_collection(uint64_t key_count) +{ + std::lock_guard lg(_mtx); + if (_session.get() == nullptr) + _session = connection_manager::instance().create_session(); + if (_collection_create_config.empty()) + testutil_die(EINVAL, "database_model: no collection create config specified!"); + uint64_t next_id = _next_collection_id++; + std::string collection_name = build_collection_name(next_id); + /* FIX-ME-Test-Framework: This will get removed when we split the model up. */ + _collections.emplace(std::piecewise_construct, std::forward_as_tuple(next_id), + std::forward_as_tuple(next_id, key_count, collection_name)); + testutil_check( + _session->create(_session.get(), collection_name.c_str(), _collection_create_config.c_str())); + _tracking->save_schema_operation( + tracking_operation::CREATE_COLLECTION, next_id, _tsm->get_next_ts()); +} + +collection & +database::get_collection(uint64_t id) +{ + std::lock_guard lg(_mtx); + const auto it = _collections.find(id); + if (it == _collections.end()) + testutil_die(EINVAL, "tried to get collection that doesn't exist."); + return (it->second); +} + +collection & +database::get_random_collection() +{ + size_t collection_count = get_collection_count(); + /* Any caller should expect at least one collection to exist. */ + testutil_assert(collection_count != 0); + return (get_collection( + random_generator::instance().generate_integer(0, collection_count - 1))); +} + +uint64_t +database::get_collection_count() +{ + std::lock_guard lg(_mtx); + return (_collections.size()); +} + +std::vector +database::get_collection_names() +{ + std::lock_guard lg(_mtx); + std::vector collection_names; + + for (auto const &it : _collections) + collection_names.push_back(it.second.name); + + return (collection_names); +} + +std::vector +database::get_collection_ids() +{ + std::lock_guard lg(_mtx); + std::vector collection_ids; + + for (auto const &it : _collections) + collection_ids.push_back(it.first); + + return (collection_ids); +} + +void +database::set_timestamp_manager(timestamp_manager *tsm) +{ + testutil_assert(_tsm == nullptr); + _tsm = tsm; +} + +void +database::set_workload_tracking(workload_tracking *tracking) +{ + testutil_assert(_tracking == nullptr); + _tracking = tracking; +} + +void +database::set_create_config(bool use_compression, bool use_reverse_collator) +{ + _collection_create_config = DEFAULT_FRAMEWORK_SCHEMA; + _collection_create_config += use_compression ? std::string(SNAPPY_BLK) + "," : ""; + _collection_create_config += use_reverse_collator ? std::string(REVERSE_COL_CFG) + "," : ""; +} +} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/src/main/database_model.h b/src/third_party/wiredtiger/test/cppsuite/src/main/database_model.h new file mode 100644 index 00000000000..aa239b00c75 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/main/database_model.h @@ -0,0 +1,118 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef DATABASE_MODEL_H +#define DATABASE_MODEL_H + +#include +#include +#include + +#include "src/component/workload_tracking.h" + +namespace test_harness { +/* Key/Value type. */ +typedef std::string key_value_t; + +/* A collection is made of mapped key value objects. */ +class collection { + public: + explicit collection(const uint64_t id, const uint64_t key_count, const std::string &name); + + /* Copies aren't allowed. */ + collection(const collection &) = delete; + collection &operator=(const collection &) = delete; + + uint64_t get_key_count() const; + + /* + * Adding new keys should generally be singly threaded per collection. If two threads both + * attempt to add keys using the incrementing id pattern they'd frequently conflict. + * + * The usage pattern is: + * 1. Call get_key_count to get the number of keys already existing. Add keys with id's equal + * to and greater than this value. + * 2. Once the transaction has successfully committed then call increase_key_count() with the + * number of added keys. + * + * The set of keys should always be contiguous such that other threads calling get_key_count + * will always know that the keys in existence are 0 -> _key_count - 1. + */ + void increase_key_count(uint64_t increment); + + public: + const std::string name; + const uint64_t id; + + private: + std::atomic _key_count{0}; +}; + +/* Representation of the collections in memory. */ +class database { + public: + static std::string build_collection_name(const uint64_t id); + + public: + /* + * Add a new collection, this will create the underlying collection in the database. + */ + void add_collection(uint64_t key_count = 0); + + /* Get a collection using the id of the collection. */ + collection &get_collection(uint64_t id); + + /* Get a random collection. */ + collection &get_random_collection(); + + /* + * Retrieve the current collection count, collection names are indexed from 0 so when using this + * take care to avoid an off by one error. + */ + uint64_t get_collection_count(); + + /* FIX-ME-Test-Framework: Replace usages of this with get_collection_ids. */ + std::vector get_collection_names(); + + std::vector get_collection_ids(); + void set_timestamp_manager(timestamp_manager *tsm); + void set_workload_tracking(workload_tracking *tracking); + void set_create_config(bool use_compression, bool use_reverse_collator); + + private: + std::string _collection_create_config = ""; + scoped_session _session; + timestamp_manager *_tsm = nullptr; + workload_tracking *_tracking = nullptr; + uint64_t _next_collection_id = 0; + std::map _collections; + std::mutex _mtx; +}; +} // namespace test_harness + +#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/src/main/database_operation.cpp b/src/third_party/wiredtiger/test/cppsuite/src/main/database_operation.cpp new file mode 100644 index 00000000000..0a72b79a778 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/main/database_operation.cpp @@ -0,0 +1,395 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "database_operation.h" + +#include "src/common/thread_manager.h" +#include "src/common/api_const.h" +#include "src/common/logger.h" +#include "src/common/random_generator.h" +#include "src/component/workload_validation.h" +#include "src/storage/connection_manager.h" + +namespace test_harness { +/* Static methods. */ +static void +populate_worker(thread_context *tc) +{ + uint64_t collections_per_thread = tc->collection_count / tc->thread_count; + + for (int64_t i = 0; i < collections_per_thread; ++i) { + collection &coll = tc->db.get_collection((tc->id * collections_per_thread) + i); + /* + * WiredTiger lets you open a cursor on a collection using the same pointer. When a session + * is closed, WiredTiger APIs close the cursors too. + */ + scoped_cursor cursor = tc->session.open_scoped_cursor(coll.name); + uint64_t j = 0; + while (j < tc->key_count) { + tc->transaction.begin(); + auto key = tc->pad_string(std::to_string(j), tc->key_size); + auto value = random_generator::instance().generate_pseudo_random_string(tc->value_size); + if (tc->insert(cursor, coll.id, key, value)) { + if (tc->transaction.commit()) { + ++j; + } + } else { + tc->transaction.rollback(); + } + } + } + logger::log_msg(LOG_TRACE, "Populate: thread {" + std::to_string(tc->id) + "} finished"); +} + +/* database_operation class implementation. */ +void +database_operation::populate( + database &database, timestamp_manager *tsm, configuration *config, workload_tracking *tracking) +{ + int64_t collection_count, key_count, key_size, thread_count, value_size; + std::vector workers; + std::string collection_name; + thread_manager tm; + + /* Validate our config. */ + collection_count = config->get_int(COLLECTION_COUNT); + key_count = config->get_int(KEY_COUNT_PER_COLLECTION); + value_size = config->get_int(VALUE_SIZE); + thread_count = config->get_int(THREAD_COUNT); + testutil_assert(thread_count == 0 || collection_count % thread_count == 0); + testutil_assert(value_size > 0); + key_size = config->get_int(KEY_SIZE); + testutil_assert(key_size > 0); + /* Keys must be unique. */ + testutil_assert(key_count <= pow(10, key_size)); + + logger::log_msg( + LOG_INFO, "Populate: creating " + std::to_string(collection_count) + " collections."); + + /* Create n collections as per the configuration. */ + for (int64_t i = 0; i < collection_count; ++i) + /* + * The database model will call into the API and create the collection, with its own + * session. + */ + database.add_collection(key_count); + + logger::log_msg( + LOG_INFO, "Populate: " + std::to_string(collection_count) + " collections created."); + + /* + * Spawn thread_count threads to populate the database, theoretically we should be IO bound + * here. + */ + for (int64_t i = 0; i < thread_count; ++i) { + thread_context *tc = new thread_context(i, thread_type::INSERT, config, + connection_manager::instance().create_session(), tsm, tracking, database); + workers.push_back(tc); + tm.add_thread(populate_worker, tc); + } + + /* Wait for our populate threads to finish and then join them. */ + logger::log_msg(LOG_INFO, "Populate: waiting for threads to complete."); + tm.join(); + + /* Cleanup our workers. */ + for (auto &it : workers) { + delete it; + it = nullptr; + } + logger::log_msg(LOG_INFO, "Populate: finished."); +} + +void +database_operation::custom_operation(thread_context *tc) +{ + logger::log_msg( + LOG_INFO, type_string(tc->type) + " thread {" + std::to_string(tc->id) + "} commencing."); +} + +void +database_operation::insert_operation(thread_context *tc) +{ + logger::log_msg( + LOG_INFO, type_string(tc->type) + " thread {" + std::to_string(tc->id) + "} commencing."); + + /* Helper struct which stores a pointer to a collection and a cursor associated with it. */ + struct collection_cursor { + collection_cursor(collection &coll, scoped_cursor &&cursor) + : coll(coll), cursor(std::move(cursor)) + { + } + collection &coll; + scoped_cursor cursor; + }; + + /* Collection cursor vector. */ + std::vector ccv; + uint64_t collection_count = tc->db.get_collection_count(); + testutil_assert(collection_count != 0); + uint64_t collections_per_thread = collection_count / tc->thread_count; + /* Must have unique collections for each thread. */ + testutil_assert(collection_count % tc->thread_count == 0); + for (int i = tc->id * collections_per_thread; + i < (tc->id * collections_per_thread) + collections_per_thread && tc->running(); ++i) { + collection &coll = tc->db.get_collection(i); + scoped_cursor cursor = tc->session.open_scoped_cursor(coll.name); + ccv.push_back({coll, std::move(cursor)}); + } + + uint64_t counter = 0; + while (tc->running()) { + uint64_t start_key = ccv[counter].coll.get_key_count(); + uint64_t added_count = 0; + tc->transaction.begin(); + + /* Collection cursor. */ + auto &cc = ccv[counter]; + while (tc->transaction.active() && tc->running()) { + /* Insert a key value pair, rolling back the transaction if required. */ + auto key = tc->pad_string(std::to_string(start_key + added_count), tc->key_size); + auto value = random_generator::instance().generate_pseudo_random_string(tc->value_size); + if (!tc->insert(cc.cursor, cc.coll.id, key, value)) { + added_count = 0; + tc->transaction.rollback(); + } else { + added_count++; + if (tc->transaction.can_commit()) { + if (tc->transaction.commit()) { + /* + * We need to inform the database model that we've added these keys as some + * other thread may rely on the key_count data. Only do so if we + * successfully committed. + */ + cc.coll.increase_key_count(added_count); + } else { + added_count = 0; + } + } + } + + /* Sleep the duration defined by the op_rate. */ + tc->sleep(); + } + /* Reset our cursor to avoid pinning content. */ + testutil_check(cc.cursor->reset(cc.cursor.get())); + counter++; + if (counter == collections_per_thread) + counter = 0; + testutil_assert(counter < collections_per_thread); + } + /* Make sure the last transaction is rolled back now the work is finished. */ + if (tc->transaction.active()) + tc->transaction.rollback(); +} + +void +database_operation::read_operation(thread_context *tc) +{ + logger::log_msg( + LOG_INFO, type_string(tc->type) + " thread {" + std::to_string(tc->id) + "} commencing."); + + std::map cursors; + while (tc->running()) { + /* Get a collection and find a cached cursor. */ + collection &coll = tc->db.get_random_collection(); + + if (cursors.find(coll.id) == cursors.end()) + cursors.emplace(coll.id, std::move(tc->session.open_scoped_cursor(coll.name))); + + /* Do a second lookup now that we know it exists. */ + auto &cursor = cursors[coll.id]; + + tc->transaction.begin(); + while (tc->transaction.active() && tc->running()) { + auto ret = cursor->next(cursor.get()); + if (ret != 0) { + if (ret == WT_NOTFOUND) { + cursor->reset(cursor.get()); + } else if (ret == WT_ROLLBACK) { + tc->transaction.rollback(); + tc->sleep(); + continue; + } else + testutil_die(ret, "Unexpected error returned from cursor->next()"); + } + tc->transaction.add_op(); + tc->transaction.try_rollback(); + tc->sleep(); + } + /* Reset our cursor to avoid pinning content. */ + testutil_check(cursor->reset(cursor.get())); + } + /* Make sure the last transaction is rolled back now the work is finished. */ + if (tc->transaction.active()) + tc->transaction.rollback(); +} + +void +database_operation::remove_operation(thread_context *tc) +{ + logger::log_msg( + LOG_INFO, type_string(tc->type) + " thread {" + std::to_string(tc->id) + "} commencing."); + + /* + * We need two types of cursors. One cursor is a random cursor to randomly select a key and the + * other one is a standard cursor to remove the random key. This is required as the random + * cursor does not support the remove operation. + */ + std::map rnd_cursors, cursors; + + /* Loop while the test is running. */ + while (tc->running()) { + /* + * Sleep the period defined by the op_rate in the configuration. Do this at the start of the + * loop as it could be skipped by a subsequent continue call. + */ + tc->sleep(); + + /* Choose a random collection to update. */ + collection &coll = tc->db.get_random_collection(); + + /* Look for existing cursors in our cursor cache. */ + if (cursors.find(coll.id) == cursors.end()) { + logger::log_msg(LOG_TRACE, + "Thread {" + std::to_string(tc->id) + + "} Creating cursor for collection: " + coll.name); + /* Open the two cursors for the chosen collection. */ + scoped_cursor rnd_cursor = + tc->session.open_scoped_cursor(coll.name, "next_random=true"); + rnd_cursors.emplace(coll.id, std::move(rnd_cursor)); + scoped_cursor cursor = tc->session.open_scoped_cursor(coll.name); + cursors.emplace(coll.id, std::move(cursor)); + } + + /* Start a transaction if possible. */ + tc->transaction.try_begin(); + + /* Get the cursor associated with the collection. */ + scoped_cursor &rnd_cursor = rnd_cursors[coll.id]; + scoped_cursor &cursor = cursors[coll.id]; + + /* Choose a random key to delete. */ + const char *key_str; + int ret = rnd_cursor->next(rnd_cursor.get()); + /* It is possible not to find anything if the collection is empty. */ + testutil_assert(ret == 0 || ret == WT_NOTFOUND); + if (ret == WT_NOTFOUND) { + /* + * If we cannot find any record, finish the current transaction as we might be able to + * see new records after starting a new one. + */ + WT_IGNORE_RET_BOOL(tc->transaction.commit()); + continue; + } + testutil_check(rnd_cursor->get_key(rnd_cursor.get(), &key_str)); + if (!tc->remove(cursor, coll.id, key_str)) { + tc->transaction.rollback(); + } + + /* Reset our cursor to avoid pinning content. */ + testutil_check(cursor->reset(cursor.get())); + + /* Commit the current transaction if we're able to. */ + if (tc->transaction.can_commit()) + WT_IGNORE_RET_BOOL(tc->transaction.commit()); + } + + /* Make sure the last operation is rolled back now the work is finished. */ + if (tc->transaction.active()) + tc->transaction.rollback(); +} + +void +database_operation::update_operation(thread_context *tc) +{ + logger::log_msg( + LOG_INFO, type_string(tc->type) + " thread {" + std::to_string(tc->id) + "} commencing."); + /* Cursor map. */ + std::map cursors; + + /* + * Loop while the test is running. + */ + while (tc->running()) { + /* + * Sleep the period defined by the op_rate in the configuration. Do this at the start of the + * loop as it could be skipped by a subsequent continue call. + */ + tc->sleep(); + + /* Choose a random collection to update. */ + collection &coll = tc->db.get_random_collection(); + + /* Look for existing cursors in our cursor cache. */ + if (cursors.find(coll.id) == cursors.end()) { + logger::log_msg(LOG_TRACE, + "Thread {" + std::to_string(tc->id) + + "} Creating cursor for collection: " + coll.name); + /* Open a cursor for the chosen collection. */ + scoped_cursor cursor = tc->session.open_scoped_cursor(coll.name); + cursors.emplace(coll.id, std::move(cursor)); + } + + /* Start a transaction if possible. */ + tc->transaction.try_begin(); + + /* Get the cursor associated with the collection. */ + scoped_cursor &cursor = cursors[coll.id]; + + /* Choose a random key to update. */ + testutil_assert(coll.get_key_count() != 0); + auto key_id = + random_generator::instance().generate_integer(0, coll.get_key_count() - 1); + auto key = tc->pad_string(std::to_string(key_id), tc->key_size); + auto value = random_generator::instance().generate_pseudo_random_string(tc->value_size); + if (!tc->update(cursor, coll.id, key, value)) { + tc->transaction.rollback(); + } + + /* Reset our cursor to avoid pinning content. */ + testutil_check(cursor->reset(cursor.get())); + + /* Commit the current transaction if we're able to. */ + if (tc->transaction.can_commit()) + WT_IGNORE_RET_BOOL(tc->transaction.commit()); + } + + /* Make sure the last operation is rolled back now the work is finished. */ + if (tc->transaction.active()) + tc->transaction.rollback(); +} + +void +database_operation::validate(const std::string &operation_table_name, + const std::string &schema_table_name, const std::vector &known_collection_ids) +{ + workload_validation wv; + wv.validate(operation_table_name, schema_table_name, known_collection_ids); +} +} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/src/main/database_operation.h b/src/third_party/wiredtiger/test/cppsuite/src/main/database_operation.h new file mode 100644 index 00000000000..946a322f25a --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/main/database_operation.h @@ -0,0 +1,71 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef DATABASE_OPERATION_H +#define DATABASE_OPERATION_H + +#include "database_model.h" +#include "thread_context.h" + +namespace test_harness { +class database_operation { + public: + /* + * Function that performs the following steps using the configuration that is defined by the + * test: + * - Creates N collections as per the configuration. + * - Creates M threads as per the configuration, each thread will: + * - Open a cursor on each collection. + * - Insert K key/value pairs in each collection. Values are random strings which size is + * defined by the configuration. + */ + virtual void populate(database &database, timestamp_manager *tsm, configuration *config, + workload_tracking *tracking); + + /* Custom operation without a default implementation. */ + virtual void custom_operation(thread_context *tc); + + /* Basic insert operation that adds a new key every rate tick. */ + virtual void insert_operation(thread_context *tc); + + /* Basic read operation that chooses a random collection and walks a cursor. */ + virtual void read_operation(thread_context *tc); + + /* Basic remove operation that chooses a random key and deletes it. */ + virtual void remove_operation(thread_context *tc); + + /* Basic update operation that chooses a random key and updates it. */ + virtual void update_operation(thread_context *tc); + + virtual void validate(const std::string &operation_table_name, + const std::string &schema_table_name, const std::vector &known_collection_ids); + + virtual ~database_operation() = default; +}; +} // namespace test_harness +#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/src/main/test.cpp b/src/third_party/wiredtiger/test/cppsuite/src/main/test.cpp new file mode 100644 index 00000000000..288ca08e2c4 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/main/test.cpp @@ -0,0 +1,190 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "test.h" + +#include "src/common/api_const.h" +#include "src/common/logger.h" +#include "src/component/perf_plotter.h" + +namespace test_harness { +test::test(const test_args &args) : _args(args) +{ + _config = new configuration(args.test_name, args.test_config); + _checkpoint_manager = new checkpoint_manager(_config->get_subconfig(CHECKPOINT_MANAGER)); + _runtime_monitor = + new runtime_monitor(args.test_name, _config->get_subconfig(RUNTIME_MONITOR), _database); + _timestamp_manager = new timestamp_manager(_config->get_subconfig(TIMESTAMP_MANAGER)); + _workload_generator = new workload_generator( + _config->get_subconfig(WORKLOAD_GENERATOR), this, _timestamp_manager, _database); + _thread_manager = new thread_manager(); + + _database.set_timestamp_manager(_timestamp_manager); + _database.set_create_config( + _config->get_bool(COMPRESSION_ENABLED), _config->get_bool(REVERSE_COLLATOR)); + + /* + * Ordering is not important here, any dependencies between components should be resolved + * internally by the components. + */ + _components = {_workload_generator, _timestamp_manager, _runtime_monitor, _checkpoint_manager}; +} + +void +test::init_tracking(workload_tracking *tracking) +{ + delete _workload_tracking; + if (tracking == nullptr) { + /* Fallback to default behavior. */ + tracking = new workload_tracking(_config->get_subconfig(WORKLOAD_TRACKING), + _config->get_bool(COMPRESSION_ENABLED), *_timestamp_manager); + } + _workload_tracking = tracking; + _workload_generator->set_workload_tracking(_workload_tracking); + _database.set_workload_tracking(_workload_tracking); + _components.push_back(_workload_tracking); +} + +test::~test() +{ + delete _config; + delete _checkpoint_manager; + delete _runtime_monitor; + delete _timestamp_manager; + delete _thread_manager; + delete _workload_generator; + delete _workload_tracking; + _config = nullptr; + _checkpoint_manager = nullptr; + _runtime_monitor = nullptr; + _timestamp_manager = nullptr; + _thread_manager = nullptr; + _workload_generator = nullptr; + _workload_tracking = nullptr; + + _components.clear(); +} + +void +test::run() +{ + int64_t cache_max_wait_ms, cache_size_mb, duration_seconds; + bool enable_logging, statistics_logging; + configuration *statistics_config; + std::string statistics_type; + /* Build the database creation config string. */ + std::string db_create_config = CONNECTION_CREATE; + + /* Enable snappy compression or reverse collator if required. */ + if (_config->get_bool(COMPRESSION_ENABLED) || _config->get_bool(REVERSE_COLLATOR)) { + db_create_config += ",extensions=["; + db_create_config += + _config->get_bool(COMPRESSION_ENABLED) ? std::string(SNAPPY_PATH) + "," : ""; + db_create_config += + _config->get_bool(REVERSE_COLLATOR) ? std::string(REVERSE_COLLATOR_PATH) : ""; + db_create_config += "]"; + } + + /* Get the cache size. */ + cache_size_mb = _config->get_int(CACHE_SIZE_MB); + db_create_config += ",cache_size=" + std::to_string(cache_size_mb) + "MB"; + + /* Get the statistics configuration for this run. */ + statistics_config = _config->get_subconfig(STATISTICS_CONFIG); + statistics_type = statistics_config->get_string(TYPE); + statistics_logging = statistics_config->get_bool(ENABLE_LOGGING); + db_create_config += statistics_logging ? "," + STATISTICS_LOG : ""; + db_create_config += ",statistics=(" + statistics_type + ")"; + /* Don't forget to delete. */ + delete statistics_config; + + /* Enable or disable write ahead logging. */ + enable_logging = _config->get_bool(ENABLE_LOGGING); + db_create_config += ",log=(enabled=" + std::string(enable_logging ? "true" : "false") + ")"; + + /* Maximum waiting time for the cache to get unstuck. */ + cache_max_wait_ms = _config->get_int(CACHE_MAX_WAIT_MS); + db_create_config += ",cache_max_wait_ms=" + std::to_string(cache_max_wait_ms); + + /* Add the user supplied wiredtiger open config. */ + db_create_config += _args.wt_open_config; + + /* + * Set up the test environment. A smart pointer is used here so that the connection can + * automatically be closed by the scoped_connection's destructor when the test finishes and the + * pointer goes out of scope. + */ + _scoped_conn = std::make_shared(db_create_config); + + /* Initiate the load stage of each component. */ + for (const auto &it : _components) + it->load(); + + /* Spawn threads for all component::run() functions. */ + for (const auto &it : _components) + _thread_manager->add_thread(&component::run, it); + + /* The initial population phase needs to be finished before starting the actual test. */ + while (_workload_generator->enabled() && !_workload_generator->db_populated()) + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + + /* The test will run for the duration as defined in the config. */ + duration_seconds = _config->get_int(DURATION_SECONDS); + testutil_assert(duration_seconds >= 0); + logger::log_msg(LOG_INFO, + "Waiting {" + std::to_string(duration_seconds) + "} seconds for testing to complete."); + std::this_thread::sleep_for(std::chrono::seconds(duration_seconds)); + + /* Notify components that they should complete their last iteration. */ + for (const auto &it : _components) + it->end_run(); + + /* Call join on the components threads so we know they have finished their loop. */ + logger::log_msg(LOG_INFO, + "Joining all component threads.\n This could take a while as we need to wait" + " for all components to finish their current loop."); + _thread_manager->join(); + + /* End the test by calling finish on all known components. */ + for (const auto &it : _components) + it->finish(); + + /* Validation stage. */ + if (_workload_tracking->enabled()) { + std::unique_ptr tracking_config(_config->get_subconfig(WORKLOAD_TRACKING)); + this->validate(_workload_tracking->get_operation_table_name(), + _workload_tracking->get_schema_table_name(), + _workload_generator->get_database().get_collection_ids()); + } + + /* Log perf stats. */ + perf_plotter::instance().output_perf_file(_args.test_name); + + logger::log_msg(LOG_INFO, "SUCCESS"); +} +} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/src/main/test.h b/src/third_party/wiredtiger/test/cppsuite/src/main/test.h new file mode 100644 index 00000000000..513df520e74 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/main/test.h @@ -0,0 +1,99 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef TEST_H +#define TEST_H + +#include + +#include "database_operation.h" +#include "src/component/checkpoint_manager.h" +#include "src/component/runtime_monitor.h" +#include "src/component/workload_generator.h" +#include "src/storage/connection_manager.h" +#include "src/storage/scoped_connection.h" + +namespace test_harness { +class test_args { + public: + test_args(const std::string &config, const std::string &name, const std::string &wt_open_config) + : test_config(config), test_name(name), wt_open_config(wt_open_config) + { + } + const std::string test_config; + const std::string test_name; + const std::string wt_open_config; +}; + +/* + * The base class for a test, the standard usage pattern is to just call run(). + */ +class test : public database_operation { + public: + explicit test(const test_args &args); + virtual ~test(); + + /* Delete the copy constructor and the assignment operator. */ + test(const test &) = delete; + test &operator=(const test &) = delete; + + /* Initialize the tracking component and its dependencies. */ + void init_tracking(workload_tracking *tracking = nullptr); + + /* + * The primary run function that most tests will be able to utilize without much other code. + */ + virtual void run(); + + /* + * Getters for all the major components, used if a test wants more control over the test + * program. + */ + workload_generator *get_workload_generator(); + runtime_monitor *get_runtime_monitor(); + timestamp_manager *get_timestamp_manager(); + thread_manager *get_thread_manager(); + + protected: + const test_args &_args; + configuration *_config; + timestamp_manager *_timestamp_manager = nullptr; + workload_tracking *_workload_tracking = nullptr; + + private: + std::vector _components; + checkpoint_manager *_checkpoint_manager = nullptr; + runtime_monitor *_runtime_monitor = nullptr; + thread_manager *_thread_manager = nullptr; + workload_generator *_workload_generator = nullptr; + std::shared_ptr _scoped_conn; + database _database; +}; +} // namespace test_harness + +#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/src/main/thread_context.cpp b/src/third_party/wiredtiger/test/cppsuite/src/main/thread_context.cpp new file mode 100644 index 00000000000..b48b985b91f --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/main/thread_context.cpp @@ -0,0 +1,349 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "thread_context.h" + +#include "src/common/api_const.h" +#include "src/common/logger.h" +#include "src/common/random_generator.h" + +namespace test_harness { + +const std::string +type_string(thread_type type) +{ + switch (type) { + case thread_type::CUSTOM: + return ("custom"); + case thread_type::INSERT: + return ("insert"); + case thread_type::READ: + return ("read"); + case thread_type::REMOVE: + return ("remove"); + case thread_type::UPDATE: + return ("update"); + default: + testutil_die(EINVAL, "unexpected thread_type: %d", static_cast(type)); + } +} + +/* transaction_context class implementation */ +transaction_context::transaction_context( + configuration *config, timestamp_manager *timestamp_manager, WT_SESSION *session) + : _timestamp_manager(timestamp_manager), _session(session) +{ + /* Use optional here as our populate threads don't define this configuration. */ + configuration *transaction_config = config->get_optional_subconfig(OPS_PER_TRANSACTION); + if (transaction_config != nullptr) { + _min_op_count = transaction_config->get_optional_int(MIN, 1); + _max_op_count = transaction_config->get_optional_int(MAX, 1); + delete transaction_config; + } +} + +bool +transaction_context::active() const +{ + return (_in_txn); +} + +void +transaction_context::add_op() +{ + _op_count++; +} + +void +transaction_context::begin(const std::string &config) +{ + testutil_assert(!_in_txn); + testutil_check( + _session->begin_transaction(_session, config.empty() ? nullptr : config.c_str())); + /* This randomizes the number of operations to be executed in one transaction. */ + _target_op_count = + random_generator::instance().generate_integer(_min_op_count, _max_op_count); + _op_count = 0; + _in_txn = true; + _needs_rollback = false; +} + +void +transaction_context::try_begin(const std::string &config) +{ + if (!_in_txn) + begin(config); +} + +/* + * It's possible to receive rollback in commit, when this happens the API will rollback the + * transaction internally. + */ +bool +transaction_context::commit(const std::string &config) +{ + WT_DECL_RET; + testutil_assert(_in_txn && !_needs_rollback); + + ret = _session->commit_transaction(_session, config.empty() ? nullptr : config.c_str()); + /* + * FIXME-WT-9198 Now we are accepting the error code EINVAL because of possible invalid + * timestamps as we know it can happen due to the nature of the framework. The framework may set + * the stable/oldest timestamps to a more recent date than the commit timestamp of the + * transaction which makes the transaction invalid. We only need to check against the stable + * timestamp as, by definition, the oldest timestamp is older than the stable one. + */ + testutil_assert(ret == 0 || ret == EINVAL || ret == WT_ROLLBACK); + + if (ret != 0) + logger::log_msg(LOG_WARN, + "Failed to commit transaction in commit, received error code: " + std::to_string(ret)); + _op_count = 0; + _in_txn = false; + return (ret == 0); +} + +void +transaction_context::rollback(const std::string &config) +{ + testutil_assert(_in_txn); + testutil_check( + _session->rollback_transaction(_session, config.empty() ? nullptr : config.c_str())); + _needs_rollback = false; + _op_count = 0; + _in_txn = false; +} + +void +transaction_context::try_rollback(const std::string &config) +{ + if (can_rollback()) + rollback(config); +} + +/* + * FIXME: WT-9198 We're concurrently doing a transaction that contains a bunch of operations while + * moving the stable timestamp. Eat the occasional EINVAL from the transaction's first commit + * timestamp being earlier than the stable timestamp. + */ +int +transaction_context::set_commit_timestamp(wt_timestamp_t ts) +{ + /* We don't want to set zero timestamps on transactions if we're not using timestamps. */ + if (!_timestamp_manager->enabled()) + return 0; + const std::string config = COMMIT_TS + "=" + timestamp_manager::decimal_to_hex(ts); + return _session->timestamp_transaction(_session, config.c_str()); +} + +void +transaction_context::set_needs_rollback(bool rollback) +{ + _needs_rollback = rollback; +} + +bool +transaction_context::can_commit() +{ + return (!_needs_rollback && can_rollback()); +} + +bool +transaction_context::can_rollback() +{ + return (_in_txn && _op_count >= _target_op_count); +} + +/* thread_context class implementation */ +thread_context::thread_context(uint64_t id, thread_type type, configuration *config, + scoped_session &&created_session, timestamp_manager *timestamp_manager, + workload_tracking *tracking, database &dbase) + : /* These won't exist for certain threads which is why we use optional here. */ + collection_count(config->get_optional_int(COLLECTION_COUNT, 1)), + key_count(config->get_optional_int(KEY_COUNT_PER_COLLECTION, 1)), + key_size(config->get_optional_int(KEY_SIZE, 1)), + value_size(config->get_optional_int(VALUE_SIZE, 1)), + thread_count(config->get_int(THREAD_COUNT)), type(type), id(id), db(dbase), + session(std::move(created_session)), tsm(timestamp_manager), + transaction(transaction_context(config, timestamp_manager, session.get())), + tracking(tracking), _throttle(config) +{ + if (tracking->enabled()) + op_track_cursor = session.open_scoped_cursor(tracking->get_operation_table_name()); + + testutil_assert(key_size > 0 && value_size > 0); +} + +void +thread_context::finish() +{ + _running = false; +} + +std::string +thread_context::pad_string(const std::string &value, uint64_t size) +{ + uint64_t diff = size > value.size() ? size - value.size() : 0; + std::string s(diff, '0'); + return (s.append(value)); +} + +bool +thread_context::update( + scoped_cursor &cursor, uint64_t collection_id, const std::string &key, const std::string &value) +{ + WT_DECL_RET; + + testutil_assert(tracking != nullptr); + testutil_assert(cursor.get() != nullptr); + + wt_timestamp_t ts = tsm->get_next_ts(); + ret = transaction.set_commit_timestamp(ts); + testutil_assert(ret == 0 || ret == EINVAL); + if (ret != 0) { + transaction.set_needs_rollback(true); + return (false); + } + + cursor->set_key(cursor.get(), key.c_str()); + cursor->set_value(cursor.get(), value.c_str()); + ret = cursor->update(cursor.get()); + + if (ret != 0) { + if (ret == WT_ROLLBACK) { + transaction.set_needs_rollback(true); + return (false); + } else + testutil_die(ret, "unhandled error while trying to update a key"); + } + + uint64_t txn_id = ((WT_SESSION_IMPL *)session.get())->txn->id; + ret = tracking->save_operation( + txn_id, tracking_operation::INSERT, collection_id, key, value, ts, op_track_cursor); + + if (ret == 0) + transaction.add_op(); + else if (ret == WT_ROLLBACK) + transaction.set_needs_rollback(true); + else + testutil_die(ret, "unhandled error while trying to save an update to the tracking table"); + return (ret == 0); +} + +bool +thread_context::insert( + scoped_cursor &cursor, uint64_t collection_id, const std::string &key, const std::string &value) +{ + WT_DECL_RET; + + testutil_assert(tracking != nullptr); + testutil_assert(cursor.get() != nullptr); + + wt_timestamp_t ts = tsm->get_next_ts(); + ret = transaction.set_commit_timestamp(ts); + testutil_assert(ret == 0 || ret == EINVAL); + if (ret != 0) { + transaction.set_needs_rollback(true); + return (false); + } + + cursor->set_key(cursor.get(), key.c_str()); + cursor->set_value(cursor.get(), value.c_str()); + ret = cursor->insert(cursor.get()); + + if (ret != 0) { + if (ret == WT_ROLLBACK) { + transaction.set_needs_rollback(true); + return (false); + } else + testutil_die(ret, "unhandled error while trying to insert a key"); + } + + uint64_t txn_id = ((WT_SESSION_IMPL *)session.get())->txn->id; + ret = tracking->save_operation( + txn_id, tracking_operation::INSERT, collection_id, key, value, ts, op_track_cursor); + + if (ret == 0) + transaction.add_op(); + else if (ret == WT_ROLLBACK) + transaction.set_needs_rollback(true); + else + testutil_die(ret, "unhandled error while trying to save an insert to the tracking table"); + return (ret == 0); +} + +bool +thread_context::remove(scoped_cursor &cursor, uint64_t collection_id, const std::string &key) +{ + WT_DECL_RET; + testutil_assert(tracking != nullptr); + testutil_assert(cursor.get() != nullptr); + + wt_timestamp_t ts = tsm->get_next_ts(); + ret = transaction.set_commit_timestamp(ts); + testutil_assert(ret == 0 || ret == EINVAL); + if (ret != 0) { + transaction.set_needs_rollback(true); + return (false); + } + + cursor->set_key(cursor.get(), key.c_str()); + ret = cursor->remove(cursor.get()); + if (ret != 0) { + if (ret == WT_ROLLBACK) { + transaction.set_needs_rollback(true); + return (false); + } else + testutil_die(ret, "unhandled error while trying to remove a key"); + } + + uint64_t txn_id = ((WT_SESSION_IMPL *)session.get())->txn->id; + ret = tracking->save_operation( + txn_id, tracking_operation::DELETE_KEY, collection_id, key, "", ts, op_track_cursor); + + if (ret == 0) + transaction.add_op(); + else if (ret == WT_ROLLBACK) + transaction.set_needs_rollback(true); + else + testutil_die(ret, "unhandled error while trying to save a remove to the tracking table"); + return (ret == 0); +} + +void +thread_context::sleep() +{ + _throttle.sleep(); +} + +bool +thread_context::running() const +{ + return (_running); +} +} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/src/main/thread_context.h b/src/third_party/wiredtiger/test/cppsuite/src/main/thread_context.h new file mode 100644 index 00000000000..c97038f11cf --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/main/thread_context.h @@ -0,0 +1,164 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef THREAD_CONTEXT_H +#define THREAD_CONTEXT_H + +#include + +#include "database_model.h" +#include "src/component/workload_tracking.h" +#include "src/component/timestamp_manager.h" +#include "src/main/configuration.h" +#include "src/storage/scoped_types.h" + +namespace test_harness { +enum thread_type { CUSTOM, INSERT, READ, REMOVE, UPDATE }; + +const std::string type_string(thread_type type); + +class transaction_context { + public: + explicit transaction_context( + configuration *config, timestamp_manager *timestamp_manager, WT_SESSION *session); + + bool active() const; + void add_op(); + void begin(const std::string &config = ""); + /* Begin a transaction if we are not currently in one. */ + void try_begin(const std::string &config = ""); + /* + * Commit a transaction and return true if the commit was successful. + */ + bool commit(const std::string &config = ""); + /* Rollback a transaction, failure will abort the test. */ + void rollback(const std::string &config = ""); + /* Attempt to rollback the transaction given the requirements are met. */ + void try_rollback(const std::string &config = ""); + /* Set a commit timestamp. */ + int set_commit_timestamp(wt_timestamp_t ts); + /* Set that the transaction needs to be rolled back. */ + void set_needs_rollback(bool rollback); + /* + * Returns true if a transaction can be committed as determined by the op count and the state of + * the transaction. + */ + bool can_commit(); + /* + * Returns true if a transaction can be rolled back as determined by the op count and the state + * of the transaction. + */ + bool can_rollback(); + + private: + bool _in_txn = false; + bool _needs_rollback = false; + + /* + * _min_op_count and _max_op_count are the minimum and maximum number of operations within one + * transaction. is the current maximum number of operations that can be executed in the current + * transaction. + */ + int64_t _max_op_count = INT64_MAX; + int64_t _min_op_count = 0; + /* + * op_count is the current number of operations that have been executed in the current + * transaction. + */ + int64_t _op_count = 0; + int64_t _target_op_count = 0; + + timestamp_manager *_timestamp_manager = nullptr; + WT_SESSION *_session = nullptr; +}; + +/* Container class for a thread and any data types it may need to interact with the database. */ +class thread_context { + public: + thread_context(uint64_t id, thread_type type, configuration *config, + scoped_session &&created_session, timestamp_manager *timestamp_manager, + workload_tracking *tracking, database &dbase); + + virtual ~thread_context() = default; + + void finish(); + + /* If the value's size is less than the given size, padding of '0' is added to the value. */ + std::string pad_string(const std::string &value, uint64_t size); + + /* + * Generic update function, takes a collection_id, key and value. + * + * Return true if the operation was successful, a return value of false implies the transaction + * needs to be rolled back. + */ + bool update(scoped_cursor &cursor, uint64_t collection_id, const std::string &key, + const std::string &value); + + /* + * Generic insert function, takes a collection_id, key and value. + * + * Return true if the operation was successful, a return value of false implies the transaction + * needs to be rolled back. + */ + bool insert(scoped_cursor &cursor, uint64_t collection_id, const std::string &key, + const std::string &value); + + /* + * Generic remove function, takes a collection_id and key and will delete the key if it exists. + * + * Return true if the operation was successful, a return value of false implies the transaction + * needs to be rolled back. + */ + bool remove(scoped_cursor &cursor, uint64_t collection_id, const std::string &key); + void sleep(); + bool running() const; + + public: + const int64_t collection_count; + const int64_t key_count; + const int64_t key_size; + const int64_t value_size; + const int64_t thread_count; + const thread_type type; + const uint64_t id; + database &db; + scoped_session session; + scoped_cursor op_track_cursor; + scoped_cursor stat_cursor; + timestamp_manager *tsm; + transaction_context transaction; + workload_tracking *tracking; + + private: + bool _running = true; + throttle _throttle; +}; +} // namespace test_harness + +#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/src/main/throttle.cpp b/src/third_party/wiredtiger/test/cppsuite/src/main/throttle.cpp new file mode 100644 index 00000000000..98263140d4d --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/main/throttle.cpp @@ -0,0 +1,79 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "throttle.h" + +#include + +#include "src/common/api_const.h" + +extern "C" { +#include "test_util.h" +} + +namespace test_harness { +throttle::throttle(const std::string &throttle_rate) +{ + std::string magnitude; + uint64_t multiplier = 0; + /* + * Find the ms, s, or m in the string. Searching for "ms" first as the following two searches + * would match as well. + */ + size_t pos = throttle_rate.find("ms"); + if (pos != std::string::npos) + multiplier = 1; + else { + pos = throttle_rate.find("s"); + if (pos != std::string::npos) + multiplier = 1000; + else { + pos = throttle_rate.find("m"); + if (pos != std::string::npos) + multiplier = 60 * 1000; + else + testutil_die(-1, "no rate specifier given"); + } + } + magnitude = throttle_rate.substr(0, pos); + /* This will throw if it can't cast, which is fine. */ + _ms = std::stoi(magnitude) * multiplier; +} + +/* Use optional and default to 1s per op in case something doesn't define this. */ +throttle::throttle(configuration *config) : throttle(config->get_optional_string(OP_RATE, "1s")) {} + +/* Default to a second per operation. */ +throttle::throttle() : throttle("1s") {} + +void +throttle::sleep() +{ + std::this_thread::sleep_for(std::chrono::milliseconds(_ms)); +} +} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/src/main/throttle.h b/src/third_party/wiredtiger/test/cppsuite/src/main/throttle.h new file mode 100644 index 00000000000..29dd622f872 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/main/throttle.h @@ -0,0 +1,54 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef THROTTLE_H +#define THROTTLE_H + +#include + +#include "configuration.h" + +namespace test_harness { +class throttle { + public: + explicit throttle(const std::string &throttle_rate); + + /* Use optional and default to 1s per op in case something doesn't define this. */ + explicit throttle(configuration *config); + + /* Default to a second per operation. */ + throttle(); + + void sleep(); + + private: + uint64_t _ms = 1000; +}; +} // namespace test_harness + +#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/src/storage/connection_manager.cpp b/src/third_party/wiredtiger/test/cppsuite/src/storage/connection_manager.cpp new file mode 100644 index 00000000000..f5bf44637bb --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/storage/connection_manager.cpp @@ -0,0 +1,103 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "connection_manager.h" + +#include "src/common/logger.h" + +extern "C" { +#include "test_util.h" +} + +namespace test_harness { +connection_manager & +connection_manager::instance() +{ + static connection_manager _instance; + return (_instance); +} + +void +connection_manager::close() +{ + if (_conn != nullptr) { + testutil_check(_conn->close(_conn, nullptr)); + _conn = nullptr; + } +} + +void +connection_manager::create(const std::string &config, const std::string &home) +{ + if (_conn != nullptr) { + logger::log_msg(LOG_ERROR, "Connection is not NULL, cannot be re-opened."); + testutil_die(EINVAL, "Connection is not NULL"); + } + logger::log_msg(LOG_INFO, "wiredtiger_open config: " + config); + + /* Create the working dir. */ + testutil_make_work_dir(home.c_str()); + + /* Open conn. */ + testutil_check(wiredtiger_open(home.c_str(), nullptr, config.c_str(), &_conn)); +} + +scoped_session +connection_manager::create_session() +{ + if (_conn == nullptr) { + logger::log_msg(LOG_ERROR, + "Connection is NULL, did you forget to call " + "connection_manager::create ?"); + testutil_die(EINVAL, "Connection is NULL"); + } + + std::lock_guard lg(_conn_mutex); + scoped_session session(_conn); + + return (session); +} + +WT_CONNECTION * +connection_manager::get_connection() +{ + return (_conn); +} + +/* + * set_timestamp calls into the connection API in a thread safe manner to set global timestamps. + */ +void +connection_manager::set_timestamp(const std::string &config) +{ + std::lock_guard lg(_conn_mutex); + testutil_check(_conn->set_timestamp(_conn, config.c_str())); +} + +connection_manager::connection_manager() {} +} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/src/storage/connection_manager.h b/src/third_party/wiredtiger/test/cppsuite/src/storage/connection_manager.h new file mode 100644 index 00000000000..20161126736 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/storage/connection_manager.h @@ -0,0 +1,78 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef CONN_API_H +#define CONN_API_H + +/* Following definitions are required in order to use printing format specifiers in C++. */ +#ifndef __STDC_LIMIT_MACROS +#define __STDC_LIMIT_MACROS +#endif +#ifndef __STDC_FORMAT_MACROS +#define __STDC_FORMAT_MACROS +#endif + +#include + +#include "scoped_types.h" + +namespace test_harness { +/* + * Singleton class owning the database connection, provides access to sessions and any other + * required connection API calls. + */ +class connection_manager { + public: + static connection_manager &instance(); + + public: + /* No copies of the singleton allowed. */ + connection_manager(connection_manager const &) = delete; + connection_manager &operator=(connection_manager const &) = delete; + + void close(); + void create(const std::string &config, const std::string &home); + scoped_session create_session(); + + WT_CONNECTION *get_connection(); + + /* + * set_timestamp calls into the connection API in a thread safe manner to set global timestamps. + */ + void set_timestamp(const std::string &config); + + private: + connection_manager(); + + private: + WT_CONNECTION *_conn = nullptr; + std::mutex _conn_mutex; +}; +} // namespace test_harness + +#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/src/storage/scoped_connection.cpp b/src/third_party/wiredtiger/test/cppsuite/src/storage/scoped_connection.cpp new file mode 100644 index 00000000000..ca879aa4da9 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/storage/scoped_connection.cpp @@ -0,0 +1,45 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "scoped_connection.h" + +#include "connection_manager.h" + +namespace test_harness { + +scoped_connection::scoped_connection(const std::string &db_conn_config, const std::string &home) +{ + connection_manager::instance().create(db_conn_config, home); +} + +scoped_connection::~scoped_connection() +{ + connection_manager::instance().close(); +} + +} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/src/storage/scoped_connection.h b/src/third_party/wiredtiger/test/cppsuite/src/storage/scoped_connection.h new file mode 100644 index 00000000000..2d76b800235 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/storage/scoped_connection.h @@ -0,0 +1,56 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef SCOPED_CONNECTION_H +#define SCOPED_CONNECTION_H + +/* Following definitions are required in order to use printing format specifiers in C++. */ +#ifndef __STDC_LIMIT_MACROS +#define __STDC_LIMIT_MACROS +#endif +#ifndef __STDC_FORMAT_MACROS +#define __STDC_FORMAT_MACROS +#endif + +#include + +extern "C" { +#include "test_util.h" +} + +namespace test_harness { + +class scoped_connection { + public: + explicit scoped_connection( + const std::string &db_conn_config, const std::string &home = DEFAULT_DIR); + ~scoped_connection(); +}; + +} // namespace test_harness +#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/src/storage/scoped_types.cpp b/src/third_party/wiredtiger/test/cppsuite/src/storage/scoped_types.cpp new file mode 100644 index 00000000000..4c97213abc8 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/storage/scoped_types.cpp @@ -0,0 +1,170 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "scoped_types.h" + +extern "C" { +#include "test_util.h" +} + +namespace test_harness { + +/* scoped_cursor implementation */ +scoped_cursor::scoped_cursor(WT_SESSION *session, const std::string &uri, const std::string &cfg) +{ + reinit(session, uri, cfg); +} + +scoped_cursor::scoped_cursor(scoped_cursor &&other) +{ + std::swap(_cursor, other._cursor); +} + +scoped_cursor::~scoped_cursor() +{ + if (_cursor != nullptr) { + testutil_check(_cursor->close(_cursor)); + _cursor = nullptr; + } +} + +/* + * Implement move assignment by move constructing a temporary and swapping its internals with the + * current cursor. This means that the currently held WT_CURSOR will get destroyed as the temporary + * falls out of the scope and we will steal the one that we're move assigning from. + */ +scoped_cursor & +scoped_cursor::operator=(scoped_cursor &&other) +{ + scoped_cursor tmp(std::move(other)); + std::swap(_cursor, tmp._cursor); + return (*this); +} + +void +scoped_cursor::reinit(WT_SESSION *session, const std::string &uri, const std::string &cfg) +{ + testutil_assert(!uri.empty()); + if (_cursor != nullptr) { + testutil_check(_cursor->close(_cursor)); + _cursor = nullptr; + } + if (session != nullptr) + testutil_check(session->open_cursor( + session, uri.c_str(), nullptr, cfg.empty() ? nullptr : cfg.c_str(), &_cursor)); +} + +/* + * Override the dereference operators. The idea is that we should able to use this class as if it is + * a pointer to a WT_CURSOR. + */ +WT_CURSOR & +scoped_cursor::operator*() +{ + return (*_cursor); +} + +WT_CURSOR * +scoped_cursor::operator->() +{ + return (_cursor); +} + +WT_CURSOR * +scoped_cursor::get() +{ + return (_cursor); +} + +/* scoped_session implementation */ +scoped_session::scoped_session(WT_CONNECTION *conn) +{ + reinit(conn); +} + +scoped_session::~scoped_session() +{ + if (_session != nullptr) { + testutil_check(_session->close(_session, nullptr)); + _session = nullptr; + } +} + +scoped_session::scoped_session(scoped_session &&other) +{ + std::swap(_session, other._session); +} + +/* + * Implement move assignment by move constructing a temporary and swapping its internals with the + * current session. This means that the currently held WT_SESSION will get destroyed as the + * temporary falls out of the scope and we will steal the one that we're move assigning from. + */ +scoped_session & +scoped_session::operator=(scoped_session &&other) +{ + scoped_session tmp(std::move(other)); + std::swap(_session, tmp._session); + return (*this); +} + +void +scoped_session::reinit(WT_CONNECTION *conn) +{ + if (_session != nullptr) { + testutil_check(_session->close(_session, nullptr)); + _session = nullptr; + } + if (conn != nullptr) + testutil_check(conn->open_session(conn, nullptr, nullptr, &_session)); +} + +WT_SESSION & +scoped_session::operator*() +{ + return (*_session); +} + +WT_SESSION * +scoped_session::operator->() +{ + return (_session); +} + +WT_SESSION * +scoped_session::get() +{ + return (_session); +} + +scoped_cursor +scoped_session::open_scoped_cursor(const std::string &uri, const std::string &cfg) +{ + return (scoped_cursor(_session, uri, cfg)); +} +} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/src/storage/scoped_types.h b/src/third_party/wiredtiger/test/cppsuite/src/storage/scoped_types.h new file mode 100644 index 00000000000..e6f33931319 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/src/storage/scoped_types.h @@ -0,0 +1,105 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef SCOPED_TYPES_H +#define SCOPED_TYPES_H + +/* Following definitions are required in order to use printing format specifiers in C++. */ +#ifndef __STDC_LIMIT_MACROS +#define __STDC_LIMIT_MACROS +#endif +#ifndef __STDC_FORMAT_MACROS +#define __STDC_FORMAT_MACROS +#endif + +#include + +extern "C" { +#include "wiredtiger.h" +} + +namespace test_harness { +class scoped_cursor { + public: + scoped_cursor() = default; + explicit scoped_cursor(WT_SESSION *session, const std::string &uri, const std::string &cfg); + + /* Moving is ok but copying is not. */ + scoped_cursor(scoped_cursor &&other); + + ~scoped_cursor(); + + scoped_cursor &operator=(scoped_cursor &&other); + scoped_cursor(const scoped_cursor &) = delete; + scoped_cursor &operator=(const scoped_cursor &) = delete; + + void reinit(WT_SESSION *session, const std::string &uri, const std::string &cfg); + + WT_CURSOR &operator*(); + WT_CURSOR *operator->(); + + WT_CURSOR *get(); + + private: + WT_CURSOR *_cursor = nullptr; +}; + +class scoped_session { + public: + scoped_session() = default; + explicit scoped_session(WT_CONNECTION *conn); + + ~scoped_session(); + + /* Moving is ok but copying is not. */ + scoped_session(scoped_session &&other); + + scoped_session &operator=(scoped_session &&other); + + scoped_session(const scoped_session &) = delete; + scoped_session &operator=(const scoped_session &) = delete; + + void reinit(WT_CONNECTION *conn); + + /* + * Override the dereference operators. The idea is that we should able to use this class as if + * it is a pointer to a WT_SESSION. + */ + WT_SESSION &operator*(); + WT_SESSION *operator->(); + + WT_SESSION *get(); + + scoped_cursor open_scoped_cursor(const std::string &uri, const std::string &cfg = ""); + + private: + WT_SESSION *_session = nullptr; +}; + +} // namespace test_harness +#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/checkpoint_manager.cpp b/src/third_party/wiredtiger/test/cppsuite/test_harness/checkpoint_manager.cpp deleted file mode 100644 index 66836325acf..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/checkpoint_manager.cpp +++ /dev/null @@ -1,57 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#include "checkpoint_manager.h" -#include "connection_manager.h" -#include "util/api_const.h" -#include "util/logger.h" - -namespace test_harness { -checkpoint_manager::checkpoint_manager(configuration *configuration) - : component(CHECKPOINT_MANAGER, configuration) -{ -} - -void -checkpoint_manager::load() -{ - /* Load the general component things. */ - component::load(); - - /* Create session that we'll use for checkpointing. */ - if (_enabled) - _session = connection_manager::instance().create_session(); -} - -void -checkpoint_manager::do_work() -{ - logger::log_msg(LOG_INFO, "Running checkpoint"); - testutil_check(_session->checkpoint(_session.get(), nullptr)); -} -} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/checkpoint_manager.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/checkpoint_manager.h deleted file mode 100644 index a127ee63657..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/checkpoint_manager.h +++ /dev/null @@ -1,53 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef CHECKPOINT_MANAGER_H -#define CHECKPOINT_MANAGER_H - -#include "core/component.h" -#include "util/scoped_types.h" - -namespace test_harness { -class checkpoint_manager : public component { - public: - explicit checkpoint_manager(configuration *configuration); - virtual ~checkpoint_manager() = default; - - /* Delete the copy constructor and the assignment operator. */ - checkpoint_manager(const checkpoint_manager &) = delete; - checkpoint_manager &operator=(const checkpoint_manager &) = delete; - - void load() override final; - void do_work() override final; - - private: - scoped_session _session; -}; -} // namespace test_harness - -#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/connection_manager.cpp b/src/third_party/wiredtiger/test/cppsuite/test_harness/connection_manager.cpp deleted file mode 100644 index 292cfbbfeab..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/connection_manager.cpp +++ /dev/null @@ -1,100 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#include "connection_manager.h" -#include "util/api_const.h" -#include "util/logger.h" -#include "util/scoped_connection.h" - -namespace test_harness { -connection_manager & -connection_manager::instance() -{ - static connection_manager _instance; - return (_instance); -} - -void -connection_manager::close() -{ - if (_conn != nullptr) { - testutil_check(_conn->close(_conn, nullptr)); - _conn = nullptr; - } -} - -void -connection_manager::create(const std::string &config, const std::string &home) -{ - if (_conn != nullptr) { - logger::log_msg(LOG_ERROR, "Connection is not NULL, cannot be re-opened."); - testutil_die(EINVAL, "Connection is not NULL"); - } - logger::log_msg(LOG_INFO, "wiredtiger_open config: " + config); - - /* Create the working dir. */ - testutil_make_work_dir(home.c_str()); - - /* Open conn. */ - testutil_check(wiredtiger_open(home.c_str(), nullptr, config.c_str(), &_conn)); -} - -scoped_session -connection_manager::create_session() -{ - if (_conn == nullptr) { - logger::log_msg(LOG_ERROR, - "Connection is NULL, did you forget to call " - "connection_manager::create ?"); - testutil_die(EINVAL, "Connection is NULL"); - } - - std::lock_guard lg(_conn_mutex); - scoped_session session(_conn); - - return (session); -} - -WT_CONNECTION * -connection_manager::get_connection() -{ - return (_conn); -} - -/* - * set_timestamp calls into the connection API in a thread safe manner to set global timestamps. - */ -void -connection_manager::set_timestamp(const std::string &config) -{ - std::lock_guard lg(_conn_mutex); - testutil_check(_conn->set_timestamp(_conn, config.c_str())); -} - -connection_manager::connection_manager() {} -} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/connection_manager.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/connection_manager.h deleted file mode 100644 index 2fef81d2af8..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/connection_manager.h +++ /dev/null @@ -1,83 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef CONN_API_H -#define CONN_API_H - -/* Following definitions are required in order to use printing format specifiers in C++. */ -#ifndef __STDC_LIMIT_MACROS -#define __STDC_LIMIT_MACROS -#endif -#ifndef __STDC_FORMAT_MACROS -#define __STDC_FORMAT_MACROS -#endif - -#include - -extern "C" { -#include "test_util.h" -#include "wiredtiger.h" -} - -#include "util/scoped_types.h" - -namespace test_harness { -/* - * Singleton class owning the database connection, provides access to sessions and any other - * required connection API calls. - */ -class connection_manager { - public: - static connection_manager &instance(); - - public: - /* No copies of the singleton allowed. */ - connection_manager(connection_manager const &) = delete; - connection_manager &operator=(connection_manager const &) = delete; - - void close(); - void create(const std::string &config, const std::string &home); - scoped_session create_session(); - - WT_CONNECTION *get_connection(); - - /* - * set_timestamp calls into the connection API in a thread safe manner to set global timestamps. - */ - void set_timestamp(const std::string &config); - - private: - connection_manager(); - - private: - WT_CONNECTION *_conn = nullptr; - std::mutex _conn_mutex; -}; -} // namespace test_harness - -#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/core/component.cpp b/src/third_party/wiredtiger/test/cppsuite/test_harness/core/component.cpp deleted file mode 100644 index aeb691fe674..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/core/component.cpp +++ /dev/null @@ -1,85 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#include "component.h" -#include "test_harness/util/api_const.h" - -namespace test_harness { -component::component(const std::string &name, configuration *config) : _config(config), _name(name) -{ -} - -component::~component() -{ - delete _config; -} - -void -component::load() -{ - logger::log_msg(LOG_INFO, "Loading component: " + _name); - _enabled = _config->get_optional_bool(ENABLED, true); - _throttle = throttle(_config); - /* If we're not enabled we shouldn't be running. */ - _running = _enabled; -} - -void -component::run() -{ - logger::log_msg(LOG_INFO, "Running component: " + _name); - while (_enabled && _running) { - do_work(); - _throttle.sleep(); - } -} - -void -component::do_work() -{ - /* Not implemented. */ -} - -bool -component::enabled() const -{ - return (_enabled); -} - -void -component::end_run() -{ - _running = false; -} - -void -component::finish() -{ - logger::log_msg(LOG_INFO, "Running finish stage of component: " + _name); -} -} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/core/component.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/core/component.h deleted file mode 100644 index 5a8f8f81cf5..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/core/component.h +++ /dev/null @@ -1,95 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef COMPONENT_H -#define COMPONENT_H - -#include "configuration.h" -#include "throttle.h" - -namespace test_harness { -/* - * A component is a class that defines 4 unique stages in its life-cycle, the stages must be run in - * the following order: load, run, end_run, finish. - * - * - */ -class component { - public: - explicit component(const std::string &name, configuration *config); - virtual ~component(); - - /* Delete the copy constructor and the assignment operator. */ - component(const component &) = delete; - component &operator=(const component &) = delete; - - /* - * The load function should perform all tasks required to setup the component for the main phase - * of the test. An example operation performed in the load phase would be populating a database. - */ - virtual void load(); - - /* - * The run function provides a top level loop that calls the do_work function every X seconds as - * defined by the throttle. Each run() method defined by the components is called in its own - * thread by the top level test class. - * - * If a component does not wish to use the standard run function, it can be overloaded. - */ - virtual void run(); - - /* end_run informs the component that is no longer running which closes out its run loop. */ - void end_run(); - - /* - * do_work is called every X seconds as defined by the throttle. Generally most components - * should do their "operation" in the do_work function. - */ - virtual void do_work(); - - /* Gets the value of the _enabled variable. */ - bool enabled() const; - - /* - * The finish phase is a cleanup phase. Created objects are destroyed here and any final testing - * requirements can be performed in this phase. An example could be the verification of the - * database, or checking some relevant statistics. - */ - virtual void finish(); - - protected: - bool _enabled = false; - volatile bool _running = false; - throttle _throttle; - configuration *_config; - - private: - std::string _name; -}; -} // namespace test_harness -#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/core/configuration.cpp b/src/third_party/wiredtiger/test/cppsuite/test_harness/core/configuration.cpp deleted file mode 100644 index f508dc76bea..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/core/configuration.cpp +++ /dev/null @@ -1,305 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#include -#include - -#include "configuration.h" - -namespace test_harness { -/* Static methods implementation. */ -static bool -config_item_to_bool(const WT_CONFIG_ITEM item) -{ - return (item.val != 0); -} - -static int64_t -config_item_to_int(const WT_CONFIG_ITEM item) -{ - return (item.val); -} - -static std::string -config_item_to_string(const WT_CONFIG_ITEM item) -{ - return std::string(item.str, item.len); -} - -static std::vector -config_item_to_list(const WT_CONFIG_ITEM item) -{ - auto str = config_item_to_string(item); - - /* Get rid of the brackets. */ - testutil_assert(!str.empty() && str.front() == '[' && str.back() == ']'); - str.pop_back(); - str.erase(0, 1); - - return (split_string(str, ',')); -} - -/* configuration class implementation. */ -configuration::configuration(const std::string &test_config_name, const std::string &config) -{ - const auto *config_entry = __wt_test_config_match(test_config_name.c_str()); - if (config_entry == nullptr) - testutil_die(EINVAL, "failed to match test config name"); - std::string default_config = std::string(config_entry->base); - /* Merge in the default configuration. */ - _config = merge_default_config(default_config, config); - logger::log_msg(LOG_INFO, "Full config: " + _config); - - int ret = - wiredtiger_test_config_validate(nullptr, nullptr, test_config_name.c_str(), _config.c_str()); - if (ret != 0) - testutil_die(EINVAL, "failed to validate given config, ensure test config exists"); - ret = wiredtiger_config_parser_open(nullptr, _config.c_str(), _config.size(), &_config_parser); - if (ret != 0) - testutil_die(EINVAL, "failed to create configuration parser for provided config"); -} - -configuration::configuration(const WT_CONFIG_ITEM &nested) -{ - if (nested.type != WT_CONFIG_ITEM::WT_CONFIG_ITEM_STRUCT) - testutil_die(EINVAL, "provided config item isn't a structure"); - int ret = wiredtiger_config_parser_open(nullptr, nested.str, nested.len, &_config_parser); - if (ret != 0) - testutil_die(EINVAL, "failed to create configuration parser for provided sub config"); -} - -configuration::~configuration() -{ - if (_config_parser != nullptr) { - _config_parser->close(_config_parser); - _config_parser = nullptr; - } -} - -std::string -configuration::get_string(const std::string &key) -{ - return get(key, false, types::STRING, "", config_item_to_string); -} - -std::string -configuration::get_optional_string(const std::string &key, const std::string &def) -{ - return get(key, true, types::STRING, def, config_item_to_string); -} - -bool -configuration::get_bool(const std::string &key) -{ - return get(key, false, types::BOOL, false, config_item_to_bool); -} - -bool -configuration::get_optional_bool(const std::string &key, const bool def) -{ - return get(key, true, types::BOOL, def, config_item_to_bool); -} - -int64_t -configuration::get_int(const std::string &key) -{ - return get(key, false, types::INT, 0, config_item_to_int); -} - -int64_t -configuration::get_optional_int(const std::string &key, const int64_t def) -{ - return get(key, true, types::INT, def, config_item_to_int); -} - -configuration * -configuration::get_subconfig(const std::string &key) -{ - return get(key, false, types::STRUCT, nullptr, - [](WT_CONFIG_ITEM item) { return new configuration(item); }); -} - -configuration * -configuration::get_optional_subconfig(const std::string &key) -{ - return get(key, true, types::STRUCT, nullptr, - [](WT_CONFIG_ITEM item) { return new configuration(item); }); -} - -std::vector -configuration::get_list(const std::string &key) -{ - return get>(key, false, types::LIST, {}, config_item_to_list); -} - -template -T -configuration::get( - const std::string &key, bool optional, types type, T def, T (*func)(WT_CONFIG_ITEM item)) -{ - WT_DECL_RET; - WT_CONFIG_ITEM value = {"", 0, 1, WT_CONFIG_ITEM::WT_CONFIG_ITEM_BOOL}; - - ret = _config_parser->get(_config_parser, key.c_str(), &value); - if (ret == WT_NOTFOUND && optional) - return (def); - else if (ret != 0) - testutil_die(ret, ("Error while finding config with key \"" + key + "\"").c_str()); - - const char *error_msg = "Configuration value doesn't match requested type"; - if (type == types::STRING && - (value.type != WT_CONFIG_ITEM::WT_CONFIG_ITEM_STRING && - value.type != WT_CONFIG_ITEM::WT_CONFIG_ITEM_ID)) - testutil_die(-1, error_msg); - else if (type == types::BOOL && value.type != WT_CONFIG_ITEM::WT_CONFIG_ITEM_BOOL) - testutil_die(-1, error_msg); - else if (type == types::INT && value.type != WT_CONFIG_ITEM::WT_CONFIG_ITEM_NUM) - testutil_die(-1, error_msg); - else if (type == types::STRUCT && value.type != WT_CONFIG_ITEM::WT_CONFIG_ITEM_STRUCT) - testutil_die(-1, error_msg); - else if (type == types::LIST && value.type != WT_CONFIG_ITEM::WT_CONFIG_ITEM_STRUCT) - testutil_die(-1, error_msg); - - return func(value); -} - -std::string -configuration::merge_default_config( - const std::string &default_config, const std::string &user_config) -{ - std::string merged_config; - auto split_default_config = split_config(default_config); - auto split_user_config = split_config(user_config); - auto user_it = split_user_config.begin(); - for (auto default_it = split_default_config.begin(); default_it != split_default_config.end(); - ++default_it) { - if (user_it == split_user_config.end() || user_it->first != default_it->first) - /* The default does not exist in the user configuration, add it. */ - merged_config += default_it->first + "=" + default_it->second; - else { - /* If we have a sub config merge it in. */ - if (user_it->second[0] == '(') - merged_config += default_it->first + "=(" + - merge_default_config(default_it->second, user_it->second) + ')'; - else - /* Add the user configuration as it exists. */ - merged_config += user_it->first + "=" + user_it->second; - ++user_it; - } - /* Add a comma after every item we add except the last one. */ - if (split_default_config.end() - default_it != 1) - merged_config += ","; - } - /* Add any remaining user config items. */ - while (user_it != split_user_config.end()) { - merged_config += "," + user_it->first + "=" + user_it->second; - ++user_it; - } - return (merged_config); -} - -std::vector> -configuration::split_config(const std::string &config) -{ - std::string cut_config = config; - std::vector> split_config; - std::string key = "", value = ""; - bool in_subconfig = false; - bool expect_value = false; - std::stack parens; - - /* All configuration strings must be at least 2 characters. */ - testutil_assert(config.size() > 1); - - /* Remove prefix and trailing "()". */ - if (config[0] == '(') - cut_config = config.substr(1, config.size() - 2); - - size_t start = 0, len = 0; - for (size_t i = 0; i < cut_config.size(); ++i) { - if (cut_config[i] == '(' || cut_config[i] == '[') { - parens.push(cut_config[i]); - in_subconfig = true; - } - if (cut_config[i] == ')' || cut_config[i] == ']') { - parens.pop(); - in_subconfig = !parens.empty(); - } - if (cut_config[i] == '=' && !in_subconfig) { - if (len == 0) { - testutil_die(EINVAL, "error parsing config: detected empty key"); - } - if (expect_value) { - testutil_die(EINVAL, - "error parsing config: syntax error parsing value for key ['%s']: '%s'", - key.c_str(), cut_config.substr(start, len).c_str()); - } - expect_value = true; - key = cut_config.substr(start, len); - start += len + 1; - len = 0; - continue; - } - if (cut_config[i] == ',' && !in_subconfig) { - if (len == 0) { - testutil_die( - EINVAL, "error parsing config: detected empty value for key:'%s'", key.c_str()); - } - if (!expect_value) { - testutil_die(EINVAL, - "error parsing config: syntax error parsing key value pair: '%s'", - cut_config.substr(start, len).c_str()); - } - expect_value = false; - if (start + len >= cut_config.size()) - break; - value = cut_config.substr(start, len); - start += len + 1; - len = 0; - split_config.push_back(std::make_pair(key, value)); - continue; - } - ++len; - } - if (expect_value) { - value = cut_config.substr(start, len); - split_config.push_back(std::make_pair(key, value)); - } - - /* We have to sort the config here otherwise we will match incorrectly while merging. */ - std::sort(split_config.begin(), split_config.end(), comparator); - return (split_config); -} - -bool -configuration::comparator( - std::pair a, std::pair b) -{ - return (a.first < b.first); -} -} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/core/configuration.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/core/configuration.h deleted file mode 100644 index b61defc9d3f..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/core/configuration.h +++ /dev/null @@ -1,114 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef CONFIGURATION_H -#define CONFIGURATION_H - -#include -#include - -#include "test_harness/util/logger.h" - -extern "C" { -#include "test_util.h" -} - -namespace test_harness { -inline std::vector -split_string(const std::string &str, const char delim) -{ - std::vector splits; - std::string current_str; - for (const auto c : str) { - if (c == delim) { - if (!current_str.empty()) { - splits.push_back(current_str); - current_str.clear(); - } - } else - current_str.push_back(c); - } - if (!current_str.empty()) - splits.push_back(std::move(current_str)); - return (splits); -} - -class configuration { - public: - explicit configuration(const std::string &test_config_name, const std::string &config); - explicit configuration(const WT_CONFIG_ITEM &nested); - - ~configuration(); - - /* - * Wrapper functions for retrieving basic configuration values. Ideally tests can avoid using - * the config item struct provided by wiredtiger. - * - * When getting a configuration value that may not exist for that configuration string or - * component, the optional forms of the functions can be used. In this case a default value must - * be passed and it will be set to that value. - */ - std::string get_string(const std::string &key); - std::string get_optional_string(const std::string &key, const std::string &def); - bool get_bool(const std::string &key); - bool get_optional_bool(const std::string &key, const bool def); - int64_t get_int(const std::string &key); - int64_t get_optional_int(const std::string &key, const int64_t def); - configuration *get_subconfig(const std::string &key); - configuration *get_optional_subconfig(const std::string &key); - std::vector get_list(const std::string &key); - std::vector get_optional_list(const std::string &key); - - private: - enum class types { BOOL, INT, LIST, STRING, STRUCT }; - - template - T get(const std::string &key, bool optional, types type, T def, T (*func)(WT_CONFIG_ITEM item)); - - /* - * Merge together two configuration strings, the user one and the default one. - */ - static std::string merge_default_config( - const std::string &default_config, const std::string &user_config); - - /* - * Split a config string into keys and values, taking care to not split incorrectly when we have - * a sub config or array. - */ - static std::vector> split_config(const std::string &config); - - static bool comparator( - std::pair a, std::pair b); - - private: - std::string _config; - WT_CONFIG_PARSER *_config_parser = nullptr; -}; -} // namespace test_harness - -#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/core/op_tracker.cpp b/src/third_party/wiredtiger/test/cppsuite/test_harness/core/op_tracker.cpp deleted file mode 100644 index af806927768..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/core/op_tracker.cpp +++ /dev/null @@ -1,37 +0,0 @@ -#include "test_harness/test.h" -#include "test_harness/util/perf_plotter.h" -#include "op_tracker.h" - -namespace test_harness { -op_tracker::op_tracker(const std::string id, const std::string &test_name) - : _id(id), _test_name(test_name), _it_count(0), _total_time_taken(0) -{ -} - -void -op_tracker::append_stats() -{ - uint64_t avg = (uint64_t)_total_time_taken / _it_count; - std::string stat = "{\"name\":\"" + _id + "\",\"value\":" + std::to_string(avg) + "}"; - perf_plotter::instance().add_stat(stat); -} - -template -auto -op_tracker::track(T lambda) -{ - auto _start_time = std::chrono::steady_clock::now(); - int ret = lambda(); - auto _end_time = std::chrono::steady_clock::now(); - _total_time_taken += (_end_time - _start_time).count(); - _it_count += 1; - - return ret; -} - -op_tracker::~op_tracker() -{ - if (_it_count != 0) - append_stats(); -} -}; // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/core/op_tracker.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/core/op_tracker.h deleted file mode 100644 index eea99b2414c..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/core/op_tracker.h +++ /dev/null @@ -1,58 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#include -#include -#include - -namespace test_harness { - -/* - * Class that tracks the performance of given operations and appends the stats to the perf file. - */ -class op_tracker { - public: - explicit op_tracker(const std::string id, const std::string &test_name); - virtual ~op_tracker(); - - /* Calculates the average time and appends the stat to the perf file. */ - void append_stats(); - - /* - * Does timing for a given operation and keeps track of how many operations have been executed - * as well as total time taken. - */ - template auto track(T lambda); - - private: - std::string _id; - std::string _test_name; - int _it_count; - uint64_t _total_time_taken; -}; -} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/core/throttle.cpp b/src/third_party/wiredtiger/test/cppsuite/test_harness/core/throttle.cpp deleted file mode 100644 index af7b94e8f98..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/core/throttle.cpp +++ /dev/null @@ -1,75 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#include - -#include "configuration.h" -#include "test_harness/util/api_const.h" -#include "throttle.h" - -namespace test_harness { -throttle::throttle(const std::string &throttle_rate) -{ - std::string magnitude; - uint64_t multiplier = 0; - /* - * Find the ms, s, or m in the string. Searching for "ms" first as the following two searches - * would match as well. - */ - size_t pos = throttle_rate.find("ms"); - if (pos != std::string::npos) - multiplier = 1; - else { - pos = throttle_rate.find("s"); - if (pos != std::string::npos) - multiplier = 1000; - else { - pos = throttle_rate.find("m"); - if (pos != std::string::npos) - multiplier = 60 * 1000; - else - testutil_die(-1, "no rate specifier given"); - } - } - magnitude = throttle_rate.substr(0, pos); - /* This will throw if it can't cast, which is fine. */ - _ms = std::stoi(magnitude) * multiplier; -} - -/* Use optional and default to 1s per op in case something doesn't define this. */ -throttle::throttle(configuration *config) : throttle(config->get_optional_string(OP_RATE, "1s")) {} - -/* Default to a second per operation. */ -throttle::throttle() : throttle("1s") {} - -void -throttle::sleep() -{ - std::this_thread::sleep_for(std::chrono::milliseconds(_ms)); -} -} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/core/throttle.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/core/throttle.h deleted file mode 100644 index 24161ea956e..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/core/throttle.h +++ /dev/null @@ -1,55 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef THROTTLE_H -#define THROTTLE_H - -#include - -/* Forward declarations for classes to reduce compilation time and modules coupling. */ -class configuration; - -namespace test_harness { -class throttle { - public: - explicit throttle(const std::string &throttle_rate); - - /* Use optional and default to 1s per op in case something doesn't define this. */ - explicit throttle(configuration *config); - - /* Default to a second per operation. */ - throttle(); - - void sleep(); - - private: - uint64_t _ms = 1000; -}; -} // namespace test_harness - -#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.cpp b/src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.cpp deleted file mode 100644 index 7298175e858..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.cpp +++ /dev/null @@ -1,370 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#include - -#include "connection_manager.h" -#include "core/component.h" -#include "core/configuration.h" -#include "core/throttle.h" -#include "runtime_monitor.h" -#include "util/api_const.h" -#include "util/logger.h" -#include "util/perf_plotter.h" - -namespace test_harness { - -/* Static methods implementation. */ -static std::string -collection_name_to_file_name(const std::string &collection_name) -{ - /* Strip out the URI prefix. */ - const size_t colon_pos = collection_name.find(':'); - testutil_assert(colon_pos != std::string::npos); - const auto stripped_name = collection_name.substr(colon_pos + 1); - - /* Now add the directory and file extension. */ - return (std::string(DEFAULT_DIR) + "/" + stripped_name + ".wt"); -} - -/* Inline methods implementation. */ - -/* - * The WiredTiger configuration API doesn't accept string statistic names when retrieving statistic - * values. This function provides the required mapping to statistic id. We should consider - * generating it programmatically in `stat.py` to avoid having to manually add a condition every - * time we want to observe a new postrun statistic. - */ -inline int -get_stat_field(const std::string &name) -{ - if (name == CACHE_HS_INSERT) - return (WT_STAT_CONN_CACHE_HS_INSERT); - else if (name == CC_PAGES_REMOVED) - return (WT_STAT_CONN_CC_PAGES_REMOVED); - testutil_die(EINVAL, "get_stat_field: Stat \"%s\" is unrecognized", name.c_str()); -} - -/* statistics class implementation */ -statistics::statistics(configuration &config, const std::string &stat_name, int stat_field) - : field(stat_field), max(config.get_int(MAX)), min(config.get_int(MIN)), name(stat_name), - postrun(config.get_bool(POSTRUN_STATISTICS)), runtime(config.get_bool(RUNTIME_STATISTICS)), - save(config.get_bool(SAVE)) -{ -} - -void -statistics::check(scoped_cursor &cursor) -{ - int64_t stat_value; - runtime_monitor::get_stat(cursor, field, &stat_value); - if (stat_value < min || stat_value > max) { - const std::string error_string = "runtime_monitor: Postrun stat \"" + name + - "\" was outside of the specified limits. Min=" + std::to_string(min) + - " Max=" + std::to_string(max) + " Actual=" + std::to_string(stat_value); - testutil_die(-1, error_string.c_str()); - } else - logger::log_msg(LOG_TRACE, name + " usage: " + std::to_string(stat_value)); -} - -std::string -statistics::get_value_str(scoped_cursor &cursor) -{ - int64_t stat_value; - runtime_monitor::get_stat(cursor, field, &stat_value); - return std::to_string(stat_value); -} - -int -statistics::get_field() const -{ - return field; -} - -int64_t -statistics::get_max() const -{ - return max; -} - -int64_t -statistics::get_min() const -{ - return min; -} - -const std::string & -statistics::get_name() const -{ - return name; -} - -bool -statistics::get_postrun() const -{ - return postrun; -} - -bool -statistics::get_runtime() const -{ - return runtime; -} - -bool -statistics::get_save() const -{ - return save; -} - -/* cache_limit_statistic class implementation */ -cache_limit_statistic::cache_limit_statistic(configuration &config, const std::string &name) - : statistics(config, name, -1) -{ -} - -void -cache_limit_statistic::check(scoped_cursor &cursor) -{ - double use_percent = get_cache_value(cursor); - if (use_percent > max) { - const std::string error_string = - "runtime_monitor: Cache usage exceeded during test! Limit: " + std::to_string(max) + - " usage: " + std::to_string(use_percent); - testutil_die(-1, error_string.c_str()); - } else - logger::log_msg(LOG_TRACE, name + " usage: " + std::to_string(use_percent)); -} - -std::string -cache_limit_statistic::get_value_str(scoped_cursor &cursor) -{ - return std::to_string(get_cache_value(cursor)); -} - -double -cache_limit_statistic::get_cache_value(scoped_cursor &cursor) -{ - int64_t cache_bytes_image, cache_bytes_other, cache_bytes_max; - double use_percent; - /* Three statistics are required to compute cache use percentage. */ - runtime_monitor::get_stat(cursor, WT_STAT_CONN_CACHE_BYTES_IMAGE, &cache_bytes_image); - runtime_monitor::get_stat(cursor, WT_STAT_CONN_CACHE_BYTES_OTHER, &cache_bytes_other); - runtime_monitor::get_stat(cursor, WT_STAT_CONN_CACHE_BYTES_MAX, &cache_bytes_max); - /* - * Assert that we never exceed our configured limit for cache usage. Add 0.0 to avoid floating - * point conversion errors. - */ - testutil_assert(cache_bytes_max > 0); - use_percent = ((cache_bytes_image + cache_bytes_other + 0.0) / cache_bytes_max) * 100; - return use_percent; -} - -/* db_size_statistic class implementation */ -db_size_statistic::db_size_statistic( - configuration &config, const std::string &name, database &database) - : statistics(config, name, -1), _database(database) -{ -#ifdef _WIN32 - Logger::log_msg("Database size checking is not implemented on Windows", LOG_ERROR); -#endif -} - -void -db_size_statistic::check(scoped_cursor &) -{ -#ifndef _WIN32 - const auto file_names = get_file_names(); - size_t db_size = get_db_size(); - logger::log_msg(LOG_TRACE, "Current database size is " + std::to_string(db_size) + " bytes"); - - if (db_size > max) { - const std::string error_string = - "runtime_monitor: Database size limit exceeded during test! Limit: " + - std::to_string(max) + " db size: " + std::to_string(db_size); - testutil_die(-1, error_string.c_str()); - } -#endif -} - -std::string -db_size_statistic::get_value_str(scoped_cursor &) -{ - return std::to_string(get_db_size()); -} - -size_t -db_size_statistic::get_db_size() const -{ - const auto file_names = get_file_names(); - size_t db_size = 0; - - for (const auto &name : file_names) { - struct stat sb; - if (stat(name.c_str(), &sb) == 0) { - db_size += sb.st_size; - logger::log_msg(LOG_TRACE, name + " was " + std::to_string(sb.st_size) + " bytes"); - } else - /* The only good reason for this to fail is if the file hasn't been created yet. */ - testutil_assert(errno == ENOENT); - } - - return db_size; -} - -const std::vector -db_size_statistic::get_file_names() const -{ - std::vector file_names; - for (const auto &name : _database.get_collection_names()) - file_names.push_back(collection_name_to_file_name(name)); - - /* Add WiredTiger internal tables. */ - file_names.push_back(std::string(DEFAULT_DIR) + "/" + WT_HS_FILE); - file_names.push_back(std::string(DEFAULT_DIR) + "/" + WT_METAFILE); - - return (file_names); -} - -/* runtime_monitor class implementation */ -void -runtime_monitor::get_stat(scoped_cursor &cursor, int stat_field, int64_t *valuep) -{ - const char *desc, *pvalue; - cursor->set_key(cursor.get(), stat_field); - testutil_check(cursor->search(cursor.get())); - testutil_check(cursor->get_value(cursor.get(), &desc, &pvalue, valuep)); - testutil_check(cursor->reset(cursor.get())); -} - -runtime_monitor::runtime_monitor( - const std::string &test_name, configuration *config, database &database) - : component(RUNTIME_MONITOR, config), _test_name(test_name), _database(database) -{ -} - -void -runtime_monitor::load() -{ - /* Load the general component things. */ - component::load(); - - /* If the component is enabled, load all the known statistics. */ - if (_enabled) { - - std::unique_ptr stat_config(_config->get_subconfig(STAT_CACHE_SIZE)); - _stats.push_back(std::unique_ptr( - new cache_limit_statistic(*stat_config, STAT_CACHE_SIZE))); - - stat_config.reset(_config->get_subconfig(STAT_DB_SIZE)); - _stats.push_back(std::unique_ptr( - new db_size_statistic(*stat_config, STAT_DB_SIZE, _database))); - - stat_config.reset(_config->get_subconfig(CACHE_HS_INSERT)); - _stats.push_back(std::unique_ptr( - new statistics(*stat_config, CACHE_HS_INSERT, get_stat_field(CACHE_HS_INSERT)))); - - stat_config.reset(_config->get_subconfig(CC_PAGES_REMOVED)); - _stats.push_back(std::unique_ptr( - new statistics(*stat_config, CC_PAGES_REMOVED, get_stat_field(CC_PAGES_REMOVED)))); - - /* Open our statistic cursor. */ - _session = connection_manager::instance().create_session(); - _cursor = _session.open_scoped_cursor(STATISTICS_URI); - } -} - -void -runtime_monitor::do_work() -{ - /* Check runtime statistics. */ - for (const auto &stat : _stats) { - if (stat->get_runtime()) - stat->check(_cursor); - } -} - -void -runtime_monitor::finish() -{ - component::finish(); - - /* Append stats to perf plotter. */ - append_stats(); - - /* Check the post run statistics now. */ - bool success = true; - int64_t stat_max, stat_min, stat_value; - std::string stat_name; - - for (const auto &stat : _stats) { - - if (!stat->get_postrun()) - continue; - - stat_max = stat->get_max(); - stat_min = stat->get_min(); - stat_name = stat->get_name(); - - stat_value = std::stoi(stat->get_value_str(_cursor)); - - if (stat_value < stat_min || stat_value > stat_max) { - const std::string error_string = "runtime_monitor: Postrun stat \"" + stat_name + - "\" was outside of the specified limits. Min=" + std::to_string(stat_min) + - " Max=" + std::to_string(stat_max) + " Actual=" + std::to_string(stat_value); - logger::log_msg(LOG_ERROR, error_string); - success = false; - } - - logger::log_msg(LOG_INFO, - "runtime_monitor: Final value of stat " + stat_name + - " is: " + std::to_string(stat_value)); - } - - if (!success) - testutil_die(-1, - "runtime_monitor: One or more postrun statistics were outside of their specified " - "limits."); -} - -/* - * This function appends the values of the different statistics that need to be saved as indicated - * by the configuration file to the perf plotter. - */ -void -runtime_monitor::append_stats() -{ - for (const auto &stat : _stats) { - if (stat->get_save()) { - auto stat_str = "{\"name\":\"" + stat->get_name() + - "\",\"value\":" + stat->get_value_str(_cursor) + "}"; - perf_plotter::instance().add_stat(stat_str); - } - } -} -} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.h deleted file mode 100644 index a02d9979801..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.h +++ /dev/null @@ -1,141 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef RUNTIME_MONITOR_H -#define RUNTIME_MONITOR_H - -#include -#include - -extern "C" { -#include "test_util.h" -#include "wiredtiger.h" -} - -#include "util/scoped_types.h" -#include "workload/database_model.h" - -/* Forward declarations for classes to reduce compilation time and modules coupling. */ -class configuration; - -namespace test_harness { - -class statistics { - public: - statistics() = default; - explicit statistics(configuration &config, const std::string &stat_name, int stat_field); - virtual ~statistics() = default; - - /* Check that the statistics are within bounds. */ - virtual void check(scoped_cursor &cursor); - - /* Retrieve the value associated to the stat in a string format. */ - virtual std::string get_value_str(scoped_cursor &cursor); - - /* Getters. */ - int get_field() const; - int64_t get_max() const; - int64_t get_min() const; - const std::string &get_name() const; - bool get_postrun() const; - bool get_runtime() const; - bool get_save() const; - - protected: - int field; - int64_t max; - int64_t min; - std::string name; - bool postrun; - bool runtime; - bool save; -}; - -class cache_limit_statistic : public statistics { - public: - explicit cache_limit_statistic(configuration &config, const std::string &name); - virtual ~cache_limit_statistic() = default; - - void check(scoped_cursor &cursor) override final; - std::string get_value_str(scoped_cursor &cursor) override final; - - private: - double get_cache_value(scoped_cursor &cursor); -}; - -class db_size_statistic : public statistics { - public: - explicit db_size_statistic(configuration &config, const std::string &name, database &database); - virtual ~db_size_statistic() = default; - - /* Don't need the stat cursor for these. */ - void check(scoped_cursor &) override final; - std::string get_value_str(scoped_cursor &) override final; - - private: - size_t get_db_size() const; - const std::vector get_file_names() const; - - private: - database &_database; -}; - -/* - * The runtime monitor class is designed to track various statistics or other runtime signals - * relevant to the given workload. - */ -class runtime_monitor : public component { - public: - static void get_stat(scoped_cursor &, int, int64_t *); - - public: - explicit runtime_monitor( - const std::string &test_name, configuration *config, database &database); - virtual ~runtime_monitor() = default; - - /* Delete the copy constructor and the assignment operator. */ - runtime_monitor(const runtime_monitor &) = delete; - runtime_monitor &operator=(const runtime_monitor &) = delete; - - void load() override final; - void do_work() override final; - void finish() override final; - - private: - void append_stats(); - - private: - scoped_session _session; - scoped_cursor _cursor; - const std::string _test_name; - std::vector> _stats; - database &_database; -}; -} // namespace test_harness - -#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/test.cpp b/src/third_party/wiredtiger/test/cppsuite/test_harness/test.cpp deleted file mode 100644 index 9f93415ac64..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/test.cpp +++ /dev/null @@ -1,198 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -/* Required to build using older versions of g++. */ -#include -#include -#include - -#include "connection_manager.h" -#include "core/component.h" -#include "core/configuration.h" -#include "test.h" -#include "thread_manager.h" -#include "timestamp_manager.h" -#include "util/api_const.h" -#include "util/perf_plotter.h" - -namespace test_harness { -test::test(const test_args &args) : _args(args) -{ - _config = new configuration(args.test_name, args.test_config); - _checkpoint_manager = new checkpoint_manager(_config->get_subconfig(CHECKPOINT_MANAGER)); - _runtime_monitor = - new runtime_monitor(args.test_name, _config->get_subconfig(RUNTIME_MONITOR), _database); - _timestamp_manager = new timestamp_manager(_config->get_subconfig(TIMESTAMP_MANAGER)); - _workload_generator = new workload_generator( - _config->get_subconfig(WORKLOAD_GENERATOR), this, _timestamp_manager, _database); - _thread_manager = new thread_manager(); - - _database.set_timestamp_manager(_timestamp_manager); - _database.set_create_config( - _config->get_bool(COMPRESSION_ENABLED), _config->get_bool(REVERSE_COLLATOR)); - - /* - * Ordering is not important here, any dependencies between components should be resolved - * internally by the components. - */ - _components = {_workload_generator, _timestamp_manager, _runtime_monitor, _checkpoint_manager}; -} - -void -test::init_tracking(workload_tracking *tracking) -{ - delete _workload_tracking; - if (tracking == nullptr) { - /* Fallback to default behavior. */ - tracking = new workload_tracking(_config->get_subconfig(WORKLOAD_TRACKING), - _config->get_bool(COMPRESSION_ENABLED), *_timestamp_manager); - } - _workload_tracking = tracking; - _workload_generator->set_workload_tracking(_workload_tracking); - _database.set_workload_tracking(_workload_tracking); - _components.push_back(_workload_tracking); -} - -test::~test() -{ - delete _config; - delete _checkpoint_manager; - delete _runtime_monitor; - delete _timestamp_manager; - delete _thread_manager; - delete _workload_generator; - delete _workload_tracking; - _config = nullptr; - _checkpoint_manager = nullptr; - _runtime_monitor = nullptr; - _timestamp_manager = nullptr; - _thread_manager = nullptr; - _workload_generator = nullptr; - _workload_tracking = nullptr; - - _components.clear(); -} - -void -test::run() -{ - int64_t cache_max_wait_ms, cache_size_mb, duration_seconds; - bool enable_logging, statistics_logging; - configuration *statistics_config; - std::string statistics_type; - /* Build the database creation config string. */ - std::string db_create_config = CONNECTION_CREATE; - - /* Enable snappy compression or reverse collator if required. */ - if (_config->get_bool(COMPRESSION_ENABLED) || _config->get_bool(REVERSE_COLLATOR)) { - db_create_config += ",extensions=["; - db_create_config += - _config->get_bool(COMPRESSION_ENABLED) ? std::string(SNAPPY_PATH) + "," : ""; - db_create_config += - _config->get_bool(REVERSE_COLLATOR) ? std::string(REVERSE_COLLATOR_PATH) : ""; - db_create_config += "]"; - } - - /* Get the cache size. */ - cache_size_mb = _config->get_int(CACHE_SIZE_MB); - db_create_config += ",cache_size=" + std::to_string(cache_size_mb) + "MB"; - - /* Get the statistics configuration for this run. */ - statistics_config = _config->get_subconfig(STATISTICS_CONFIG); - statistics_type = statistics_config->get_string(TYPE); - statistics_logging = statistics_config->get_bool(ENABLE_LOGGING); - db_create_config += statistics_logging ? "," + STATISTICS_LOG : ""; - db_create_config += ",statistics=(" + statistics_type + ")"; - /* Don't forget to delete. */ - delete statistics_config; - - /* Enable or disable write ahead logging. */ - enable_logging = _config->get_bool(ENABLE_LOGGING); - db_create_config += ",log=(enabled=" + std::string(enable_logging ? "true" : "false") + ")"; - - /* Maximum waiting time for the cache to get unstuck. */ - cache_max_wait_ms = _config->get_int(CACHE_MAX_WAIT_MS); - db_create_config += ",cache_max_wait_ms=" + std::to_string(cache_max_wait_ms); - - /* Add the user supplied wiredtiger open config. */ - db_create_config += _args.wt_open_config; - - /* - * Set up the test environment. A smart pointer is used here so that the connection can - * automatically be closed by the scoped_connection's destructor when the test finishes and the - * pointer goes out of scope. - */ - _scoped_conn = std::make_shared(db_create_config); - - /* Initiate the load stage of each component. */ - for (const auto &it : _components) - it->load(); - - /* Spawn threads for all component::run() functions. */ - for (const auto &it : _components) - _thread_manager->add_thread(&component::run, it); - - /* The initial population phase needs to be finished before starting the actual test. */ - while (_workload_generator->enabled() && !_workload_generator->db_populated()) - std::this_thread::sleep_for(std::chrono::milliseconds(10)); - - /* The test will run for the duration as defined in the config. */ - duration_seconds = _config->get_int(DURATION_SECONDS); - testutil_assert(duration_seconds >= 0); - logger::log_msg(LOG_INFO, - "Waiting {" + std::to_string(duration_seconds) + "} seconds for testing to complete."); - std::this_thread::sleep_for(std::chrono::seconds(duration_seconds)); - - /* Notify components that they should complete their last iteration. */ - for (const auto &it : _components) - it->end_run(); - - /* Call join on the components threads so we know they have finished their loop. */ - logger::log_msg(LOG_INFO, - "Joining all component threads.\n This could take a while as we need to wait" - " for all components to finish their current loop."); - _thread_manager->join(); - - /* End the test by calling finish on all known components. */ - for (const auto &it : _components) - it->finish(); - - /* Validation stage. */ - if (_workload_tracking->enabled()) { - std::unique_ptr tracking_config(_config->get_subconfig(WORKLOAD_TRACKING)); - this->validate(_workload_tracking->get_operation_table_name(), - _workload_tracking->get_schema_table_name(), - _workload_generator->get_database().get_collection_ids()); - } - - /* Log perf stats. */ - perf_plotter::instance().output_perf_file(_args.test_name); - - logger::log_msg(LOG_INFO, "SUCCESS"); -} -} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/test.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/test.h deleted file mode 100644 index 5e851cf7d03..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/test.h +++ /dev/null @@ -1,105 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef TEST_H -#define TEST_H - -#include -#include -#include - -extern "C" { -#include "wiredtiger.h" -} - -#include "checkpoint_manager.h" -#include "connection_manager.h" -#include "runtime_monitor.h" -#include "util/scoped_connection.h" -#include "workload/database_operation.h" -#include "workload_generator.h" - -namespace test_harness { -class test_args { - public: - test_args(const std::string &config, const std::string &name, const std::string &wt_open_config) - : test_config(config), test_name(name), wt_open_config(wt_open_config) - { - } - const std::string test_config; - const std::string test_name; - const std::string wt_open_config; -}; - -/* - * The base class for a test, the standard usage pattern is to just call run(). - */ -class test : public database_operation { - public: - explicit test(const test_args &args); - virtual ~test(); - - /* Delete the copy constructor and the assignment operator. */ - test(const test &) = delete; - test &operator=(const test &) = delete; - - /* Initialize the tracking component and its dependencies. */ - void init_tracking(workload_tracking *tracking = nullptr); - - /* - * The primary run function that most tests will be able to utilize without much other code. - */ - virtual void run(); - - /* - * Getters for all the major components, used if a test wants more control over the test - * program. - */ - workload_generator *get_workload_generator(); - runtime_monitor *get_runtime_monitor(); - timestamp_manager *get_timestamp_manager(); - thread_manager *get_thread_manager(); - - protected: - const test_args &_args; - configuration *_config; - timestamp_manager *_timestamp_manager = nullptr; - workload_tracking *_workload_tracking = nullptr; - - private: - std::vector _components; - checkpoint_manager *_checkpoint_manager = nullptr; - runtime_monitor *_runtime_monitor = nullptr; - thread_manager *_thread_manager = nullptr; - workload_generator *_workload_generator = nullptr; - std::shared_ptr _scoped_conn; - database _database; -}; -} // namespace test_harness - -#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/thread_manager.cpp b/src/third_party/wiredtiger/test/cppsuite/test_harness/thread_manager.cpp deleted file mode 100644 index 29d1e547151..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/thread_manager.cpp +++ /dev/null @@ -1,59 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#include "test_harness/util/logger.h" -#include "thread_manager.h" - -namespace test_harness { -thread_manager::~thread_manager() -{ - for (auto &it : _workers) { - if (it != nullptr && it->joinable()) { - logger::log_msg(LOG_ERROR, "You should've called join on the thread manager"); - it->join(); - } - delete it; - it = nullptr; - } - _workers.clear(); -} - -void -thread_manager::join() -{ - for (const auto &it : _workers) { - while (!it->joinable()) { - /* Helpful for diagnosing hangs. */ - logger::log_msg(LOG_TRACE, "Thread manager: Waiting to join."); - /* Check every so often to avoid spamming the log. */ - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - } - it->join(); - } -} -} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/thread_manager.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/thread_manager.h deleted file mode 100644 index 0fce6bfeb45..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/thread_manager.h +++ /dev/null @@ -1,62 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef THREAD_MANAGER_H -#define THREAD_MANAGER_H - -#include -#include - -namespace test_harness { -/* Class that handles threads, from their initialization to their deletion. */ -class thread_manager { - public: - ~thread_manager(); - - /* - * Generic function to create threads that call member function of classes. - */ - template - void - add_thread(Callable &&fct, Args &&... args) - { - std::thread *t = new std::thread(fct, std::forward(args)...); - _workers.push_back(t); - } - - /* - * Complete the operations for all threads. - */ - void join(); - - private: - std::vector _workers; -}; -} // namespace test_harness - -#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/timestamp_manager.cpp b/src/third_party/wiredtiger/test/cppsuite/test_harness/timestamp_manager.cpp deleted file mode 100644 index fce4b3ca789..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/timestamp_manager.cpp +++ /dev/null @@ -1,151 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#include -#include -#include - -#include "connection_manager.h" -#include "core/configuration.h" -#include "timestamp_manager.h" -#include "util/api_const.h" -#include "workload/random_generator.h" - -namespace test_harness { -const std::string -timestamp_manager::decimal_to_hex(uint64_t value) -{ - std::stringstream ss; - ss << std::hex << value; - std::string res(ss.str()); - return (res); -} - -timestamp_manager::timestamp_manager(configuration *config) : component("timestamp_manager", config) -{ -} - -void -timestamp_manager::load() -{ - component::load(); - int64_t oldest_lag = _config->get_int(OLDEST_LAG); - testutil_assert(oldest_lag >= 0); - /* Cast and then shift left to match the seconds position. */ - _oldest_lag = oldest_lag; - _oldest_lag <<= 32; - - int64_t stable_lag = _config->get_int(STABLE_LAG); - testutil_assert(stable_lag >= 0); - /* Cast and then shift left to match the seconds position. */ - _stable_lag = stable_lag; - _stable_lag <<= 32; -} - -void -timestamp_manager::do_work() -{ - std::string config, log_msg; - /* latest_ts_s represents the time component of the latest timestamp provided. */ - wt_timestamp_t latest_ts_s = get_time_now_s(); - - /* - * Keep a time window between the latest and stable ts less than the max defined in the - * configuration. - */ - wt_timestamp_t new_stable_ts = _stable_ts; - testutil_assert(latest_ts_s >= _stable_ts); - if ((latest_ts_s - _stable_ts) > _stable_lag) { - log_msg = "Timestamp_manager: Stable timestamp expired."; - new_stable_ts = latest_ts_s - _stable_lag; - config += STABLE_TS + "=" + decimal_to_hex(new_stable_ts); - } - - /* - * Keep a time window between the stable and oldest ts less than the max defined in the - * configuration. - */ - wt_timestamp_t new_oldest_ts = _oldest_ts; - testutil_assert(_stable_ts >= _oldest_ts); - if ((new_stable_ts - _oldest_ts) > _oldest_lag) { - if (log_msg.empty()) - log_msg = "Timestamp_manager: Oldest timestamp expired."; - else - log_msg += " Oldest timestamp expired."; - new_oldest_ts = new_stable_ts - _oldest_lag; - if (!config.empty()) - config += ","; - config += OLDEST_TS + "=" + decimal_to_hex(new_oldest_ts); - } - - if (!log_msg.empty()) - logger::log_msg(LOG_TRACE, log_msg); - - /* - * Save the new timestamps. Any timestamps that we're viewing from another thread should be set - * AFTER we've saved the new timestamps to avoid races where we sweep data that is not yet - * obsolete. - */ - if (!config.empty()) { - connection_manager::instance().set_timestamp(config); - _oldest_ts = new_oldest_ts; - _stable_ts = new_stable_ts; - } -} - -wt_timestamp_t -timestamp_manager::get_next_ts() -{ - uint64_t current_time = get_time_now_s(); - uint64_t increment = _increment_ts.fetch_add(1); - current_time |= (increment & 0x00000000FFFFFFFF); - return (current_time); -} - -wt_timestamp_t -timestamp_manager::get_oldest_ts() const -{ - return (_oldest_ts); -} - -wt_timestamp_t -timestamp_manager::get_random_ts() const -{ - return random_generator::instance().generate_integer( - _oldest_ts, static_cast(get_time_now_s())); -} - -uint64_t -timestamp_manager::get_time_now_s() const -{ - auto now = std::chrono::system_clock::now().time_since_epoch(); - uint64_t current_time_s = - static_cast(std::chrono::duration_cast(now).count()) << 32; - return (current_time_s); -} -} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/timestamp_manager.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/timestamp_manager.h deleted file mode 100644 index 88cff40c30e..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/timestamp_manager.h +++ /dev/null @@ -1,85 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef TIMESTAMP_MANAGER_H -#define TIMESTAMP_MANAGER_H - -#include -#include - -#include "core/component.h" - -/* Forward declarations for classes to reduce compilation time and modules coupling. */ -class configuration; - -namespace test_harness { -/* - * The timestamp monitor class manages global timestamp state for all components in the test - * harness. It also manages the global timestamps within WiredTiger. - * - * The format of a timestamp is as follows, the first 32 bits represent the epoch time in seconds. - * The last 32 bits represent an increment for uniqueness. - */ -class timestamp_manager : public component { - public: - static const std::string decimal_to_hex(uint64_t value); - - public: - explicit timestamp_manager(configuration *config); - virtual ~timestamp_manager() = default; - - void load() override final; - void do_work() override final; - - /* Get a unique timestamp. */ - wt_timestamp_t get_next_ts(); - - /* Get oldest timestamp. */ - wt_timestamp_t get_oldest_ts() const; - - /* Generate a random timestamp between the oldest timestamp and now. */ - wt_timestamp_t get_random_ts() const; - - private: - /* Get the current time in seconds, bit shifted to the expected location. */ - uint64_t get_time_now_s() const; - - private: - std::atomic _increment_ts{0}; - /* The tracking table sweep needs to read the oldest timestamp. */ - std::atomic _oldest_ts{0U}; - wt_timestamp_t _stable_ts = 0U; - /* - * _oldest_lag is the time window between the stable and oldest timestamps. - * _stable_lag is the time window between the latest and stable timestamps. - */ - uint64_t _oldest_lag = 0U, _stable_lag = 0U; -}; -} // namespace test_harness - -#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/api_const.cpp b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/api_const.cpp deleted file mode 100644 index db313b93ebc..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/api_const.cpp +++ /dev/null @@ -1,92 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#include "api_const.h" - -/* Define all constants related to WiredTiger APIs and testing. */ -namespace test_harness { - -/* Component names. */ -const std::string CHECKPOINT_MANAGER = "checkpoint_manager"; -const std::string RUNTIME_MONITOR = "runtime_monitor"; -const std::string TIMESTAMP_MANAGER = "timestamp_manager"; -const std::string WORKLOAD_GENERATOR = "workload_generator"; -const std::string WORKLOAD_TRACKING = "workload_tracking"; - -/* Configuration API consts. */ -const std::string CACHE_HS_INSERT = "cache_hs_insert"; -const std::string CACHE_MAX_WAIT_MS = "cache_max_wait_ms"; -const std::string CACHE_SIZE_MB = "cache_size_mb"; -const std::string CC_PAGES_REMOVED = "cc_pages_removed"; -const std::string COLLECTION_COUNT = "collection_count"; -const std::string COMPRESSION_ENABLED = "compression_enabled"; -const std::string CUSTOM_OP_CONFIG = "custom_config"; -const std::string DURATION_SECONDS = "duration_seconds"; -const std::string ENABLED = "enabled"; -const std::string ENABLE_LOGGING = "enable_logging"; -const std::string INSERT_OP_CONFIG = "insert_config"; -const std::string KEY_COUNT_PER_COLLECTION = "key_count_per_collection"; -const std::string KEY_SIZE = "key_size"; -const std::string LIMIT = "limit"; -const std::string MAX = "max"; -const std::string MIN = "min"; -const std::string OLDEST_LAG = "oldest_lag"; -const std::string OP_RATE = "op_rate"; -const std::string OPS_PER_TRANSACTION = "ops_per_transaction"; -const std::string POPULATE_CONFIG = "populate_config"; -const std::string POSTRUN_STATISTICS = "postrun"; -const std::string READ_OP_CONFIG = "read_config"; -const std::string REMOVE_OP_CONFIG = "remove_config"; -const std::string REVERSE_COLLATOR = "reverse_collator"; -const std::string RUNTIME_STATISTICS = "runtime"; -const std::string SAVE = "save"; -const std::string STABLE_LAG = "stable_lag"; -const std::string STAT_CACHE_SIZE = "stat_cache_size"; -const std::string STAT_DB_SIZE = "stat_db_size"; -const std::string STATISTICS_CONFIG = "statistics_config"; -const std::string THREAD_COUNT = "thread_count"; -const std::string TRACKING_KEY_FORMAT = "tracking_key_format"; -const std::string TRACKING_VALUE_FORMAT = "tracking_value_format"; -const std::string TYPE = "type"; -const std::string UPDATE_OP_CONFIG = "update_config"; -const std::string VALUE_SIZE = "value_size"; - -/* WiredTiger API consts. */ -const std::string COMMIT_TS = "commit_timestamp"; -const std::string CONNECTION_CREATE = "create"; -const std::string OLDEST_TS = "oldest_timestamp"; -const std::string STABLE_TS = "stable_timestamp"; -const std::string STATISTICS_LOG = "statistics_log=(json,wait=1)"; - -/* Test harness consts. */ -const std::string DEFAULT_FRAMEWORK_SCHEMA = "key_format=S,value_format=S,"; -const std::string TABLE_OPERATION_TRACKING = "table:operation_tracking"; -const std::string TABLE_SCHEMA_TRACKING = "table:schema_tracking"; -const std::string STATISTICS_URI = "statistics:"; - -} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/api_const.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/api_const.h deleted file mode 100644 index 8c0038fdee5..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/api_const.h +++ /dev/null @@ -1,116 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef API_CONST_H -#define API_CONST_H - -#include - -/* Define all constants related to WiredTiger APIs and testing. */ -namespace test_harness { - -/* Component names. */ -extern const std::string CHECKPOINT_MANAGER; -extern const std::string RUNTIME_MONITOR; -extern const std::string TIMESTAMP_MANAGER; -extern const std::string WORKLOAD_GENERATOR; -extern const std::string WORKLOAD_TRACKING; - -/* Configuration API consts. */ -extern const std::string CACHE_HS_INSERT; -extern const std::string CACHE_MAX_WAIT_MS; -extern const std::string CACHE_SIZE_MB; -extern const std::string CC_PAGES_REMOVED; -extern const std::string COLLECTION_COUNT; -extern const std::string COMPRESSION_ENABLED; -extern const std::string CUSTOM_OP_CONFIG; -extern const std::string DURATION_SECONDS; -extern const std::string ENABLED; -extern const std::string ENABLE_LOGGING; -extern const std::string INSERT_OP_CONFIG; -extern const std::string KEY_COUNT_PER_COLLECTION; -extern const std::string KEY_SIZE; -extern const std::string LIMIT; -extern const std::string MAX; -extern const std::string MIN; -extern const std::string OLDEST_LAG; -extern const std::string OP_RATE; -extern const std::string OPS_PER_TRANSACTION; -extern const std::string POPULATE_CONFIG; -extern const std::string POSTRUN_STATISTICS; -extern const std::string READ_OP_CONFIG; -extern const std::string REMOVE_OP_CONFIG; -extern const std::string REVERSE_COLLATOR; -extern const std::string RUNTIME_STATISTICS; -extern const std::string SAVE; -extern const std::string STABLE_LAG; -extern const std::string STAT_CACHE_SIZE; -extern const std::string STAT_DB_SIZE; -extern const std::string STATISTICS_CONFIG; -extern const std::string THREAD_COUNT; -extern const std::string TRACKING_KEY_FORMAT; -extern const std::string TRACKING_VALUE_FORMAT; -extern const std::string TYPE; -extern const std::string UPDATE_OP_CONFIG; -extern const std::string VALUE_SIZE; - -/* WiredTiger API consts. */ -extern const std::string COMMIT_TS; -extern const std::string CONNECTION_CREATE; -extern const std::string OLDEST_TS; -extern const std::string STABLE_TS; -extern const std::string STATISTICS_LOG; - -/* - * Use the Snappy compressor for stress testing to avoid excessive disk space usage. Our builds can - * pre-specify 'EXTSUBPATH' to indicate any special sub-directories the module is located. If unset - * we fallback to the '.libs' sub-directory used by autoconf. - */ -#define BLKCMP_PFX "block_compressor=" -#define SNAPPY_BLK BLKCMP_PFX "snappy" -#define EXTPATH "../../ext/" -#ifndef EXTSUBPATH -#define EXTSUBPATH ".libs/" -#endif - -/* Use reverse collator to test changes that deal different table sorting orders. */ -#define REVERSE_COL_CFG "collator=reverse" - -#define SNAPPY_PATH EXTPATH "compressors/snappy/" EXTSUBPATH "libwiredtiger_snappy.so" -#define REVERSE_COLLATOR_PATH \ - EXTPATH "collators/reverse/" EXTSUBPATH "libwiredtiger_reverse_collator.so" - -/* Test harness consts. */ -extern const std::string DEFAULT_FRAMEWORK_SCHEMA; -extern const std::string TABLE_OPERATION_TRACKING; -extern const std::string TABLE_SCHEMA_TRACKING; -extern const std::string STATISTICS_URI; - -} // namespace test_harness - -#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/logger.cpp b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/logger.cpp deleted file mode 100644 index 67b72eac172..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/logger.cpp +++ /dev/null @@ -1,96 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#include -#include -#include -#include -#include - -#include "logger.h" - -/* Define helpful functions related to debugging. */ -namespace test_harness { -/* Order of elements in this array corresponds to the definitions above. */ -const char *const LOG_LEVELS[] = {"ERROR", "WARN", "INFO", "TRACE"}; - -/* Mutex used by logger to synchronize printing. */ -static std::mutex _logger_mtx; - -/* Set default log level. */ -int64_t logger::trace_level = LOG_WARN; - -/* Include date in the logs by default. */ -bool logger::include_date = true; - -void -get_time(char *time_buf, size_t buf_size) -{ - size_t alloc_size; - struct tm *tm, _tm; - - /* Get time since epoch in nanoseconds. */ - uint64_t epoch_nanosec = std::chrono::high_resolution_clock::now().time_since_epoch().count(); - - /* Calculate time since epoch in seconds. */ - time_t time_epoch_sec = epoch_nanosec / WT_BILLION; - - tm = localtime_r(&time_epoch_sec, &_tm); - testutil_assert(tm != nullptr); - - alloc_size = - strftime(time_buf, buf_size, logger::include_date ? "[%Y-%m-%dT%H:%M:%S" : "[%H:%M:%S", tm); - - testutil_assert(alloc_size <= buf_size); - WT_IGNORE_RET(__wt_snprintf(&time_buf[alloc_size], buf_size - alloc_size, ".%" PRIu64 "Z]", - (uint64_t)epoch_nanosec % WT_BILLION)); -} - -/* Used to print out traces for debugging purpose. */ -void -logger::log_msg(int64_t trace_type, const std::string &str) -{ - if (logger::trace_level >= trace_type) { - testutil_assert( - trace_type >= LOG_ERROR && trace_type < sizeof(LOG_LEVELS) / sizeof(LOG_LEVELS[0])); - - char time_buf[64]; - get_time(time_buf, sizeof(time_buf)); - - std::ostringstream ss; - ss << time_buf << "[TID:" << std::this_thread::get_id() << "][" << LOG_LEVELS[trace_type] - << "]: " << str << std::endl; - - std::lock_guard lg(_logger_mtx); - if (trace_type == LOG_ERROR) - std::cerr << ss.str(); - else - std::cout << ss.str(); - } -} -} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/logger.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/logger.h deleted file mode 100644 index 3b30ede7bd4..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/logger.h +++ /dev/null @@ -1,82 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef DEBUG_UTILS_H -#define DEBUG_UTILS_H - -/* Following definitions are required in order to use printing format specifiers in C++. */ -#ifndef __STDC_LIMIT_MACROS -#define __STDC_LIMIT_MACROS -#endif -#ifndef __STDC_FORMAT_MACROS -#define __STDC_FORMAT_MACROS -#endif - -#include -#include -#include -#include -#include - -extern "C" { -#include "test_util.h" -} - -/* Define helpful functions related to debugging. */ -namespace test_harness { - -/* Possible log levels. If you change anything here make sure you update LOG_LEVELS accordingly. */ -#define LOG_ERROR 0 -#define LOG_WARN 1 -#define LOG_INFO 2 -/* - * The trace log level can incur a performance overhead since the current logging implementation - * requires per-line locking. - */ -#define LOG_TRACE 3 - -void get_time(char *time_buf, size_t buf_size); - -class logger { - public: - /* Current log level. Default is LOG_WARN. */ - static int64_t trace_level; - - /* Include date in the logs if enabled. Default is true. */ - static bool include_date; - - public: - /* Used to print out traces for debugging purpose. */ - static void log_msg(int64_t trace_type, const std::string &str); - - /* Make sure the class will not be instantiated. */ - logger() = delete; -}; -} // namespace test_harness - -#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/perf_plotter.cpp b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/perf_plotter.cpp deleted file mode 100644 index d02c1ae61a9..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/perf_plotter.cpp +++ /dev/null @@ -1,65 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ -#include "perf_plotter.h" - -namespace test_harness { -void -perf_plotter::add_stat(const std::string &stat_string) -{ - std::lock_guard lg(_stat_mutex); - _stats.push_back(stat_string); -} - -void -perf_plotter::output_perf_file(const std::string &test_name) -{ - std::ofstream perf_file; - std::string stat_info = "[{\"info\":{\"test_name\": \"" + test_name + "\"},\"metrics\": ["; - - perf_file.open(test_name + ".json"); - - for (const auto &stat : _stats) - stat_info += stat + ","; - - /* Remove last extra comma. */ - if (stat_info.back() == ',') - stat_info.pop_back(); - - perf_file << stat_info << "]}]"; - perf_file.close(); -} - -perf_plotter & -perf_plotter::instance() -{ - static perf_plotter _instance; - return (_instance); -} - -perf_plotter::perf_plotter() {} -} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/perf_plotter.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/perf_plotter.h deleted file mode 100644 index 36cd9a5817e..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/perf_plotter.h +++ /dev/null @@ -1,58 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ -#ifndef PERF_PLOTTER_H -#define PERF_PLOTTER_H -#include -#include -#include -namespace test_harness { -/* - * Singleton class owning the perf plot json, provides access to add statistics, and a central call - * point to output the file. - */ -class perf_plotter { - public: - static perf_plotter &instance(); - - public: - /* No copies of the singleton allowed. */ - perf_plotter(perf_plotter const &) = delete; - perf_plotter &operator=(perf_plotter const &) = delete; - - /* Anyone can get the perf_plotter and add a statistic to it. */ - void add_stat(const std::string &stat_string); - void output_perf_file(const std::string &test_name); - - private: - perf_plotter(); - std::vector _stats; - std::mutex _stat_mutex; -}; -} // namespace test_harness - -#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_connection.cpp b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_connection.cpp deleted file mode 100644 index 39a8fede916..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_connection.cpp +++ /dev/null @@ -1,46 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ -#include -#include - -#include "../connection_manager.h" -#include "scoped_connection.h" - -namespace test_harness { - -scoped_connection::scoped_connection(const std::string &db_conn_config, const std::string &home) -{ - connection_manager::instance().create(db_conn_config, home); -} - -scoped_connection::~scoped_connection() -{ - connection_manager::instance().close(); -} - -} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_connection.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_connection.h deleted file mode 100644 index 6e999b6db54..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_connection.h +++ /dev/null @@ -1,56 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef SCOPED_CONNECTION_H -#define SCOPED_CONNECTION_H - -/* Following definitions are required in order to use printing format specifiers in C++. */ -#ifndef __STDC_LIMIT_MACROS -#define __STDC_LIMIT_MACROS -#endif -#ifndef __STDC_FORMAT_MACROS -#define __STDC_FORMAT_MACROS -#endif - -extern "C" { -#include "test_util.h" -} - -#include "../connection_manager.h" - -namespace test_harness { - -class scoped_connection { - public: - explicit scoped_connection( - const std::string &db_conn_config, const std::string &home = DEFAULT_DIR); - ~scoped_connection(); -}; - -} // namespace test_harness -#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_types.cpp b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_types.cpp deleted file mode 100644 index b0b20ccc0f8..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_types.cpp +++ /dev/null @@ -1,169 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#include -#include - -#include "scoped_types.h" - -namespace test_harness { - -/* scoped_cursor implementation */ -scoped_cursor::scoped_cursor(WT_SESSION *session, const std::string &uri, const std::string &cfg) -{ - reinit(session, uri, cfg); -} - -scoped_cursor::scoped_cursor(scoped_cursor &&other) -{ - std::swap(_cursor, other._cursor); -} - -scoped_cursor::~scoped_cursor() -{ - if (_cursor != nullptr) { - testutil_check(_cursor->close(_cursor)); - _cursor = nullptr; - } -} - -/* - * Implement move assignment by move constructing a temporary and swapping its internals with the - * current cursor. This means that the currently held WT_CURSOR will get destroyed as the temporary - * falls out of the scope and we will steal the one that we're move assigning from. - */ -scoped_cursor & -scoped_cursor::operator=(scoped_cursor &&other) -{ - scoped_cursor tmp(std::move(other)); - std::swap(_cursor, tmp._cursor); - return (*this); -} - -void -scoped_cursor::reinit(WT_SESSION *session, const std::string &uri, const std::string &cfg) -{ - testutil_assert(!uri.empty()); - if (_cursor != nullptr) { - testutil_check(_cursor->close(_cursor)); - _cursor = nullptr; - } - if (session != nullptr) - testutil_check(session->open_cursor( - session, uri.c_str(), nullptr, cfg.empty() ? nullptr : cfg.c_str(), &_cursor)); -} - -/* - * Override the dereference operators. The idea is that we should able to use this class as if it is - * a pointer to a WT_CURSOR. - */ -WT_CURSOR & -scoped_cursor::operator*() -{ - return (*_cursor); -} - -WT_CURSOR * -scoped_cursor::operator->() -{ - return (_cursor); -} - -WT_CURSOR * -scoped_cursor::get() -{ - return (_cursor); -} - -/* scoped_session implementation */ -scoped_session::scoped_session(WT_CONNECTION *conn) -{ - reinit(conn); -} - -scoped_session::~scoped_session() -{ - if (_session != nullptr) { - testutil_check(_session->close(_session, nullptr)); - _session = nullptr; - } -} - -scoped_session::scoped_session(scoped_session &&other) -{ - std::swap(_session, other._session); -} - -/* - * Implement move assignment by move constructing a temporary and swapping its internals with the - * current session. This means that the currently held WT_SESSION will get destroyed as the - * temporary falls out of the scope and we will steal the one that we're move assigning from. - */ -scoped_session & -scoped_session::operator=(scoped_session &&other) -{ - scoped_session tmp(std::move(other)); - std::swap(_session, tmp._session); - return (*this); -} - -void -scoped_session::reinit(WT_CONNECTION *conn) -{ - if (_session != nullptr) { - testutil_check(_session->close(_session, nullptr)); - _session = nullptr; - } - if (conn != nullptr) - testutil_check(conn->open_session(conn, nullptr, nullptr, &_session)); -} - -WT_SESSION & -scoped_session::operator*() -{ - return (*_session); -} - -WT_SESSION * -scoped_session::operator->() -{ - return (_session); -} - -WT_SESSION * -scoped_session::get() -{ - return (_session); -} - -scoped_cursor -scoped_session::open_scoped_cursor(const std::string &uri, const std::string &cfg) -{ - return (scoped_cursor(_session, uri, cfg)); -} -} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_types.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_types.h deleted file mode 100644 index c015c0c909b..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_types.h +++ /dev/null @@ -1,103 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef SCOPED_TYPES_H -#define SCOPED_TYPES_H - -/* Following definitions are required in order to use printing format specifiers in C++. */ -#ifndef __STDC_LIMIT_MACROS -#define __STDC_LIMIT_MACROS -#endif -#ifndef __STDC_FORMAT_MACROS -#define __STDC_FORMAT_MACROS -#endif - -extern "C" { -#include "test_util.h" -} - -namespace test_harness { -class scoped_cursor { - public: - scoped_cursor() = default; - explicit scoped_cursor(WT_SESSION *session, const std::string &uri, const std::string &cfg); - - /* Moving is ok but copying is not. */ - scoped_cursor(scoped_cursor &&other); - - ~scoped_cursor(); - - scoped_cursor &operator=(scoped_cursor &&other); - scoped_cursor(const scoped_cursor &) = delete; - scoped_cursor &operator=(const scoped_cursor &) = delete; - - void reinit(WT_SESSION *session, const std::string &uri, const std::string &cfg); - - WT_CURSOR &operator*(); - WT_CURSOR *operator->(); - - WT_CURSOR *get(); - - private: - WT_CURSOR *_cursor = nullptr; -}; - -class scoped_session { - public: - scoped_session() = default; - explicit scoped_session(WT_CONNECTION *conn); - - ~scoped_session(); - - /* Moving is ok but copying is not. */ - scoped_session(scoped_session &&other); - - scoped_session &operator=(scoped_session &&other); - - scoped_session(const scoped_session &) = delete; - scoped_session &operator=(const scoped_session &) = delete; - - void reinit(WT_CONNECTION *conn); - - /* - * Override the dereference operators. The idea is that we should able to use this class as if - * it is a pointer to a WT_SESSION. - */ - WT_SESSION &operator*(); - WT_SESSION *operator->(); - - WT_SESSION *get(); - - scoped_cursor open_scoped_cursor(const std::string &uri, const std::string &cfg = ""); - - private: - WT_SESSION *_session = nullptr; -}; - -} // namespace test_harness -#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_model.cpp b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_model.cpp deleted file mode 100644 index 1fbd40012ed..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_model.cpp +++ /dev/null @@ -1,156 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#include -#include - -#include "../connection_manager.h" -#include "../timestamp_manager.h" -#include "../util/api_const.h" -#include "database_model.h" -#include "random_generator.h" -#include "workload_tracking.h" - -namespace test_harness { -/* collection class implementation */ -collection::collection(const uint64_t id, const uint64_t key_count, const std::string &name) - : name(name), id(id), _key_count(key_count) -{ -} - -uint64_t -collection::get_key_count() const -{ - return (_key_count); -} - -void -collection::increase_key_count(uint64_t increment) -{ - _key_count += increment; -} - -/* database class implementation */ -std::string -database::build_collection_name(const uint64_t id) -{ - return (std::string("table:collection_" + std::to_string(id))); -} - -void -database::add_collection(uint64_t key_count) -{ - std::lock_guard lg(_mtx); - if (_session.get() == nullptr) - _session = connection_manager::instance().create_session(); - if (_collection_create_config.empty()) - testutil_die(EINVAL, "database_model: no collection create config specified!"); - uint64_t next_id = _next_collection_id++; - std::string collection_name = build_collection_name(next_id); - /* FIX-ME-Test-Framework: This will get removed when we split the model up. */ - _collections.emplace(std::piecewise_construct, std::forward_as_tuple(next_id), - std::forward_as_tuple(next_id, key_count, collection_name)); - testutil_check( - _session->create(_session.get(), collection_name.c_str(), _collection_create_config.c_str())); - _tracking->save_schema_operation( - tracking_operation::CREATE_COLLECTION, next_id, _tsm->get_next_ts()); -} - -collection & -database::get_collection(uint64_t id) -{ - std::lock_guard lg(_mtx); - const auto it = _collections.find(id); - if (it == _collections.end()) - testutil_die(EINVAL, "tried to get collection that doesn't exist."); - return (it->second); -} - -collection & -database::get_random_collection() -{ - size_t collection_count = get_collection_count(); - /* Any caller should expect at least one collection to exist. */ - testutil_assert(collection_count != 0); - return (get_collection( - random_generator::instance().generate_integer(0, collection_count - 1))); -} - -uint64_t -database::get_collection_count() -{ - std::lock_guard lg(_mtx); - return (_collections.size()); -} - -std::vector -database::get_collection_names() -{ - std::lock_guard lg(_mtx); - std::vector collection_names; - - for (auto const &it : _collections) - collection_names.push_back(it.second.name); - - return (collection_names); -} - -std::vector -database::get_collection_ids() -{ - std::lock_guard lg(_mtx); - std::vector collection_ids; - - for (auto const &it : _collections) - collection_ids.push_back(it.first); - - return (collection_ids); -} - -void -database::set_timestamp_manager(timestamp_manager *tsm) -{ - testutil_assert(_tsm == nullptr); - _tsm = tsm; -} - -void -database::set_workload_tracking(workload_tracking *tracking) -{ - testutil_assert(_tracking == nullptr); - _tracking = tracking; -} - -void -database::set_create_config(bool use_compression, bool use_reverse_collator) -{ - _collection_create_config = DEFAULT_FRAMEWORK_SCHEMA; - _collection_create_config += use_compression ? std::string(SNAPPY_BLK) + "," : ""; - _collection_create_config += use_reverse_collator ? std::string(REVERSE_COL_CFG) + "," : ""; -} -} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_model.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_model.h deleted file mode 100644 index d907dbfbda1..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_model.h +++ /dev/null @@ -1,121 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef DATABASE_MODEL_H -#define DATABASE_MODEL_H - -#include -#include -#include - -#include "workload_tracking.h" - -/* Forward declarations for classes to reduce compilation time and modules coupling. */ -class timestamp_manager; - -namespace test_harness { -/* Key/Value type. */ -typedef std::string key_value_t; - -/* A collection is made of mapped key value objects. */ -class collection { - public: - explicit collection(const uint64_t id, const uint64_t key_count, const std::string &name); - - /* Copies aren't allowed. */ - collection(const collection &) = delete; - collection &operator=(const collection &) = delete; - - uint64_t get_key_count() const; - - /* - * Adding new keys should generally be singly threaded per collection. If two threads both - * attempt to add keys using the incrementing id pattern they'd frequently conflict. - * - * The usage pattern is: - * 1. Call get_key_count to get the number of keys already existing. Add keys with id's equal - * to and greater than this value. - * 2. Once the transaction has successfully committed then call increase_key_count() with the - * number of added keys. - * - * The set of keys should always be contiguous such that other threads calling get_key_count - * will always know that the keys in existence are 0 -> _key_count - 1. - */ - void increase_key_count(uint64_t increment); - - public: - const std::string name; - const uint64_t id; - - private: - std::atomic _key_count{0}; -}; - -/* Representation of the collections in memory. */ -class database { - public: - static std::string build_collection_name(const uint64_t id); - - public: - /* - * Add a new collection, this will create the underlying collection in the database. - */ - void add_collection(uint64_t key_count = 0); - - /* Get a collection using the id of the collection. */ - collection &get_collection(uint64_t id); - - /* Get a random collection. */ - collection &get_random_collection(); - - /* - * Retrieve the current collection count, collection names are indexed from 0 so when using this - * take care to avoid an off by one error. - */ - uint64_t get_collection_count(); - - /* FIX-ME-Test-Framework: Replace usages of this with get_collection_ids. */ - std::vector get_collection_names(); - - std::vector get_collection_ids(); - void set_timestamp_manager(timestamp_manager *tsm); - void set_workload_tracking(workload_tracking *tracking); - void set_create_config(bool use_compression, bool use_reverse_collator); - - private: - std::string _collection_create_config = ""; - scoped_session _session; - timestamp_manager *_tsm = nullptr; - workload_tracking *_tracking = nullptr; - uint64_t _next_collection_id = 0; - std::map _collections; - std::mutex _mtx; -}; -} // namespace test_harness - -#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.cpp b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.cpp deleted file mode 100644 index 549a84b5387..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.cpp +++ /dev/null @@ -1,397 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#include -#include - -#include "../connection_manager.h" -#include "../thread_manager.h" -#include "../util/api_const.h" -#include "database_operation.h" -#include "random_generator.h" -#include "workload_tracking.h" -#include "workload_validation.h" - -namespace test_harness { -/* Static methods. */ -static void -populate_worker(thread_context *tc) -{ - uint64_t collections_per_thread = tc->collection_count / tc->thread_count; - - for (int64_t i = 0; i < collections_per_thread; ++i) { - collection &coll = tc->db.get_collection((tc->id * collections_per_thread) + i); - /* - * WiredTiger lets you open a cursor on a collection using the same pointer. When a session - * is closed, WiredTiger APIs close the cursors too. - */ - scoped_cursor cursor = tc->session.open_scoped_cursor(coll.name); - uint64_t j = 0; - while (j < tc->key_count) { - tc->transaction.begin(); - auto key = tc->pad_string(std::to_string(j), tc->key_size); - auto value = random_generator::instance().generate_pseudo_random_string(tc->value_size); - if (tc->insert(cursor, coll.id, key, value)) { - if (tc->transaction.commit()) { - ++j; - } - } else { - tc->transaction.rollback(); - } - } - } - logger::log_msg(LOG_TRACE, "Populate: thread {" + std::to_string(tc->id) + "} finished"); -} - -/* database_operation class implementation. */ -void -database_operation::populate( - database &database, timestamp_manager *tsm, configuration *config, workload_tracking *tracking) -{ - int64_t collection_count, key_count, key_size, thread_count, value_size; - std::vector workers; - std::string collection_name; - thread_manager tm; - - /* Validate our config. */ - collection_count = config->get_int(COLLECTION_COUNT); - key_count = config->get_int(KEY_COUNT_PER_COLLECTION); - value_size = config->get_int(VALUE_SIZE); - thread_count = config->get_int(THREAD_COUNT); - testutil_assert(thread_count == 0 || collection_count % thread_count == 0); - testutil_assert(value_size > 0); - key_size = config->get_int(KEY_SIZE); - testutil_assert(key_size > 0); - /* Keys must be unique. */ - testutil_assert(key_count <= pow(10, key_size)); - - logger::log_msg( - LOG_INFO, "Populate: creating " + std::to_string(collection_count) + " collections."); - - /* Create n collections as per the configuration. */ - for (int64_t i = 0; i < collection_count; ++i) - /* - * The database model will call into the API and create the collection, with its own - * session. - */ - database.add_collection(key_count); - - logger::log_msg( - LOG_INFO, "Populate: " + std::to_string(collection_count) + " collections created."); - - /* - * Spawn thread_count threads to populate the database, theoretically we should be IO bound - * here. - */ - for (int64_t i = 0; i < thread_count; ++i) { - thread_context *tc = new thread_context(i, thread_type::INSERT, config, - connection_manager::instance().create_session(), tsm, tracking, database); - workers.push_back(tc); - tm.add_thread(populate_worker, tc); - } - - /* Wait for our populate threads to finish and then join them. */ - logger::log_msg(LOG_INFO, "Populate: waiting for threads to complete."); - tm.join(); - - /* Cleanup our workers. */ - for (auto &it : workers) { - delete it; - it = nullptr; - } - logger::log_msg(LOG_INFO, "Populate: finished."); -} - -void -database_operation::custom_operation(thread_context *tc) -{ - logger::log_msg( - LOG_INFO, type_string(tc->type) + " thread {" + std::to_string(tc->id) + "} commencing."); -} - -void -database_operation::insert_operation(thread_context *tc) -{ - logger::log_msg( - LOG_INFO, type_string(tc->type) + " thread {" + std::to_string(tc->id) + "} commencing."); - - /* Helper struct which stores a pointer to a collection and a cursor associated with it. */ - struct collection_cursor { - collection_cursor(collection &coll, scoped_cursor &&cursor) - : coll(coll), cursor(std::move(cursor)) - { - } - collection &coll; - scoped_cursor cursor; - }; - - /* Collection cursor vector. */ - std::vector ccv; - uint64_t collection_count = tc->db.get_collection_count(); - testutil_assert(collection_count != 0); - uint64_t collections_per_thread = collection_count / tc->thread_count; - /* Must have unique collections for each thread. */ - testutil_assert(collection_count % tc->thread_count == 0); - for (int i = tc->id * collections_per_thread; - i < (tc->id * collections_per_thread) + collections_per_thread && tc->running(); ++i) { - collection &coll = tc->db.get_collection(i); - scoped_cursor cursor = tc->session.open_scoped_cursor(coll.name); - ccv.push_back({coll, std::move(cursor)}); - } - - uint64_t counter = 0; - while (tc->running()) { - uint64_t start_key = ccv[counter].coll.get_key_count(); - uint64_t added_count = 0; - tc->transaction.begin(); - - /* Collection cursor. */ - auto &cc = ccv[counter]; - while (tc->transaction.active() && tc->running()) { - /* Insert a key value pair, rolling back the transaction if required. */ - auto key = tc->pad_string(std::to_string(start_key + added_count), tc->key_size); - auto value = random_generator::instance().generate_pseudo_random_string(tc->value_size); - if (!tc->insert(cc.cursor, cc.coll.id, key, value)) { - added_count = 0; - tc->transaction.rollback(); - } else { - added_count++; - if (tc->transaction.can_commit()) { - if (tc->transaction.commit()) { - /* - * We need to inform the database model that we've added these keys as some - * other thread may rely on the key_count data. Only do so if we - * successfully committed. - */ - cc.coll.increase_key_count(added_count); - } else { - added_count = 0; - } - } - } - - /* Sleep the duration defined by the op_rate. */ - tc->sleep(); - } - /* Reset our cursor to avoid pinning content. */ - testutil_check(cc.cursor->reset(cc.cursor.get())); - counter++; - if (counter == collections_per_thread) - counter = 0; - testutil_assert(counter < collections_per_thread); - } - /* Make sure the last transaction is rolled back now the work is finished. */ - if (tc->transaction.active()) - tc->transaction.rollback(); -} - -void -database_operation::read_operation(thread_context *tc) -{ - logger::log_msg( - LOG_INFO, type_string(tc->type) + " thread {" + std::to_string(tc->id) + "} commencing."); - - std::map cursors; - while (tc->running()) { - /* Get a collection and find a cached cursor. */ - collection &coll = tc->db.get_random_collection(); - - if (cursors.find(coll.id) == cursors.end()) - cursors.emplace(coll.id, std::move(tc->session.open_scoped_cursor(coll.name))); - - /* Do a second lookup now that we know it exists. */ - auto &cursor = cursors[coll.id]; - - tc->transaction.begin(); - while (tc->transaction.active() && tc->running()) { - auto ret = cursor->next(cursor.get()); - if (ret != 0) { - if (ret == WT_NOTFOUND) { - cursor->reset(cursor.get()); - } else if (ret == WT_ROLLBACK) { - tc->transaction.rollback(); - tc->sleep(); - continue; - } else - testutil_die(ret, "Unexpected error returned from cursor->next()"); - } - tc->transaction.add_op(); - tc->transaction.try_rollback(); - tc->sleep(); - } - /* Reset our cursor to avoid pinning content. */ - testutil_check(cursor->reset(cursor.get())); - } - /* Make sure the last transaction is rolled back now the work is finished. */ - if (tc->transaction.active()) - tc->transaction.rollback(); -} - -void -database_operation::remove_operation(thread_context *tc) -{ - logger::log_msg( - LOG_INFO, type_string(tc->type) + " thread {" + std::to_string(tc->id) + "} commencing."); - - /* - * We need two types of cursors. One cursor is a random cursor to randomly select a key and the - * other one is a standard cursor to remove the random key. This is required as the random - * cursor does not support the remove operation. - */ - std::map rnd_cursors, cursors; - - /* Loop while the test is running. */ - while (tc->running()) { - /* - * Sleep the period defined by the op_rate in the configuration. Do this at the start of the - * loop as it could be skipped by a subsequent continue call. - */ - tc->sleep(); - - /* Choose a random collection to update. */ - collection &coll = tc->db.get_random_collection(); - - /* Look for existing cursors in our cursor cache. */ - if (cursors.find(coll.id) == cursors.end()) { - logger::log_msg(LOG_TRACE, - "Thread {" + std::to_string(tc->id) + - "} Creating cursor for collection: " + coll.name); - /* Open the two cursors for the chosen collection. */ - scoped_cursor rnd_cursor = - tc->session.open_scoped_cursor(coll.name, "next_random=true"); - rnd_cursors.emplace(coll.id, std::move(rnd_cursor)); - scoped_cursor cursor = tc->session.open_scoped_cursor(coll.name); - cursors.emplace(coll.id, std::move(cursor)); - } - - /* Start a transaction if possible. */ - tc->transaction.try_begin(); - - /* Get the cursor associated with the collection. */ - scoped_cursor &rnd_cursor = rnd_cursors[coll.id]; - scoped_cursor &cursor = cursors[coll.id]; - - /* Choose a random key to delete. */ - const char *key_str; - int ret = rnd_cursor->next(rnd_cursor.get()); - /* It is possible not to find anything if the collection is empty. */ - testutil_assert(ret == 0 || ret == WT_NOTFOUND); - if (ret == WT_NOTFOUND) { - /* - * If we cannot find any record, finish the current transaction as we might be able to - * see new records after starting a new one. - */ - WT_IGNORE_RET_BOOL(tc->transaction.commit()); - continue; - } - testutil_check(rnd_cursor->get_key(rnd_cursor.get(), &key_str)); - if (!tc->remove(cursor, coll.id, key_str)) { - tc->transaction.rollback(); - } - - /* Reset our cursor to avoid pinning content. */ - testutil_check(cursor->reset(cursor.get())); - - /* Commit the current transaction if we're able to. */ - if (tc->transaction.can_commit()) - WT_IGNORE_RET_BOOL(tc->transaction.commit()); - } - - /* Make sure the last operation is rolled back now the work is finished. */ - if (tc->transaction.active()) - tc->transaction.rollback(); -} - -void -database_operation::update_operation(thread_context *tc) -{ - logger::log_msg( - LOG_INFO, type_string(tc->type) + " thread {" + std::to_string(tc->id) + "} commencing."); - /* Cursor map. */ - std::map cursors; - - /* - * Loop while the test is running. - */ - while (tc->running()) { - /* - * Sleep the period defined by the op_rate in the configuration. Do this at the start of the - * loop as it could be skipped by a subsequent continue call. - */ - tc->sleep(); - - /* Choose a random collection to update. */ - collection &coll = tc->db.get_random_collection(); - - /* Look for existing cursors in our cursor cache. */ - if (cursors.find(coll.id) == cursors.end()) { - logger::log_msg(LOG_TRACE, - "Thread {" + std::to_string(tc->id) + - "} Creating cursor for collection: " + coll.name); - /* Open a cursor for the chosen collection. */ - scoped_cursor cursor = tc->session.open_scoped_cursor(coll.name); - cursors.emplace(coll.id, std::move(cursor)); - } - - /* Start a transaction if possible. */ - tc->transaction.try_begin(); - - /* Get the cursor associated with the collection. */ - scoped_cursor &cursor = cursors[coll.id]; - - /* Choose a random key to update. */ - testutil_assert(coll.get_key_count() != 0); - auto key_id = - random_generator::instance().generate_integer(0, coll.get_key_count() - 1); - auto key = tc->pad_string(std::to_string(key_id), tc->key_size); - auto value = random_generator::instance().generate_pseudo_random_string(tc->value_size); - if (!tc->update(cursor, coll.id, key, value)) { - tc->transaction.rollback(); - } - - /* Reset our cursor to avoid pinning content. */ - testutil_check(cursor->reset(cursor.get())); - - /* Commit the current transaction if we're able to. */ - if (tc->transaction.can_commit()) - WT_IGNORE_RET_BOOL(tc->transaction.commit()); - } - - /* Make sure the last operation is rolled back now the work is finished. */ - if (tc->transaction.active()) - tc->transaction.rollback(); -} - -void -database_operation::validate(const std::string &operation_table_name, - const std::string &schema_table_name, const std::vector &known_collection_ids) -{ - workload_validation wv; - wv.validate(operation_table_name, schema_table_name, known_collection_ids); -} -} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.h deleted file mode 100644 index 946a322f25a..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.h +++ /dev/null @@ -1,71 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef DATABASE_OPERATION_H -#define DATABASE_OPERATION_H - -#include "database_model.h" -#include "thread_context.h" - -namespace test_harness { -class database_operation { - public: - /* - * Function that performs the following steps using the configuration that is defined by the - * test: - * - Creates N collections as per the configuration. - * - Creates M threads as per the configuration, each thread will: - * - Open a cursor on each collection. - * - Insert K key/value pairs in each collection. Values are random strings which size is - * defined by the configuration. - */ - virtual void populate(database &database, timestamp_manager *tsm, configuration *config, - workload_tracking *tracking); - - /* Custom operation without a default implementation. */ - virtual void custom_operation(thread_context *tc); - - /* Basic insert operation that adds a new key every rate tick. */ - virtual void insert_operation(thread_context *tc); - - /* Basic read operation that chooses a random collection and walks a cursor. */ - virtual void read_operation(thread_context *tc); - - /* Basic remove operation that chooses a random key and deletes it. */ - virtual void remove_operation(thread_context *tc); - - /* Basic update operation that chooses a random key and updates it. */ - virtual void update_operation(thread_context *tc); - - virtual void validate(const std::string &operation_table_name, - const std::string &schema_table_name, const std::vector &known_collection_ids); - - virtual ~database_operation() = default; -}; -} // namespace test_harness -#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/random_generator.cpp b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/random_generator.cpp deleted file mode 100644 index 3973af7242c..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/random_generator.cpp +++ /dev/null @@ -1,114 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#include "random_generator.h" -#include - -extern "C" { -#include "test_util.h" -} - -namespace test_harness { -random_generator & -random_generator::instance() -{ - thread_local random_generator _instance; - return (_instance); -} - -std::string -random_generator::generate_random_string(std::size_t length, characters_type type) -{ - const std::string characters = get_characters(type); - std::string str; - - while (str.size() < length) - str += characters; - - std::shuffle(str.begin(), str.end(), _generator); - return (str.substr(0, length)); -} - -std::string -random_generator::generate_pseudo_random_string(std::size_t length, characters_type type) -{ - std::string random_string; - std::uniform_int_distribution<> &distribution = get_distribution(type); - std::size_t start_location = distribution(_generator); - const std::string &characters = get_characters(type); - - for (std::size_t i = 0; i < length; ++i) { - random_string += characters[start_location]; - if (start_location == characters.size() - 1) - start_location = 0; - else - start_location++; - } - return (random_string); -} - -random_generator::random_generator() -{ - _generator = std::mt19937(std::random_device{}()); - _alphanum_distrib = std::uniform_int_distribution<>(0, _pseudo_alphanum.size() - 1); - _alpha_distrib = std::uniform_int_distribution<>(0, _alphabet.size() - 1); -} - -std::uniform_int_distribution<> & -random_generator::get_distribution(characters_type type) -{ - switch (type) { - case characters_type::ALPHABET: - return (_alpha_distrib); - break; - case characters_type::PSEUDO_ALPHANUMERIC: - return (_alphanum_distrib); - break; - default: - testutil_die(type, "Unexpected characters_type"); - break; - } -} - -const std::string & -random_generator::get_characters(characters_type type) -{ - switch (type) { - case characters_type::ALPHABET: - return (_alphabet); - break; - case characters_type::PSEUDO_ALPHANUMERIC: - return (_pseudo_alphanum); - break; - default: - testutil_die(type, "Unexpected characters_type"); - break; - } -} - -} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/random_generator.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/random_generator.h deleted file mode 100644 index 967d5566ce1..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/random_generator.h +++ /dev/null @@ -1,89 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef RANDOM_GENERATOR_H -#define RANDOM_GENERATOR_H - -/* Following definitions are required in order to use printing format specifiers in C++. */ -#ifndef __STDC_LIMIT_MACROS -#define __STDC_LIMIT_MACROS -#endif -#ifndef __STDC_FORMAT_MACROS -#define __STDC_FORMAT_MACROS -#endif - -#include -#include - -namespace test_harness { -/* Helper class to generate random values using uniform distributions. */ - -enum characters_type { PSEUDO_ALPHANUMERIC, ALPHABET }; - -class random_generator { - public: - static random_generator &instance(); - - /* No copies of the singleton allowed. */ - random_generator(random_generator const &) = delete; - random_generator &operator=(random_generator const &) = delete; - - /* Generate a random string of a given length. */ - std::string generate_random_string( - std::size_t length, characters_type type = PSEUDO_ALPHANUMERIC); - - /* - * Generate a pseudo random string which compresses better. It should not be used to generate - * keys due to the limited randomness. - */ - std::string generate_pseudo_random_string( - std::size_t length, characters_type type = PSEUDO_ALPHANUMERIC); - - /* Generate a random integer between min and max. */ - template - T - generate_integer(T min, T max) - { - std::uniform_int_distribution dis(min, max); - return dis(_generator); - } - - private: - random_generator(); - std::uniform_int_distribution<> &get_distribution(characters_type type); - const std::string &get_characters(characters_type type); - - std::mt19937 _generator; - std::uniform_int_distribution<> _alphanum_distrib, _alpha_distrib; - const std::string _alphabet = "abcdefghijklmnopqrstuvwxyz"; - const std::string _pseudo_alphanum = - "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; -}; -} // namespace test_harness - -#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.cpp b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.cpp deleted file mode 100644 index 75f1c338f74..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.cpp +++ /dev/null @@ -1,351 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#include "../core/configuration.h" -#include "../timestamp_manager.h" -#include "../util/api_const.h" -#include "../util/logger.h" -#include "workload_tracking.h" -#include "random_generator.h" -#include "thread_context.h" - -namespace test_harness { - -const std::string -type_string(thread_type type) -{ - switch (type) { - case thread_type::CUSTOM: - return ("custom"); - case thread_type::INSERT: - return ("insert"); - case thread_type::READ: - return ("read"); - case thread_type::REMOVE: - return ("remove"); - case thread_type::UPDATE: - return ("update"); - default: - testutil_die(EINVAL, "unexpected thread_type: %d", static_cast(type)); - } -} - -/* transaction_context class implementation */ -transaction_context::transaction_context( - configuration *config, timestamp_manager *timestamp_manager, WT_SESSION *session) - : _timestamp_manager(timestamp_manager), _session(session) -{ - /* Use optional here as our populate threads don't define this configuration. */ - configuration *transaction_config = config->get_optional_subconfig(OPS_PER_TRANSACTION); - if (transaction_config != nullptr) { - _min_op_count = transaction_config->get_optional_int(MIN, 1); - _max_op_count = transaction_config->get_optional_int(MAX, 1); - delete transaction_config; - } -} - -bool -transaction_context::active() const -{ - return (_in_txn); -} - -void -transaction_context::add_op() -{ - _op_count++; -} - -void -transaction_context::begin(const std::string &config) -{ - testutil_assert(!_in_txn); - testutil_check( - _session->begin_transaction(_session, config.empty() ? nullptr : config.c_str())); - /* This randomizes the number of operations to be executed in one transaction. */ - _target_op_count = - random_generator::instance().generate_integer(_min_op_count, _max_op_count); - _op_count = 0; - _in_txn = true; - _needs_rollback = false; -} - -void -transaction_context::try_begin(const std::string &config) -{ - if (!_in_txn) - begin(config); -} - -/* - * It's possible to receive rollback in commit, when this happens the API will rollback the - * transaction internally. - */ -bool -transaction_context::commit(const std::string &config) -{ - WT_DECL_RET; - testutil_assert(_in_txn && !_needs_rollback); - - ret = _session->commit_transaction(_session, config.empty() ? nullptr : config.c_str()); - /* - * FIXME-WT-9198 Now we are accepting the error code EINVAL because of possible invalid - * timestamps as we know it can happen due to the nature of the framework. The framework may set - * the stable/oldest timestamps to a more recent date than the commit timestamp of the - * transaction which makes the transaction invalid. We only need to check against the stable - * timestamp as, by definition, the oldest timestamp is older than the stable one. - */ - testutil_assert(ret == 0 || ret == EINVAL || ret == WT_ROLLBACK); - - if (ret != 0) - logger::log_msg(LOG_WARN, - "Failed to commit transaction in commit, received error code: " + std::to_string(ret)); - _op_count = 0; - _in_txn = false; - return (ret == 0); -} - -void -transaction_context::rollback(const std::string &config) -{ - testutil_assert(_in_txn); - testutil_check( - _session->rollback_transaction(_session, config.empty() ? nullptr : config.c_str())); - _needs_rollback = false; - _op_count = 0; - _in_txn = false; -} - -void -transaction_context::try_rollback(const std::string &config) -{ - if (can_rollback()) - rollback(config); -} - -/* - * FIXME: WT-9198 We're concurrently doing a transaction that contains a bunch of operations while - * moving the stable timestamp. Eat the occasional EINVAL from the transaction's first commit - * timestamp being earlier than the stable timestamp. - */ -int -transaction_context::set_commit_timestamp(wt_timestamp_t ts) -{ - /* We don't want to set zero timestamps on transactions if we're not using timestamps. */ - if (!_timestamp_manager->enabled()) - return 0; - const std::string config = COMMIT_TS + "=" + timestamp_manager::decimal_to_hex(ts); - return _session->timestamp_transaction(_session, config.c_str()); -} - -void -transaction_context::set_needs_rollback(bool rollback) -{ - _needs_rollback = rollback; -} - -bool -transaction_context::can_commit() -{ - return (!_needs_rollback && can_rollback()); -} - -bool -transaction_context::can_rollback() -{ - return (_in_txn && _op_count >= _target_op_count); -} - -/* thread_context class implementation */ -thread_context::thread_context(uint64_t id, thread_type type, configuration *config, - scoped_session &&created_session, timestamp_manager *timestamp_manager, - workload_tracking *tracking, database &dbase) - : /* These won't exist for certain threads which is why we use optional here. */ - collection_count(config->get_optional_int(COLLECTION_COUNT, 1)), - key_count(config->get_optional_int(KEY_COUNT_PER_COLLECTION, 1)), - key_size(config->get_optional_int(KEY_SIZE, 1)), - value_size(config->get_optional_int(VALUE_SIZE, 1)), - thread_count(config->get_int(THREAD_COUNT)), type(type), id(id), db(dbase), - session(std::move(created_session)), tsm(timestamp_manager), - transaction(transaction_context(config, timestamp_manager, session.get())), - tracking(tracking), _throttle(config) -{ - if (tracking->enabled()) - op_track_cursor = session.open_scoped_cursor(tracking->get_operation_table_name()); - - testutil_assert(key_size > 0 && value_size > 0); -} - -void -thread_context::finish() -{ - _running = false; -} - -std::string -thread_context::pad_string(const std::string &value, uint64_t size) -{ - uint64_t diff = size > value.size() ? size - value.size() : 0; - std::string s(diff, '0'); - return (s.append(value)); -} - -bool -thread_context::update( - scoped_cursor &cursor, uint64_t collection_id, const std::string &key, const std::string &value) -{ - WT_DECL_RET; - - testutil_assert(tracking != nullptr); - testutil_assert(cursor.get() != nullptr); - - wt_timestamp_t ts = tsm->get_next_ts(); - ret = transaction.set_commit_timestamp(ts); - testutil_assert(ret == 0 || ret == EINVAL); - if (ret != 0) { - transaction.set_needs_rollback(true); - return (false); - } - - cursor->set_key(cursor.get(), key.c_str()); - cursor->set_value(cursor.get(), value.c_str()); - ret = cursor->update(cursor.get()); - - if (ret != 0) { - if (ret == WT_ROLLBACK) { - transaction.set_needs_rollback(true); - return (false); - } else - testutil_die(ret, "unhandled error while trying to update a key"); - } - - uint64_t txn_id = ((WT_SESSION_IMPL *)session.get())->txn->id; - ret = tracking->save_operation( - txn_id, tracking_operation::INSERT, collection_id, key, value, ts, op_track_cursor); - - if (ret == 0) - transaction.add_op(); - else if (ret == WT_ROLLBACK) - transaction.set_needs_rollback(true); - else - testutil_die(ret, "unhandled error while trying to save an update to the tracking table"); - return (ret == 0); -} - -bool -thread_context::insert( - scoped_cursor &cursor, uint64_t collection_id, const std::string &key, const std::string &value) -{ - WT_DECL_RET; - - testutil_assert(tracking != nullptr); - testutil_assert(cursor.get() != nullptr); - - wt_timestamp_t ts = tsm->get_next_ts(); - ret = transaction.set_commit_timestamp(ts); - testutil_assert(ret == 0 || ret == EINVAL); - if (ret != 0) { - transaction.set_needs_rollback(true); - return (false); - } - - cursor->set_key(cursor.get(), key.c_str()); - cursor->set_value(cursor.get(), value.c_str()); - ret = cursor->insert(cursor.get()); - - if (ret != 0) { - if (ret == WT_ROLLBACK) { - transaction.set_needs_rollback(true); - return (false); - } else - testutil_die(ret, "unhandled error while trying to insert a key"); - } - - uint64_t txn_id = ((WT_SESSION_IMPL *)session.get())->txn->id; - ret = tracking->save_operation( - txn_id, tracking_operation::INSERT, collection_id, key, value, ts, op_track_cursor); - - if (ret == 0) - transaction.add_op(); - else if (ret == WT_ROLLBACK) - transaction.set_needs_rollback(true); - else - testutil_die(ret, "unhandled error while trying to save an insert to the tracking table"); - return (ret == 0); -} - -bool -thread_context::remove(scoped_cursor &cursor, uint64_t collection_id, const std::string &key) -{ - WT_DECL_RET; - testutil_assert(tracking != nullptr); - testutil_assert(cursor.get() != nullptr); - - wt_timestamp_t ts = tsm->get_next_ts(); - ret = transaction.set_commit_timestamp(ts); - testutil_assert(ret == 0 || ret == EINVAL); - if (ret != 0) { - transaction.set_needs_rollback(true); - return (false); - } - - cursor->set_key(cursor.get(), key.c_str()); - ret = cursor->remove(cursor.get()); - if (ret != 0) { - if (ret == WT_ROLLBACK) { - transaction.set_needs_rollback(true); - return (false); - } else - testutil_die(ret, "unhandled error while trying to remove a key"); - } - - uint64_t txn_id = ((WT_SESSION_IMPL *)session.get())->txn->id; - ret = tracking->save_operation( - txn_id, tracking_operation::DELETE_KEY, collection_id, key, "", ts, op_track_cursor); - - if (ret == 0) - transaction.add_op(); - else if (ret == WT_ROLLBACK) - transaction.set_needs_rollback(true); - else - testutil_die(ret, "unhandled error while trying to save a remove to the tracking table"); - return (ret == 0); -} - -void -thread_context::sleep() -{ - _throttle.sleep(); -} - -bool -thread_context::running() const -{ - return (_running); -} -} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h deleted file mode 100644 index e968db51ec9..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h +++ /dev/null @@ -1,170 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef THREAD_CONTEXT_H -#define THREAD_CONTEXT_H - -#include - -extern "C" { -#include "test_util.h" -} - -#include "../core/throttle.h" -#include "../workload/database_model.h" - -/* Forward declarations for classes to reduce compilation time and modules coupling. */ -class configuration; -class timestamp_manager; -class workload_tracking; - -namespace test_harness { -enum thread_type { CUSTOM, INSERT, READ, REMOVE, UPDATE }; - -const std::string type_string(thread_type type); - -class transaction_context { - public: - explicit transaction_context( - configuration *config, timestamp_manager *timestamp_manager, WT_SESSION *session); - - bool active() const; - void add_op(); - void begin(const std::string &config = ""); - /* Begin a transaction if we are not currently in one. */ - void try_begin(const std::string &config = ""); - /* - * Commit a transaction and return true if the commit was successful. - */ - bool commit(const std::string &config = ""); - /* Rollback a transaction, failure will abort the test. */ - void rollback(const std::string &config = ""); - /* Attempt to rollback the transaction given the requirements are met. */ - void try_rollback(const std::string &config = ""); - /* Set a commit timestamp. */ - int set_commit_timestamp(wt_timestamp_t ts); - /* Set that the transaction needs to be rolled back. */ - void set_needs_rollback(bool rollback); - /* - * Returns true if a transaction can be committed as determined by the op count and the state of - * the transaction. - */ - bool can_commit(); - /* - * Returns true if a transaction can be rolled back as determined by the op count and the state - * of the transaction. - */ - bool can_rollback(); - - private: - bool _in_txn = false; - bool _needs_rollback = false; - - /* - * _min_op_count and _max_op_count are the minimum and maximum number of operations within one - * transaction. is the current maximum number of operations that can be executed in the current - * transaction. - */ - int64_t _max_op_count = INT64_MAX; - int64_t _min_op_count = 0; - /* - * op_count is the current number of operations that have been executed in the current - * transaction. - */ - int64_t _op_count = 0; - int64_t _target_op_count = 0; - - timestamp_manager *_timestamp_manager = nullptr; - WT_SESSION *_session = nullptr; -}; - -/* Container class for a thread and any data types it may need to interact with the database. */ -class thread_context { - public: - thread_context(uint64_t id, thread_type type, configuration *config, - scoped_session &&created_session, timestamp_manager *timestamp_manager, - workload_tracking *tracking, database &dbase); - - virtual ~thread_context() = default; - - void finish(); - - /* If the value's size is less than the given size, padding of '0' is added to the value. */ - std::string pad_string(const std::string &value, uint64_t size); - - /* - * Generic update function, takes a collection_id, key and value. - * - * Return true if the operation was successful, a return value of false implies the transaction - * needs to be rolled back. - */ - bool update(scoped_cursor &cursor, uint64_t collection_id, const std::string &key, - const std::string &value); - - /* - * Generic insert function, takes a collection_id, key and value. - * - * Return true if the operation was successful, a return value of false implies the transaction - * needs to be rolled back. - */ - bool insert(scoped_cursor &cursor, uint64_t collection_id, const std::string &key, - const std::string &value); - - /* - * Generic remove function, takes a collection_id and key and will delete the key if it exists. - * - * Return true if the operation was successful, a return value of false implies the transaction - * needs to be rolled back. - */ - bool remove(scoped_cursor &cursor, uint64_t collection_id, const std::string &key); - void sleep(); - bool running() const; - - public: - const int64_t collection_count; - const int64_t key_count; - const int64_t key_size; - const int64_t value_size; - const int64_t thread_count; - const thread_type type; - const uint64_t id; - database &db; - scoped_session session; - scoped_cursor op_track_cursor; - scoped_cursor stat_cursor; - timestamp_manager *tsm; - transaction_context transaction; - workload_tracking *tracking; - - private: - bool _running = true; - throttle _throttle; -}; -} // namespace test_harness - -#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_tracking.cpp b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_tracking.cpp deleted file mode 100644 index 9b459bd4ad8..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_tracking.cpp +++ /dev/null @@ -1,229 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#include "../connection_manager.h" -#include "../core/configuration.h" -#include "../util/api_const.h" -#include "../util/scoped_types.h" -#include "workload_tracking.h" - -namespace test_harness { -workload_tracking::workload_tracking( - configuration *_config, const bool use_compression, timestamp_manager &tsm) - : component(WORKLOAD_TRACKING, _config), _operation_table_name(TABLE_OPERATION_TRACKING), - _schema_table_config(SCHEMA_TRACKING_TABLE_CONFIG), _schema_table_name(TABLE_SCHEMA_TRACKING), - _use_compression(use_compression), _tsm(tsm) -{ - _operation_table_config = "key_format=" + _config->get_string(TRACKING_KEY_FORMAT) + - ",value_format=" + _config->get_string(TRACKING_VALUE_FORMAT) + ",log=(enabled=true)"; -} - -const std::string & -workload_tracking::get_schema_table_name() const -{ - return (_schema_table_name); -} - -const std::string & -workload_tracking::get_operation_table_name() const -{ - return (_operation_table_name); -} - -void -workload_tracking::load() -{ - component::load(); - - if (!_enabled) - return; - - /* Initiate schema tracking. */ - _session = connection_manager::instance().create_session(); - testutil_check( - _session->create(_session.get(), _schema_table_name.c_str(), _schema_table_config.c_str())); - _schema_track_cursor = _session.open_scoped_cursor(_schema_table_name); - logger::log_msg(LOG_TRACE, "Schema tracking initiated"); - - /* Initiate operations tracking. */ - testutil_check(_session->create( - _session.get(), _operation_table_name.c_str(), _operation_table_config.c_str())); - logger::log_msg(LOG_TRACE, "Operations tracking created"); - - /* - * Open sweep cursor in a dedicated sweep session. This cursor will be used to clear out - * obsolete data from the tracking table. - */ - _sweep_session = connection_manager::instance().create_session(); - _sweep_cursor = _sweep_session.open_scoped_cursor(_operation_table_name); - logger::log_msg(LOG_TRACE, "Tracking table sweep initialized"); -} - -void -workload_tracking::do_work() -{ - WT_DECL_RET; - wt_timestamp_t ts, oldest_ts; - uint64_t collection_id, sweep_collection_id; - int op_type; - const char *key, *value; - char *sweep_key; - bool globally_visible_update_found; - - /* - * This function prunes old data from the tracking table as the default validation logic doesn't - * use it. User-defined validation may need this data, so don't allow it to be removed. - */ - const std::string key_format(_sweep_cursor->key_format); - const std::string value_format(_sweep_cursor->value_format); - if (key_format != OPERATION_TRACKING_KEY_FORMAT || - value_format != OPERATION_TRACKING_VALUE_FORMAT) - return; - - key = sweep_key = nullptr; - globally_visible_update_found = false; - - /* Take a copy of the oldest so that we sweep with a consistent timestamp. */ - oldest_ts = _tsm.get_oldest_ts(); - - /* We need to check if the component is still running to avoid unnecessary iterations. */ - while (_running && (ret = _sweep_cursor->prev(_sweep_cursor.get())) == 0) { - testutil_check(_sweep_cursor->get_key(_sweep_cursor.get(), &collection_id, &key, &ts)); - testutil_check(_sweep_cursor->get_value(_sweep_cursor.get(), &op_type, &value)); - /* - * If we're on a new key, reset the check. We want to track whether we have a globally - * visible update for the current key. - */ - if (sweep_key == nullptr || sweep_collection_id != collection_id || - strcmp(sweep_key, key) != 0) { - globally_visible_update_found = false; - if (sweep_key != nullptr) - free(sweep_key); - sweep_key = static_cast(dstrdup(key)); - sweep_collection_id = collection_id; - } - if (ts <= oldest_ts) { - if (globally_visible_update_found) { - if (logger::trace_level == LOG_TRACE) - logger::log_msg(LOG_TRACE, - std::string("workload tracking: Obsoleted update, key=") + sweep_key + - ", collection_id=" + std::to_string(collection_id) + - ", timestamp=" + std::to_string(ts) + - ", oldest_timestamp=" + std::to_string(oldest_ts) + ", value=" + value); - /* - * Wrap the removal in a transaction as we need to specify we aren't using a - * timestamp on purpose. - */ - testutil_check( - _sweep_session->begin_transaction(_sweep_session.get(), "no_timestamp=true")); - testutil_check(_sweep_cursor->remove(_sweep_cursor.get())); - testutil_check(_sweep_session->commit_transaction(_sweep_session.get(), nullptr)); - } else if (static_cast(op_type) == tracking_operation::INSERT) { - if (logger::trace_level == LOG_TRACE) - logger::log_msg(LOG_TRACE, - std::string("workload tracking: Found globally visible update, key=") + - sweep_key + ", collection_id=" + std::to_string(collection_id) + - ", timestamp=" + std::to_string(ts) + - ", oldest_timestamp=" + std::to_string(oldest_ts) + ", value=" + value); - globally_visible_update_found = true; - } - } - } - - free(sweep_key); - - /* - * If we get here and the test is still running, it means we must have reached the end of the - * table. We can also get here because the test is no longer running. In this case, the cursor - * can either be at the end of the table or still on a valid entry since we interrupted the - * work. - */ - if (ret != 0 && ret != WT_NOTFOUND) - testutil_die(LOG_ERROR, - "Tracking table sweep failed: cursor->next() returned an unexpected error %d.", ret); - - /* If we have a position, give it up. */ - testutil_check(_sweep_cursor->reset(_sweep_cursor.get())); -} - -void -workload_tracking::save_schema_operation( - const tracking_operation &operation, const uint64_t &collection_id, wt_timestamp_t ts) -{ - std::string error_message; - - if (!_enabled) - return; - - if (operation == tracking_operation::CREATE_COLLECTION || - operation == tracking_operation::DELETE_COLLECTION) { - _schema_track_cursor->set_key(_schema_track_cursor.get(), collection_id, ts); - _schema_track_cursor->set_value(_schema_track_cursor.get(), static_cast(operation)); - testutil_check(_schema_track_cursor->insert(_schema_track_cursor.get())); - } else { - error_message = - "save_schema_operation: invalid operation " + std::to_string(static_cast(operation)); - testutil_die(EINVAL, error_message.c_str()); - } -} - -int -workload_tracking::save_operation(const uint64_t txn_id, const tracking_operation &operation, - const uint64_t &collection_id, const std::string &key, const std::string &value, - wt_timestamp_t ts, scoped_cursor &op_track_cursor) -{ - WT_DECL_RET; - - if (!_enabled) - return (0); - - testutil_assert(op_track_cursor.get() != nullptr); - - if (operation == tracking_operation::CREATE_COLLECTION || - operation == tracking_operation::DELETE_COLLECTION) { - const std::string error_message = - "save_operation: invalid operation " + std::to_string(static_cast(operation)); - testutil_die(EINVAL, error_message.c_str()); - } else { - set_tracking_cursor(txn_id, operation, collection_id, key, value, ts, op_track_cursor); - ret = op_track_cursor->insert(op_track_cursor.get()); - } - return (ret); -} - -/* Note that the transaction id is not used in the default implementation of the tracking table. */ -void -workload_tracking::set_tracking_cursor(const uint64_t txn_id, const tracking_operation &operation, - const uint64_t &collection_id, const std::string &key, const std::string &value, - wt_timestamp_t ts, scoped_cursor &op_track_cursor) -{ - op_track_cursor->set_key(op_track_cursor.get(), collection_id, key.c_str(), ts); - op_track_cursor->set_value(op_track_cursor.get(), static_cast(operation), value.c_str()); -} - -} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_tracking.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_tracking.h deleted file mode 100644 index 0599c0b5a55..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_tracking.h +++ /dev/null @@ -1,99 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef WORKLOAD_TRACKING_H -#define WORKLOAD_TRACKING_H - -#include "../core/component.h" -#include "../timestamp_manager.h" -#include "../util/scoped_types.h" - -/* - * Default schema for tracking operations on collections (key_format: Collection id / Key / - * Timestamp, value_format: Operation type / Value) - */ -#define OPERATION_TRACKING_KEY_FORMAT WT_UNCHECKED_STRING(QSQ) -#define OPERATION_TRACKING_VALUE_FORMAT WT_UNCHECKED_STRING(iS) - -/* - * Default schema for tracking schema operations on collections (key_format: Collection id / - * Timestamp, value_format: Operation type) - */ -#define SCHEMA_TRACKING_KEY_FORMAT WT_UNCHECKED_STRING(QQ) -#define SCHEMA_TRACKING_VALUE_FORMAT WT_UNCHECKED_STRING(i) -#define SCHEMA_TRACKING_TABLE_CONFIG \ - "key_format=" SCHEMA_TRACKING_KEY_FORMAT ",value_format=" SCHEMA_TRACKING_VALUE_FORMAT \ - ",log=(enabled=true)" - -namespace test_harness { -/* Tracking operations. */ -enum class tracking_operation { CREATE_COLLECTION, CUSTOM, DELETE_COLLECTION, DELETE_KEY, INSERT }; - -/* Class used to track operations performed on collections */ -class workload_tracking : public component { - public: - workload_tracking(configuration *_config, const bool use_compression, timestamp_manager &tsm); - virtual ~workload_tracking() = default; - - const std::string &get_schema_table_name() const; - const std::string &get_operation_table_name() const; - void load() override final; - - /* - * As every operation is tracked in the tracking table we need to clear out obsolete operations - * otherwise the file size grow continuously, as such we cleanup operations that are no longer - * relevant, i.e. older than the oldest timestamp. - */ - void do_work() override final; - - void save_schema_operation( - const tracking_operation &operation, const uint64_t &collection_id, wt_timestamp_t ts); - - virtual void set_tracking_cursor(const uint64_t txn_id, const tracking_operation &operation, - const uint64_t &collection_id, const std::string &key, const std::string &value, - wt_timestamp_t ts, scoped_cursor &op_track_cursor); - - int save_operation(const uint64_t txn_id, const tracking_operation &operation, - const uint64_t &collection_id, const std::string &key, const std::string &value, - wt_timestamp_t ts, scoped_cursor &op_track_cursor); - - private: - scoped_session _session; - scoped_session _sweep_session; - scoped_cursor _schema_track_cursor; - scoped_cursor _sweep_cursor; - std::string _operation_table_config; - const std::string _operation_table_name; - const std::string _schema_table_config; - const std::string _schema_table_name; - const bool _use_compression; - timestamp_manager &_tsm; -}; -} // namespace test_harness - -#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_validation.cpp b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_validation.cpp deleted file mode 100644 index a64ef7e5b7b..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_validation.cpp +++ /dev/null @@ -1,291 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#include - -#include "../connection_manager.h" -#include "database_model.h" -#include "workload_validation.h" - -namespace test_harness { -void -workload_validation::validate(const std::string &operation_table_name, - const std::string &schema_table_name, const std::vector &known_collection_ids) -{ - WT_DECL_RET; - wt_timestamp_t tracked_timestamp; - std::vector created_collections, deleted_collections; - uint64_t tracked_collection_id; - const char *tracked_key, *tracked_value; - int tracked_op_type; - uint64_t current_collection_id = 0; - - logger::log_msg(LOG_INFO, "Beginning validation."); - - scoped_session session = connection_manager::instance().create_session(); - scoped_cursor cursor = session.open_scoped_cursor(operation_table_name); - - /* - * Default validation depends on specific fields being present in the tracking table. If the - * tracking table schema has been modified the user must define their own validation. - */ - const std::string key_format(cursor->key_format); - const std::string value_format(cursor->value_format); - if (key_format != OPERATION_TRACKING_KEY_FORMAT || - value_format != OPERATION_TRACKING_VALUE_FORMAT) { - testutil_die(EINVAL, - "Attempting to perform default validation on a test with a user-defined tracking " - "table. Please define validation for your test"); - } - - /* Retrieve the collections that were created and deleted during the test. */ - parse_schema_tracking_table( - session, schema_table_name, created_collections, deleted_collections); - - /* - * Make sure the deleted collections do not exist on disk. The created collections are checked - * in check_reference. - */ - for (auto const &it : deleted_collections) { - if (!verify_collection_file_state(session, it, false)) - testutil_die(LOG_ERROR, - "Validation failed: collection %s present on disk while it has been tracked as " - "deleted.", - database::build_collection_name(it).c_str()); - } - - /* - * All collections in memory should match those created in the schema tracking table. Dropping - * is currently not supported. - */ - std::sort(created_collections.begin(), created_collections.end()); - auto on_disk_collection_id = created_collections.begin(); - if (created_collections.size() != known_collection_ids.size()) - testutil_die(LOG_ERROR, - "Validation failed: collection state mismatch, expected %lu" - " collections to exist but have %lu on disk", - created_collections.size(), known_collection_ids.size()); - for (const auto id : known_collection_ids) { - if (id != *on_disk_collection_id) - testutil_die(LOG_ERROR, - "Validation failed: collection state mismatch expected " - "collection id %lu but got %lu.", - id, *on_disk_collection_id); - on_disk_collection_id++; - } - - /* Parse the tracking table. */ - validation_collection current_collection_records; - while ((ret = cursor->next(cursor.get())) == 0) { - testutil_check( - cursor->get_key(cursor.get(), &tracked_collection_id, &tracked_key, &tracked_timestamp)); - testutil_check(cursor->get_value(cursor.get(), &tracked_op_type, &tracked_value)); - - logger::log_msg(LOG_TRACE, - "Retrieved tracked values. \n Collection id: " + std::to_string(tracked_collection_id) + - "\n Key: " + std::string(tracked_key) + - "\n Timestamp: " + std::to_string(tracked_timestamp) + "\n Operation type: " + - std::to_string(tracked_op_type) + "\n Value: " + std::string(tracked_value)); - - /* - * Check if we've stepped over to the next collection. The tracking table is sorted by - * collection_id so this is correct. - */ - if (tracked_collection_id != current_collection_id) { - if (std::find(known_collection_ids.begin(), known_collection_ids.end(), - tracked_collection_id) == known_collection_ids.end()) - testutil_die(LOG_ERROR, - "Validation failed: The collection id %lu is not part of the known " - "collection set.", - tracked_collection_id); - if (tracked_collection_id < current_collection_id) - testutil_die(LOG_ERROR, "Validation failed: The collection id %lu is out of order.", - tracked_collection_id); - - /* - * Given that we've stepped over to the next collection we've built a full picture of - * the current collection and can now validate it. - */ - verify_collection(session, current_collection_id, current_collection_records); - - /* Begin processing the next collection. */ - current_collection_id = tracked_collection_id; - current_collection_records.clear(); - } - - /* - * Add the values from the tracking table to the current collection model. - */ - update_data_model(static_cast(tracked_op_type), - current_collection_records, current_collection_id, tracked_key, tracked_value); - }; - - /* The value of ret should be WT_NOTFOUND once the cursor has read all rows. */ - if (ret != WT_NOTFOUND) - testutil_die( - LOG_ERROR, "Validation failed: cursor->next() return an unexpected error %d.", ret); - - /* - * We still need to validate the last collection. But we can also end up here if there aren't - * any collections, check for that. - */ - if (known_collection_ids.size() != 0) - verify_collection(session, current_collection_id, current_collection_records); -} - -void -workload_validation::parse_schema_tracking_table(scoped_session &session, - const std::string &tracking_table_name, std::vector &created_collections, - std::vector &deleted_collections) -{ - wt_timestamp_t key_timestamp; - uint64_t key_collection_id; - int value_operation_type; - - scoped_cursor cursor = session.open_scoped_cursor(tracking_table_name); - - while (cursor->next(cursor.get()) == 0) { - testutil_check(cursor->get_key(cursor.get(), &key_collection_id, &key_timestamp)); - testutil_check(cursor->get_value(cursor.get(), &value_operation_type)); - - logger::log_msg(LOG_TRACE, "Collection id is " + std::to_string(key_collection_id)); - logger::log_msg(LOG_TRACE, "Timestamp is " + std::to_string(key_timestamp)); - logger::log_msg(LOG_TRACE, "Operation type is " + std::to_string(value_operation_type)); - - if (static_cast(value_operation_type) == - tracking_operation::CREATE_COLLECTION) { - deleted_collections.erase(std::remove(deleted_collections.begin(), - deleted_collections.end(), key_collection_id), - deleted_collections.end()); - created_collections.push_back(key_collection_id); - } else if (static_cast(value_operation_type) == - tracking_operation::DELETE_COLLECTION) { - created_collections.erase(std::remove(created_collections.begin(), - created_collections.end(), key_collection_id), - created_collections.end()); - deleted_collections.push_back(key_collection_id); - } - } -} - -void -workload_validation::update_data_model(const tracking_operation &operation, - validation_collection &collection, const uint64_t collection_id, const char *key, - const char *value) -{ - if (operation == tracking_operation::DELETE_KEY) { - /* Search for the key validating that it exists. */ - const auto it = collection.find(key); - if (it == collection.end()) - testutil_die(LOG_ERROR, - "Validation failed: key deleted that doesn't exist. Collection id: %lu Key: %s", - collection_id, key); - else if (it->second.exists == false) - /* The key has been deleted twice. */ - testutil_die(LOG_ERROR, - "Validation failed: deleted key deleted again. Collection id: %lu Key: %s", - collection_id, it->first.c_str()); - - /* Update the key_state to deleted. */ - it->second.exists = false; - } else if (operation == tracking_operation::INSERT) - collection[key_value_t(key)] = key_state{true, key_value_t(value)}; - else - testutil_die(LOG_ERROR, "Validation failed: unexpected operation in the tracking table: %d", - static_cast(operation)); -} - -void -workload_validation::verify_collection( - scoped_session &session, const uint64_t collection_id, validation_collection &collection) -{ - /* Check the collection exists on disk. */ - if (!verify_collection_file_state(session, collection_id, true)) - testutil_die(LOG_ERROR, - "Validation failed: collection %lu not present on disk while it has been tracked as " - "created.", - collection_id); - - /* Walk through each key/value pair of the current collection. */ - for (const auto &record : collection) - verify_key_value(session, collection_id, record.first, record.second); -} - -bool -workload_validation::verify_collection_file_state( - scoped_session &session, const uint64_t collection_id, bool exists) const -{ - /* - * We don't necessarily expect to successfully open the cursor so don't create a scoped cursor. - */ - WT_CURSOR *cursor; - int ret = session->open_cursor(session.get(), - database::build_collection_name(collection_id).c_str(), nullptr, nullptr, &cursor); - if (ret == 0) - testutil_check(cursor->close(cursor)); - return (exists ? (ret == 0) : (ret != 0)); -} - -void -workload_validation::verify_key_value(scoped_session &session, const uint64_t collection_id, - const std::string &key, const key_state &key_state) -{ - WT_DECL_RET; - const char *retrieved_value; - - scoped_cursor cursor = - session.open_scoped_cursor(database::build_collection_name(collection_id)); - cursor->set_key(cursor.get(), key.c_str()); - ret = cursor->search(cursor.get()); - testutil_assertfmt(ret == 0 || ret == WT_NOTFOUND, - "Validation failed: Unexpected error returned %d while searching for a key. Key: %s, " - "Collection_id: %lu", - ret, key.c_str(), collection_id); - if (ret == WT_NOTFOUND && key_state.exists) - testutil_die(LOG_ERROR, - "Validation failed: Search failed to find key that should exist. Key: %s, " - "Collection_id: %lu", - key.c_str(), collection_id); - else if (ret == 0 && key_state.exists == false) { - testutil_die(LOG_ERROR, - "Validation failed: Key exists when it is expected to be deleted. Key: %s, " - "Collection_id: %lu", - key.c_str(), collection_id); - } - - if (key_state.exists == false) - return; - - testutil_check(cursor->get_value(cursor.get(), &retrieved_value)); - if (key_state.value != key_value_t(retrieved_value)) - testutil_die(LOG_ERROR, - "Validation failed: Value mismatch for key. Key: %s, Collection_id: %lu, Expected " - "value: %s, Found value: %s", - key.c_str(), collection_id, key_state.value.c_str(), retrieved_value); -} -} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_validation.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_validation.h deleted file mode 100644 index ba745a225df..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_validation.h +++ /dev/null @@ -1,99 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef WORKLOAD_VALIDATION_H -#define WORKLOAD_VALIDATION_H - -#include -#include -#include - -extern "C" { -#include "wiredtiger.h" -} - -#include "../util/scoped_types.h" - -namespace test_harness { -struct key_state { - key_state() = default; - key_state(bool exists, const key_value_t &value) : exists(exists), value(value) {} - bool exists = false; - key_value_t value; -}; -typedef std::map validation_collection; -/* - * Class that can validate database state and collection data. - */ -class workload_validation { - public: - /* - * Validate the on disk data against what has been tracked during the test. This is done by - * replaying the tracked operations so a representation in memory of the collections is created. - * This representation is then compared to what is on disk. - * - * - operation_table_name: Table that contains all the operations performed on keys. - * - schema_table_name: Table that contains all the schema operations performed. - */ - void validate(const std::string &operation_table_name, const std::string &schema_table_name, - const std::vector &known_collection_ids); - - private: - /* - * Read the tracking table to retrieve the created and deleted collections during the test. - * collection_name: collection that contains the operations on the different collections during - * the test. - */ - void parse_schema_tracking_table(scoped_session &session, - const std::string &tracking_table_name, std::vector &created_collections, - std::vector &deleted_collections); - - /* Update the data model. */ - void update_data_model(const tracking_operation &operation, validation_collection &collection, - const uint64_t collection_id, const char *key, const char *value); - - /* - * Compare the tracked operations against what has been saved on disk. - */ - void verify_collection( - scoped_session &session, const uint64_t collection_id, validation_collection &collection); - - /* - * Check whether a collection exists on disk. exists: needs to be set to true if the collection - * is expected to be existing, false otherwise. - */ - bool verify_collection_file_state( - scoped_session &session, const uint64_t collection_id, bool exists) const; - - /* Verify the given expected value is the same on disk. */ - void verify_key_value(scoped_session &session, const uint64_t collection_id, - const std::string &key, const key_state &key_state); -}; -} // namespace test_harness - -#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_generator.cpp b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_generator.cpp deleted file mode 100644 index 026f05b2cb5..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_generator.cpp +++ /dev/null @@ -1,163 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#include -#include - -#include "connection_manager.h" -#include "core/configuration.h" -#include "core/throttle.h" -#include "util/api_const.h" -#include "workload/database_model.h" -#include "workload/database_operation.h" -#include "workload/random_generator.h" -#include "workload/workload_tracking.h" -#include "workload_generator.h" - -namespace test_harness { -/* operation_config class implementation */ -operation_config::operation_config(configuration *config, thread_type type) - : config(config), type(type), thread_count(config->get_int(THREAD_COUNT)) -{ -} - -std::function -operation_config::get_func(database_operation *dbo) -{ - switch (type) { - case thread_type::CUSTOM: - return (std::bind(&database_operation::custom_operation, dbo, std::placeholders::_1)); - case thread_type::INSERT: - return (std::bind(&database_operation::insert_operation, dbo, std::placeholders::_1)); - case thread_type::READ: - return (std::bind(&database_operation::read_operation, dbo, std::placeholders::_1)); - case thread_type::REMOVE: - return (std::bind(&database_operation::remove_operation, dbo, std::placeholders::_1)); - case thread_type::UPDATE: - return (std::bind(&database_operation::update_operation, dbo, std::placeholders::_1)); - default: - /* This may cause a separate testutil_die in type_string but that should be okay. */ - testutil_die(EINVAL, "unexpected thread_type: %s", type_string(type).c_str()); - } -} - -/* workload_generator class implementation */ -workload_generator::workload_generator(configuration *configuration, - database_operation *db_operation, timestamp_manager *timestamp_manager, database &database) - : component(WORKLOAD_GENERATOR, configuration), _database(database), - _database_operation(db_operation), _timestamp_manager(timestamp_manager) -{ -} - -workload_generator::~workload_generator() -{ - for (auto &it : _workers) - delete it; -} - -void -workload_generator::set_workload_tracking(workload_tracking *tracking) -{ - testutil_assert(_tracking == nullptr); - _tracking = tracking; -} - -void -workload_generator::run() -{ - configuration *populate_config; - std::vector operation_configs; - uint64_t thread_id = 0; - - /* Retrieve useful parameters from the test configuration. */ - operation_configs.push_back( - operation_config(_config->get_subconfig(CUSTOM_OP_CONFIG), thread_type::CUSTOM)); - operation_configs.push_back( - operation_config(_config->get_subconfig(INSERT_OP_CONFIG), thread_type::INSERT)); - operation_configs.push_back( - operation_config(_config->get_subconfig(READ_OP_CONFIG), thread_type::READ)); - operation_configs.push_back( - operation_config(_config->get_subconfig(REMOVE_OP_CONFIG), thread_type::REMOVE)); - operation_configs.push_back( - operation_config(_config->get_subconfig(UPDATE_OP_CONFIG), thread_type::UPDATE)); - populate_config = _config->get_subconfig(POPULATE_CONFIG); - - /* Populate the database. */ - _database_operation->populate(_database, _timestamp_manager, populate_config, _tracking); - _db_populated = true; - delete populate_config; - - /* Generate threads to execute read operations on the collections. */ - for (auto &it : operation_configs) { - if (it.thread_count != 0) - logger::log_msg(LOG_INFO, - "Workload_generator: Creating " + std::to_string(it.thread_count) + " " + - type_string(it.type) + " threads."); - for (size_t i = 0; i < it.thread_count && _running; ++i) { - thread_context *tc = new thread_context(thread_id++, it.type, it.config, - connection_manager::instance().create_session(), _timestamp_manager, _tracking, - _database); - _workers.push_back(tc); - _thread_manager.add_thread(it.get_func(_database_operation), tc); - } - /* - * Don't forget to delete the config we created earlier. While we do pass the config into - * the thread context it is not saved, so we are safe to do this. - */ - delete it.config; - - /* - * Reset the thread_id counter to 0 as we're only interested in knowing per operation type - * which thread we are. - */ - thread_id = 0; - } -} - -void -workload_generator::finish() -{ - component::finish(); - for (const auto &it : _workers) - it->finish(); - _thread_manager.join(); - logger::log_msg(LOG_TRACE, "Workload generator: run stage done"); -} - -database & -workload_generator::get_database() -{ - return (_database); -} - -bool -workload_generator::db_populated() const -{ - return (_db_populated); -} -} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_generator.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_generator.h deleted file mode 100644 index 11049e53063..00000000000 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_generator.h +++ /dev/null @@ -1,97 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef WORKLOAD_GENERATOR_H -#define WORKLOAD_GENERATOR_H - -#include -#include - -#include "core/component.h" -#include "workload/thread_context.h" -#include "thread_manager.h" -#include "workload/database_operation.h" - -/* Forward declarations for classes to reduce compilation time and modules coupling. */ -class configuration; -class database; -class workload_tracking; - -namespace test_harness { -/* - * Helper class to enable scalable operation types in the database_operation. - */ -class operation_config { - public: - explicit operation_config(configuration *config, thread_type type); - - /* Returns a function pointer to the member function of the supplied database operation. */ - std::function get_func(database_operation *dbo); - - public: - configuration *config; - const thread_type type; - const int64_t thread_count; -}; - -/* - * Class that can execute operations based on a given configuration. - */ -class workload_generator : public component { - public: - explicit workload_generator(configuration *configuration, database_operation *db_operation, - timestamp_manager *timestamp_manager, database &database); - - ~workload_generator(); - - /* Delete the copy constructor and the assignment operator. */ - workload_generator(const workload_generator &) = delete; - workload_generator &operator=(const workload_generator &) = delete; - - /* Do the work of the main part of the workload. */ - void run() override final; - void finish() override final; - - database &get_database(); - bool db_populated() const; - - /* Set the tracking component. */ - void set_workload_tracking(workload_tracking *tracking); - - private: - database &_database; - database_operation *_database_operation = nullptr; - thread_manager _thread_manager; - timestamp_manager *_timestamp_manager = nullptr; - workload_tracking *_tracking = nullptr; - std::vector _workers; - bool _db_populated = false; -}; -} // namespace test_harness - -#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/bounded_cursor_perf.cpp b/src/third_party/wiredtiger/test/cppsuite/tests/bounded_cursor_perf.cpp index 7f4dfea30b0..faa002f06af 100644 --- a/src/third_party/wiredtiger/test/cppsuite/tests/bounded_cursor_perf.cpp +++ b/src/third_party/wiredtiger/test/cppsuite/tests/bounded_cursor_perf.cpp @@ -26,14 +26,10 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include +#include "src/component/op_tracker.cpp" +#include "src/main/test.h" -#include "test_harness/core/op_tracker.cpp" -#include "test_harness/test.h" -#include "test_harness/util/api_const.h" -#include "test_harness/workload/random_generator.h" - -namespace test_harness { +using namespace test_harness; /* * This test performs cursor traversal operations next() and prev() on a collection with both @@ -50,9 +46,11 @@ class bounded_cursor_perf : public test { static void set_bounds(scoped_cursor &cursor) { - cursor->set_key(cursor.get(), std::string(1, ('0' - 1)).c_str()); + std::string lower_bound(1, ('0' - 1)); + cursor->set_key(cursor.get(), lower_bound.c_str()); cursor->bound(cursor.get(), "bound=lower"); - cursor->set_key(cursor.get(), std::string(1, ('9' + 1)).c_str()); + std::string upper_bound(1, ('9' + 1)); + cursor->set_key(cursor.get(), upper_bound.c_str()); cursor->bound(cursor.get(), "bound=upper"); } @@ -114,4 +112,3 @@ class bounded_cursor_perf : public test { } } }; -} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/burst_inserts.cpp b/src/third_party/wiredtiger/test/cppsuite/tests/burst_inserts.cpp index 97341e356b3..64ae392d14b 100644 --- a/src/third_party/wiredtiger/test/cppsuite/tests/burst_inserts.cpp +++ b/src/third_party/wiredtiger/test/cppsuite/tests/burst_inserts.cpp @@ -26,9 +26,8 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include "test_harness/test.h" -#include "test_harness/workload/random_generator.h" -#include "test_harness/timestamp_manager.h" +#include "src/common/random_generator.h" +#include "src/main/test.h" using namespace test_harness; diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/cache_resize.cpp b/src/third_party/wiredtiger/test/cppsuite/tests/cache_resize.cpp index ccffed30420..98b357b7eb8 100644 --- a/src/third_party/wiredtiger/test/cppsuite/tests/cache_resize.cpp +++ b/src/third_party/wiredtiger/test/cppsuite/tests/cache_resize.cpp @@ -26,11 +26,15 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include "test_harness/test.h" +#include "src/common/api_const.h" +#include "src/common/random_generator.h" +#include "src/component/workload_tracking.h" +#include "src/main/test.h" + +using namespace test_harness; -namespace test_harness { /* Defines what data is written to the tracking table for use in custom validation. */ -class tracking_table_cache_resize : public test_harness::workload_tracking { +class tracking_table_cache_resize : public workload_tracking { public: tracking_table_cache_resize( @@ -55,16 +59,16 @@ class tracking_table_cache_resize : public test_harness::workload_tracking { * than the cache size they are rejected, so only transactions made when cache size is 500MB should * be allowed. */ -class cache_resize : public test_harness::test { +class cache_resize : public test { public: - cache_resize(const test_harness::test_args &args) : test(args) + cache_resize(const test_args &args) : test(args) { init_tracking(new tracking_table_cache_resize(_config->get_subconfig(WORKLOAD_TRACKING), _config->get_bool(COMPRESSION_ENABLED), *_timestamp_manager)); } void - custom_operation(test_harness::thread_context *tc) override final + custom_operation(thread_context *tc) override final { WT_CONNECTION *conn = connection_manager::instance().get_connection(); WT_CONNECTION_IMPL *conn_impl = (WT_CONNECTION_IMPL *)conn; @@ -120,7 +124,7 @@ class cache_resize : public test_harness::test { } void - insert_operation(test_harness::thread_context *tc) override final + insert_operation(thread_context *tc) override final { const uint64_t collection_count = tc->db.get_collection_count(); testutil_assert(collection_count > 0); @@ -233,5 +237,3 @@ class cache_resize : public test_harness::test { */ } }; - -} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/csuite_style_example_test.cpp b/src/third_party/wiredtiger/test/cppsuite/tests/csuite_style_example_test.cpp index 0b5a2966b2f..155ffc101e9 100644 --- a/src/third_party/wiredtiger/test/cppsuite/tests/csuite_style_example_test.cpp +++ b/src/third_party/wiredtiger/test/cppsuite/tests/csuite_style_example_test.cpp @@ -33,15 +33,16 @@ * test_template.cpp and create_script.sh. */ -#include "test_harness/connection_manager.h" -#include "test_harness/thread_manager.h" -#include "test_harness/util/api_const.h" -#include "test_harness/util/logger.h" -#include "test_harness/util/scoped_connection.h" -#include "test_harness/workload/random_generator.h" +#include "src/common/api_const.h" +#include "src/common/logger.h" +#include "src/common/random_generator.h" +#include "src/common/thread_manager.h" +#include "src/storage/connection_manager.h" +#include "src/storage/scoped_connection.h" extern "C" { #include "wiredtiger.h" +#include "test_util.h" } using namespace test_harness; diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/cursor_bound_01.cpp b/src/third_party/wiredtiger/test/cppsuite/tests/cursor_bound_01.cpp index f3f733dfe25..ba048dbbca8 100644 --- a/src/third_party/wiredtiger/test/cppsuite/tests/cursor_bound_01.cpp +++ b/src/third_party/wiredtiger/test/cppsuite/tests/cursor_bound_01.cpp @@ -26,9 +26,11 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include "test_harness/test.h" -#include "test_harness/util/api_const.h" -#include "test_harness/workload/random_generator.h" +#include "src/common/api_const.h" +#include "src/common/random_generator.h" +#include "src/main/test.h" + +using namespace test_harness; /* * In this test, we want to verify the usage of the cursor bound API and check that the cursor @@ -43,8 +45,6 @@ * random bounds set. Both next() and prev() calls with bounds set is verified against the * default cursor next() and prev() calls. */ -namespace test_harness { - class cursor_bound_01 : public test { /* Class helper to represent the lower and uppers bounds for the range cursor. */ class bound { @@ -605,7 +605,7 @@ class cursor_bound_01 : public test { } scoped_cursor normal_cursor = tc->session.open_scoped_cursor(coll.name); - wt_timestamp_t ts = tc->tsm->get_random_ts(); + wt_timestamp_t ts = tc->tsm->get_valid_read_ts(); /* * The oldest timestamp might move ahead and the reading timestamp might become invalid. * To tackle this issue, we round the timestamp to the oldest timestamp value. @@ -677,7 +677,7 @@ class cursor_bound_01 : public test { } scoped_cursor normal_cursor = tc->session.open_scoped_cursor(coll.name); - wt_timestamp_t ts = tc->tsm->get_random_ts(); + wt_timestamp_t ts = tc->tsm->get_valid_read_ts(); /* * The oldest timestamp might move ahead and the reading timestamp might become invalid. * To tackle this issue, we round the timestamp to the oldest timestamp value. @@ -699,5 +699,3 @@ class cursor_bound_01 : public test { tc->transaction.rollback(); } }; - -} // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/hs_cleanup.cpp b/src/third_party/wiredtiger/test/cppsuite/tests/hs_cleanup.cpp index 475153ae9c7..a92b53066fb 100644 --- a/src/third_party/wiredtiger/test/cppsuite/tests/hs_cleanup.cpp +++ b/src/third_party/wiredtiger/test/cppsuite/tests/hs_cleanup.cpp @@ -26,9 +26,8 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include "test_harness/test.h" - -#include "test_harness/connection_manager.h" +#include "src/common/random_generator.h" +#include "src/main/test.h" using namespace test_harness; diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/operations_test.cpp b/src/third_party/wiredtiger/test/cppsuite/tests/operations_test.cpp index b96ba0ebd86..7be490958ff 100644 --- a/src/third_party/wiredtiger/test/cppsuite/tests/operations_test.cpp +++ b/src/third_party/wiredtiger/test/cppsuite/tests/operations_test.cpp @@ -26,7 +26,9 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include "test_harness/test.h" +#include "src/main/test.h" + +using namespace test_harness; /* * The "base test" that the framework uses, because its not overloading any of the database @@ -34,9 +36,9 @@ * * Can be used to create stress tests in various ways. */ -class operations_test : public test_harness::test { +class operations_test : public test { public: - operations_test(const test_harness::test_args &args) : test(args) + operations_test(const test_args &args) : test(args) { init_tracking(); } diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/run.cpp b/src/third_party/wiredtiger/test/cppsuite/tests/run.cpp index 6d52caee4fb..c79c0cb4220 100755 --- a/src/third_party/wiredtiger/test/cppsuite/tests/run.cpp +++ b/src/third_party/wiredtiger/test/cppsuite/tests/run.cpp @@ -26,12 +26,12 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include +#include #include #include -#include "test_harness/util/logger.h" -#include "test_harness/test.h" +#include "src/common/logger.h" +#include "src/main/test.h" #include "bounded_cursor_perf.cpp" #include "burst_inserts.cpp" @@ -44,6 +44,10 @@ #include "search_near_03.cpp" #include "test_template.cpp" +extern "C" { +#include "test_util.h" +} + /* Declarations to avoid the error raised by -Werror=missing-prototypes. */ const std::string parse_configuration_from_file(const std::string &filename); void print_help(); diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/search_near_01.cpp b/src/third_party/wiredtiger/test/cppsuite/tests/search_near_01.cpp index dcb6692f643..b70c5573f74 100644 --- a/src/third_party/wiredtiger/test/cppsuite/tests/search_near_01.cpp +++ b/src/third_party/wiredtiger/test/cppsuite/tests/search_near_01.cpp @@ -26,13 +26,12 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include "test_harness/util/api_const.h" -#include "test_harness/workload/random_generator.h" -#include "test_harness/workload/thread_context.h" -#include "test_harness/test.h" -#include "test_harness/thread_manager.h" +#include "src/common/api_const.h" +#include "src/common/random_generator.h" +#include "src/main/test.h" using namespace test_harness; + /* * In this test, we want to verify that search_near with prefix enabled only traverses the portion * of the tree that follows the prefix portion of the search key. The test is composed of a populate @@ -43,7 +42,7 @@ using namespace test_harness; * - Using WiredTiger statistics to validate that the number of entries traversed is within * bounds of the search key. */ -class search_near_01 : public test_harness::test { +class search_near_01 : public test { uint64_t keys_per_prefix = 0; uint64_t srchkey_len = 0; const std::string ALPHABET{"abcdefghijklmnopqrstuvwxyz"}; @@ -102,14 +101,14 @@ class search_near_01 : public test_harness::test { } public: - search_near_01(const test_harness::test_args &args) : test(args) + search_near_01(const test_args &args) : test(args) { init_tracking(); } void - populate(test_harness::database &database, test_harness::timestamp_manager *tsm, - test_harness::configuration *config, test_harness::workload_tracking *tracking) override final + populate(database &database, timestamp_manager *tsm, configuration *config, + workload_tracking *tracking) override final { uint64_t collection_count, key_size; std::vector workers; @@ -180,8 +179,8 @@ class search_near_01 : public test_harness::test { } static void - perform_search_near(test_harness::thread_context *tc, std::string collection_name, - uint64_t srchkey_len, std::atomic &z_key_searches) + perform_search_near(thread_context *tc, std::string collection_name, uint64_t srchkey_len, + std::atomic &z_key_searches) { std::string srch_key; int cmpp = 0; @@ -223,13 +222,13 @@ class search_near_01 : public test_harness::test { } void - read_operation(test_harness::thread_context *tc) override final + read_operation(thread_context *tc) override final { /* Make sure that thread statistics cursor is null before we open it. */ testutil_assert(tc->stat_cursor.get() == nullptr); /* This test will only work with one read thread. */ testutil_assert(tc->thread_count == 1); - test_harness::configuration *workload_config, *read_config; + configuration *workload_config, *read_config; std::vector workers; std::atomic z_key_searches; int64_t entries_stat, expected_entries, prefix_stat, prev_entries_stat, prev_prefix_stat; diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/search_near_02.cpp b/src/third_party/wiredtiger/test/cppsuite/tests/search_near_02.cpp index 63d0354c9c8..dd98a88e6dc 100644 --- a/src/third_party/wiredtiger/test/cppsuite/tests/search_near_02.cpp +++ b/src/third_party/wiredtiger/test/cppsuite/tests/search_near_02.cpp @@ -26,9 +26,9 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include "test_harness/test.h" -#include "test_harness/util/api_const.h" -#include "test_harness/workload/random_generator.h" +#include "src/common/api_const.h" +#include "src/common/random_generator.h" +#include "src/main/test.h" using namespace test_harness; @@ -39,16 +39,16 @@ using namespace test_harness; * - M threads will execute search_near calls with prefix enabled using random prefixes as well. * Each search_near call with prefix enabled is verified using the default search_near. */ -class search_near_02 : public test_harness::test { +class search_near_02 : public test { public: - search_near_02(const test_harness::test_args &args) : test(args) + search_near_02(const test_args &args) : test(args) { init_tracking(); } void - populate(test_harness::database &database, test_harness::timestamp_manager *, - test_harness::configuration *config, test_harness::workload_tracking *) override final + populate(database &database, timestamp_manager *, configuration *config, + workload_tracking *) override final { /* * The populate phase only creates empty collections. The number of collections is defined @@ -66,7 +66,7 @@ class search_near_02 : public test_harness::test { } void - insert_operation(test_harness::thread_context *tc) override final + insert_operation(thread_context *tc) override final { /* Each insert operation will insert new keys in the collections. */ logger::log_msg( @@ -145,7 +145,7 @@ class search_near_02 : public test_harness::test { } void - read_operation(test_harness::thread_context *tc) override final + read_operation(thread_context *tc) override final { /* * Each read operation performs search_near calls with and without prefix enabled on random @@ -176,7 +176,7 @@ class search_near_02 : public test_harness::test { auto &cursor_prefix = cursors[coll.id]; - wt_timestamp_t ts = tc->tsm->get_random_ts(); + wt_timestamp_t ts = tc->tsm->get_valid_read_ts(); /* * The oldest timestamp might move ahead and the reading timestamp might become invalid. * To tackle this issue, we round the timestamp to the oldest timestamp value. diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/search_near_03.cpp b/src/third_party/wiredtiger/test/cppsuite/tests/search_near_03.cpp index dc6b4a0352a..b4b365a9792 100644 --- a/src/third_party/wiredtiger/test/cppsuite/tests/search_near_03.cpp +++ b/src/third_party/wiredtiger/test/cppsuite/tests/search_near_03.cpp @@ -26,9 +26,9 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include "test_harness/test.h" -#include "test_harness/util/api_const.h" -#include "test_harness/workload/random_generator.h" +#include "src/common/api_const.h" +#include "src/common/random_generator.h" +#include "src/main/test.h" using namespace test_harness; @@ -40,13 +40,13 @@ using namespace test_harness; * - M threads will traverse the collections and ensure that the number of records in the * collections don't change. */ -class search_near_03 : public test_harness::test { +class search_near_03 : public test { /* A 2D array consisted of a mapping between each collection and their inserted prefixes. */ std::vector> prefixes_map; const std::string ALPHABET{"abcdefghijklmnopqrstuvwxyz"}; public: - search_near_03(const test_harness::test_args &args) : test(args) + search_near_03(const test_args &args) : test(args) { init_tracking(); } @@ -143,8 +143,8 @@ class search_near_03 : public test_harness::test { } void - populate(test_harness::database &database, test_harness::timestamp_manager *tsm, - test_harness::configuration *config, test_harness::workload_tracking *tracking) override final + populate(database &database, timestamp_manager *tsm, configuration *config, + workload_tracking *tracking) override final { uint64_t collection_count, key_count, key_size; std::vector workers; @@ -217,7 +217,7 @@ class search_near_03 : public test_harness::test { } void - insert_operation(test_harness::thread_context *tc) override final + insert_operation(thread_context *tc) override final { std::map cursors; std::string prefix_key; @@ -261,7 +261,7 @@ class search_near_03 : public test_harness::test { } void - read_operation(test_harness::thread_context *tc) override final + read_operation(thread_context *tc) override final { uint64_t key_count = 0; int ret = 0; diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/test_template.cpp b/src/third_party/wiredtiger/test/cppsuite/tests/test_template.cpp index 96e9ff29fdc..5ddf4826597 100644 --- a/src/third_party/wiredtiger/test/cppsuite/tests/test_template.cpp +++ b/src/third_party/wiredtiger/test/cppsuite/tests/test_template.cpp @@ -26,11 +26,14 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include "test_harness/test.h" +#include + +#include "src/common/api_const.h" +#include "src/main/test.h" namespace test_harness { /* Defines what data is written to the tracking table for use in custom validation. */ -class tracking_table_template : public test_harness::workload_tracking { +class tracking_table_template : public workload_tracking { public: tracking_table_template( @@ -54,9 +57,9 @@ class tracking_table_template : public test_harness::workload_tracking { * Class that defines operations that do nothing as an example. This shows how database operations * can be overridden and customized. */ -class test_template : public test_harness::test { +class test_template : public test { public: - test_template(const test_harness::test_args &args) : test(args) + test_template(const test_args &args) : test(args) { init_tracking(new tracking_table_template(_config->get_subconfig(WORKLOAD_TRACKING), _config->get_bool(COMPRESSION_ENABLED), *_timestamp_manager)); @@ -70,38 +73,37 @@ class test_template : public test_harness::test { } void - populate(test_harness::database &, test_harness::timestamp_manager *, - test_harness::configuration *, test_harness::workload_tracking *) override final + populate(database &, timestamp_manager *, configuration *, workload_tracking *) override final { std::cout << "populate: nothing done." << std::endl; } void - custom_operation(test_harness::thread_context *) override final + custom_operation(thread_context *) override final { std::cout << "custom_operation: nothing done." << std::endl; } void - insert_operation(test_harness::thread_context *) override final + insert_operation(thread_context *) override final { std::cout << "insert_operation: nothing done." << std::endl; } void - read_operation(test_harness::thread_context *) override final + read_operation(thread_context *) override final { std::cout << "read_operation: nothing done." << std::endl; } void - remove_operation(test_harness::thread_context *) override final + remove_operation(thread_context *) override final { std::cout << "remove_operation: nothing done." << std::endl; } void - update_operation(test_harness::thread_context *) override final + update_operation(thread_context *) override final { std::cout << "update_operation: nothing done." << std::endl; } diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index d431af14ddf..105abffe6d7 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -1030,6 +1030,8 @@ tasks: - func: "compile wiredtiger" vars: posix_configure_flags: -DENABLE_STRICT=1 -DHAVE_DIAGNOSTIC=1 -DENABLE_LZ4=1 -DENABLE_SNAPPY=1 -DENABLE_ZLIB=1 -DHAVE_UNITTEST=1 + # Different flags for windows based on what is supported and enabled by default. + windows_configure_flags: -DHAVE_DIAGNOSTIC=1 -DHAVE_UNITTEST=1 - command: shell.exec params: working_dir: "wiredtiger/cmake_build" @@ -4353,6 +4355,7 @@ buildvariants: - name: make-check-test - name: ".unit_test" - name: fops + - name: unittest-test - name: macos-1014 display_name: "OS X 10.14" diff --git a/src/third_party/wiredtiger/test/evergreen/build_windows.ps1 b/src/third_party/wiredtiger/test/evergreen/build_windows.ps1 index 001a93dbab4..56a68b0ab55 100644 --- a/src/third_party/wiredtiger/test/evergreen/build_windows.ps1 +++ b/src/third_party/wiredtiger/test/evergreen/build_windows.ps1 @@ -27,7 +27,8 @@ cd cmake_build # Configure build with CMake. if( $configure -eq $true) { - C:\cmake\bin\cmake --no-warn-unused-cli -DSWIG_DIR="C:\swigwin-3.0.2" -DSWIG_EXECUTABLE="C:\swigwin-3.0.2\swig.exe" -DCMAKE_BUILD_TYPE='None' -DENABLE_PYTHON=1 -DENABLE_STRICT=1 -DCMAKE_TOOLCHAIN_FILE='..\cmake\toolchains\cl.cmake' -G "Ninja" ..\. + # Note that ${args} are all the command line options that are not automatically parsed by the param function. + C:\cmake\bin\cmake --no-warn-unused-cli -DSWIG_DIR='C:\swigwin-3.0.2' -DSWIG_EXECUTABLE='C:\swigwin-3.0.2\swig.exe' -DCMAKE_BUILD_TYPE='None' -DENABLE_PYTHON=1 -DENABLE_STRICT=1 -DCMAKE_TOOLCHAIN_FILE='..\cmake\toolchains\cl.cmake' ${args} -G "Ninja" ..\. } # Execute Ninja build. diff --git a/src/third_party/wiredtiger/test/suite/test_cursor_bound01.py b/src/third_party/wiredtiger/test/suite/test_cursor_bound01.py index 7e806650f33..c0baf30ee22 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor_bound01.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor_bound01.py @@ -30,25 +30,34 @@ import wiredtiger, wttest from wtscenario import make_scenarios # test_cursor_bound01.py -# Basic cursor bound API validation +# Basic cursor bound API validation. class test_cursor_bound01(wttest.WiredTigerTestCase): file_name = 'test_cursor_bound01' types = [ - ('file', dict(uri='file:', use_index = False)), - ('table', dict(uri='table:', use_index = False)), - ('lsm', dict(uri='lsm:', use_index = False)), - ('index', dict(uri='table:', use_index = True)), + ('file', dict(uri='file:', use_index = False, use_colgroup = False)), + ('table', dict(uri='table:', use_index = False, use_colgroup = False)), + ('lsm', dict(uri='lsm:', use_index = False, use_colgroup = False)), + ('colgroup', dict(uri='table:', use_index = False, use_colgroup = False)), + #FIXME: Turn on once index cursor bound implementation is done. + #('index', dict(uri='table:', use_index = True)), ] - scenarios = make_scenarios(types) def test_bound_api(self): uri = self.uri + self.file_name create_params = 'value_format=S,key_format=i' - if self.use_index: + if self.use_index or self.use_colgroup: create_params += ",columns=(k,v)" + if self.use_colgroup: + create_params += ',colgroups=(g0)' self.session.create(uri, create_params) + # Add in column group. + if self.use_colgroup: + create_params = 'columns=(v),' + suburi = 'colgroup:table0:g0' + self.session.create(suburi, create_params) + cursor = None if self.use_index: # Test Index Cursors bound API support. @@ -69,7 +78,9 @@ class test_cursor_bound01(wttest.WiredTigerTestCase): '/Invalid argument/') # Check that bound configuration works properly. + cursor.set_key(0) cursor.bound("bound=lower") + cursor.set_key(10) cursor.bound("bound=upper") # Clear and inclusive configuration are not compatible with each other. diff --git a/src/third_party/wiredtiger/test/suite/test_cursor_bound02.py b/src/third_party/wiredtiger/test/suite/test_cursor_bound02.py new file mode 100644 index 00000000000..5e8ec36f422 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_cursor_bound02.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtscenario import make_scenarios + +# test_cursor_bound02.py +# Test that setting bounds of different key formats works in the cursor bound API. Make +# sure that WiredTiger complains when the upper and lower bounds overlap and that clearing the +# bounds through the bound API and reset calls work appriopately. +class test_cursor_bound02(wttest.WiredTigerTestCase): + file_name = 'test_cursor_bound02' + + types = [ + ('file', dict(uri='file:', use_colgroup=False)), + ('table', dict(uri='table:', use_colgroup=False)), + ('colgroup', dict(uri='table:', use_colgroup=True)) + ] + + key_format_values = [ + ('string', dict(key_format='S')), + ('var', dict(key_format='r')), + ('int', dict(key_format='i')), + ('bytes', dict(key_format='u')), + ('composite_string', dict(key_format='SSS')), + ('composite_int_string', dict(key_format='iS')), + ('composite_complex', dict(key_format='iSru')), + ] + + inclusive = [ + ('inclusive', dict(inclusive=True)), + ('no-inclusive', dict(inclusive=False)) + ] + scenarios = make_scenarios(types, key_format_values, inclusive) + + def gen_key(self, i): + tuple_key = [] + for key in self.key_format: + if key == 'S' or key == 'u': + tuple_key.append('key' + str(i)) + elif key == "r": + tuple_key.append(self.recno(i)) + elif key == "i": + tuple_key.append(i) + + if (len(self.key_format) == 1): + return tuple_key[0] + else: + return tuple(tuple_key) + + def gen_colgroup_create_param(self): + create_params = ",columns=(" + start = 0 + for _ in self.key_format: + create_params += "k{0},".format(str(start)) + start += 1 + create_params += "v),colgroups=(g0)" + return create_params + + def set_bounds(self, cursor, bound_config): + inclusive_config = ",inclusive=false" if self.inclusive == False else "" + return cursor.bound("bound={0}{1}".format(bound_config, inclusive_config)) + + def test_bound_api(self): + uri = self.uri + self.file_name + create_params = 'value_format=S,key_format={}'.format(self.key_format) + if self.use_colgroup: + create_params += self.gen_colgroup_create_param() + self.session.create(uri, create_params) + # Add in column group. + if self.use_colgroup: + create_params = 'columns=(v),' + suburi = 'colgroup:{0}:g0'.format(self.file_name) + self.session.create(suburi, create_params) + + cursor = self.session.open_cursor(uri) + + # Test bound API: Basic usage. + cursor.set_key(self.gen_key(40)) + self.assertEqual(self.set_bounds(cursor, "lower"), 0) + cursor.set_key(self.gen_key(90)) + self.assertEqual(self.set_bounds(cursor, "upper"), 0) + + # Test bound API: Upper bound < lower bound. + cursor.set_key(self.gen_key(30)) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.set_bounds(cursor, "upper"), '/Invalid argument/') + + # Test bound API: Lower bound > upper bound. + cursor.set_key(self.gen_key(95)) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.set_bounds(cursor, "lower"), '/Invalid argument/') + + # Test bound API: Test setting lower bound to 20, which would succeed setting upper + # bound to 30 + cursor.set_key(self.gen_key(20)) + self.assertEqual(self.set_bounds(cursor, "lower"), 0) + cursor.set_key(self.gen_key(30)) + self.assertEqual(self.set_bounds(cursor, "upper"), 0) + + # Test bound API: Test setting upper bound to 99, which would succeed setting lower + # bound to 90 + cursor.set_key(self.gen_key(99)) + self.assertEqual(self.set_bounds(cursor, "upper"), 0) + cursor.set_key(self.gen_key(90)) + self.assertEqual(self.set_bounds(cursor, "lower"), 0) + + # Test bound API: No key set. + cursor.reset() + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.set_bounds(cursor, "lower"), '/Invalid argument/') + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.set_bounds(cursor, "upper"), '/Invalid argument/') + + # Test bound API: Test that the key persists after lower bound call. + cursor.set_key(self.gen_key(30)) + cursor.set_value("30") + self.assertEqual(self.set_bounds(cursor, "lower"), 0) + cursor.insert() + + # Test bound API: Test that the key persists after upper bound call. + cursor.set_key(self.gen_key(90)) + cursor.set_value("90") + self.assertEqual(self.set_bounds(cursor, "upper"), 0) + cursor.insert() + + # Test bound API: that if lower bound is equal to the upper bound, that both bounds needs to + # have inclusive configured. + cursor.bound("action=clear") + cursor.set_key(self.gen_key(50)) + self.assertEqual(cursor.bound("bound=lower,inclusive=true"), 0) + self.assertEqual(cursor.bound("bound=upper,inclusive=true"), 0) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: cursor.bound("bound=lower,inclusive=false"), '/Invalid argument/') + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: cursor.bound("bound=upper,inclusive=false"), '/Invalid argument/') + + # Test bound API: Test that only setting one of the bound inclusive config to true, should + # fail too. + cursor.bound("action=clear") + self.assertEqual(cursor.bound("bound=lower,inclusive=false"), 0) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: cursor.bound("bound=upper,inclusive=false"), '/Invalid argument/') + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: cursor.bound("bound=upper,inclusive=true"), '/Invalid argument/') + + cursor.bound("action=clear") + self.assertEqual(cursor.bound("bound=upper,inclusive=false"), 0) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: cursor.bound("bound=lower,inclusive=false"), '/Invalid argument/') + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: cursor.bound("bound=lower,inclusive=true"), '/Invalid argument/') + + + def test_bound_api_reset(self): + uri = self.uri + self.file_name + create_params = 'value_format=S,key_format={}'.format(self.key_format) + if self.use_colgroup: + create_params += self.gen_colgroup_create_param() + self.session.create(uri, create_params) + # Add in column group. + if self.use_colgroup: + create_params = 'columns=(v),' + suburi = 'colgroup:{0}:g0'.format(self.file_name) + self.session.create(suburi, create_params) + cursor = self.session.open_cursor(uri) + + cursor.set_key(self.gen_key(30)) + self.assertEqual(self.set_bounds(cursor, "lower"), 0) + cursor.set_key(self.gen_key(90)) + self.assertEqual(self.set_bounds(cursor, "upper"), 0) + + # Test bound API: Test that cursor reset works on the lower bound. + cursor.set_key(self.gen_key(10)) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.set_bounds(cursor, "upper"), '/Invalid argument/') + cursor.reset() + cursor.set_key(self.gen_key(10)) + self.assertEqual(self.set_bounds(cursor, "upper"), 0) + + # Test bound API: Test that cursor reset works on the upper bound. + cursor.set_key(self.gen_key(99)) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.set_bounds(cursor, "lower"), '/Invalid argument/') + cursor.reset() + cursor.set_key(self.gen_key(99)) + self.assertEqual(self.set_bounds(cursor, "lower"), 0) + + # Test bound API: Test that cursor reset works the clearing bounds both ways. + cursor.set_key(self.gen_key(50)) + self.assertEqual(self.set_bounds(cursor, "lower"), 0) + cursor.reset() + cursor.set_key(self.gen_key(20)) + self.assertEqual(self.set_bounds(cursor, "lower"), 0) + cursor.set_key(self.gen_key(99)) + self.assertEqual(self.set_bounds(cursor, "upper"), 0) + + cursor.set_key(self.gen_key(55)) + self.assertEqual(self.set_bounds(cursor, "upper"), 0) + cursor.reset() + cursor.set_key(self.gen_key(90)) + self.assertEqual(self.set_bounds(cursor, "lower"), 0) + cursor.set_key(self.gen_key(99)) + self.assertEqual(self.set_bounds(cursor, "upper"), 0) + + # Test bound API: Make sure that a clear and reset works sequentially. + cursor.reset() + self.assertEqual(cursor.bound("action=clear"), 0) + + cursor.set_key(self.gen_key(30)) + self.assertEqual(self.set_bounds(cursor, "lower"), 0) + cursor.set_key(self.gen_key(90)) + self.assertEqual(self.set_bounds(cursor, "upper"), 0) + self.assertEqual(cursor.bound("action=clear"), 0) + cursor.reset() + + # Test bound API: Test that reset works after a reset. + cursor.set_key(self.gen_key(30)) + self.assertEqual(self.set_bounds(cursor, "lower"), 0) + cursor.set_key(self.gen_key(90)) + self.assertEqual(self.set_bounds(cursor, "upper"), 0) + cursor.reset() + cursor.reset() + + def test_bound_api_clear(self): + uri = self.uri + self.file_name + create_params = 'value_format=S,key_format={}'.format(self.key_format) + if self.use_colgroup: + create_params += self.gen_colgroup_create_param() + self.session.create(uri, create_params) + # Add in column group. + if self.use_colgroup: + create_params = 'columns=(v),' + suburi = 'colgroup:{0}:g0'.format(self.file_name) + self.session.create(suburi, create_params) + cursor = self.session.open_cursor(uri) + + cursor.set_key(self.gen_key(30)) + self.assertEqual(self.set_bounds(cursor, "lower"), 0) + cursor.set_key(self.gen_key(90)) + self.assertEqual(self.set_bounds(cursor, "upper"), 0) + + # Test bound API: Test that clearing the lower bound works. + cursor.set_key(self.gen_key(10)) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.set_bounds(cursor, "upper"), '/Invalid argument/') + self.assertEqual(cursor.bound("action=clear,bound=lower"), 0) + cursor.set_key(self.gen_key(10)) + self.assertEqual(self.set_bounds(cursor, "upper"), 0) + + # Test bound API: Test that clearing the upper bound works. + cursor.set_key(self.gen_key(99)) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.set_bounds(cursor, "lower"), '/Invalid argument/') + self.assertEqual(cursor.bound("action=clear,bound=upper"), 0) + cursor.set_key(self.gen_key(99)) + self.assertEqual(self.set_bounds(cursor, "lower"), 0) + + # Test bound API: Test that clearing both of the bounds works. + cursor.reset() + cursor.set_key(self.gen_key(50)) + self.assertEqual(self.set_bounds(cursor, "upper"), 0) + self.assertEqual(cursor.bound("action=clear"), 0) + cursor.set_key(self.gen_key(90)) + self.assertEqual(self.set_bounds(cursor, "lower"), 0) + cursor.set_key(self.gen_key(99)) + self.assertEqual(self.set_bounds(cursor, "upper"), 0) + + cursor.reset() + cursor.set_key(self.gen_key(50)) + self.assertEqual(self.set_bounds(cursor, "lower"), 0) + self.assertEqual(cursor.bound("action=clear"), 0) + cursor.set_key(self.gen_key(20)) + self.assertEqual(self.set_bounds(cursor, "lower"), 0) + cursor.set_key(self.gen_key(99)) + self.assertEqual(self.set_bounds(cursor, "upper"), 0) + + # Test bound API: Test that clear works after a clear. + self.assertEqual(cursor.bound("action=clear"), 0) + self.assertEqual(cursor.bound("action=clear"), 0) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/unittest/tests/test_reconciliation_tracking.cpp b/src/third_party/wiredtiger/test/unittest/tests/test_reconciliation_tracking.cpp index 59dfebc19ba..a3461ccd814 100644 --- a/src/third_party/wiredtiger/test/unittest/tests/test_reconciliation_tracking.cpp +++ b/src/third_party/wiredtiger/test/unittest/tests/test_reconciliation_tracking.cpp @@ -37,7 +37,7 @@ TEST_CASE("Reconciliation tracking: ovfl_discard_verbose", "[reconciliation]") SECTION("handle null page and tag") { - REQUIRE(__ut_ovfl_discard_verbose(session, nullptr, nullptr, nullptr) == 0); + REQUIRE(__ut_ovfl_discard_verbose(session, nullptr, nullptr, nullptr) == EINVAL); } } diff --git a/src/third_party/wiredtiger/test/unittest/tests/wrappers/connection_wrapper.cpp b/src/third_party/wiredtiger/test/unittest/tests/wrappers/connection_wrapper.cpp index 81d838d7f3b..0bb79182546 100644 --- a/src/third_party/wiredtiger/test/unittest/tests/wrappers/connection_wrapper.cpp +++ b/src/third_party/wiredtiger/test/unittest/tests/wrappers/connection_wrapper.cpp @@ -16,7 +16,20 @@ ConnectionWrapper::ConnectionWrapper(const std::string &db_home) : _conn_impl(nullptr), _conn(nullptr), _db_home(db_home) { - utils::throwIfNonZero(mkdir(_db_home.c_str(), 0700)); + struct stat sb; + /* + * Check if the DB Home exists and is a directory, without this the mkdir can fail on some + * platforms (win). + */ + if (stat(_db_home.c_str(), &sb) == 0) { + if (!S_ISDIR(sb.st_mode)) { + std::string errorMessage("Path exists and is not a directory: " + db_home); + throw std::runtime_error(errorMessage); + } + /* We are happy that it is an existing directory. */ + } else { + utils::throwIfNonZero(mkdir(_db_home.c_str(), 0700)); + } utils::throwIfNonZero(wiredtiger_open(_db_home.c_str(), nullptr, "create", &_conn)); } diff --git a/src/third_party/wiredtiger/test/unittest/tests/wrappers/connection_wrapper.h b/src/third_party/wiredtiger/test/unittest/tests/wrappers/connection_wrapper.h index 1093e4465d9..c064639a903 100644 --- a/src/third_party/wiredtiger/test/unittest/tests/wrappers/connection_wrapper.h +++ b/src/third_party/wiredtiger/test/unittest/tests/wrappers/connection_wrapper.h @@ -13,6 +13,10 @@ #include #include "wt_internal.h" +#ifdef _WIN32 +#include "windows_shim.h" +#endif + /* * Prefer a "real" class over a mock class when you need a fully fleshed-out connection or session. * There's a speed cost to this, since it will write a bunch of files to disk during the test, which diff --git a/src/third_party/wiredtiger/test/windows/windows_shim.h b/src/third_party/wiredtiger/test/windows/windows_shim.h index 15bd2ac19dc..06b60360c69 100644 --- a/src/third_party/wiredtiger/test/windows/windows_shim.h +++ b/src/third_party/wiredtiger/test/windows/windows_shim.h @@ -44,6 +44,7 @@ * Emulate */ #define mkdir(path, mode) _mkdir(path) +#define S_ISDIR(mode) ((mode & _S_IFDIR) == _S_IFDIR) /* * Emulate -- cgit v1.2.1