diff options
author | Chenhao Qu <chenhao.qu@mongodb.com> | 2021-07-14 08:23:04 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-07-14 08:48:19 +0000 |
commit | b836677cfbcf1fd8678b8bb123336a50c2525b71 (patch) | |
tree | a105fc64492a9b56afc74e9d8ef0c966a64e231b | |
parent | d2efb2cbc1fccd9ea8ad9cff8a6e815b2cc714af (diff) | |
download | mongo-b836677cfbcf1fd8678b8bb123336a50c2525b71.tar.gz |
Import wiredtiger: c3b5de971512297ebc97d304d548007a1b615789 from branch mongodb-master
ref: b5de6d5d07..c3b5de9715
for: 5.1.0
WT-7836 Fixing a number of small issues in the cppsuite test framework
10 files changed, 101 insertions, 73 deletions
diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 4b5ec0689b0..8a2a3443f02 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-master", - "commit": "b5de6d5d072ecc980676f998e8817b2b2ff10cbd" + "commit": "c3b5de971512297ebc97d304d548007a1b615789" } diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_types.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_types.cxx index 9b70228e8b0..599a8214be2 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_types.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_types.cxx @@ -113,8 +113,8 @@ scoped_session::scoped_session(scoped_session &&other) } /* - * Implement move assignment by move constructing a temporary and swapping its internals with - * the current session. This means that the currently held WT_SESSION will get destroyed as the + * Implement move assignment by move constructing a temporary and swapping its internals with the + * current session. This means that the currently held WT_SESSION will get destroyed as the * temporary falls out of the scope and we will steal the one that we're move assigning from. */ scoped_session & diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.cxx index 6fbf130b4da..9c6e021c3ee 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.cxx @@ -29,6 +29,7 @@ #include <cmath> #include <map> +#include "../connection_manager.h" #include "../thread_manager.h" #include "../util/api_const.h" #include "database_operation.h" @@ -50,10 +51,10 @@ populate_worker(thread_context *tc) scoped_cursor cursor = tc->session.open_scoped_cursor(coll.name.c_str()); for (uint64_t i = 0; i < tc->key_count; ++i) { /* Start a txn. */ - tc->transaction.begin(tc->session.get(), ""); + tc->transaction.begin(); if (!tc->insert(cursor, coll.id, i)) testutil_die(-1, "Got a rollback in populate, this is currently not handled."); - tc->transaction.commit(tc->session.get(), ""); + tc->transaction.commit(); } } logger::log_msg(LOG_TRACE, "Populate: thread {" + std::to_string(tc->id) + "} finished"); @@ -97,8 +98,8 @@ database_operation::populate( * here. */ for (int64_t i = 0; i < thread_count; ++i) { - thread_context *tc = - new thread_context(i, thread_type::INSERT, config, tsm, tracking, database); + thread_context *tc = new thread_context(i, thread_type::INSERT, config, + connection_manager::instance().create_session(), tsm, tracking, database); workers.push_back(tc); tm.add_thread(populate_worker, tc); } @@ -144,31 +145,37 @@ database_operation::insert_operation(thread_context *tc) uint64_t start_key = ccv[counter].coll.get_key_count(); uint64_t added_count = 0; bool committed = true; - tc->transaction.begin(tc->session.get(), ""); + tc->transaction.begin(); + + /* Collection cursor. */ + auto &cc = ccv[counter]; while (tc->transaction.active() && tc->running()) { /* Insert a key value pair. */ - if (!tc->insert(ccv[counter].cursor, ccv[counter].coll.id, start_key + added_count)) { + if (!tc->insert(cc.cursor, cc.coll.id, start_key + added_count)) { committed = false; break; } added_count++; - tc->transaction.try_commit(tc->session.get(), ""); + tc->transaction.try_commit(); /* Sleep the duration defined by the op_rate. */ tc->sleep(); } + /* Reset our cursor to avoid pinning content. */ + testutil_check(cc.cursor->reset(cc.cursor.get())); + /* * We need to inform the database model that we've added these keys as some other thread may * rely on the key_count data. Only do so if we successfully committed. */ if (committed) - ccv[counter].coll.increase_key_count(added_count); + cc.coll.increase_key_count(added_count); counter++; if (counter == collections_per_thread) counter = 0; } /* Make sure the last transaction is rolled back now the work is finished. */ if (tc->transaction.active()) - tc->transaction.rollback(tc->session.get(), ""); + tc->transaction.rollback(); } void @@ -190,7 +197,7 @@ database_operation::read_operation(thread_context *tc) } else cursor = it->second.get(); - tc->transaction.begin(tc->session.get(), ""); + tc->transaction.begin(); while (tc->transaction.active() && tc->running()) { ret = cursor->next(cursor); /* If we get not found here we walked off the end. */ @@ -199,13 +206,15 @@ database_operation::read_operation(thread_context *tc) } else if (ret != 0) testutil_die(ret, "cursor->next() failed"); tc->transaction.add_op(); - tc->transaction.try_rollback(tc->session.get(), ""); + tc->transaction.try_rollback(); tc->sleep(); } + /* Reset our cursor to avoid pinning content. */ + testutil_check(cursor->reset(cursor)); } /* Make sure the last transaction is rolled back now the work is finished. */ if (tc->transaction.active()) - tc->transaction.rollback(tc->session.get(), ""); + tc->transaction.rollback(); } void @@ -215,7 +224,6 @@ database_operation::update_operation(thread_context *tc) LOG_INFO, type_string(tc->type) + " thread {" + std::to_string(tc->id) + "} commencing."); /* Cursor map. */ std::map<uint64_t, scoped_cursor> cursors; - uint64_t collection_id = 0; /* * Loop while the test is running. @@ -241,7 +249,7 @@ database_operation::update_operation(thread_context *tc) } /* Start a transaction if possible. */ - tc->transaction.try_begin(tc->session.get(), ""); + tc->transaction.try_begin(); /* Get the cursor associated with the collection. */ scoped_cursor &cursor = cursors[coll.id]; @@ -249,15 +257,21 @@ database_operation::update_operation(thread_context *tc) /* Choose a random key to update. */ uint64_t key_id = random_generator::instance().generate_integer<uint64_t>(0, coll.get_key_count() - 1); - if (!tc->update(cursor, collection_id, tc->key_to_string(key_id))) + bool successful_update = tc->update(cursor, coll.id, tc->key_to_string(key_id)); + + /* Reset our cursor to avoid pinning content. */ + testutil_check(cursor->reset(cursor.get())); + + /* We received a rollback in update. */ + if (!successful_update) continue; /* Commit the current transaction if we're able to. */ - tc->transaction.try_commit(tc->session.get(), ""); + tc->transaction.try_commit(); } /* Make sure the last operation is committed now the work is finished. */ if (tc->transaction.active()) - tc->transaction.commit(tc->session.get(), ""); + tc->transaction.commit(); } } // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.cxx index 7bb56f2744a..ea425028ade 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.cxx @@ -26,7 +26,6 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include "../connection_manager.h" #include "../core/configuration.h" #include "../timestamp_manager.h" #include "../util/api_const.h" @@ -37,8 +36,8 @@ namespace test_harness { /* transaction_context class implementation */ transaction_context::transaction_context( - configuration *config, timestamp_manager *timestamp_manager) - : _timestamp_manager(timestamp_manager) + configuration *config, timestamp_manager *timestamp_manager, WT_SESSION *session) + : _timestamp_manager(timestamp_manager), _session(session) { /* Use optional here as our populate threads don't define this configuration. */ configuration *transaction_config = config->get_optional_subconfig(OPS_PER_TRANSACTION); @@ -62,10 +61,11 @@ transaction_context::add_op() } void -transaction_context::begin(WT_SESSION *session, const std::string &config) +transaction_context::begin(const std::string &config) { testutil_assert(!_in_txn); - testutil_check(session->begin_transaction(session, config.empty() ? nullptr : config.c_str())); + testutil_check( + _session->begin_transaction(_session, config.empty() ? nullptr : config.c_str())); /* This randomizes the number of operations to be executed in one transaction. */ _target_op_count = random_generator::instance().generate_integer<int64_t>(_min_op_count, _max_op_count); @@ -74,53 +74,54 @@ transaction_context::begin(WT_SESSION *session, const std::string &config) } void -transaction_context::try_begin(WT_SESSION *session, const std::string &config) +transaction_context::try_begin(const std::string &config) { if (!_in_txn) - begin(session, config); + begin(config); } void -transaction_context::commit(WT_SESSION *session, const std::string &config) +transaction_context::commit(const std::string &config) { testutil_assert(_in_txn); - testutil_check(session->commit_transaction(session, config.empty() ? nullptr : config.c_str())); + testutil_check( + _session->commit_transaction(_session, config.empty() ? nullptr : config.c_str())); _op_count = 0; _in_txn = false; } void -transaction_context::try_commit(WT_SESSION *session, const std::string &config) +transaction_context::try_commit(const std::string &config) { if (can_commit_rollback()) - commit(session, config); + commit(config); } void -transaction_context::rollback(WT_SESSION *session, const std::string &config) +transaction_context::rollback(const std::string &config) { testutil_assert(_in_txn); testutil_check( - session->rollback_transaction(session, config.empty() ? nullptr : config.c_str())); + _session->rollback_transaction(_session, config.empty() ? nullptr : config.c_str())); _op_count = 0; _in_txn = false; } void -transaction_context::try_rollback(WT_SESSION *session, const std::string &config) +transaction_context::try_rollback(const std::string &config) { if (can_commit_rollback()) - rollback(session, config); + rollback(config); } void -transaction_context::set_commit_timestamp(WT_SESSION *session, wt_timestamp_t ts) +transaction_context::set_commit_timestamp(wt_timestamp_t ts) { /* We don't want to set zero timestamps on transactions if we're not using timestamps. */ if (!_timestamp_manager->enabled()) return; std::string config = std::string(COMMIT_TS) + "=" + timestamp_manager::decimal_to_hex(ts); - testutil_check(session->timestamp_transaction(session, config.c_str())); + testutil_check(_session->timestamp_transaction(_session, config.c_str())); } bool @@ -131,9 +132,11 @@ transaction_context::can_commit_rollback() /* thread_context class implementation */ thread_context::thread_context(uint64_t id, thread_type type, configuration *config, - timestamp_manager *timestamp_manager, workload_tracking *tracking, database &dbase) + scoped_session &&created_session, timestamp_manager *timestamp_manager, + workload_tracking *tracking, database &dbase) : id(id), type(type), db(dbase), tsm(timestamp_manager), tracking(tracking), - transaction(transaction_context(config, timestamp_manager)), + session(std::move(created_session)), + transaction(transaction_context(config, timestamp_manager, session.get())), /* These won't exist for certain threads which is why we use optional here. */ collection_count(config->get_optional_int(COLLECTION_COUNT, 1)), key_count(config->get_optional_int(KEY_COUNT_PER_COLLECTION, 1)), @@ -141,9 +144,7 @@ thread_context::thread_context(uint64_t id, thread_type type, configuration *con value_size(config->get_optional_int(VALUE_SIZE, 1)), thread_count(config->get_int(THREAD_COUNT)) { - session = connection_manager::instance().create_session(); _throttle = throttle(config); - if (tracking->enabled()) op_track_cursor = session.open_scoped_cursor(tracking->get_operation_table_name().c_str()); @@ -176,14 +177,14 @@ thread_context::update(scoped_cursor &cursor, uint64_t collection_id, const std: testutil_assert(tracking != nullptr); testutil_assert(cursor.get() != nullptr); - transaction.set_commit_timestamp(session.get(), ts); + transaction.set_commit_timestamp(ts); value = random_generator::instance().generate_string(value_size); cursor->set_key(cursor.get(), key.c_str()); cursor->set_value(cursor.get(), value.c_str()); ret = cursor->update(cursor.get()); if (ret != 0) { if (ret == WT_ROLLBACK) { - transaction.rollback(session.get(), ""); + transaction.rollback(); return (false); } else testutil_die(ret, "unhandled error while trying to update a key"); @@ -192,7 +193,7 @@ thread_context::update(scoped_cursor &cursor, uint64_t collection_id, const std: tracking_operation::INSERT, collection_id, key.c_str(), value.c_str(), ts, op_track_cursor); if (ret != 0) { if (ret == WT_ROLLBACK) { - transaction.rollback(session.get(), ""); + transaction.rollback(); return (false); } else testutil_die( @@ -215,7 +216,7 @@ thread_context::insert(scoped_cursor &cursor, uint64_t collection_id, uint64_t k * it will return a value for the tracking table. */ wt_timestamp_t ts = tsm->get_next_ts(); - transaction.set_commit_timestamp(session.get(), ts); + transaction.set_commit_timestamp(ts); key = key_to_string(key_id); value = random_generator::instance().generate_string(value_size); @@ -225,7 +226,7 @@ thread_context::insert(scoped_cursor &cursor, uint64_t collection_id, uint64_t k ret = cursor->insert(cursor.get()); if (ret != 0) { if (ret == WT_ROLLBACK) { - transaction.rollback(session.get(), ""); + transaction.rollback(); return (false); } else testutil_die(ret, "unhandled error while trying to insert a key"); @@ -234,7 +235,7 @@ thread_context::insert(scoped_cursor &cursor, uint64_t collection_id, uint64_t k tracking_operation::INSERT, collection_id, key.c_str(), value.c_str(), ts, op_track_cursor); if (ret != 0) { if (ret == WT_ROLLBACK) { - transaction.rollback(session.get(), ""); + transaction.rollback(); return (false); } else testutil_die( diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h index 1d0ec474104..77b8d1ef7a3 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h @@ -63,21 +63,22 @@ type_string(thread_type type) class transaction_context { public: - explicit transaction_context(configuration *config, timestamp_manager *timestamp_manager); + explicit transaction_context( + configuration *config, timestamp_manager *timestamp_manager, WT_SESSION *session); bool active() const; void add_op(); - void begin(WT_SESSION *session, const std::string &config); + void begin(const std::string &config = ""); /* Begin a transaction if we are not currently in one. */ - void try_begin(WT_SESSION *session, const std::string &config); - void commit(WT_SESSION *session, const std::string &config); + void try_begin(const std::string &config = ""); + void commit(const std::string &config = ""); /* Attempt to commit the transaction given the requirements are met. */ - void try_commit(WT_SESSION *session, const std::string &config); - void rollback(WT_SESSION *session, const std::string &config); + void try_commit(const std::string &config = ""); + void rollback(const std::string &config = ""); /* Attempt to rollback the transaction given the requirements are met. */ - void try_rollback(WT_SESSION *session, const std::string &config); + void try_rollback(const std::string &config = ""); /* Set a commit timestamp. */ - void set_commit_timestamp(WT_SESSION *session, wt_timestamp_t ts); + void set_commit_timestamp(wt_timestamp_t ts); private: bool can_commit_rollback(); @@ -99,6 +100,7 @@ class transaction_context { int64_t _target_op_count = 0; bool _in_txn = false; + WT_SESSION *_session = nullptr; timestamp_manager *_timestamp_manager = nullptr; }; @@ -106,7 +108,8 @@ class transaction_context { class thread_context { public: thread_context(uint64_t id, thread_type type, configuration *config, - timestamp_manager *timestamp_manager, workload_tracking *tracking, database &db); + scoped_session &&created_session, timestamp_manager *timestamp_manager, + workload_tracking *tracking, database &dbase); virtual ~thread_context() = default; diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_tracking.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_tracking.cxx index 97def79e8a2..48a7b09f555 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_tracking.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_tracking.cxx @@ -99,9 +99,6 @@ workload_tracking::do_work() /* Take a copy of the oldest so that we sweep with a consistent timestamp. */ oldest_ts = _tsm.get_oldest_ts(); - /* If we have a position, give it up. */ - testutil_check(_sweep_cursor->reset(_sweep_cursor.get())); - while ((ret = _sweep_cursor->prev(_sweep_cursor.get())) == 0) { testutil_check(_sweep_cursor->get_key(_sweep_cursor.get(), &collection_id, &key, &ts)); testutil_check(_sweep_cursor->get_value(_sweep_cursor.get(), &op_type, &value)); @@ -143,6 +140,9 @@ workload_tracking::do_work() if (ret != WT_NOTFOUND) testutil_die(LOG_ERROR, "Tracking table sweep failed: cursor->next() returned an unexpected error %d.", ret); + + /* If we have a position, give it up. */ + testutil_check(_sweep_cursor->reset(_sweep_cursor.get())); } void diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_validation.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_validation.cxx index fe1e16aa9c0..5ebd17b3e5e 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_validation.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_validation.cxx @@ -136,6 +136,13 @@ workload_validation::validate(const std::string &operation_table_name, if (ret != WT_NOTFOUND) testutil_die( LOG_ERROR, "Validation failed: cursor->next() return an unexpected error %d.", ret); + + /* + * We still need to validate the last collection. But we can also end up here if there aren't + * any collections, check for that. + */ + if (known_collection_ids.size() != 0) + verify_collection(session, current_collection_id, current_collection_records); } void diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_generator.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_generator.cxx index 200ab291670..32676a59a6f 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_generator.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_generator.cxx @@ -29,6 +29,7 @@ #include <atomic> #include <map> +#include "connection_manager.h" #include "core/configuration.h" #include "core/throttle.h" #include "util/api_const.h" @@ -99,12 +100,14 @@ workload_generator::run() /* Generate threads to execute read operations on the collections. */ for (auto &it : operation_configs) { - logger::log_msg(LOG_INFO, - "Workload_generator: Creating " + std::to_string(it.thread_count) + " " + - type_string(it.type) + " threads."); + if (it.thread_count != 0) + logger::log_msg(LOG_INFO, + "Workload_generator: Creating " + std::to_string(it.thread_count) + " " + + type_string(it.type) + " threads."); for (size_t i = 0; i < it.thread_count && _running; ++i) { - thread_context *tc = new thread_context( - thread_id++, it.type, it.config, _timestamp_manager, _tracking, _database); + thread_context *tc = new thread_context(thread_id++, it.type, it.config, + connection_manager::instance().create_session(), _timestamp_manager, _tracking, + _database); _workers.push_back(tc); _thread_manager.add_thread(it.get_func(_database_operation), tc); } diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/example_test.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/example_test.cxx index ba8be05bca2..a226fa33ff3 100644 --- a/src/third_party/wiredtiger/test/cppsuite/tests/example_test.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/tests/example_test.cxx @@ -29,8 +29,8 @@ #include "test_harness/test.h" /* - * Class that defines operations that do nothing as an example. - * This shows how database operations can be overriden and customized. + * Class that defines operations that do nothing as an example. This shows how database operations + * can be overriden and customized. */ class example_test : public test_harness::test { public: @@ -38,8 +38,8 @@ class example_test : public test_harness::test { void populate(test_harness::database &database, test_harness::timestamp_manager *_timestamp_manager, - test_harness::configuration *_config, test_harness::workload_tracking *tracking) - override final + test_harness::configuration *_config, + test_harness::workload_tracking *tracking) override final { std::cout << "populate: nothing done." << std::endl; } diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/hs_cleanup.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/hs_cleanup.cxx index a42209a6eba..027e63dcf77 100644 --- a/src/third_party/wiredtiger/test/cppsuite/tests/hs_cleanup.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/tests/hs_cleanup.cxx @@ -49,12 +49,12 @@ class hs_cleanup : public test { { WT_DECL_RET; const char *key_tmp; - scoped_session session = connection_manager::instance().create_session(); + collection &coll = tc->db.get_collection(tc->id); /* In this test each thread gets a single collection. */ testutil_assert(tc->db.get_collection_count() == tc->thread_count); - scoped_cursor cursor = session.open_scoped_cursor(coll.name.c_str()); + scoped_cursor cursor = tc->session.open_scoped_cursor(coll.name.c_str()); /* We don't know the keyrange we're operating over here so we can't be much smarter here. */ while (tc->running()) { @@ -70,7 +70,7 @@ class hs_cleanup : public test { testutil_check(cursor->get_key(cursor.get(), &key_tmp)); /* Start a transaction if possible. */ - tc->transaction.try_begin(tc->session.get(), ""); + tc->transaction.try_begin(); /* * The retrieved key needs to be passed inside the update function. However, the update @@ -81,10 +81,10 @@ class hs_cleanup : public test { continue; /* Commit our transaction. */ - tc->transaction.try_commit(tc->session.get(), ""); + tc->transaction.try_commit(); } /* Ensure our last transaction is resolved. */ if (tc->transaction.active()) - tc->transaction.commit(tc->session.get(), ""); + tc->transaction.commit(); } }; |