diff options
Diffstat (limited to 'src/mongo/db')
-rw-r--r-- | src/mongo/db/repl/SConscript | 1 | ||||
-rw-r--r-- | src/mongo/db/repl/database_cloner.cpp | 70 | ||||
-rw-r--r-- | src/mongo/db/repl/database_cloner.h | 27 | ||||
-rw-r--r-- | src/mongo/db/repl/database_cloner_test.cpp | 151 |
4 files changed, 232 insertions, 17 deletions
diff --git a/src/mongo/db/repl/SConscript b/src/mongo/db/repl/SConscript index 616fbd35e32..1736bcc2c0c 100644 --- a/src/mongo/db/repl/SConscript +++ b/src/mongo/db/repl/SConscript @@ -869,6 +869,7 @@ env.CppUnitTest( 'base_cloner_test_fixture', '$BUILD_DIR/mongo/db/auth/authorization_manager_mock_init', '$BUILD_DIR/mongo/db/service_context_noop_init', + '$BUILD_DIR/mongo/unittest/task_executor_proxy', ], ) diff --git a/src/mongo/db/repl/database_cloner.cpp b/src/mongo/db/repl/database_cloner.cpp index 224d6d15a08..ba3f52ed048 100644 --- a/src/mongo/db/repl/database_cloner.cpp +++ b/src/mongo/db/repl/database_cloner.cpp @@ -152,7 +152,7 @@ std::string DatabaseCloner::_getDiagnosticString_inlock() const { output << " source: " << _source.toString(); output << " database: " << _dbname; output << " listCollections filter" << _listCollectionsFilter; - output << " active: " << _active; + output << " active: " << _isActive_inlock(); output << " collection info objects (empty if listCollections is in progress): " << _collectionInfos.size(); return output; @@ -160,14 +160,26 @@ std::string DatabaseCloner::_getDiagnosticString_inlock() const { bool DatabaseCloner::isActive() const { LockGuard lk(_mutex); - return _active; + return _isActive_inlock(); +} + +bool DatabaseCloner::_isActive_inlock() const { + return State::kRunning == _state || State::kShuttingDown == _state; } Status DatabaseCloner::startup() noexcept { LockGuard lk(_mutex); - if (_active) { - return Status(ErrorCodes::IllegalOperation, "database cloner already started"); + switch (_state) { + case State::kPreStart: + _state = State::kRunning; + break; + case State::kRunning: + return Status(ErrorCodes::InternalError, "database cloner already started"); + case State::kShuttingDown: + return Status(ErrorCodes::ShutdownInProgress, "database cloner shutting down"); + case State::kComplete: + return Status(ErrorCodes::ShutdownInProgress, "database cloner completed"); } _stats.start = _executor->now(); @@ -176,25 +188,31 @@ Status DatabaseCloner::startup() noexcept { if (!scheduleResult.isOK()) { error() << "Error scheduling listCollections for database: " << _dbname << ", error:" << scheduleResult; + _state = State::kComplete; return scheduleResult; } - _active = true; - return Status::OK(); } void DatabaseCloner::shutdown() { - { - LockGuard lk(_mutex); - - if (!_active) { + stdx::lock_guard<stdx::mutex> lock(_mutex); + switch (_state) { + case State::kPreStart: + // Transition directly from PreStart to Complete if not started yet. + _state = State::kComplete; return; - } + case State::kRunning: + _state = State::kShuttingDown; + break; + case State::kShuttingDown: + case State::kComplete: + // Nothing to do if we are already in ShuttingDown or Complete state. + return; + } - for (auto&& collectionCloner : _collectionCloners) { - collectionCloner.shutdown(); - } + for (auto&& collectionCloner : _collectionCloners) { + collectionCloner.shutdown(); } _listCollectionsFetcher.shutdown(); @@ -211,7 +229,7 @@ DatabaseCloner::Stats DatabaseCloner::getStats() const { void DatabaseCloner::join() { UniqueLock lk(_mutex); - _condition.wait(lk, [this]() { return !_active; }); + _condition.wait(lk, [this]() { return !_isActive_inlock(); }); } void DatabaseCloner::setScheduleDbWorkFn_forTest(const CollectionCloner::ScheduleDbWorkFn& work) { @@ -225,6 +243,11 @@ void DatabaseCloner::setStartCollectionClonerFn( _startCollectionCloner = startCollectionCloner; } +DatabaseCloner::State DatabaseCloner::getState_forTest() const { + stdx::lock_guard<stdx::mutex> lk(_mutex); + return _state; +} + void DatabaseCloner::_listCollectionsCallback(const StatusWith<Fetcher::QueryResponse>& result, Fetcher::NextAction* nextAction, BSONObjBuilder* getMoreBob) { @@ -413,7 +436,8 @@ void DatabaseCloner::_collectionClonerCallback(const Status& status, const Names void DatabaseCloner::_finishCallback(const Status& status) { _onCompletion(status); LockGuard lk(_mutex); - _active = false; + invariant(_state != State::kComplete); + _state = State::kComplete; _condition.notify_all(); _stats.end = _executor->now(); LOG(1) << " database: " << _dbname << ", stats: " << _stats.toString(); @@ -461,5 +485,19 @@ void DatabaseCloner::Stats::append(BSONObjBuilder* builder) const { } } +std::ostream& operator<<(std::ostream& os, const DatabaseCloner::State& state) { + switch (state) { + case DatabaseCloner::State::kPreStart: + return os << "PreStart"; + case DatabaseCloner::State::kRunning: + return os << "Running"; + case DatabaseCloner::State::kShuttingDown: + return os << "ShuttingDown"; + case DatabaseCloner::State::kComplete: + return os << "Complete"; + } + MONGO_UNREACHABLE; +} + } // namespace repl } // namespace mongo diff --git a/src/mongo/db/repl/database_cloner.h b/src/mongo/db/repl/database_cloner.h index 4acfba304e8..97a4c0cbc4c 100644 --- a/src/mongo/db/repl/database_cloner.h +++ b/src/mongo/db/repl/database_cloner.h @@ -28,6 +28,7 @@ #pragma once +#include <iosfwd> #include <list> #include <string> #include <vector> @@ -161,7 +162,23 @@ public: */ void setStartCollectionClonerFn(const StartCollectionClonerFn& startCollectionCloner); + // State transitions: + // PreStart --> Running --> ShuttingDown --> Complete + // It is possible to skip intermediate states. For example, + // Calling shutdown() when the cloner has not started will transition from PreStart directly + // to Complete. + // This enum class is made public for testing. + enum class State { kPreStart, kRunning, kShuttingDown, kComplete }; + + /** + * Returns current database cloner state. + * For testing only. + */ + State getState_forTest() const; + private: + bool _isActive_inlock() const; + /** * Read collection names and options from listCollections result. */ @@ -209,7 +226,6 @@ private: CollectionCallbackFn _collectionWork; // (R) Invoked once for every successfully started collection cloner. CallbackFn _onCompletion; // (R) Invoked once when cloning completes or fails. - bool _active = false; // _active is true when database cloner is started. Fetcher _listCollectionsFetcher; // (R) Fetcher instance for running listCollections command. // Collection info objects returned from listCollections. // Format of each document: @@ -227,7 +243,16 @@ private: _scheduleDbWorkFn; // (RT) Function for scheduling database work using the executor. StartCollectionClonerFn _startCollectionCloner; // (RT) Stats _stats; // (M) Stats about what this instance did. + + // Current database cloner state. See comments for State enum class for details. + State _state = State::kPreStart; // (M) }; +/** + * Insertion operator for DatabaseCloner::State. Formats database cloner state for output stream. + * For testing only. + */ +std::ostream& operator<<(std::ostream& os, const DatabaseCloner::State& state); + } // namespace repl } // namespace mongo diff --git a/src/mongo/db/repl/database_cloner_test.cpp b/src/mongo/db/repl/database_cloner_test.cpp index 5eeca00cfcd..f5fa63181e4 100644 --- a/src/mongo/db/repl/database_cloner_test.cpp +++ b/src/mongo/db/repl/database_cloner_test.cpp @@ -37,6 +37,7 @@ #include "mongo/db/repl/base_cloner_test_fixture.h" #include "mongo/db/repl/database_cloner.h" #include "mongo/db/repl/storage_interface.h" +#include "mongo/unittest/task_executor_proxy.h" #include "mongo/unittest/unittest.h" #include "mongo/util/mongoutils/str.h" @@ -187,8 +188,61 @@ TEST_F(DatabaseClonerTest, ClonerLifeCycle) { testLifeCycle(); } +TEST_F(DatabaseClonerTest, DatabaseClonerTransitionsToCompleteIfShutdownBeforeStartup) { + ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest()); + + _databaseCloner->shutdown(); + ASSERT_EQUALS(DatabaseCloner::State::kComplete, _databaseCloner->getState_forTest()); + ASSERT_EQUALS(ErrorCodes::ShutdownInProgress, _databaseCloner->startup()); +} + +class TaskExecutorWithFailureInScheduleRemoteCommand : public unittest::TaskExecutorProxy { +public: + using ShouldFailRequestFn = stdx::function<bool(const executor::RemoteCommandRequest&)>; + + TaskExecutorWithFailureInScheduleRemoteCommand(executor::TaskExecutor* executor, + ShouldFailRequestFn shouldFailRequest) + : unittest::TaskExecutorProxy(executor), _shouldFailRequest(shouldFailRequest) {} + + StatusWith<CallbackHandle> scheduleRemoteCommand(const executor::RemoteCommandRequest& request, + const RemoteCommandCallbackFn& cb) override { + if (_shouldFailRequest(request)) { + return Status(ErrorCodes::OperationFailed, "failed to schedule remote command"); + } + return getExecutor()->scheduleRemoteCommand(request, cb); + } + +private: + ShouldFailRequestFn _shouldFailRequest; +}; + +TEST_F(DatabaseClonerTest, + DatabaseClonerReturnsScheduleErrorOnFailingToScheduleListCollectionsCommand) { + TaskExecutorWithFailureInScheduleRemoteCommand executorProxy( + &getExecutor(), [](const executor::RemoteCommandRequest& request) { + return str::equals("listCollections", request.cmdObj.firstElementFieldName()); + }); + + DatabaseCloner databaseCloner(&executorProxy, + dbWorkThreadPool.get(), + target, + dbname, + BSONObj(), + DatabaseCloner::ListCollectionsPredicateFn(), + storageInterface.get(), + [](const Status&, const NamespaceString&) {}, + [](const Status&) {}); + ASSERT_EQUALS(DatabaseCloner::State::kPreStart, databaseCloner.getState_forTest()); + + ASSERT_EQUALS(ErrorCodes::OperationFailed, databaseCloner.startup()); + ASSERT_EQUALS(DatabaseCloner::State::kComplete, databaseCloner.getState_forTest()); +} + TEST_F(DatabaseClonerTest, FirstRemoteCommandWithoutFilter) { + ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest()); + ASSERT_OK(_databaseCloner->startup()); + ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest()); auto net = getNet(); executor::NetworkInterfaceMock::InNetworkGuard guard(net); @@ -222,7 +276,10 @@ TEST_F(DatabaseClonerTest, FirstRemoteCommandWithFilter) { stdx::placeholders::_1, stdx::placeholders::_2), stdx::bind(&DatabaseClonerTest::setStatus, this, stdx::placeholders::_1))); + ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest()); + ASSERT_OK(_databaseCloner->startup()); + ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest()); auto net = getNet(); executor::NetworkInterfaceMock::InNetworkGuard guard(net); @@ -238,10 +295,14 @@ TEST_F(DatabaseClonerTest, FirstRemoteCommandWithFilter) { filterElement.Obj()); ASSERT_FALSE(net->hasReadyRequests()); ASSERT_TRUE(_databaseCloner->isActive()); + ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest()); } TEST_F(DatabaseClonerTest, InvalidListCollectionsFilter) { + ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest()); + ASSERT_OK(_databaseCloner->startup()); + ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest()); { executor::NetworkInterfaceMock::InNetworkGuard guard(getNet()); @@ -253,11 +314,15 @@ TEST_F(DatabaseClonerTest, InvalidListCollectionsFilter) { ASSERT_EQUALS(ErrorCodes::BadValue, getStatus().code()); ASSERT_FALSE(_databaseCloner->isActive()); + ASSERT_EQUALS(DatabaseCloner::State::kComplete, _databaseCloner->getState_forTest()); } // A database may have no collections. Nothing to do for the database cloner. TEST_F(DatabaseClonerTest, ListCollectionsReturnedNoCollections) { + ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest()); + ASSERT_OK(_databaseCloner->startup()); + ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest()); // Keep going even if initial batch is empty. { @@ -276,6 +341,7 @@ TEST_F(DatabaseClonerTest, ListCollectionsReturnedNoCollections) { ASSERT_OK(getStatus()); ASSERT_FALSE(_databaseCloner->isActive()); + ASSERT_EQUALS(DatabaseCloner::State::kComplete, _databaseCloner->getState_forTest()); } TEST_F(DatabaseClonerTest, ListCollectionsPredicate) { @@ -295,7 +361,10 @@ TEST_F(DatabaseClonerTest, ListCollectionsPredicate) { stdx::placeholders::_1, stdx::placeholders::_2), stdx::bind(&DatabaseClonerTest::setStatus, this, stdx::placeholders::_1))); + ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest()); + ASSERT_OK(_databaseCloner->startup()); + ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest()); const std::vector<BSONObj> sourceInfos = {BSON("name" << "a" @@ -317,6 +386,7 @@ TEST_F(DatabaseClonerTest, ListCollectionsPredicate) { ASSERT_EQUALS(getDetectableErrorStatus(), getStatus()); ASSERT_TRUE(_databaseCloner->isActive()); + ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest()); const std::vector<BSONObj>& collectionInfos = _databaseCloner->getCollectionInfos_forTest(); ASSERT_EQUALS(2U, collectionInfos.size()); @@ -325,7 +395,10 @@ TEST_F(DatabaseClonerTest, ListCollectionsPredicate) { } TEST_F(DatabaseClonerTest, ListCollectionsMultipleBatches) { + ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest()); + ASSERT_OK(_databaseCloner->startup()); + ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest()); const std::vector<BSONObj> sourceInfos = {BSON("name" << "a" @@ -364,34 +437,52 @@ TEST_F(DatabaseClonerTest, ListCollectionsMultipleBatches) { ASSERT_BSONOBJ_EQ(sourceInfos[0], collectionInfos[0]); ASSERT_BSONOBJ_EQ(sourceInfos[1], collectionInfos[1]); } + + ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest()); } TEST_F(DatabaseClonerTest, CollectionInfoNameFieldMissing) { + ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest()); + ASSERT_OK(_databaseCloner->startup()); + ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest()); + { executor::NetworkInterfaceMock::InNetworkGuard guard(getNet()); processNetworkResponse( createListCollectionsResponse(0, BSON_ARRAY(BSON("options" << BSONObj())))); } + ASSERT_EQUALS(ErrorCodes::FailedToParse, getStatus().code()); ASSERT_STRING_CONTAINS(getStatus().reason(), "must contain 'name' field"); ASSERT_FALSE(_databaseCloner->isActive()); + ASSERT_EQUALS(DatabaseCloner::State::kComplete, _databaseCloner->getState_forTest()); } TEST_F(DatabaseClonerTest, CollectionInfoNameNotAString) { + ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest()); + ASSERT_OK(_databaseCloner->startup()); + ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest()); + { executor::NetworkInterfaceMock::InNetworkGuard guard(getNet()); processNetworkResponse(createListCollectionsResponse( 0, BSON_ARRAY(BSON("name" << 123 << "options" << BSONObj())))); } + ASSERT_EQUALS(ErrorCodes::TypeMismatch, getStatus().code()); ASSERT_STRING_CONTAINS(getStatus().reason(), "'name' field must be a string"); ASSERT_FALSE(_databaseCloner->isActive()); + ASSERT_EQUALS(DatabaseCloner::State::kComplete, _databaseCloner->getState_forTest()); } TEST_F(DatabaseClonerTest, CollectionInfoNameEmpty) { + ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest()); + ASSERT_OK(_databaseCloner->startup()); + ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest()); + { executor::NetworkInterfaceMock::InNetworkGuard guard(getNet()); processNetworkResponse(createListCollectionsResponse(0, @@ -400,13 +491,19 @@ TEST_F(DatabaseClonerTest, CollectionInfoNameEmpty) { << "options" << BSONObj())))); } + ASSERT_EQUALS(ErrorCodes::BadValue, getStatus().code()); ASSERT_STRING_CONTAINS(getStatus().reason(), "invalid collection namespace: db."); ASSERT_FALSE(_databaseCloner->isActive()); + ASSERT_EQUALS(DatabaseCloner::State::kComplete, _databaseCloner->getState_forTest()); } TEST_F(DatabaseClonerTest, CollectionInfoNameDuplicate) { + ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest()); + ASSERT_OK(_databaseCloner->startup()); + ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest()); + { executor::NetworkInterfaceMock::InNetworkGuard guard(getNet()); processNetworkResponse(createListCollectionsResponse(0, @@ -419,26 +516,38 @@ TEST_F(DatabaseClonerTest, CollectionInfoNameDuplicate) { << "options" << BSONObj())))); } + ASSERT_EQUALS(ErrorCodes::DuplicateKey, getStatus().code()); ASSERT_STRING_CONTAINS(getStatus().reason(), "duplicate collection name 'a'"); ASSERT_FALSE(_databaseCloner->isActive()); + ASSERT_EQUALS(DatabaseCloner::State::kComplete, _databaseCloner->getState_forTest()); } TEST_F(DatabaseClonerTest, CollectionInfoOptionsFieldMissing) { + ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest()); + ASSERT_OK(_databaseCloner->startup()); + ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest()); + { executor::NetworkInterfaceMock::InNetworkGuard guard(getNet()); processNetworkResponse(createListCollectionsResponse(0, BSON_ARRAY(BSON("name" << "a")))); } + ASSERT_EQUALS(ErrorCodes::FailedToParse, getStatus().code()); ASSERT_STRING_CONTAINS(getStatus().reason(), "must contain 'options' field"); ASSERT_FALSE(_databaseCloner->isActive()); + ASSERT_EQUALS(DatabaseCloner::State::kComplete, _databaseCloner->getState_forTest()); } TEST_F(DatabaseClonerTest, CollectionInfoOptionsNotAnObject) { + ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest()); + ASSERT_OK(_databaseCloner->startup()); + ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest()); + { executor::NetworkInterfaceMock::InNetworkGuard guard(getNet()); processNetworkResponse(createListCollectionsResponse(0, @@ -447,13 +556,19 @@ TEST_F(DatabaseClonerTest, CollectionInfoOptionsNotAnObject) { << "options" << 123)))); } + ASSERT_EQUALS(ErrorCodes::TypeMismatch, getStatus().code()); ASSERT_STRING_CONTAINS(getStatus().reason(), "'options' field must be an object"); ASSERT_FALSE(_databaseCloner->isActive()); + ASSERT_EQUALS(DatabaseCloner::State::kComplete, _databaseCloner->getState_forTest()); } TEST_F(DatabaseClonerTest, InvalidCollectionOptions) { + ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest()); + ASSERT_OK(_databaseCloner->startup()); + ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest()); + { executor::NetworkInterfaceMock::InNetworkGuard guard(getNet()); processNetworkResponse( @@ -463,12 +578,18 @@ TEST_F(DatabaseClonerTest, InvalidCollectionOptions) { << "options" << BSON("storageEngine" << 1))))); } + ASSERT_EQUALS(ErrorCodes::BadValue, getStatus().code()); ASSERT_FALSE(_databaseCloner->isActive()); + ASSERT_EQUALS(DatabaseCloner::State::kComplete, _databaseCloner->getState_forTest()); } TEST_F(DatabaseClonerTest, DatabaseClonerResendsListCollectionsRequestOnRetriableError) { + ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest()); + ASSERT_OK(_databaseCloner->startup()); + ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest()); + auto net = getNet(); executor::NetworkInterfaceMock::InNetworkGuard guard(net); @@ -479,6 +600,7 @@ TEST_F(DatabaseClonerTest, DatabaseClonerResendsListCollectionsRequestOnRetriabl // DatabaseCloner stays active because it resends the listCollections request. ASSERT_TRUE(_databaseCloner->isActive()); + ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest()); // DatabaseCloner should resend listCollections request. auto noi = net->getNextReadyRequest(); @@ -500,7 +622,10 @@ TEST_F(DatabaseClonerTest, ListCollectionsReturnsEmptyCollectionName) { stdx::placeholders::_1, stdx::placeholders::_2), stdx::bind(&DatabaseClonerTest::setStatus, this, stdx::placeholders::_1))); + ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest()); + ASSERT_OK(_databaseCloner->startup()); + ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest()); { executor::NetworkInterfaceMock::InNetworkGuard guard(getNet()); @@ -510,13 +635,18 @@ TEST_F(DatabaseClonerTest, ListCollectionsReturnsEmptyCollectionName) { << "options" << BSONObj())))); } + ASSERT_EQUALS(ErrorCodes::BadValue, getStatus().code()); ASSERT_STRING_CONTAINS(getStatus().reason(), "invalid collection namespace: db."); ASSERT_FALSE(_databaseCloner->isActive()); + ASSERT_EQUALS(DatabaseCloner::State::kComplete, _databaseCloner->getState_forTest()); } TEST_F(DatabaseClonerTest, StartFirstCollectionClonerFailed) { + ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest()); + ASSERT_OK(_databaseCloner->startup()); + ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest()); _databaseCloner->setStartCollectionClonerFn([](CollectionCloner& cloner) { return Status(ErrorCodes::OperationFailed, @@ -531,12 +661,18 @@ TEST_F(DatabaseClonerTest, StartFirstCollectionClonerFailed) { << "options" << BSONObj())))); } + ASSERT_EQUALS(ErrorCodes::OperationFailed, getStatus().code()); ASSERT_FALSE(_databaseCloner->isActive()); + ASSERT_EQUALS(DatabaseCloner::State::kComplete, _databaseCloner->getState_forTest()); } TEST_F(DatabaseClonerTest, StartSecondCollectionClonerFailed) { + ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest()); + ASSERT_OK(_databaseCloner->startup()); + ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest()); + const Status errStatus{ErrorCodes::OperationFailed, "StartSecondCollectionClonerFailed injected failure."}; @@ -566,10 +702,15 @@ TEST_F(DatabaseClonerTest, StartSecondCollectionClonerFailed) { _databaseCloner->join(); ASSERT_FALSE(_databaseCloner->isActive()); ASSERT_EQUALS(errStatus, getStatus()); + ASSERT_EQUALS(DatabaseCloner::State::kComplete, _databaseCloner->getState_forTest()); } TEST_F(DatabaseClonerTest, ShutdownCancelsCollectionCloning) { + ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest()); + ASSERT_OK(_databaseCloner->startup()); + ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest()); + auto net = getNet(); { executor::NetworkInterfaceMock::InNetworkGuard guard(net); @@ -596,12 +737,14 @@ TEST_F(DatabaseClonerTest, ShutdownCancelsCollectionCloning) { } _databaseCloner->shutdown(); + ASSERT_EQUALS(DatabaseCloner::State::kShuttingDown, _databaseCloner->getState_forTest()); // Deliver cancellation event to cloners. executor::NetworkInterfaceMock::InNetworkGuard(net)->runReadyNetworkOperations(); _databaseCloner->join(); ASSERT_FALSE(_databaseCloner->isActive()); + ASSERT_EQUALS(DatabaseCloner::State::kComplete, _databaseCloner->getState_forTest()); // This is the error code from attempting to start up the last (of 2) collection cloner which // was shut down before it was ever started. @@ -609,7 +752,10 @@ TEST_F(DatabaseClonerTest, ShutdownCancelsCollectionCloning) { } TEST_F(DatabaseClonerTest, FirstCollectionListIndexesFailed) { + ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest()); + ASSERT_OK(_databaseCloner->startup()); + ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest()); const std::vector<BSONObj> sourceInfos = {BSON("name" << "a" @@ -644,6 +790,7 @@ TEST_F(DatabaseClonerTest, FirstCollectionListIndexesFailed) { _databaseCloner->join(); ASSERT_EQ(getStatus().code(), ErrorCodes::InitialSyncFailure); ASSERT_FALSE(_databaseCloner->isActive()); + ASSERT_EQUALS(DatabaseCloner::State::kComplete, _databaseCloner->getState_forTest()); ASSERT_EQUALS(2U, _collections.size()); @@ -661,7 +808,10 @@ TEST_F(DatabaseClonerTest, FirstCollectionListIndexesFailed) { } TEST_F(DatabaseClonerTest, CreateCollections) { + ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest()); + ASSERT_OK(_databaseCloner->startup()); + ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest()); const std::vector<BSONObj> sourceInfos = {BSON("name" << "a" @@ -707,6 +857,7 @@ TEST_F(DatabaseClonerTest, CreateCollections) { _databaseCloner->join(); ASSERT_OK(getStatus()); ASSERT_FALSE(_databaseCloner->isActive()); + ASSERT_EQUALS(DatabaseCloner::State::kComplete, _databaseCloner->getState_forTest()); ASSERT_EQUALS(2U, _collections.size()); |