SERVER-27611 make DatabaseCloner single-use only. If a DatabaseCloner is shut down before startup, calling startup() will fail.

4 files changed, 232 insertions, 17 deletions

diff --git a/src/mongo/db/repl/SConscript b/src/mongo/db/repl/SConscript
index 616fbd35e32..1736bcc2c0c 100644
--- a/src/mongo/db/repl/SConscript
+++ b/src/mongo/db/repl/SConscript
@@ -869,6 +869,7 @@ env.CppUnitTest(
         'base_cloner_test_fixture',
         '$BUILD_DIR/mongo/db/auth/authorization_manager_mock_init',
         '$BUILD_DIR/mongo/db/service_context_noop_init',
+        '$BUILD_DIR/mongo/unittest/task_executor_proxy',
     ],
 )
 
diff --git a/src/mongo/db/repl/database_cloner.cpp b/src/mongo/db/repl/database_cloner.cpp
index 224d6d15a08..ba3f52ed048 100644
--- a/src/mongo/db/repl/database_cloner.cpp
+++ b/src/mongo/db/repl/database_cloner.cpp
@@ -152,7 +152,7 @@ std::string DatabaseCloner::_getDiagnosticString_inlock() const {
     output << " source: " << _source.toString();
     output << " database: " << _dbname;
     output << " listCollections filter" << _listCollectionsFilter;
-    output << " active: " << _active;
+    output << " active: " << _isActive_inlock();
     output << " collection info objects (empty if listCollections is in progress): "
            << _collectionInfos.size();
     return output;
@@ -160,14 +160,26 @@ std::string DatabaseCloner::_getDiagnosticString_inlock() const {
 
 bool DatabaseCloner::isActive() const {
     LockGuard lk(_mutex);
-    return _active;
+    return _isActive_inlock();
+}
+
+bool DatabaseCloner::_isActive_inlock() const {
+    return State::kRunning == _state || State::kShuttingDown == _state;
 }
 
 Status DatabaseCloner::startup() noexcept {
     LockGuard lk(_mutex);
 
-    if (_active) {
-        return Status(ErrorCodes::IllegalOperation, "database cloner already started");
+    switch (_state) {
+        case State::kPreStart:
+            _state = State::kRunning;
+            break;
+        case State::kRunning:
+            return Status(ErrorCodes::InternalError, "database cloner already started");
+        case State::kShuttingDown:
+            return Status(ErrorCodes::ShutdownInProgress, "database cloner shutting down");
+        case State::kComplete:
+            return Status(ErrorCodes::ShutdownInProgress, "database cloner completed");
     }
 
     _stats.start = _executor->now();
@@ -176,25 +188,31 @@ Status DatabaseCloner::startup() noexcept {
     if (!scheduleResult.isOK()) {
         error() << "Error scheduling listCollections for database: " << _dbname
                 << ", error:" << scheduleResult;
+        _state = State::kComplete;
         return scheduleResult;
     }
 
-    _active = true;
-
     return Status::OK();
 }
 
 void DatabaseCloner::shutdown() {
-    {
-        LockGuard lk(_mutex);
-
-        if (!_active) {
+    stdx::lock_guard<stdx::mutex> lock(_mutex);
+    switch (_state) {
+        case State::kPreStart:
+            // Transition directly from PreStart to Complete if not started yet.
+            _state = State::kComplete;
             return;
-        }
+        case State::kRunning:
+            _state = State::kShuttingDown;
+            break;
+        case State::kShuttingDown:
+        case State::kComplete:
+            // Nothing to do if we are already in ShuttingDown or Complete state.
+            return;
+    }
 
-        for (auto&& collectionCloner : _collectionCloners) {
-            collectionCloner.shutdown();
-        }
+    for (auto&& collectionCloner : _collectionCloners) {
+        collectionCloner.shutdown();
     }
 
     _listCollectionsFetcher.shutdown();
@@ -211,7 +229,7 @@ DatabaseCloner::Stats DatabaseCloner::getStats() const {
 
 void DatabaseCloner::join() {
     UniqueLock lk(_mutex);
-    _condition.wait(lk, [this]() { return !_active; });
+    _condition.wait(lk, [this]() { return !_isActive_inlock(); });
 }
 
 void DatabaseCloner::setScheduleDbWorkFn_forTest(const CollectionCloner::ScheduleDbWorkFn& work) {
@@ -225,6 +243,11 @@ void DatabaseCloner::setStartCollectionClonerFn(
     _startCollectionCloner = startCollectionCloner;
 }
 
+DatabaseCloner::State DatabaseCloner::getState_forTest() const {
+    stdx::lock_guard<stdx::mutex> lk(_mutex);
+    return _state;
+}
+
 void DatabaseCloner::_listCollectionsCallback(const StatusWith<Fetcher::QueryResponse>& result,
                                               Fetcher::NextAction* nextAction,
                                               BSONObjBuilder* getMoreBob) {
@@ -413,7 +436,8 @@ void DatabaseCloner::_collectionClonerCallback(const Status& status, const Names
 void DatabaseCloner::_finishCallback(const Status& status) {
     _onCompletion(status);
     LockGuard lk(_mutex);
-    _active = false;
+    invariant(_state != State::kComplete);
+    _state = State::kComplete;
     _condition.notify_all();
     _stats.end = _executor->now();
     LOG(1) << "    database: " << _dbname << ", stats: " << _stats.toString();
@@ -461,5 +485,19 @@ void DatabaseCloner::Stats::append(BSONObjBuilder* builder) const {
     }
 }
 
+std::ostream& operator<<(std::ostream& os, const DatabaseCloner::State& state) {
+    switch (state) {
+        case DatabaseCloner::State::kPreStart:
+            return os << "PreStart";
+        case DatabaseCloner::State::kRunning:
+            return os << "Running";
+        case DatabaseCloner::State::kShuttingDown:
+            return os << "ShuttingDown";
+        case DatabaseCloner::State::kComplete:
+            return os << "Complete";
+    }
+    MONGO_UNREACHABLE;
+}
+
 }  // namespace repl
 }  // namespace mongo
diff --git a/src/mongo/db/repl/database_cloner.h b/src/mongo/db/repl/database_cloner.h
index 4acfba304e8..97a4c0cbc4c 100644
--- a/src/mongo/db/repl/database_cloner.h
+++ b/src/mongo/db/repl/database_cloner.h
@@ -28,6 +28,7 @@
 
 #pragma once
 
+#include <iosfwd>
 #include <list>
 #include <string>
 #include <vector>
@@ -161,7 +162,23 @@ public:
      */
     void setStartCollectionClonerFn(const StartCollectionClonerFn& startCollectionCloner);
 
+    // State transitions:
+    // PreStart --> Running --> ShuttingDown --> Complete
+    // It is possible to skip intermediate states. For example,
+    // Calling shutdown() when the cloner has not started will transition from PreStart directly
+    // to Complete.
+    // This enum class is made public for testing.
+    enum class State { kPreStart, kRunning, kShuttingDown, kComplete };
+
+    /**
+     * Returns current database cloner state.
+     * For testing only.
+     */
+    State getState_forTest() const;
+
 private:
+    bool _isActive_inlock() const;
+
     /**
      * Read collection names and options from listCollections result.
      */
@@ -209,7 +226,6 @@ private:
     CollectionCallbackFn
         _collectionWork;       // (R) Invoked once for every successfully started collection cloner.
     CallbackFn _onCompletion;  // (R) Invoked once when cloning completes or fails.
-    bool _active = false;      // _active is true when database cloner is started.
     Fetcher _listCollectionsFetcher;  // (R) Fetcher instance for running listCollections command.
     // Collection info objects returned from listCollections.
     // Format of each document:
@@ -227,7 +243,16 @@ private:
         _scheduleDbWorkFn;  // (RT) Function for scheduling database work using the executor.
     StartCollectionClonerFn _startCollectionCloner;  // (RT)
     Stats _stats;                                    // (M) Stats about what this instance did.
+
+    // Current database cloner state. See comments for State enum class for details.
+    State _state = State::kPreStart;  // (M)
 };
 
+/**
+ * Insertion operator for DatabaseCloner::State. Formats database cloner state for output stream.
+ * For testing only.
+ */
+std::ostream& operator<<(std::ostream& os, const DatabaseCloner::State& state);
+
 }  // namespace repl
 }  // namespace mongo
diff --git a/src/mongo/db/repl/database_cloner_test.cpp b/src/mongo/db/repl/database_cloner_test.cpp
index 5eeca00cfcd..f5fa63181e4 100644
--- a/src/mongo/db/repl/database_cloner_test.cpp
+++ b/src/mongo/db/repl/database_cloner_test.cpp
@@ -37,6 +37,7 @@
 #include "mongo/db/repl/base_cloner_test_fixture.h"
 #include "mongo/db/repl/database_cloner.h"
 #include "mongo/db/repl/storage_interface.h"
+#include "mongo/unittest/task_executor_proxy.h"
 #include "mongo/unittest/unittest.h"
 #include "mongo/util/mongoutils/str.h"
 
@@ -187,8 +188,61 @@ TEST_F(DatabaseClonerTest, ClonerLifeCycle) {
     testLifeCycle();
 }
 
+TEST_F(DatabaseClonerTest, DatabaseClonerTransitionsToCompleteIfShutdownBeforeStartup) {
+    ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest());
+
+    _databaseCloner->shutdown();
+    ASSERT_EQUALS(DatabaseCloner::State::kComplete, _databaseCloner->getState_forTest());
+    ASSERT_EQUALS(ErrorCodes::ShutdownInProgress, _databaseCloner->startup());
+}
+
+class TaskExecutorWithFailureInScheduleRemoteCommand : public unittest::TaskExecutorProxy {
+public:
+    using ShouldFailRequestFn = stdx::function<bool(const executor::RemoteCommandRequest&)>;
+
+    TaskExecutorWithFailureInScheduleRemoteCommand(executor::TaskExecutor* executor,
+                                                   ShouldFailRequestFn shouldFailRequest)
+        : unittest::TaskExecutorProxy(executor), _shouldFailRequest(shouldFailRequest) {}
+
+    StatusWith<CallbackHandle> scheduleRemoteCommand(const executor::RemoteCommandRequest& request,
+                                                     const RemoteCommandCallbackFn& cb) override {
+        if (_shouldFailRequest(request)) {
+            return Status(ErrorCodes::OperationFailed, "failed to schedule remote command");
+        }
+        return getExecutor()->scheduleRemoteCommand(request, cb);
+    }
+
+private:
+    ShouldFailRequestFn _shouldFailRequest;
+};
+
+TEST_F(DatabaseClonerTest,
+       DatabaseClonerReturnsScheduleErrorOnFailingToScheduleListCollectionsCommand) {
+    TaskExecutorWithFailureInScheduleRemoteCommand executorProxy(
+        &getExecutor(), [](const executor::RemoteCommandRequest& request) {
+            return str::equals("listCollections", request.cmdObj.firstElementFieldName());
+        });
+
+    DatabaseCloner databaseCloner(&executorProxy,
+                                  dbWorkThreadPool.get(),
+                                  target,
+                                  dbname,
+                                  BSONObj(),
+                                  DatabaseCloner::ListCollectionsPredicateFn(),
+                                  storageInterface.get(),
+                                  [](const Status&, const NamespaceString&) {},
+                                  [](const Status&) {});
+    ASSERT_EQUALS(DatabaseCloner::State::kPreStart, databaseCloner.getState_forTest());
+
+    ASSERT_EQUALS(ErrorCodes::OperationFailed, databaseCloner.startup());
+    ASSERT_EQUALS(DatabaseCloner::State::kComplete, databaseCloner.getState_forTest());
+}
+
 TEST_F(DatabaseClonerTest, FirstRemoteCommandWithoutFilter) {
+    ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest());
+
     ASSERT_OK(_databaseCloner->startup());
+    ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest());
 
     auto net = getNet();
     executor::NetworkInterfaceMock::InNetworkGuard guard(net);
@@ -222,7 +276,10 @@ TEST_F(DatabaseClonerTest, FirstRemoteCommandWithFilter) {
                    stdx::placeholders::_1,
                    stdx::placeholders::_2),
         stdx::bind(&DatabaseClonerTest::setStatus, this, stdx::placeholders::_1)));
+    ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest());
+
     ASSERT_OK(_databaseCloner->startup());
+    ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest());
 
     auto net = getNet();
     executor::NetworkInterfaceMock::InNetworkGuard guard(net);
@@ -238,10 +295,14 @@ TEST_F(DatabaseClonerTest, FirstRemoteCommandWithFilter) {
                       filterElement.Obj());
     ASSERT_FALSE(net->hasReadyRequests());
     ASSERT_TRUE(_databaseCloner->isActive());
+    ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest());
 }
 
 TEST_F(DatabaseClonerTest, InvalidListCollectionsFilter) {
+    ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest());
+
     ASSERT_OK(_databaseCloner->startup());
+    ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest());
 
     {
         executor::NetworkInterfaceMock::InNetworkGuard guard(getNet());
@@ -253,11 +314,15 @@ TEST_F(DatabaseClonerTest, InvalidListCollectionsFilter) {
 
     ASSERT_EQUALS(ErrorCodes::BadValue, getStatus().code());
     ASSERT_FALSE(_databaseCloner->isActive());
+    ASSERT_EQUALS(DatabaseCloner::State::kComplete, _databaseCloner->getState_forTest());
 }
 
 // A database may have no collections. Nothing to do for the database cloner.
 TEST_F(DatabaseClonerTest, ListCollectionsReturnedNoCollections) {
+    ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest());
+
     ASSERT_OK(_databaseCloner->startup());
+    ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest());
 
     // Keep going even if initial batch is empty.
     {
@@ -276,6 +341,7 @@ TEST_F(DatabaseClonerTest, ListCollectionsReturnedNoCollections) {
 
     ASSERT_OK(getStatus());
     ASSERT_FALSE(_databaseCloner->isActive());
+    ASSERT_EQUALS(DatabaseCloner::State::kComplete, _databaseCloner->getState_forTest());
 }
 
 TEST_F(DatabaseClonerTest, ListCollectionsPredicate) {
@@ -295,7 +361,10 @@ TEST_F(DatabaseClonerTest, ListCollectionsPredicate) {
                    stdx::placeholders::_1,
                    stdx::placeholders::_2),
         stdx::bind(&DatabaseClonerTest::setStatus, this, stdx::placeholders::_1)));
+    ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest());
+
     ASSERT_OK(_databaseCloner->startup());
+    ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest());
 
     const std::vector<BSONObj> sourceInfos = {BSON("name"
                                                    << "a"
@@ -317,6 +386,7 @@ TEST_F(DatabaseClonerTest, ListCollectionsPredicate) {
 
     ASSERT_EQUALS(getDetectableErrorStatus(), getStatus());
     ASSERT_TRUE(_databaseCloner->isActive());
+    ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest());
 
     const std::vector<BSONObj>& collectionInfos = _databaseCloner->getCollectionInfos_forTest();
     ASSERT_EQUALS(2U, collectionInfos.size());
@@ -325,7 +395,10 @@ TEST_F(DatabaseClonerTest, ListCollectionsPredicate) {
 }
 
 TEST_F(DatabaseClonerTest, ListCollectionsMultipleBatches) {
+    ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest());
+
     ASSERT_OK(_databaseCloner->startup());
+    ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest());
 
     const std::vector<BSONObj> sourceInfos = {BSON("name"
                                                    << "a"
@@ -364,34 +437,52 @@ TEST_F(DatabaseClonerTest, ListCollectionsMultipleBatches) {
         ASSERT_BSONOBJ_EQ(sourceInfos[0], collectionInfos[0]);
         ASSERT_BSONOBJ_EQ(sourceInfos[1], collectionInfos[1]);
     }
+
+    ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest());
 }
 
 TEST_F(DatabaseClonerTest, CollectionInfoNameFieldMissing) {
+    ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest());
+
     ASSERT_OK(_databaseCloner->startup());
+    ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest());
+
     {
         executor::NetworkInterfaceMock::InNetworkGuard guard(getNet());
         processNetworkResponse(
             createListCollectionsResponse(0, BSON_ARRAY(BSON("options" << BSONObj()))));
     }
+
     ASSERT_EQUALS(ErrorCodes::FailedToParse, getStatus().code());
     ASSERT_STRING_CONTAINS(getStatus().reason(), "must contain 'name' field");
     ASSERT_FALSE(_databaseCloner->isActive());
+    ASSERT_EQUALS(DatabaseCloner::State::kComplete, _databaseCloner->getState_forTest());
 }
 
 TEST_F(DatabaseClonerTest, CollectionInfoNameNotAString) {
+    ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest());
+
     ASSERT_OK(_databaseCloner->startup());
+    ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest());
+
     {
         executor::NetworkInterfaceMock::InNetworkGuard guard(getNet());
         processNetworkResponse(createListCollectionsResponse(
             0, BSON_ARRAY(BSON("name" << 123 << "options" << BSONObj()))));
     }
+
     ASSERT_EQUALS(ErrorCodes::TypeMismatch, getStatus().code());
     ASSERT_STRING_CONTAINS(getStatus().reason(), "'name' field must be a string");
     ASSERT_FALSE(_databaseCloner->isActive());
+    ASSERT_EQUALS(DatabaseCloner::State::kComplete, _databaseCloner->getState_forTest());
 }
 
 TEST_F(DatabaseClonerTest, CollectionInfoNameEmpty) {
+    ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest());
+
     ASSERT_OK(_databaseCloner->startup());
+    ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest());
+
     {
         executor::NetworkInterfaceMock::InNetworkGuard guard(getNet());
         processNetworkResponse(createListCollectionsResponse(0,
@@ -400,13 +491,19 @@ TEST_F(DatabaseClonerTest, CollectionInfoNameEmpty) {
                                                                              << "options"
                                                                              << BSONObj()))));
     }
+
     ASSERT_EQUALS(ErrorCodes::BadValue, getStatus().code());
     ASSERT_STRING_CONTAINS(getStatus().reason(), "invalid collection namespace: db.");
     ASSERT_FALSE(_databaseCloner->isActive());
+    ASSERT_EQUALS(DatabaseCloner::State::kComplete, _databaseCloner->getState_forTest());
 }
 
 TEST_F(DatabaseClonerTest, CollectionInfoNameDuplicate) {
+    ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest());
+
     ASSERT_OK(_databaseCloner->startup());
+    ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest());
+
     {
         executor::NetworkInterfaceMock::InNetworkGuard guard(getNet());
         processNetworkResponse(createListCollectionsResponse(0,
@@ -419,26 +516,38 @@ TEST_F(DatabaseClonerTest, CollectionInfoNameDuplicate) {
                                                                                 << "options"
                                                                                 << BSONObj()))));
     }
+
     ASSERT_EQUALS(ErrorCodes::DuplicateKey, getStatus().code());
     ASSERT_STRING_CONTAINS(getStatus().reason(), "duplicate collection name 'a'");
     ASSERT_FALSE(_databaseCloner->isActive());
+    ASSERT_EQUALS(DatabaseCloner::State::kComplete, _databaseCloner->getState_forTest());
 }
 
 TEST_F(DatabaseClonerTest, CollectionInfoOptionsFieldMissing) {
+    ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest());
+
     ASSERT_OK(_databaseCloner->startup());
+    ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest());
+
     {
         executor::NetworkInterfaceMock::InNetworkGuard guard(getNet());
         processNetworkResponse(createListCollectionsResponse(0,
                                                              BSON_ARRAY(BSON("name"
                                                                              << "a"))));
     }
+
     ASSERT_EQUALS(ErrorCodes::FailedToParse, getStatus().code());
     ASSERT_STRING_CONTAINS(getStatus().reason(), "must contain 'options' field");
     ASSERT_FALSE(_databaseCloner->isActive());
+    ASSERT_EQUALS(DatabaseCloner::State::kComplete, _databaseCloner->getState_forTest());
 }
 
 TEST_F(DatabaseClonerTest, CollectionInfoOptionsNotAnObject) {
+    ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest());
+
     ASSERT_OK(_databaseCloner->startup());
+    ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest());
+
     {
         executor::NetworkInterfaceMock::InNetworkGuard guard(getNet());
         processNetworkResponse(createListCollectionsResponse(0,
@@ -447,13 +556,19 @@ TEST_F(DatabaseClonerTest, CollectionInfoOptionsNotAnObject) {
                                                                              << "options"
                                                                              << 123))));
     }
+
     ASSERT_EQUALS(ErrorCodes::TypeMismatch, getStatus().code());
     ASSERT_STRING_CONTAINS(getStatus().reason(), "'options' field must be an object");
     ASSERT_FALSE(_databaseCloner->isActive());
+    ASSERT_EQUALS(DatabaseCloner::State::kComplete, _databaseCloner->getState_forTest());
 }
 
 TEST_F(DatabaseClonerTest, InvalidCollectionOptions) {
+    ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest());
+
     ASSERT_OK(_databaseCloner->startup());
+    ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest());
+
     {
         executor::NetworkInterfaceMock::InNetworkGuard guard(getNet());
         processNetworkResponse(
@@ -463,12 +578,18 @@ TEST_F(DatabaseClonerTest, InvalidCollectionOptions) {
                                                           << "options"
                                                           << BSON("storageEngine" << 1)))));
     }
+
     ASSERT_EQUALS(ErrorCodes::BadValue, getStatus().code());
     ASSERT_FALSE(_databaseCloner->isActive());
+    ASSERT_EQUALS(DatabaseCloner::State::kComplete, _databaseCloner->getState_forTest());
 }
 
 TEST_F(DatabaseClonerTest, DatabaseClonerResendsListCollectionsRequestOnRetriableError) {
+    ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest());
+
     ASSERT_OK(_databaseCloner->startup());
+    ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest());
+
     auto net = getNet();
     executor::NetworkInterfaceMock::InNetworkGuard guard(net);
 
@@ -479,6 +600,7 @@ TEST_F(DatabaseClonerTest, DatabaseClonerResendsListCollectionsRequestOnRetriabl
 
     // DatabaseCloner stays active because it resends the listCollections request.
     ASSERT_TRUE(_databaseCloner->isActive());
+    ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest());
 
     // DatabaseCloner should resend listCollections request.
     auto noi = net->getNextReadyRequest();
@@ -500,7 +622,10 @@ TEST_F(DatabaseClonerTest, ListCollectionsReturnsEmptyCollectionName) {
                    stdx::placeholders::_1,
                    stdx::placeholders::_2),
         stdx::bind(&DatabaseClonerTest::setStatus, this, stdx::placeholders::_1)));
+    ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest());
+
     ASSERT_OK(_databaseCloner->startup());
+    ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest());
 
     {
         executor::NetworkInterfaceMock::InNetworkGuard guard(getNet());
@@ -510,13 +635,18 @@ TEST_F(DatabaseClonerTest, ListCollectionsReturnsEmptyCollectionName) {
                                                                              << "options"
                                                                              << BSONObj()))));
     }
+
     ASSERT_EQUALS(ErrorCodes::BadValue, getStatus().code());
     ASSERT_STRING_CONTAINS(getStatus().reason(), "invalid collection namespace: db.");
     ASSERT_FALSE(_databaseCloner->isActive());
+    ASSERT_EQUALS(DatabaseCloner::State::kComplete, _databaseCloner->getState_forTest());
 }
 
 TEST_F(DatabaseClonerTest, StartFirstCollectionClonerFailed) {
+    ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest());
+
     ASSERT_OK(_databaseCloner->startup());
+    ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest());
 
     _databaseCloner->setStartCollectionClonerFn([](CollectionCloner& cloner) {
         return Status(ErrorCodes::OperationFailed,
@@ -531,12 +661,18 @@ TEST_F(DatabaseClonerTest, StartFirstCollectionClonerFailed) {
                                                                              << "options"
                                                                              << BSONObj()))));
     }
+
     ASSERT_EQUALS(ErrorCodes::OperationFailed, getStatus().code());
     ASSERT_FALSE(_databaseCloner->isActive());
+    ASSERT_EQUALS(DatabaseCloner::State::kComplete, _databaseCloner->getState_forTest());
 }
 
 TEST_F(DatabaseClonerTest, StartSecondCollectionClonerFailed) {
+    ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest());
+
     ASSERT_OK(_databaseCloner->startup());
+    ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest());
+
     const Status errStatus{ErrorCodes::OperationFailed,
                            "StartSecondCollectionClonerFailed injected failure."};
 
@@ -566,10 +702,15 @@ TEST_F(DatabaseClonerTest, StartSecondCollectionClonerFailed) {
     _databaseCloner->join();
     ASSERT_FALSE(_databaseCloner->isActive());
     ASSERT_EQUALS(errStatus, getStatus());
+    ASSERT_EQUALS(DatabaseCloner::State::kComplete, _databaseCloner->getState_forTest());
 }
 
 TEST_F(DatabaseClonerTest, ShutdownCancelsCollectionCloning) {
+    ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest());
+
     ASSERT_OK(_databaseCloner->startup());
+    ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest());
+
     auto net = getNet();
     {
         executor::NetworkInterfaceMock::InNetworkGuard guard(net);
@@ -596,12 +737,14 @@ TEST_F(DatabaseClonerTest, ShutdownCancelsCollectionCloning) {
     }
 
     _databaseCloner->shutdown();
+    ASSERT_EQUALS(DatabaseCloner::State::kShuttingDown, _databaseCloner->getState_forTest());
 
     // Deliver cancellation event to cloners.
     executor::NetworkInterfaceMock::InNetworkGuard(net)->runReadyNetworkOperations();
 
     _databaseCloner->join();
     ASSERT_FALSE(_databaseCloner->isActive());
+    ASSERT_EQUALS(DatabaseCloner::State::kComplete, _databaseCloner->getState_forTest());
 
     // This is the error code from attempting to start up the last (of 2) collection cloner which
     // was shut down before it was ever started.
@@ -609,7 +752,10 @@ TEST_F(DatabaseClonerTest, ShutdownCancelsCollectionCloning) {
 }
 
 TEST_F(DatabaseClonerTest, FirstCollectionListIndexesFailed) {
+    ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest());
+
     ASSERT_OK(_databaseCloner->startup());
+    ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest());
 
     const std::vector<BSONObj> sourceInfos = {BSON("name"
                                                    << "a"
@@ -644,6 +790,7 @@ TEST_F(DatabaseClonerTest, FirstCollectionListIndexesFailed) {
     _databaseCloner->join();
     ASSERT_EQ(getStatus().code(), ErrorCodes::InitialSyncFailure);
     ASSERT_FALSE(_databaseCloner->isActive());
+    ASSERT_EQUALS(DatabaseCloner::State::kComplete, _databaseCloner->getState_forTest());
 
     ASSERT_EQUALS(2U, _collections.size());
 
@@ -661,7 +808,10 @@ TEST_F(DatabaseClonerTest, FirstCollectionListIndexesFailed) {
 }
 
 TEST_F(DatabaseClonerTest, CreateCollections) {
+    ASSERT_EQUALS(DatabaseCloner::State::kPreStart, _databaseCloner->getState_forTest());
+
     ASSERT_OK(_databaseCloner->startup());
+    ASSERT_EQUALS(DatabaseCloner::State::kRunning, _databaseCloner->getState_forTest());
 
     const std::vector<BSONObj> sourceInfos = {BSON("name"
                                                    << "a"
@@ -707,6 +857,7 @@ TEST_F(DatabaseClonerTest, CreateCollections) {
     _databaseCloner->join();
     ASSERT_OK(getStatus());
     ASSERT_FALSE(_databaseCloner->isActive());
+    ASSERT_EQUALS(DatabaseCloner::State::kComplete, _databaseCloner->getState_forTest());
 
     ASSERT_EQUALS(2U, _collections.size());