SERVER-48815 Create TenantAllDatabaseCloner

8 files changed, 816 insertions, 17 deletions

diff --git a/src/mongo/db/repl/SConscript b/src/mongo/db/repl/SConscript
index e06386f1469..0cc508c438f 100644
--- a/src/mongo/db/repl/SConscript
+++ b/src/mongo/db/repl/SConscript
@@ -977,7 +977,7 @@ env.Library(
     ],
     LIBDEPS = [
         'base_cloner',
-        'database_cloner_common',
+        'cloner_utils',
         'initial_sync_shared_data',
         'member_data',
         'replication_consistency_markers_impl',
@@ -1000,12 +1000,13 @@ env.Library(
 env.Library(
     target='tenant_migration_cloners',
     source=[
+        'tenant_all_database_cloner.cpp',
         'tenant_collection_cloner.cpp',
         'tenant_database_cloner.cpp',
     ],
     LIBDEPS = [
         'base_cloner',
-        'database_cloner_common',
+        'cloner_utils',
         'initial_sync_shared_data',
         '$BUILD_DIR/mongo/base',
     ],
@@ -1122,8 +1123,9 @@ env.Library(
 )
 
 env.Library(
-    target='database_cloner_common',
+    target='cloner_utils',
     source=[
+        'cloner_utils.cpp',
         'database_cloner_common.cpp',
         env.Idlc('database_cloner.idl')[0],
     ],
@@ -1132,7 +1134,9 @@ env.Library(
     ],
     LIBDEPS_PRIVATE=[
         '$BUILD_DIR/mongo/db/catalog/collection_options',
+        '$BUILD_DIR/mongo/db/namespace_string',
         '$BUILD_DIR/mongo/idl/idl_parser',
+        'read_concern_args',
     ]
 )
 
@@ -1488,6 +1492,7 @@ env.CppUnitTest(
         'collection_cloner_test.cpp',
         'database_cloner_test.cpp',
         'initial_sync_shared_data_test.cpp',
+        'tenant_all_database_cloner_test.cpp',
         'tenant_database_cloner_test.cpp'
     ],
     LIBDEPS=[
diff --git a/src/mongo/db/repl/cloner_utils.cpp b/src/mongo/db/repl/cloner_utils.cpp
new file mode 100644
index 00000000000..988259c0b46
--- /dev/null
+++ b/src/mongo/db/repl/cloner_utils.cpp
@@ -0,0 +1,55 @@
+/**
+ *    Copyright (C) 2020-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/bson/bsonobjbuilder.h"
+#include "mongo/db/namespace_string.h"
+#include "mongo/db/repl/cloner_utils.h"
+#include "mongo/db/repl/read_concern_args.h"
+
+namespace mongo {
+namespace repl {
+
+BSONObj ClonerUtils::makeTenantDatabaseFilter(StringData prefix) {
+    return BSON("name" << BSON("$regexp"
+                               << "/^" + prefix + "_.*/"));
+}
+
+BSONObj ClonerUtils::buildMajorityWaitRequest(Timestamp operationTime) {
+    BSONObjBuilder bob;
+    bob.append("find", NamespaceString::kSystemReplSetNamespace.toString());
+    bob.append("filter", BSONObj());
+    ReadConcernArgs readConcern(LogicalTime(operationTime), ReadConcernLevel::kMajorityReadConcern);
+    readConcern.appendInfo(&bob);
+    return bob.obj();
+}
+
+}  // namespace repl
+}  // namespace mongo
\ No newline at end of file
diff --git a/src/mongo/db/repl/cloner_utils.h b/src/mongo/db/repl/cloner_utils.h
new file mode 100644
index 00000000000..9f37ac91648
--- /dev/null
+++ b/src/mongo/db/repl/cloner_utils.h
@@ -0,0 +1,61 @@
+/**
+ *    Copyright (C) 2020-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#pragma once
+
+#include <string>
+
+#include "mongo/bson/bsonobj.h"
+#include "mongo/bson/timestamp.h"
+
+namespace mongo {
+namespace repl {
+
+/**
+ * Contains a variety of static helper methods used by any set of the replication cloners.
+ */
+class ClonerUtils {
+    ClonerUtils(const ClonerUtils&) = delete;
+    ClonerUtils& operator=(const ClonerUtils&) = delete;
+
+public:
+    /**
+     * Builds a filter that matches database names prefixed with a specific tenantId.
+     */
+    static BSONObj makeTenantDatabaseFilter(StringData prefix);
+
+    /**
+     * Assembles a majority read using the operationTime specified as the afterClusterTime.
+     */
+    static BSONObj buildMajorityWaitRequest(Timestamp operationTime);
+};
+
+
+}  // namespace repl
+}  // namespace mongo
\ No newline at end of file
diff --git a/src/mongo/db/repl/tenant_all_database_cloner.cpp b/src/mongo/db/repl/tenant_all_database_cloner.cpp
new file mode 100644
index 00000000000..1005ddc5b25
--- /dev/null
+++ b/src/mongo/db/repl/tenant_all_database_cloner.cpp
@@ -0,0 +1,189 @@
+/**
+ *    Copyright (C) 2020-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kReplicationInitialSync
+
+#include "mongo/platform/basic.h"
+
+#include <algorithm>
+
+#include "mongo/db/repl/cloner_utils.h"
+#include "mongo/db/repl/tenant_all_database_cloner.h"
+#include "mongo/db/repl/tenant_database_cloner.h"
+#include "mongo/logv2/log.h"
+#include "mongo/rpc/get_status_from_command_result.h"
+#include "mongo/util/assert_util.h"
+
+namespace mongo {
+namespace repl {
+
+// Failpoint which the tenant database cloner to hang after it has successully run listDatabases
+// and recorded the results and the operationTime.
+MONGO_FAIL_POINT_DEFINE(tenantAllDatabaseClonerHangAfterGettingOperationTime);
+
+TenantAllDatabaseCloner::TenantAllDatabaseCloner(InitialSyncSharedData* sharedData,
+                                                 const HostAndPort& source,
+                                                 DBClientConnection* client,
+                                                 StorageInterface* storageInterface,
+                                                 ThreadPool* dbPool,
+                                                 StringData databasePrefix)
+    : BaseCloner(
+          "TenantAllDatabaseCloner"_sd, sharedData, source, client, storageInterface, dbPool),
+      _databasePrefix(databasePrefix),
+      _listDatabasesStage("listDatabases", this, &TenantAllDatabaseCloner::listDatabasesStage) {}
+
+BaseCloner::ClonerStages TenantAllDatabaseCloner::getStages() {
+    return {&_listDatabasesStage};
+}
+
+BaseCloner::AfterStageBehavior TenantAllDatabaseCloner::listDatabasesStage() {
+    BSONObj res;
+    const BSONObj filter = ClonerUtils::makeTenantDatabaseFilter(_databasePrefix);
+    auto databasesArray = getClient()->getDatabaseInfos(filter, true /* nameOnly */);
+
+    // Do a speculative majority read on the sync source to make sure the databases listed
+    // exist on a majority of nodes in the set. We do not check the rollbackId - rollback
+    // would lead to the sync source closing connections so the stage would fail.
+    _operationTime = getClient()->getOperationTime();
+
+    if (MONGO_unlikely(tenantAllDatabaseClonerHangAfterGettingOperationTime.shouldFail())) {
+        LOGV2(4881504,
+              "Failpoint 'tenantAllDatabaseClonerHangAfterGettingOperationTime' enabled. Blocking "
+              "until it is disabled.");
+        tenantAllDatabaseClonerHangAfterGettingOperationTime.pauseWhileSet();
+    }
+
+    BSONObj readResult;
+    BSONObj cmd = ClonerUtils::buildMajorityWaitRequest(_operationTime);
+    getClient()->runCommand("admin", cmd, readResult, QueryOption_SlaveOk);
+    uassertStatusOKWithContext(
+        getStatusFromCommandResult(readResult),
+        "TenantAllDatabaseCloner failed to get listDatabases result majority-committed");
+
+    // Process and verify the listDatabases results.
+    for (const auto& dbBSON : databasesArray) {
+        LOGV2_DEBUG(4881508, 2, "Cloner received listDatabases entry", "db"_attr = dbBSON);
+        uassert(4881505, "Result from donor must have 'name' set", dbBSON.hasField("name"));
+
+        const auto& dbName = dbBSON["name"].str();
+        _databases.emplace_back(dbName);
+    }
+
+    std::sort(_databases.begin(), _databases.end());
+    return kContinueNormally;
+}
+
+void TenantAllDatabaseCloner::postStage() {
+    {
+        stdx::lock_guard<Latch> lk(_mutex);
+        _stats.databasesCloned = 0;
+        _stats.databaseStats.reserve(_databases.size());
+        for (const auto& dbName : _databases) {
+            _stats.databaseStats.emplace_back();
+            _stats.databaseStats.back().dbname = dbName;
+        }
+    }
+    for (const auto& dbName : _databases) {
+        {
+            stdx::lock_guard<Latch> lk(_mutex);
+            _currentDatabaseCloner = std::make_unique<TenantDatabaseCloner>(dbName,
+                                                                            getSharedData(),
+                                                                            getSource(),
+                                                                            getClient(),
+                                                                            getStorageInterface(),
+                                                                            getDBPool());
+        }
+        auto dbStatus = _currentDatabaseCloner->run();
+        if (dbStatus.isOK()) {
+            LOGV2_DEBUG(4881500,
+                        1,
+                        "Tenant migration database clone finished",
+                        "dbName"_attr = dbName,
+                        "status"_attr = dbStatus);
+        } else {
+            LOGV2_WARNING(4881501,
+                          "Tenant migration database clone failed",
+                          "dbName"_attr = dbName,
+                          "dbNumber"_attr = (_stats.databasesCloned + 1),
+                          "totalDbs"_attr = _databases.size(),
+                          "error"_attr = dbStatus.toString());
+            setInitialSyncFailedStatus(dbStatus);
+            return;
+        }
+        {
+            stdx::lock_guard<Latch> lk(_mutex);
+            _stats.databaseStats[_stats.databasesCloned] = _currentDatabaseCloner->getStats();
+            _currentDatabaseCloner = nullptr;
+            _stats.databasesCloned++;
+        }
+    }
+}
+
+TenantAllDatabaseCloner::Stats TenantAllDatabaseCloner::getStats() const {
+    stdx::lock_guard<Latch> lk(_mutex);
+    TenantAllDatabaseCloner::Stats stats = _stats;
+    if (_currentDatabaseCloner) {
+        stats.databaseStats[_stats.databasesCloned] = _currentDatabaseCloner->getStats();
+    }
+    return stats;
+}
+
+std::string TenantAllDatabaseCloner::toString() const {
+    stdx::lock_guard<Latch> lk(_mutex);
+    return str::stream() << "tenant migration --"
+                         << " active:" << isActive(lk) << " status:" << getStatus(lk).toString()
+                         << " source:" << getSource()
+                         << " db cloners completed:" << _stats.databasesCloned;
+}
+
+std::string TenantAllDatabaseCloner::Stats::toString() const {
+    return toBSON().toString();
+}
+
+BSONObj TenantAllDatabaseCloner::Stats::toBSON() const {
+    BSONObjBuilder bob;
+    append(&bob);
+    return bob.obj();
+}
+
+void TenantAllDatabaseCloner::Stats::append(BSONObjBuilder* builder) const {
+    builder->appendNumber("databasesCloned", databasesCloned);
+    for (auto&& db : databaseStats) {
+        BSONObjBuilder dbBuilder(builder->subobjStart(db.dbname));
+        db.append(&dbBuilder);
+        dbBuilder.doneFast();
+    }
+}
+
+Timestamp TenantAllDatabaseCloner::getOperationTime_forTest() {
+    return _operationTime;
+}
+
+}  // namespace repl
+}  // namespace mongo
diff --git a/src/mongo/db/repl/tenant_all_database_cloner.h b/src/mongo/db/repl/tenant_all_database_cloner.h
new file mode 100644
index 00000000000..75859f95383
--- /dev/null
+++ b/src/mongo/db/repl/tenant_all_database_cloner.h
@@ -0,0 +1,125 @@
+/**
+ *    Copyright (C) 2020-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#pragma once
+
+#include <vector>
+
+#include "mongo/db/repl/base_cloner.h"
+#include "mongo/db/repl/tenant_database_cloner.h"
+
+namespace mongo {
+namespace repl {
+
+class TenantAllDatabaseCloner final : public BaseCloner {
+public:
+    struct Stats {
+        size_t databasesCloned{0};
+        std::vector<TenantDatabaseCloner::Stats> databaseStats;
+
+        std::string toString() const;
+        BSONObj toBSON() const;
+        void append(BSONObjBuilder* builder) const;
+    };
+
+    TenantAllDatabaseCloner(InitialSyncSharedData* sharedData,
+                            const HostAndPort& source,
+                            DBClientConnection* client,
+                            StorageInterface* storageInterface,
+                            ThreadPool* dbPool,
+                            StringData databasePrefix);
+
+    virtual ~TenantAllDatabaseCloner() = default;
+
+    Stats getStats() const;
+
+    std::string toString() const;
+
+    Timestamp getOperationTime_forTest();
+
+protected:
+    ClonerStages getStages() final;
+
+private:
+    friend class TenantAllDatabaseClonerTest;
+
+    class TenantAllDatabaseClonerStage : public ClonerStage<TenantAllDatabaseCloner> {
+    public:
+        TenantAllDatabaseClonerStage(std::string name,
+                                     TenantAllDatabaseCloner* cloner,
+                                     ClonerRunFn stageFunc)
+            : ClonerStage<TenantAllDatabaseCloner>(name, cloner, stageFunc) {}
+
+        bool isTransientError(const Status& status) override {
+            // Always abort on error.
+            return false;
+        }
+    };
+
+    /**
+     * Stage function that retrieves database information from the sync source.
+     */
+    AfterStageBehavior listDatabasesStage();
+
+    /**
+     *
+     * The postStage creates and runs the individual TenantDatabaseCloners on each database found on
+     * the sync source.
+     */
+    void postStage() final;
+
+    std::string describeForFuzzer(BaseClonerStage* stage) const final {
+        return "admin db: { " + stage->getName() + ": 1 }";
+    }
+
+    // All member variables are labeled with one of the following codes indicating the
+    // synchronization rules for accessing them.
+    //
+    // (R)  Read-only in concurrent operation; no synchronization required.
+    // (S)  Self-synchronizing; access according to classes own rules.
+    // (M)  Reads and writes guarded by _mutex (defined in base class).
+    // (X)  Access only allowed from the main flow of control called from run() or constructor.
+    // (MX) Write access with mutex from main flow of control, read access with mutex from other
+    //      threads, read access allowed from main flow without mutex.
+    std::vector<std::string> _databases;                           // (X)
+    std::unique_ptr<TenantDatabaseCloner> _currentDatabaseCloner;  // (MX)
+
+    // The database name prefix of the tenant associated with this migration.
+    std::string _databasePrefix;  // (R)
+
+    TenantAllDatabaseClonerStage _listDatabasesStage;  // (R)
+
+    // The operationTime returned with the listDatabases result.
+    Timestamp _operationTime;  // (X)
+
+    Stats _stats;  // (MX)
+};
+
+}  // namespace repl
+}  // namespace mongo
diff --git a/src/mongo/db/repl/tenant_all_database_cloner_test.cpp b/src/mongo/db/repl/tenant_all_database_cloner_test.cpp
new file mode 100644
index 00000000000..9ae8973c5a3
--- /dev/null
+++ b/src/mongo/db/repl/tenant_all_database_cloner_test.cpp
@@ -0,0 +1,376 @@
+/**
+ *    Copyright (C) 2020-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kTest
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/db/repl/cloner_test_fixture.h"
+#include "mongo/db/repl/storage_interface.h"
+#include "mongo/db/repl/storage_interface_mock.h"
+#include "mongo/db/repl/tenant_all_database_cloner.h"
+#include "mongo/db/service_context_test_fixture.h"
+#include "mongo/dbtests/mock/mock_dbclient_connection.h"
+#include "mongo/logv2/log.h"
+#include "mongo/unittest/unittest.h"
+#include "mongo/util/clock_source_mock.h"
+#include "mongo/util/concurrency/thread_pool.h"
+
+namespace mongo {
+namespace repl {
+
+class TenantAllDatabaseClonerTest : public ClonerTestFixture {
+public:
+    TenantAllDatabaseClonerTest() {}
+
+protected:
+    void setUp() override {
+        ClonerTestFixture::setUp();
+        _mockClient->setOperationTime(_operationTime);
+    }
+
+    std::unique_ptr<TenantAllDatabaseCloner> makeAllDatabaseCloner() {
+        return std::make_unique<TenantAllDatabaseCloner>(_sharedData.get(),
+                                                         _source,
+                                                         _mockClient.get(),
+                                                         &_storageInterface,
+                                                         _dbWorkThreadPool.get(),
+                                                         _databasePrefix);
+    }
+
+    std::vector<std::string> getDatabasesFromCloner(TenantAllDatabaseCloner* cloner) {
+        return cloner->_databases;
+    }
+
+    BSONObj createFindResponse(ErrorCodes::Error code = ErrorCodes::OK) {
+        BSONObjBuilder bob;
+        if (code != ErrorCodes::OK) {
+            bob.append("ok", 0);
+            bob.append("code", code);
+        } else {
+            bob.append("ok", 1);
+        }
+        return bob.obj();
+    }
+
+    static Timestamp _operationTime;
+    static std::string _databasePrefix;
+    static std::string _tenantDbA;
+    static std::string _tenantDbAAB;
+    static std::string _tenantDbABC;
+    static std::string _tenantDbB;
+};
+
+/* static */
+Timestamp TenantAllDatabaseClonerTest::_operationTime = Timestamp(12345, 67);
+std::string TenantAllDatabaseClonerTest::_databasePrefix = "tenant42";
+std::string TenantAllDatabaseClonerTest::_tenantDbA = _databasePrefix + "_a";
+std::string TenantAllDatabaseClonerTest::_tenantDbAAB = _databasePrefix + "_aab";
+std::string TenantAllDatabaseClonerTest::_tenantDbABC = _databasePrefix + "_abc";
+std::string TenantAllDatabaseClonerTest::_tenantDbB = _databasePrefix + "_b";
+
+TEST_F(TenantAllDatabaseClonerTest, FailsOnListDatabases) {
+    Status expectedResult{ErrorCodes::BadValue, "foo"};
+    _mockServer->setCommandReply("listDatabases", expectedResult);
+    _mockServer->setCommandReply("find", createFindResponse());
+    auto cloner = makeAllDatabaseCloner();
+
+    auto result = cloner->run();
+
+    ASSERT_EQ(result, expectedResult);
+}
+
+TEST_F(TenantAllDatabaseClonerTest, DatabasesSortedByNameOne) {
+    // Passed in as b -> aab -> abc -> a.
+    _mockServer->setCommandReply("listDatabases",
+                                 fromjson("{ok:1, databases:[{name:'" + _tenantDbB + "'}, {name:'" +
+                                          _tenantDbAAB + "'}, {name:'" + _tenantDbABC +
+                                          "'}, {name:'" + _tenantDbA + "'}]}"));
+
+    _mockServer->setCommandReply("find", createFindResponse());
+    auto cloner = makeAllDatabaseCloner();
+    cloner->setStopAfterStage_forTest("listDatabases");
+
+    ASSERT_OK(cloner->run());
+
+    auto databases = getDatabasesFromCloner(cloner.get());
+    ASSERT_EQUALS(4u, databases.size());
+    ASSERT_EQUALS(_tenantDbA, databases[0]);
+    ASSERT_EQUALS(_tenantDbAAB, databases[1]);
+    ASSERT_EQUALS(_tenantDbABC, databases[2]);
+    ASSERT_EQUALS(_tenantDbB, databases[3]);
+}
+
+TEST_F(TenantAllDatabaseClonerTest, DatabasesSortedByNameTwo) {
+    // Passed in as a -> aab -> abc -> b.
+    _mockServer->setCommandReply("listDatabases",
+                                 fromjson("{ok:1, databases:[{name:'" + _tenantDbA + "'}, {name:'" +
+                                          _tenantDbAAB + "'}, {name:'" + _tenantDbABC +
+                                          "'}, {name:'" + _tenantDbB + "'}]}"));
+
+    _mockServer->setCommandReply("find", createFindResponse());
+    auto cloner = makeAllDatabaseCloner();
+    cloner->setStopAfterStage_forTest("listDatabases");
+
+    ASSERT_OK(cloner->run());
+
+    auto databases = getDatabasesFromCloner(cloner.get());
+    ASSERT_EQUALS(4u, databases.size());
+    ASSERT_EQUALS(_tenantDbA, databases[0]);
+    ASSERT_EQUALS(_tenantDbAAB, databases[1]);
+    ASSERT_EQUALS(_tenantDbABC, databases[2]);
+    ASSERT_EQUALS(_tenantDbB, databases[3]);
+}
+
+TEST_F(TenantAllDatabaseClonerTest, DatabaseStats) {
+    _mockServer->setCommandReply("listDatabases",
+                                 fromjson("{ok:1, databases:[{name:'" + _tenantDbA + "'}, {name:'" +
+                                          _tenantDbAAB + "'}, {name:'" + _tenantDbABC + "'}]}"));
+    _mockServer->setCommandReply("find", createFindResponse());
+
+    // Make the DatabaseCloner do nothing
+    _mockServer->setCommandReply("listCollections", createCursorResponse("admin.$cmd", {}));
+    auto cloner = makeAllDatabaseCloner();
+
+    // Set up the DatabaseCloner to pause so we can check stats.
+    // We need to use two fail points to do this because fail points cannot have their data
+    // modified atomically.
+    auto dbClonerBeforeFailPoint = globalFailPointRegistry().find("hangBeforeClonerStage");
+    auto dbClonerAfterFailPoint = globalFailPointRegistry().find("hangAfterClonerStage");
+    auto timesEntered = dbClonerBeforeFailPoint->setMode(
+        FailPoint::alwaysOn,
+        0,
+        fromjson("{cloner: 'TenantDatabaseCloner', stage: 'listCollections', database: '" +
+                 _tenantDbA + "'}"));
+    dbClonerAfterFailPoint->setMode(
+        FailPoint::alwaysOn,
+        0,
+        fromjson("{cloner: 'TenantDatabaseCloner', stage: 'listCollections', database: '" +
+                 _tenantDbA + "'}"));
+
+    _clock.advance(Minutes(1));
+    // Run the cloner in a separate thread.
+    stdx::thread clonerThread([&] {
+        Client::initThread("ClonerRunner");
+        ASSERT_OK(cloner->run());
+    });
+
+    // Wait for the failpoint to be reached
+    dbClonerBeforeFailPoint->waitForTimesEntered(timesEntered + 1);
+
+    auto databases = getDatabasesFromCloner(cloner.get());
+    ASSERT_EQUALS(3u, databases.size());
+    ASSERT_EQUALS(_tenantDbA, databases[0]);
+    ASSERT_EQUALS(_tenantDbAAB, databases[1]);
+    ASSERT_EQUALS(_tenantDbABC, databases[2]);
+
+    auto stats = cloner->getStats();
+    ASSERT_EQUALS(0, stats.databasesCloned);
+    ASSERT_EQUALS(3, stats.databaseStats.size());
+    ASSERT_EQUALS(_tenantDbA, stats.databaseStats[0].dbname);
+    ASSERT_EQUALS(_tenantDbAAB, stats.databaseStats[1].dbname);
+    ASSERT_EQUALS(_tenantDbABC, stats.databaseStats[2].dbname);
+    ASSERT_EQUALS(_clock.now(), stats.databaseStats[0].start);
+    ASSERT_EQUALS(Date_t(), stats.databaseStats[0].end);
+    ASSERT_EQUALS(Date_t(), stats.databaseStats[1].start);
+    ASSERT_EQUALS(Date_t(), stats.databaseStats[1].end);
+    ASSERT_EQUALS(Date_t(), stats.databaseStats[2].start);
+    ASSERT_EQUALS(Date_t(), stats.databaseStats[2].end);
+    _clock.advance(Minutes(1));
+
+    // Allow the cloner to move to the next DB.
+    timesEntered = dbClonerBeforeFailPoint->setMode(
+        FailPoint::alwaysOn,
+        0,
+        fromjson("{cloner: 'TenantDatabaseCloner', stage: 'listCollections', database: '" +
+                 _tenantDbAAB + "'}"));
+    dbClonerAfterFailPoint->setMode(
+        FailPoint::alwaysOn,
+        0,
+        fromjson("{cloner: 'TenantDatabaseCloner', stage: 'listCollections', database: '" +
+                 _tenantDbAAB + "'}"));
+
+    // Wait for the failpoint to be reached.
+    dbClonerBeforeFailPoint->waitForTimesEntered(timesEntered + 1);
+
+    stats = cloner->getStats();
+    ASSERT_EQUALS(1, stats.databasesCloned);
+    ASSERT_EQUALS(3, stats.databaseStats.size());
+    ASSERT_EQUALS(_tenantDbA, stats.databaseStats[0].dbname);
+    ASSERT_EQUALS(_tenantDbAAB, stats.databaseStats[1].dbname);
+    ASSERT_EQUALS(_tenantDbABC, stats.databaseStats[2].dbname);
+    ASSERT_EQUALS(_clock.now(), stats.databaseStats[0].end);
+    ASSERT_EQUALS(_clock.now(), stats.databaseStats[1].start);
+    ASSERT_EQUALS(Date_t(), stats.databaseStats[1].end);
+    ASSERT_EQUALS(Date_t(), stats.databaseStats[2].start);
+    ASSERT_EQUALS(Date_t(), stats.databaseStats[2].end);
+    _clock.advance(Minutes(1));
+
+    // Allow the cloner to move to the last DB.
+    timesEntered = dbClonerBeforeFailPoint->setMode(
+        FailPoint::alwaysOn,
+        0,
+        fromjson("{cloner: 'TenantDatabaseCloner', stage: 'listCollections', database: '" +
+                 _tenantDbABC + "'}"));
+    dbClonerAfterFailPoint->setMode(
+        FailPoint::alwaysOn,
+        0,
+        fromjson("{cloner: 'TenantDatabaseCloner', stage: 'listCollections', database: '" +
+                 _tenantDbABC + "'}"));
+
+    // Wait for the failpoint to be reached.
+    dbClonerBeforeFailPoint->waitForTimesEntered(timesEntered + 1);
+
+    stats = cloner->getStats();
+    ASSERT_EQUALS(2, stats.databasesCloned);
+    ASSERT_EQUALS(3, stats.databaseStats.size());
+    ASSERT_EQUALS(_tenantDbA, stats.databaseStats[0].dbname);
+    ASSERT_EQUALS(_tenantDbAAB, stats.databaseStats[1].dbname);
+    ASSERT_EQUALS(_tenantDbABC, stats.databaseStats[2].dbname);
+    ASSERT_EQUALS(_clock.now(), stats.databaseStats[1].end);
+    ASSERT_EQUALS(_clock.now(), stats.databaseStats[2].start);
+    ASSERT_EQUALS(Date_t(), stats.databaseStats[2].end);
+    _clock.advance(Minutes(1));
+
+    // Allow the cloner to finish
+    dbClonerBeforeFailPoint->setMode(FailPoint::off, 0);
+    dbClonerAfterFailPoint->setMode(FailPoint::off, 0);
+    clonerThread.join();
+
+    stats = cloner->getStats();
+    ASSERT_EQUALS(3, stats.databasesCloned);
+    ASSERT_EQUALS(_tenantDbA, stats.databaseStats[0].dbname);
+    ASSERT_EQUALS(_tenantDbAAB, stats.databaseStats[1].dbname);
+    ASSERT_EQUALS(_tenantDbABC, stats.databaseStats[2].dbname);
+    ASSERT_EQUALS(_clock.now(), stats.databaseStats[2].end);
+}
+
+TEST_F(TenantAllDatabaseClonerTest, FailsOnListCollectionsOnOnlyDatabase) {
+    _mockServer->setCommandReply("listDatabases",
+                                 fromjson("{ok:1, databases:[{name:'" + _tenantDbA + "'}]}"));
+    _mockServer->setCommandReply("find", createFindResponse());
+    _mockServer->setCommandReply("listCollections", Status{ErrorCodes::NoSuchKey, "fake"});
+
+    auto cloner = makeAllDatabaseCloner();
+    ASSERT_NOT_OK(cloner->run());
+}
+
+TEST_F(TenantAllDatabaseClonerTest, ClonesDatabasesForTenant) {
+    auto listDatabasesReply =
+        "{ok:1, databases:[{name:'" + _tenantDbA + "'}, {name:'" + _tenantDbAAB + "'}]}";
+    _mockServer->setCommandReply("listDatabases", fromjson(listDatabasesReply));
+    _mockServer->setCommandReply("find", createFindResponse());
+
+    auto cloner = makeAllDatabaseCloner();
+    cloner->setStopAfterStage_forTest("listDatabases");
+
+    ASSERT_OK(cloner->run());
+
+    auto databases = getDatabasesFromCloner(cloner.get());
+    ASSERT_EQUALS(2u, databases.size());
+    ASSERT_EQUALS(_tenantDbA, databases[0]);
+    ASSERT_EQUALS(_tenantDbAAB, databases[1]);
+}
+
+TEST_F(TenantAllDatabaseClonerTest, ListDatabasesMajorityReadFailsWithSpecificError) {
+    auto listDatabasesReply =
+        "{ok:1, databases:[{name:'" + _tenantDbA + "'}, {name:'" + _tenantDbAAB + "'}]}";
+    _mockServer->setCommandReply("listDatabases", fromjson(listDatabasesReply));
+    _mockServer->setCommandReply("find", createFindResponse(ErrorCodes::OperationFailed));
+
+    auto cloner = makeAllDatabaseCloner();
+    cloner->setStopAfterStage_forTest("listDatabases");
+
+    auto status = cloner->run();
+    ASSERT_NOT_OK(status);
+    ASSERT_EQUALS(ErrorCodes::OperationFailed, status.code());
+}
+
+TEST_F(TenantAllDatabaseClonerTest, ListCollectionsRemoteUnreachableBeforeMajorityFind) {
+    auto listDatabasesReply =
+        "{ok:1, databases:[{name:'" + _tenantDbA + "'}, {name:'" + _tenantDbAAB + "'}]}";
+    _mockServer->setCommandReply("listDatabases", fromjson(listDatabasesReply));
+
+    auto cloner = makeAllDatabaseCloner();
+    cloner->setStopAfterStage_forTest("listDatabases");
+
+    auto clonerOperationTimeFP =
+        globalFailPointRegistry().find("tenantAllDatabaseClonerHangAfterGettingOperationTime");
+    auto timesEntered = clonerOperationTimeFP->setMode(FailPoint::alwaysOn, 0);
+
+    // Run the cloner in a separate thread.
+    stdx::thread clonerThread([&] {
+        Client::initThread("ClonerRunner");
+        ASSERT_NOT_OK(cloner->run());
+    });
+
+    // Wait for the failpoint to be reached
+    clonerOperationTimeFP->waitForTimesEntered(timesEntered + 1);
+    _mockServer->shutdown();
+
+    // Finish test
+    clonerOperationTimeFP->setMode(FailPoint::off, 0);
+    clonerThread.join();
+}
+
+TEST_F(TenantAllDatabaseClonerTest, ListDatabasesRecordsCorrectOperationTime) {
+    auto listDatabasesReply =
+        "{ok:1, databases:[{name:'" + _tenantDbA + "'}, {name:'" + _tenantDbAAB + "'}]}";
+    _mockServer->setCommandReply("listDatabases", fromjson(listDatabasesReply));
+    _mockServer->setCommandReply("find", createFindResponse());
+
+    auto clonerOperationTimeFP =
+        globalFailPointRegistry().find("tenantAllDatabaseClonerHangAfterGettingOperationTime");
+    auto timesEntered = clonerOperationTimeFP->setMode(FailPoint::alwaysOn, 0);
+
+    auto cloner = makeAllDatabaseCloner();
+    cloner->setStopAfterStage_forTest("listDatabases");
+
+    // Run the cloner in a separate thread.
+    stdx::thread clonerThread([&] {
+        Client::initThread("ClonerRunner");
+        ASSERT_OK(cloner->run());
+    });
+
+    // Wait for the failpoint to be reached
+    clonerOperationTimeFP->waitForTimesEntered(timesEntered + 1);
+    ASSERT_EQUALS(_operationTime, cloner->getOperationTime_forTest());
+
+    // Finish test
+    clonerOperationTimeFP->setMode(FailPoint::off, 0);
+    clonerThread.join();
+
+    auto databases = getDatabasesFromCloner(cloner.get());
+    ASSERT_EQUALS(2u, databases.size());
+    ASSERT_EQUALS(_tenantDbA, databases[0]);
+    ASSERT_EQUALS(_tenantDbAAB, databases[1]);
+}
+
+}  // namespace repl
+}  // namespace mongo
\ No newline at end of file
diff --git a/src/mongo/db/repl/tenant_database_cloner.cpp b/src/mongo/db/repl/tenant_database_cloner.cpp
index 1513a7186b3..8baee18daca 100644
--- a/src/mongo/db/repl/tenant_database_cloner.cpp
+++ b/src/mongo/db/repl/tenant_database_cloner.cpp
@@ -33,8 +33,8 @@
 
 #include "mongo/base/string_data.h"
 #include "mongo/db/commands/list_collections_filter.h"
+#include "mongo/db/repl/cloner_utils.h"
 #include "mongo/db/repl/database_cloner_gen.h"
-#include "mongo/db/repl/read_concern_args.h"
 #include "mongo/db/repl/tenant_collection_cloner.h"
 #include "mongo/db/repl/tenant_database_cloner.h"
 #include "mongo/logv2/log.h"
@@ -70,16 +70,6 @@ void TenantDatabaseCloner::preStage() {
     _stats.start = getSharedData()->getClock()->now();
 }
 
-/* static */
-BSONObj TenantDatabaseCloner::buildMajorityWaitRequest(Timestamp operationTime) {
-    BSONObjBuilder bob;
-    bob.append("find", NamespaceString::kSystemReplSetNamespace.toString());
-    bob.append("filter", BSONObj());
-    ReadConcernArgs readConcern(LogicalTime(operationTime), ReadConcernLevel::kMajorityReadConcern);
-    readConcern.appendInfo(&bob);
-    return bob.obj();
-}
-
 BaseCloner::AfterStageBehavior TenantDatabaseCloner::listCollectionsStage() {
     // This will be set after a successful listCollections command.
     _operationTime = Timestamp(0, 0);
@@ -111,7 +101,7 @@ BaseCloner::AfterStageBehavior TenantDatabaseCloner::listCollectionsStage() {
         });
 
     BSONObj readResult;
-    BSONObj cmd = TenantDatabaseCloner::buildMajorityWaitRequest(_operationTime);
+    BSONObj cmd = ClonerUtils::buildMajorityWaitRequest(_operationTime);
     getClient()->runCommand("admin", cmd, readResult, QueryOption_SlaveOk);
     uassertStatusOKWithContext(
         getStatusFromCommandResult(readResult),
diff --git a/src/mongo/db/repl/tenant_database_cloner.h b/src/mongo/db/repl/tenant_database_cloner.h
index fa7a860e400..1684231ab0d 100644
--- a/src/mongo/db/repl/tenant_database_cloner.h
+++ b/src/mongo/db/repl/tenant_database_cloner.h
@@ -65,8 +65,6 @@ public:
 
     std::string toString() const;
 
-    static BSONObj buildMajorityWaitRequest(Timestamp operationTime);
-
     Timestamp getOperationTime_forTest();
 
 protected: