SERVER-25966 added CollectionCloner tests for metadata retries

4 files changed, 158 insertions, 0 deletions

diff --git a/src/mongo/db/repl/collection_cloner_test.cpp b/src/mongo/db/repl/collection_cloner_test.cpp
index 51c063c24b8..7153497499f 100644
--- a/src/mongo/db/repl/collection_cloner_test.cpp
+++ b/src/mongo/db/repl/collection_cloner_test.cpp
@@ -427,6 +427,30 @@ TEST_F(CollectionClonerTest, ListIndexesReturnedNamespaceNotFound) {
     ASSERT_EQ(collNss, nss);
 }
 
+
+TEST_F(CollectionClonerTest, CollectionClonerResendsListIndexesCommandOnRetriableError) {
+    ASSERT_OK(collectionCloner->startup());
+
+    auto net = getNet();
+    executor::NetworkInterfaceMock::InNetworkGuard guard(net);
+
+    // First request sent by CollectionCloner. CollectionCollection sends listIndexes request
+    // irrespective of collection size in a successful count response.
+    assertRemoteCommandNameEquals("count", net->scheduleSuccessfulResponse(createCountResponse(0)));
+    net->runReadyNetworkOperations();
+
+    // Respond to first listIndexes request with a retriable error.
+    assertRemoteCommandNameEquals("listIndexes",
+                                  net->scheduleErrorResponse(Status(ErrorCodes::HostNotFound, "")));
+    net->runReadyNetworkOperations();
+    ASSERT_TRUE(collectionCloner->isActive());
+
+    // Confirm that CollectionCloner resends the listIndexes request.
+    auto noi = net->getNextReadyRequest();
+    assertRemoteCommandNameEquals("listIndexes", noi->getRequest());
+    net->blackHole(noi);
+}
+
 TEST_F(CollectionClonerTest,
        ListIndexesReturnedNamespaceNotFoundAndCreateCollectionCallbackCanceled) {
     ASSERT_OK(collectionCloner->startup());
@@ -689,6 +713,36 @@ TEST_F(CollectionClonerTest, FindCommandFailed) {
     ASSERT_FALSE(collectionCloner->isActive());
 }
 
+TEST_F(CollectionClonerTest, CollectionClonerResendsFindCommandOnRetriableError) {
+    ASSERT_OK(collectionCloner->startup());
+
+    auto net = getNet();
+    executor::NetworkInterfaceMock::InNetworkGuard guard(net);
+
+    // CollectionCollection sends listIndexes request irrespective of collection size in a
+    // successful count response.
+    assertRemoteCommandNameEquals("count", net->scheduleSuccessfulResponse(createCountResponse(0)));
+    net->runReadyNetworkOperations();
+
+    // CollectionCloner requires a successful listIndexes response in order to send the find request
+    // for the documents in the collection.
+    assertRemoteCommandNameEquals(
+        "listIndexes",
+        net->scheduleSuccessfulResponse(createListIndexesResponse(0, BSON_ARRAY(idIndexSpec))));
+    net->runReadyNetworkOperations();
+
+    // Respond to the find request with a retriable error.
+    assertRemoteCommandNameEquals("find",
+                                  net->scheduleErrorResponse(Status(ErrorCodes::HostNotFound, "")));
+    net->runReadyNetworkOperations();
+    ASSERT_TRUE(collectionCloner->isActive());
+
+    // Confirm that CollectionCloner resends the find request.
+    auto noi = net->getNextReadyRequest();
+    assertRemoteCommandNameEquals("find", noi->getRequest());
+    net->blackHole(noi);
+}
+
 TEST_F(CollectionClonerTest, FindCommandCanceled) {
     ASSERT_OK(collectionCloner->startup());
 
diff --git a/src/mongo/db/repl/data_replicator_test.cpp b/src/mongo/db/repl/data_replicator_test.cpp
index 524c95125b7..7626ac9824c 100644
--- a/src/mongo/db/repl/data_replicator_test.cpp
+++ b/src/mongo/db/repl/data_replicator_test.cpp
@@ -643,6 +643,34 @@ TEST_F(DataReplicatorTest, DataReplicatorReturnsCallbackCanceledIfShutdownImmedi
     ASSERT_EQUALS(ErrorCodes::CallbackCanceled, _lastApplied);
 }
 
+TEST_F(DataReplicatorTest,
+       DataReplicatorRetriesSyncSourceSelectionIfChooseNewSyncSourceReturnsInvalidSyncSource) {
+    auto dr = &getDR();
+    auto txn = makeOpCtx();
+
+    // Override chooseNewSyncSource() result in SyncSourceSelectorMock before calling startup()
+    // because DataReplicator will look for a valid sync source immediately after startup.
+    _syncSourceSelector->setChooseNewSyncSourceResult_forTest(HostAndPort());
+
+    ASSERT_OK(dr->startup(txn.get(), maxAttempts));
+
+    // Run first sync source selection attempt.
+    executor::NetworkInterfaceMock::InNetworkGuard(getNet())->runReadyNetworkOperations();
+
+    // DataReplicator will not drop user databases while looking for a valid sync source.
+    ASSERT_FALSE(_storageInterfaceWorkDone.droppedUserDBs);
+
+    // First sync source selection attempt failed. Update SyncSourceSelectorMock to return valid
+    // sync source next time chooseNewSyncSource() is called.
+    _syncSourceSelector->setChooseNewSyncSourceResult_forTest(HostAndPort("localhost", 12345));
+
+    // Advance clock until the next sync source selection attempt.
+    advanceClock(getNet(), _options.syncSourceRetryWait);
+
+    // DataReplictor drops user databases after obtaining a valid sync source.
+    ASSERT_TRUE(_storageInterfaceWorkDone.droppedUserDBs);
+}
+
 const std::uint32_t chooseSyncSourceMaxAttempts = 10U;
 
 /**
@@ -1086,6 +1114,33 @@ TEST_F(DataReplicatorTest,
 }
 
 TEST_F(DataReplicatorTest,
+       DataReplicatorResendsFindCommandIfLastOplogEntryFetcherReturnsRetriableError) {
+    auto dr = &getDR();
+    auto txn = makeOpCtx();
+
+    _syncSourceSelector->setChooseNewSyncSourceResult_forTest(HostAndPort("localhost", 12345));
+    ASSERT_OK(dr->startup(txn.get(), maxAttempts));
+
+    auto net = getNet();
+    executor::NetworkInterfaceMock::InNetworkGuard guard(net);
+
+    // Base rollback ID.
+    net->scheduleSuccessfulResponse(makeRollbackCheckerResponse(1));
+    net->runReadyNetworkOperations();
+
+    // Last oplog entry first attempt - retriable error.
+    assertRemoteCommandNameEquals("find",
+                                  net->scheduleErrorResponse(Status(ErrorCodes::HostNotFound, "")));
+    net->runReadyNetworkOperations();
+
+    // DataReplicator stays active because it resends the find request for the last oplog entry.
+    ASSERT_TRUE(dr->isActive());
+
+    // Last oplog entry second attempt.
+    processSuccessfulLastOplogEntryFetcherResponse({makeOplogEntry(1)});
+}
+
+TEST_F(DataReplicatorTest,
        DataReplicatorReturnsNoSuchKeyIfLastOplogEntryFetcherReturnsEntryWithMissingHash) {
     auto dr = &getDR();
     auto txn = makeOpCtx();
diff --git a/src/mongo/db/repl/database_cloner_test.cpp b/src/mongo/db/repl/database_cloner_test.cpp
index e5cb2b464e2..c2e13b2e8a9 100644
--- a/src/mongo/db/repl/database_cloner_test.cpp
+++ b/src/mongo/db/repl/database_cloner_test.cpp
@@ -467,6 +467,25 @@ TEST_F(DatabaseClonerTest, InvalidCollectionOptions) {
     ASSERT_FALSE(_databaseCloner->isActive());
 }
 
+TEST_F(DatabaseClonerTest, DatabaseClonerResendsListCollectionsRequestOnRetriableError) {
+    ASSERT_OK(_databaseCloner->startup());
+    auto net = getNet();
+    executor::NetworkInterfaceMock::InNetworkGuard guard(net);
+
+    // Respond to first listCollections request with a retriable error.
+    assertRemoteCommandNameEquals("listCollections",
+                                  net->scheduleErrorResponse(Status(ErrorCodes::HostNotFound, "")));
+    net->runReadyNetworkOperations();
+
+    // DatabaseCloner stays active because it resends the listCollections request.
+    ASSERT_TRUE(_databaseCloner->isActive());
+
+    // DatabaseCloner should resend listCollections request.
+    auto noi = net->getNextReadyRequest();
+    assertRemoteCommandNameEquals("listCollections", noi->getRequest());
+    net->blackHole(noi);
+}
+
 TEST_F(DatabaseClonerTest, ListCollectionsReturnsEmptyCollectionName) {
     _databaseCloner.reset(new DatabaseCloner(
         &getExecutor(),
diff --git a/src/mongo/db/repl/databases_cloner_test.cpp b/src/mongo/db/repl/databases_cloner_test.cpp
index 81552201485..366e8c04a7c 100644
--- a/src/mongo/db/repl/databases_cloner_test.cpp
+++ b/src/mongo/db/repl/databases_cloner_test.cpp
@@ -472,6 +472,36 @@ TEST_F(DBsClonerTest, FailsOnListDatabases) {
     ASSERT_EQ(result, expectedResult);
 }
 
+TEST_F(DBsClonerTest, DatabasesClonerResendsListDatabasesRequestOnRetriableError) {
+    Status result{Status::OK()};
+    DatabasesCloner cloner{&getStorage(),
+                           &getExecutor(),
+                           &getDbWorkThreadPool(),
+                           HostAndPort{"local:1234"},
+                           [](const BSONObj&) { return true; },
+                           [](const Status&) {}};
+    ON_BLOCK_EXIT([this] { getExecutor().shutdown(); });
+
+    ASSERT_OK(cloner.startup());
+    ASSERT_TRUE(cloner.isActive());
+
+    auto net = getNet();
+    executor::NetworkInterfaceMock::InNetworkGuard guard(net);
+
+    // Respond to first listDatabases request with a retriable error.
+    assertRemoteCommandNameEquals("listDatabases",
+                                  net->scheduleErrorResponse(Status(ErrorCodes::HostNotFound, "")));
+    net->runReadyNetworkOperations();
+
+    // DatabasesCloner stays active because it resends the listDatabases request.
+    ASSERT_TRUE(cloner.isActive());
+
+    // DatabasesCloner should resend listDatabases request.
+    auto noi = net->getNextReadyRequest();
+    assertRemoteCommandNameEquals("listDatabases", noi->getRequest());
+    net->blackHole(noi);
+}
+
 TEST_F(DBsClonerTest, DatabasesClonerReturnsCallbackCanceledIfShutdownDuringListDatabasesCommand) {
     Status result{Status::OK()};
     DatabasesCloner cloner{&getStorage(),