SERVER-25593: check shutdown/failure status during initial sync when acquiring lock.

4 files changed, 37 insertions, 3 deletions

diff --git a/src/mongo/db/repl/data_replicator.cpp b/src/mongo/db/repl/data_replicator.cpp
index 038a0d31b71..99766b9dc65 100644
--- a/src/mongo/db/repl/data_replicator.cpp
+++ b/src/mongo/db/repl/data_replicator.cpp
@@ -658,12 +658,20 @@ Status DataReplicator::_runInitialSyncAttempt_inlock(OperationContext* txn,
 
 StatusWith<OpTimeWithHash> DataReplicator::doInitialSync(OperationContext* txn,
                                                          std::size_t maxAttempts) {
+    const Status shutdownStatus{ErrorCodes::ShutdownInProgress,
+                                "Shutting down while in doInitialSync."};
     if (!txn) {
         std::string msg = "Initial Sync attempted but no OperationContext*, so aborting.";
         error() << msg;
         return Status{ErrorCodes::InitialSyncFailure, msg};
     }
     UniqueLock lk(_mutex);
+    if (_inShutdown || (_initialSyncState && !_initialSyncState->status.isOK())) {
+        const auto retStatus = (_initialSyncState && !_initialSyncState->status.isOK())
+            ? _initialSyncState->status
+            : shutdownStatus;
+        return retStatus;
+    }
     _stats.initialSyncStart = _exec->now();
     if (_state != DataReplicatorState::Uninitialized) {
         if (_state == DataReplicatorState::InitialSync)
@@ -688,6 +696,10 @@ StatusWith<OpTimeWithHash> DataReplicator::doInitialSync(OperationContext* txn,
     _stats.maxFailedInitialSyncAttempts = maxAttempts;
     _stats.failedInitialSyncAttempts = 0;
     while (_stats.failedInitialSyncAttempts < _stats.maxFailedInitialSyncAttempts) {
+        if (_inShutdown) {
+            return shutdownStatus;
+        }
+
         Status attemptErrorStatus(Status::OK());
 
         ON_BLOCK_EXIT([this, txn, &lk, &attemptErrorStatus]() {
@@ -766,6 +778,12 @@ StatusWith<OpTimeWithHash> DataReplicator::doInitialSync(OperationContext* txn,
                 lk.lock();
             }
         }
+        if (_inShutdown) {
+            const auto retStatus = (_initialSyncState && !_initialSyncState->status.isOK())
+                ? _initialSyncState->status
+                : shutdownStatus;
+            return retStatus;
+        }
 
         // Cleanup
         _cancelAllHandles_inlock();
@@ -1388,6 +1406,7 @@ Status DataReplicator::scheduleShutdown(OperationContext* txn) {
     {
         LockGuard lk(_mutex);
         invariant(!_onShutdown.isValid());
+        _inShutdown = true;
         _onShutdown = eventStatus.getValue();
         if (_initialSyncState) {
             _initialSyncState->status = {ErrorCodes::ShutdownInProgress,
@@ -1435,6 +1454,12 @@ void DataReplicator::_enqueueDocuments(Fetcher::Documents::const_iterator begin,
         return;
     }
 
+    {
+        LockGuard lk{_mutex};
+        if (_inShutdown) {
+            return;
+        }
+    }
     invariant(_oplogBuffer);
 
     // Wait for enough space.
diff --git a/src/mongo/db/repl/data_replicator.h b/src/mongo/db/repl/data_replicator.h
index b41de048043..8bbb4807960 100644
--- a/src/mongo/db/repl/data_replicator.h
+++ b/src/mongo/db/repl/data_replicator.h
@@ -370,6 +370,7 @@ private:
     OpTimeWithHash _lastFetched;                                                // (MX)
     OpTimeWithHash _lastApplied;                                                // (MX)
     std::unique_ptr<OplogBuffer> _oplogBuffer;                                  // (M)
+    bool _inShutdown = false;                                                   // (M)
     Event _onShutdown;                                                          // (M)
     Timestamp _rollbackCommonOptime;                                            // (MX)
     CollectionCloner::ScheduleDbWorkFn _scheduleDbWorkFn;                       // (M)
diff --git a/src/mongo/db/repl/data_replicator_test.cpp b/src/mongo/db/repl/data_replicator_test.cpp
index 4fe4426b60a..a855de28c71 100644
--- a/src/mongo/db/repl/data_replicator_test.cpp
+++ b/src/mongo/db/repl/data_replicator_test.cpp
@@ -649,7 +649,8 @@ protected:
 
     void verifySync(NetworkInterfaceMock* net, ErrorCodes::Error code) {
         // Check result
-        ASSERT_EQ(_isbr->getResult(net).getStatus().code(), code) << "status codes differ";
+        const auto status = _isbr->getResult(net).getStatus();
+        ASSERT_EQ(status.code(), code) << "status codes differ, status: " << status;
     }
 
     BSONObj getInitialSyncProgress() {
@@ -1232,7 +1233,8 @@ TEST_F(InitialSyncTest, InitialSyncStateIsResetAfterFailure) {
     log() << "done playing first response of second round of responses";
 
     auto dr = &getDR();
-    ASSERT_TRUE(dr->getState() == DataReplicatorState::InitialSync);
+    ASSERT_TRUE(dr->getState() == DataReplicatorState::InitialSync) << ", state: "
+                                                                    << dr->getDiagnosticString();
     ASSERT_EQUALS(dr->getLastFetched(), OpTimeWithHash());
     ASSERT_EQUALS(dr->getLastApplied(), OpTimeWithHash());
 
diff --git a/src/mongo/db/repl/replication_coordinator_impl.cpp b/src/mongo/db/repl/replication_coordinator_impl.cpp
index dd8a57cd60f..81bc5a0f32d 100644
--- a/src/mongo/db/repl/replication_coordinator_impl.cpp
+++ b/src/mongo/db/repl/replication_coordinator_impl.cpp
@@ -620,7 +620,13 @@ void ReplicationCoordinatorImpl::_startDataReplication(OperationContext* txn,
 
             drCopy.reset();
             lk.lock();
-            if (!_inShutdown) {
+
+            if (ErrorCodes::CallbackCanceled == status ||
+                ErrorCodes::ShutdownInProgress == status) {
+
+                log() << "Initial Sync has been cancelled: " << status.getStatus();
+                return;
+            } else if (!_inShutdown) {
                 fassertNoTrace(40088, status.getStatus());
             } else if (!status.isOK()) {
                 log() << "Initial Sync failed during shutdown due to " << status.getStatus();