SERVER-50140: Initial sync cannot survive unclean restart of the sync source

(cherry picked from commit 06169f718a7aec04f952979ffa6590e4334dea5a)
(cherry picked from commit d598744856190df68f36c3f7a88decd30fa8e912)

17 files changed, 161 insertions, 19 deletions

diff --git a/buildscripts/resmokeconfig/suites/replica_sets_multiversion.yml b/buildscripts/resmokeconfig/suites/replica_sets_multiversion.yml
index 555630879b9..8bef7eb96c9 100644
--- a/buildscripts/resmokeconfig/suites/replica_sets_multiversion.yml
+++ b/buildscripts/resmokeconfig/suites/replica_sets_multiversion.yml
@@ -8,6 +8,7 @@ selector:
   - requires_fcv_44
   exclude_files:
   - jstests/replsets/apply_prepare_txn_write_conflict_robustness.js
+  - jstests/replsets/initial_sync_fails_unclean_restart.js
 executor:
   config:
     shell_options:
diff --git a/etc/backports_required_for_multiversion_tests.yml b/etc/backports_required_for_multiversion_tests.yml
index 2433f07a7a0..97e8cd6134e 100644
--- a/etc/backports_required_for_multiversion_tests.yml
+++ b/etc/backports_required_for_multiversion_tests.yml
@@ -75,6 +75,8 @@ replica_sets_multiversion:
   test_file: jstests/replsets/disallow_adding_initialized_node1.js
 - ticket: SERVER-49471
   test_file: jstests/replsets/apply_prepare_txn_write_conflict_robustness.js
+- ticket: SERVER-50140
+  test_file: jstests/replsets/initial_sync_fails_unclean_restart.js
 
 sharding_multiversion:
 - ticket: SERVER-38691
diff --git a/jstests/replsets/initial_sync_fails_unclean_restart.js b/jstests/replsets/initial_sync_fails_unclean_restart.js
new file mode 100644
index 00000000000..020e2097b0f
--- /dev/null
+++ b/jstests/replsets/initial_sync_fails_unclean_restart.js
@@ -0,0 +1,104 @@
+/**
+ * Tests that initial sync will abort an attempt if the sync source restarts from an unclean
+ * shutdown. And the sync source node increments its rollback id after the unclean shutdown.
+ *
+ * This is to test resumable initial sync behavior when the sync source restarts after an unclean
+ * shutdown. See SERVER-50140 for more details.
+ * @tags: [requires_persistence]
+ */
+(function() {
+"use strict";
+
+load("jstests/libs/fail_point_util.js");
+load("jstests/replsets/rslib.js");
+
+const dbName = "test";
+const collName = "coll";
+
+const rst = new ReplSetTest({nodes: 1});
+rst.startSet();
+rst.initiate();
+
+let syncSourceNode = rst.getPrimary();
+const syncSourceColl = syncSourceNode.getDB(dbName)[collName];
+
+// Insert some initial data to be cloned.
+assert.commandWorked(syncSourceColl.insert([{_id: 1}, {_id: 2}, {_id: 3}]));
+
+jsTest.log("Adding a new node to the replica set");
+const initialSyncNode = rst.add({
+    rsConfig: {priority: 0, votes: 0},
+    setParameter: {
+        'failpoint.initialSyncHangBeforeCopyingDatabases': tojson({mode: 'alwaysOn'}),
+        // Wait for the cloners to finish.
+        'failpoint.initialSyncHangAfterDataCloning': tojson({mode: 'alwaysOn'}),
+        'numInitialSyncAttempts': 1,
+    }
+});
+rst.reInitiate();
+
+jsTestLog("The initialSyncNode should hang before the database cloning phase");
+checkLog.contains(initialSyncNode, "initialSyncHangBeforeCopyingDatabases fail point enabled");
+
+// Pauses the journal flusher and writes with {j: false}. So this data will be lost after the
+// syncSourceNode restarts after an unclean shutdown.
+const journalFp = configureFailPoint(syncSourceNode, "pauseJournalFlusherThread");
+journalFp.wait();
+assert.commandWorked(syncSourceColl.insert({_id: 4}));
+
+// Hang the initialSyncNode before initial sync finishes so we can check initial sync failure.
+const beforeFinishFailPoint = configureFailPoint(initialSyncNode, "initialSyncHangBeforeFinish");
+
+jsTestLog("Resuming database cloner on the initialSyncNode");
+assert.commandWorked(initialSyncNode.adminCommand(
+    {configureFailPoint: 'initialSyncHangBeforeCopyingDatabases', mode: 'off'}));
+
+jsTestLog("Waiting for data cloning to complete on the initialSyncNode");
+assert.commandWorked(initialSyncNode.adminCommand({
+    waitForFailPoint: "initialSyncHangAfterDataCloning",
+    timesEntered: 1,
+    maxTimeMS: kDefaultWaitForFailPointTimeout
+}));
+
+// Get the rollback id of the sync source before the unclean shutdown.
+const rollbackIdBefore = syncSourceNode.getDB("local").system.rollback.id.findOne();
+
+jsTestLog("Shutting down the syncSourceNode uncleanly");
+rst.stop(syncSourceNode,
+         9,
+         {allowedExitCode: MongoRunner.EXIT_SIGKILL},
+         {forRestart: true, waitPid: true});
+
+// Make sure some retries happen due to resumable initial sync and the initial sync does not
+// immediately fail while the sync source is completely down.
+const nRetries = 2;
+checkLog.containsWithAtLeastCount(initialSyncNode, "Trying to reconnect", nRetries);
+
+// Restart the sync source and wait for it to become primary again.
+jsTestLog("Restarting the syncSourceNode");
+rst.start(syncSourceNode, {waitForConnect: true}, true /* restart */);
+syncSourceNode = rst.getPrimary();
+
+// Test that the rollback id is incremented after the unclean shutdown.
+const rollbackIdAfter = syncSourceNode.getDB("local").system.rollback.id.findOne();
+assert.eq(rollbackIdAfter.rollbackId,
+          rollbackIdBefore.rollbackId + 1,
+          () => "rollbackIdBefore: " + tojson(rollbackIdBefore) +
+              " rollbackIdAfter: " + tojson(rollbackIdAfter));
+
+jsTestLog("Resuming initial sync after the data cloning phase on the initialSyncNode");
+assert.commandWorked(initialSyncNode.adminCommand(
+    {configureFailPoint: "initialSyncHangAfterDataCloning", mode: "off"}));
+
+jsTestLog("Waiting for initial sync to fail on the initialSyncNode");
+beforeFinishFailPoint.wait();
+const res = assert.commandWorked(initialSyncNode.adminCommand({replSetGetStatus: 1}));
+// The initial sync should have failed.
+assert.eq(res.initialSyncStatus.failedInitialSyncAttempts, 1, () => tojson(res.initialSyncStatus));
+
+// Get rid of the failed node so the fixture can stop properly.
+rst.stop(initialSyncNode);
+rst.remove(initialSyncNode);
+
+rst.stopSet();
+})();
diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp
index 0033c9e7c1d..83c6667d003 100644
--- a/src/mongo/db/db.cpp
+++ b/src/mongo/db/db.cpp
@@ -356,7 +356,8 @@ ExitCode _initAndListen(ServiceContext* serviceContext, int listenPort) {
                      std::make_unique<FlowControl>(
                          serviceContext, repl::ReplicationCoordinator::get(serviceContext)));
 
-    initializeStorageEngine(serviceContext, StorageEngineInitFlags::kNone);
+    auto lastStorageEngineShutdownState =
+        initializeStorageEngine(serviceContext, StorageEngineInitFlags::kNone);
 
 #ifdef MONGO_CONFIG_WIREDTIGER_ENABLED
     if (EncryptionHooks::get(serviceContext)->restartRequired()) {
@@ -603,7 +604,7 @@ ExitCode _initAndListen(ServiceContext* serviceContext, int listenPort) {
         uassert(ErrorCodes::BadValue,
                 str::stream() << "Cannot use queryableBackupMode in a replica set",
                 !replCoord->isReplEnabled());
-        replCoord->startup(startupOpCtx.get());
+        replCoord->startup(startupOpCtx.get(), lastStorageEngineShutdownState);
     }
 
     if (!storageGlobalParams.readOnly) {
@@ -656,7 +657,7 @@ ExitCode _initAndListen(ServiceContext* serviceContext, int listenPort) {
             ReplicaSetNodeProcessInterface::getReplicaSetNodeExecutor(serviceContext)->startup();
         }
 
-        replCoord->startup(startupOpCtx.get());
+        replCoord->startup(startupOpCtx.get(), lastStorageEngineShutdownState);
         if (getReplSetMemberInStandaloneMode(serviceContext)) {
             LOGV2_WARNING_OPTIONS(
                 20547,
diff --git a/src/mongo/db/repl/replication_coordinator.h b/src/mongo/db/repl/replication_coordinator.h
index 314ae8ccd4a..cec8785f9e8 100644
--- a/src/mongo/db/repl/replication_coordinator.h
+++ b/src/mongo/db/repl/replication_coordinator.h
@@ -41,6 +41,7 @@
 #include "mongo/db/repl/repl_settings.h"
 #include "mongo/db/repl/split_horizon.h"
 #include "mongo/db/repl/sync_source_selector.h"
+#include "mongo/db/storage/storage_engine_init.h"
 #include "mongo/executor/task_executor.h"
 #include "mongo/rpc/topology_version_gen.h"
 #include "mongo/util/net/hostandport.h"
@@ -115,7 +116,8 @@ public:
      * components of the replication system to start up whatever threads and do whatever
      * initialization they need.
      */
-    virtual void startup(OperationContext* opCtx) = 0;
+    virtual void startup(OperationContext* opCtx,
+                         LastStorageEngineShutdownState lastStorageEngineShutdownState) = 0;
 
     /**
      * Start terminal shutdown.  This causes the topology coordinator to refuse to vote in any
diff --git a/src/mongo/db/repl/replication_coordinator_impl.cpp b/src/mongo/db/repl/replication_coordinator_impl.cpp
index 7fffcffaa17..2a819414283 100644
--- a/src/mongo/db/repl/replication_coordinator_impl.cpp
+++ b/src/mongo/db/repl/replication_coordinator_impl.cpp
@@ -406,7 +406,8 @@ void ReplicationCoordinatorImpl::appendConnectionStats(executor::ConnectionPoolS
     _replExecutor->appendConnectionStats(stats);
 }
 
-bool ReplicationCoordinatorImpl::_startLoadLocalConfig(OperationContext* opCtx) {
+bool ReplicationCoordinatorImpl::_startLoadLocalConfig(
+    OperationContext* opCtx, LastStorageEngineShutdownState lastStorageEngineShutdownState) {
     // Create necessary replication collections to guarantee that if a checkpoint sees data after
     // initial sync has completed, it also sees these collections.
     fassert(50708, _replicationProcess->getConsistencyMarkers()->createInternalCollections(opCtx));
@@ -448,6 +449,9 @@ bool ReplicationCoordinatorImpl::_startLoadLocalConfig(OperationContext* opCtx)
                                 "Error loading local Rollback ID document at startup",
                                 "error"_attr = status);
         }
+    } else if (lastStorageEngineShutdownState == LastStorageEngineShutdownState::kUnclean) {
+        LOGV2(501401, "Incrementing the rollback ID after unclean shutdown");
+        fassert(501402, _replicationProcess->incrementRollbackID(opCtx));
     }
 
     StatusWith<BSONObj> cfg = _externalState->loadLocalConfigDocument(opCtx);
@@ -807,7 +811,8 @@ void ReplicationCoordinatorImpl::_startDataReplication(OperationContext* opCtx,
     }
 }
 
-void ReplicationCoordinatorImpl::startup(OperationContext* opCtx) {
+void ReplicationCoordinatorImpl::startup(
+    OperationContext* opCtx, LastStorageEngineShutdownState lastStorageEngineShutdownState) {
     if (!isReplEnabled()) {
         if (ReplSettings::shouldRecoverFromOplogAsStandalone()) {
             uassert(ErrorCodes::InvalidOptions,
@@ -866,7 +871,7 @@ void ReplicationCoordinatorImpl::startup(OperationContext* opCtx) {
 
     _replExecutor->startup();
 
-    bool doneLoadingConfig = _startLoadLocalConfig(opCtx);
+    bool doneLoadingConfig = _startLoadLocalConfig(opCtx, lastStorageEngineShutdownState);
     if (doneLoadingConfig) {
         // If we're not done loading the config, then the config state will be set by
         // _finishLoadLocalConfig.
diff --git a/src/mongo/db/repl/replication_coordinator_impl.h b/src/mongo/db/repl/replication_coordinator_impl.h
index 8bbb4df7400..bde6f3136b8 100644
--- a/src/mongo/db/repl/replication_coordinator_impl.h
+++ b/src/mongo/db/repl/replication_coordinator_impl.h
@@ -102,7 +102,8 @@ public:
 
     // ================== Members of public ReplicationCoordinator API ===================
 
-    virtual void startup(OperationContext* opCtx) override;
+    virtual void startup(OperationContext* opCtx,
+                         LastStorageEngineShutdownState lastStorageEngineShutdownState) override;
 
     virtual void enterTerminalShutdown() override;
 
@@ -976,8 +977,12 @@ private:
      * was no local config at all or it was invalid in some way, and false if there was a valid
      * config detected but more work is needed to set it as the local config (which will be
      * handled by the callback to _finishLoadLocalConfig).
+     *
+     * Increments the rollback ID if the lastStorageEngineShutdownState indicates that the server
+     * was shut down uncleanly.
      */
-    bool _startLoadLocalConfig(OperationContext* opCtx);
+    bool _startLoadLocalConfig(OperationContext* opCtx,
+                               LastStorageEngineShutdownState lastStorageEngineShutdownState);
 
     /**
      * Callback that finishes the work started in _startLoadLocalConfig and sets _rsConfigState
diff --git a/src/mongo/db/repl/replication_coordinator_mock.cpp b/src/mongo/db/repl/replication_coordinator_mock.cpp
index 07a15b346ad..a06f8acf20e 100644
--- a/src/mongo/db/repl/replication_coordinator_mock.cpp
+++ b/src/mongo/db/repl/replication_coordinator_mock.cpp
@@ -73,7 +73,8 @@ ReplicationCoordinatorMock::ReplicationCoordinatorMock(ServiceContext* service)
 
 ReplicationCoordinatorMock::~ReplicationCoordinatorMock() {}
 
-void ReplicationCoordinatorMock::startup(OperationContext* opCtx) {
+void ReplicationCoordinatorMock::startup(
+    OperationContext* opCtx, LastStorageEngineShutdownState lastStorageEngineShutdownState) {
     // TODO
 }
 
diff --git a/src/mongo/db/repl/replication_coordinator_mock.h b/src/mongo/db/repl/replication_coordinator_mock.h
index 1336fba406e..bd43047be54 100644
--- a/src/mongo/db/repl/replication_coordinator_mock.h
+++ b/src/mongo/db/repl/replication_coordinator_mock.h
@@ -66,7 +66,8 @@ public:
 
     virtual ~ReplicationCoordinatorMock();
 
-    virtual void startup(OperationContext* opCtx);
+    virtual void startup(OperationContext* opCtx,
+                         LastStorageEngineShutdownState lastStorageEngineShutdownState);
 
     virtual void enterTerminalShutdown();
 
diff --git a/src/mongo/db/repl/replication_coordinator_noop.cpp b/src/mongo/db/repl/replication_coordinator_noop.cpp
index 4c5fc88a601..a591118079e 100644
--- a/src/mongo/db/repl/replication_coordinator_noop.cpp
+++ b/src/mongo/db/repl/replication_coordinator_noop.cpp
@@ -37,7 +37,8 @@ namespace repl {
 ReplicationCoordinatorNoOp::ReplicationCoordinatorNoOp(ServiceContext* service)
     : _service(service) {}
 
-void ReplicationCoordinatorNoOp::startup(OperationContext* opCtx) {}
+void ReplicationCoordinatorNoOp::startup(
+    OperationContext* opCtx, LastStorageEngineShutdownState lastStorageEngineShutdownState) {}
 
 void ReplicationCoordinatorNoOp::enterTerminalShutdown() {}
 
diff --git a/src/mongo/db/repl/replication_coordinator_noop.h b/src/mongo/db/repl/replication_coordinator_noop.h
index 48af2f87a9f..59d16ef5f89 100644
--- a/src/mongo/db/repl/replication_coordinator_noop.h
+++ b/src/mongo/db/repl/replication_coordinator_noop.h
@@ -47,7 +47,8 @@ public:
     ReplicationCoordinatorNoOp(ReplicationCoordinatorNoOp&) = delete;
     ReplicationCoordinatorNoOp& operator=(ReplicationCoordinatorNoOp&) = delete;
 
-    void startup(OperationContext* opCtx) final;
+    void startup(OperationContext* opCtx,
+                 LastStorageEngineShutdownState lastStorageEngineShutdownState) final;
 
     void enterTerminalShutdown() final;
 
diff --git a/src/mongo/db/repl/replication_coordinator_test_fixture.cpp b/src/mongo/db/repl/replication_coordinator_test_fixture.cpp
index 7d882e454cc..42d15f0e941 100644
--- a/src/mongo/db/repl/replication_coordinator_test_fixture.cpp
+++ b/src/mongo/db/repl/replication_coordinator_test_fixture.cpp
@@ -182,7 +182,7 @@ void ReplCoordTest::start() {
     }
 
     const auto opCtx = makeOperationContext();
-    _repl->startup(opCtx.get());
+    _repl->startup(opCtx.get(), LastStorageEngineShutdownState::kClean);
     _repl->waitForStartUpComplete_forTest();
     // _rsConfig should be written down at this point, so populate _memberData accordingly.
     _topo->populateAllMembersConfigVersionAndTerm_forTest();
diff --git a/src/mongo/db/storage/storage_engine_init.cpp b/src/mongo/db/storage/storage_engine_init.cpp
index 4b008057543..6f685fc38c1 100644
--- a/src/mongo/db/storage/storage_engine_init.cpp
+++ b/src/mongo/db/storage/storage_engine_init.cpp
@@ -61,7 +61,8 @@ void createLockFile(ServiceContext* service);
 
 extern bool _supportsDocLocking;
 
-void initializeStorageEngine(ServiceContext* service, const StorageEngineInitFlags initFlags) {
+LastStorageEngineShutdownState initializeStorageEngine(ServiceContext* service,
+                                                       const StorageEngineInitFlags initFlags) {
     // This should be set once.
     invariant(!service->getStorageEngine());
 
@@ -173,6 +174,12 @@ void initializeStorageEngine(ServiceContext* service, const StorageEngineInitFla
     guard.dismiss();
 
     _supportsDocLocking = service->getStorageEngine()->supportsDocLocking();
+
+    if (lockFile && lockFile->createdByUncleanShutdown()) {
+        return LastStorageEngineShutdownState::kUnclean;
+    } else {
+        return LastStorageEngineShutdownState::kClean;
+    }
 }
 
 void shutdownGlobalStorageEngineCleanly(ServiceContext* service) {
diff --git a/src/mongo/db/storage/storage_engine_init.h b/src/mongo/db/storage/storage_engine_init.h
index 9c92653ee56..fac7b24ff4f 100644
--- a/src/mongo/db/storage/storage_engine_init.h
+++ b/src/mongo/db/storage/storage_engine_init.h
@@ -48,9 +48,16 @@ enum StorageEngineInitFlags {
 };
 
 /**
+ * Information on last server shutdown state that is relevant to the recovery process.
+ * Determined by initializeStorageEngine() during mongod.lock initialization.
+ */
+enum class LastStorageEngineShutdownState { kClean, kUnclean };
+
+/**
  * Initializes the storage engine on "service".
  */
-void initializeStorageEngine(ServiceContext* service, StorageEngineInitFlags initFlags);
+LastStorageEngineShutdownState initializeStorageEngine(ServiceContext* service,
+                                                       StorageEngineInitFlags initFlags);
 
 /**
  * Shuts down storage engine cleanly and releases any locks on mongod.lock.
diff --git a/src/mongo/embedded/embedded.cpp b/src/mongo/embedded/embedded.cpp
index 8001b821549..27c36e1216f 100644
--- a/src/mongo/embedded/embedded.cpp
+++ b/src/mongo/embedded/embedded.cpp
@@ -237,7 +237,9 @@ ServiceContext* initialize(const char* yaml_config) {
     serviceContext->setPeriodicRunner(std::move(periodicRunner));
 
     setUpCatalog(serviceContext);
-    initializeStorageEngine(serviceContext, StorageEngineInitFlags::kAllowNoLockFile);
+    auto lastStorageEngineShutdownState =
+        initializeStorageEngine(serviceContext, StorageEngineInitFlags::kAllowNoLockFile);
+    invariant(LastStorageEngineShutdownState::kClean == lastStorageEngineShutdownState);
 
     // Warn if we detect configurations for multiple registered storage engines in the same
     // configuration file/environment.
diff --git a/src/mongo/embedded/replication_coordinator_embedded.cpp b/src/mongo/embedded/replication_coordinator_embedded.cpp
index 4867fdba23c..0f3505ac042 100644
--- a/src/mongo/embedded/replication_coordinator_embedded.cpp
+++ b/src/mongo/embedded/replication_coordinator_embedded.cpp
@@ -47,7 +47,8 @@ ReplicationCoordinatorEmbedded::ReplicationCoordinatorEmbedded(ServiceContext* s
 
 ReplicationCoordinatorEmbedded::~ReplicationCoordinatorEmbedded() = default;
 
-void ReplicationCoordinatorEmbedded::startup(OperationContext* opCtx) {}
+void ReplicationCoordinatorEmbedded::startup(
+    OperationContext* opCtx, LastStorageEngineShutdownState lastStorageEngineShutdownState) {}
 
 void ReplicationCoordinatorEmbedded::enterTerminalShutdown() {}
 
diff --git a/src/mongo/embedded/replication_coordinator_embedded.h b/src/mongo/embedded/replication_coordinator_embedded.h
index 1c7278590d3..e8f55226073 100644
--- a/src/mongo/embedded/replication_coordinator_embedded.h
+++ b/src/mongo/embedded/replication_coordinator_embedded.h
@@ -45,7 +45,8 @@ public:
 
     // Members that are implemented and safe to call of public ReplicationCoordinator API
 
-    void startup(OperationContext* opCtx) override;
+    void startup(OperationContext* opCtx,
+                 LastStorageEngineShutdownState lastStorageEngineShutdownState) override;
 
     void enterTerminalShutdown() override;