diff options
17 files changed, 161 insertions, 19 deletions
diff --git a/buildscripts/resmokeconfig/suites/replica_sets_multiversion.yml b/buildscripts/resmokeconfig/suites/replica_sets_multiversion.yml index 555630879b9..8bef7eb96c9 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_multiversion.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_multiversion.yml @@ -8,6 +8,7 @@ selector: - requires_fcv_44 exclude_files: - jstests/replsets/apply_prepare_txn_write_conflict_robustness.js + - jstests/replsets/initial_sync_fails_unclean_restart.js executor: config: shell_options: diff --git a/etc/backports_required_for_multiversion_tests.yml b/etc/backports_required_for_multiversion_tests.yml index 2433f07a7a0..97e8cd6134e 100644 --- a/etc/backports_required_for_multiversion_tests.yml +++ b/etc/backports_required_for_multiversion_tests.yml @@ -75,6 +75,8 @@ replica_sets_multiversion: test_file: jstests/replsets/disallow_adding_initialized_node1.js - ticket: SERVER-49471 test_file: jstests/replsets/apply_prepare_txn_write_conflict_robustness.js +- ticket: SERVER-50140 + test_file: jstests/replsets/initial_sync_fails_unclean_restart.js sharding_multiversion: - ticket: SERVER-38691 diff --git a/jstests/replsets/initial_sync_fails_unclean_restart.js b/jstests/replsets/initial_sync_fails_unclean_restart.js new file mode 100644 index 00000000000..020e2097b0f --- /dev/null +++ b/jstests/replsets/initial_sync_fails_unclean_restart.js @@ -0,0 +1,104 @@ +/** + * Tests that initial sync will abort an attempt if the sync source restarts from an unclean + * shutdown. And the sync source node increments its rollback id after the unclean shutdown. + * + * This is to test resumable initial sync behavior when the sync source restarts after an unclean + * shutdown. See SERVER-50140 for more details. + * @tags: [requires_persistence] + */ +(function() { +"use strict"; + +load("jstests/libs/fail_point_util.js"); +load("jstests/replsets/rslib.js"); + +const dbName = "test"; +const collName = "coll"; + +const rst = new ReplSetTest({nodes: 1}); +rst.startSet(); +rst.initiate(); + +let syncSourceNode = rst.getPrimary(); +const syncSourceColl = syncSourceNode.getDB(dbName)[collName]; + +// Insert some initial data to be cloned. +assert.commandWorked(syncSourceColl.insert([{_id: 1}, {_id: 2}, {_id: 3}])); + +jsTest.log("Adding a new node to the replica set"); +const initialSyncNode = rst.add({ + rsConfig: {priority: 0, votes: 0}, + setParameter: { + 'failpoint.initialSyncHangBeforeCopyingDatabases': tojson({mode: 'alwaysOn'}), + // Wait for the cloners to finish. + 'failpoint.initialSyncHangAfterDataCloning': tojson({mode: 'alwaysOn'}), + 'numInitialSyncAttempts': 1, + } +}); +rst.reInitiate(); + +jsTestLog("The initialSyncNode should hang before the database cloning phase"); +checkLog.contains(initialSyncNode, "initialSyncHangBeforeCopyingDatabases fail point enabled"); + +// Pauses the journal flusher and writes with {j: false}. So this data will be lost after the +// syncSourceNode restarts after an unclean shutdown. +const journalFp = configureFailPoint(syncSourceNode, "pauseJournalFlusherThread"); +journalFp.wait(); +assert.commandWorked(syncSourceColl.insert({_id: 4})); + +// Hang the initialSyncNode before initial sync finishes so we can check initial sync failure. +const beforeFinishFailPoint = configureFailPoint(initialSyncNode, "initialSyncHangBeforeFinish"); + +jsTestLog("Resuming database cloner on the initialSyncNode"); +assert.commandWorked(initialSyncNode.adminCommand( + {configureFailPoint: 'initialSyncHangBeforeCopyingDatabases', mode: 'off'})); + +jsTestLog("Waiting for data cloning to complete on the initialSyncNode"); +assert.commandWorked(initialSyncNode.adminCommand({ + waitForFailPoint: "initialSyncHangAfterDataCloning", + timesEntered: 1, + maxTimeMS: kDefaultWaitForFailPointTimeout +})); + +// Get the rollback id of the sync source before the unclean shutdown. +const rollbackIdBefore = syncSourceNode.getDB("local").system.rollback.id.findOne(); + +jsTestLog("Shutting down the syncSourceNode uncleanly"); +rst.stop(syncSourceNode, + 9, + {allowedExitCode: MongoRunner.EXIT_SIGKILL}, + {forRestart: true, waitPid: true}); + +// Make sure some retries happen due to resumable initial sync and the initial sync does not +// immediately fail while the sync source is completely down. +const nRetries = 2; +checkLog.containsWithAtLeastCount(initialSyncNode, "Trying to reconnect", nRetries); + +// Restart the sync source and wait for it to become primary again. +jsTestLog("Restarting the syncSourceNode"); +rst.start(syncSourceNode, {waitForConnect: true}, true /* restart */); +syncSourceNode = rst.getPrimary(); + +// Test that the rollback id is incremented after the unclean shutdown. +const rollbackIdAfter = syncSourceNode.getDB("local").system.rollback.id.findOne(); +assert.eq(rollbackIdAfter.rollbackId, + rollbackIdBefore.rollbackId + 1, + () => "rollbackIdBefore: " + tojson(rollbackIdBefore) + + " rollbackIdAfter: " + tojson(rollbackIdAfter)); + +jsTestLog("Resuming initial sync after the data cloning phase on the initialSyncNode"); +assert.commandWorked(initialSyncNode.adminCommand( + {configureFailPoint: "initialSyncHangAfterDataCloning", mode: "off"})); + +jsTestLog("Waiting for initial sync to fail on the initialSyncNode"); +beforeFinishFailPoint.wait(); +const res = assert.commandWorked(initialSyncNode.adminCommand({replSetGetStatus: 1})); +// The initial sync should have failed. +assert.eq(res.initialSyncStatus.failedInitialSyncAttempts, 1, () => tojson(res.initialSyncStatus)); + +// Get rid of the failed node so the fixture can stop properly. +rst.stop(initialSyncNode); +rst.remove(initialSyncNode); + +rst.stopSet(); +})(); diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp index 0033c9e7c1d..83c6667d003 100644 --- a/src/mongo/db/db.cpp +++ b/src/mongo/db/db.cpp @@ -356,7 +356,8 @@ ExitCode _initAndListen(ServiceContext* serviceContext, int listenPort) { std::make_unique<FlowControl>( serviceContext, repl::ReplicationCoordinator::get(serviceContext))); - initializeStorageEngine(serviceContext, StorageEngineInitFlags::kNone); + auto lastStorageEngineShutdownState = + initializeStorageEngine(serviceContext, StorageEngineInitFlags::kNone); #ifdef MONGO_CONFIG_WIREDTIGER_ENABLED if (EncryptionHooks::get(serviceContext)->restartRequired()) { @@ -603,7 +604,7 @@ ExitCode _initAndListen(ServiceContext* serviceContext, int listenPort) { uassert(ErrorCodes::BadValue, str::stream() << "Cannot use queryableBackupMode in a replica set", !replCoord->isReplEnabled()); - replCoord->startup(startupOpCtx.get()); + replCoord->startup(startupOpCtx.get(), lastStorageEngineShutdownState); } if (!storageGlobalParams.readOnly) { @@ -656,7 +657,7 @@ ExitCode _initAndListen(ServiceContext* serviceContext, int listenPort) { ReplicaSetNodeProcessInterface::getReplicaSetNodeExecutor(serviceContext)->startup(); } - replCoord->startup(startupOpCtx.get()); + replCoord->startup(startupOpCtx.get(), lastStorageEngineShutdownState); if (getReplSetMemberInStandaloneMode(serviceContext)) { LOGV2_WARNING_OPTIONS( 20547, diff --git a/src/mongo/db/repl/replication_coordinator.h b/src/mongo/db/repl/replication_coordinator.h index 314ae8ccd4a..cec8785f9e8 100644 --- a/src/mongo/db/repl/replication_coordinator.h +++ b/src/mongo/db/repl/replication_coordinator.h @@ -41,6 +41,7 @@ #include "mongo/db/repl/repl_settings.h" #include "mongo/db/repl/split_horizon.h" #include "mongo/db/repl/sync_source_selector.h" +#include "mongo/db/storage/storage_engine_init.h" #include "mongo/executor/task_executor.h" #include "mongo/rpc/topology_version_gen.h" #include "mongo/util/net/hostandport.h" @@ -115,7 +116,8 @@ public: * components of the replication system to start up whatever threads and do whatever * initialization they need. */ - virtual void startup(OperationContext* opCtx) = 0; + virtual void startup(OperationContext* opCtx, + LastStorageEngineShutdownState lastStorageEngineShutdownState) = 0; /** * Start terminal shutdown. This causes the topology coordinator to refuse to vote in any diff --git a/src/mongo/db/repl/replication_coordinator_impl.cpp b/src/mongo/db/repl/replication_coordinator_impl.cpp index 7fffcffaa17..2a819414283 100644 --- a/src/mongo/db/repl/replication_coordinator_impl.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl.cpp @@ -406,7 +406,8 @@ void ReplicationCoordinatorImpl::appendConnectionStats(executor::ConnectionPoolS _replExecutor->appendConnectionStats(stats); } -bool ReplicationCoordinatorImpl::_startLoadLocalConfig(OperationContext* opCtx) { +bool ReplicationCoordinatorImpl::_startLoadLocalConfig( + OperationContext* opCtx, LastStorageEngineShutdownState lastStorageEngineShutdownState) { // Create necessary replication collections to guarantee that if a checkpoint sees data after // initial sync has completed, it also sees these collections. fassert(50708, _replicationProcess->getConsistencyMarkers()->createInternalCollections(opCtx)); @@ -448,6 +449,9 @@ bool ReplicationCoordinatorImpl::_startLoadLocalConfig(OperationContext* opCtx) "Error loading local Rollback ID document at startup", "error"_attr = status); } + } else if (lastStorageEngineShutdownState == LastStorageEngineShutdownState::kUnclean) { + LOGV2(501401, "Incrementing the rollback ID after unclean shutdown"); + fassert(501402, _replicationProcess->incrementRollbackID(opCtx)); } StatusWith<BSONObj> cfg = _externalState->loadLocalConfigDocument(opCtx); @@ -807,7 +811,8 @@ void ReplicationCoordinatorImpl::_startDataReplication(OperationContext* opCtx, } } -void ReplicationCoordinatorImpl::startup(OperationContext* opCtx) { +void ReplicationCoordinatorImpl::startup( + OperationContext* opCtx, LastStorageEngineShutdownState lastStorageEngineShutdownState) { if (!isReplEnabled()) { if (ReplSettings::shouldRecoverFromOplogAsStandalone()) { uassert(ErrorCodes::InvalidOptions, @@ -866,7 +871,7 @@ void ReplicationCoordinatorImpl::startup(OperationContext* opCtx) { _replExecutor->startup(); - bool doneLoadingConfig = _startLoadLocalConfig(opCtx); + bool doneLoadingConfig = _startLoadLocalConfig(opCtx, lastStorageEngineShutdownState); if (doneLoadingConfig) { // If we're not done loading the config, then the config state will be set by // _finishLoadLocalConfig. diff --git a/src/mongo/db/repl/replication_coordinator_impl.h b/src/mongo/db/repl/replication_coordinator_impl.h index 8bbb4df7400..bde6f3136b8 100644 --- a/src/mongo/db/repl/replication_coordinator_impl.h +++ b/src/mongo/db/repl/replication_coordinator_impl.h @@ -102,7 +102,8 @@ public: // ================== Members of public ReplicationCoordinator API =================== - virtual void startup(OperationContext* opCtx) override; + virtual void startup(OperationContext* opCtx, + LastStorageEngineShutdownState lastStorageEngineShutdownState) override; virtual void enterTerminalShutdown() override; @@ -976,8 +977,12 @@ private: * was no local config at all or it was invalid in some way, and false if there was a valid * config detected but more work is needed to set it as the local config (which will be * handled by the callback to _finishLoadLocalConfig). + * + * Increments the rollback ID if the lastStorageEngineShutdownState indicates that the server + * was shut down uncleanly. */ - bool _startLoadLocalConfig(OperationContext* opCtx); + bool _startLoadLocalConfig(OperationContext* opCtx, + LastStorageEngineShutdownState lastStorageEngineShutdownState); /** * Callback that finishes the work started in _startLoadLocalConfig and sets _rsConfigState diff --git a/src/mongo/db/repl/replication_coordinator_mock.cpp b/src/mongo/db/repl/replication_coordinator_mock.cpp index 07a15b346ad..a06f8acf20e 100644 --- a/src/mongo/db/repl/replication_coordinator_mock.cpp +++ b/src/mongo/db/repl/replication_coordinator_mock.cpp @@ -73,7 +73,8 @@ ReplicationCoordinatorMock::ReplicationCoordinatorMock(ServiceContext* service) ReplicationCoordinatorMock::~ReplicationCoordinatorMock() {} -void ReplicationCoordinatorMock::startup(OperationContext* opCtx) { +void ReplicationCoordinatorMock::startup( + OperationContext* opCtx, LastStorageEngineShutdownState lastStorageEngineShutdownState) { // TODO } diff --git a/src/mongo/db/repl/replication_coordinator_mock.h b/src/mongo/db/repl/replication_coordinator_mock.h index 1336fba406e..bd43047be54 100644 --- a/src/mongo/db/repl/replication_coordinator_mock.h +++ b/src/mongo/db/repl/replication_coordinator_mock.h @@ -66,7 +66,8 @@ public: virtual ~ReplicationCoordinatorMock(); - virtual void startup(OperationContext* opCtx); + virtual void startup(OperationContext* opCtx, + LastStorageEngineShutdownState lastStorageEngineShutdownState); virtual void enterTerminalShutdown(); diff --git a/src/mongo/db/repl/replication_coordinator_noop.cpp b/src/mongo/db/repl/replication_coordinator_noop.cpp index 4c5fc88a601..a591118079e 100644 --- a/src/mongo/db/repl/replication_coordinator_noop.cpp +++ b/src/mongo/db/repl/replication_coordinator_noop.cpp @@ -37,7 +37,8 @@ namespace repl { ReplicationCoordinatorNoOp::ReplicationCoordinatorNoOp(ServiceContext* service) : _service(service) {} -void ReplicationCoordinatorNoOp::startup(OperationContext* opCtx) {} +void ReplicationCoordinatorNoOp::startup( + OperationContext* opCtx, LastStorageEngineShutdownState lastStorageEngineShutdownState) {} void ReplicationCoordinatorNoOp::enterTerminalShutdown() {} diff --git a/src/mongo/db/repl/replication_coordinator_noop.h b/src/mongo/db/repl/replication_coordinator_noop.h index 48af2f87a9f..59d16ef5f89 100644 --- a/src/mongo/db/repl/replication_coordinator_noop.h +++ b/src/mongo/db/repl/replication_coordinator_noop.h @@ -47,7 +47,8 @@ public: ReplicationCoordinatorNoOp(ReplicationCoordinatorNoOp&) = delete; ReplicationCoordinatorNoOp& operator=(ReplicationCoordinatorNoOp&) = delete; - void startup(OperationContext* opCtx) final; + void startup(OperationContext* opCtx, + LastStorageEngineShutdownState lastStorageEngineShutdownState) final; void enterTerminalShutdown() final; diff --git a/src/mongo/db/repl/replication_coordinator_test_fixture.cpp b/src/mongo/db/repl/replication_coordinator_test_fixture.cpp index 7d882e454cc..42d15f0e941 100644 --- a/src/mongo/db/repl/replication_coordinator_test_fixture.cpp +++ b/src/mongo/db/repl/replication_coordinator_test_fixture.cpp @@ -182,7 +182,7 @@ void ReplCoordTest::start() { } const auto opCtx = makeOperationContext(); - _repl->startup(opCtx.get()); + _repl->startup(opCtx.get(), LastStorageEngineShutdownState::kClean); _repl->waitForStartUpComplete_forTest(); // _rsConfig should be written down at this point, so populate _memberData accordingly. _topo->populateAllMembersConfigVersionAndTerm_forTest(); diff --git a/src/mongo/db/storage/storage_engine_init.cpp b/src/mongo/db/storage/storage_engine_init.cpp index 4b008057543..6f685fc38c1 100644 --- a/src/mongo/db/storage/storage_engine_init.cpp +++ b/src/mongo/db/storage/storage_engine_init.cpp @@ -61,7 +61,8 @@ void createLockFile(ServiceContext* service); extern bool _supportsDocLocking; -void initializeStorageEngine(ServiceContext* service, const StorageEngineInitFlags initFlags) { +LastStorageEngineShutdownState initializeStorageEngine(ServiceContext* service, + const StorageEngineInitFlags initFlags) { // This should be set once. invariant(!service->getStorageEngine()); @@ -173,6 +174,12 @@ void initializeStorageEngine(ServiceContext* service, const StorageEngineInitFla guard.dismiss(); _supportsDocLocking = service->getStorageEngine()->supportsDocLocking(); + + if (lockFile && lockFile->createdByUncleanShutdown()) { + return LastStorageEngineShutdownState::kUnclean; + } else { + return LastStorageEngineShutdownState::kClean; + } } void shutdownGlobalStorageEngineCleanly(ServiceContext* service) { diff --git a/src/mongo/db/storage/storage_engine_init.h b/src/mongo/db/storage/storage_engine_init.h index 9c92653ee56..fac7b24ff4f 100644 --- a/src/mongo/db/storage/storage_engine_init.h +++ b/src/mongo/db/storage/storage_engine_init.h @@ -48,9 +48,16 @@ enum StorageEngineInitFlags { }; /** + * Information on last server shutdown state that is relevant to the recovery process. + * Determined by initializeStorageEngine() during mongod.lock initialization. + */ +enum class LastStorageEngineShutdownState { kClean, kUnclean }; + +/** * Initializes the storage engine on "service". */ -void initializeStorageEngine(ServiceContext* service, StorageEngineInitFlags initFlags); +LastStorageEngineShutdownState initializeStorageEngine(ServiceContext* service, + StorageEngineInitFlags initFlags); /** * Shuts down storage engine cleanly and releases any locks on mongod.lock. diff --git a/src/mongo/embedded/embedded.cpp b/src/mongo/embedded/embedded.cpp index 8001b821549..27c36e1216f 100644 --- a/src/mongo/embedded/embedded.cpp +++ b/src/mongo/embedded/embedded.cpp @@ -237,7 +237,9 @@ ServiceContext* initialize(const char* yaml_config) { serviceContext->setPeriodicRunner(std::move(periodicRunner)); setUpCatalog(serviceContext); - initializeStorageEngine(serviceContext, StorageEngineInitFlags::kAllowNoLockFile); + auto lastStorageEngineShutdownState = + initializeStorageEngine(serviceContext, StorageEngineInitFlags::kAllowNoLockFile); + invariant(LastStorageEngineShutdownState::kClean == lastStorageEngineShutdownState); // Warn if we detect configurations for multiple registered storage engines in the same // configuration file/environment. diff --git a/src/mongo/embedded/replication_coordinator_embedded.cpp b/src/mongo/embedded/replication_coordinator_embedded.cpp index 4867fdba23c..0f3505ac042 100644 --- a/src/mongo/embedded/replication_coordinator_embedded.cpp +++ b/src/mongo/embedded/replication_coordinator_embedded.cpp @@ -47,7 +47,8 @@ ReplicationCoordinatorEmbedded::ReplicationCoordinatorEmbedded(ServiceContext* s ReplicationCoordinatorEmbedded::~ReplicationCoordinatorEmbedded() = default; -void ReplicationCoordinatorEmbedded::startup(OperationContext* opCtx) {} +void ReplicationCoordinatorEmbedded::startup( + OperationContext* opCtx, LastStorageEngineShutdownState lastStorageEngineShutdownState) {} void ReplicationCoordinatorEmbedded::enterTerminalShutdown() {} diff --git a/src/mongo/embedded/replication_coordinator_embedded.h b/src/mongo/embedded/replication_coordinator_embedded.h index 1c7278590d3..e8f55226073 100644 --- a/src/mongo/embedded/replication_coordinator_embedded.h +++ b/src/mongo/embedded/replication_coordinator_embedded.h @@ -45,7 +45,8 @@ public: // Members that are implemented and safe to call of public ReplicationCoordinator API - void startup(OperationContext* opCtx) override; + void startup(OperationContext* opCtx, + LastStorageEngineShutdownState lastStorageEngineShutdownState) override; void enterTerminalShutdown() override; |