diff options
-rw-r--r-- | src/mongo/db/repl/minvalid.h | 4 | ||||
-rw-r--r-- | src/mongo/db/repl/oplogreader.cpp | 10 | ||||
-rw-r--r-- | src/mongo/db/repl/rs_initialsync.cpp | 5 | ||||
-rw-r--r-- | src/mongo/db/repl/sync_tail.cpp | 30 |
4 files changed, 38 insertions, 11 deletions
diff --git a/src/mongo/db/repl/minvalid.h b/src/mongo/db/repl/minvalid.h index 17f63a8274d..d92d5d0c4a6 100644 --- a/src/mongo/db/repl/minvalid.h +++ b/src/mongo/db/repl/minvalid.h @@ -38,8 +38,8 @@ namespace repl { struct BatchBoundaries { BatchBoundaries(const OpTime s, const OpTime e) : start(s), end(e) {} - const OpTime start; - const OpTime end; + OpTime start; + OpTime end; }; enum class DurableRequirement { diff --git a/src/mongo/db/repl/oplogreader.cpp b/src/mongo/db/repl/oplogreader.cpp index 4ba200694a6..c1b6e37f82f 100644 --- a/src/mongo/db/repl/oplogreader.cpp +++ b/src/mongo/db/repl/oplogreader.cpp @@ -161,11 +161,15 @@ void OplogReader::connectToSyncSource(OperationContext* txn, // Connected to at least one member, but in all cases we were too stale to use them // as a sync source. - error() << "too stale to catch up"; + error() << "too stale to catch up -- entering maintenance mode"; log() << "our last optime : " << lastOpTimeFetched; log() << "oldest available is " << oldestOpTimeSeen; log() << "See http://dochub.mongodb.org/core/resyncingaverystalereplicasetmember"; setMinValid(txn, {lastOpTimeFetched, oldestOpTimeSeen}); + auto status = replCoord->setMaintenanceMode(true); + if (!status.isOK()) { + warning() << "Failed to transition into maintenance mode."; + } bool worked = replCoord->setFollowerMode(MemberState::RS_RECOVERING); if (!worked) { warning() << "Failed to transition into " << MemberState(MemberState::RS_RECOVERING) @@ -200,6 +204,10 @@ void OplogReader::connectToSyncSource(OperationContext* txn, continue; } + + // TODO: If we were too stale (recovering with maintenance mode on), then turn it off, to + // allow becoming secondary/etc. + // Got a valid sync source. return; } // while (true) diff --git a/src/mongo/db/repl/rs_initialsync.cpp b/src/mongo/db/repl/rs_initialsync.cpp index aeec194cebb..57224dca3bc 100644 --- a/src/mongo/db/repl/rs_initialsync.cpp +++ b/src/mongo/db/repl/rs_initialsync.cpp @@ -439,6 +439,11 @@ Status _initialSync() { // Clear the initial sync flag -- cannot be done under a db lock, or recursive. clearInitialSyncFlag(&txn); + // Clear maint. mode. + while (replCoord->getMaintenanceMode()) { + replCoord->setMaintenanceMode(false); + } + log() << "initial sync done"; return Status::OK(); } diff --git a/src/mongo/db/repl/sync_tail.cpp b/src/mongo/db/repl/sync_tail.cpp index dd85ce8573c..107a47fd0de 100644 --- a/src/mongo/db/repl/sync_tail.cpp +++ b/src/mongo/db/repl/sync_tail.cpp @@ -484,7 +484,10 @@ OpTime SyncTail::multiApply(OperationContext* txn, const OpQueue& ops) { } namespace { -void tryToGoLiveAsASecondary(OperationContext* txn, ReplicationCoordinator* replCoord) { +void tryToGoLiveAsASecondary(OperationContext* txn, + ReplicationCoordinator* replCoord, + const BatchBoundaries& minValidBoundaries, + const OpTime& lastWriteOpTime) { if (replCoord->isInPrimaryOrSecondaryState()) { return; } @@ -503,8 +506,15 @@ void tryToGoLiveAsASecondary(OperationContext* txn, ReplicationCoordinator* repl return; } - BatchBoundaries boundaries = getMinValid(txn); - if (!boundaries.start.isNull() || boundaries.end > replCoord->getMyLastOptime()) { + // If an apply batch is active then we cannot transition. + if (!minValidBoundaries.start.isNull()) { + return; + } + + // Must have applied/written to minvalid, so return if not. + // -- If 'lastWriteOpTime' is null/uninitialized then we can't transition. + // -- If 'lastWriteOpTime' is less than the end of the last batch then we can't transition. + if (lastWriteOpTime.isNull() || minValidBoundaries.end > lastWriteOpTime) { return; } @@ -621,7 +631,9 @@ void SyncTail::oplogApplication() { auto replCoord = ReplicationCoordinator::get(&txn); ApplyBatchFinalizer finalizer(replCoord); - OpTime originalEndOpTime(getMinValid(&txn).end); + auto minValidBoundaries = getMinValid(&txn); + OpTime originalEndOpTime(minValidBoundaries.end); + OpTime lastWriteOpTime{replCoord->getMyLastOptime()}; while (!inShutdown()) { OpQueue ops; @@ -631,7 +643,7 @@ void SyncTail::oplogApplication() { return; } - tryToGoLiveAsASecondary(&txn, replCoord); + tryToGoLiveAsASecondary(&txn, replCoord, minValidBoundaries, lastWriteOpTime); // Blocks up to a second waiting for a batch to be ready to apply. If one doesn't become // ready in time, we'll loop again so we can do the above checks periodically. @@ -682,11 +694,13 @@ void SyncTail::oplogApplication() { // This write will not journal/checkpoint. setMinValid(&txn, {start, end}); - OpTime finalOpTime = multiApply(&txn, ops); - setNewTimestamp(finalOpTime.getTimestamp()); + lastWriteOpTime = multiApply(&txn, ops); + setNewTimestamp(lastWriteOpTime.getTimestamp()); setMinValid(&txn, end, DurableRequirement::None); - finalizer.record(finalOpTime); + minValidBoundaries.start = {}; + minValidBoundaries.end = end; + finalizer.record(lastWriteOpTime); } } |