SERVER-21478: use lastWrite instead of replCoord::lastOpTime when transitioning to secondaryr3.2.0-rc3

4 files changed, 38 insertions, 11 deletions

diff --git a/src/mongo/db/repl/minvalid.h b/src/mongo/db/repl/minvalid.h
index 17f63a8274d..d92d5d0c4a6 100644
--- a/src/mongo/db/repl/minvalid.h
+++ b/src/mongo/db/repl/minvalid.h
@@ -38,8 +38,8 @@ namespace repl {
 
 struct BatchBoundaries {
     BatchBoundaries(const OpTime s, const OpTime e) : start(s), end(e) {}
-    const OpTime start;
-    const OpTime end;
+    OpTime start;
+    OpTime end;
 };
 
 enum class DurableRequirement {
diff --git a/src/mongo/db/repl/oplogreader.cpp b/src/mongo/db/repl/oplogreader.cpp
index 4ba200694a6..c1b6e37f82f 100644
--- a/src/mongo/db/repl/oplogreader.cpp
+++ b/src/mongo/db/repl/oplogreader.cpp
@@ -161,11 +161,15 @@ void OplogReader::connectToSyncSource(OperationContext* txn,
 
             // Connected to at least one member, but in all cases we were too stale to use them
             // as a sync source.
-            error() << "too stale to catch up";
+            error() << "too stale to catch up -- entering maintenance mode";
             log() << "our last optime : " << lastOpTimeFetched;
             log() << "oldest available is " << oldestOpTimeSeen;
             log() << "See http://dochub.mongodb.org/core/resyncingaverystalereplicasetmember";
             setMinValid(txn, {lastOpTimeFetched, oldestOpTimeSeen});
+            auto status = replCoord->setMaintenanceMode(true);
+            if (!status.isOK()) {
+                warning() << "Failed to transition into maintenance mode.";
+            }
             bool worked = replCoord->setFollowerMode(MemberState::RS_RECOVERING);
             if (!worked) {
                 warning() << "Failed to transition into " << MemberState(MemberState::RS_RECOVERING)
@@ -200,6 +204,10 @@ void OplogReader::connectToSyncSource(OperationContext* txn,
             continue;
         }
 
+
+        // TODO: If we were too stale (recovering with maintenance mode on), then turn it off, to
+        //       allow becoming secondary/etc.
+
         // Got a valid sync source.
         return;
     }  // while (true)
diff --git a/src/mongo/db/repl/rs_initialsync.cpp b/src/mongo/db/repl/rs_initialsync.cpp
index aeec194cebb..57224dca3bc 100644
--- a/src/mongo/db/repl/rs_initialsync.cpp
+++ b/src/mongo/db/repl/rs_initialsync.cpp
@@ -439,6 +439,11 @@ Status _initialSync() {
     // Clear the initial sync flag -- cannot be done under a db lock, or recursive.
     clearInitialSyncFlag(&txn);
 
+    // Clear maint. mode.
+    while (replCoord->getMaintenanceMode()) {
+        replCoord->setMaintenanceMode(false);
+    }
+
     log() << "initial sync done";
     return Status::OK();
 }
diff --git a/src/mongo/db/repl/sync_tail.cpp b/src/mongo/db/repl/sync_tail.cpp
index dd85ce8573c..107a47fd0de 100644
--- a/src/mongo/db/repl/sync_tail.cpp
+++ b/src/mongo/db/repl/sync_tail.cpp
@@ -484,7 +484,10 @@ OpTime SyncTail::multiApply(OperationContext* txn, const OpQueue& ops) {
 }
 
 namespace {
-void tryToGoLiveAsASecondary(OperationContext* txn, ReplicationCoordinator* replCoord) {
+void tryToGoLiveAsASecondary(OperationContext* txn,
+                             ReplicationCoordinator* replCoord,
+                             const BatchBoundaries& minValidBoundaries,
+                             const OpTime& lastWriteOpTime) {
     if (replCoord->isInPrimaryOrSecondaryState()) {
         return;
     }
@@ -503,8 +506,15 @@ void tryToGoLiveAsASecondary(OperationContext* txn, ReplicationCoordinator* repl
         return;
     }
 
-    BatchBoundaries boundaries = getMinValid(txn);
-    if (!boundaries.start.isNull() || boundaries.end > replCoord->getMyLastOptime()) {
+    // If an apply batch is active then we cannot transition.
+    if (!minValidBoundaries.start.isNull()) {
+        return;
+    }
+
+    // Must have applied/written to minvalid, so return if not.
+    // -- If 'lastWriteOpTime' is null/uninitialized then we can't transition.
+    // -- If 'lastWriteOpTime' is less than the end of the last batch then we can't transition.
+    if (lastWriteOpTime.isNull() || minValidBoundaries.end > lastWriteOpTime) {
         return;
     }
 
@@ -621,7 +631,9 @@ void SyncTail::oplogApplication() {
     auto replCoord = ReplicationCoordinator::get(&txn);
     ApplyBatchFinalizer finalizer(replCoord);
 
-    OpTime originalEndOpTime(getMinValid(&txn).end);
+    auto minValidBoundaries = getMinValid(&txn);
+    OpTime originalEndOpTime(minValidBoundaries.end);
+    OpTime lastWriteOpTime{replCoord->getMyLastOptime()};
     while (!inShutdown()) {
         OpQueue ops;
 
@@ -631,7 +643,7 @@ void SyncTail::oplogApplication() {
                 return;
             }
 
-            tryToGoLiveAsASecondary(&txn, replCoord);
+            tryToGoLiveAsASecondary(&txn, replCoord, minValidBoundaries, lastWriteOpTime);
 
             // Blocks up to a second waiting for a batch to be ready to apply. If one doesn't become
             // ready in time, we'll loop again so we can do the above checks periodically.
@@ -682,11 +694,13 @@ void SyncTail::oplogApplication() {
         // This write will not journal/checkpoint.
         setMinValid(&txn, {start, end});
 
-        OpTime finalOpTime = multiApply(&txn, ops);
-        setNewTimestamp(finalOpTime.getTimestamp());
+        lastWriteOpTime = multiApply(&txn, ops);
+        setNewTimestamp(lastWriteOpTime.getTimestamp());
 
         setMinValid(&txn, end, DurableRequirement::None);
-        finalizer.record(finalOpTime);
+        minValidBoundaries.start = {};
+        minValidBoundaries.end = end;
+        finalizer.record(lastWriteOpTime);
     }
 }