SERVER-46721 Secondary readers should read at the no-overlap time instead of lastApplied

The no-overlap time, ReadSource::kNoOverlap, is the minimum of replication's lastApplied timestamp
and WiredTiger's all_durable time. This time is independent of replication state and ensures
queries do not see oplog holes after state transitions from secondary to primary.

1 files changed, 111 insertions, 3 deletions

diff --git a/src/mongo/db/db_raii_test.cpp b/src/mongo/db/db_raii_test.cpp
index 4b9a3bac3c4..5258a51a33b 100644
--- a/src/mongo/db/db_raii_test.cpp
+++ b/src/mongo/db/db_raii_test.cpp
@@ -37,6 +37,8 @@
 #include "mongo/db/client.h"
 #include "mongo/db/concurrency/lock_state.h"
 #include "mongo/db/db_raii.h"
+#include "mongo/db/query/internal_plans.h"
+#include "mongo/db/storage/snapshot_manager.h"
 #include "mongo/logv2/log.h"
 #include "mongo/unittest/unittest.h"
 #include "mongo/util/time_support.h"
@@ -199,8 +201,13 @@ TEST_F(DBRAIITestFixture,
     ASSERT_OK(
         storageInterface()->createCollection(client1.second.get(), nss, defaultCollectionOptions));
     ASSERT_OK(replCoord->setFollowerMode(repl::MemberState::RS_SECONDARY));
+
+    // Don't call into the ReplicationCoordinator to update lastApplied because it is only a mock
+    // class and does not update the correct state in the SnapshotManager.
     repl::OpTime opTime(Timestamp(200, 1), 1);
-    replCoord->setMyLastAppliedOpTimeAndWallTime({opTime, Date_t() + Seconds(1)});
+    auto snapshotManager =
+        client1.second.get()->getServiceContext()->getStorageEngine()->getSnapshotManager();
+    snapshotManager->setLastApplied(opTime.getTimestamp());
     Lock::DBLock dbLock1(client1.second.get(), nss.db(), MODE_IX);
     ASSERT(client1.second->lockState()->isDbLockedForMode(nss.db(), MODE_IX));
 
@@ -220,18 +227,119 @@ TEST_F(DBRAIITestFixture,
     // for the collection.  If we now manually set our last applied time to something very early, we
     // will be guaranteed to hit the logic that triggers when the minimum snapshot time is greater
     // than the read-at time, since we default to reading at last-applied when in SECONDARY state.
+
+    // Don't call into the ReplicationCoordinator to update lastApplied because it is only a mock
+    // class and does not update the correct state in the SnapshotManager.
     repl::OpTime opTime(Timestamp(2, 1), 1);
-    replCoord->setMyLastAppliedOpTimeAndWallTime({opTime, Date_t() + Seconds(1)});
+    auto snapshotManager =
+        client1.second.get()->getServiceContext()->getStorageEngine()->getSnapshotManager();
+    snapshotManager->setLastApplied(opTime.getTimestamp());
+
     Lock::DBLock dbLock1(client1.second.get(), nss.db(), MODE_IX);
     ASSERT(client1.second->lockState()->isDbLockedForMode(nss.db(), MODE_IX));
+
     AutoGetCollectionForRead coll(client2.second.get(), NamespaceString("local.system.js"));
+    // Reading from an unreplicated collection does not change the ReadSource to kNoOverlap.
+    ASSERT_EQ(client2.second.get()->recoveryUnit()->getTimestampReadSource(),
+              RecoveryUnit::ReadSource::kUnset);
 
-    // The current code uasserts in this situation, so we confirm that happens here.
+    // Reading from a replicated collection will try to switch to kNoOverlap. Because we are
+    // already reading without a timestamp and we can't reacquire the PBWM lock to continue reading
+    // without a timestamp, we uassert in this situation.
     ASSERT_THROWS_CODE(AutoGetCollectionForRead(client2.second.get(), nss),
                        DBException,
                        ErrorCodes::SnapshotUnavailable);
 }
 
+TEST_F(DBRAIITestFixture, AutoGetCollectionForReadLastAppliedConflict) {
+    // This test simulates a situation where AutoGetCollectionForRead cant read at the no-overlap
+    // point (minimum of all_durable and lastApplied) because it is set to a point earlier than the
+    // catalog change. We expect to read without a timestamp and hold the PBWM lock.
+    auto replCoord = repl::ReplicationCoordinator::get(client1.second.get());
+    CollectionOptions defaultCollectionOptions;
+    ASSERT_OK(
+        storageInterface()->createCollection(client1.second.get(), nss, defaultCollectionOptions));
+    ASSERT_OK(replCoord->setFollowerMode(repl::MemberState::RS_SECONDARY));
+
+    // Note that when the collection was created, above, the system chooses a minimum snapshot time
+    // for the collection.  If we now manually set our last applied time to something very early, we
+    // will be guaranteed to hit the logic that triggers when the minimum snapshot time is greater
+    // than the read-at time, since we default to reading at last-applied when in SECONDARY state.
+
+    // Don't call into the ReplicationCoordinator to update lastApplied because it is only a mock
+    // class and does not update the correct state in the SnapshotManager.
+    repl::OpTime opTime(Timestamp(2, 1), 1);
+    auto snapshotManager =
+        client1.second.get()->getServiceContext()->getStorageEngine()->getSnapshotManager();
+    snapshotManager->setLastApplied(opTime.getTimestamp());
+    AutoGetCollectionForRead coll(client1.second.get(), nss);
+
+    // We can't read from kNoOverlap in this scenario because there is a catalog conflict. Resort
+    // to taking the PBWM lock and reading without a timestamp.
+    ASSERT_EQ(client1.second.get()->recoveryUnit()->getTimestampReadSource(),
+              RecoveryUnit::ReadSource::kUnset);
+    ASSERT_TRUE(client1.second.get()->lockState()->isLockHeldForMode(
+        resourceIdParallelBatchWriterMode, MODE_IS));
+}
+
+TEST_F(DBRAIITestFixture, AutoGetCollectionForReadLastAppliedUnavailable) {
+    // This test simulates a situation where AutoGetCollectionForRead reads at the no-overlap
+    // point (minimum of all_durable and lastApplied) even though lastApplied is not available.
+    auto replCoord = repl::ReplicationCoordinator::get(client1.second.get());
+    CollectionOptions defaultCollectionOptions;
+    ASSERT_OK(
+        storageInterface()->createCollection(client1.second.get(), nss, defaultCollectionOptions));
+    ASSERT_OK(replCoord->setFollowerMode(repl::MemberState::RS_SECONDARY));
+
+    // Note that when the collection was created, above, the system chooses a minimum snapshot time
+    // for the collection. Since last-applied isn't available, we default to all_durable, which is
+    // available, and is greater than the collection minimum snapshot.
+    auto snapshotManager =
+        client1.second.get()->getServiceContext()->getStorageEngine()->getSnapshotManager();
+    ASSERT_FALSE(snapshotManager->getLastApplied());
+    AutoGetCollectionForRead coll(client1.second.get(), nss);
+
+    // Even though lastApplied isn't available, the ReadSource is set to kNoOverlap, which reads
+    // at the all_durable time.
+    ASSERT_EQ(client1.second.get()->recoveryUnit()->getTimestampReadSource(),
+              RecoveryUnit::ReadSource::kNoOverlap);
+    ASSERT_TRUE(client1.second.get()->recoveryUnit()->getPointInTimeReadTimestamp());
+    ASSERT_FALSE(client1.second.get()->lockState()->isLockHeldForMode(
+        resourceIdParallelBatchWriterMode, MODE_IS));
+}
+
+TEST_F(DBRAIITestFixture, AutoGetCollectionForReadUsesNoOverlapOnSecondary) {
+    auto opCtx = client1.second.get();
+    ASSERT_OK(storageInterface()->createCollection(opCtx, nss, {}));
+    ASSERT_OK(
+        repl::ReplicationCoordinator::get(opCtx)->setFollowerMode(repl::MemberState::RS_SECONDARY));
+    AutoGetCollectionForRead autoColl(opCtx, nss);
+    auto exec = InternalPlanner::collectionScan(opCtx,
+                                                nss.ns(),
+                                                autoColl.getCollection(),
+                                                PlanExecutor::YIELD_MANUAL,
+                                                InternalPlanner::FORWARD);
+
+    // The collection scan should use the default ReadSource on a secondary.
+    ASSERT_EQ(RecoveryUnit::ReadSource::kNoOverlap,
+              opCtx->recoveryUnit()->getTimestampReadSource());
+
+    // While yielding the collection scan, simulate stepping-up to a primary.
+    exec->saveState();
+    Locker::LockSnapshot lockSnapshot;
+    ASSERT_TRUE(opCtx->lockState()->saveLockStateAndUnlock(&lockSnapshot));
+    ASSERT_OK(
+        repl::ReplicationCoordinator::get(opCtx)->setFollowerMode(repl::MemberState::RS_PRIMARY));
+
+    // After restoring, the collection scan should now be reading with kNoOverlap, the default on
+    // secondaries.
+    opCtx->lockState()->restoreLockState(opCtx, lockSnapshot);
+    exec->restoreState();
+    ASSERT_EQ(RecoveryUnit::ReadSource::kNoOverlap,
+              opCtx->recoveryUnit()->getTimestampReadSource());
+    BSONObj obj;
+    ASSERT_EQUALS(PlanExecutor::IS_EOF, exec->getNext(&obj, nullptr));
+}
 
 }  // namespace
 }  // namespace mongo