SERVER-39310: Call checkCanServeReadsFor() in 'getMore'

- added call to checkCanServeReadsFor() in getmore_cmd.cpp after getting readlocks
- introduced two fail points: 1. pause 'getMore' before readlocks 2. pause rollback after state transition
- added testcase read_operations_during_rollback.js
- added call to checkCanServeReadsFor() in find_cmd.cpp after getting readlocks (cherry picked from commit e8fe32029aded4d0e909f531196edff43c96cfff)
- added assert.includes (cherry picked from commit dd9be1adf2425c7ddd746ff6da75d564474ebed3)

(cherry picked from commit b885fa6feb7da00dc367e917c53ba16a41b75af4)

8 files changed, 138 insertions, 1 deletions

diff --git a/jstests/replsets/read_operations_during_rollback.js b/jstests/replsets/read_operations_during_rollback.js
new file mode 100644
index 00000000000..92bdb569943
--- /dev/null
+++ b/jstests/replsets/read_operations_during_rollback.js
@@ -0,0 +1,95 @@
+/*
+ * This test makes sure 'find' and 'getMore' commands fail correctly during rollback.
+ */
+(function() {
+    "use strict";
+
+    load("jstests/replsets/libs/rollback_test.js");
+
+    const dbName = "test";
+    const collName = "coll";
+
+    let setFailPoint = (node, failpoint) => {
+        jsTestLog("Setting fail point " + failpoint);
+        assert.commandWorked(node.adminCommand({configureFailPoint: failpoint, mode: "alwaysOn"}));
+    };
+
+    let clearFailPoint = (node, failpoint) => {
+        jsTestLog("Clearing fail point " + failpoint);
+        assert.commandWorked(node.adminCommand({configureFailPoint: failpoint, mode: "off"}));
+    };
+
+    // Set up Rollback Test.
+    let rollbackTest = new RollbackTest();
+
+    // Insert a document to be read later.
+    assert.writeOK(rollbackTest.getPrimary().getDB(dbName)[collName].insert({}));
+
+    let rollbackNode = rollbackTest.transitionToRollbackOperations();
+
+    setFailPoint(rollbackNode, "rollbackHangAfterTransitionToRollback");
+
+    setFailPoint(rollbackNode, "GetMoreHangBeforeReadLock");
+
+    const joinGetMoreThread = startParallelShell(() => {
+        db.getMongo().setSlaveOk();
+        const cursorID =
+            assert.commandWorked(db.runCommand({"find": "coll", batchSize: 0})).cursor.id;
+        let res = assert.throws(function() {
+            db.runCommand({"getMore": cursorID, collection: "coll"});
+        }, [], "network error");
+        // Make sure the connection of an outstanding read operation gets closed during rollback
+        // even though the read was started before rollback.
+        assert.includes(res.toString(), "network error while attempting to run command");
+    }, rollbackNode.port);
+
+    const cursorIdToBeReadDuringRollback =
+        assert
+            .commandWorked(rollbackNode.getDB(dbName).runCommand({"find": collName, batchSize: 0}))
+            .cursor.id;
+
+    // Wait for 'getMore' to hang.
+    checkLog.contains(rollbackNode, "GetMoreHangBeforeReadLock fail point enabled.");
+
+    // Start rollback.
+    rollbackTest.transitionToSyncSourceOperationsBeforeRollback();
+    rollbackTest.transitionToSyncSourceOperationsDuringRollback();
+
+    jsTestLog("Reconnecting to " + rollbackNode.host + " after rollback");
+    reconnect(rollbackNode.getDB(dbName));
+
+    // Wait for rollback to hang.
+    checkLog.contains(rollbackNode, "rollbackHangAfterTransitionToRollback fail point enabled.");
+
+    clearFailPoint(rollbackNode, "GetMoreHangBeforeReadLock");
+
+    jsTestLog("Wait for 'getMore' thread to join.");
+    joinGetMoreThread();
+
+    jsTestLog("Reading during rollback.");
+    // Make sure that read operations fail during rollback.
+    assert.commandFailedWithCode(rollbackNode.getDB(dbName).runCommand({"find": collName}),
+                                 ErrorCodes.NotMasterOrSecondary);
+    assert.commandFailedWithCode(
+        rollbackNode.getDB(dbName).runCommand(
+            {"getMore": cursorIdToBeReadDuringRollback, collection: collName}),
+        ErrorCodes.NotMasterOrSecondary);
+
+    // Disable the best-effort check for primary-ness in the service entry point, so that we
+    // exercise the real check for primary-ness in 'find' and 'getMore' commands.
+    setFailPoint(rollbackNode, "skipCheckingForNotMasterInCommandDispatch");
+    jsTestLog("Reading during rollback (again with command dispatch checks disabled).");
+    assert.commandFailedWithCode(rollbackNode.getDB(dbName).runCommand({"find": collName}),
+                                 ErrorCodes.NotMasterOrSecondary);
+    assert.commandFailedWithCode(
+        rollbackNode.getDB(dbName).runCommand(
+            {"getMore": cursorIdToBeReadDuringRollback, collection: collName}),
+        ErrorCodes.NotMasterOrSecondary);
+
+    clearFailPoint(rollbackNode, "rollbackHangAfterTransitionToRollback");
+
+    rollbackTest.transitionToSteadyStateOperations();
+
+    // Check the replica set.
+    rollbackTest.stop();
+}());
diff --git a/src/mongo/db/commands/find_cmd.cpp b/src/mongo/db/commands/find_cmd.cpp
index b6c496ed6ac..e4a17acc8e6 100644
--- a/src/mongo/db/commands/find_cmd.cpp
+++ b/src/mongo/db/commands/find_cmd.cpp
@@ -240,11 +240,11 @@ public:
             return appendCommandStatus(result, qrStatus.getStatus());
         }
 
+        auto replCoord = repl::ReplicationCoordinator::get(opCtx);
         auto& qr = qrStatus.getValue();
 
         // Validate term before acquiring locks, if provided.
         if (auto term = qr->getReplicationTerm()) {
-            auto replCoord = repl::ReplicationCoordinator::get(opCtx);
             Status status = replCoord->updateTerm(opCtx, *term);
             // Note: updateTerm returns ok if term stayed the same.
             if (!status.isOK()) {
@@ -258,6 +258,10 @@ public:
         const NamespaceString nss(parseNsOrUUID(opCtx, dbname, cmdObj));
         qr->refreshNSS(opCtx);
 
+        // Check whether we are allowed to read from this node after acquiring our locks.
+        uassertStatusOK(replCoord->checkCanServeReadsFor(
+            opCtx, nss, ReadPreferenceSetting::get(opCtx).canRunOnSecondary()));
+
         // Fill out curop information.
         //
         // We pass negative values for 'ntoreturn' and 'ntoskip' to indicate that these values
diff --git a/src/mongo/db/commands/getmore_cmd.cpp b/src/mongo/db/commands/getmore_cmd.cpp
index 3715d7cf1ef..83e59456479 100644
--- a/src/mongo/db/commands/getmore_cmd.cpp
+++ b/src/mongo/db/commands/getmore_cmd.cpp
@@ -68,6 +68,7 @@ namespace mongo {
 namespace {
 
 MONGO_FP_DECLARE(rsStopGetMoreCmd);
+MONGO_FP_DECLARE(GetMoreHangBeforeReadLock);
 
 /**
  * A command for running getMore() against an existing cursor registered with a CursorManager.
@@ -206,6 +207,12 @@ public:
                                      dbProfilingLevel);
             }
         } else {
+            if (MONGO_FAIL_POINT(GetMoreHangBeforeReadLock)) {
+                log() << "GetMoreHangBeforeReadLock fail point enabled. Blocking until fail "
+                         "point is disabled.";
+                MONGO_FAIL_POINT_PAUSE_WHILE_SET(GetMoreHangBeforeReadLock);
+            }
+
             readLock.emplace(opCtx, request.nss);
             const int doNotChangeProfilingLevel = 0;
             statsTracker.emplace(opCtx,
@@ -215,6 +222,10 @@ public:
                                  readLock->getDb() ? readLock->getDb()->getProfilingLevel()
                                                    : doNotChangeProfilingLevel);
 
+            // Check whether we are allowed to read from this node after acquiring our locks.
+            uassertStatusOK(repl::ReplicationCoordinator::get(opCtx)->checkCanServeReadsFor(
+                opCtx, request.nss, true));
+
             Collection* collection = readLock->getCollection();
             if (!collection) {
                 return appendCommandStatus(result,
diff --git a/src/mongo/db/repl/roll_back_local_operations.cpp b/src/mongo/db/repl/roll_back_local_operations.cpp
index 9b2843ff617..56bd2da369a 100644
--- a/src/mongo/db/repl/roll_back_local_operations.cpp
+++ b/src/mongo/db/repl/roll_back_local_operations.cpp
@@ -51,6 +51,8 @@ MONGO_FP_DECLARE(rollbackHangBeforeFinish);
 // Failpoint which causes rollback to hang and then fail after minValid is written.
 MONGO_FP_DECLARE(rollbackHangThenFailAfterWritingMinValid);
 
+// This is needed by rs_rollback and rollback_impl.
+MONGO_FP_DECLARE(rollbackHangAfterTransitionToRollback);
 
 namespace {
 
diff --git a/src/mongo/db/repl/roll_back_local_operations.h b/src/mongo/db/repl/roll_back_local_operations.h
index 6dcf7bbb082..ba8dcfe79ae 100644
--- a/src/mongo/db/repl/roll_back_local_operations.h
+++ b/src/mongo/db/repl/roll_back_local_operations.h
@@ -51,6 +51,9 @@ namespace repl {
 MONGO_FP_FORWARD_DECLARE(rollbackHangBeforeFinish);
 MONGO_FP_FORWARD_DECLARE(rollbackHangThenFailAfterWritingMinValid);
 
+// This is needed by rs_rollback and rollback_impl.
+MONGO_FP_FORWARD_DECLARE(rollbackHangAfterTransitionToRollback);
+
 class RollBackLocalOperations {
     MONGO_DISALLOW_COPYING(RollBackLocalOperations);
 
diff --git a/src/mongo/db/repl/rollback_impl.cpp b/src/mongo/db/repl/rollback_impl.cpp
index 12dee8e5ac7..ec8350037d9 100644
--- a/src/mongo/db/repl/rollback_impl.cpp
+++ b/src/mongo/db/repl/rollback_impl.cpp
@@ -93,6 +93,12 @@ Status RollbackImpl::runRollback(OperationContext* opCtx) {
     }
     _listener->onTransitionToRollback();
 
+    if (MONGO_FAIL_POINT(rollbackHangAfterTransitionToRollback)) {
+        log() << "rollbackHangAfterTransitionToRollback fail point enabled. Blocking until fail "
+                 "point is disabled (rollback_impl).";
+        MONGO_FAIL_POINT_PAUSE_WHILE_SET(rollbackHangAfterTransitionToRollback);
+    }
+
     auto commonPointSW = _findCommonPoint();
     if (!commonPointSW.isOK()) {
         return commonPointSW.getStatus();
diff --git a/src/mongo/db/repl/rs_rollback.cpp b/src/mongo/db/repl/rs_rollback.cpp
index b4315f20670..33e15f3ae85 100644
--- a/src/mongo/db/repl/rs_rollback.cpp
+++ b/src/mongo/db/repl/rs_rollback.cpp
@@ -1487,6 +1487,12 @@ void rollback(OperationContext* opCtx,
         }
     }
 
+    if (MONGO_FAIL_POINT(rollbackHangAfterTransitionToRollback)) {
+        log() << "rollbackHangAfterTransitionToRollback fail point enabled. Blocking until fail "
+                 "point is disabled (rs_rollback).";
+        MONGO_FAIL_POINT_PAUSE_WHILE_SET(rollbackHangAfterTransitionToRollback);
+    }
+
     try {
         auto status = syncRollback(
             opCtx, localOplog, rollbackSource, requiredRBID, replCoord, replicationProcess);
diff --git a/src/mongo/shell/assert.js b/src/mongo/shell/assert.js
index 364a9b453ed..cae620c7c70 100644
--- a/src/mongo/shell/assert.js
+++ b/src/mongo/shell/assert.js
@@ -687,3 +687,13 @@ assert.gleErrorRegex = function(dbOrGLEDoc, regex, msg) {
                  " :" + msg);
     }
 };
+
+assert.includes = function(haystack, needle, msg) {
+    if (!haystack.includes(needle)) {
+        var assertMsg = "string [" + haystack + "] does not include [" + needle + "]";
+        if (msg) {
+            assertMsg += ", " + msg;
+        }
+        doassert(assertMsg);
+    }
+};