SERVER-46091 Add new failpoint to support driver testing of ResumableChangeStreamError label

10 files changed, 192 insertions, 1 deletions

diff --git a/jstests/noPassthrough/change_stream_error_label.js b/jstests/noPassthrough/change_stream_error_label.js
index 19c78ba25a3..e999a324526 100644
--- a/jstests/noPassthrough/change_stream_error_label.js
+++ b/jstests/noPassthrough/change_stream_error_label.js
@@ -25,5 +25,42 @@ assert.commandFailedWithCode(err, ErrorCodes.NotMasterNoSlaveOk);
 assert("errorLabels" in err, err);
 assert.contains("ResumableChangeStreamError", err.errorLabels, err);
 
+// Now verify that the 'failGetMoreAfterCursorCheckout' failpoint can effectively exercise the
+// error label generation logic for change stream getMores.
+function testFailGetMoreAfterCursorCheckoutFailpoint({errorCode, expectedLabel}) {
+    // Re-enable "slaveOk" on the test connection.
+    testDB.getMongo().setSlaveOk(true);
+
+    // Activate the failpoint and set the exception that it will throw.
+    assert.commandWorked(testDB.adminCommand({
+        configureFailPoint: "failGetMoreAfterCursorCheckout",
+        mode: "alwaysOn",
+        data: {"errorCode": errorCode}
+    }));
+
+    // Now open a valid $changeStream cursor...
+    const aggCmdRes = assert.commandWorked(
+        coll.runCommand("aggregate", {pipeline: [{$changeStream: {}}], cursor: {}}));
+
+    // ... run a getMore using the cursorID from the original command response, and confirm that the
+    // expected error was thrown...
+    const getMoreRes = assert.commandFailedWithCode(
+        testDB.runCommand({getMore: aggCmdRes.cursor.id, collection: coll.getName()}), errorCode);
+
+    /// ... and confirm that the label is present or absent depending on the "expectedLabel" value.
+    const errorLabels = (getMoreRes.errorLabels || []);
+    assert.eq("errorLabels" in getMoreRes, expectedLabel, getMoreRes);
+    assert.eq(errorLabels.includes("ResumableChangeStreamError"), expectedLabel, getMoreRes);
+
+    // Finally, disable the failpoint.
+    assert.commandWorked(
+        testDB.adminCommand({configureFailPoint: "failGetMoreAfterCursorCheckout", mode: "off"}));
+}
+// Test the expected output for both resumable and non-resumable error codes.
+testFailGetMoreAfterCursorCheckoutFailpoint(
+    {errorCode: ErrorCodes.ShutdownInProgress, expectedLabel: true});
+testFailGetMoreAfterCursorCheckoutFailpoint(
+    {errorCode: ErrorCodes.FailedToParse, expectedLabel: false});
+
 rst.stopSet();
 }());
\ No newline at end of file
diff --git a/jstests/noPassthrough/fail_point_getmore_after_cursor_checkout.js b/jstests/noPassthrough/fail_point_getmore_after_cursor_checkout.js
new file mode 100644
index 00000000000..047cb2f2983
--- /dev/null
+++ b/jstests/noPassthrough/fail_point_getmore_after_cursor_checkout.js
@@ -0,0 +1,51 @@
+/**
+ * Test that 'failGetMoreAfterCursorCheckout' works in both legacy and command read modes.
+ * @tags: [requires_replication, requires_journaling]
+ */
+(function() {
+"use strict";
+
+const rst = new ReplSetTest({nodes: 1});
+rst.startSet();
+rst.initiate();
+
+const testDB = rst.getPrimary().getDB(jsTestName());
+const coll = testDB.test;
+
+// Insert a set of test documents into the collection.
+for (let i = 0; i < 10; ++i) {
+    assert.commandWorked(coll.insert({_id: i}));
+}
+
+// Run the test for both 'commands' and 'legacy' read modes.
+for (let readMode of ["commands", "legacy"]) {
+    // Set the appropriate read mode for this test case.
+    testDB.getMongo().forceReadMode(readMode);
+
+    // Perform the test for both 'find' and 'aggregate' cursors.
+    for (let testCursor of [coll.find({}).sort({_id: 1}).batchSize(2),
+                            coll.aggregate([{$sort: {_id: 1}}], {cursor: {batchSize: 2}})]) {
+        // Activate the failpoint and set the exception that it will throw.
+        assert.commandWorked(testDB.adminCommand({
+            configureFailPoint: "failGetMoreAfterCursorCheckout",
+            mode: "alwaysOn",
+            data: {"errorCode": ErrorCodes.ShutdownInProgress}
+        }));
+
+        // Consume the documents from the first batch, leaving the cursor open.
+        assert.docEq(testCursor.next(), {_id: 0});
+        assert.docEq(testCursor.next(), {_id: 1});
+        assert.eq(testCursor.objsLeftInBatch(), 0);
+
+        // Issue a getMore and confirm that the failpoint throws the expected exception.
+        const getMoreRes = assert.throws(() => testCursor.hasNext() && testCursor.next());
+        assert.commandFailedWithCode(getMoreRes, ErrorCodes.ShutdownInProgress);
+
+        // Disable the failpoint.
+        assert.commandWorked(testDB.adminCommand(
+            {configureFailPoint: "failGetMoreAfterCursorCheckout", mode: "off"}));
+    }
+}
+
+rst.stopSet();
+}());
diff --git a/jstests/sharding/change_stream_error_label.js b/jstests/sharding/change_stream_error_label.js
index 15250e9e84b..c164d282de7 100644
--- a/jstests/sharding/change_stream_error_label.js
+++ b/jstests/sharding/change_stream_error_label.js
@@ -35,7 +35,42 @@ const expectedStopShardErrors = [
     ErrorCodes.NotMasterOrSecondary
 ];
 
-// Shard the collection, split at {_id: 0}, and move the upper chunk to the other shard.
+// First, verify that the 'failGetMoreAfterCursorCheckout' failpoint can effectively exercise the
+// error label generation logic for change stream getMores.
+function testFailGetMoreAfterCursorCheckoutFailpoint({errorCode, expectedLabel}) {
+    // Activate the failpoint and set the exception that it will throw.
+    assert.commandWorked(testDB.adminCommand({
+        configureFailPoint: "failGetMoreAfterCursorCheckout",
+        mode: "alwaysOn",
+        data: {"errorCode": errorCode}
+    }));
+
+    // Now open a valid $changeStream cursor...
+    const aggCmdRes = assert.commandWorked(
+        coll.runCommand("aggregate", {pipeline: [{$changeStream: {}}], cursor: {}}));
+
+    // ... run a getMore using the cursorID from the original command response, and confirm that the
+    // expected error was thrown...
+    const getMoreRes = assert.commandFailedWithCode(
+        testDB.runCommand({getMore: aggCmdRes.cursor.id, collection: coll.getName()}), errorCode);
+
+    /// ... and confirm that the label is present or absent depending on the "expectedLabel" value.
+    const errorLabels = (getMoreRes.errorLabels || []);
+    assert.eq("errorLabels" in getMoreRes, expectedLabel, getMoreRes);
+    assert.eq(errorLabels.includes("ResumableChangeStreamError"), expectedLabel, getMoreRes);
+
+    // Finally, disable the failpoint.
+    assert.commandWorked(
+        testDB.adminCommand({configureFailPoint: "failGetMoreAfterCursorCheckout", mode: "off"}));
+}
+// Test the expected output for both resumable and non-resumable error codes.
+testFailGetMoreAfterCursorCheckoutFailpoint(
+    {errorCode: ErrorCodes.ShutdownInProgress, expectedLabel: true});
+testFailGetMoreAfterCursorCheckoutFailpoint(
+    {errorCode: ErrorCodes.FailedToParse, expectedLabel: false});
+
+// Now test both aggregate and getMore under conditions of an actual cluster outage. Shard the
+// collection, split at {_id: 0}, and move the upper chunk to the other shard.
 st.shardColl(coll, {_id: 1}, {_id: 0}, {_id: 0});
 
 // Open a change stream on the collection...
diff --git a/src/mongo/db/commands.cpp b/src/mongo/db/commands.cpp
index e29e7128410..ccd2707ae8e 100644
--- a/src/mongo/db/commands.cpp
+++ b/src/mongo/db/commands.cpp
@@ -534,7 +534,13 @@ bool CommandHelpers::shouldActivateFailCommandFailPoint(const BSONObj& data,
     } catch (const ExceptionFor<ErrorCodes::InvalidNamespace>&) {
         return false;
     }
+    return shouldActivateFailCommandFailPoint(data, nss, cmd, client);
+}
 
+bool CommandHelpers::shouldActivateFailCommandFailPoint(const BSONObj& data,
+                                                        const NamespaceString& nss,
+                                                        const Command* cmd,
+                                                        Client* client) {
     if (cmd->getName() == "configureFailPoint"_sd)  // Banned even if in failCommands.
         return false;
 
diff --git a/src/mongo/db/commands.h b/src/mongo/db/commands.h
index a9467d34c0b..3eff8eb062d 100644
--- a/src/mongo/db/commands.h
+++ b/src/mongo/db/commands.h
@@ -272,6 +272,14 @@ struct CommandHelpers {
                                                    Client* client);
 
     /**
+     * Checks if the command passed in is in the list of failCommands defined in the fail point.
+     */
+    static bool shouldActivateFailCommandFailPoint(const BSONObj& data,
+                                                   const NamespaceString& nss,
+                                                   const Command* cmd,
+                                                   Client* client);
+
+    /**
      * Possibly uasserts according to the "failCommand" fail point.
      */
     static void evaluateFailCommandFailPoint(OperationContext* opCtx,
diff --git a/src/mongo/db/commands/getmore_cmd.cpp b/src/mongo/db/commands/getmore_cmd.cpp
index 91b6804f691..15b84ad2be1 100644
--- a/src/mongo/db/commands/getmore_cmd.cpp
+++ b/src/mongo/db/commands/getmore_cmd.cpp
@@ -555,6 +555,22 @@ public:
                 curOp->setGenericCursor_inlock(cursorPin->toGenericCursor());
             }
 
+            // If the 'failGetMoreAfterCursorCheckout' failpoint is enabled, throw an exception with
+            // the given 'errorCode' value, or ErrorCodes::InternalError if 'errorCode' is omitted.
+            failGetMoreAfterCursorCheckout.executeIf(
+                [](const BSONObj& data) {
+                    auto errorCode = (data["errorCode"] ? data["errorCode"].safeNumberLong()
+                                                        : ErrorCodes::InternalError);
+                    uasserted(errorCode, "Hit the 'failGetMoreAfterCursorCheckout' failpoint");
+                },
+                [&opCtx, &cursorPin](const BSONObj& data) {
+                    auto dataForFailCommand =
+                        data.addField(BSON("failCommands" << BSON_ARRAY("getMore")).firstElement());
+                    auto* getMoreCommand = CommandHelpers::findCommand("getMore");
+                    return CommandHelpers::shouldActivateFailCommandFailPoint(
+                        dataForFailCommand, cursorPin->nss(), getMoreCommand, opCtx->getClient());
+                });
+
             CursorId respondWithId = 0;
 
             CursorResponseBuilder nextBatch(reply, CursorResponseBuilder::Options());
diff --git a/src/mongo/db/query/find.cpp b/src/mongo/db/query/find.cpp
index 197475a9173..ffbae30a9ee 100644
--- a/src/mongo/db/query/find.cpp
+++ b/src/mongo/db/query/find.cpp
@@ -458,6 +458,22 @@ Message getMore(OperationContext* opCtx,
         curOp.setGenericCursor_inlock(cursorPin->toGenericCursor());
     }
 
+    // If the 'failGetMoreAfterCursorCheckout' failpoint is enabled, throw an exception with the
+    // specified 'errorCode' value, or ErrorCodes::InternalError if 'errorCode' is omitted.
+    failGetMoreAfterCursorCheckout.executeIf(
+        [](const BSONObj& data) {
+            auto errorCode = (data["errorCode"] ? data["errorCode"].safeNumberLong()
+                                                : ErrorCodes::InternalError);
+            uasserted(errorCode, "Hit the 'failGetMoreAfterCursorCheckout' failpoint");
+        },
+        [&opCtx, &nss](const BSONObj& data) {
+            auto dataForFailCommand =
+                data.addField(BSON("failCommands" << BSON_ARRAY("getMore")).firstElement());
+            auto* getMoreCommand = CommandHelpers::findCommand("getMore");
+            return CommandHelpers::shouldActivateFailCommandFailPoint(
+                dataForFailCommand, nss, getMoreCommand, opCtx->getClient());
+        });
+
     PlanExecutor::ExecState state;
 
     // We report keysExamined and docsExamined to OpDebug for a given getMore operation. To obtain
diff --git a/src/mongo/db/query/find_common.cpp b/src/mongo/db/query/find_common.cpp
index 6b8a3a97b30..78568ac6153 100644
--- a/src/mongo/db/query/find_common.cpp
+++ b/src/mongo/db/query/find_common.cpp
@@ -54,6 +54,8 @@ MONGO_FAIL_POINT_DEFINE(waitWithPinnedCursorDuringGetMoreBatch);
 
 MONGO_FAIL_POINT_DEFINE(waitBeforeUnpinningOrDeletingCursorAfterGetMoreBatch);
 
+MONGO_FAIL_POINT_DEFINE(failGetMoreAfterCursorCheckout);
+
 const OperationContext::Decoration<AwaitDataState> awaitDataState =
     OperationContext::declareDecoration<AwaitDataState>();
 
diff --git a/src/mongo/db/query/find_common.h b/src/mongo/db/query/find_common.h
index a0a4f747cda..44b941f7667 100644
--- a/src/mongo/db/query/find_common.h
+++ b/src/mongo/db/query/find_common.h
@@ -75,6 +75,10 @@ extern FailPoint waitWithPinnedCursorDuringGetMoreBatch;
 // has completed building the current batch.
 extern FailPoint waitBeforeUnpinningOrDeletingCursorAfterGetMoreBatch;
 
+// Enabling this failpoint will cause a getMore to fail with a specified exception after it has
+// checked out its cursor and the originating command has been made available to CurOp.
+extern FailPoint failGetMoreAfterCursorCheckout;
+
 /**
  * Suite of find/getMore related functions used in both the mongod and mongos query paths.
  */
diff --git a/src/mongo/s/query/cluster_find.cpp b/src/mongo/s/query/cluster_find.cpp
index 9598e1560e6..e06b79dcb6a 100644
--- a/src/mongo/s/query/cluster_find.cpp
+++ b/src/mongo/s/query/cluster_find.cpp
@@ -716,6 +716,22 @@ StatusWith<CursorResponse> ClusterFind::runGetMore(OperationContext* opCtx,
         CurOp::get(opCtx)->setGenericCursor_inlock(pinnedCursor.getValue().toGenericCursor());
     }
 
+    // If the 'failGetMoreAfterCursorCheckout' failpoint is enabled, throw an exception with the
+    // specified 'errorCode' value, or ErrorCodes::InternalError if 'errorCode' is omitted.
+    failGetMoreAfterCursorCheckout.executeIf(
+        [](const BSONObj& data) {
+            auto errorCode = (data["errorCode"] ? data["errorCode"].safeNumberLong()
+                                                : ErrorCodes::InternalError);
+            uasserted(errorCode, "Hit the 'failGetMoreAfterCursorCheckout' failpoint");
+        },
+        [&opCtx, &request](const BSONObj& data) {
+            auto dataForFailCommand =
+                data.addField(BSON("failCommands" << BSON_ARRAY("getMore")).firstElement());
+            auto* getMoreCommand = CommandHelpers::findCommand("getMore");
+            return CommandHelpers::shouldActivateFailCommandFailPoint(
+                dataForFailCommand, request.nss, getMoreCommand, opCtx->getClient());
+        });
+
     // If the 'waitAfterPinningCursorBeforeGetMoreBatch' fail point is enabled, set the 'msg'
     // field of this operation's CurOp to signal that we've hit this point.
     if (MONGO_unlikely(waitAfterPinningCursorBeforeGetMoreBatch.shouldFail())) {