SERVER-43941: Add "errorLabels" field to failCommand failpoint

11 files changed, 288 insertions, 11 deletions

diff --git a/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml b/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml
index 1c4b7bd6daa..332c698ab63 100644
--- a/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml
+++ b/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml
@@ -69,6 +69,8 @@ selector:
   - jstests/sharding/movechunk_interrupt_at_primary_stepdown.js
   # Enable after SERVER-31083 is backported and available on 4.2 and 4.0 binaries
   - jstests/sharding/enable_sharding_with_primary.js
+  # Enable after SERVER-43941 is backported or 4.4 becomes last-stable
+  - jstests/sharding/transactions_error_labels.js
 executor:
   config:
     shell_options:
diff --git a/jstests/core/failcommand_failpoint.js b/jstests/core/failcommand_failpoint.js
index 4e94e052f8a..ff6c5aeae26 100644
--- a/jstests/core/failcommand_failpoint.js
+++ b/jstests/core/failcommand_failpoint.js
@@ -4,6 +4,8 @@
 (function() {
 "use strict";
 
+load("jstests/libs/fixture_helpers.js");
+
 const testDB = db.getSiblingDB("test_failcommand");
 const adminDB = db.getSiblingDB("admin");
 
@@ -328,4 +330,159 @@ assert.commandWorked(
 res = assert.commandWorkedIgnoringWriteConcernErrors(testDB.runCommand(
     {insert: "foo", documents: [{x: "doc_for_namespace_case_should_trigger_wce"}]}));
 assert.eq(res.writeConcernError, {code: ErrorCodes.InternalError, errmsg: "foo"});
+
+// Test failing with error labels will not make `times` decrement twice.
+assert.commandWorked(adminDB.runCommand({
+    configureFailPoint: "failCommand",
+    mode: {times: 1},
+    data: {
+        errorCode: ErrorCodes.BadValue,
+        failCommands: ["ping"],
+        errorLabels: ["Foo", "Bar"],
+        threadName: threadName
+    }
+}));
+res = assert.commandFailedWithCode(testDB.runCommand({ping: 1}), ErrorCodes.BadValue);
+assert.eq(res.errorLabels, ["Foo", "Bar"], res);
+assert.commandWorked(testDB.runCommand({ping: 1}));
+
+// Test specifying both writeConcernError and errorLabels.
+assert.commandWorked(adminDB.runCommand({
+    configureFailPoint: "failCommand",
+    mode: {times: 1},
+    data: {
+        writeConcernError: {code: 12345, errmsg: "hello"},
+        failCommands: ["insert"],
+        errorLabels: ["Foo", "Bar"],
+        threadName: threadName
+    }
+}));
+res = testDB.runCommand({insert: "c", documents: [{}]});
+assert.eq(res.writeConcernError, {code: 12345, errmsg: "hello"});
+assert.eq(res.errorLabels, ["Foo", "Bar"], res);
+
+// Test failCommand with empty errorLabels.
+assert.commandWorked(adminDB.runCommand({
+    configureFailPoint: "failCommand",
+    mode: {times: 1},
+    data: {
+        errorCode: ErrorCodes.BadValue,
+        failCommands: ["ping"],
+        errorLabels: [],
+        threadName: threadName
+    }
+}));
+res = assert.commandFailedWithCode(testDB.runCommand({ping: 1}), ErrorCodes.BadValue);
+// There should be no errorLabels field if no error labels provided in failCommand.
+assert(!res.hasOwnProperty("errorLabels"));
+
+// Test specifying both writeConcernError and empty errorLabels.
+assert.commandWorked(adminDB.runCommand({
+    configureFailPoint: "failCommand",
+    mode: {times: 1},
+    data: {
+        writeConcernError: {code: 12345, errmsg: "hello"},
+        failCommands: ["insert"],
+        errorLabels: [],
+        threadName: threadName
+    }
+}));
+res = testDB.runCommand({insert: "c", documents: [{}]});
+assert.eq(res.writeConcernError, {code: 12345, errmsg: "hello"});
+// There should be no errorLabels field if no error labels provided in failCommand.
+assert(!res.hasOwnProperty("errorLabels"));
+
+// Test specifying errorLabels without errorCode or writeConcernError.
+assert.commandWorked(adminDB.runCommand({
+    configureFailPoint: "failCommand",
+    mode: {times: 1},
+    data: {failCommands: ["ping"], errorLabels: ["Foo", "Bar"], threadName: threadName}
+}));
+// The command should not fail if no errorCode or writeConcernError specified.
+res = assert.commandWorked(testDB.runCommand({ping: 1}));
+// As the command does not fail, there should not be any error labels even if errorLabels is
+// specified in the failCommand.
+assert(!res.hasOwnProperty("errorLabels"), res);
+assert.commandWorked(adminDB.runCommand({configureFailPoint: "failCommand", mode: "off"}));
+
+// Only run error labels override tests for replica set because the tests require retryable writes.
+// And mongos doesn't return RetryableWriteError labels.
+if (!FixtureHelpers.isReplSet(adminDB)) {
+    jsTestLog("Skipping error labels override tests");
+    return;
+}
+
+// Test error labels override.
+assert.commandWorked(adminDB.runCommand({
+    configureFailPoint: "failCommand",
+    mode: {times: 1},
+    data: {
+        errorCode: ErrorCodes.NotMaster,
+        failCommands: ["insert"],
+        errorLabels: ["Foo"],
+        threadName: threadName
+    }
+}));
+// This normally fails with RetryableWriteError label.
+res = assert.commandFailedWithCode(
+    testDB.runCommand(
+        {insert: "test", documents: [{x: "retryable_write"}], txnNumber: NumberLong(0)}),
+    ErrorCodes.NotMaster);
+// Test that failCommand overrides the error label to "Foo".
+assert.eq(res.errorLabels, ["Foo"], res);
+
+// Test error labels override while specifying writeConcernError.
+assert.commandWorked(adminDB.runCommand({
+    configureFailPoint: "failCommand",
+    mode: {times: 1},
+    data: {
+        writeConcernError: {code: ErrorCodes.NotMaster, errmsg: "hello"},
+        failCommands: ["insert"],
+        errorLabels: ["Foo"],
+        threadName: threadName
+    }
+}));
+// This normally fails with RetryableWriteError label.
+res = testDB.runCommand(
+    {insert: "test", documents: [{x: "retryable_write"}], txnNumber: NumberLong(0)});
+assert.eq(res.writeConcernError, {code: ErrorCodes.NotMaster, errmsg: "hello"});
+// Test that failCommand overrides the error label to "Foo".
+assert.eq(res.errorLabels, ["Foo"], res);
+
+// Test error labels override with empty errorLabels.
+assert.commandWorked(adminDB.runCommand({
+    configureFailPoint: "failCommand",
+    mode: {times: 1},
+    data: {
+        errorCode: ErrorCodes.NotMaster,
+        failCommands: ["insert"],
+        errorLabels: [],
+        threadName: threadName
+    }
+}));
+// This normally fails with RetryableWriteError label.
+res = assert.commandFailedWithCode(
+    testDB.runCommand(
+        {insert: "test", documents: [{x: "retryable_write"}], txnNumber: NumberLong(0)}),
+    ErrorCodes.NotMaster);
+// There should be no errorLabels field if no error labels provided in failCommand.
+assert(!res.hasOwnProperty("errorLabels"), res);
+
+// Test error labels override with empty errorLabels while specifying writeConcernError.
+assert.commandWorked(adminDB.runCommand({
+    configureFailPoint: "failCommand",
+    mode: {times: 1},
+    data: {
+        writeConcernError: {code: ErrorCodes.NotMaster, errmsg: "hello"},
+        failCommands: ["insert"],
+        errorLabels: [],
+        threadName: threadName
+    }
+}));
+// This normally fails with RetryableWriteError label.
+res = testDB.runCommand(
+    {insert: "test", documents: [{x: "retryable_write"}], txnNumber: NumberLong(0)});
+assert.eq(res.writeConcernError, {code: ErrorCodes.NotMaster, errmsg: "hello"});
+// There should be no errorLabels field if no error labels provided in failCommand.
+assert(!res.hasOwnProperty("errorLabels"), res);
 }());
diff --git a/jstests/replsets/transient_txn_error_labels.js b/jstests/replsets/transient_txn_error_labels.js
index b82220b6ce1..f48ff6f4aaa 100644
--- a/jstests/replsets/transient_txn_error_labels.js
+++ b/jstests/replsets/transient_txn_error_labels.js
@@ -44,6 +44,28 @@ let res = secondarySessionDb.runCommand({
 assert.commandFailedWithCode(res, ErrorCodes.NotMaster);
 assert.eq(res.errorLabels, ["TransientTransactionError"]);
 
+jsTest.log("failCommand with errorLabels but without errorCode or writeConcernError should not " +
+           "interfere with server's error labels attaching");
+txnNumber++;
+// This failCommand should have no effect.
+assert.commandWorked(secondary.adminCommand({
+    configureFailPoint: "failCommand",
+    mode: "alwaysOn",
+    data: {errorLabels: ["foo"], failCommands: ["insert"]}
+}));
+res = secondarySessionDb.runCommand({
+    insert: collName,
+    documents: [{_id: "insert-1"}],
+    readConcern: {level: "snapshot"},
+    txnNumber: NumberLong(txnNumber),
+    startTransaction: true,
+    autocommit: false
+});
+assert.commandFailedWithCode(res, ErrorCodes.NotMaster);
+// Server should continue to return TransientTransactionError label.
+assert.eq(res.errorLabels, ["TransientTransactionError"]);
+assert.commandWorked(secondary.adminCommand({configureFailPoint: "failCommand", mode: "off"}));
+
 jsTest.log("Insert as a retryable write on secondary should fail with retryable error labels");
 txnNumber++;
 // Insert as a retryable write.
diff --git a/jstests/sharding/transactions_error_labels.js b/jstests/sharding/transactions_error_labels.js
index f295dddf588..fe296509678 100644
--- a/jstests/sharding/transactions_error_labels.js
+++ b/jstests/sharding/transactions_error_labels.js
@@ -119,6 +119,20 @@ const runCommitTests = function(commandSentToShard) {
     checkMongosResponse(res, ErrorCodes.NoSuchTransaction, "TransientTransactionError", null);
     turnOffFailCommand(st.rs0);
 
+    jsTest.log("failCommand with errorLabels but without errorCode or writeConcernError should " +
+               "not interfere with mongos' error labels attaching");
+    assert.commandWorked(st.s.adminCommand({
+        configureFailPoint: "failCommand",
+        mode: "alwaysOn",
+        data: {failCommands: ["insert"], errorLabels: ["foo"]}
+    }));
+    assert.commandWorked(startTransaction(mongosSession, dbName, collName));
+    abortTransactionDirectlyOnParticipant(
+        st.rs0, mongosSession.getSessionId(), mongosSession.getTxnNumber_forTesting());
+    res = mongosSession.commitTransaction_forTesting();
+    checkMongosResponse(res, ErrorCodes.NoSuchTransaction, "TransientTransactionError", null);
+    assert.commandWorked(st.s.adminCommand({configureFailPoint: "failCommand", mode: "off"}));
+
     jsTest.log("Mongos does not attach any error label if " + commandSentToShard +
                " returns NoSuchTransaction with writeConcern error.");
     failCommandWithWriteConcernError(st.rs0, commandSentToShard);
@@ -174,6 +188,43 @@ turnOffFailCommand(st.rs0);
 assert.commandFailedWithCode(mongosSession.abortTransaction_forTesting(),
                              ErrorCodes.NoSuchTransaction);
 
+jsTest.log("'TransientTransactionError' label is attached if write statement returns " +
+           "WriteConflict via failCommand on mongos");
+assert.commandWorked(st.s.adminCommand({
+    configureFailPoint: "failCommand",
+    mode: "alwaysOn",
+    data: {failCommands: ["insert"], errorCode: ErrorCodes.WriteConflict}
+}));
+res = startTransaction(mongosSession, dbName, collName);
+checkMongosResponse(res, ErrorCodes.WriteConflict, "TransientTransactionError", null);
+assert.commandWorked(st.s.adminCommand({configureFailPoint: "failCommand", mode: "off"}));
+assert.commandFailedWithCode(mongosSession.abortTransaction_forTesting(),
+                             ErrorCodes.NoSuchTransaction);
+
+jsTest.log("failCommand with errorLabels should override labels attached by mongos");
+assert.commandWorked(st.s.adminCommand({
+    configureFailPoint: "failCommand",
+    mode: "alwaysOn",
+    data: {failCommands: ["insert"], errorCode: ErrorCodes.WriteConflict, errorLabels: ["foo"]}
+}));
+res = startTransaction(mongosSession, dbName, collName);
+checkMongosResponse(res, ErrorCodes.WriteConflict, "foo", null);
+assert.commandWorked(st.s.adminCommand({configureFailPoint: "failCommand", mode: "off"}));
+assert.commandFailedWithCode(mongosSession.abortTransaction_forTesting(),
+                             ErrorCodes.NoSuchTransaction);
+
+jsTest.log("failCommand with empty errorLabels should suppress labels attached by mongos");
+assert.commandWorked(st.s.adminCommand({
+    configureFailPoint: "failCommand",
+    mode: "alwaysOn",
+    data: {failCommands: ["insert"], errorCode: ErrorCodes.WriteConflict, errorLabels: []}
+}));
+res = startTransaction(mongosSession, dbName, collName);
+checkMongosResponse(res, ErrorCodes.WriteConflict, null, null);
+assert.commandWorked(st.s.adminCommand({configureFailPoint: "failCommand", mode: "off"}));
+assert.commandFailedWithCode(mongosSession.abortTransaction_forTesting(),
+                             ErrorCodes.NoSuchTransaction);
+
 // statements prior to commit network error
 failCommandWithError(
     st.rs0, {commandToFail: "insert", errorCode: ErrorCodes.InternalError, closeConnection: true});
diff --git a/src/mongo/db/SConscript b/src/mongo/db/SConscript
index 3375d49d295..8c051a3916a 100644
--- a/src/mongo/db/SConscript
+++ b/src/mongo/db/SConscript
@@ -1335,6 +1335,7 @@ env.Library(
         'error_labels.cpp',
     ],
     LIBDEPS=[
+        'commands',
         'logical_session_id',
     ],
 )
diff --git a/src/mongo/db/commands.cpp b/src/mongo/db/commands.cpp
index fd6c28d145c..0345bd31680 100644
--- a/src/mongo/db/commands.cpp
+++ b/src/mongo/db/commands.cpp
@@ -50,6 +50,7 @@
 #include "mongo/db/command_generic_argument.h"
 #include "mongo/db/commands/test_commands_enabled.h"
 #include "mongo/db/curop.h"
+#include "mongo/db/error_labels.h"
 #include "mongo/db/jsobj.h"
 #include "mongo/db/namespace_string.h"
 #include "mongo/db/read_write_concern_defaults.h"
@@ -471,6 +472,11 @@ constexpr StringData CommandHelpers::kHelpFieldName;
 MONGO_FAIL_POINT_DEFINE(failCommand);
 MONGO_FAIL_POINT_DEFINE(waitInCommandMarkKillOnClientDisconnect);
 
+// A decoration representing error labels specified in a failCommand failpoint that has affected a
+// command in this OperationContext.
+const OperationContext::Decoration<boost::optional<BSONArray>> errorLabelsOverride =
+    OperationContext::declareDecoration<boost::optional<BSONArray>>();
+
 bool CommandHelpers::shouldActivateFailCommandFailPoint(const BSONObj& data,
                                                         StringData cmdName,
                                                         Client* client,
@@ -515,7 +521,16 @@ void CommandHelpers::evaluateFailCommandFailPoint(OperationContext* opCtx,
     bool hasErrorCode;
     long long errorCode;
     failCommand.executeIf(
-        [&](const BSONObj&) {
+        [&](const BSONObj& data) {
+            if (data.hasField(kErrorLabelsFieldName) &&
+                data[kErrorLabelsFieldName].type() == Array) {
+                // Propagate error labels specified in the failCommand failpoint to the
+                // OperationContext decoration to override getErrorLabels() behaviors.
+                invariant(!errorLabelsOverride(opCtx));
+                errorLabelsOverride(opCtx).emplace(
+                    data.getObjectField(kErrorLabelsFieldName).getOwned());
+            }
+
             if (closeConnection) {
                 opCtx->getClient()->session()->end();
                 log() << "Failing command '" << commandName
diff --git a/src/mongo/db/commands.h b/src/mongo/db/commands.h
index 6f174a40dfa..ea62ac51fc0 100644
--- a/src/mongo/db/commands.h
+++ b/src/mongo/db/commands.h
@@ -56,6 +56,7 @@ namespace mongo {
 
 extern FailPoint failCommand;
 extern FailPoint waitInCommandMarkKillOnClientDisconnect;
+extern const OperationContext::Decoration<boost::optional<BSONArray>> errorLabelsOverride;
 
 class Command;
 class CommandInvocation;
diff --git a/src/mongo/db/error_labels.cpp b/src/mongo/db/error_labels.cpp
index d56ba6dd82d..383f4f77551 100644
--- a/src/mongo/db/error_labels.cpp
+++ b/src/mongo/db/error_labels.cpp
@@ -28,6 +28,7 @@
  */
 
 #include "mongo/db/error_labels.h"
+#include "mongo/db/commands.h"
 
 namespace mongo {
 
@@ -94,17 +95,28 @@ bool ErrorLabelBuilder::_isCommitOrAbort() const {
         _commandName == "abortTransaction";
 }
 
-BSONObj getErrorLabels(const OperationSessionInfoFromClient& sessionOptions,
+BSONObj getErrorLabels(OperationContext* opCtx,
+                       const OperationSessionInfoFromClient& sessionOptions,
                        const std::string& commandName,
                        boost::optional<ErrorCodes::Error> code,
                        boost::optional<ErrorCodes::Error> wcCode,
                        bool isInternalClient) {
-    BSONArrayBuilder labelArray;
+    if (MONGO_unlikely(errorLabelsOverride(opCtx))) {
+        // This command was failed by a failCommand failpoint. Thus, we return the errorLabels
+        // specified in the failpoint to supress any other error labels that would otherwise be
+        // returned by the ErrorLabelBuilder.
+        if (errorLabelsOverride(opCtx).get().isEmpty()) {
+            return BSONObj();
+        } else {
+            return BSON(kErrorLabelsFieldName << errorLabelsOverride(opCtx).get());
+        }
+    }
 
+    BSONArrayBuilder labelArray;
     ErrorLabelBuilder labelBuilder(sessionOptions, commandName, code, wcCode, isInternalClient);
     labelBuilder.build(labelArray);
 
-    return (labelArray.arrSize() > 0) ? BSON("errorLabels" << labelArray.arr()) : BSONObj();
+    return (labelArray.arrSize() > 0) ? BSON(kErrorLabelsFieldName << labelArray.arr()) : BSONObj();
 }
 
 bool isTransientTransactionError(ErrorCodes::Error code,
diff --git a/src/mongo/db/error_labels.h b/src/mongo/db/error_labels.h
index 7490d862372..6066963fa98 100644
--- a/src/mongo/db/error_labels.h
+++ b/src/mongo/db/error_labels.h
@@ -32,7 +32,7 @@
 #include "mongo/db/logical_session_id.h"
 
 namespace mongo {
-
+static constexpr StringData kErrorLabelsFieldName = "errorLabels"_sd;
 namespace ErrorLabel {
 // PLEASE CONSULT DRIVERS BEFORE ADDING NEW ERROR LABELS.
 static constexpr StringData kTransientTransaction = "TransientTransactionError"_sd;
@@ -71,7 +71,8 @@ private:
 /**
  * Returns the error labels for the given error.
  */
-BSONObj getErrorLabels(const OperationSessionInfoFromClient& sessionOptions,
+BSONObj getErrorLabels(OperationContext* opCtx,
+                       const OperationSessionInfoFromClient& sessionOptions,
                        const std::string& commandName,
                        boost::optional<ErrorCodes::Error> code,
                        boost::optional<ErrorCodes::Error> wcCode,
diff --git a/src/mongo/db/service_entry_point_common.cpp b/src/mongo/db/service_entry_point_common.cpp
index c64241e5bff..4a42657c92a 100644
--- a/src/mongo/db/service_entry_point_common.cpp
+++ b/src/mongo/db/service_entry_point_common.cpp
@@ -628,6 +628,14 @@ bool runCommandImpl(OperationContext* opCtx,
                 [&](const BSONObj& data) {
                     bb.append(data["writeConcernError"]);
                     reallyWait = false;
+                    if (data.hasField(kErrorLabelsFieldName) &&
+                        data[kErrorLabelsFieldName].type() == Array) {
+                        // Propagate error labels specified in the failCommand failpoint to the
+                        // OperationContext decoration to override getErrorLabels() behaviors.
+                        invariant(!errorLabelsOverride(opCtx));
+                        errorLabelsOverride(opCtx).emplace(
+                            data.getObjectField(kErrorLabelsFieldName).getOwned());
+                    }
                 },
                 [&](const BSONObj& data) {
                     return CommandHelpers::shouldActivateFailCommandFailPoint(
@@ -719,8 +727,8 @@ bool runCommandImpl(OperationContext* opCtx,
         }
         auto isInternalClient = opCtx->getClient()->session() &&
             (opCtx->getClient()->session()->getTags() & transport::Session::kInternalClient);
-        auto errorLabels =
-            getErrorLabels(sessionOptions, command->getName(), code, wcCode, isInternalClient);
+        auto errorLabels = getErrorLabels(
+            opCtx, sessionOptions, command->getName(), code, wcCode, isInternalClient);
         replyBuilder->getBodyBuilder().appendElements(errorLabels);
     }
 
@@ -1024,8 +1032,8 @@ void execCommandDatabase(OperationContext* opCtx,
         }
         auto isInternalClient = opCtx->getClient()->session() &&
             (opCtx->getClient()->session()->getTags() & transport::Session::kInternalClient);
-        auto errorLabels =
-            getErrorLabels(sessionOptions, command->getName(), e.code(), wcCode, isInternalClient);
+        auto errorLabels = getErrorLabels(
+            opCtx, sessionOptions, command->getName(), e.code(), wcCode, isInternalClient);
         extraFieldsBuilder.appendElements(errorLabels);
 
         BSONObjBuilder metadataBob;
diff --git a/src/mongo/s/commands/strategy.cpp b/src/mongo/s/commands/strategy.cpp
index 6a4e5f6da85..2f734c11654 100644
--- a/src/mongo/s/commands/strategy.cpp
+++ b/src/mongo/s/commands/strategy.cpp
@@ -274,6 +274,13 @@ void execCommandClient(OperationContext* opCtx,
         failCommand.executeIf(
             [&](const BSONObj& data) {
                 result->getBodyBuilder().append(data["writeConcernError"]);
+                if (data.hasField(kErrorLabelsFieldName) &&
+                    data[kErrorLabelsFieldName].type() == Array) {
+                    auto labels = data.getObjectField(kErrorLabelsFieldName).getOwned();
+                    if (!labels.isEmpty()) {
+                        result->getBodyBuilder().append(kErrorLabelsFieldName, BSONArray(labels));
+                    }
+                }
             },
             [&](const BSONObj& data) {
                 return CommandHelpers::shouldActivateFailCommandFailPoint(
@@ -629,7 +636,7 @@ void runCommand(OperationContext* opCtx,
         // isInternalClient is set to true to suppress mongos from returning the RetryableWriteError
         // label.
         auto errorLabels = getErrorLabels(
-            osi, command->getName(), e.code(), boost::none, true /* isInternalClient */);
+            opCtx, osi, command->getName(), e.code(), boost::none, true /* isInternalClient */);
         errorBuilder->appendElements(errorLabels);
         throw;
     }