SERVER-40631 Attach TransientTransactionError error label for network error codes

6 files changed, 118 insertions, 16 deletions

diff --git a/jstests/replsets/transient_txn_error_labels.js b/jstests/replsets/transient_txn_error_labels.js
index 18d6b4d4b2b..06385622e60 100644
--- a/jstests/replsets/transient_txn_error_labels.js
+++ b/jstests/replsets/transient_txn_error_labels.js
@@ -200,6 +200,40 @@
     thread.join();
     assert.commandWorked(thread.returnData());
 
+    // Re-create the collection for later test cases.
+    assert.commandWorked(testDB.createCollection(collName, {writeConcern: {w: "majority"}}));
+
+    jsTest.log("Network errors for in-progress statements should be transient");
+    session.startTransaction();
+    assert.commandWorked(testDB.adminCommand({
+        configureFailPoint: "failCommand",
+        mode: "alwaysOn",
+        data: {errorCode: ErrorCodes.HostUnreachable, failCommands: ["aggregate"]}
+    }));
+    res = sessionDb.runCommand({aggregate: collName, pipeline: [{$match: {}}], cursor: {}});
+    assert.commandFailedWithCode(res, ErrorCodes.HostUnreachable);
+    assert.eq(res.errorLabels, ["TransientTransactionError"]);
+    session.abortTransaction_forTesting();
+    assert.commandWorked(testDB.adminCommand({configureFailPoint: "failCommand", mode: "off"}));
+
+    jsTest.log("Network errors for commit should not be transient");
+    session.startTransaction();
+    assert.commandWorked(sessionColl.insert({_id: "commitTransaction-network-error"}));
+    assert.commandWorked(testDB.adminCommand({
+        configureFailPoint: "failCommand",
+        mode: "alwaysOn",
+        data: {errorCode: ErrorCodes.HostUnreachable, failCommands: ["commitTransaction"]}
+    }));
+    res = sessionDb.adminCommand({
+        commitTransaction: 1,
+        txnNumber: NumberLong(session.getTxnNumber_forTesting()),
+        autocommit: false
+    });
+    assert.commandFailedWithCode(res, ErrorCodes.HostUnreachable);
+    assert(!res.hasOwnProperty("errorLabels"), tojson(res));
+    session.abortTransaction_forTesting();
+    assert.commandWorked(testDB.adminCommand({configureFailPoint: "failCommand", mode: "off"}));
+
     session.endSession();
 
     st.stop();
diff --git a/jstests/sharding/transactions_error_labels.js b/jstests/sharding/transactions_error_labels.js
index 50864cde1c6..d0b4188c1a7 100644
--- a/jstests/sharding/transactions_error_labels.js
+++ b/jstests/sharding/transactions_error_labels.js
@@ -9,12 +9,13 @@
     const collName = "foo";
     const ns = dbName + "." + collName;
 
-    const failCommandWithError = function(rst, commandToFail, errorCode) {
+    const failCommandWithError = function(rst, {commandToFail, errorCode, closeConnection}) {
         rst.nodes.forEach(function(node) {
             assert.commandWorked(node.getDB("admin").runCommand({
                 configureFailPoint: "failCommand",
                 mode: "alwaysOn",
                 data: {
+                    closeConnection: closeConnection,
                     errorCode: errorCode,
                     failCommands: [commandToFail],
                     failInternalCommands: true  // mongod sees mongos as an internal client
@@ -127,6 +128,18 @@
         res = mongosSession.commitTransaction_forTesting();
         checkMongosResponse(res, ErrorCodes.NoSuchTransaction, null, true);
         turnOffFailCommand(st.rs0);
+
+        jsTest.log("No error label for network error if " + commandSentToShard +
+                   " returns network error");
+        assert.commandWorked(startTransaction(mongosSession, dbName, collName));
+        failCommandWithError(st.rs0, {
+            commandToFail: commandSentToShard,
+            errorCode: ErrorCodes.InternalError,
+            closeConnection: true
+        });
+        res = mongosSession.commitTransaction_forTesting();
+        checkMongosResponse(res, ErrorCodes.HostUnreachable, false /* expectedErrorLabel */, null);
+        turnOffFailCommand(st.rs0);
     };
 
     let st = new ShardingTest({shards: 2, config: 1, mongosOptions: {verbose: 3}});
@@ -153,12 +166,23 @@
     // write statement
     jsTest.log(
         "'TransientTransactionError' label is attached if write statement returns WriteConflict");
-    failCommandWithError(st.rs0, "insert", ErrorCodes.WriteConflict);
+    failCommandWithError(
+        st.rs0,
+        {commandToFail: "insert", errorCode: ErrorCodes.WriteConflict, closeConnection: false});
     res = startTransaction(mongosSession, dbName, collName);
     checkMongosResponse(res, ErrorCodes.WriteConflict, "TransientTransactionError", null);
     turnOffFailCommand(st.rs0);
     mongosSession.abortTransaction();
 
+    // statements prior to commit network error
+    failCommandWithError(
+        st.rs0,
+        {commandToFail: "insert", errorCode: ErrorCodes.InternalError, closeConnection: true});
+    res = startTransaction(mongosSession, dbName, collName);
+    checkMongosResponse(res, ErrorCodes.HostUnreachable, "TransientTransactionError", null);
+    turnOffFailCommand(st.rs0);
+    mongosSession.abortTransaction();
+
     // commitTransaction for single-shard transaction (mongos sends commitTransaction)
     runCommitTests("commitTransaction");
 
diff --git a/jstests/sharding/transactions_writes_not_retryable.js b/jstests/sharding/transactions_writes_not_retryable.js
index d0253653362..76cb3af7854 100644
--- a/jstests/sharding/transactions_writes_not_retryable.js
+++ b/jstests/sharding/transactions_writes_not_retryable.js
@@ -54,19 +54,11 @@
             "expected write in transaction not to be retried on closed connection, cmd: " +
                 tojson(writeCmd) + ", sharded: " + isSharded);
 
-        let confirmIsNetworkError = function(response) {
-            if (response.hasOwnProperty('writeErrors')) {
-                assert.neq(null, res.writeErrors[0]);
-                assert(ErrorCodes.isNetworkError(res.writeErrors[0].code),
-                       "expected network error, got: " + tojson(res.code));
-            } else {
-                assert.eq('findAndModify', writeCmdName);
-                assert(ErrorCodes.isNetworkError(res.code),
-                       "expected network error, got: " + tojson(res.code));
-            }
-        };
-
-        confirmIsNetworkError(res);
+        // Network errors during sharded transactions are transient transaction errors, so they're
+        // returned as top level codes for all commands, including batch writes.
+        assert(ErrorCodes.isNetworkError(res.code),
+               "expected network error, got: " + tojson(res.code));
+        assert.eq(res.errorLabels, ["TransientTransactionError"]);
         assert.commandFailedWithCode(session.abortTransaction_forTesting(),
                                      ErrorCodes.NoSuchTransaction);
 
diff --git a/src/mongo/base/SConscript b/src/mongo/base/SConscript
index 56617fcdd66..d58f57ebb61 100644
--- a/src/mongo/base/SConscript
+++ b/src/mongo/base/SConscript
@@ -49,6 +49,7 @@ env.CppUnitTest('base_test',
                  'status_test.cpp',
                  'status_with_test.cpp',
                  'string_data_test.cpp',
+                 'transaction_error_test.cpp',
                  'uuid_test.cpp',
                 ])
 
diff --git a/src/mongo/base/transaction_error.cpp b/src/mongo/base/transaction_error.cpp
index 80a720c8dae..da395b64174 100644
--- a/src/mongo/base/transaction_error.cpp
+++ b/src/mongo/base/transaction_error.cpp
@@ -57,7 +57,7 @@ bool isTransientTransactionError(ErrorCodes::Error code,
         // For commands other than "commitTransaction", we know there's no side-effect for these
         // errors, but it's not true for "commitTransaction" if a failover happens.
         isTransient |= ErrorCodes::isNotMasterError(code) || ErrorCodes::isShutdownError(code) ||
-            code == ErrorCodes::NoSuchTransaction;
+            ErrorCodes::isNetworkError(code) || code == ErrorCodes::NoSuchTransaction;
     }
 
     return isTransient;
diff --git a/src/mongo/base/transaction_error_test.cpp b/src/mongo/base/transaction_error_test.cpp
new file mode 100644
index 00000000000..5baa8457e67
--- /dev/null
+++ b/src/mongo/base/transaction_error_test.cpp
@@ -0,0 +1,51 @@
+/**
+ *    Copyright (C) 2019-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/base/transaction_error.h"
+#include "mongo/unittest/unittest.h"
+
+namespace mongo {
+namespace {
+
+TEST(IsTransientTransactionErrorTest, NetworkErrorsAreTransientBeforeCommit) {
+    ASSERT(isTransientTransactionError(ErrorCodes::HostUnreachable,
+                                       false /* hasWriteConcernError */,
+                                       false /* isCommitTransaction */));
+}
+
+TEST(IsTransientTransactionErrorTest, NetworkErrorsAreNotTransientOnCommit) {
+    ASSERT(!isTransientTransactionError(ErrorCodes::HostUnreachable,
+                                        false /* hasWriteConcernError */,
+                                        true /* isCommitTransaction */));
+}
+
+}  // namespace
+}  // namespace mongo