SERVER-55648 Return top-level batch-write error in case of shutdown

3 files changed, 77 insertions, 0 deletions

diff --git a/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml b/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml
index 392607b4af9..a63bac55d57 100644
--- a/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml
+++ b/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml
@@ -132,6 +132,8 @@ selector:
   - jstests/sharding/safe_secondary_reads_causal_consistency.js
   # Enable the following once SERVER-55725 is backported to 4.0 (and prev. versions checked here).
   - jstests/sharding/time_zone_info_mongos.js
+  # Requires behavior that does not and will never exist in 3.6.
+  - jstests/sharding/retryable_mongos_write_errors.js
 executor:
   config:
     shell_options:
diff --git a/jstests/sharding/retryable_mongos_write_errors.js b/jstests/sharding/retryable_mongos_write_errors.js
new file mode 100644
index 00000000000..51b43c13b2c
--- /dev/null
+++ b/jstests/sharding/retryable_mongos_write_errors.js
@@ -0,0 +1,68 @@
+/**
+ * Tests that retryable write errors in MongoS return a top level error message.
+ */
+
+(function() {
+    "use strict";
+
+    load('jstests/libs/parallelTester.js');  // for ScopedThread.
+
+    const dbName = "test";
+    const collName = "retryable_mongos_write_errors";
+    const ns = dbName + "." + collName;
+
+    const st = new ShardingTest({config: 1, mongos: 1, shards: 1});
+    const shard0Primary = st.rs0.getPrimary();
+
+    // Creates a new connection, uses it to get the database from the parameter name and inserts
+    // multiple documents to the provided collection.
+    function insertHandler(host, dbName, collName, testData) {
+        try {
+            TestData = testData;
+            const conn = new Mongo(host);
+            const database = conn.getDB(dbName);
+            // creates an array with 10 documents
+            const docs = Array.from(Array(10).keys()).map((i) => ({a: i, b: "retryable"}));
+            const commandResponse = database.runCommand({insert: collName, documents: docs});
+            // assert that retryableInsertRes failed with the HostUnreachableError or
+            // InterruptedAtShutdown error code
+            assert.commandFailedWithCode(commandResponse, ErrorCodes.InterruptedAtShutdown);
+            jsTest.log("Command Response: " + tojson(commandResponse) + "." + commandResponse.code);
+            return {ok: 1};
+        } catch (e) {
+            if (!isNetworkError(e)) {
+                return {ok: 0, error: e.toString(), stack: e.stack};
+            }
+
+            return {ok: 1};
+        }
+    }
+
+    const failpointName = 'hangAfterCollectionInserts';
+    const executeFailPointCommand = (mode) => {
+        assert.commandWorked(shard0Primary.adminCommand(
+            {configureFailPoint: failpointName, mode, data: {collectionNS: ns}}));
+    };
+
+    executeFailPointCommand("alwaysOn");
+
+    const insertThread = new ScopedThread(insertHandler, st.s.host, dbName, collName, TestData);
+    jsTest.log("Starting To Insert Documents");
+    insertThread.start();
+
+    checkLog.contains(shard0Primary, `${failpointName} fail point enabled`);
+    jsTest.log("Starting to shutdown MongoS.");
+    MongoRunner.stopMongos(st.s);
+
+    try {
+        assert.commandWorked(insertThread.returnData());
+    } finally {
+        jsTest.log("Finished Assertions, Turning Off Failpoint");
+        executeFailPointCommand("off");
+    }
+
+    st.s = MongoRunner.runMongos(st.s);
+
+    jsTest.log('Shutting down sharding test');
+    st.stop();
+}());
\ No newline at end of file
diff --git a/src/mongo/s/write_ops/batch_write_exec.cpp b/src/mongo/s/write_ops/batch_write_exec.cpp
index 54bfdf6572f..bf2dfbe912b 100644
--- a/src/mongo/s/write_ops/batch_write_exec.cpp
+++ b/src/mongo/s/write_ops/batch_write_exec.cpp
@@ -46,6 +46,7 @@
 #include "mongo/s/grid.h"
 #include "mongo/s/write_ops/batch_write_op.h"
 #include "mongo/s/write_ops/write_error_detail.h"
+#include "mongo/util/exit.h"
 #include "mongo/util/log.h"
 
 namespace mongo {
@@ -303,6 +304,12 @@ void BatchWriteExec::executeBatch(OperationContext* opCtx,
                                            : OID());
                 } else {
                     // Error occurred dispatching, note it
+                    if (ErrorCodes::isShutdownError(responseStatus.code()) &&
+                        globalInShutdownDeprecated()) {
+                        // Throw an error since the mongos itself is shutting down so this should
+                        // be a top level error instead of a write error.
+                        uassertStatusOK(responseStatus);
+                    }
                     const Status status = responseStatus.withContext(
                         str::stream() << "Write results unavailable from " << shardHost);