SERVER-47394 Disconnect from client on SplitHorizonChange error

(cherry picked from commit 39f506463b2a88310ec874330347f1fd3ce07778)

4 files changed, 36 insertions, 11 deletions

diff --git a/jstests/replsets/awaitable_ismaster_errors_on_horizon_change.js b/jstests/replsets/awaitable_ismaster_errors_on_horizon_change.js
index d7c2c3ca4bc..a595f512618 100644
--- a/jstests/replsets/awaitable_ismaster_errors_on_horizon_change.js
+++ b/jstests/replsets/awaitable_ismaster_errors_on_horizon_change.js
@@ -1,6 +1,6 @@
 /**
- * Tests that doing a reconfig that changes the SplitHorizon will return an error to waiting
- * isMaster requests.
+ * Tests that doing a reconfig that changes the SplitHorizon will cause the server to disconnect
+ * from clients with waiting isMaster requests.
  * @tags: [requires_fcv_44]
  */
 (function() {
@@ -8,6 +8,9 @@
 load("jstests/libs/parallel_shell_helpers.js");
 load("jstests/libs/fail_point_util.js");
 
+// Never retry on network error, because this test needs to detect the network error.
+TestData.skipRetryOnNetworkError = true;
+
 const replTest = new ReplSetTest({nodes: [{}, {rsConfig: {priority: 0, votes: 0}}]});
 replTest.startSet();
 replTest.initiate();
@@ -19,14 +22,15 @@ const secondary = replTest.getSecondary();
 const secondaryDB = secondary.getDB(dbName);
 
 function runAwaitableIsMasterBeforeHorizonChange(topologyVersionField) {
-    const result = assert.commandFailedWithCode(db.runCommand({
+    let res = assert.throws(() => db.runCommand({
         isMaster: 1,
         topologyVersion: topologyVersionField,
         maxAwaitTimeMS: 99999999,
-    }),
-                                                ErrorCodes.SplitHorizonChange);
+    }));
+    assert(isNetworkError(res));
 
-    assert.eq(result.errmsg, "Received a reconfig that changed the horizon mappings.");
+    // We should automatically reconnect after the failed command.
+    assert.commandWorked(db.adminCommand({ping: 1}));
 }
 
 function runAwaitableIsMaster(topologyVersionField) {
diff --git a/jstests/replsets/awaitable_ismaster_on_nodes_with_invalid_configs.js b/jstests/replsets/awaitable_ismaster_on_nodes_with_invalid_configs.js
index f39720e2ed5..8c04749c7a9 100644
--- a/jstests/replsets/awaitable_ismaster_on_nodes_with_invalid_configs.js
+++ b/jstests/replsets/awaitable_ismaster_on_nodes_with_invalid_configs.js
@@ -7,6 +7,9 @@
 load("jstests/libs/parallel_shell_helpers.js");
 load("jstests/libs/fail_point_util.js");
 
+// Never retry on network error, because this test needs to detect the network error.
+TestData.skipRetryOnNetworkError = true;
+
 const replTest = new ReplSetTest({nodes: [{}, {rsConfig: {priority: 0, votes: 0}}]});
 // Start the replica set but don't initiate yet.
 replTest.startSet();
@@ -33,13 +36,15 @@ function runAwaitableIsMaster(topologyVersionField) {
 
 // Waiting isMasters should error when a node rejoins a replica set.
 function runAwaitableIsMasterOnRejoiningSet(topologyVersionField) {
-    const result = assert.commandFailedWithCode(db.runCommand({
+    const result = assert.throws(() => db.runCommand({
         isMaster: 1,
         topologyVersion: topologyVersionField,
         maxAwaitTimeMS: 99999999,
-    }),
-                                                ErrorCodes.SplitHorizonChange);
-    assert.eq(result.errmsg, "Received a reconfig that changed the horizon mappings.");
+    }));
+    assert(isNetworkError(result));
+
+    // We should automatically reconnect after the failed command.
+    assert.commandWorked(db.adminCommand({ping: 1}));
 }
 
 // A failpoint signalling that the servers have received the isMaster request and are waiting for a
diff --git a/src/mongo/base/error_codes.yml b/src/mongo/base/error_codes.yml
index 4e5071dfcc5..07f245a848e 100644
--- a/src/mongo/base/error_codes.yml
+++ b/src/mongo/base/error_codes.yml
@@ -21,6 +21,10 @@ error_categories:
     - VoteAbortError
     - NonResumableChangeStreamError
     - RetriableError
+    # isCloseConnectionError() includes all codes that indicate that it is no longer safe or
+    # desirable to maintain a connection with the client. The server will close the connection to
+    # get the client to through server selection again.
+    - CloseConnectionError
 
 error_codes:
     - {code: 0,name: OK}
@@ -339,7 +343,7 @@ error_codes:
     - {code: 300,name: RangeDeletionAbandonedBecauseCollectionWithUUIDDoesNotExist}
     - {code: 301,name: DataCorruptionDetected}
     - {code: 302,name: OCSPCertificateStatusUnknown}
-    - {code: 303,name: SplitHorizonChange}
+    - {code: 303,name: SplitHorizonChange,categories: [CloseConnectionError]}
 
     # This code should only be used upon finding that a shard has been marked stale in the sharding
     # catalog cache, and as such does not belong in the StaleShardVersionError or
diff --git a/src/mongo/db/service_entry_point_common.cpp b/src/mongo/db/service_entry_point_common.cpp
index 705f5c10c0c..55e50d2a6cd 100644
--- a/src/mongo/db/service_entry_point_common.cpp
+++ b/src/mongo/db/service_entry_point_common.cpp
@@ -1280,6 +1280,12 @@ void execCommandDatabase(OperationContext* opCtx,
                     "error"_attr = redact(e.toString()));
 
         generateErrorResponse(opCtx, replyBuilder, e, metadataBob.obj(), extraFieldsBuilder.obj());
+
+        if (ErrorCodes::isA<ErrorCategory::CloseConnectionError>(e.code())) {
+            // Rethrow the exception to the top to signal that the client connection should be
+            // closed.
+            throw;
+        }
     }
 }
 
@@ -1398,6 +1404,12 @@ DbResponse receivedCommands(OperationContext* opCtx,
 
             generateErrorResponse(
                 opCtx, replyBuilder.get(), ex, metadataBob.obj(), extraFieldsBuilder.obj());
+
+            if (ErrorCodes::isA<ErrorCategory::CloseConnectionError>(ex.code())) {
+                // Rethrow the exception to the top to signal that the client connection should be
+                // closed.
+                throw;
+            }
         }
     }();