SERVER-23544 Always fail to create SyncClusterConnections after CatalogManager swap to CSRS mode

3 files changed, 29 insertions, 8 deletions

diff --git a/src/mongo/s/client/sharding_network_connection_hook.cpp b/src/mongo/s/client/sharding_network_connection_hook.cpp
index 6652f6a19e1..1581a05617a 100644
--- a/src/mongo/s/client/sharding_network_connection_hook.cpp
+++ b/src/mongo/s/client/sharding_network_connection_hook.cpp
@@ -45,11 +45,13 @@ namespace mongo {
 
 Status ShardingNetworkConnectionHook::validateHost(
     const HostAndPort& remoteHost, const executor::RemoteCommandResponse& isMasterReply) {
-    return validateHostImpl(remoteHost, isMasterReply);
+    return validateHostImpl(remoteHost, isMasterReply, false);
 }
 
 Status ShardingNetworkConnectionHook::validateHostImpl(
-    const HostAndPort& remoteHost, const executor::RemoteCommandResponse& isMasterReply) {
+    const HostAndPort& remoteHost,
+    const executor::RemoteCommandResponse& isMasterReply,
+    bool forSCC) {
     auto shard = grid.shardRegistry()->getShardForHostNoReload(remoteHost);
     if (!shard) {
         return {ErrorCodes::ShardNotFound,
@@ -69,10 +71,23 @@ Status ShardingNetworkConnectionHook::validateHostImpl(
             }
             using ConfigServerMode = CatalogManager::ConfigServerMode;
             const BSONElement setName = isMasterReply.data["setName"];
-            return grid.forwardingCatalogManager()->scheduleReplaceCatalogManagerIfNeeded(
-                (configServerModeNumber == 0 ? ConfigServerMode::SCCC : ConfigServerMode::CSRS),
-                (setName.type() == String ? setName.valueStringData() : StringData()),
-                remoteHost);
+            auto configServerMode =
+                (configServerModeNumber == 0 ? ConfigServerMode::SCCC : ConfigServerMode::CSRS);
+            auto catalogSwapStatus =
+                grid.forwardingCatalogManager()->scheduleReplaceCatalogManagerIfNeeded(
+                    configServerMode,
+                    (setName.type() == String ? setName.valueStringData() : StringData()),
+                    remoteHost);
+            if (configServerMode == ConfigServerMode::CSRS && catalogSwapStatus.isOK() && forSCC) {
+                // Even though scheduleReplaceCatalogManagerIfNeeded didn't indicate that a catalog
+                // manager swap is needed, if this connection is part of a SyncClusterConnection,
+                // and it's talking to a CSRS config server we still need to fail.
+                return Status(ErrorCodes::IncompatibleCatalogManager,
+                              "Need to swap sharding catalog manager. Detected config server in "
+                              "CSRS mode while using a SyncClusterConnection, which only supports "
+                              "SCCC mode config servers");
+            }
+            return catalogSwapStatus;
         }
         case ErrorCodes::NoSuchKey: {
             // The ismaster response indicates that remoteHost is not a config server, or that
diff --git a/src/mongo/s/client/sharding_network_connection_hook.h b/src/mongo/s/client/sharding_network_connection_hook.h
index 64132eabc41..074a4973e6f 100644
--- a/src/mongo/s/client/sharding_network_connection_hook.h
+++ b/src/mongo/s/client/sharding_network_connection_hook.h
@@ -53,9 +53,15 @@ public:
     /**
      * Implementation of validateHost can be called without a ShardingNetworkConnectionHook
      * instance.
+     *
+     * 'forSCC' bool will be true if this is being called on a connection created by
+     * SyncClusterConnection.  If forSCC is true we should fail to connect if the config server
+     * is detected to be in CSRS mode, regardless of whether we actually still need to swap our
+     * catalog manager or not.
      */
     static Status validateHostImpl(const HostAndPort& remoteHost,
-                                   const executor::RemoteCommandResponse& isMasterReply);
+                                   const executor::RemoteCommandResponse& isMasterReply,
+                                   bool forSCC);
 
     /**
      * Makes a SetShardVersion request for initializing sharding information on the new connection.
diff --git a/src/mongo/s/sharding_initialization.cpp b/src/mongo/s/sharding_initialization.cpp
index 7d5da250958..36a074742dd 100644
--- a/src/mongo/s/sharding_initialization.cpp
+++ b/src/mongo/s/sharding_initialization.cpp
@@ -155,7 +155,7 @@ Status initializeGlobalShardingState(OperationContext* txn,
                                      bool allowNetworking) {
     SyncClusterConnection::setConnectionValidationHook(
         [](const HostAndPort& target, const executor::RemoteCommandResponse& isMasterReply) {
-            return ShardingNetworkConnectionHook::validateHostImpl(target, isMasterReply);
+            return ShardingNetworkConnectionHook::validateHostImpl(target, isMasterReply, true);
         });
     auto network =
         executor::makeNetworkInterface("NetworkInterfaceASIO-ShardRegistry",