SERVER-35418 Allow specifying CAs for incoming and outgoing connections separately

(cherry picked from commit 17ccef2b9f0c71b60d31b84b8824215ff87f03aa)

Option names mapped from tls* to ssl*

6 files changed, 183 insertions, 31 deletions

diff --git a/jstests/ssl/ssl_cluster_ca.js b/jstests/ssl/ssl_cluster_ca.js
new file mode 100644
index 00000000000..2ee05406ea1
--- /dev/null
+++ b/jstests/ssl/ssl_cluster_ca.js
@@ -0,0 +1,82 @@
+// Verify certificates and CAs between intra-cluster
+// and client->server communication using different CAs.
+
+(function() {
+    "use strict";
+
+    function testRS(opts, succeed) {
+        const origSkipCheck = TestData.skipCheckDBHashes;
+        const rsOpts = {
+            // Use localhost so that SAN matches.
+            useHostName: false,
+            nodes: {node0: opts, node1: opts},
+        };
+        const rs = new ReplSetTest(rsOpts);
+        rs.startSet();
+        if (succeed) {
+            rs.initiate();
+            assert.commandWorked(rs.getPrimary().getDB('admin').runCommand({isMaster: 1}));
+        } else {
+            assert.throws(function() {
+                rs.initiate();
+            });
+            TestData.skipCheckDBHashes = true;
+        }
+        rs.stopSet();
+        TestData.skipCheckDBHashes = origSkipCheck;
+    }
+
+    // The name "trusted" in these certificates is misleading.
+    // They're just a separate trust chain from the ones without the name.
+    // ca.pem signed client.pem and server.pem
+    // trusted-ca.pem signed trusted-client.pem and trusted-server.pem
+    const valid_options = {
+        sslMode: 'requireSSL',
+        // Servers present trusted-server.pem to clients and each other for inbound connections.
+        // Peers validate trusted-server.pem using trusted-ca.pem when making those connections.
+        sslPEMKeyFile: 'jstests/libs/trusted-server.pem',
+        sslCAFile: 'jstests/libs/trusted-ca.pem',
+        // Servers making outbound connections to other servers present server.pem to their peers
+        // which their peers validate using ca.pem.
+        sslClusterFile: 'jstests/libs/server.pem',
+        sslClusterCAFile: 'jstests/libs/ca.pem',
+        // SERVER-36895: IP based hostname validation with SubjectAlternateName
+        sslAllowInvalidHostnames: '',
+    };
+
+    testRS(valid_options, true);
+
+    const wrong_cluster_file =
+        Object.assign({}, valid_options, {sslClusterFile: valid_options.sslPEMKeyFile});
+    testRS(wrong_cluster_file, false);
+
+    const wrong_key_file =
+        Object.assign({}, valid_options, {sslPEMKeyFile: valid_options.sslClusterFile});
+    testRS(wrong_key_file, false);
+
+    const mongod = MongoRunner.runMongod(valid_options);
+    assert(mongod, "Failed starting standalone mongod with alternate CA");
+
+    function testConnect(cert, succeed) {
+        const mongo = runMongoProgram("mongo",
+                                      "--host",
+                                      "localhost",
+                                      "--port",
+                                      mongod.port,
+                                      "--ssl",
+                                      "--sslCAFile",
+                                      valid_options.sslCAFile,
+                                      "--sslPEMKeyFile",
+                                      cert,
+                                      "--eval",
+                                      ";");
+
+        // runMongoProgram returns 0 on success
+        assert.eq(mongo === 0, succeed);
+    }
+
+    testConnect('jstests/libs/client.pem', true);
+    testConnect('jstests/libs/trusted-client.pem', false);
+
+    MongoRunner.stopMongod(mongod);
+}());
diff --git a/src/mongo/util/net/ssl_manager_apple.cpp b/src/mongo/util/net/ssl_manager_apple.cpp
index 8c4b6a05b40..8f3b6680d34 100644
--- a/src/mongo/util/net/ssl_manager_apple.cpp
+++ b/src/mongo/util/net/ssl_manager_apple.cpp
@@ -1080,7 +1080,20 @@ private:
     bool _suppressNoCertificateWarning;
     asio::ssl::apple::Context _clientCtx;
     asio::ssl::apple::Context _serverCtx;
-    CFUniquePtr<::CFArrayRef> _ca;
+
+    /* _clientCA represents the CA to use when acting as a client
+     * and validating remotes during outbound connections.
+     * This comes from, in order, --tlsCAFile, or the system CA.
+     */
+    CFUniquePtr<::CFArrayRef> _clientCA;
+
+    /* _serverCA represents the CA to use when acting as a server
+     * and validating remotes during inbound connections.
+     * This comes from --tlsClusterCAFile, if available,
+     * otherwise it inherits from _clientCA.
+     */
+    CFUniquePtr<::CFArrayRef> _serverCA;
+
     SSLConfiguration _sslConfiguration;
 };
 
@@ -1108,8 +1121,8 @@ SSLManagerApple::SSLManagerApple(const SSLParams& params, bool isServer)
 
     if (!params.sslCAFile.empty()) {
         auto ca = uassertStatusOK(loadPEM(params.sslCAFile, "", kLoadPEMStripKeys));
-        _ca = std::move(ca);
-        _sslConfiguration.hasCA = _ca && ::CFArrayGetCount(_ca.get());
+        _clientCA = std::move(ca);
+        _sslConfiguration.hasCA = _clientCA && ::CFArrayGetCount(_clientCA.get());
     }
 
     if (!params.sslCertificateSelector.empty() || !params.sslClusterCertificateSelector.empty()) {
@@ -1117,12 +1130,22 @@ SSLManagerApple::SSLManagerApple(const SSLParams& params, bool isServer)
         _sslConfiguration.hasCA = true;
     }
 
-    if (!_ca) {
+    if (!_clientCA) {
         // No explicit CA was specified, use the Keychain CA explicitly on client connects,
         // even though we're going to pretend it doesn't exist on server.
         ::CFArrayRef certs = nullptr;
         uassertOSStatusOK(SecTrustCopyAnchorCertificates(&certs));
-        _ca.reset(certs);
+        _clientCA.reset(certs);
+    }
+
+    if (!params.sslClusterCAFile.empty()) {
+        auto ca = uassertStatusOK(loadPEM(params.sslClusterCAFile, "", kLoadPEMStripKeys));
+        _serverCA = std::move(ca);
+    } else {
+        // No inbound CA specified, share a reference with outbound CA.
+        auto ca = _clientCA.get();
+        ::CFRetain(ca);
+        _serverCA.reset(ca);
     }
 }
 
@@ -1313,8 +1336,12 @@ StatusWith<boost::optional<SSLPeerInfo>> SSLManagerApple::parseAndValidatePeerCe
         }
     }
 
-    if (_ca) {
-        auto status = ::SecTrustSetAnchorCertificates(cftrust.get(), _ca.get());
+    // When remoteHost is empty, it means we're handling an Inbound connection.
+    // In that case, we in a server role, so use the _serverCA,
+    // otherwise we're in a client role, so use that.
+    auto ca = remoteHost.empty() ? _serverCA.get() : _clientCA.get();
+    if (ca) {
+        auto status = ::SecTrustSetAnchorCertificates(cftrust.get(), ca);
         if (status == ::errSecSuccess) {
             status = ::SecTrustSetAnchorCertificatesOnly(cftrust.get(), true);
         }
diff --git a/src/mongo/util/net/ssl_manager_openssl.cpp b/src/mongo/util/net/ssl_manager_openssl.cpp
index 6a10bb7d387..d1bf3f8476c 100644
--- a/src/mongo/util/net/ssl_manager_openssl.cpp
+++ b/src/mongo/util/net/ssl_manager_openssl.cpp
@@ -731,10 +731,14 @@ Status SSLManagerOpenSSL::initSSLContext(SSL_CTX* context,
         }
     }
 
-    const auto status =
-        params.sslCAFile.empty() ? _setupSystemCA(context) : _setupCA(context, params.sslCAFile);
-    if (!status.isOK())
+    std::string cafile = params.sslCAFile;
+    if (direction == ConnectionDirection::kIncoming && !params.sslClusterCAFile.empty()) {
+        cafile = params.sslClusterCAFile;
+    }
+    const auto status = cafile.empty() ? _setupSystemCA(context) : _setupCA(context, cafile);
+    if (!status.isOK()) {
         return status;
+    }
 
     if (!params.sslCRLFile.empty()) {
         if (!_setupCRL(context, params.sslCRLFile)) {
diff --git a/src/mongo/util/net/ssl_manager_windows.cpp b/src/mongo/util/net/ssl_manager_windows.cpp
index c5dcc4c865e..7ac11406796 100644
--- a/src/mongo/util/net/ssl_manager_windows.cpp
+++ b/src/mongo/util/net/ssl_manager_windows.cpp
@@ -297,7 +297,15 @@ private:
                                 SSLX509Name* subjectName,
                                 Date_t* serverCertificateExpirationDate);
 
-    Status _initChainEngines(bool hasCAFile);
+    struct CAEngine {
+        CERT_CHAIN_ENGINE_CONFIG machineConfig;
+        UniqueCertChainEngine machine;
+        CERT_CHAIN_ENGINE_CONFIG userConfig;
+        UniqueCertChainEngine user;
+        UniqueCertStore CAstore;
+    };
+
+    Status _initChainEngines(CAEngine* engine);
 
 private:
     bool _weakValidation;
@@ -314,17 +322,21 @@ private:
     std::array<PCCERT_CONTEXT, 1> _clientCertificates;
     std::array<PCCERT_CONTEXT, 1> _serverCertificates;
 
-    UniqueCertificate _sslCertificate;
-    UniqueCertificate _sslClusterCertificate;
-
-    UniqueCertStore _certStore;
+    /* _clientEngine represents the CA to use when acting as a client
+     * and validating remotes during outbound connections.
+     * This comes from, in order, --tlsCAFile, or the system CA.
+     */
+    CAEngine _clientEngine;
 
-    std::array<HCERTSTORE, 1> _additionalCertStores;
-    CERT_CHAIN_ENGINE_CONFIG _chainEngineConfigMachine;
-    UniqueCertChainEngine _chainEngineMachine;
+    /* _serverEngine represents the CA to use when acting as a server
+     * and validating remotes during inbound connections.
+     * This comes from --tlsClusterCAFile, if available,
+     * otherwise it inherits from _clientEngine.
+     */
+    CAEngine _serverEngine;
 
-    CERT_CHAIN_ENGINE_CONFIG _chainEngineConfigUser;
-    UniqueCertChainEngine _chainEngineUser;
+    UniqueCertificate _sslCertificate;
+    UniqueCertificate _sslClusterCertificate;
 };
 
 MONGO_INITIALIZER(SSLManager)(InitializerContext*) {
@@ -411,7 +423,8 @@ SSLManagerWindows::SSLManagerWindows(const SSLParams& params, bool isServer)
             CertificateExpirationMonitor(_sslConfiguration.serverCertificateExpirationDate);
     }
 
-    uassertStatusOK(_initChainEngines(!params.sslCAFile.empty()));
+    uassertStatusOK(_initChainEngines(&_serverEngine));
+    uassertStatusOK(_initChainEngines(&_clientEngine));
 }
 
 StatusWith<UniqueCertChainEngine> initChainEngine(CERT_CHAIN_ENGINE_CONFIG* chainEngineConfig,
@@ -439,23 +452,23 @@ StatusWith<UniqueCertChainEngine> initChainEngine(CERT_CHAIN_ENGINE_CONFIG* chai
     return {chainEngine};
 }
 
-Status SSLManagerWindows::_initChainEngines(bool hasCAFile) {
-    auto swMachine =
-        initChainEngine(&_chainEngineConfigMachine, _certStore, CERT_CHAIN_USE_LOCAL_MACHINE_STORE);
+Status SSLManagerWindows::_initChainEngines(CAEngine* engine) {
+    auto swMachine = initChainEngine(
+        &engine->machineConfig, engine->CAstore, CERT_CHAIN_USE_LOCAL_MACHINE_STORE);
 
     if (!swMachine.isOK()) {
         return swMachine.getStatus();
     }
 
-    _chainEngineMachine = std::move(swMachine.getValue());
+    engine->machine = std::move(swMachine.getValue());
 
-    auto swUser = initChainEngine(&_chainEngineConfigUser, _certStore, 0);
+    auto swUser = initChainEngine(&engine->userConfig, engine->CAstore, 0);
 
     if (!swUser.isOK()) {
         return swUser.getStatus();
     }
 
-    _chainEngineUser = std::move(swUser.getValue());
+    engine->user = std::move(swUser.getValue());
 
     return Status::OK();
 }
@@ -1180,7 +1193,18 @@ Status SSLManagerWindows::_loadCertificates(const SSLParams& params) {
             return swChain.getStatus();
         }
 
-        _certStore = std::move(swChain.getValue());
+        _clientEngine.CAstore = std::move(swChain.getValue());
+    }
+
+    const auto serverCAFile =
+        params.sslClusterCAFile.empty() ? params.sslCAFile : params.sslClusterCAFile;
+    if (!serverCAFile.empty()) {
+        auto swChain = readCertChains(serverCAFile, params.sslCRLFile);
+        if (!swChain.isOK()) {
+            return swChain.getStatus();
+        }
+
+        _serverEngine.CAstore = std::move(swChain.getValue());
     }
 
     if (hasCertificateSelector(params.sslCertificateSelector)) {
@@ -1228,7 +1252,6 @@ Status SSLManagerWindows::initSSLContext(SCHANNEL_CRED* cred,
     cred->dwVersion = SCHANNEL_CRED_VERSION;
     cred->dwFlags = SCH_USE_STRONG_CRYPTO;  // Use strong crypto;
 
-    cred->hRootStore = _certStore;
 
     uint32_t supportedProtocols = 0;
 
@@ -1236,6 +1259,7 @@ Status SSLManagerWindows::initSSLContext(SCHANNEL_CRED* cred,
         supportedProtocols = SP_PROT_TLS1_SERVER | SP_PROT_TLS1_0_SERVER | SP_PROT_TLS1_1_SERVER |
             SP_PROT_TLS1_2_SERVER;
 
+        cred->hRootStore = _serverEngine.CAstore;
         cred->dwFlags = cred->dwFlags          // flags
             | SCH_CRED_REVOCATION_CHECK_CHAIN  // Check certificate revocation
             | SCH_CRED_SNI_CREDENTIAL          // Pass along SNI creds
@@ -1246,6 +1270,7 @@ Status SSLManagerWindows::initSSLContext(SCHANNEL_CRED* cred,
         supportedProtocols = SP_PROT_TLS1_CLIENT | SP_PROT_TLS1_0_CLIENT | SP_PROT_TLS1_1_CLIENT |
             SP_PROT_TLS1_2_CLIENT;
 
+        cred->hRootStore = _clientEngine.CAstore;
         cred->dwFlags = cred->dwFlags           // Flags
             | SCH_CRED_REVOCATION_CHECK_CHAIN   // Check certificate revocation
             | SCH_CRED_NO_SERVERNAME_CHECK      // Do not validate server name against cert
@@ -1702,10 +1727,12 @@ StatusWith<boost::optional<SSLPeerInfo>> SSLManagerWindows::parseAndValidatePeer
     UniqueCertificate certHolder(cert);
     SSLX509Name peerSubjectName;
 
+    auto* engine = remoteHost.empty() ? &_serverEngine : &_clientEngine;
+
     // Validate against the local machine store first since it is easier to manage programmatically.
     Status validateCertMachine = validatePeerCertificate(remoteHost,
                                                          certHolder.get(),
-                                                         _chainEngineMachine,
+                                                         engine->machine,
                                                          _allowInvalidCertificates,
                                                          _allowInvalidHostnames,
                                                          &peerSubjectName);
@@ -1714,7 +1741,7 @@ StatusWith<boost::optional<SSLPeerInfo>> SSLManagerWindows::parseAndValidatePeer
         // manage.
         Status validateCertUser = validatePeerCertificate(remoteHost,
                                                           certHolder.get(),
-                                                          _chainEngineUser,
+                                                          engine->user,
                                                           _allowInvalidCertificates,
                                                           _allowInvalidHostnames,
                                                           &peerSubjectName);
diff --git a/src/mongo/util/net/ssl_options.cpp b/src/mongo/util/net/ssl_options.cpp
index 72ee5b041c9..d4f0d6e869e 100644
--- a/src/mongo/util/net/ssl_options.cpp
+++ b/src/mongo/util/net/ssl_options.cpp
@@ -206,6 +206,11 @@ Status addSSLServerOptions(moe::OptionSection* options) {
     options->addOptionChaining(
         "net.ssl.CAFile", "sslCAFile", moe::String, "Certificate Authority file for SSL");
 
+    options->addOptionChaining("net.ssl.clusterCAFile",
+                               "sslClusterCAFile",
+                               moe::String,
+                               "CA used for verifying remotes during outbound connections");
+
     options->addOptionChaining(
         "net.ssl.CRLFile", "sslCRLFile", moe::String, "Certificate Revocation List file for SSL");
 
@@ -423,6 +428,12 @@ Status storeSSLServerOptions(const moe::Environment& params) {
                 .generic_string();
     }
 
+    if (params.count("net.ssl.clusterCAFile")) {
+        sslGlobalParams.sslClusterCAFile =
+            boost::filesystem::absolute(params["net.ssl.clusterCAFile"].as<std::string>())
+                .generic_string();
+    }
+
     if (params.count("net.ssl.CRLFile")) {
         sslGlobalParams.sslCRLFile =
             boost::filesystem::absolute(params["net.ssl.CRLFile"].as<std::string>())
diff --git a/src/mongo/util/net/ssl_options.h b/src/mongo/util/net/ssl_options.h
index f8071b782a6..d75cd8437d4 100644
--- a/src/mongo/util/net/ssl_options.h
+++ b/src/mongo/util/net/ssl_options.h
@@ -55,6 +55,7 @@ struct SSLParams {
     std::string sslClusterFile;     // --sslInternalKeyFile
     std::string sslClusterPassword;  // --sslInternalKeyPassword
     std::string sslCAFile;           // --sslCAFile
+    std::string sslClusterCAFile;    // --sslClusterCAFile
     std::string sslCRLFile;          // --sslCRLFile
     std::string sslCipherConfig;     // --sslCipherConfig