SERVER-35541 Support PKCS#8 PrivateKeyInfo in SChannel Providerr4.0.0-rc5

(cherry picked from commit 8678e9ee87d2864376922522b9fa9b5a909aac1b)

3 files changed, 149 insertions, 31 deletions

diff --git a/jstests/libs/client_privatekey.pem b/jstests/libs/client_privatekey.pem
new file mode 100644
index 00000000000..c479d868366
--- /dev/null
+++ b/jstests/libs/client_privatekey.pem
@@ -0,0 +1,51 @@
+-----BEGIN CERTIFICATE-----
+MIID6TCCAtGgAwIBAgIJAOdcrxT4uC26MA0GCSqGSIb3DQEBCwUAMHQxFzAVBgNV
+BAMTDktlcm5lbCBUZXN0IENBMQ8wDQYDVQQLEwZLZXJuZWwxEDAOBgNVBAoTB01v
+bmdvREIxFjAUBgNVBAcTDU5ldyBZb3JrIENpdHkxETAPBgNVBAgTCE5ldyBZb3Jr
+MQswCQYDVQQGEwJVUzAeFw0xODA1MDQwMzE1MDhaFw0yODA1MDEwMzE1MDhaMIGx
+MQswCQYDVQQGEwJVUzERMA8GA1UECAwITmV3IFlvcmsxFjAUBgNVBAcMDU5ldyBZ
+b3JrIENpdHkxEDAOBgNVBAoMB01vbmdvREIxEzARBgNVBAsMCktlcm5lbFVzZXIx
+DzANBgNVBAMMBmNsaWVudDEiMCAGCSqGSIb3DQEJARYTZXhhbXBsZUBtb25nb2Ri
+LmNvbTEbMBkGA1UEDAwSQSBUZXN0IENlcnRpZmljYXRlMIIBIjANBgkqhkiG9w0B
+AQEFAAOCAQ8AMIIBCgKCAQEAqJQdWFNQ3dHzkrBOAzblrz/zAkUWy8eimcayhdM3
+ycz1h3poZM9p49ZHlcNZGYJq8cvG6zCBdg9eAaZ15L20Qc4mNT0+ZIejHrQqzY/D
+MhW2n2aBMPUdZp5MZAqY5cG4EqDpx9rZ5kKuu8wprB+Sf3n67bYxLm1G0RBe5uau
+Mus4R8baE+w0ins1pwMiEOBE2aUyBgYG5Rp04E3b2ko9TJjDaGDpTqtjHfkHOi0/
+sD/z3WMhgU2sNw5A7O3qlm61rLTm+pVTeP8+PU628PoQ4IZzmCkOZUeiO1cIMAvI
+ahiulJIbMCMVYLZi0bt7MjS4u9Vmuq9maLiU/rqMGQ6VOQIDAQABo0AwPjA8Bgsr
+BgEEAYKOKQIBAQQtMSswDwwGYmFja3VwDAVhZG1pbjAYDA9yZWFkQW55RGF0YWJh
+c2UMBWFkbWluMA0GCSqGSIb3DQEBCwUAA4IBAQAfxxzBppR0KaAkPYdGeu8HsbJ9
+DHiu532pIYN5A74J19kMu/wDfc5VKHA6gZkjymtHZUCoq9fSNXWWW90VSlnu+94x
++yE6LATu4lEAvbb82oyhRH6IH0sLvbzwnsVvejONB43uTejcUWSYvFhteN4zFL+I
+eDfqCxg12XxGAfNFiu6/GYus03MkZFY06q8vLh2qjICRaGEuhrqWGWKn/rGasbgL
+S2cpP74Ze2u5si0EF6+nRrhI1FdpcKgt0P+IFkFanx6YnWuVDf301EtgB9GTK616
+UI6pjlfj2rKQLZws/39UoMKlPbtmQiyv5iaW+qSTwARgy0tVCPaAD95sNS8o
+-----END CERTIFICATE-----
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQColB1YU1Dd0fOS
+sE4DNuWvP/MCRRbLx6KZxrKF0zfJzPWHemhkz2nj1keVw1kZgmrxy8brMIF2D14B
+pnXkvbRBziY1PT5kh6MetCrNj8MyFbafZoEw9R1mnkxkCpjlwbgSoOnH2tnmQq67
+zCmsH5J/efrttjEubUbREF7m5q4y6zhHxtoT7DSKezWnAyIQ4ETZpTIGBgblGnTg
+TdvaSj1MmMNoYOlOq2Md+Qc6LT+wP/PdYyGBTaw3DkDs7eqWbrWstOb6lVN4/z49
+Trbw+hDghnOYKQ5lR6I7VwgwC8hqGK6UkhswIxVgtmLRu3syNLi71Wa6r2ZouJT+
+uowZDpU5AgMBAAECggEAF1FNcc8sa1JEF7EtSUKJqKD20wSNb6hwdEbIoLwjhlVE
+TPYAhJkv82TLSAraQy4H7uQZ4qQ19AeSIDZPLlGTIlsKa+jJvYwJYxDWrtJI1J+u
+QAQksInnBC+3L0tL9NPDy0fYrl02MKRemPlXdkhzshBERiciTGKvHfHevp9/zukC
+v7tXotC2YR0xJuAuZn4hvlp2oJUtWtdwfsnnDFaznVCl4L+xjZWoXKwf5H4+DTmC
+NUZgw0CrLAZSy93MY7mJQgjeWZi7BZ0i++BY32mvWA9KcGKi1DhCWCtGsjI0j05/
+yvNciLGgarxiIm/yH32FM5OCst50IzPENoRSjpNyEQKBgQDX5TktD6Kdd/1m7mhG
+a9k4jL0A/HG/2uo1T+zrHcF4ytzVMg4TdltmYme0tmL5ZJZ+0kpPxwxR4jStj5Dn
+6Q/ws3XME/cpjxN67mN4lzZcGCVdK6bJGmQg45EXlznvZG69EYQjRXRwpOd7bxKJ
+P0Gx9fvSaREomCN4ZdJD1bgalQKBgQDH5MSJZqrfg5QU7M/qAADbUVW8qDYnWnti
+/9UnkpqsY5Dbu1Hzi82S/bFxrMNlkqSH+HhMnMKRv3jUK9dI1w4ksVe4TNiao9oR
+R6baKQFyXkpBVI8OvGm3YhA+jjt5W220IfWch2mQayF7kXr7ZbNTGqoK+YDmQDEL
+P+gN5JALFQKBgQDOqV+X4NWawtdQhnpOnsIqPZ5WqVABhP6Y1c9GKQ+gHyBQ/D85
+JhzL3vrCo6wOatkcwRPdVf5XC5+9XCYruI9Hq6HhHmOD5q+RB000jBq/AWYSz8wx
+yQ975BppaJtOqubOGUxRrxSNF212XJZzxpeOUJcjjOud3jRl9aHWyWgYGQKBgQCa
+MOwttMVP/pYmgflrXi9K/fVtvi9vscsFtaQ9j0fCfFcMN/ViMVAU9559ui9LVY+W
+1uIs7qrjdFgHCf5Z0JG3Whj5MEXRG10mAShClmjoSLVsqYZkbo/5hipTprFXzl/U
+GHH2GTojzY+ZWsIL1Ao8q4/JQWbdtVxfeh9NQCagdQKBgHw4VkLjtM3ZR9lNeacF
+GdBqEql08iVszXBwoknpKjMsBcpJVY6OQaRcMRoicKD6SyAhAhG1BjRZmuRtaZas
+zQBexYs/w8w6jjUxRF7FcQcKjJiXV9oPhgaNdaEtGPnAgE9X8/qqPGz+OCMI+RPW
+E28R8ZvFyr1KehGrSdEkjAt/
+-----END PRIVATE KEY-----
\ No newline at end of file
diff --git a/jstests/ssl/ssl_private_key.js b/jstests/ssl/ssl_private_key.js
new file mode 100644
index 00000000000..5317d6c86fa
--- /dev/null
+++ b/jstests/ssl/ssl_private_key.js
@@ -0,0 +1,36 @@
+// Test that clients support "BEGIN PRIVATE KEY" pems with RSA keys
+load('jstests/ssl/libs/ssl_helpers.js');
+
+(function() {
+    "use strict";
+
+    const SERVER_CERT = "jstests/libs/server.pem";
+    const CA_CERT = "jstests/libs/ca.pem";
+    const CLIENT_CERT = "jstests/libs/client_privatekey.pem";
+
+    function authAndTest(port) {
+        const mongo = runMongoProgram("mongo",
+                                      "--host",
+                                      "localhost",
+                                      "--port",
+                                      port,
+                                      "--ssl",
+                                      "--sslCAFile",
+                                      CA_CERT,
+                                      "--sslPEMKeyFile",
+                                      CLIENT_CERT,
+                                      "--eval",
+                                      "1");
+
+        // runMongoProgram returns 0 on success
+        assert.eq(0, mongo, "Connection attempt failed");
+    }
+
+    const x509_options = {sslMode: "requireSSL", sslPEMKeyFile: SERVER_CERT, sslCAFile: CA_CERT};
+
+    let mongo = MongoRunner.runMongod(Object.merge(x509_options, {auth: ""}));
+
+    authAndTest(mongo.port);
+
+    MongoRunner.stopMongod(mongo);
+}());
diff --git a/src/mongo/util/net/ssl_manager_windows.cpp b/src/mongo/util/net/ssl_manager_windows.cpp
index d96595f48a5..b0dc13add24 100644
--- a/src/mongo/util/net/ssl_manager_windows.cpp
+++ b/src/mongo/util/net/ssl_manager_windows.cpp
@@ -693,26 +693,6 @@ StatusWith<UniqueCertificateWithPrivateKey> readCertPEMFile(StringData fileName,
                                        "intermediate CA certificates belong in the CA file.");
     }
 
-    // PEM files can have either private key format
-    // Also the private key can either come before or after the certificate
-    auto swPrivateKeyBlob = findPEMBlob(buf, "RSA PRIVATE KEY"_sd);
-    // We expect to find at least one certificate
-    if (!swPrivateKeyBlob.isOK()) {
-        // A "PRIVATE KEY" is actually a PKCS #8 PrivateKeyInfo ASN.1 type. We do not support it for
-        // now so tell the user how to fix it.
-        // Warn user rsa -in roles.key -out roles2.key
-        swPrivateKeyBlob = findPEMBlob(buf, "PRIVATE KEY"_sd);
-        if (!swPrivateKeyBlob.isOK()) {
-            return swPrivateKeyBlob.getStatus();
-        } else {
-            return Status(ErrorCodes::InvalidSSLConfiguration,
-                          str::stream() << "Expected to find 'RSA PRIVATE KEY' in PEM file, found "
-                                           "'PRIVATE KEY' instead.");
-        }
-    }
-
-    auto privateKeyBlob = swPrivateKeyBlob.getValue();
-
     auto swCert = decodePEMBlob(publicKeyBlob);
     if (!swCert.isOK()) {
         return swCert.getStatus();
@@ -732,19 +712,71 @@ StatusWith<UniqueCertificateWithPrivateKey> readCertPEMFile(StringData fileName,
 
     UniqueCertificate certHolder(cert);
 
-    auto swPrivateKeyBuf = decodePEMBlob(privateKeyBlob);
-    if (!swPrivateKeyBuf.isOK()) {
-        return swPrivateKeyBuf.getStatus();
-    }
+    std::vector<uint8_t> privateKey;
+
+    // PEM files can have either private key format
+    // Also the private key can either come before or after the certificate
+    auto swPrivateKeyBlob = findPEMBlob(buf, "RSA PRIVATE KEY"_sd);
+    // We expect to find at least one certificate
+    if (!swPrivateKeyBlob.isOK()) {
+        // A "PRIVATE KEY" is actually a PKCS #8 PrivateKeyInfo ASN.1 type.
+        swPrivateKeyBlob = findPEMBlob(buf, "PRIVATE KEY"_sd);
+        if (!swPrivateKeyBlob.isOK()) {
+            return swPrivateKeyBlob.getStatus();
+        }
+
+        auto privateKeyBlob = swPrivateKeyBlob.getValue();
+
+        auto swPrivateKeyBuf = decodePEMBlob(privateKeyBlob);
+        if (!swPrivateKeyBuf.isOK()) {
+            return swPrivateKeyBuf.getStatus();
+        }
+
+        auto privateKeyBuf = swPrivateKeyBuf.getValue();
+
+        auto swPrivateKey =
+            decodeObject(PKCS_PRIVATE_KEY_INFO, privateKeyBuf.data(), privateKeyBuf.size());
+        if (!swPrivateKey.isOK()) {
+            return swPrivateKey.getStatus();
+        }
+
+        CRYPT_PRIVATE_KEY_INFO* privateKeyInfo =
+            reinterpret_cast<CRYPT_PRIVATE_KEY_INFO*>(swPrivateKey.getValue().data());
+
+        if (strcmp(privateKeyInfo->Algorithm.pszObjId, szOID_RSA_RSA) != 0) {
+            return Status(ErrorCodes::InvalidSSLConfiguration,
+                          str::stream() << "Non-RSA private keys are not supported, use the "
+                                           "Windows certificate store instead");
+        }
+
+        auto swPrivateKey2 = decodeObject(PKCS_RSA_PRIVATE_KEY,
+                                          privateKeyInfo->PrivateKey.pbData,
+                                          privateKeyInfo->PrivateKey.cbData);
+        if (!swPrivateKey2.isOK()) {
+            return swPrivateKey2.getStatus();
+        }
 
-    auto privateKeyBuf = swPrivateKeyBuf.getValue();
+        privateKey = swPrivateKey2.getValue();
+    } else {
+        auto privateKeyBlob = swPrivateKeyBlob.getValue();
 
-    auto swPrivateKey =
-        decodeObject(PKCS_RSA_PRIVATE_KEY, privateKeyBuf.data(), privateKeyBuf.size());
-    if (!swPrivateKey.isOK()) {
-        return swPrivateKey.getStatus();
+        auto swPrivateKeyBuf = decodePEMBlob(privateKeyBlob);
+        if (!swPrivateKeyBuf.isOK()) {
+            return swPrivateKeyBuf.getStatus();
+        }
+
+        auto privateKeyBuf = swPrivateKeyBuf.getValue();
+
+        auto swPrivateKey =
+            decodeObject(PKCS_RSA_PRIVATE_KEY, privateKeyBuf.data(), privateKeyBuf.size());
+        if (!swPrivateKey.isOK()) {
+            return swPrivateKey.getStatus();
+        }
+
+        privateKey = swPrivateKey.getValue();
     }
 
+
     HCRYPTPROV hProv;
     std::wstring wstr;
     BOOL ret;
@@ -801,8 +833,7 @@ StatusWith<UniqueCertificateWithPrivateKey> readCertPEMFile(StringData fileName,
     UniqueCryptProvider cryptProvider(hProv);
 
     HCRYPTKEY hkey;
-    ret = CryptImportKey(
-        hProv, swPrivateKey.getValue().data(), swPrivateKey.getValue().size(), 0, 0, &hkey);
+    ret = CryptImportKey(hProv, privateKey.data(), privateKey.size(), 0, 0, &hkey);
     if (!ret) {
         DWORD gle = GetLastError();
         return Status(ErrorCodes::InvalidSSLConfiguration,