Bug 1552254 internal_error alert on Certificate Request with sha1+ecdsa in TLS 1.3

We need to be able to select Client certificates based on the schemes sent to us from the server. Rather than changing the callback function, this patch adds those schemes to the ssl socket info as suggested by Dana. In addition, two helpful functions have been added to aid User applications in properly selecting the Certificate:
   PRBool SSL_CertIsUsable(PRFileDesc *fd, CERTCertificate *cert) - returns true if the given cert matches the schemes of the server, the schemes configured on the socket, capability of the token the private key resides on, and the current policy. For future SSL protocol, additional restrictions may be parsed.
   SSL_FilterCertListBySocket(PRFileDesc *fd, CERTCertList *certlist) - removes the certs from the cert list that doesn't pass the SSL_CertIsUsable() call.

In addition the built in cert selection function (NSS_GetClientAuthData) uses the above functions to filter the list. In order to support the NSS_GetClientAuthData three new functions have been added:
   SECStatus CERT_FilterCertListByNickname(CERTCertList *certList, char *nickname, void *pwarg) -- removes the certs that don't match the 'nickname'.
   SECStatus CERT_FilterCertListByCertList(CERTCertlist *certList, const CERTCertlist *filterList ) -- removes all the certs on the first cert list that isn't on the second.
   PRBool CERT_IsInList(CERTCertificate *, const CERTCertList *certList) -- returns true if cert is on certList.

In addition
 * PK11_FindObjectForCert() is exported so the token the cert lives on can be accessed.
 * the ssle ssl_PickClientSignatureScheme() function (along with several supporing functions) have been modified so it can be used by SSL_CertIsUsable()

Differential Revision: https://phabricator.services.mozilla.com/D135715

1 files changed, 120 insertions, 0 deletions

diff --git a/gtests/ssl_gtest/ssl_auth_unittest.cc b/gtests/ssl_gtest/ssl_auth_unittest.cc
index adb4424c1..925b82721 100644
--- a/gtests/ssl_gtest/ssl_auth_unittest.cc
+++ b/gtests/ssl_gtest/ssl_auth_unittest.cc
@@ -309,6 +309,126 @@ static SECStatus GetClientAuthDataHook(void* self, PRFileDesc* fd,
   return SECSuccess;
 }
 
+typedef struct AutoClientTestStr {
+  SECStatus result;
+  const std::string cert;
+} AutoClientTest;
+
+typedef struct AutoClientResultsStr {
+  AutoClientTest isRsa2048;
+  AutoClientTest isClient;
+  AutoClientTest isNull;
+  bool hookCalled;
+} AutoClientResults;
+
+void VerifyClientCertMatch(CERTCertificate* clientCert,
+                           const std::string expectedName) {
+  const char* name = clientCert->nickname;
+  std::cout << "Match name=\"" << name << "\" expected=\"" << expectedName
+            << "\"" << std::endl;
+  EXPECT_TRUE(PORT_Strcmp(name, expectedName.c_str()) == 0)
+      << " Certmismatch: \"" << name << "\" != \"" << expectedName << "\"";
+}
+
+static SECStatus GetAutoClientAuthDataHook(void* expectResults, PRFileDesc* fd,
+                                           CERTDistNames* caNames,
+                                           CERTCertificate** clientCert,
+                                           SECKEYPrivateKey** clientKey) {
+  AutoClientResults& results = *(AutoClientResults*)expectResults;
+  SECStatus rv;
+
+  results.hookCalled = true;
+  *clientCert = NULL;
+  *clientKey = NULL;
+  rv = NSS_GetClientAuthData((void*)TlsAgent::kRsa2048.c_str(), fd, caNames,
+                             clientCert, clientKey);
+  if (rv == SECSuccess) {
+    VerifyClientCertMatch(*clientCert, results.isRsa2048.cert);
+    CERT_DestroyCertificate(*clientCert);
+    SECKEY_DestroyPrivateKey(*clientKey);
+    *clientCert = NULL;
+    *clientKey = NULL;
+  }
+  EXPECT_EQ(results.isRsa2048.result, rv);
+
+  rv = NSS_GetClientAuthData((void*)TlsAgent::kClient.c_str(), fd, caNames,
+                             clientCert, clientKey);
+  if (rv == SECSuccess) {
+    VerifyClientCertMatch(*clientCert, results.isClient.cert);
+    CERT_DestroyCertificate(*clientCert);
+    SECKEY_DestroyPrivateKey(*clientKey);
+    *clientCert = NULL;
+    *clientKey = NULL;
+  }
+  EXPECT_EQ(results.isClient.result, rv);
+  EXPECT_EQ(*clientCert, nullptr);
+  EXPECT_EQ(*clientKey, nullptr);
+  rv = NSS_GetClientAuthData(NULL, fd, caNames, clientCert, clientKey);
+  if (rv == SECSuccess) {
+    VerifyClientCertMatch(*clientCert, results.isNull.cert);
+    // return this result
+  }
+  EXPECT_EQ(results.isNull.result, rv);
+  return rv;
+}
+
+// while I would have liked to use a new INSTANTIATE macro the
+// generates the following three tests, figuring out how to make that
+// work on top of the existing TlsConnect* plumbing hurts my head.
+TEST_P(TlsConnectTls12, AutoClientSelectRsaPss) {
+  AutoClientResults rsa = {{SECSuccess, TlsAgent::kRsa2048},
+                           {SECSuccess, TlsAgent::kClient},
+                           {SECSuccess, TlsAgent::kDelegatorRsaPss2048},
+                           false};
+  static const SSLSignatureScheme kSchemes[] = {ssl_sig_rsa_pss_pss_sha256,
+                                                ssl_sig_rsa_pkcs1_sha256,
+                                                ssl_sig_rsa_pkcs1_sha1};
+  Reset("rsa_pss_noparam");
+  client_->SetupClientAuth();
+  server_->RequestClientAuth(true);
+  EXPECT_EQ(SECSuccess,
+            SSL_GetClientAuthDataHook(client_->ssl_fd(),
+                                      GetAutoClientAuthDataHook, (void*)&rsa));
+  server_->SetSignatureSchemes(kSchemes, PR_ARRAY_SIZE(kSchemes));
+  client_->SetSignatureSchemes(kSchemes, PR_ARRAY_SIZE(kSchemes));
+  Connect();
+  EXPECT_TRUE(rsa.hookCalled);
+}
+
+TEST_P(TlsConnectTls12, AutoClientSelectEcc) {
+  AutoClientResults ecc = {{SECFailure, TlsAgent::kClient},
+                           {SECFailure, TlsAgent::kClient},
+                           {SECSuccess, TlsAgent::kDelegatorEcdsa256},
+                           false};
+  static const SSLSignatureScheme kSchemes[] = {ssl_sig_ecdsa_secp256r1_sha256};
+  client_->SetupClientAuth();
+  server_->RequestClientAuth(true);
+  EXPECT_EQ(SECSuccess,
+            SSL_GetClientAuthDataHook(client_->ssl_fd(),
+                                      GetAutoClientAuthDataHook, (void*)&ecc));
+  server_->SetSignatureSchemes(kSchemes, PR_ARRAY_SIZE(kSchemes));
+  client_->SetSignatureSchemes(kSchemes, PR_ARRAY_SIZE(kSchemes));
+  Connect();
+  EXPECT_TRUE(ecc.hookCalled);
+}
+
+TEST_P(TlsConnectTls12, AutoClientSelectDsa) {
+  AutoClientResults dsa = {{SECFailure, TlsAgent::kClient},
+                           {SECFailure, TlsAgent::kClient},
+                           {SECSuccess, TlsAgent::kServerDsa},
+                           false};
+  static const SSLSignatureScheme kSchemes[] = {ssl_sig_dsa_sha256};
+  client_->SetupClientAuth();
+  server_->RequestClientAuth(true);
+  EXPECT_EQ(SECSuccess,
+            SSL_GetClientAuthDataHook(client_->ssl_fd(),
+                                      GetAutoClientAuthDataHook, (void*)&dsa));
+  server_->SetSignatureSchemes(kSchemes, PR_ARRAY_SIZE(kSchemes));
+  client_->SetSignatureSchemes(kSchemes, PR_ARRAY_SIZE(kSchemes));
+  Connect();
+  EXPECT_TRUE(dsa.hookCalled);
+}
+
 TEST_F(TlsConnectStreamTls13, PostHandshakeAuthMultiple) {
   client_->SetupClientAuth();
   EXPECT_EQ(SECSuccess, SSL_OptionSet(client_->ssl_fd(),