Bug 1130754: Avoid recalculating tbsCertificate digest, r=keeler

1 files changed, 52 insertions, 57 deletions

diff --git a/lib/mozpkix/lib/pkixnss.cpp b/lib/mozpkix/lib/pkixnss.cpp
index 9471e2ffe..04cc02052 100644
--- a/lib/mozpkix/lib/pkixnss.cpp
+++ b/lib/mozpkix/lib/pkixnss.cpp
@@ -37,48 +37,20 @@
 
 namespace mozilla { namespace pkix {
 
+namespace {
+
 Result
-VerifySignedDataNSS(const SignedDataWithSignature& sd,
-                    Input subjectPublicKeyInfo, void* pkcs11PinArg)
+VerifySignedDigest(const SignedDigest& sd,
+                   Input subjectPublicKeyInfo,
+                   SECOidTag pubKeyAlg,
+                   void* pkcs11PinArg)
 {
-  SECOidTag pubKeyAlg;
   SECOidTag digestAlg;
-  switch (sd.algorithm) {
-    case SignatureAlgorithm::ecdsa_with_sha512:
-      pubKeyAlg = SEC_OID_ANSIX962_EC_PUBLIC_KEY;
-      digestAlg = SEC_OID_SHA512;
-      break;
-    case SignatureAlgorithm::ecdsa_with_sha384:
-      pubKeyAlg = SEC_OID_ANSIX962_EC_PUBLIC_KEY;
-      digestAlg = SEC_OID_SHA384;
-      break;
-    case SignatureAlgorithm::ecdsa_with_sha256:
-      pubKeyAlg = SEC_OID_ANSIX962_EC_PUBLIC_KEY;
-      digestAlg = SEC_OID_SHA256;
-      break;
-    case SignatureAlgorithm::ecdsa_with_sha1:
-      pubKeyAlg = SEC_OID_ANSIX962_EC_PUBLIC_KEY;
-      digestAlg = SEC_OID_SHA1;
-      break;
-    case SignatureAlgorithm::rsa_pkcs1_with_sha512:
-      pubKeyAlg = SEC_OID_PKCS1_RSA_ENCRYPTION;
-      digestAlg = SEC_OID_SHA512;
-      break;
-    case SignatureAlgorithm::rsa_pkcs1_with_sha384:
-      pubKeyAlg = SEC_OID_PKCS1_RSA_ENCRYPTION;
-      digestAlg = SEC_OID_SHA384;
-      break;
-    case SignatureAlgorithm::rsa_pkcs1_with_sha256:
-      pubKeyAlg = SEC_OID_PKCS1_RSA_ENCRYPTION;
-      digestAlg = SEC_OID_SHA256;
-      break;
-    case SignatureAlgorithm::rsa_pkcs1_with_sha1:
-      pubKeyAlg = SEC_OID_PKCS1_RSA_ENCRYPTION;
-      digestAlg = SEC_OID_SHA1;
-      break;
-    case SignatureAlgorithm::unsupported_algorithm: // fall through
-      return NotReached("unknown signature algorithm",
-                        Result::ERROR_CERT_SIGNATURE_ALGORITHM_DISABLED);
+  switch (sd.digestAlgorithm) {
+    case DigestAlgorithm::sha512: digestAlg = SEC_OID_SHA512; break;
+    case DigestAlgorithm::sha384: digestAlg = SEC_OID_SHA384; break;
+    case DigestAlgorithm::sha256: digestAlg = SEC_OID_SHA256; break;
+    case DigestAlgorithm::sha1: digestAlg = SEC_OID_SHA1; break;
     MOZILLA_PKIX_UNREACHABLE_DEFAULT_ENUM
   }
 
@@ -95,19 +67,11 @@ VerifySignedDataNSS(const SignedDataWithSignature& sd,
     return MapPRErrorCodeToResult(PR_GetError());
   }
 
-  // The static_cast is safe as long as the length of the data in sd.data can
-  // fit in an int. Right now that length is stored as a uint16_t, so this
-  // works. In the future this may change, hence the assertion.
-  // See also bug 921585.
-  static_assert(sizeof(decltype(sd.data.GetLength())) < sizeof(int),
-                "sd.data.GetLength() must fit in an int");
-  SECItem dataSECItem(UnsafeMapInputToSECItem(sd.data));
+  SECItem digestSECItem(UnsafeMapInputToSECItem(sd.digest));
   SECItem signatureSECItem(UnsafeMapInputToSECItem(sd.signature));
-  SECStatus srv = VFY_VerifyDataDirect(dataSECItem.data,
-                                       static_cast<int>(dataSECItem.len),
-                                       pubKey.get(), &signatureSECItem,
-                                       pubKeyAlg, digestAlg, nullptr,
-                                       pkcs11PinArg);
+  SECStatus srv = VFY_VerifyDigestDirect(&digestSECItem, pubKey.get(),
+                                         &signatureSECItem, pubKeyAlg,
+                                         digestAlg, pkcs11PinArg);
   if (srv != SECSuccess) {
     return MapPRErrorCodeToResult(PR_GetError());
   }
@@ -115,14 +79,45 @@ VerifySignedDataNSS(const SignedDataWithSignature& sd,
   return Success;
 }
 
+} // unnamed namespace
+
+Result
+VerifyRSAPKCS1SignedDigestNSS(const SignedDigest& sd,
+                              Input subjectPublicKeyInfo,
+                              void* pkcs11PinArg)
+{
+  return VerifySignedDigest(sd, subjectPublicKeyInfo,
+                            SEC_OID_PKCS1_RSA_ENCRYPTION, pkcs11PinArg);
+}
+
+Result
+VerifyECDSASignedDigestNSS(const SignedDigest& sd,
+                           Input subjectPublicKeyInfo,
+                           void* pkcs11PinArg)
+{
+  return VerifySignedDigest(sd, subjectPublicKeyInfo,
+                            SEC_OID_ANSIX962_EC_PUBLIC_KEY, pkcs11PinArg);
+}
+
 Result
-DigestBufNSS(Input item, /*out*/ uint8_t* digestBuf, size_t digestBufLen)
+DigestBufNSS(Input item,
+             DigestAlgorithm digestAlg,
+             /*out*/ uint8_t* digestBuf,
+             size_t digestBufLen)
 {
-  static_assert(TrustDomain::DIGEST_LENGTH == SHA1_LENGTH,
-                "TrustDomain::DIGEST_LENGTH must be 20 (SHA-1 digest length)");
-  if (digestBufLen != TrustDomain::DIGEST_LENGTH) {
-    return NotReached("invalid hash length", Result::FATAL_ERROR_INVALID_ARGS);
+  SECOidTag oid;
+  size_t bits;
+  switch (digestAlg) {
+    case DigestAlgorithm::sha512: oid = SEC_OID_SHA512; bits = 512; break;
+    case DigestAlgorithm::sha384: oid = SEC_OID_SHA384; bits = 384; break;
+    case DigestAlgorithm::sha256: oid = SEC_OID_SHA256; bits = 256; break;
+    case DigestAlgorithm::sha1: oid = SEC_OID_SHA1; bits = 160; break;
+    MOZILLA_PKIX_UNREACHABLE_DEFAULT_ENUM
   }
+  if (digestBufLen != bits / 8) {
+    return Result::FATAL_ERROR_INVALID_ARGS;
+  }
+
   SECItem itemSECItem = UnsafeMapInputToSECItem(item);
   if (itemSECItem.len >
         static_cast<decltype(itemSECItem.len)>(
@@ -130,7 +125,7 @@ DigestBufNSS(Input item, /*out*/ uint8_t* digestBuf, size_t digestBufLen)
     PR_NOT_REACHED("large items should not be possible here");
     return Result::FATAL_ERROR_INVALID_ARGS;
   }
-  SECStatus srv = PK11_HashBuf(SEC_OID_SHA1, digestBuf, itemSECItem.data,
+  SECStatus srv = PK11_HashBuf(oid, digestBuf, itemSECItem.data,
                                static_cast<int32_t>(itemSECItem.len));
   if (srv != SECSuccess) {
     return MapPRErrorCodeToResult(PR_GetError());