Bug 1720228 NSS incorrectly accepting 1536 bit DH primes in FIPS mode

When NSS is in FIPS mode, it should reject all primes smaller than 2048. The ike 1536 prime is in the accepted primes table. In FIPS mode it should be rejected.

Differential Revision: https://phabricator.services.mozilla.com/D119895

5 files changed, 11 insertions, 7 deletions

diff --git a/gtests/softoken_gtest/softoken_dh_vectors.h b/gtests/softoken_gtest/softoken_dh_vectors.h
index f2e4514cf..306aded47 100644
--- a/gtests/softoken_gtest/softoken_dh_vectors.h
+++ b/gtests/softoken_gtest/softoken_dh_vectors.h
@@ -2872,7 +2872,7 @@ static const DhTestVector DH_TEST_VECTORS[] = {
      {siBuffer, (unsigned char *)g2, sizeof(g2)},
      {siBuffer, NULL, 0},
      {siBuffer, NULL, 0},
-     IKE_APPROVED,
+     SAFE_PRIME,
      CLASS_1536},
     {"IKE 2048",
      {siBuffer, (unsigned char *)prime_ike_2048, sizeof(prime_ike_2048)},
@@ -2952,7 +2952,7 @@ static const DhTestVector DH_TEST_VECTORS[] = {
      {siBuffer, (unsigned char *)sub2_prime_ike_1536,
       sizeof(sub2_prime_ike_1536)},
      {siBuffer, NULL, 0},
-     IKE_APPROVED,
+     SAFE_PRIME,
      CLASS_1536},
     {"IKE 2048 with subprime",
      {siBuffer, (unsigned char *)prime_ike_2048, sizeof(prime_ike_2048)},
diff --git a/lib/softoken/pkcs11c.c b/lib/softoken/pkcs11c.c
index c3216b3fd..201a0c728 100644
--- a/lib/softoken/pkcs11c.c
+++ b/lib/softoken/pkcs11c.c
@@ -5193,7 +5193,7 @@ sftk_PairwiseConsistencyCheck(CK_SESSION_HANDLE hSession, SFTKSlot *slot,
                 /* subprime not supplied, In this case look it up.
                  * This only works with approved primes, but in FIPS mode
                  * that's the only kine of prime that will get here */
-                subPrimePtr = sftk_VerifyDH_Prime(&prime);
+                subPrimePtr = sftk_VerifyDH_Prime(&prime, isFIPS);
                 if (subPrimePtr == NULL) {
                     crv = CKR_GENERAL_ERROR;
                     goto done;
@@ -8351,7 +8351,7 @@ NSC_DeriveKey(CK_SESSION_HANDLE hSession,
 
             /* if the prime is an approved prime, we can skip all the other
              * checks. */
-            subPrime = sftk_VerifyDH_Prime(&dhPrime);
+            subPrime = sftk_VerifyDH_Prime(&dhPrime, isFIPS);
             if (subPrime == NULL) {
                 SECItem dhSubPrime;
                 /* If the caller set the subprime value, it means that
diff --git a/lib/softoken/pkcs11i.h b/lib/softoken/pkcs11i.h
index aa212f09e..032e85fee 100644
--- a/lib/softoken/pkcs11i.h
+++ b/lib/softoken/pkcs11i.h
@@ -946,7 +946,7 @@ char **NSC_ModuleDBFunc(unsigned long function, char *parameters, void *args);
 /* dh verify functions */
 /* verify that dhPrime matches one of our known primes, and if so return
  * it's subprime value */
-const SECItem *sftk_VerifyDH_Prime(SECItem *dhPrime);
+const SECItem *sftk_VerifyDH_Prime(SECItem *dhPrime, PRBool isFIPS);
 /* check if dhSubPrime claims dhPrime is a safe prime. */
 SECStatus sftk_IsSafePrime(SECItem *dhPrime, SECItem *dhSubPrime, PRBool *isSafe);
 /* map an operation Attribute to a Mechanism flag */
diff --git a/lib/softoken/pkcs11u.c b/lib/softoken/pkcs11u.c
index 43d4ba9d5..f37aab92f 100644
--- a/lib/softoken/pkcs11u.c
+++ b/lib/softoken/pkcs11u.c
@@ -2312,7 +2312,7 @@ sftk_handleSpecial(SFTKSlot *slot, CK_MECHANISM *mech,
             if (crv != CKR_OK) {
                 return PR_FALSE;
             }
-            dhSubPrime = sftk_VerifyDH_Prime(&dhPrime);
+            dhSubPrime = sftk_VerifyDH_Prime(&dhPrime, PR_TRUE);
             SECITEM_ZfreeItem(&dhPrime, PR_FALSE);
             return (dhSubPrime) ? PR_TRUE : PR_FALSE;
         }
diff --git a/lib/softoken/sftkdhverify.c b/lib/softoken/sftkdhverify.c
index d85fba94f..6ac5e852a 100644
--- a/lib/softoken/sftkdhverify.c
+++ b/lib/softoken/sftkdhverify.c
@@ -1171,11 +1171,15 @@ static const SECItem subprime_tls_8192 =
  * verify that dhPrime matches one of our known primes
  */
 const SECItem *
-sftk_VerifyDH_Prime(SECItem *dhPrime)
+sftk_VerifyDH_Prime(SECItem *dhPrime, PRBool isFIPS)
 {
     /* use the length to decide which primes to check */
     switch (dhPrime->len) {
         case 1536 / PR_BITS_PER_BYTE:
+            /* don't accept 1536 bit primes in FIPS mode */
+            if (isFIPS) {
+                break;
+            }
             if (PORT_Memcmp(dhPrime->data, prime_ike_1536,
                             sizeof(prime_ike_1536)) == 0) {
                 return &subprime_ike_1536;