Bug 1049435: Change RSA_PrivateKeyCheck to not require p > q.

Change RSA_PrivateKeyCheck and rsa_build_from_primes (called by
RSA_NewKey and RSA_PopulatePrivateKey) to require p != q.

Continue to allow RSA_NewKey and RSA_PopulatePrivateKey to force p > q,
but add a comment to note that it is not necessary.

Remove unused variable prevbp in get_blinding_params.

r=rlb,rrelyea.

1 files changed, 21 insertions, 8 deletions

diff --git a/lib/freebl/rsa.c b/lib/freebl/rsa.c
index cc7d4fee4..498cc96bc 100644
--- a/lib/freebl/rsa.c
+++ b/lib/freebl/rsa.c
@@ -97,8 +97,8 @@ static struct RSABlindingParamsListStr blindingParamsList = { 0 };
 static PRBool nssRSAUseBlinding = PR_TRUE;
 
 static SECStatus
-rsa_build_from_primes(mp_int *p, mp_int *q, 
-		mp_int *e, PRBool needPublicExponent, 
+rsa_build_from_primes(const mp_int *p, const mp_int *q,
+		mp_int *e, PRBool needPublicExponent,
 		mp_int *d, PRBool needPrivateExponent,
 		RSAPrivateKey *key, unsigned int keySizeInBits)
 {
@@ -116,6 +116,12 @@ rsa_build_from_primes(mp_int *p, mp_int *q,
     CHECK_MPI_OK( mp_init(&psub1) );
     CHECK_MPI_OK( mp_init(&qsub1) );
     CHECK_MPI_OK( mp_init(&tmp)   );
+    /* p and q must be distinct. */
+    if (mp_cmp(p, q) == 0) {
+	PORT_SetError(SEC_ERROR_NEED_RANDOM);
+	rv = SECFailure;
+	goto cleanup;
+    }
     /* 1.  Compute n = p*q */
     CHECK_MPI_OK( mp_mul(p, q, &n) );
     /*     verify that the modulus has the desired number of bits */
@@ -280,7 +286,11 @@ RSA_NewKey(int keySizeInBits, SECItem *publicExponent)
 	PORT_SetError(0);
 	CHECK_SEC_OK( generate_prime(&p, primeLen) );
 	CHECK_SEC_OK( generate_prime(&q, primeLen) );
-	/* Assure q < p */
+	/* Assure p > q */
+	/* NOTE: PKCS #1 does not require p > q, and NSS doesn't use any
+	 * implementation optimization that requires p > q. We can remove
+	 * this code in the future.
+	 */
 	if (mp_cmp(&p, &q) < 0)
 	    mp_exch(&p, &q);
 	/* Attempt to use these primes to generate a key */
@@ -762,7 +772,11 @@ RSA_PopulatePrivateKey(RSAPrivateKey *key)
 	}
      }
 
-     /* force p to the the larger prime */
+     /* Assure p > q */
+     /* NOTE: PKCS #1 does not require p > q, and NSS doesn't use any
+      * implementation optimization that requires p > q. We can remove
+      * this code in the future.
+      */
      if (mp_cmp(&p, &q) < 0)
 	mp_exch(&p, &q);
 
@@ -1093,7 +1107,7 @@ get_blinding_params(RSAPrivateKey *key, mp_int *n, unsigned int modLen,
 {
     RSABlindingParams *rsabp           = NULL;
     blindingParams    *bpUnlinked      = NULL;
-    blindingParams    *bp, *prevbp     = NULL;
+    blindingParams    *bp;
     PRCList           *el;
     SECStatus          rv              = SECSuccess;
     mp_err             err             = MP_OKAY;
@@ -1183,7 +1197,6 @@ get_blinding_params(RSAPrivateKey *key, mp_int *n, unsigned int modLen,
 	}
 	/* We did not find a usable set of blinding params.  Can we make one? */
 	/* Find a free bp struct. */
-	prevbp = NULL;
 	if ((bp = rsabp->free) != NULL) {
 	    /* unlink this bp */
 	    rsabp->free  = bp->next;
@@ -1400,8 +1413,8 @@ RSA_PrivateKeyCheck(const RSAPrivateKey *key)
     SECITEM_TO_MPINT(key->exponent1,       &d_p);
     SECITEM_TO_MPINT(key->exponent2,       &d_q);
     SECITEM_TO_MPINT(key->coefficient,     &qInv);
-    /* p > q */
-    if (mp_cmp(&p, &q) <= 0) {
+    /* p and q must be distinct. */
+    if (mp_cmp(&p, &q) == 0) {
 	rv = SECFailure;
 	goto cleanup;
     }