Update RSA keygen to use sp800-56b by default

Fixes #11742
Fixes #11764

The newer RSA sp800-56b algorithm is being used for the normal case of a non multiprime key of at least length 2048.
Insecure key lengths and mutltiprime RSA will use the old method.

Bad public exponents are no longer allowed (i.e values less than 65537 or even). Values such as 2 that would cause a infinite loop now result in an error. The value of 3 has been marked as deprecated but is still allowed for legacy purposes.

Reviewed-by: Tomas Mraz <tmraz@fedoraproject.org>
(Merged from https://github.com/openssl/openssl/pull/11765)

1 files changed, 31 insertions, 9 deletions

diff --git a/crypto/rsa/rsa_gen.c b/crypto/rsa/rsa_gen.c
index 5712aa1791..e52bce6355 100644
--- a/crypto/rsa/rsa_gen.c
+++ b/crypto/rsa/rsa_gen.c
@@ -70,16 +70,10 @@ int RSA_generate_multi_prime_key(RSA *rsa, int bits, int primes,
     return rsa_keygen(NULL, rsa, bits, primes, e_value, cb, 0);
 }
 
-static int rsa_keygen(OPENSSL_CTX *libctx, RSA *rsa, int bits, int primes,
-                      BIGNUM *e_value, BN_GENCB *cb, int pairwise_test)
+#ifndef FIPS_MODULE
+static int rsa_multiprime_keygen(RSA *rsa, int bits, int primes,
+                                 BIGNUM *e_value, BN_GENCB *cb)
 {
-    int ok = -1;
-#ifdef FIPS_MODULE
-    if (primes != 2)
-        return 0;
-    ok = rsa_sp800_56b_generate_key(rsa, bits, e_value, cb);
-    pairwise_test = 1; /* FIPS MODE needs to always run the pairwise test */
-#else
     BIGNUM *r0 = NULL, *r1 = NULL, *r2 = NULL, *tmp, *prime;
     int n = 0, bitsr[RSA_MAX_PRIME_NUM], bitse = 0;
     int i = 0, quo = 0, rmd = 0, adj = 0, retries = 0;
@@ -88,6 +82,7 @@ static int rsa_keygen(OPENSSL_CTX *libctx, RSA *rsa, int bits, int primes,
     BN_CTX *ctx = NULL;
     BN_ULONG bitst = 0;
     unsigned long error = 0;
+    int ok = -1;
 
     if (bits < RSA_MIN_MODULUS_BITS) {
         ok = 0;             /* we set our own err */
@@ -95,6 +90,12 @@ static int rsa_keygen(OPENSSL_CTX *libctx, RSA *rsa, int bits, int primes,
         goto err;
     }
 
+    /* A bad value for e can cause infinite loops */
+    if (e_value != NULL && !rsa_check_public_exponent(e_value)) {
+        RSAerr(0, RSA_R_PUB_EXPONENT_OUT_OF_RANGE);
+        return 0;
+    }
+
     if (primes < RSA_DEFAULT_PRIME_NUM || primes > rsa_multip_cap(bits)) {
         ok = 0;             /* we set our own err */
         RSAerr(0, RSA_R_KEY_PRIME_NUM_INVALID);
@@ -407,8 +408,29 @@ static int rsa_keygen(OPENSSL_CTX *libctx, RSA *rsa, int bits, int primes,
     }
     BN_CTX_end(ctx);
     BN_CTX_free(ctx);
+    return ok;
+}
 #endif /* FIPS_MODULE */
 
+static int rsa_keygen(OPENSSL_CTX *libctx, RSA *rsa, int bits, int primes,
+                      BIGNUM *e_value, BN_GENCB *cb, int pairwise_test)
+{
+    int ok = 0;
+
+    /*
+     * Only multi-prime keys or insecure keys with a small key length will use
+     * the older rsa_multiprime_keygen().
+     */
+    if (primes == 2 && bits >= 2048)
+        ok = rsa_sp800_56b_generate_key(rsa, bits, e_value, cb);
+#ifndef FIPS_MODULE
+    else
+        ok = rsa_multiprime_keygen(rsa, bits, primes, e_value, cb);
+#endif /* FIPS_MODULE */
+
+#ifdef FIPS_MODULE
+    pairwise_test = 1; /* FIPS MODE needs to always run the pairwise test */
+#endif
     if (pairwise_test && ok > 0) {
         OSSL_CALLBACK *stcb = NULL;
         void *stcbarg = NULL;