Bug 1798823 - add checks for zero-length RSA modulus to avoid memory errors and failed assertions later r=nss-reviewers,nkulatova,jschanck

Differential Revision: https://phabricator.services.mozilla.com/D162111

2 files changed, 26 insertions, 0 deletions

diff --git a/lib/freebl/rsa.c b/lib/freebl/rsa.c
index 2b8a3bfb5..9a0edcaac 100644
--- a/lib/freebl/rsa.c
+++ b/lib/freebl/rsa.c
@@ -899,6 +899,9 @@ cleanup:
 static unsigned int
 rsa_modulusLen(SECItem *modulus)
 {
+    if (modulus->len == 0) {
+        return 0;
+    };
     unsigned char byteZero = modulus->data[0];
     unsigned int modLen = modulus->len - !byteZero;
     return modLen;
@@ -931,6 +934,13 @@ RSA_PublicKeyOp(RSAPublicKey *key,
     CHECK_MPI_OK(mp_init(&c));
     modLen = rsa_modulusLen(&key->modulus);
     expLen = rsa_modulusLen(&key->publicExponent);
+
+    if (modLen == 0) {
+        PORT_SetError(SEC_ERROR_INVALID_ARGS);
+        rv = SECFailure;
+        goto cleanup;
+    }
+
     /* 1.  Obtain public key (n, e) */
     if (BAD_RSA_KEY_SIZE(modLen, expLen)) {
         PORT_SetError(SEC_ERROR_INVALID_KEY);
@@ -1434,6 +1444,10 @@ rsa_PrivateKeyOp(RSAPrivateKey *key,
     }
     /* check input out of range (needs to be in range [0..n-1]) */
     modLen = rsa_modulusLen(&key->modulus);
+    if (modLen == 0) {
+        PORT_SetError(SEC_ERROR_INVALID_ARGS);
+        return SECFailure;
+    }
     offset = (key->modulus.data[0] == 0) ? 1 : 0; /* may be leading 0 */
     if (memcmp(input, key->modulus.data + offset, modLen) >= 0) {
         PORT_SetError(SEC_ERROR_INVALID_ARGS);
diff --git a/lib/freebl/rsapkcs.c b/lib/freebl/rsapkcs.c
index 469db3fef..91b4c7c5f 100644
--- a/lib/freebl/rsapkcs.c
+++ b/lib/freebl/rsapkcs.c
@@ -80,6 +80,10 @@ constantTimeCondition(unsigned int c,
 static unsigned int
 rsa_modulusLen(SECItem *modulus)
 {
+    if (modulus->len == 0) {
+        return 0;
+    }
+
     unsigned char byteZero = modulus->data[0];
     unsigned int modLen = modulus->len - !byteZero;
     return modLen;
@@ -88,9 +92,17 @@ rsa_modulusLen(SECItem *modulus)
 static unsigned int
 rsa_modulusBits(SECItem *modulus)
 {
+    if (modulus->len == 0) {
+        return 0;
+    }
+
     unsigned char byteZero = modulus->data[0];
     unsigned int numBits = (modulus->len - 1) * 8;
 
+    if (byteZero == 0 && modulus->len == 1) {
+        return 0;
+    }
+
     if (byteZero == 0) {
         numBits -= 8;
         byteZero = modulus->data[1];