Bug 1798823 - Additional zero-length RSA modulus checks. r=nkulatovaNSS_3_87_BETA1

Differential Revision: https://phabricator.services.mozilla.com/D163622

2 files changed, 21 insertions, 12 deletions

diff --git a/lib/cryptohi/seckey.c b/lib/cryptohi/seckey.c
index dfeb0e20d..fb353fa14 100644
--- a/lib/cryptohi/seckey.c
+++ b/lib/cryptohi/seckey.c
@@ -1044,14 +1044,18 @@ SECKEY_PublicKeyStrengthInBits(const SECKEYPublicKey *pubk)
 unsigned
 SECKEY_SignatureLen(const SECKEYPublicKey *pubk)
 {
-    unsigned char b0;
     unsigned size;
 
     switch (pubk->keyType) {
         case rsaKey:
         case rsaPssKey:
-            b0 = pubk->u.rsa.modulus.data[0];
-            return b0 ? pubk->u.rsa.modulus.len : pubk->u.rsa.modulus.len - 1;
+            if (pubk->u.rsa.modulus.len == 0) {
+                return 0;
+            }
+            if (pubk->u.rsa.modulus.data[0] == 0) {
+                return pubk->u.rsa.modulus.len - 1;
+            }
+            return pubk->u.rsa.modulus.len;
         case dsaKey:
             return pubk->u.dsa.params.subPrime.len * 2;
         case ecKey:
diff --git a/lib/softoken/lowkey.c b/lib/softoken/lowkey.c
index 1eba1ad8f..f47bda231 100644
--- a/lib/softoken/lowkey.c
+++ b/lib/softoken/lowkey.c
@@ -226,15 +226,18 @@ nsslowkey_DestroyPublicKey(NSSLOWKEYPublicKey *pubk)
 unsigned
 nsslowkey_PublicModulusLen(NSSLOWKEYPublicKey *pubk)
 {
-    unsigned char b0;
-
     /* interpret modulus length as key strength... in
      * fortezza that's the public key length */
 
     switch (pubk->keyType) {
         case NSSLOWKEYRSAKey:
-            b0 = pubk->u.rsa.modulus.data[0];
-            return b0 ? pubk->u.rsa.modulus.len : pubk->u.rsa.modulus.len - 1;
+            if (pubk->u.rsa.modulus.len == 0) {
+                return 0;
+            }
+            if (pubk->u.rsa.modulus.data[0] == 0) {
+                return pubk->u.rsa.modulus.len - 1;
+            }
+            return pubk->u.rsa.modulus.len;
         default:
             break;
     }
@@ -244,13 +247,15 @@ nsslowkey_PublicModulusLen(NSSLOWKEYPublicKey *pubk)
 unsigned
 nsslowkey_PrivateModulusLen(NSSLOWKEYPrivateKey *privk)
 {
-
-    unsigned char b0;
-
     switch (privk->keyType) {
         case NSSLOWKEYRSAKey:
-            b0 = privk->u.rsa.modulus.data[0];
-            return b0 ? privk->u.rsa.modulus.len : privk->u.rsa.modulus.len - 1;
+            if (privk->u.rsa.modulus.len == 0) {
+                return 0;
+            }
+            if (privk->u.rsa.modulus.data[0] == 0) {
+                return privk->u.rsa.modulus.len - 1;
+            }
+            return privk->u.rsa.modulus.len;
         default:
             break;
     }