Bugzilla Bug 296410: fixed the bug that prevented the verification of

RSA signatures using SHA-512 because we were using magic constant 32 and
DSA_SIGNATURE_LEN (40), which were not big enough for SHA-512 (64 bytes).
r=relyea,jpierre.
Modified Files:
 Tag: NSS_3_10_BRANCH
	secsign.c secvfy.c

2 files changed, 8 insertions, 5 deletions

diff --git a/security/nss/lib/cryptohi/secsign.c b/security/nss/lib/cryptohi/secsign.c
index 13b75abcc..68447be26 100644
--- a/security/nss/lib/cryptohi/secsign.c
+++ b/security/nss/lib/cryptohi/secsign.c
@@ -201,7 +201,7 @@ SGN_Update(SGNContext *cx, unsigned char *input, unsigned inputLen)
 SECStatus
 SGN_End(SGNContext *cx, SECItem *result)
 {
-    unsigned char digest[32];
+    unsigned char digest[HASH_LENGTH_MAX];
     unsigned part1, signatureLen;
     SECStatus rv;
     SECItem digder, sigitem;
diff --git a/security/nss/lib/cryptohi/secvfy.c b/security/nss/lib/cryptohi/secvfy.c
index a9e3970da..4a5de4e0f 100644
--- a/security/nss/lib/cryptohi/secvfy.c
+++ b/security/nss/lib/cryptohi/secvfy.c
@@ -83,7 +83,7 @@ DecryptSigBlock(SECOidTag *tagp, unsigned char *digest, SECKEYPublicKey *key,
     */
     tag = SECOID_GetAlgorithmTag(&di->digestAlgorithm);
     /* XXX Check that tag is an appropriate algorithm? */
-    if (di->digest.len > 32) {
+    if (di->digest.len > HASH_LENGTH_MAX) {
 	PORT_SetError(SEC_ERROR_OUTPUT_LEN);
 	goto loser;
     }
@@ -110,8 +110,11 @@ struct VFYContextStr {
     SECOidTag alg;
     VerifyType type;
     SECKEYPublicKey *key;
-    /* digest holds the full dsa signature... 40 bytes */
-    unsigned char digest[DSA_SIGNATURE_LEN];
+    /*
+     * digest holds either the hash (<= HASH_LENGTH_MAX=64 bytes)
+     * in the RSA signature, or the full DSA signature (40 bytes).
+     */
+    unsigned char digest[HASH_LENGTH_MAX];
     void * wincx;
     void *hashcx;
     const SECHashObject *hashobj;
@@ -350,7 +353,7 @@ VFY_Update(VFYContext *cx, unsigned char *input, unsigned inputLen)
 SECStatus
 VFY_EndWithSignature(VFYContext *cx, SECItem *sig)
 {
-    unsigned char final[32];
+    unsigned char final[HASH_LENGTH_MAX];
     unsigned part;
     SECItem hash,dsasig; /* dsasig is also used for ECDSA */
     SECStatus rv;