Fix unnecessary assertion failures occuring in SMIME testing in

debug builds only.  Partial fix for bugscape bug 53775. r=wchang0222

6 files changed, 41 insertions, 28 deletions

diff --git a/security/nss/cmd/lib/secutil.c b/security/nss/cmd/lib/secutil.c
index 5556a55ac..76d271725 100644
--- a/security/nss/cmd/lib/secutil.c
+++ b/security/nss/cmd/lib/secutil.c
@@ -514,26 +514,23 @@ secu_StdinToItem(SECItem *dst)
 	numBytes = PR_Read(PR_STDIN, buf, sizeof(buf));
 
 	if (numBytes < 0) {
-	    PORT_SetError(PR_IO_ERROR);
 	    return SECFailure;
 	}
 
 	if (numBytes == 0)
 	    break;
 
-	if (buf[numBytes-1] == '\n') {
-	    buf[numBytes-1] = '\0';
-	    notDone = PR_FALSE;
-	}
-
 	if (dst->data) {
+	    /* XXX if PORT_Realloc fails, the old buffer is leaked. */
 	    dst->data = (unsigned char*)PORT_Realloc(dst->data, 
-	                                             dst->len+numBytes);
-	    PORT_Memcpy(dst->data+dst->len, buf, numBytes);
+	                                             dst->len + numBytes);
 	} else {
 	    dst->data = (unsigned char*)PORT_Alloc(numBytes);
-	    PORT_Memcpy(dst->data, buf, numBytes);
 	}
+	if (!dst->data) {
+	    return SECFailure;
+	}
+	PORT_Memcpy(dst->data + dst->len, buf, numBytes);
 	dst->len += numBytes;
     }
 
diff --git a/security/nss/lib/smime/cmscinfo.c b/security/nss/lib/smime/cmscinfo.c
index 85756a536..d01c5962b 100644
--- a/security/nss/lib/smime/cmscinfo.c
+++ b/security/nss/lib/smime/cmscinfo.c
@@ -93,20 +93,31 @@ NSS_CMSContentInfo_Destroy(NSSCMSContentInfo *cinfo)
 NSSCMSContentInfo *
 NSS_CMSContentInfo_GetChildContentInfo(NSSCMSContentInfo *cinfo)
 {
-    switch (NSS_CMSContentInfo_GetContentTypeTag(cinfo)) {
-    case SEC_OID_PKCS7_DATA:
-	return NULL;
+    void * ptr                  = NULL;
+    NSSCMSContentInfo * ccinfo  = NULL;
+    SECOidTag tag = NSS_CMSContentInfo_GetContentTypeTag(cinfo);
+    switch (tag) {
     case SEC_OID_PKCS7_SIGNED_DATA:
-	return &(cinfo->content.signedData->contentInfo);
+	ptr    = (void *)cinfo->content.signedData;
+	ccinfo = &(cinfo->content.signedData->contentInfo);
+	break;
     case SEC_OID_PKCS7_ENVELOPED_DATA:
-	return &(cinfo->content.envelopedData->contentInfo);
+	ptr    = (void *)cinfo->content.envelopedData;
+	ccinfo = &(cinfo->content.envelopedData->contentInfo);
+	break;
     case SEC_OID_PKCS7_DIGESTED_DATA:
-	return &(cinfo->content.digestedData->contentInfo);
+	ptr    = (void *)cinfo->content.digestedData;
+	ccinfo = &(cinfo->content.digestedData->contentInfo);
+	break;
     case SEC_OID_PKCS7_ENCRYPTED_DATA:
-	return &(cinfo->content.encryptedData->contentInfo);
+	ptr    = (void *)cinfo->content.encryptedData;
+	ccinfo = &(cinfo->content.encryptedData->contentInfo);
+	break;
+    case SEC_OID_PKCS7_DATA:
     default:
-	return NULL;
+	break;
     }
+    return (ptr ? ccinfo : NULL);
 }
 
 /*
diff --git a/security/nss/lib/smime/cmscipher.c b/security/nss/lib/smime/cmscipher.c
index 27e5668aa..6d73beaff 100644
--- a/security/nss/lib/smime/cmscipher.c
+++ b/security/nss/lib/smime/cmscipher.c
@@ -598,7 +598,7 @@ NSS_CMSCipherContext_Decrypt(NSSCMSCipherContext *cc, unsigned char *output,
      */
     if (final && (padsize != 0)) {
 	unsigned int padlen = *(output + ofraglen - 1);
-	PORT_Assert (padlen > 0 && padlen <= padsize);
+
 	if (padlen == 0 || padlen > padsize) {
 	    PORT_SetError(SEC_ERROR_BAD_DATA);
 	    return SECFailure;
diff --git a/security/nss/lib/smime/cmsmessage.c b/security/nss/lib/smime/cmsmessage.c
index 21a730e6f..7e785cc1f 100644
--- a/security/nss/lib/smime/cmsmessage.c
+++ b/security/nss/lib/smime/cmsmessage.c
@@ -193,8 +193,9 @@ NSS_CMSMessage_ContentLevelCount(NSSCMSMessage *cmsg)
     NSSCMSContentInfo *cinfo;
 
     /* walk down the chain of contentinfos */
-    for (cinfo = &(cmsg->contentInfo); cinfo != NULL; cinfo = NSS_CMSContentInfo_GetChildContentInfo(cinfo)) {
+    for (cinfo = &(cmsg->contentInfo); cinfo != NULL; ) {
 	count++;
+	cinfo = NSS_CMSContentInfo_GetChildContentInfo(cinfo);
     }
     return count;
 }
diff --git a/security/nss/lib/smime/cmsutil.c b/security/nss/lib/smime/cmsutil.c
index 9c88e4ee9..11efdcd3e 100644
--- a/security/nss/lib/smime/cmsutil.c
+++ b/security/nss/lib/smime/cmsutil.c
@@ -319,10 +319,10 @@ NSSCMSContentInfo *
 NSS_CMSContent_GetContentInfo(void *msg, SECOidTag type)
 {
     NSSCMSContent c;
-    NSSCMSContentInfo *cinfo;
-
-    PORT_Assert(msg != NULL);
+    NSSCMSContentInfo *cinfo = NULL;
 
+    if (!msg)
+	return cinfo;
     c.pointer = msg;
     switch (type) {
     case SEC_OID_PKCS7_SIGNED_DATA:
diff --git a/security/nss/lib/util/secasn1d.c b/security/nss/lib/util/secasn1d.c
index 0e6ad0a79..965fa702d 100644
--- a/security/nss/lib/util/secasn1d.c
+++ b/security/nss/lib/util/secasn1d.c
@@ -403,6 +403,7 @@ sec_asn1d_push_state (SEC_ASN1DecoderContext *cx,
 	new_state->depth = state->depth;
 	if (new_depth) {
 	    if (++new_state->depth > SEC_ASN1D_MAX_DEPTH) {
+		PORT_SetError (SEC_ERROR_BAD_DER);
 		goto loser;
 	    }
 	}
@@ -1046,6 +1047,7 @@ sec_asn1d_prepare_for_contents (sec_asn1d_state *state)
 	sec_asn1d_state *parent = sec_asn1d_get_enclosing_construct(state);
 	if (parent && !parent->indefinite && 
 	    state->consumed + state->contents_length > parent->pending) {
+	    PORT_SetError (SEC_ERROR_BAD_DER);
 	    state->top->status = decodeError;
 	    return;
 	}
@@ -1137,6 +1139,7 @@ sec_asn1d_prepare_for_contents (sec_asn1d_state *state)
 	 * implement this because in practice, it seems to be unused.
 	 */
 	PORT_Assert(0);
+	PORT_SetError (SEC_ERROR_BAD_DER); /* XXX */
 	state->top->status = decodeError;
 	break;
 
@@ -1146,6 +1149,7 @@ sec_asn1d_prepare_for_contents (sec_asn1d_state *state)
 	 * An indefinite-length encoding is not alloweed.
 	 */
 	if (state->contents_length || state->indefinite) {
+	    PORT_SetError (SEC_ERROR_BAD_DER);
 	    state->top->status = decodeError;
 	    break;
 	}
@@ -1160,6 +1164,7 @@ sec_asn1d_prepare_for_contents (sec_asn1d_state *state)
       case SEC_ASN1_BMP_STRING:
 	/* Error if length is not divisable by 2 */
 	if (state->contents_length % 2) {
+	   PORT_SetError (SEC_ERROR_BAD_DER);
 	   state->top->status = decodeError;
 	   break;
 	}   
@@ -1169,6 +1174,7 @@ sec_asn1d_prepare_for_contents (sec_asn1d_state *state)
       case SEC_ASN1_UNIVERSAL_STRING:
 	/* Error if length is not divisable by 4 */
 	if (state->contents_length % 4) {
+	   PORT_SetError (SEC_ERROR_BAD_DER);
 	   state->top->status = decodeError;
 	   break;
 	}   
@@ -2185,13 +2191,9 @@ sec_asn1d_absorb_child (sec_asn1d_state *state)
     /*
      * Inherit the missing status of our child, and do the ugly
      * backing-up if necessary.
-     * (Only IMPLICIT or POINTER should encounter such; all other cases
-     * should have confirmed a tag *before* pushing a child.)
      */
     state->missing = state->child->missing;
     if (state->missing) {
-	PORT_Assert (state->place == afterImplicit
-		     || state->place == afterPointer);
 	state->found_tag_number = state->child->found_tag_number;
 	state->found_tag_modifiers = state->child->found_tag_modifiers;
 	state->endofcontents = state->child->endofcontents;
@@ -2651,6 +2653,7 @@ SEC_ASN1DecoderUpdate (SEC_ASN1DecoderContext *cx,
 	    }
 	    if (cx->status == needBytes) {
 		/* recursive call wanted more data. Fatal. Clean up below. */
+		PORT_SetError (SEC_ERROR_BAD_DER);
 		cx->status = decodeError;
 	    }
 	    break;
@@ -2725,9 +2728,10 @@ SEC_ASN1DecoderUpdate (SEC_ASN1DecoderContext *cx,
 	 * length which is greater than the entire encoding.  So, we cannot
 	 * have this be an error.
 	 */
-	    if (len > 0)
+	    if (len > 0) {
+		PORT_SetError (SEC_ERROR_BAD_DER);
 		cx->status = decodeError;
-	    else
+	    } else
 #endif
 		cx->status = allDone;
 	    break;