more backwards compatibility fixes

* always send DER of serial number to PKCS#11 queries
* in softoken, construct key for certificate using decoded serial number with (possibly) a leading zero, for compatibility with version 7 db
* in softoken, decode serial number *without* removing leading zero for searches

4 files changed, 54 insertions, 24 deletions

diff --git a/security/nss/lib/certdb/stanpcertdb.c b/security/nss/lib/certdb/stanpcertdb.c
index c5dda65ff..cb89bd90a 100644
--- a/security/nss/lib/certdb/stanpcertdb.c
+++ b/security/nss/lib/certdb/stanpcertdb.c
@@ -258,19 +258,7 @@ CERT_FindCertByIssuerAndSN(CERTCertDBHandle *handle, CERTIssuerAndSN *issuerAndS
 {
     PK11SlotInfo *slot;
     CERTCertificate *cert;
-    NSSDER issuer, serial;
-    NSSCryptoContext *cc;
-    NSSCertificate *c;
 
-    NSSITEM_FROM_SECITEM(&issuer, &issuerAndSN->derIssuer);
-    NSSITEM_FROM_SECITEM(&serial, &issuerAndSN->serialNumber);
-    cc = STAN_GetDefaultCryptoContext();
-    c = NSSCryptoContext_FindCertificateByIssuerAndSerialNumber(cc, 
-                                                                &issuer, 
-                                                                &serial);
-    if (c) {
-	return STAN_GetCERTCertificate(c);
-    }
     cert = PK11_FindCertByIssuerAndSN(&slot,issuerAndSN,NULL);
     if (cert && slot) {
         PK11_FreeSlot(slot);
diff --git a/security/nss/lib/pk11wrap/pk11cert.c b/security/nss/lib/pk11wrap/pk11cert.c
index a7a90b414..587c991b6 100644
--- a/security/nss/lib/pk11wrap/pk11cert.c
+++ b/security/nss/lib/pk11wrap/pk11cert.c
@@ -2156,11 +2156,31 @@ PK11_FindCertByIssuerAndSN(PK11SlotInfo **slotPtr, CERTIssuerAndSN *issuerSN,
     CERTCertificate *rvCert = NULL;
     NSSCertificate *cert;
     NSSDER issuer, serial;
-    issuer.data = (void *)issuerSN->derIssuer.data;
-    issuer.size = (PRUint32)issuerSN->derIssuer.len;
-    serial.data = (void *)issuerSN->serialNumber.data;
-    serial.size = (PRUint32)issuerSN->serialNumber.len;
-    /* XXX login to slots */
+    NSSCryptoContext *cc;
+    SECItem *derSerial;
+
+    /* PKCS#11 needs to use DER-encoded serial numbers.  Create a
+     * CERTIssuerAndSN that actually has the encoded value and pass that
+     * to PKCS#11 (and the crypto context).
+     */
+    derSerial = SEC_ASN1EncodeItem(NULL, NULL,
+                                   &issuerSN->serialNumber,
+                                   SEC_IntegerTemplate);
+    if (!derSerial) {
+	return NULL;
+    }
+
+    NSSITEM_FROM_SECITEM(&issuer, &issuerSN->derIssuer);
+    NSSITEM_FROM_SECITEM(&serial, derSerial);
+
+    cc = STAN_GetDefaultCryptoContext();
+    cert = NSSCryptoContext_FindCertificateByIssuerAndSerialNumber(cc, 
+                                                                &issuer, 
+                                                                &serial);
+    if (cert) {
+	SECITEM_FreeItem(derSerial, PR_TRUE);
+	return STAN_GetCERTCertificate(cert);
+    }
     cert = NSSTrustDomain_FindCertificateByIssuerAndSerialNumber(
                                                   STAN_GetDefaultTrustDomain(),
                                                   &issuer,
@@ -2169,6 +2189,7 @@ PK11_FindCertByIssuerAndSN(PK11SlotInfo **slotPtr, CERTIssuerAndSN *issuerSN,
 	rvCert = STAN_GetCERTCertificate(cert);
 	if (slotPtr) *slotPtr = PK11_ReferenceSlot(rvCert->slot);
     }
+    SECITEM_FreeItem(derSerial, PR_TRUE);
     return rvCert;
 #endif
 }
diff --git a/security/nss/lib/softoken/lowcert.c b/security/nss/lib/softoken/lowcert.c
index 91b7d0dc0..272edc2b9 100644
--- a/security/nss/lib/softoken/lowcert.c
+++ b/security/nss/lib/softoken/lowcert.c
@@ -339,22 +339,45 @@ nsslowcert_FixupEmailAddr(char *emailAddr)
     return(retaddr);
 }
 
+/* NSS has traditionally keyed certificate entries in the cert database
+ * by (serial number, DER_ISSUER).  The serial number may have a leading zero
+ * in order to make it a signed integer.  However, the ASN.1 decoder now 
+ * strips the leading zero, treating any INTEGER as unsigned.  In order to
+ * be compatible with version 7 of the database, it is necessary to reapply
+ * that leading zero to the serial number when needed, before computing the
+ * database key.
+ */
 static SECStatus
 nsslowcert_KeyFromIssuerAndSN(PRArenaPool *arena, SECItem *issuer, SECItem *sn,
 			SECItem *key)
 {
+    PRBool leadingZero = PR_FALSE;
+    int start;
+
     key->len = sn->len + issuer->len;
+
+    if (sn->data[0] & 0x80) {
+	leadingZero = PR_TRUE;
+	key->len++;
+    }
     
     key->data = (unsigned char*)PORT_ArenaAlloc(arena, key->len);
     if ( !key->data ) {
 	goto loser;
     }
 
+    if (leadingZero) {
+	key->data[0] = 0;
+	start = 1;
+    } else {
+	start = 0;
+    }
+
     /* copy the serialNumber */
-    PORT_Memcpy(key->data, sn->data, sn->len);
+    PORT_Memcpy(key->data + start, sn->data, sn->len);
 
     /* copy the issuer */
-    PORT_Memcpy(&key->data[sn->len], issuer->data, issuer->len);
+    PORT_Memcpy(&key->data[start + sn->len], issuer->data, issuer->len);
 
     return(SECSuccess);
 
diff --git a/security/nss/lib/softoken/pcertdb.c b/security/nss/lib/softoken/pcertdb.c
index 80972ab88..a81247031 100644
--- a/security/nss/lib/softoken/pcertdb.c
+++ b/security/nss/lib/softoken/pcertdb.c
@@ -4162,11 +4162,9 @@ nsslowcert_FindCertByIssuerAndSN(NSSLOWCERTCertDBHandle *handle, NSSLOWCERTIssue
 		}
 	    } 
 	}
-	while (sn->data[index] == 0) {
-	    index++;
-	    data_len--;
-	    data_left--;
-	}
+	/* XXX leaving any leading zeros on the serial number for backwards
+	 * compatibility
+	 */
 	/* not a valid der, must be just an unlucky serial number value */
 	if (data_len != data_left) {
 	    data_len = sn->len;