more client auth fixes

4 files changed, 55 insertions, 30 deletions

diff --git a/security/nss/lib/pki/volatiledomain.c b/security/nss/lib/pki/volatiledomain.c
index f2913858d..7ddef1fc9 100644
--- a/security/nss/lib/pki/volatiledomain.c
+++ b/security/nss/lib/pki/volatiledomain.c
@@ -674,7 +674,7 @@ nssVolatileDomain_FindCertsByNickname (
 {
     PRStatus status;
     PRUint32 i;
-    NSSCert **certs, **tdCerts;
+    NSSCert **certs, **tdCerts = NULL;
     NSSUTF8 *cNick;
     struct cert_array_str cert_array;
 
@@ -693,10 +693,18 @@ nssVolatileDomain_FindCertsByNickname (
     }
     PZ_Unlock(vd->objectLock);
 
+    if (maximumOpt) { 
+	maximumOpt -= cert_array.count;
+	if (maximumOpt == 0) { /* already full */
+	    goto finish;
+	}
+    }
+
     tdCerts = nssTrustDomain_FindCertsByNickname(vd->td, name, 
                                    rvOpt ? rvOpt + cert_array.count : NULL,
-                                   maximumOpt - cert_array.count, arenaOpt);
+                                   maximumOpt, arenaOpt);
 
+finish:
     return finish_cert_array(&cert_array, status, tdCerts);
 }
 
@@ -804,7 +812,7 @@ nssVolatileDomain_FindCertsBySubject (
 {
     PRStatus status;
     PRUint32 i;
-    NSSCert **certs, **tdCerts;
+    NSSCert **certs, **tdCerts = NULL;
     NSSDER *certSubject;
     struct cert_array_str cert_array;
 
@@ -823,10 +831,18 @@ nssVolatileDomain_FindCertsBySubject (
     }
     PZ_Unlock(vd->objectLock);
 
+    if (maximumOpt) { 
+	maximumOpt -= cert_array.count;
+	if (maximumOpt == 0) { /* already full */
+	    goto finish;
+	}
+    }
+
     tdCerts = nssTrustDomain_FindCertsBySubject(vd->td, subject, 
                                    rvOpt ? rvOpt + cert_array.count : NULL,
-                                   maximumOpt - cert_array.count, arenaOpt);
+                                   maximumOpt, arenaOpt);
 
+finish:
     return finish_cert_array(&cert_array, status, tdCerts);
 }
 
@@ -925,7 +941,7 @@ nssVolatileDomain_FindCertsByEmail (
 {
     PRStatus status;
     PRUint32 i;
-    NSSCert **certs, **tdCerts;
+    NSSCert **certs, **tdCerts = NULL;
     NSSASCII7 *cEmail;
     struct cert_array_str cert_array;
 
@@ -944,10 +960,18 @@ nssVolatileDomain_FindCertsByEmail (
     }
     PZ_Unlock(vd->objectLock);
 
+    if (maximumOpt) { 
+	maximumOpt -= cert_array.count;
+	if (maximumOpt == 0) { /* already full */
+	    goto finish;
+	}
+    }
+
     tdCerts = nssTrustDomain_FindCertsByEmail(vd->td, email, 
                                    rvOpt ? rvOpt + cert_array.count : NULL,
-                                   maximumOpt - cert_array.count, arenaOpt);
+                                   maximumOpt, arenaOpt);
 
+finish:
     return finish_cert_array(&cert_array, status, tdCerts);
 }
 
diff --git a/security/nss/lib/softoken/pkcs11c.c b/security/nss/lib/softoken/pkcs11c.c
index 1dc76c3bf..04b7ac7e9 100644
--- a/security/nss/lib/softoken/pkcs11c.c
+++ b/security/nss/lib/softoken/pkcs11c.c
@@ -2138,6 +2138,7 @@ CK_RV NSC_Sign(CK_SESSION_HANDLE hSession,
 
     if (!pSignature) {
 	*pulSignatureLen = context->maxLen;
+	rv = SECSuccess;
 	goto finish;
     }
 
diff --git a/security/nss/lib/ssl/ssl3con.c b/security/nss/lib/ssl/ssl3con.c
index da9841df5..3767456b4 100644
--- a/security/nss/lib/ssl/ssl3con.c
+++ b/security/nss/lib/ssl/ssl3con.c
@@ -910,8 +910,8 @@ ssl3_VerifySignedHashes(SSL3Hashes *hash, NSSCert *cert,
          == PR_FAILURE) 
     {
 	ssl_MapLowLevelError(SSL_ERROR_BAD_HANDSHAKE_HASH_VALUE);
-	rv = PR_FAILURE;
-    }
+	rv = SECFailure;
+    } else rv = SECSuccess;
     NSSPublicKey_Destroy(key);
     if (signature) {
     	nss_ZFreeIf(signature->data);
diff --git a/security/nss/tests/ssl/sslauth.txt b/security/nss/tests/ssl/sslauth.txt
index a2d8e7ddd..0051e9dce 100644
--- a/security/nss/tests/ssl/sslauth.txt
+++ b/security/nss/tests/ssl/sslauth.txt
@@ -6,26 +6,26 @@
 #   value  params     params
 #  ------  ------     ------                         ---------------
      0       -r           -w_nss                   TLS Request don't require client auth (client does not provide auth)
-     0       -r           -w_bogus_-n_TestUser     TLS Request don't require client auth (bad password)
-     0       -r           -w_nss_-n_TestUser       TLS Request don't require client auth (client auth)
-#     0       -r_-r        -w_nss                   TLS Require client auth (client does not provide auth)
-#    254      -r_-r        -w_bogus_-n_TestUser     TLS Require client auth (bad password)
-#     0       -r_-r        -w_nss_-n_TestUser_      TLS Require client auth (client auth)
+     0       -r           -w_bogus_-n_client     TLS Request don't require client auth (bad password)
+     0       -r           -w_nss_-n_client       TLS Request don't require client auth (client auth)
+    254      -r_-r        -w_nss                   TLS Require client auth (client does not provide auth)
+    254      -r_-r        -w_bogus_-n_client     TLS Require client auth (bad password)
+     0       -r_-r        -w_nss_-n_client_      TLS Require client auth (client auth)
      0       -r           -T_-w_nss                SSL3 Request don't require client auth (client does not provide auth)
-     0       -r           -T_-n_TestUser_-w_bogus  SSL3 Request don't require client auth (bad password)
-     0       -r           -T_-n_TestUser_-w_nss    SSL3 Request don't require client auth (client auth)
-#     0       -r_-r        -T_-w_nss                SSL3 Require client auth (client does not provide auth)
-#    254      -r_-r        -T_-n_TestUser_-w_bogus  SSL3 Require client auth (bad password)
-#     0       -r_-r        -T_-n_TestUser_-w_nss    SSL3 Require client auth (client auth)
-     0       -r_-r_-r     -w_nss                   TLS Request don't require client auth on 2nd hs (client does not provide auth)
-     0       -r_-r_-r     -w_bogus_-n_TestUser     TLS Request don't require client auth on 2nd hs (bad password)
-     0       -r_-r_-r     -w_nss_-n_TestUser       TLS Request don't require client auth on 2nd hs (client auth)
-     0       -r_-r_-r_-r  -w_nss                   TLS Require client auth on 2nd hs (client does not provide auth)
-     1       -r_-r_-r_-r  -w_bogus_-n_TestUser     TLS Require client auth on 2nd hs (bad password)
-     0       -r_-r_-r_-r  -w_nss_-n_TestUser_      TLS Require client auth on 2nd hs (client auth)
-     0       -r_-r_-r     -T_-w_nss                SSL3 Request don't require client auth on 2nd hs (client does not provide auth)
-     0       -r_-r_-r     -T_-n_TestUser_-w_bogus  SSL3 Request don't require client auth on 2nd hs (bad password)
-     0       -r_-r_-r     -T_-n_TestUser_-w_nss    SSL3 Request don't require client auth on 2nd hs (client auth)
-     0       -r_-r_-r_-r  -T_-w_nss                SSL3 Require client auth on 2nd hs (client does not provide auth)
-     1       -r_-r_-r_-r  -T_-n_TestUser_-w_bogus  SSL3 Require client auth on 2nd hs (bad password)
-     0       -r_-r_-r_-r  -T_-n_TestUser_-w_nss    SSL3 Require client auth on 2nd hs (client auth)
+     0       -r           -T_-n_client_-w_bogus  SSL3 Request don't require client auth (bad password)
+     0       -r           -T_-n_client_-w_nss    SSL3 Request don't require client auth (client auth)
+    254      -r_-r        -T_-w_nss                SSL3 Require client auth (client does not provide auth)
+    254      -r_-r        -T_-n_client_-w_bogus  SSL3 Require client auth (bad password)
+     0       -r_-r        -T_-n_client_-w_nss    SSL3 Require client auth (client auth)
+    254      -r_-r_-r     -w_nss                   TLS Request don't require client auth on 2nd hs (client does not provide auth)
+     0       -r_-r_-r     -w_bogus_-n_client     TLS Request don't require client auth on 2nd hs (bad password)
+     0       -r_-r_-r     -w_nss_-n_client       TLS Request don't require client auth on 2nd hs (client auth)
+    254      -r_-r_-r_-r  -w_nss                   TLS Require client auth on 2nd hs (client does not provide auth)
+     1       -r_-r_-r_-r  -w_bogus_-n_client     TLS Require client auth on 2nd hs (bad password)
+     0       -r_-r_-r_-r  -w_nss_-n_client_      TLS Require client auth on 2nd hs (client auth)
+    254      -r_-r_-r     -T_-w_nss                SSL3 Request don't require client auth on 2nd hs (client does not provide auth)
+     0       -r_-r_-r     -T_-n_client_-w_bogus  SSL3 Request don't require client auth on 2nd hs (bad password)
+     0       -r_-r_-r     -T_-n_client_-w_nss    SSL3 Request don't require client auth on 2nd hs (client auth)
+    254      -r_-r_-r_-r  -T_-w_nss                SSL3 Require client auth on 2nd hs (client does not provide auth)
+     1       -r_-r_-r_-r  -T_-n_client_-w_bogus  SSL3 Require client auth on 2nd hs (bad password)
+     0       -r_-r_-r_-r  -T_-n_client_-w_nss    SSL3 Require client auth on 2nd hs (client auth)