Bug 186586: If at NSS shutdown there are still certs in the cert caches,

cause NSS shutdown and the next NSS initialization to fail but do not
destroy the cert caches (and the crypto context and trust domain containing
them) to avoid a crash if the NSS client destroys the certs later.  New
error codes needed to be added to indicate the failure of NSS shutdown and
NSS initialization due to this cause.
Modified Files:
	base/errorval.c nss/nssinit.c pki/pki3hack.c pki/pki3hack.h
	pki/pkistore.c pki/pkistore.h pki/tdcache.c pki/trustdomain.c
	util/secerr.h

9 files changed, 63 insertions, 12 deletions

diff --git a/security/nss/lib/base/errorval.c b/security/nss/lib/base/errorval.c
index 7a5892d92..88e406c84 100644
--- a/security/nss/lib/base/errorval.c
+++ b/security/nss/lib/base/errorval.c
@@ -88,4 +88,6 @@ const NSSError NSS_ERROR_CERTIFICATE_IN_CACHE           = 32;
 const NSSError NSS_ERROR_HASH_COLLISION                 = 33;
 const NSSError NSS_ERROR_DEVICE_ERROR                   = 34;
 const NSSError NSS_ERROR_INVALID_CERTIFICATE            = 35;
+const NSSError NSS_ERROR_BUSY                           = 36;
+const NSSError NSS_ERROR_ALREADY_INITIALIZED            = 37;
 
diff --git a/security/nss/lib/nss/nssinit.c b/security/nss/lib/nss/nssinit.c
index 0297bbd25..c2f72091a 100644
--- a/security/nss/lib/nss/nssinit.c
+++ b/security/nss/lib/nss/nssinit.c
@@ -49,6 +49,8 @@
 #include "nss.h"
 #include "secrng.h"
 #include "pk11func.h"
+#include "secerr.h"
+#include "nssbase.h"
 
 #include "pki3hack.h"
 #include "certi.h"
@@ -464,8 +466,9 @@ loser:
     }
 
     if (rv == SECSuccess) {
-	/* can this function fail?? */
-	STAN_LoadDefaultNSS3TrustDomain();
+	if (STAN_LoadDefaultNSS3TrustDomain() != PR_SUCCESS) {
+	    return SECFailure;
+	}
 	CERT_SetDefaultCertDB((CERTCertDBHandle *)
 				STAN_GetDefaultTrustDomain());
 #ifndef XP_MAC
@@ -540,18 +543,27 @@ NSS_NoDB_Init(const char * configdir)
 			PR_TRUE,PR_TRUE,PR_TRUE,PR_TRUE,PR_TRUE,PR_TRUE);
 }
 
+extern const NSSError NSS_ERROR_BUSY;
+
 SECStatus
 NSS_Shutdown(void)
 {
     SECStatus rv;
+    PRStatus status;
 
     ShutdownCRLCache();
     SECOID_Shutdown();
-    STAN_Shutdown();
+    status = STAN_Shutdown();
     cert_DestroySubjectKeyIDHashTable();
     SECMOD_CleanupCallOnce();
     rv = SECMOD_Shutdown();
     pk11sdr_Shutdown();
+    if (status == PR_FAILURE) {
+	if (NSS_GetError() == NSS_ERROR_BUSY) {
+	    PORT_SetError(SEC_ERROR_BUSY);
+	}
+	rv = SECFailure;
+    }
     nss_IsInitted = PR_FALSE;
     return rv;
 }
diff --git a/security/nss/lib/pki/pki3hack.c b/security/nss/lib/pki/pki3hack.c
index a828f98c8..ae096667d 100644
--- a/security/nss/lib/pki/pki3hack.c
+++ b/security/nss/lib/pki/pki3hack.c
@@ -87,6 +87,8 @@ STAN_GetDefaultCryptoContext()
     return g_default_crypto_context;
 }
 
+extern const NSSError NSS_ERROR_ALREADY_INITIALIZED;
+
 NSS_IMPLEMENT PRStatus
 STAN_LoadDefaultNSS3TrustDomain (
   void
@@ -98,6 +100,11 @@ STAN_LoadDefaultNSS3TrustDomain (
     SECMODListLock *moduleLock = SECMOD_GetDefaultModuleListLock();
     int i;
 
+    if (g_default_trust_domain || g_default_crypto_context) {
+	/* Stan is already initialized or a previous shutdown failed. */
+	nss_SetError(NSS_ERROR_ALREADY_INITIALIZED);
+	return PR_FAILURE;
+    }
     td = NSSTrustDomain_Create(NULL, NULL, NULL, NULL);
     if (!td) {
 	return PR_FAILURE;
@@ -160,15 +167,25 @@ STAN_RemoveModuleFromDefaultTrustDomain (
     return SECSuccess;
 }
 
-NSS_IMPLEMENT void
+NSS_IMPLEMENT PRStatus
 STAN_Shutdown()
 {
+    PRStatus status = PR_SUCCESS;
     if (g_default_trust_domain) {
-	NSSTrustDomain_Destroy(g_default_trust_domain);
+	if (NSSTrustDomain_Destroy(g_default_trust_domain) == PR_SUCCESS) {
+	    g_default_trust_domain = NULL;
+	} else {
+	    status = PR_FAILURE;
+	}
     }
     if (g_default_crypto_context) {
-	NSSCryptoContext_Destroy(g_default_crypto_context);
+	if (NSSCryptoContext_Destroy(g_default_crypto_context) == PR_SUCCESS) {
+	    g_default_crypto_context = NULL;
+	} else {
+	    status = PR_FAILURE;
+	}
     }
+    return status;
 }
 
 /* this function should not be a hack; it will be needed in 4.0 (rename) */
diff --git a/security/nss/lib/pki/pki3hack.h b/security/nss/lib/pki/pki3hack.h
index 7343b52f2..da68269c3 100644
--- a/security/nss/lib/pki/pki3hack.h
+++ b/security/nss/lib/pki/pki3hack.h
@@ -72,7 +72,7 @@ STAN_LoadDefaultNSS3TrustDomain
   void
 );
 
-NSS_EXTERN void
+NSS_EXTERN PRStatus
 STAN_Shutdown();
 
 NSS_EXTERN SECStatus
diff --git a/security/nss/lib/pki/pkistore.c b/security/nss/lib/pki/pkistore.c
index 9d843312c..08250db63 100644
--- a/security/nss/lib/pki/pkistore.c
+++ b/security/nss/lib/pki/pkistore.c
@@ -160,11 +160,17 @@ loser:
     return NULL;
 }
 
-NSS_IMPLEMENT void
+extern const NSSError NSS_ERROR_BUSY;
+
+NSS_IMPLEMENT PRStatus
 nssCertificateStore_Destroy (
   nssCertificateStore *store
 )
 {
+    if (nssHash_Count(store->issuer_and_serial) > 0) {
+	nss_SetError(NSS_ERROR_BUSY);
+	return PR_FAILURE;
+    }
     PZ_DestroyLock(store->lock);
     nssHash_Destroy(store->issuer_and_serial);
     nssHash_Destroy(store->subject);
@@ -173,6 +179,7 @@ nssCertificateStore_Destroy (
     } else {
 	nss_ZFreeIf(store);
     }
+    return PR_SUCCESS;
 }
 
 static PRStatus
diff --git a/security/nss/lib/pki/pkistore.h b/security/nss/lib/pki/pkistore.h
index 175298891..0fd9e679d 100644
--- a/security/nss/lib/pki/pkistore.h
+++ b/security/nss/lib/pki/pkistore.h
@@ -72,7 +72,7 @@ nssCertificateStore_Create
   NSSArena *arenaOpt
 );
 
-NSS_EXTERN void
+NSS_EXTERN PRStatus
 nssCertificateStore_Destroy
 (
   nssCertificateStore *store
diff --git a/security/nss/lib/pki/tdcache.c b/security/nss/lib/pki/tdcache.c
index 8f88899e1..1f1d3fa97 100644
--- a/security/nss/lib/pki/tdcache.c
+++ b/security/nss/lib/pki/tdcache.c
@@ -234,12 +234,20 @@ loser:
  * clean shutdown, it is necessary for there to be no certs in the cache.
  */
 
+extern const NSSError NSS_ERROR_INTERNAL_ERROR;
+extern const NSSError NSS_ERROR_BUSY;
+
 NSS_IMPLEMENT PRStatus
 nssTrustDomain_DestroyCache (
   NSSTrustDomain *td
 )
 {
     if (!td->cache) {
+	nss_SetError(NSS_ERROR_INTERNAL_ERROR);
+	return PR_FAILURE;
+    }
+    if (nssHash_Count(td->cache->issuerAndSN) > 0) {
+	nss_SetError(NSS_ERROR_BUSY);
 	return PR_FAILURE;
     }
     PZ_DestroyLock(td->cache->lock);
diff --git a/security/nss/lib/pki/trustdomain.c b/security/nss/lib/pki/trustdomain.c
index d26e4ec76..a023ce3f8 100644
--- a/security/nss/lib/pki/trustdomain.c
+++ b/security/nss/lib/pki/trustdomain.c
@@ -116,6 +116,7 @@ NSSTrustDomain_Destroy (
   NSSTrustDomain *td
 )
 {
+    PRStatus status = PR_SUCCESS;
     if (--td->refCount == 0) {
 	/* Destroy each token in the list of tokens */
 	if (td->tokens) {
@@ -123,11 +124,14 @@ NSSTrustDomain_Destroy (
 	    nssList_Clear(td->tokenList, token_destructor);
 	    nssList_Destroy(td->tokenList);
 	}
-	nssTrustDomain_DestroyCache(td);
+	status = nssTrustDomain_DestroyCache(td);
+	if (status == PR_FAILURE) {
+	    return status;
+	}
 	/* Destroy the trust domain */
 	nssArena_Destroy(td->arena);
     }
-    return PR_SUCCESS;
+    return status;
 }
 
 /* XXX uses tokens until slot list is in place */
diff --git a/security/nss/lib/util/secerr.h b/security/nss/lib/util/secerr.h
index 525ed513b..7e139cae1 100644
--- a/security/nss/lib/util/secerr.h
+++ b/security/nss/lib/util/secerr.h
@@ -183,7 +183,8 @@ SEC_ERROR_UNSUPPORTED_MESSAGE_TYPE	    =	(SEC_ERROR_BASE + 134),
 SEC_ERROR_MODULE_STUCK			    =	(SEC_ERROR_BASE + 135),
 SEC_ERROR_BAD_TEMPLATE			    =	(SEC_ERROR_BASE + 136),
 SEC_ERROR_CRL_NOT_FOUND 		    =	(SEC_ERROR_BASE + 137),
-SEC_ERROR_REUSED_ISSUER_AND_SERIAL         =   (SEC_ERROR_BASE + 138)
+SEC_ERROR_REUSED_ISSUER_AND_SERIAL          =   (SEC_ERROR_BASE + 138),
+SEC_ERROR_BUSY                              =   (SEC_ERROR_BASE + 139)
 
 } SECErrorCodes;
 #endif /* NO_SECURITY_ERROR_ENUM */