Fix for bug 403240 . Threads hanging in nss_InitLock . Stop using nss_InitLock in libssl. r=nelson,wtc

5 files changed, 52 insertions, 19 deletions

diff --git a/security/nss/lib/ssl/ssl3con.c b/security/nss/lib/ssl/ssl3con.c
index abb1e5ca0..8a90ec7c6 100644
--- a/security/nss/lib/ssl/ssl3con.c
+++ b/security/nss/lib/ssl/ssl3con.c
@@ -60,7 +60,6 @@
 
 #include "pk11func.h"
 #include "secmod.h"
-#include "nsslocks.h"
 #include "ec.h"
 #include "blapi.h"
 
@@ -3913,11 +3912,10 @@ SSL3_ShutdownServerCache(void)
     return SECSuccess;
 }
 
-void ssl_InitSymWrapKeysLock(void)
+SECStatus ssl_InitSymWrapKeysLock(void)
 {
-    /* atomically initialize the lock */
-    if (!symWrapKeysLock)
-	nss_InitLock(&symWrapKeysLock, nssILockOther);
+    symWrapKeysLock = PZ_NewLock(nssILockOther);
+    return symWrapKeysLock ? SECSuccess : SECFailure;
 }
 
 /* Try to get wrapping key for mechanism from in-memory array.
@@ -3956,7 +3954,7 @@ getWrappingKey( sslSocket *       ss,
 
     pSymWrapKey = &symWrapKeys[symWrapMechIndex].symWrapKey[exchKeyType];
 
-    ssl_InitSymWrapKeysLock();
+    ssl_InitLocks(PR_TRUE);
 
     PZ_Lock(symWrapKeysLock);
 
diff --git a/security/nss/lib/ssl/ssl3ecc.c b/security/nss/lib/ssl/ssl3ecc.c
index 14c12e28d..d404d4f9a 100644
--- a/security/nss/lib/ssl/ssl3ecc.c
+++ b/security/nss/lib/ssl/ssl3ecc.c
@@ -63,7 +63,6 @@
 
 #include "pk11func.h"
 #include "secmod.h"
-#include "nsslocks.h"
 #include "ec.h"
 #include "blapi.h"
 
diff --git a/security/nss/lib/ssl/sslimpl.h b/security/nss/lib/ssl/sslimpl.h
index 7710f76ce..874f3dec3 100644
--- a/security/nss/lib/ssl/sslimpl.h
+++ b/security/nss/lib/ssl/sslimpl.h
@@ -1445,9 +1445,9 @@ ssl_SetWrappingKey(SSLWrappedSymWrappingKey *wswk);
 /* get rid of the symmetric wrapping key references. */
 extern SECStatus SSL3_ShutdownServerCache(void);
 
-extern void ssl_InitClientSessionCacheLock(void);
+extern SECStatus ssl_InitSymWrapKeysLock(void);
 
-extern void ssl_InitSymWrapKeysLock(void);
+extern SECStatus ssl_InitLocks(PRBool lateInit);
 
 /********************** misc calls *********************/
 
diff --git a/security/nss/lib/ssl/sslnonce.c b/security/nss/lib/ssl/sslnonce.c
index e2dbcf0a5..d913d39fb 100644
--- a/security/nss/lib/ssl/sslnonce.c
+++ b/security/nss/lib/ssl/sslnonce.c
@@ -46,7 +46,6 @@
 #include "sslimpl.h"
 #include "sslproto.h"
 #include "nssilock.h"
-#include "nsslocks.h"
 #if (defined(XP_UNIX) || defined(XP_WIN) || defined(_WINDOWS) || defined(XP_BEOS)) && !defined(_WIN32_WCE)
 #include <time.h>
 #endif
@@ -68,16 +67,56 @@ static PZLock *      cacheLock = NULL;
 #define LOCK_CACHE 	lock_cache()
 #define UNLOCK_CACHE	PZ_Unlock(cacheLock)
 
-void ssl_InitClientSessionCacheLock(void)
+static SECStatus
+ssl_InitClientSessionCacheLock(void)
 {
-    if (!cacheLock)
-	nss_InitLock(&cacheLock, nssILockCache);
+    cacheLock = PZ_NewLock(nssILockCache);
+    return cacheLock ? SECSuccess : SECFailure;
+}
+
+static PRBool LocksInitializedEarly = PR_FALSE;
+
+static PRStatus
+initLocks(void)
+{
+    SECStatus rv1, rv2;
+    rv1 = ssl_InitSymWrapKeysLock();
+    rv2 = ssl_InitClientSessionCacheLock();
+    if ( (SECSuccess == rv1) && (SECSuccess == rv2) ) {
+        return PR_SUCCESS;
+    }
+    return PR_FAILURE;
+}
+
+static PRCallOnceType lockOnce;
+
+/* lateInit means that the call is not happening during a 1-time
+ * initialization function, but rather during dynamic, lazy initialization
+ */
+SECStatus
+ssl_InitLocks(PRBool lateInit)
+{
+    if (LocksInitializedEarly) {
+        return SECSuccess;
+    }
+
+    if (lateInit) {
+        return (PR_SUCCESS == PR_CallOnce(&lockOnce, initLocks)) ? 
+             SECSuccess : SECFailure;
+    }
+     
+    if (PR_SUCCESS == initLocks()) {
+        LocksInitializedEarly = PR_TRUE;
+        return SECSuccess;
+    }
+
+    return SECFailure;
 }
 
 static void 
 lock_cache(void)
 {
-    ssl_InitClientSessionCacheLock();
+    ssl_InitLocks(PR_TRUE);
     PZ_Lock(cacheLock);
 }
 
diff --git a/security/nss/lib/ssl/sslsnce.c b/security/nss/lib/ssl/sslsnce.c
index 583b500a4..c1af80942 100644
--- a/security/nss/lib/ssl/sslsnce.c
+++ b/security/nss/lib/ssl/sslsnce.c
@@ -105,7 +105,6 @@
 #define SET_ERROR_CODE /* reminder */
 
 #include "nspr.h"
-#include "nsslocks.h"
 #include "sslmutex.h"
 
 #ifdef XP_OS2_VACPP
@@ -1178,8 +1177,7 @@ SSL_ConfigServerSessionIDCache(	int      maxCacheEntries,
 			       	PRUint32 ssl3_timeout, 
 			  const char *   directory)
 {
-    ssl_InitClientSessionCacheLock();
-    ssl_InitSymWrapKeysLock();
+    ssl_InitLocks(PR_FALSE);
     return SSL_ConfigServerSessionIDCacheInstance(&globalCache, 
     		maxCacheEntries, ssl2_timeout, ssl3_timeout, directory, PR_FALSE);
 }
@@ -1292,8 +1290,7 @@ SSL_InheritMPServerSIDCacheInstance(cacheDesc *cache, const char * envString)
     	return SECSuccess;	/* already done. */
     }
 
-    ssl_InitClientSessionCacheLock();
-    ssl_InitSymWrapKeysLock();
+    ssl_InitLocks(PR_FALSE);
 
     ssl_sid_lookup  = ServerSessionIDLookup;
     ssl_sid_cache   = ServerSessionIDCache;