Fix for 249488 - root cert module requires locking functions in CK_C_INITIALIZE_ARGS . r=ian, sr=relyea

7 files changed, 119 insertions, 76 deletions

diff --git a/security/nss/lib/ckfw/ckfw.h b/security/nss/lib/ckfw/ckfw.h
index 2e83f8c35..719b6fa4a 100644
--- a/security/nss/lib/ckfw/ckfw.h
+++ b/security/nss/lib/ckfw/ckfw.h
@@ -107,6 +107,7 @@ NSS_EXTERN NSSCKFWInstance *
 nssCKFWInstance_Create
 (
   CK_C_INITIALIZE_ARGS_PTR pInitArgs,
+  CryptokiLockingState LockingState,
   NSSCKMDInstance *mdInstance,
   CK_RV *pError
 );
@@ -1821,6 +1822,7 @@ NSS_EXTERN NSSCKFWMutex *
 nssCKFWMutex_Create
 (
   CK_C_INITIALIZE_ARGS_PTR pInitArgs,
+  CryptokiLockingState LockingState,
   NSSArena *arena,
   CK_RV *pError
 );
diff --git a/security/nss/lib/ckfw/ckfwm.h b/security/nss/lib/ckfw/ckfwm.h
index 615efe2b6..8940cdc96 100644
--- a/security/nss/lib/ckfw/ckfwm.h
+++ b/security/nss/lib/ckfw/ckfwm.h
@@ -158,9 +158,10 @@ nssCKFWHash_Iterate
   void *closure
 );
 
-NSS_EXTERN void
+NSS_EXTERN CK_RV
 nssSetLockArgs(
-   CK_C_INITIALIZE_ARGS_PTR pInitArgs
+   CK_C_INITIALIZE_ARGS_PTR pInitArgs,
+   CryptokiLockingState* returned
 );
 
 
diff --git a/security/nss/lib/ckfw/instance.c b/security/nss/lib/ckfw/instance.c
index 103726573..93011b30d 100644
--- a/security/nss/lib/ckfw/instance.c
+++ b/security/nss/lib/ckfw/instance.c
@@ -97,6 +97,8 @@ struct NSSCKFWInstanceStr {
   NSSArena *arena;
   NSSCKMDInstance *mdInstance;
   CK_C_INITIALIZE_ARGS_PTR pInitArgs;
+  CK_C_INITIALIZE_ARGS initArgs;
+  CryptokiLockingState LockingState;
   CK_BBOOL mayCreatePthreads;
   NSSUTF8 *configurationData;
   CK_ULONG nSlots;
@@ -188,6 +190,7 @@ NSS_IMPLEMENT NSSCKFWInstance *
 nssCKFWInstance_Create
 (
   CK_C_INITIALIZE_ARGS_PTR pInitArgs,
+  CryptokiLockingState LockingState,
   NSSCKMDInstance *mdInstance,
   CK_RV *pError
 )
@@ -221,9 +224,11 @@ nssCKFWInstance_Create
 
   fwInstance->arena = arena;
   fwInstance->mdInstance = mdInstance;
-  fwInstance->pInitArgs = pInitArgs;
 
+  fwInstance->LockingState = LockingState;
   if( (CK_C_INITIALIZE_ARGS_PTR)NULL != pInitArgs ) {
+    fwInstance->initArgs = *pInitArgs;
+    fwInstance->pInitArgs = &fwInstance->initArgs;
     if( pInitArgs->flags & CKF_LIBRARY_CANT_CREATE_OS_THREADS ) {
       fwInstance->mayCreatePthreads = CK_FALSE;
     } else {
@@ -234,7 +239,8 @@ nssCKFWInstance_Create
     fwInstance->mayCreatePthreads = CK_TRUE;
   }
 
-  fwInstance->mutex = nssCKFWMutex_Create(pInitArgs, arena, pError);
+  fwInstance->mutex = nssCKFWMutex_Create(pInitArgs, LockingState, arena,
+                                          pError);
   if( (NSSCKFWMutex *)NULL == fwInstance->mutex ) {
     if( CKR_OK == *pError ) {
       *pError = CKR_GENERAL_ERROR;
@@ -358,7 +364,9 @@ nssCKFWInstance_Create
     }
   }
 
-  (void)NSSArena_Destroy(arena);
+  if (arena) {
+    (void)NSSArena_Destroy(arena);
+  }
   return (NSSCKFWInstance *)NULL;
 }
 
@@ -499,7 +507,8 @@ nssCKFWInstance_CreateMutex
   }
 #endif /* NSSDEBUG */
 
-  mutex = nssCKFWMutex_Create(fwInstance->pInitArgs, arena, pError);
+  mutex = nssCKFWMutex_Create(fwInstance->pInitArgs, fwInstance->LockingState,
+                              arena, pError);
   if( (NSSCKFWMutex *)NULL == mutex ) {
     if( CKR_OK == *pError ) {
       *pError = CKR_GENERAL_ERROR;
diff --git a/security/nss/lib/ckfw/mutex.c b/security/nss/lib/ckfw/mutex.c
index 789616683..ea9700d3e 100644
--- a/security/nss/lib/ckfw/mutex.c
+++ b/security/nss/lib/ckfw/mutex.c
@@ -129,79 +129,39 @@ NSS_EXTERN NSSCKFWMutex *
 nssCKFWMutex_Create
 (
   CK_C_INITIALIZE_ARGS_PTR pInitArgs,
+  CryptokiLockingState LockingState,
   NSSArena *arena,
   CK_RV *pError
 )
 {
   NSSCKFWMutex *mutex;
-  CK_ULONG count = (CK_ULONG)0;
-  CK_BBOOL os_ok = CK_FALSE;
-
-  if( (CK_C_INITIALIZE_ARGS_PTR)NULL != pInitArgs ) {
-    if( (CK_CREATEMUTEX )NULL != pInitArgs->CreateMutex  ) count++;
-    if( (CK_DESTROYMUTEX)NULL != pInitArgs->DestroyMutex ) count++;
-    if( (CK_LOCKMUTEX   )NULL != pInitArgs->LockMutex    ) count++;
-    if( (CK_UNLOCKMUTEX )NULL != pInitArgs->UnlockMutex  ) count++;
-    os_ok = (pInitArgs->flags & CKF_OS_LOCKING_OK) ? CK_TRUE : CK_FALSE;
-
-    if( (0 != count) && (4 != count) ) {
-      *pError = CKR_ARGUMENTS_BAD;
-      return (NSSCKFWMutex *)NULL;
-    }
-  }
-
-  if( (0 == count) && (CK_TRUE == os_ok) ) {
-    /*
-     * This is case #2 in the description of C_Initialize:
-     * The library will be called in a multithreaded way, but
-     * no routines were specified: os locking calls should be
-     * used.  Unfortunately, this can be hard.. like, I think
-     * I may have to dynamically look up the entry points in
-     * the instance of NSPR already going in the application.
-     *
-     * I know that *we* always specify routines, so this only
-     * comes up if someone is using NSS to create their own
-     * PCKS#11 modules for other products.  Oh, heck, I'll 
-     * worry about this then.
-     */
-    *pError = CKR_CANT_LOCK;
-    return (NSSCKFWMutex *)NULL;
-  }
-
+  
   mutex = nss_ZNEW(arena, NSSCKFWMutex);
   if( (NSSCKFWMutex *)NULL == mutex ) {
     *pError = CKR_HOST_MEMORY;
     return (NSSCKFWMutex *)NULL;
   }
 
-  if( 0 == count ) {
-    /*
-     * With the above test out of the way, we know this is case
-     * #1 in the description of C_Initialize: this library will
-     * not be called in a multithreaded way.  I'll just return
-     * an object with noop calls.
-     */
-
-    mutex->Destroy = (CK_DESTROYMUTEX)mutex_noop;
-    mutex->Lock    = (CK_LOCKMUTEX   )mutex_noop;
-    mutex->Unlock  = (CK_UNLOCKMUTEX )mutex_noop;
-  } else {
-    /*
-     * We know that we're in either case #3 or #4 in the description
-     * of C_Initialize.  Case #3 says we should use the specified
-     * functions, case #4 cays we can use either the specified ones
-     * or the OS ones.  I'll use the specified ones.
-     */
-
-    mutex->Destroy = pInitArgs->DestroyMutex;
-    mutex->Lock    = pInitArgs->LockMutex;
-    mutex->Unlock  = pInitArgs->UnlockMutex;
+  switch (LockingState)
+  {
+      default:
+      case SingleThreaded:
+          mutex->Destroy = (CK_DESTROYMUTEX)mutex_noop;
+          mutex->Lock    = (CK_LOCKMUTEX   )mutex_noop;
+          mutex->Unlock  = (CK_UNLOCKMUTEX )mutex_noop;
+          break;
+
+      case MultiThreaded:
+          *pError = pInitArgs->CreateMutex(&mutex->etc);
+          mutex->Destroy = pInitArgs->DestroyMutex;
+          mutex->Lock    = pInitArgs->LockMutex;
+          mutex->Unlock  = pInitArgs->UnlockMutex;
+          break;
+  }
     
-    *pError = pInitArgs->CreateMutex(&mutex->etc);
-    if( CKR_OK != *pError ) {
-      (void)nss_ZFreeIf(mutex);
-      return (NSSCKFWMutex *)NULL;
-    }
+  if( CKR_OK != *pError ) {
+    (void)nss_ZFreeIf(mutex);
+    return (NSSCKFWMutex *)NULL;
   }
 
 #ifdef DEBUG
@@ -343,3 +303,4 @@ NSSCKFWMutex_Unlock
 
   return nssCKFWMutex_Unlock(mutex);
 }
+
diff --git a/security/nss/lib/ckfw/nsprstub.c b/security/nss/lib/ckfw/nsprstub.c
index 0801023d4..68549357f 100644
--- a/security/nss/lib/ckfw/nsprstub.c
+++ b/security/nss/lib/ckfw/nsprstub.c
@@ -345,15 +345,76 @@ PR_IMPLEMENT(PRInt32) PR_AtomicSet(PRInt32 *val) { return ++(*val); }
 PR_IMPLEMENT(PRInt32) PR_AtomicIncrement(PRInt32 *val) { return ++(*val); }
 #endif /* ! (WIN32 && GCC) */
 
-CK_C_INITIALIZE_ARGS_PTR nssstub_initArgs = NULL;
-NSSArena *nssstub_arena = NULL;
-PR_IMPLEMENT(void)
-nssSetLockArgs(CK_C_INITIALIZE_ARGS_PTR pInitArgs)
+static CK_C_INITIALIZE_ARGS_PTR nssstub_pInitArgs = NULL;
+static CK_C_INITIALIZE_ARGS nssstub_initArgs;
+static NSSArena *nssstub_arena = NULL;
+static CryptokiLockingState nssstub_LockingState = SingleThreaded;
+
+PR_IMPLEMENT(CK_RV)
+nssSetLockArgs(CK_C_INITIALIZE_ARGS_PTR pInitArgs, CryptokiLockingState* returned)
 {
-    if (nssstub_initArgs == NULL) {
-	nssstub_initArgs = pInitArgs;
+    CK_ULONG count = (CK_ULONG)0;
+    CK_BBOOL os_ok = CK_FALSE;
+    CK_RV rv = CKR_OK;
+    if (nssstub_pInitArgs == NULL) {
+	if (pInitArgs != NULL) {
+	    nssstub_initArgs = *pInitArgs;
+	    nssstub_pInitArgs = &nssstub_initArgs;
+            if( (CK_CREATEMUTEX )NULL != pInitArgs->CreateMutex  ) count++;
+            if( (CK_DESTROYMUTEX)NULL != pInitArgs->DestroyMutex ) count++;
+            if( (CK_LOCKMUTEX   )NULL != pInitArgs->LockMutex    ) count++;
+            if( (CK_UNLOCKMUTEX )NULL != pInitArgs->UnlockMutex  ) count++;
+            os_ok = (pInitArgs->flags & CKF_OS_LOCKING_OK) ? CK_TRUE : CK_FALSE;
+
+            if( (0 != count) && (4 != count) ) {
+                rv = CKR_ARGUMENTS_BAD;
+                goto loser;
+            }
+	} else {
+	    nssstub_pInitArgs = pInitArgs;
+	}
 	/* nssstub_arena = NSSArena_Create(); */
     }
+
+    if( (0 == count) && (CK_TRUE == os_ok) ) {
+      /*
+       * This is case #2 in the description of C_Initialize:
+       * The library will be called in a multithreaded way, but
+       * no routines were specified: os locking calls should be
+       * used.  Unfortunately, this can be hard.. like, I think
+       * I may have to dynamically look up the entry points in
+       * the instance of NSPR already going in the application.
+       *
+       * I know that *we* always specify routines, so this only
+       * comes up if someone is using NSS to create their own
+       * PCKS#11 modules for other products.  Oh, heck, I'll 
+       * worry about this then.
+       */
+      rv = CKR_CANT_LOCK;
+      goto loser;
+    }
+
+    if( 0 == count ) {
+      /*
+       * With the above test out of the way, we know this is case
+       * #1 in the description of C_Initialize: this library will
+       * not be called in a multithreaded way.
+       */
+
+      nssstub_LockingState = SingleThreaded;
+    } else {
+      /*
+       * We know that we're in either case #3 or #4 in the description
+       * of C_Initialize.  Case #3 says we should use the specified
+       * functions, case #4 cays we can use either the specified ones
+       * or the OS ones.  I'll use the specified ones.
+       */
+      nssstub_LockingState = MultiThreaded;
+    }
+
+    loser:
+    *returned = nssstub_LockingState;
+    return rv;
 }
 
 /*
@@ -369,7 +430,7 @@ PR_NewLock(void) {
 	NSSCKFWMutex *mlock = NULL;
 	CK_RV error;
 
-	mlock = nssCKFWMutex_Create(nssstub_initArgs,nssstub_arena,&error);
+	mlock = nssCKFWMutex_Create(nssstub_pInitArgs,nssstub_LockingState,nssstub_arena,&error);
 	lock = (PRLock *)mlock;
 
 	/* if we don't have a lock, nssCKFWMutex can deal with things */
diff --git a/security/nss/lib/ckfw/nssckfwt.h b/security/nss/lib/ckfw/nssckfwt.h
index 13be0f325..f8855be0e 100644
--- a/security/nss/lib/ckfw/nssckfwt.h
+++ b/security/nss/lib/ckfw/nssckfwt.h
@@ -108,4 +108,9 @@ typedef struct NSSCKFWFindObjectsStr NSSCKFWFindObjects;
 struct NSSCKFWMutexStr;
 typedef struct NSSCKFWMutexStr NSSCKFWMutex;
 
+typedef enum {
+    SingleThreaded,
+    MultiThreaded
+} CryptokiLockingState ;
+
 #endif /* NSSCKFWT_H */
diff --git a/security/nss/lib/ckfw/wrap.c b/security/nss/lib/ckfw/wrap.c
index 08ad62baf..caba8c45d 100644
--- a/security/nss/lib/ckfw/wrap.c
+++ b/security/nss/lib/ckfw/wrap.c
@@ -135,6 +135,7 @@ NSSCKFWC_Initialize
 )
 {
   CK_RV error = CKR_OK;
+  CryptokiLockingState locking_state;
 
   if( (NSSCKFWInstance **)NULL == pFwInstance ) {
     error = CKR_GENERAL_ERROR;
@@ -154,9 +155,12 @@ NSSCKFWC_Initialize
   /* remember the locking args for those times we need to get a lock in code
    * outside the framework.
    */
-  nssSetLockArgs(pInitArgs);
+  error = nssSetLockArgs(pInitArgs, &locking_state);
+  if (CKR_OK != error) {
+      goto loser;
+  }
 
-  *pFwInstance = nssCKFWInstance_Create(pInitArgs, mdInstance, &error);
+  *pFwInstance = nssCKFWInstance_Create(pInitArgs, locking_state, mdInstance, &error);
   if( (NSSCKFWInstance *)NULL == *pFwInstance ) {
     goto loser;
   }