bug 145322, reduce the number of PKCS#11 sessions used in SSL connections, implement new function PK11_SaveContextAlloc

r=relyea

4 files changed, 119 insertions, 20 deletions

diff --git a/security/nss/lib/nss/nss.def b/security/nss/lib/nss/nss.def
index 4931b6964..5ccad1b9d 100644
--- a/security/nss/lib/nss/nss.def
+++ b/security/nss/lib/nss/nss.def
@@ -683,6 +683,7 @@ SECMOD_CanDeleteInternalModule;
 ;+NSS_3.6 { 	# cert creation APIs used by certutil
 ;+    global:
 PK11_GetPBEIV;
+PK11_SaveContextAlloc;
 ;+    local:
 ;+       *;
 ;+};
diff --git a/security/nss/lib/pk11wrap/pk11func.h b/security/nss/lib/pk11wrap/pk11func.h
index 61f1fd705..6e1ea5588 100644
--- a/security/nss/lib/pk11wrap/pk11func.h
+++ b/security/nss/lib/pk11wrap/pk11func.h
@@ -481,6 +481,21 @@ SECStatus PK11_DigestFinal(PK11Context *context, unsigned char *data,
 PRBool PK11_HashOK(SECOidTag hashAlg);
 SECStatus PK11_SaveContext(PK11Context *cx,unsigned char *save,
 						int *len, int saveLength);
+
+/* Save the context's state, with possible allocation.
+ * The caller may supply an already allocated buffer in preAllocBuf,
+ * with length pabLen.  If the buffer is large enough for the context's
+ * state, it will receive the state.
+ * If the buffer is not large enough (or NULL), then a new buffer will
+ * be allocated with PORT_Alloc.
+ * In either case, the state will be returned as a buffer, and the length
+ * of the state will be given in *stateLen.
+ */
+unsigned char *
+PK11_SaveContextAlloc(PK11Context *cx,
+                      unsigned char *preAllocBuf, unsigned int pabLen,
+                      unsigned int *stateLen);
+
 SECStatus PK11_RestoreContext(PK11Context *cx,unsigned char *save,int len);
 SECStatus PK11_GenerateFortezzaIV(PK11SymKey *symKey,unsigned char *iv,int len);
 SECStatus PK11_ReadSlotCerts(PK11SlotInfo *slot);
diff --git a/security/nss/lib/pk11wrap/pk11skey.c b/security/nss/lib/pk11wrap/pk11skey.c
index 236fa1e57..e6b6117bd 100644
--- a/security/nss/lib/pk11wrap/pk11skey.c
+++ b/security/nss/lib/pk11wrap/pk11skey.c
@@ -298,17 +298,6 @@ PK11_SymKeyFromHandle(PK11SlotInfo *slot, PK11SymKey *parent, PK11Origin origin,
     symKey->origin = origin;
     symKey->owner = owner;
 
-    /* adopt the parent's session */
-    /* This is only used by SSL. What we really want here is a session
-     * structure with a ref count so  the session goes away only after all the
-     * keys do. */
-    if (owner && parent) {
-	pk11_CloseSession(symKey->slot, symKey->session,symKey->sessionOwner);
-	symKey->sessionOwner = parent->sessionOwner;
-	symKey->session = parent->session;
-	parent->sessionOwner = PR_FALSE;
-    }
-
     return symKey;
 }
 
@@ -3243,7 +3232,16 @@ PK11_ExitContextMonitor(PK11Context *cx) {
 void
 PK11_DestroyContext(PK11Context *context, PRBool freeit)
 {
-    pk11_CloseSession(context->slot,context->session,context->ownSession);
+    if (context->ownSession && context->key && /* context owns session & key */
+        context->key->session == context->session && /* sharing session */
+        !context->key->sessionOwner)              /* sanity check */
+    {
+	/* session still valid, let the key free it as necessary */
+        (void)PK11_Finalize(context); /* end any ongoing activity */
+	context->key->sessionOwner = PR_TRUE;
+    } else {
+	pk11_CloseSession(context->slot,context->session,context->ownSession);
+    }
     /* initialize the critical fields of the context */
     if (context->savedData != NULL ) PORT_Free(context->savedData);
     if (context->key) PK11_FreeSymKey(context->key);
@@ -3427,7 +3425,14 @@ static PK11Context *pk11_CreateNewContextInSlot(CK_MECHANISM_TYPE type,
     context->operation = operation;
     context->key = symKey ? PK11_ReferenceSymKey(symKey) : NULL;
     context->slot = PK11_ReferenceSlot(slot);
-    context->session = pk11_GetNewSession(slot,&context->ownSession);
+    if (symKey && symKey->sessionOwner) {
+	/* The symkey owns a session.  Adopt that session. */
+	context->session = symKey->session;
+	context->ownSession = symKey->sessionOwner;
+	symKey->sessionOwner = PR_FALSE;
+    } else {
+	context->session = pk11_GetNewSession(slot, &context->ownSession);
+    }
     context->cx = symKey ? symKey->cx : NULL;
     /* get our session */
     context->savedData = NULL;
@@ -3647,6 +3652,55 @@ PK11_SaveContext(PK11Context *cx,unsigned char *save,int *len, int saveLength)
     return (data != NULL) ? SECSuccess : SECFailure;
 }
 
+/* same as above, but may allocate the return buffer. */
+unsigned char *
+PK11_SaveContextAlloc(PK11Context *cx,
+                      unsigned char *preAllocBuf, unsigned int pabLen,
+                      unsigned int *stateLen)
+{
+    unsigned char *stateBuf = NULL;
+    unsigned long length = (unsigned long)pabLen;
+    PRBool callerHasBuf = (preAllocBuf != NULL);
+
+    if (cx->ownSession) {
+        PK11_EnterContextMonitor(cx);
+	stateBuf = (unsigned char *)pk11_saveContextHelper(cx, preAllocBuf,
+	                                                   &length,
+	                                                   callerHasBuf, 
+	                                                   PR_FALSE);
+	if (stateBuf == NULL && callerHasBuf) {
+	    /* pk11_saveContextHelper will attempt to free the supplied
+	     * buffer if staticBuffer == PR_FALSE.  However, it won't
+	     * allocate a new buffer unless staticBuffer == PR_FALSE.  We
+	     * want to allocate a new buffer if this one isn't big enough,
+	     * but we don't want the caller buffer to be freed.  So, 
+	     * we have to try again.
+	     */
+	    length = 0;
+	    stateBuf = (unsigned char *)pk11_saveContextHelper(cx, NULL,
+	                                                       &length, 
+	                                                       PR_FALSE, 
+	                                                       PR_FALSE);
+	}
+        PK11_ExitContextMonitor(cx);
+	*stateLen = (stateBuf != NULL) ? length : 0;
+    } else {
+	if (pabLen < cx->savedLength) {
+	    stateBuf = (unsigned char *)PORT_Alloc(cx->savedLength);
+	    if (!stateBuf) {
+		return (unsigned char *)NULL;
+	    }
+	} else {
+	    stateBuf = preAllocBuf;
+	}
+	if (cx->savedData) {
+	    PORT_Memcpy(stateBuf, cx->savedData, cx->savedLength);
+	}
+	*stateLen = cx->savedLength;
+    }
+    return stateBuf;
+}
+
 /*
  * restore the context state into a new running context. Also required for
  * FORTEZZA .
diff --git a/security/nss/lib/ssl/ssl3con.c b/security/nss/lib/ssl/ssl3con.c
index 1e2da2e94..e220bdc0f 100644
--- a/security/nss/lib/ssl/ssl3con.c
+++ b/security/nss/lib/ssl/ssl3con.c
@@ -2418,22 +2418,31 @@ ssl3_ComputeHandshakeHashes(sslSocket *     ss,
     SSL3Opaque    md5_inner[MAX_MAC_LENGTH];
     SSL3Opaque    sha_inner[MAX_MAC_LENGTH];
     unsigned char s[4];
+    unsigned char md5StackBuf[256];
+    unsigned char shaStackBuf[512];
+    unsigned char *md5StateBuf = NULL;
+    unsigned char *shaStateBuf = NULL;
+    unsigned int  md5StateLen, shaStateLen;
 
     PORT_Assert( ssl_HaveSSL3HandshakeLock(ss) );
 
     isTLS = (PRBool)(spec->version > SSL_LIBRARY_VERSION_3_0);
 
-    md5 = PK11_CloneContext(ssl3->hs.md5);
-    if (md5 == NULL) {
+    md5StateBuf = PK11_SaveContextAlloc(ssl3->hs.md5, md5StackBuf,
+                                        sizeof md5StackBuf, &md5StateLen);
+    if (md5StateBuf == NULL) {
 	ssl_MapLowLevelError(SSL_ERROR_MD5_DIGEST_FAILURE);
-    	return SECFailure;
+	goto loser;
     }
+    md5 = ssl3->hs.md5;
 
-    sha = PK11_CloneContext(ssl3->hs.sha);
-    if (sha == NULL) {
+    shaStateBuf = PK11_SaveContextAlloc(ssl3->hs.sha, shaStackBuf,
+                                        sizeof shaStackBuf, &shaStateLen);
+    if (shaStateBuf == NULL) {
 	ssl_MapLowLevelError(SSL_ERROR_SHA_DIGEST_FAILURE);
 	goto loser;
     }
+    sha = ssl3->hs.sha;
 
     if (!isTLS) {
 	/* compute hashes for SSL3. */
@@ -2522,8 +2531,28 @@ ssl3_ComputeHandshakeHashes(sslSocket *     ss,
     rv = SECSuccess;
 
 loser:
-    if (md5) PK11_DestroyContext(md5, PR_TRUE);
-    if (sha) PK11_DestroyContext(sha, PR_TRUE);
+    if (md5StateBuf) {
+	if (PK11_RestoreContext(ssl3->hs.md5, md5StateBuf, md5StateLen)
+	     != SECSuccess) 
+	{
+	    ssl_MapLowLevelError(SSL_ERROR_MD5_DIGEST_FAILURE);
+	    rv = SECFailure;
+	}
+	if (md5StateBuf != md5StackBuf) {
+	    PORT_ZFree(md5StateBuf, md5StateLen);
+	}
+    }
+    if (shaStateBuf) {
+	if (PK11_RestoreContext(ssl3->hs.sha, shaStateBuf, shaStateLen)
+	     != SECSuccess) 
+	{
+	    ssl_MapLowLevelError(SSL_ERROR_SHA_DIGEST_FAILURE);
+	    rv = SECFailure;
+	}
+	if (shaStateBuf != shaStackBuf) {
+	    PORT_ZFree(shaStateBuf, shaStateLen);
+	}
+    }
 
     return rv;
 }