487884 - object leak in libpkix library upon error. r=nelson

3 files changed, 31 insertions, 12 deletions

diff --git a/security/nss/lib/certhigh/certvfypkix.c b/security/nss/lib/certhigh/certvfypkix.c
index 7a3747d90..d4dac5b79 100644
--- a/security/nss/lib/certhigh/certvfypkix.c
+++ b/security/nss/lib/certhigh/certvfypkix.c
@@ -1223,7 +1223,15 @@ cert_VerifyCertChainPkix(
     int  memLeakLoopCount = 0;
     int  objCountTable[PKIX_NUMTYPES]; 
     int  fnInvLocalCount = 0;
+    PKIX_Boolean savedUsePkixEngFlag = usePKIXValidationEngine;
 
+    if (usePKIXValidationEngine) {
+        /* current memory leak testing implementation does not allow
+         * to run simultaneous tests one the same or a different threads.
+         * Setting the variable to false, to make additional chain
+         * validations be handled by old nss. */
+        usePKIXValidationEngine = PR_FALSE;
+    }
     testStartFnStackPosition = 2;
     fnStackNameArr[0] = "cert_VerifyCertChainPkix";
     fnStackInvCountArr[0] = 0;
@@ -1337,6 +1345,7 @@ cleanup:
 
     runningLeakTest = PKIX_FALSE; 
     PR_AtomicDecrement(&parallelFnInvocationCount);
+    usePKIXValidationEngine = savedUsePkixEngFlag;
 #endif /* PKIX_OBJECT_LEAK_TEST */
 
     return rv;
@@ -2045,6 +2054,15 @@ SECStatus CERT_PKIXVerifyCert(
     int  memLeakLoopCount = 0;
     int  objCountTable[PKIX_NUMTYPES];
     int  fnInvLocalCount = 0;
+    PKIX_Boolean savedUsePkixEngFlag = usePKIXValidationEngine;
+
+    if (usePKIXValidationEngine) {
+        /* current memory leak testing implementation does not allow
+         * to run simultaneous tests one the same or a different threads.
+         * Setting the variable to false, to make additional chain
+         * validations be handled by old nss. */
+        usePKIXValidationEngine = PR_FALSE;
+    }
     testStartFnStackPosition = 1;
     fnStackNameArr[0] = "CERT_PKIXVerifyCert";
     fnStackInvCountArr[0] = 0;
@@ -2264,6 +2282,7 @@ cleanup:
 
     runningLeakTest = PKIX_FALSE; 
     PR_AtomicDecrement(&parallelFnInvocationCount);
+    usePKIXValidationEngine = savedUsePkixEngFlag;
 #endif /* PKIX_OBJECT_LEAK_TEST */
 
     return r;
diff --git a/security/nss/lib/libpkix/pkix/checker/pkix_ocspchecker.c b/security/nss/lib/libpkix/pkix/checker/pkix_ocspchecker.c
index 78e3b3293..2f32e86bb 100644
--- a/security/nss/lib/libpkix/pkix/checker/pkix_ocspchecker.c
+++ b/security/nss/lib/libpkix/pkix/checker/pkix_ocspchecker.c
@@ -251,7 +251,7 @@ pkix_OcspChecker_CheckExternal(
         PKIX_RevocationStatus revStatus = PKIX_RevStatus_NoInfo;
         void *nbioContext = NULL;
 
-        PKIX_ENTER(OCSPCHECKER, "pkix_OcspChecker_Check");
+        PKIX_ENTER(OCSPCHECKER, "pkix_OcspChecker_CheckExternal");
 
         PKIX_CHECK(
             pkix_CheckType((PKIX_PL_Object*)checkerObject,
diff --git a/security/nss/lib/libpkix/pkix_pl_nss/pki/pkix_pl_ocspresponse.c b/security/nss/lib/libpkix/pkix_pl_nss/pki/pkix_pl_ocspresponse.c
index 0eba65a26..5ffa2dcfc 100644
--- a/security/nss/lib/libpkix/pkix_pl_nss/pki/pkix_pl_ocspresponse.c
+++ b/security/nss/lib/libpkix/pkix_pl_nss/pki/pkix_pl_ocspresponse.c
@@ -502,7 +502,9 @@ pkix_pl_OcspResponse_Create(
                         ocspResponse->request = request;
                         ocspResponse->httpClient = httpClient;
                         ocspResponse->serverSession = serverSession;
+                        serverSession = NULL;
                         ocspResponse->sessionRequest = sessionRequest;
+                        sessionRequest = NULL;
                         ocspResponse->verifyFcn = verifyFcn;
                         ocspResponse->handle = CERT_GetDefaultCertDB();
                         ocspResponse->encodedResponse = NULL;
@@ -560,26 +562,24 @@ pkix_pl_OcspResponse_Create(
                             responseData, responseDataLen);
         }
         *pResponse = ocspResponse;
+        ocspResponse = NULL;
 
 cleanup:
 
         if (path != NULL) {
             PORT_Free(path);
         }
-
         if (hostname != NULL) {
             PORT_Free(hostname);
         }
-
-        if (PKIX_ERROR_RECEIVED){
-            if (ocspResponse) {
-                PKIX_DECREF(ocspResponse);
-            } else {
-                if (serverSession) 
-                    hcv1->freeSessionFcn(serverSession);
-                if (sessionRequest)
-                    hcv1->freeFcn(sessionRequest);
-            }
+        if (ocspResponse) {
+            PKIX_DECREF(ocspResponse);
+        }
+        if (serverSession) {
+            hcv1->freeSessionFcn(serverSession);
+        }
+        if (sessionRequest) {
+            hcv1->freeFcn(sessionRequest);
         }
 
         PKIX_RETURN(OCSPRESPONSE);