Bug 398397: Patch. r=nelson,r=julien

Bug 391815: Patch. r=alexei,r=julien

Adding at once because requires much code merging.

2 files changed, 263 insertions, 121 deletions

diff --git a/security/nss/tests/iopr/ocsp_iopr.sh b/security/nss/tests/iopr/ocsp_iopr.sh
index c592d2a0c..c7d9b899a 100644
--- a/security/nss/tests/iopr/ocsp_iopr.sh
+++ b/security/nss/tests/iopr/ocsp_iopr.sh
@@ -96,16 +96,21 @@ ocsp_get_cert_status() {
         clntParam="-l $respUrl -t $defRespCert"
     fi
 
-    outFile=$dbDir/ocsptest.out.$$
-    echo "ocspclnt -d $dbDir -S $cert $clntParam"
-    ocspclnt -d $dbDir -S $cert $clntParam >$outFile 2>&1
-    ret=$?
-    echo "ocspclnt output:"
-    cat $outFile
-    [ -z "`grep succeeded $outFile`" ] && ret=1
+    if [ -z "${MEMLEAK_DBG}" ]; then
+        outFile=$dbDir/ocsptest.out.$$
+        echo "ocspclnt -d $dbDir -S $cert $clntParam"
+        ocspclnt -d $dbDir -S $cert $clntParam >$outFile 2>&1
+        ret=$?
+        echo "ocspclnt output:"
+        cat $outFile
+        [ -z "`grep succeeded $outFile`" ] && ret=1
     
-    rm -f $outFile
-    return $ret
+        rm -f $outFile
+        return $ret
+    fi
+
+    OCSP_ATTR="-d $dbDir -S $cert $clntParam"
+    ${RUN_COMMAND_DBG} ocspclnt ${OCSP_ATTR}
 }
 
 ########################################################################
@@ -129,8 +134,10 @@ ocsp_iopr() {
         return 0
     fi
     
-    html_head "OCSP testing with responder at $IOPR_HOSTADDR. <br>" \
-        "Test Type: $testDescription"
+    if [ -z "${MEMLEAK_DBG}" ]; then
+        html_head "OCSP testing with responder at $IOPR_HOSTADDR. <br>" \
+            "Test Type: $testDescription"
+    fi
 
     if [ -n "$testResponder" ]; then
         responderUrl="$testProto://$servName:$testPort"
@@ -138,25 +145,35 @@ ocsp_iopr() {
         responderUrl=""
     fi
 
-    for certName in $testValidCertNames; do
-        ocsp_get_cert_status $dbDir $certName "$responderUrl" "$testResponder"
-        html_msg $? 0 "Getting status of a valid cert ($certName)" \
-            "produced a returncode of $ret, expected is 0."
-    done
+    if [ -z "${MEMLEAK_DBG}" ]; then
+        for certName in $testValidCertNames; do
+            ocsp_get_cert_status $dbDir $certName "$responderUrl" \
+                "$testResponder"
+            html_msg $? 0 "Getting status of a valid cert ($certName)" \
+                "produced a returncode of $ret, expected is 0."
+        done
 
-    for certName in $testRevokedCertNames; do
-        ocsp_get_cert_status $dbDir $certName "$responderUrl" "$testResponder"
-        html_msg $? 1 "Getting status of a unvalid cert ($certName)" \
-            "produced a returncode of $ret, expected is 1." 
-    done
+        for certName in $testRevokedCertNames; do
+            ocsp_get_cert_status $dbDir $certName "$responderUrl" \
+                "$testResponder"
+            html_msg $? 1 "Getting status of a unvalid cert ($certName)" \
+                "produced a returncode of $ret, expected is 1." 
+        done
 
-    for certName in $testStatUnknownCertNames; do
-        ocsp_get_cert_status $dbDir $certName "$responderUrl" "$testResponder"
-        html_msg $? 1 "Getting status of a cert with unknown status " \
-                    "($certName) produced a returncode of $ret, expected is 1."
-    done
+        for certName in $testStatUnknownCertNames; do
+            ocsp_get_cert_status $dbDir $certName "$responderUrl" \
+                "$testResponder"
+            html_msg $? 1 "Getting status of a cert with unknown status " \
+                        "($certName) produced a returncode of $ret, expected is 1."
+        done
+    else
+        for certName in $testValidCertNames $testRevokedCertNames \
+            $testStatUnknownCertName; do
+            ocsp_get_cert_status $dbDir $certName "$responderUrl" \
+                "$testResponder" 
+        done
+    fi
 }
-
   
 #####################################################################
 # Initial point for running ocsp test againt multiple hosts involved in
@@ -175,6 +192,10 @@ ocsp_iopr_run() {
     fi
     cd ${CLIENTDIR}
 
+    if [ -n "${MEMLEAK_DBG}" ]; then
+        html_head "Memory leak checking - IOPR"
+    fi
+
     num=1
     IOPR_HOST_PARAM=`echo "${IOPR_HOSTADDR_LIST} " | cut -f $num -d' '`
     while [ "$IOPR_HOST_PARAM" ]; do
@@ -198,22 +219,44 @@ ocsp_iopr_run() {
         [ -z "`echo ${supportedTests_new} | grep -i ocsp`" ] && continue;
 
         # Testing directories defined by webserver.
+        if [ -n "${MEMLEAK_DBG}" ]; then
+            LOGNAME=iopr-${IOPR_HOSTADDR}
+            LOGFILE=${LOGDIR}/${LOGNAME}.log
+        fi
+       
+        # Testing directories defined by webserver.
         echo "Testing ocsp interoperability.
                 Client: local(tstclnt).
                 Responder: remote($IOPR_HOSTADDR)"
-        
+
         for ocspTestType in ${supportedTests_new}; do
             if [ -z "`echo $ocspTestType | grep -i ocsp`" ]; then
                 continue
             fi
+            if [ -n "${MEMLEAK_DBG}" ]; then
+                ocsp_iopr $ocspTestType ${IOPR_HOSTADDR} \
+                    ${IOPR_OCSP_CLIENTDIR}_${IOPR_HOSTADDR} 2>> ${LOGFILE}
+            else
+                ocsp_iopr $ocspTestType ${IOPR_HOSTADDR} \
+                    ${IOPR_OCSP_CLIENTDIR}_${IOPR_HOSTADDR}
+            fi
+        done
 
-            ocsp_iopr $ocspTestType ${IOPR_HOSTADDR} \
-                ${IOPR_OCSP_CLIENTDIR}_${IOPR_HOSTADDR}
+        if [ -n "${MEMLEAK_DBG}" ]; then
+            log_parse
+            ret=$?
+            html_msg ${ret} 0 "${LOGNAME}" \
+                "produced a returncode of $ret, expected is 0"
+        fi
 
-        done
         echo "================================================"
         echo "Done testing ocsp interoperability with $IOPR_HOSTADDR"
     done
+
+    if [ -n "${MEMLEAK_DBG}" ]; then
+        html "</TABLE><BR>"
+    fi
+
     NO_ECC_CERTS=0
     return 0
 }
diff --git a/security/nss/tests/memleak/memleak.sh b/security/nss/tests/memleak/memleak.sh
index 8e3f74a87..f579787d6 100644
--- a/security/nss/tests/memleak/memleak.sh
+++ b/security/nss/tests/memleak/memleak.sh
@@ -79,6 +79,7 @@ memleak_init()
 	OLD_LIBRARY_PATH=${LD_LIBRARY_PATH}
 	TMP_LIBDIR="${HOSTDIR}/tmp$$"
 	TMP_STACKS="${HOSTDIR}/stacks$$"
+	TMP_SORTED="${HOSTDIR}/sorted$$"
 	TMP_COUNT="${HOSTDIR}/count$$"
 	
 	PORT=${PORT:-8443}
@@ -193,6 +194,7 @@ memleak_init()
 ########################################################################
 memleak_cleanup()
 {
+	unset MEMLEAK_DBG
 	unset NSS_DISABLE_UNLOAD
 	
 	. ${QADIR}/common/cleanup.sh
@@ -306,7 +308,7 @@ run ${ATTR}
 	echo "${DBX} ${COMMAND}"
 	echo "${SCRIPTNAME}: -------- DBX commands:"
 	echo "${DBX_CMD}"
-	echo "${DBX_CMD}" | ${DBX} ${COMMAND} 2>/dev/null | grep -v Reading
+	echo "${DBX_CMD}" | ${DBX} ${COMMAND} 2>/dev/null | grep -v Reading 1>&2
 }
 
 ######################### run_command_valgrind #########################
@@ -321,7 +323,7 @@ run_command_valgrind()
 	echo "${SCRIPTNAME}: -------- Running ${COMMAND} under Valgrind:"
 	echo "${VALGRIND} --tool=memcheck --leak-check=yes --show-reachable=yes --partial-loads-ok=yes --leak-resolution=high --num-callers=50 ${COMMAND} ${ATTR}"
 	echo "Running: ${COMMAND} ${ATTR}" 1>&2
-	${VALGRIND} --tool=memcheck --leak-check=yes --show-reachable=yes --partial-loads-ok=yes --leak-resolution=high --num-callers=50 ${COMMAND} ${ATTR}
+	${VALGRIND} --tool=memcheck --leak-check=yes --show-reachable=yes --partial-loads-ok=yes --leak-resolution=high --num-callers=50 ${COMMAND} ${ATTR} 1>&2
 	echo "==0==" 
 }
 
@@ -381,6 +383,64 @@ run_strsclnt_dbg()
 	tstclnt ${TSTCLNT_ATTR} < ${REQUEST_FILE}
 }
 
+stat_clear()
+{
+	stat_minbytes=9999999
+	stat_maxbytes=0
+	stat_minblocks=9999999
+	stat_maxblocks=0
+	stat_bytes=0
+	stat_blocks=0
+	stat_runs=0
+}
+
+stat_add()
+{
+	read hash lbytes bytes_str lblocks blocks_str in_str lruns runs_str \
+		minbytes minbytes_str maxbytes maxbytes_str minblocks \
+		minblocks_str maxblocks maxblocks_str rest < ${TMP_COUNT} 
+	rm ${TMP_COUNT}
+	
+	tbytes=`expr ${tbytes} + ${lbytes}`
+	tblocks=`expr ${tblocks} + ${lblocks}`
+	truns=`expr ${truns} + ${lruns}`
+	
+	if [ ${stat_minbytes} -gt ${minbytes} ]; then
+		stat_minbytes=${minbytes}
+	fi
+			
+	if [ ${stat_maxbytes} -lt ${maxbytes} ]; then
+		stat_maxbytes=${maxbytes}
+	fi
+			
+	if [ ${stat_minblocks} -gt ${minblocks} ]; then
+		stat_minblocks=${minblocks}
+	fi
+			
+	if [ ${stat_maxblocks} -lt ${maxblocks} ]; then
+		stat_maxblocks=${maxblocks}
+	fi
+			
+	stat_bytes=`expr ${stat_bytes} + ${lbytes}`
+	stat_blocks=`expr ${stat_blocks} + ${lblocks}`
+	stat_runs=`expr ${stat_runs} + ${lruns}`
+}
+
+stat_print()
+{
+	if [ ${stat_runs} -gt 0 ]; then
+		stat_avgbytes=`expr "${stat_bytes}" / "${stat_runs}"`
+		stat_avgblocks=`expr "${stat_blocks}" / "${stat_runs}"`
+		
+		echo
+		echo "$1 statistics:"
+		echo "Leaked bytes: ${stat_minbytes} min, ${stat_avgbytes} avg, ${stat_maxbytes} max"
+		echo "Leaked blocks: ${stat_minblocks} min, ${stat_avgblocks} avg, ${stat_maxblocks} max"
+		echo "Total runs: ${stat_runs}"
+		echo
+	fi
+}
+
 ########################## run_ciphers_server ##########################
 # local shell function to test server part of code (selfserv)
 ########################################################################
@@ -388,6 +448,8 @@ run_ciphers_server()
 {
 	html_head "Memory leak checking - server"
 	
+	stat_clear
+	
 	client_mode="NORMAL"	
 	for server_mode in ${MODE_LIST}; do
 		set_test_mode
@@ -399,19 +461,22 @@ run_ciphers_server()
 			LOGFILE=${LOGDIR}/${LOGNAME}.log
 			echo "Running ${LOGNAME}"
 			
-			run_selfserv_dbg 2>&1 | tee ${LOGFILE} &
+			run_selfserv_dbg 2>> ${LOGFILE} &
 			sleep 5
 			run_strsclnt
 			
 			sleep 20
 			clear_freebl
 			
-			log_parse >> ${FOUNDLEAKS}
+			log_parse
 			ret=$?
+			
 			html_msg ${ret} 0 "${LOGNAME}" "produced a returncode of $ret, expected is 0"
 		done
 	done
 	
+	stat_print "Selfserv"
+	
 	html "</TABLE><BR>"
 }
 
@@ -422,6 +487,8 @@ run_ciphers_client()
 {
 	html_head "Memory leak checking - client"
 	
+	stat_clear
+	
 	server_mode="NORMAL"
 	for client_mode in ${MODE_LIST}; do
 		set_test_mode
@@ -435,17 +502,19 @@ run_ciphers_client()
 			
 			run_selfserv &
 			sleep 5
-			run_strsclnt_dbg 2>&1 | tee ${LOGFILE}
+			run_strsclnt_dbg 2>> ${LOGFILE}
 			
 			sleep 20
 			clear_freebl
-
-			log_parse >> ${FOUNDLEAKS}
+			
+			log_parse
 			ret=$?
 			html_msg ${ret} 0 "${LOGNAME}" "produced a returncode of $ret, expected is 0"
 		done
 	done
 	
+	stat_print "Strsclnt"
+	
 	html "</TABLE><BR>"
 }
 
@@ -459,7 +528,13 @@ parse_logfile_dbx()
 		in_mel = 0
 		mel_line = 0
 		bytes = 0
+		lbytes = 0
+		minbytes = 9999999
+		maxbytes = 0
 		blocks = 0
+		lblocks = 0
+		minblocks = 9999999
+		maxblocks = 0
 		runs = 0
 		stack_string = ""
 		bin_name = ""
@@ -468,7 +543,7 @@ parse_logfile_dbx()
 	/Possible memory leak -- address in block \(aib\):/ ||
 	/Block in use \(biu\):/ {
 		in_mel = 1
-		stack_string=""
+		stack_string = ""
 		next
 	}
 	in_mel == 1 && /^$/ {
@@ -478,7 +553,7 @@ parse_logfile_dbx()
 		next
 	}
 	in_mel == 1 {
-		mel_line += 1;
+		mel_line += 1
 	}
 	/Found leaked block of size/ {
 		bytes += $6
@@ -501,8 +576,8 @@ parse_logfile_dbx()
 		next
 	}
 	mel_line > 2 {
-		gsub(/\(\)/,"")
-		new_line = $2;
+		gsub(/\(\)/, "")
+		new_line = $2
 		stack_string = "/" new_line stack_string
 		next
 	}
@@ -512,17 +587,23 @@ parse_logfile_dbx()
 	}
 	/execution completed/ {
 		runs += 1
+		lbytes += bytes
+		minbytes = (minbytes < bytes) ? minbytes : bytes
+		maxbytes = (maxbytes > bytes) ? maxbytes : bytes
+		bytes = 0
+		lblocks += blocks
+		minblocks = (minblocks < blocks) ? minblocks : blocks
+		maxblocks = (maxblocks > blocks) ? maxblocks : blocks
+		blocks = 0
 		next
 	}
 	END {
-		print "# " bytes " bytes " blocks " blocks in " runs " runs" > "/dev/stderr";
+		print "# " lbytes " bytes " lblocks " blocks in " runs " runs " \
+		minbytes " minbytes " maxbytes " maxbytes " minblocks " minblocks " \
+		maxblocks " maxblocks " > "/dev/stderr"
 	}' 2> ${TMP_COUNT}
 	
-	read hash lbytes bytes_str lblocks blocks_str in_str lruns rest < ${TMP_COUNT}
-	tbytes=`expr "${tbytes}" + "${lbytes}"`
-	tblocks=`expr "${tblocks}" + "${lblocks}"`
-	truns=`expr "${truns}" + "${lruns}"`
-	rm ${TMP_COUNT}
+	stat_add
 }
 
 ######################## parse_logfile_valgrind ########################
@@ -531,68 +612,80 @@ parse_logfile_dbx()
 parse_logfile_valgrind()
 {
 	${AWK} '
-	 BEGIN {
-		 in_mel=0;
-		 in_sum=0;
-		 bytes=0;
-		 blocks=0;
-		 runs=0;
-		 stack_string="";
-		 bin_name=""; }
-	 !/==[0-9]*==/ { 
+	BEGIN {
+		in_mel = 0
+		in_sum = 0
+		bytes = 0
+		lbytes = 0
+		minbytes = 9999999
+		maxbytes = 0
+		blocks = 0
+		lblocks = 0
+		minblocks = 9999999
+		maxblocks = 0
+		runs = 0
+		stack_string = ""
+		bin_name = "" 
+	}
+	!/==[0-9]*==/ { 
 		if ( $1 == "Running:" ) 
 			bin_name = $2 
-		next;
-	 }
-	 /blocks are/ {
-		in_mel = 1;
-		stack_string="";
-		next;
-	 }
-	 /LEAK SUMMARY/ {
-		in_sum=1;
-		runs += 1;
-		next;
-	 }
-	 /^==[0-9]*== *$/ { 
-	    	if (in_mel)
-			print bin_name stack_string;
-		in_sum = 0;
-		in_mel = 0;
-		next;
-	 }
-	 in_mel == 1 {	
-		new_line = $4;
+		next
+	}
+	/blocks are/ {
+		in_mel = 1
+		stack_string = ""
+		next
+	}
+	/LEAK SUMMARY/ {
+		in_sum = 1
+		next
+	}
+	/^==[0-9]*== *$/ { 
+		if (in_mel)
+			print bin_name stack_string
+		if (in_sum) {
+			runs += 1
+			lbytes += bytes
+			minbytes = (minbytes < bytes) ? minbytes : bytes
+			maxbytes = (maxbytes > bytes) ? maxbytes : bytes
+			bytes = 0
+			lblocks += blocks
+			minblocks = (minblocks < blocks) ? minblocks : blocks
+			maxblocks = (maxblocks > blocks) ? maxblocks : blocks
+			blocks = 0
+		}
+		in_sum = 0
+		in_mel = 0
+		next
+	}
+	in_mel == 1 {	
+		new_line = $4
 		if ( new_line == "(within")
-			new_line = "*";
-		stack_string = "/" new_line stack_string;
-	 }
-	 in_sum == 1 {
-		for (i=2; i <= NF; i++) {
+			new_line = "*"
+		stack_string = "/" new_line stack_string
+	}
+	in_sum == 1 {
+		for (i = 2; i <= NF; i++) {
 			if ($i == "bytes") {
-				str = $(i-1);
-				gsub(",","",str);
-				bytes += str;
+				str = $(i - 1)
+				gsub(",", "", str)
+				bytes += str
 			}
 			if ($i == "blocks.") {
-				str = $(i-1);
-				gsub(",","",str);
-				blocks += str;
+				str = $(i - 1)
+				gsub(",", "", str)
+				blocks += str
 			}
 		}
-	 }
-	 END {
-		print "# " bytes " bytes " blocks " blocks in " runs " runs" > "/dev/stderr";
-	 }' 2> ${TMP_COUNT}
-
-	# sigh it would be nice to just pipe stderr and let stdout go by I've never been 
-	# able to convince any shell to do that correctly, so we are reduced to using a temp
-	# file
-	read hash lbytes bytes_str lblocks blocks_str in_str lruns rest < ${TMP_COUNT}
-	tbytes=`expr "${tbytes}" + "${lbytes}"`
-	tblocks=`expr "${tblocks}" + "${lblocks}"`
-	truns=`expr "${truns}" + "${lruns}"`
-	rm ${TMP_COUNT}
+	}
+	END {
+		print "# " lbytes " bytes " lblocks " blocks in " runs " runs " \
+		minbytes " minbytes " maxbytes " maxbytes " minblocks " minblocks " \
+		maxblocks " maxblocks " > "/dev/stderr"
+	}' 2> ${TMP_COUNT}
+	
+	stat_add
 }
 
 ############################# check_ignored ############################
@@ -600,7 +693,7 @@ parse_logfile_valgrind()
 ########################################################################
 check_ignored()
 {
-    ${AWK} -F/ '
+	${AWK} -F/ '
 	BEGIN {
 		ignore = "'${IGNORED_STACKS}'"
 		# read in the ignore file
@@ -616,25 +709,25 @@ check_ignored()
 				continue
 			} else {
 				bugnum_array[count] = BUGNUM
- 				# Create a regular expression for the ignored stack:
- 				# replace * with % so we can later replace them with regular expressions
- 				# without messing up everything (the regular expressions contain *)
- 				gsub("\\*","%",line)
- 				# replace %% with .*
- 				gsub("%%",".*",line)
- 				# replace % with [^/]*
- 				gsub("%","[^/]*",line)
- 				# add ^ at the beginning
- 				# add $ at the end
- 				line_array[count] = "^" line "$"
+				# Create a regular expression for the ignored stack:
+				# replace * with % so we can later replace them with regular expressions
+				# without messing up everything (the regular expressions contain *)
+				gsub("\\*", "%", line)
+				# replace %% with .*
+				gsub("%%", ".*", line)
+				# replace % with [^/]*
+				gsub("%", "[^/]*", line)
+				# add ^ at the beginning
+				# add $ at the end
+				line_array[count] = "^" line "$"
 				count++
 			}
 		}
 	}
 	{
-		match_found=0
+		match_found = 0
 		# Look for matching ignored stack
-	 	for (i=0 ; i < count; i++) {
+		for (i = 0; i < count; i++) {
 			if ($0 ~ line_array[i]) {
 				# found a match
 				match_found = 1
@@ -666,13 +759,15 @@ check_ignored()
 ########################################################################
 log_parse()
 {
-	echo "${SCRIPTNAME}: Processing log ${LOGNAME}:"
 	${PARSE_LOGFILE} < ${LOGFILE} > ${TMP_STACKS}
-	cat ${TMP_STACKS} | sort -u | check_ignored
+	echo "${SCRIPTNAME}: Processing log ${LOGNAME}:" > ${TMP_SORTED}
+	cat ${TMP_STACKS} | sort -u | check_ignored >> ${TMP_SORTED}
 	ret=$?
-	rm ${TMP_STACKS}
-	echo ""
-
+	echo >> ${TMP_SORTED}
+	
+	cat ${TMP_SORTED} | tee -a ${FOUNDLEAKS}
+	rm ${TMP_STACKS} ${TMP_SORTED}
+	
 	return ${ret}
 }
 
@@ -693,9 +788,13 @@ cnt_total()
 ########################################################################
 run_ocsp()
 {
+	stat_clear
+	
 	cd ${QADIR}/iopr
 	. ./ocsp_iopr.sh
 	ocsp_iopr_run
+	
+	stat_print "Ocspclnt"
 }
 
 ################################# main #################################