run-tests improvements:

  Add XLEAK section, this is made necessary by the support
  for valgrind tools other than memcheck.
  XLEAK is to a leaking test what XFAIL is to a failing test.
  Tidy up summary a little to exclude Expected Fail and Expected Leak
  when there are no expected fail/leaks

1 files changed, 48 insertions, 14 deletions

diff --git a/run-tests.php b/run-tests.php
index df95991486..7abc4aedee 100755
--- a/run-tests.php
+++ b/run-tests.php
@@ -300,6 +300,7 @@ NO_PROC_OPEN_ERROR;
 		'WARNED' => array(),
 		'LEAKED' => array(),
 		'XFAILED' => array(),
+		'XLEAKED' => array(),
 		'SLOW' => array()
 	);
 
@@ -566,7 +567,7 @@ Options:
                 with value 'bar').
 
     -g          Comma separated list of groups to show during test run
-                (possible values: PASS, FAIL, XFAIL, SKIP, BORK, WARN, LEAK, REDIRECT).
+                (possible values: PASS, FAIL, XFAIL, XLEAK, SKIP, BORK, WARN, LEAK, REDIRECT).
 
     -m          Test for memory leaks with Valgrind (equivalent to -M memcheck).
 
@@ -1313,7 +1314,7 @@ function run_all_tests($test_files, $env, $redir_tested = null)
 		$test_idx++;
 
 		if ($workerID) {
-			$PHP_FAILED_TESTS = ['BORKED' => [], 'FAILED' => [], 'WARNED' => [], 'LEAKED' => [], 'XFAILED' => [], 'SLOW' => []];
+			$PHP_FAILED_TESTS = ['BORKED' => [], 'FAILED' => [], 'WARNED' => [], 'LEAKED' => [], 'XFAILED' => [], 'XLEAKED' => [], 'SLOW' => []];
 			ob_start();
 		}
 
@@ -1336,7 +1337,7 @@ function run_all_tests($test_files, $env, $redir_tested = null)
 			}
 
 			$test_results[$index] = $result;
-			if ($failed_tests_file && ($result == 'XFAILED' || $result == 'FAILED' || $result == 'WARNED' || $result == 'LEAKED')) {
+			if ($failed_tests_file && ($result == 'XFAILED' || $result == 'XLEAKED' || $result == 'FAILED' || $result == 'WARNED' || $result == 'LEAKED')) {
 				fwrite($failed_tests_file, "$index\n");
 			}
 			if ($result_tests_file) {
@@ -1594,7 +1595,7 @@ escape:
 							if (!is_array($name) && $result != 'REDIR') {
 								$test_results[$index] = $result;
 
-								if ($failed_tests_file && ($result == 'XFAILED' || $result == 'FAILED' || $result == 'WARNED' || $result == 'LEAKED')) {
+								if ($failed_tests_file && ($result == 'XFAILED' || $result == 'XLEAKED' || $result == 'FAILED' || $result == 'WARNED' || $result == 'LEAKED')) {
 									fwrite($failed_tests_file, "$index\n");
 								}
 								if ($result_tests_file) {
@@ -1822,7 +1823,7 @@ TEST $file
 				'FILE', 'FILEEOF', 'FILE_EXTERNAL', 'REDIRECTTEST',
 				'CAPTURE_STDIO', 'STDIN', 'CGI', 'PHPDBG',
 				'INI', 'ENV', 'EXTENSIONS',
-				'SKIPIF', 'XFAIL', 'CLEAN',
+				'SKIPIF', 'XFAIL', 'XLEAK', 'CLEAN',
 				'CREDITS', 'DESCRIPTION', 'CONFLICTS',
 			))) {
 				$bork_info = 'Unknown section "' . $section . '"';
@@ -2625,7 +2626,10 @@ COMMAND $cmd
 				if (isset($section_text['XFAIL'])) {
 					$warn = true;
 					$info = " (warn: XFAIL section but test passes)";
-				} else {
+				} else if (isset($section_text['XLEAK'])) {
+					$warn = true;
+					$info = " (warn: XLEAK section but test passes)";
+                } else {
 					show_result("PASS", $tested, $tested_file, '', $temp_filenames);
 					junit_mark_test_as('PASS', $shortname, $tested);
 					return 'PASSED';
@@ -2651,7 +2655,10 @@ COMMAND $cmd
 				if (isset($section_text['XFAIL'])) {
 					$warn = true;
 					$info = " (warn: XFAIL section but test passes)";
-				} else {
+				} if (isset($section_text['XLEAK'])) {
+					$warn = true;
+					$info = " (warn: XLEAK section but test passes)";      
+                } else {
 					show_result("PASS", $tested, $tested_file, '', $temp_filenames);
 					junit_mark_test_as('PASS', $shortname, $tested);
 					return 'PASSED';
@@ -2674,7 +2681,8 @@ COMMAND $cmd
 	}
 
 	if ($leaked) {
-		$restype[] = 'LEAK';
+        $restype[] = isset($section_text['XLEAK']) ? 
+                        'XLEAK' : 'LEAK';
 	}
 
 	if ($warn) {
@@ -2685,7 +2693,10 @@ COMMAND $cmd
 		if (isset($section_text['XFAIL'])) {
 			$restype[] = 'XFAIL';
 			$info = '  XFAIL REASON: ' . rtrim($section_text['XFAIL']);
-		} else {
+		} else if ($section_text['XLEAK']) {
+            $restype[] = 'XLEAK';
+			$info = '  XLEAK REASON: ' . rtrim($section_text['XLEAK']);
+        } else {
 			$restype[] = 'FAIL';
 		}
 	}
@@ -2962,7 +2973,8 @@ function compute_summary()
 		'FAILED' => 0,
 		'BORKED' => 0,
 		'LEAKED' => 0,
-		'XFAILED' => 0
+		'XFAILED' => 0,
+		'XLEAKED' => 0
 	);
 
 	foreach ($test_results as $v) {
@@ -2987,10 +2999,11 @@ function get_summary($show_ext_summary, $show_html)
 		$x_warned = (100.0 * $sum_results['WARNED']) / $x_total;
 		$x_failed = (100.0 * $sum_results['FAILED']) / $x_total;
 		$x_xfailed = (100.0 * $sum_results['XFAILED']) / $x_total;
+		$x_xleaked = (100.0 * $sum_results['XLEAKED']) / $x_total;
 		$x_leaked = (100.0 * $sum_results['LEAKED']) / $x_total;
 		$x_passed = (100.0 * $sum_results['PASSED']) / $x_total;
 	} else {
-		$x_warned = $x_failed = $x_passed = $x_leaked = $x_xfailed = 0;
+		$x_warned = $x_failed = $x_passed = $x_leaked = $x_xfailed = $x_xleaked = 0;
 	}
 
 	$summary = '';
@@ -3021,12 +3034,20 @@ Tests borked    : ' . sprintf('%4d (%5.1f%%)', $sum_results['BORKED'], $percent_
 	$summary .= '
 Tests skipped   : ' . sprintf('%4d (%5.1f%%)', $sum_results['SKIPPED'], $percent_results['SKIPPED']) . ' --------
 Tests warned    : ' . sprintf('%4d (%5.1f%%)', $sum_results['WARNED'], $percent_results['WARNED']) . ' ' . sprintf('(%5.1f%%)', $x_warned) . '
-Tests failed    : ' . sprintf('%4d (%5.1f%%)', $sum_results['FAILED'], $percent_results['FAILED']) . ' ' . sprintf('(%5.1f%%)', $x_failed) . '
+Tests failed    : ' . sprintf('%4d (%5.1f%%)', $sum_results['FAILED'], $percent_results['FAILED']) . ' ' . sprintf('(%5.1f%%)', $x_failed);
+
+    if ($sum_results['XFAILED']) {
+        $summary .= '
 Expected fail   : ' . sprintf('%4d (%5.1f%%)', $sum_results['XFAILED'], $percent_results['XFAILED']) . ' ' . sprintf('(%5.1f%%)', $x_xfailed);
+    }
 
 	if ($valgrind) {
 		$summary .= '
 Tests leaked    : ' . sprintf('%4d (%5.1f%%)', $sum_results['LEAKED'], $percent_results['LEAKED']) . ' ' . sprintf('(%5.1f%%)', $x_leaked);
+        if ($sum_results['XLEAKED']) {
+            $summary .= '
+Expected leak   : ' . sprintf('%4d (%5.1f%%)', $sum_results['XLEAKED'], $percent_results['XLEAKED']) . ' ' . sprintf('(%5.1f%%)', $x_xleaked);
+        }
 	}
 
 	$summary .= '
@@ -3115,6 +3136,19 @@ LEAKED TEST SUMMARY
 		$failed_test_summary .= "=====================================================================\n";
 	}
 
+	if (count($PHP_FAILED_TESTS['XLEAKED'])) {
+		$failed_test_summary .= '
+=====================================================================
+EXPECTED LEAK TEST SUMMARY
+---------------------------------------------------------------------
+';
+		foreach ($PHP_FAILED_TESTS['XLEAKED'] as $failed_test_data) {
+			$failed_test_summary .= $failed_test_data['test_name'] . $failed_test_data['info'] . "\n";
+		}
+
+		$failed_test_summary .= "=====================================================================\n";
+	}
+
 	if ($failed_test_summary && !getenv('NO_PHPTEST_SUMMARY')) {
 		$summary .= $failed_test_summary;
 	}
@@ -3390,13 +3424,13 @@ function junit_mark_test_as($type, $file_name, $test_name, $time = null, $messag
 
 	if (is_array($type)) {
 		$output_type = $type[0] . 'ED';
-		$temp = array_intersect(array('XFAIL', 'FAIL', 'WARN'), $type);
+		$temp = array_intersect(array('XFAIL', 'XLEAK', 'FAIL', 'WARN'), $type);
 		$type = reset($temp);
 	} else {
 		$output_type = $type . 'ED';
 	}
 
-	if ('PASS' == $type || 'XFAIL' == $type) {
+	if ('PASS' == $type || 'XFAIL' == $type || 'XLEAK' == $type) {
 		junit_suite_record($suite, 'test_pass');
 	} elseif ('BORK' == $type) {
 		junit_suite_record($suite, 'test_error');