diff options
Diffstat (limited to 'run-tests.php')
-rwxr-xr-x | run-tests.php | 232 |
1 files changed, 157 insertions, 75 deletions
diff --git a/run-tests.php b/run-tests.php index 0f38e488ff..85e59e54bc 100755 --- a/run-tests.php +++ b/run-tests.php @@ -94,6 +94,28 @@ if (ob_get_level()) echo "Not all buffers were deleted.\n"; error_reporting(E_ALL); $environment = isset($_ENV) ? $_ENV : array(); +// Note: php.ini-development sets variables_order="GPCS" not "EGPCS", in which case $_ENV is NOT populated. +// detect and handle this case, or die or warn +if (empty($environment)) { + // not documented, but returns array of all environment variables + $environment = getenv(); +} +if (empty($environment['TEMP'])) { + $environment['TEMP'] = sys_get_temp_dir(); + + if (empty($environment['TEMP'])) { + // for example, OpCache on Windows will fail in this case because child processes (for tests) will not get + // a TEMP variable, so GetTempPath() will fallback to c:\windows, while GetTempPath() will return %TEMP% for parent + // (likely a different path). The parent will initialize the OpCache in that path, and child will fail to reattach to + // the OpCache because it will be using the wrong path. + die("TEMP environment is NOT set"); + } else if (count($environment)==1) { + // not having other environment variables, only having TEMP, is probably ok, but strange and may make a + // difference in the test pass rate, so warn the user. + echo "WARNING: Only 1 environment variable will be available to tests(TEMP environment variable)".PHP_EOL; + } +} +// if ((substr(PHP_OS, 0, 3) == "WIN") && empty($environment["SystemRoot"])) { $environment["SystemRoot"] = getenv("SystemRoot"); } @@ -217,7 +239,7 @@ $ini_overwrites = array( 'display_startup_errors=1', 'log_errors=0', 'html_errors=0', - 'track_errors=1', + 'track_errors=0', 'report_memleaks=1', 'report_zend_debug=0', 'docref_root=', @@ -232,11 +254,12 @@ $ini_overwrites = array( 'log_errors_max_len=0', 'opcache.fast_shutdown=0', 'opcache.file_update_protection=0', + 'zend.assertions=1', ); $no_file_cache = '-d opcache.file_cache= -d opcache.file_cache_only=0'; -function write_information($show_html) +function write_information() { global $cwd, $php, $php_cgi, $phpdbg, $php_info, $user_tests, $ini_overwrites, $pass_options, $exts_to_test, $leak_check, $valgrind_header, $no_file_cache; @@ -279,7 +302,7 @@ More .INIs : " , (function_exists(\'php_ini_scanned_files\') ? str_replace("\n" @unlink($info_file); // load list of enabled extensions - save_text($info_file, '<?php echo str_replace("Zend OPcache", "opcache", join(",", get_loaded_extensions())); ?>'); + save_text($info_file, '<?php echo str_replace("Zend OPcache", "opcache", implode(",", get_loaded_extensions())); ?>'); $exts_to_test = explode(',',`$php $pass_options $info_params $no_file_cache "$info_file"`); // check for extensions that need special handling and regenerate $info_params_ex = array( @@ -324,7 +347,7 @@ define('TRAVIS_CI' , (bool) getenv('TRAVIS')); function save_or_mail_results() { global $sum_results, $just_save_results, $failed_test_summary, - $PHP_FAILED_TESTS, $CUR_DIR, $php, $output_file, $compression; + $PHP_FAILED_TESTS, $CUR_DIR, $php, $output_file; /* We got failed Tests, offer the user to send an e-mail to QA team, unless NO_INTERACTION is set */ if (!getenv('NO_INTERACTION') && !TRAVIS_CI) { @@ -435,7 +458,7 @@ function save_or_mail_results() $failed_tests_data .= $sep . "PHPINFO" . $sep; $failed_tests_data .= shell_exec($php . ' -ddisplay_errors=stderr -dhtml_errors=0 -i 2> /dev/null'); - if ($just_save_results || !mail_qa_team($failed_tests_data, $compression, $status) && !TRAVIS_CI) { + if (($just_save_results || !mail_qa_team($failed_tests_data, $status)) && !TRAVIS_CI) { file_put_contents($output_file, $failed_tests_data); if (!$just_save_results) { @@ -456,20 +479,16 @@ function save_or_mail_results() $test_files = array(); $redir_tests = array(); $test_results = array(); -$PHP_FAILED_TESTS = array('BORKED' => array(), 'FAILED' => array(), 'WARNED' => array(), 'LEAKED' => array(), 'XFAILED' => array()); +$PHP_FAILED_TESTS = array('BORKED' => array(), 'FAILED' => array(), 'WARNED' => array(), 'LEAKED' => array(), 'XFAILED' => array(), 'SLOW' => array()); // If parameters given assume they represent selected tests to run. +$result_tests_file= false; $failed_tests_file= false; $pass_option_n = false; $pass_options = ''; -$compression = 0; $output_file = $CUR_DIR . '/php_test_results_' . date('Ymd_Hi') . '.txt'; -if ($compression && in_array("compress.zlib", stream_get_filters())) { - $output_file = 'compress.zlib://' . $output_file . '.gz'; -} - $just_save_results = false; $leak_check = false; $html_output = false; @@ -479,6 +498,7 @@ $temp_target = null; $temp_urlbase = null; $conf_passed = null; $no_clean = false; +$slow_min_ms = INF; $cfgtypes = array('show', 'keep'); $cfgfiles = array('skip', 'php', 'clean', 'out', 'diff', 'exp'); @@ -494,7 +514,7 @@ foreach($cfgtypes as $type) { if (getenv('TEST_PHP_ARGS')) { - if (!isset($argc) || !$argc || !isset($argv)) { + if (!isset($argc, $argv) || !$argc) { $argv = array(__FILE__); } @@ -563,6 +583,9 @@ if (isset($argc) && $argc > 1) { case 'a': $failed_tests_file = fopen($argv[++$i], 'a+t'); break; + case 'W': + $result_tests_file = fopen($argv[++$i], 'w+t'); + break; case 'c': $conf_passed = $argv[++$i]; break; @@ -636,6 +659,9 @@ if (isset($argc) && $argc > 1) { $cfg['show'][$file] = true; } break; + case '--show-slow': + $slow_min_ms = $argv[++$i]; + break; case '--temp-source': $temp_source = $argv[++$i]; break; @@ -695,6 +721,8 @@ Options: -a <file> Same as -w but append rather then truncating <file>. + -W <file> Write a list of all tests and their result status to <file>. + -c <file> Look for php.ini in directory <file> or use <file> as ini. -n Pass -n option to the php binary (Do not use a php.ini). @@ -751,6 +779,9 @@ Options: get written independent of the log format, however 'diff' only exists when a test fails. + --show-slow [n] + Show all tests that took longer than [n] milliseconds to run. + --no-clean Do not execute clean section if any. HELP; @@ -802,7 +833,7 @@ HELP; if ($test_cnt) { putenv('NO_INTERACTION=1'); verify_config(); - write_information($html_output); + write_information(); usort($test_files, "test_sort"); $start_time = time(); @@ -824,6 +855,10 @@ HELP; fclose($failed_tests_file); } + if ($result_tests_file) { + fclose($result_tests_file); + } + compute_summary(); if ($html_output) { fwrite($html_file, "<hr/>\n" . get_summary(false, true)); @@ -852,7 +887,7 @@ HELP; } verify_config(); -write_information($html_output); +write_information(); // Compile a list of all test files (*.phpt). $test_files = array(); @@ -884,7 +919,7 @@ foreach ($user_tests as $dir) { function find_files($dir, $is_ext_dir = false, $ignore = false) { - global $test_files, $exts_to_test, $ignored_by_ext, $exts_skipped, $exts_tested; + global $test_files, $exts_to_test, $ignored_by_ext, $exts_skipped; $o = opendir($dir) or error("cannot open directory: $dir"); @@ -959,6 +994,10 @@ if ($failed_tests_file) { fclose($failed_tests_file); } +if ($result_tests_file) { + fclose($result_tests_file); +} + // Summarize results if (0 == count($test_results)) { @@ -988,7 +1027,7 @@ exit(0); // Send Email to QA Team // -function mail_qa_team($data, $compression, $status = false) +function mail_qa_team($data, $status = false) { $url_bits = parse_url(QA_SUBMISSION_PAGE); @@ -1077,7 +1116,7 @@ function error_report($testname, $logname, $tested) } } -function system_with_timeout($commandline, $env = null, $stdin = null) +function system_with_timeout($commandline, $env = null, $stdin = null, $captureStdIn = true, $captureStdOut = true, $captureStdErr = true) { global $leak_check, $cwd; @@ -1088,21 +1127,29 @@ function system_with_timeout($commandline, $env = null, $stdin = null) $bin_env[$key] = $value; } - $proc = proc_open($commandline, array( - 0 => array('pipe', 'r'), - 1 => array('pipe', 'w'), - 2 => array('pipe', 'w') - ), $pipes, $cwd, $bin_env, array('suppress_errors' => true, 'binary_pipes' => true)); + $descriptorspec = array(); + if ($captureStdIn) { + $descriptorspec[0] = array('pipe', 'r'); + } + if ($captureStdOut) { + $descriptorspec[1] = array('pipe', 'w'); + } + if ($captureStdErr) { + $descriptorspec[2] = array('pipe', 'w'); + } + $proc = proc_open($commandline, $descriptorspec, $pipes, $cwd, $bin_env, array('suppress_errors' => true, 'binary_pipes' => true)); if (!$proc) { return false; } - if (!is_null($stdin)) { - fwrite($pipes[0], $stdin); + if ($captureStdIn) { + if (!is_null($stdin)) { + fwrite($pipes[0], $stdin); + } + fclose($pipes[0]); + unset($pipes[0]); } - fclose($pipes[0]); - unset($pipes[0]); $timeout = $leak_check ? 300 : (isset($env['TEST_TIMEOUT']) ? $env['TEST_TIMEOUT'] : 60); @@ -1122,7 +1169,13 @@ function system_with_timeout($commandline, $env = null, $stdin = null) proc_terminate($proc, 9); return $data; } else if ($n > 0) { - $line = fread($pipes[1], 8192); + if ($captureStdOut) { + $line = fread($pipes[1], 8192); + } elseif ($captureStdErr) { + $line = fread($pipes[2], 8192); + } else { + $line = ''; + } if (strlen($line) == 0) { /* EOF */ break; @@ -1140,13 +1193,13 @@ function system_with_timeout($commandline, $env = null, $stdin = null) $data .= "\nTermsig=" . ($stat["exitcode"] - 128) . "\n"; } - $code = proc_close($proc); + proc_close($proc); return $data; } function run_all_tests($test_files, $env, $redir_tested = null) { - global $test_results, $failed_tests_file, $php, $test_cnt, $test_idx; + global $test_results, $failed_tests_file, $result_tests_file, $php, $test_idx; foreach($test_files as $name) { @@ -1169,6 +1222,9 @@ function run_all_tests($test_files, $env, $redir_tested = null) if ($failed_tests_file && ($result == 'XFAILED' || $result == 'FAILED' || $result == 'WARNED' || $result == 'LEAKED')) { fwrite($failed_tests_file, "$index\n"); } + if ($result_tests_file) { + fwrite($result_tests_file, "$result\t$index\n"); + } } } } @@ -1197,14 +1253,14 @@ function show_file_block($file, $block, $section = null) // function run_test($php, $file, $env) { - global $log_format, $info_params, $ini_overwrites, $cwd, $PHP_FAILED_TESTS; + global $log_format, $ini_overwrites, $cwd, $PHP_FAILED_TESTS; global $pass_options, $DETAILED, $IN_REDIRECT, $test_cnt, $test_idx; global $leak_check, $temp_source, $temp_target, $cfg, $environment; global $no_clean; global $valgrind_version; - global $JUNIT; global $SHOW_ONLY_GROUPS; global $no_file_cache; + global $slow_min_ms; $temp_filenames = null; $org_file = $file; @@ -1311,7 +1367,7 @@ TEST $file unset($section_text['FILEEOF']); } - foreach (array( 'FILE', 'EXPECT', 'EXPECTF', 'EXPECTREGEX' ) as $prefix) { + foreach (array( 'FILE', 'EXPECT', 'EXPECTF', 'EXPECTREGEX' ) as $prefix) { $key = $prefix . '_EXTERNAL'; if (@count($section_text[$key]) == 1) { @@ -1353,6 +1409,21 @@ TEST $file return 'BORKED'; } + if (isset($section_text['CAPTURE_STDIO'])) { + $captureStdIn = stripos($section_text['CAPTURE_STDIO'], 'STDIN') !== false; + $captureStdOut = stripos($section_text['CAPTURE_STDIO'], 'STDOUT') !== false; + $captureStdErr = stripos($section_text['CAPTURE_STDIO'], 'STDERR') !== false; + } else { + $captureStdIn = true; + $captureStdOut = true; + $captureStdErr = true; + } + if ($captureStdOut && $captureStdErr) { + $cmdRedirect = ' 2>&1'; + } else { + $cmdRedirect = ''; + } + $tested = trim($section_text['TEST']); /* For GET/POST/PUT tests, check if cgi sapi is available and if it is, use it. */ @@ -1577,10 +1648,6 @@ TEST $file show_result('SKIP', $tested, $tested_file, '', $temp_filenames); } - if (isset($old_php)) { - $php = $old_php; - } - if (!$cfg['keep']['skip']) { @unlink($test_skipif); } @@ -1609,9 +1676,9 @@ TEST $file } } } - + if (!extension_loaded("zlib") - && ( array_key_exists("GZIP_POST", $section_text) + && ( array_key_exists("GZIP_POST", $section_text) || array_key_exists("DEFLATE_POST", $section_text)) ) { $message = "ext/zlib required"; @@ -1628,7 +1695,7 @@ TEST $file $IN_REDIRECT['dir'] = realpath(dirname($file)); $IN_REDIRECT['prefix'] = trim($section_text['TEST']); - if (count($IN_REDIRECT['TESTS']) == 1) { + if (!empty($IN_REDIRECT['TESTS'])) { if (is_array($org_file)) { $test_files[] = $org_file[1]; @@ -1760,7 +1827,7 @@ TEST $file } save_text($tmp_post, $request); - $cmd = "$php $pass_options $ini_settings -f \"$test_file\" 2>&1 < \"$tmp_post\""; + $cmd = "$php $pass_options $ini_settings -f \"$test_file\"$cmdRedirect < \"$tmp_post\""; } elseif (array_key_exists('PUT', $section_text) && !empty($section_text['PUT'])) { @@ -1794,7 +1861,7 @@ TEST $file } save_text($tmp_post, $request); - $cmd = "$php $pass_options $ini_settings -f \"$test_file\" 2>&1 < \"$tmp_post\""; + $cmd = "$php $pass_options $ini_settings -f \"$test_file\"$cmdRedirect < \"$tmp_post\""; } else if (array_key_exists('POST', $section_text) && !empty($section_text['POST'])) { @@ -1811,7 +1878,7 @@ TEST $file $env['CONTENT_LENGTH'] = $content_length; } - $cmd = "$php $pass_options $ini_settings -f \"$test_file\" 2>&1 < \"$tmp_post\""; + $cmd = "$php $pass_options $ini_settings -f \"$test_file\"$cmdRedirect < \"$tmp_post\""; } else if (array_key_exists('GZIP_POST', $section_text) && !empty($section_text['GZIP_POST'])) { @@ -1826,7 +1893,7 @@ TEST $file $env['CONTENT_TYPE'] = 'application/x-www-form-urlencoded'; $env['CONTENT_LENGTH'] = $content_length; - $cmd = "$php $pass_options $ini_settings -f \"$test_file\" 2>&1 < \"$tmp_post\""; + $cmd = "$php $pass_options $ini_settings -f \"$test_file\"$cmdRedirect < \"$tmp_post\""; } else if (array_key_exists('DEFLATE_POST', $section_text) && !empty($section_text['DEFLATE_POST'])) { $post = trim($section_text['DEFLATE_POST']); @@ -1839,7 +1906,7 @@ TEST $file $env['CONTENT_TYPE'] = 'application/x-www-form-urlencoded'; $env['CONTENT_LENGTH'] = $content_length; - $cmd = "$php $pass_options $ini_settings -f \"$test_file\" 2>&1 < \"$tmp_post\""; + $cmd = "$php $pass_options $ini_settings -f \"$test_file\"$cmdRedirect < \"$tmp_post\""; } else { @@ -1847,21 +1914,26 @@ TEST $file $env['CONTENT_TYPE'] = ''; $env['CONTENT_LENGTH'] = ''; - $cmd = "$php $pass_options $ini_settings -f \"$test_file\" $args 2>&1"; + $cmd = "$php $pass_options $ini_settings -f \"$test_file\" $args$cmdRedirect"; } if ($leak_check) { $env['USE_ZEND_ALLOC'] = '0'; $env['ZEND_DONT_UNLOAD_MODULES'] = 1; + $valgrind_cmd = "valgrind -q --tool=memcheck --trace-children=yes"; + if (strpos($test_file, "pcre") !== false) { + $valgrind_cmd .= " --smc-check=all"; + } + /* --vex-iropt-register-updates=allregs-at-mem-access is necessary for phpdbg watchpoint tests */ if (version_compare($valgrind_version, '3.8.0', '>=')) { /* valgrind 3.3.0+ doesn't have --log-file-exactly option */ - $cmd = "valgrind -q --tool=memcheck --trace-children=yes --vex-iropt-register-updates=allregs-at-mem-access --log-file=$memcheck_filename $cmd"; + $cmd = "$valgrind_cmd --vex-iropt-register-updates=allregs-at-mem-access --log-file=$memcheck_filename $cmd"; } elseif (version_compare($valgrind_version, '3.3.0', '>=')) { - $cmd = "valgrind -q --tool=memcheck --trace-children=yes --vex-iropt-precise-memory-exns=yes --log-file=$memcheck_filename $cmd"; + $cmd = "$valgrind_cmd --vex-iropt-precise-memory-exns=yes --log-file=$memcheck_filename $cmd"; } else { - $cmd = "valgrind -q --tool=memcheck --trace-children=yes --vex-iropt-precise-memory-exns=yes --log-file-exactly=$memcheck_filename $cmd"; + $cmd = "$valgrind_cmd --vex-iropt-precise-memory-exns=yes --log-file-exactly=$memcheck_filename $cmd"; } } else { @@ -1882,10 +1954,21 @@ COMMAND $cmd "; junit_start_timer($shortname); + $startTime = microtime(true); - $out = system_with_timeout($cmd, $env, isset($section_text['STDIN']) ? $section_text['STDIN'] : null); + $out = system_with_timeout($cmd, $env, isset($section_text['STDIN']) ? $section_text['STDIN'] : null, $captureStdIn, $captureStdOut, $captureStdErr); junit_finish_timer($shortname); + $time = microtime(true) - $startTime; + if ($time * 1000 >= $slow_min_ms) { + $PHP_FAILED_TESTS['SLOW'][] = array( + 'name' => $file, + 'test_name' => (is_array($IN_REDIRECT) ? $IN_REDIRECT['via'] : '') . $tested . " [$tested_file]", + 'output' => '', + 'diff' => '', + 'info' => $time, + ); + } if (array_key_exists('CLEAN', $section_text) && (!$no_clean || $cfg['keep']['clean'])) { @@ -1925,12 +2008,11 @@ COMMAND $cmd $output = preg_replace("/\r\n/", "\n", trim($out)); /* when using CGI, strip the headers from the output */ - $headers = ""; + $headers = array(); if (!empty($uses_cgi) && preg_match("/^(.*?)\r?\n\r?\n(.*)/s", $out, $match)) { $output = trim($match[2]); $rh = preg_split("/[\n\r]+/", $match[1]); - $headers = array(); foreach ($rh as $line) { if (strpos($line, ':') !== false) { @@ -1955,26 +2037,23 @@ COMMAND $cmd } } - $org_headers = $headers; - $headers = array(); $output_headers = array(); foreach($want as $k => $v) { - if (isset($org_headers[$k])) { - $headers = $org_headers[$k]; - $output_headers[] = $k . ': ' . $org_headers[$k]; + if (isset($headers[$k])) { + $output_headers[] = $k . ': ' . $headers[$k]; } - if (!isset($org_headers[$k]) || $org_headers[$k] != $v) { + if (!isset($headers[$k]) || $headers[$k] != $v) { $failed_headers = true; } } ksort($wanted_headers); - $wanted_headers = join("\n", $wanted_headers); + $wanted_headers = implode("\n", $wanted_headers); ksort($output_headers); - $output_headers = join("\n", $output_headers); + $output_headers = implode("\n", $output_headers); } show_file_block('out', $output); @@ -2064,9 +2143,6 @@ COMMAND $cmd if (!$cfg['keep']['php']) { @unlink($test_file); } - if (isset($old_php)) { - $php = $old_php; - } if (!$leaked && !$failed_headers) { if (isset($section_text['XFAIL'] )) { @@ -2094,10 +2170,6 @@ COMMAND $cmd @unlink($test_file); } - if (isset($old_php)) { - $php = $old_php; - } - if (!$leaked && !$failed_headers) { if (isset($section_text['XFAIL'] )) { $warn = true; @@ -2197,10 +2269,6 @@ $output ); } - if (isset($old_php)) { - $php = $old_php; - } - $diff = empty($diff) ? '' : preg_replace('/\e/', '<esc>', $diff); junit_mark_test_as($restype, str_replace($cwd . '/', '', $tested_file), $tested, null, $info, $diff); @@ -2261,8 +2329,8 @@ function count_array_diff($ar1, $ar2, $is_reg, $w, $idx1, $idx2, $cnt1, $cnt2, $ function generate_array_diff($ar1, $ar2, $is_reg, $w) { - $idx1 = 0; $ofs1 = 0; $cnt1 = @count($ar1); - $idx2 = 0; $ofs2 = 0; $cnt2 = @count($ar2); + $idx1 = 0; $cnt1 = @count($ar1); + $idx2 = 0; $cnt2 = @count($ar2); $diff = array(); $old1 = array(); $old2 = array(); @@ -2280,10 +2348,8 @@ function generate_array_diff($ar1, $ar2, $is_reg, $w) if ($c1 > $c2) { $old1[$idx1] = sprintf("%03d- ", $idx1+1) . $w[$idx1++]; - $last = 1; } else if ($c2 > 0) { $old2[$idx2] = sprintf("%03d+ ", $idx2+1) . $ar2[$idx2++]; - $last = 2; } else { $old1[$idx1] = sprintf("%03d- ", $idx1+1) . $w[$idx1++]; $old2[$idx2] = sprintf("%03d+ ", $idx2+1) . $ar2[$idx2++]; @@ -2419,7 +2485,7 @@ function compute_summary() $sum_results['SKIPPED'] += $ignored_by_ext; $percent_results = array(); - while (list($v, $n) = each($sum_results)) { + foreach ($sum_results as $v => $n) { $percent_results[$v] = (100.0 * $n) / $n_total; } } @@ -2484,6 +2550,22 @@ Time taken : ' . sprintf('%4d seconds', $end_time - $start_time) . ' '; $failed_test_summary = ''; + if (count($PHP_FAILED_TESTS['SLOW'])) { + usort($PHP_FAILED_TESTS['SLOW'], function($a, $b) { + return $a['info'] < $b['info'] ? 1 : -1; + }); + + $failed_test_summary .= ' +===================================================================== +SLOW TEST SUMMARY +--------------------------------------------------------------------- +'; + foreach ($PHP_FAILED_TESTS['SLOW'] as $failed_test_data) { + $failed_test_summary .= sprintf('(%.3f s) ', $failed_test_data['info']) . $failed_test_data['test_name'] . "\n"; + } + $failed_test_summary .= "=====================================================================\n"; + } + if (count($PHP_FAILED_TESTS['XFAILED'])) { $failed_test_summary .= ' ===================================================================== |