Add --repeat testing mode

This testing mode executes the test multiple times in the same
process (but in different requests). It is primarily intended to
catch tracing JIT bugs, but also catches state leaks across
requests.

Closes GH-6365.

18 files changed, 144 insertions, 29 deletions

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index dabf35b3d2..1932b1cdee 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -103,3 +103,8 @@ jobs:
         configurationName: DEBUG_NTS_FILE_CACHE
         configurationParameters: '--enable-debug --disable-zts'
         timeoutInMinutes: 90
+    - template: azure/job.yml
+      parameters:
+        configurationName: DEBUG_NTS_REPEAT
+        configurationParameters: '--enable-debug --disable-zts'
+        runTestsParameters: '--repeat 2'
diff --git a/ext/opcache/tests/bug65915.phpt b/ext/opcache/tests/bug65915.phpt
index c616c4fb5f..b62dbd8ea7 100644
--- a/ext/opcache/tests/bug65915.phpt
+++ b/ext/opcache/tests/bug65915.phpt
@@ -5,7 +5,11 @@ opcache.enable=1
 opcache.enable_cli=1
 opcache.file_cache_only=0
 --SKIPIF--
-<?php require_once('skipif.inc'); ?>
+<?php
+require_once('skipif.inc');
+// We don't invalidate the file after the second write.
+if (getenv('SKIP_REPEAT')) die("skip Not repeatable");
+?>
 --FILE--
 <?php
 $tmp = __DIR__ . "/bug65915.inc.php";
diff --git a/ext/opcache/tests/zzz_basic_logging.phpt b/ext/opcache/tests/zzz_basic_logging.phpt
index bf04b50861..0fbf308a51 100644
--- a/ext/opcache/tests/zzz_basic_logging.phpt
+++ b/ext/opcache/tests/zzz_basic_logging.phpt
@@ -12,7 +12,11 @@ opcache.log_verbosity_level=4
 opcache.huge_code_pages=0
 opcache.preload=
 --SKIPIF--
-<?php require_once('skipif.inc'); ?>
+<?php
+require_once('skipif.inc');
+// Prints "Debug Restarting!" message on next request.
+if (getenv('SKIP_REPEAT')) die("skip Not repeatable");
+?>
 --FILE--
 <?php
 echo "Foo Bar\n";
diff --git a/ext/pcntl/tests/002.phpt b/ext/pcntl/tests/002.phpt
index 68b2643092..254da86136 100644
--- a/ext/pcntl/tests/002.phpt
+++ b/ext/pcntl/tests/002.phpt
@@ -7,6 +7,7 @@ pcntl: pcntl_sigprocmask(), pcntl_sigwaitinfo(), pcntl_sigtimedwait()
     elseif (!function_exists('pcntl_sigwaitinfo') or !function_exists('pcntl_sigtimedwait')) die('skip required functionality is not available');
     elseif (!defined('CLD_EXITED')) die('skip CLD_EXITED not defined');
     elseif (getenv('SKIP_ASAN')) die('skip Fails intermittently under asan/msan');
+    elseif (getenv("SKIP_REPEAT")) die("skip cannot be repeated");
 ?>
 --FILE--
 <?php
diff --git a/ext/pcntl/tests/pcntl_unshare_01.phpt b/ext/pcntl/tests/pcntl_unshare_01.phpt
index fcbf112e4d..6eb5a20bdd 100644
--- a/ext/pcntl/tests/pcntl_unshare_01.phpt
+++ b/ext/pcntl/tests/pcntl_unshare_01.phpt
@@ -9,6 +9,7 @@ if (!defined("CLONE_NEWUSER")) die("skip flag unavailable");
 if (@pcntl_unshare(CLONE_NEWUSER) == false && pcntl_get_last_error() == PCNTL_EPERM) {
     die("skip Insufficient privileges to use CLONE_NEWUSER");
 }
+if (getenv("SKIP_REPEAT")) die("skip cannot be repeated");
 ?>
 --FILE--
 <?php
diff --git a/ext/pcntl/tests/pcntl_unshare_02.phpt b/ext/pcntl/tests/pcntl_unshare_02.phpt
index fdf07bc81c..3e90a3920c 100644
--- a/ext/pcntl/tests/pcntl_unshare_02.phpt
+++ b/ext/pcntl/tests/pcntl_unshare_02.phpt
@@ -15,6 +15,7 @@ if (posix_getuid() !== 0 &&
 if (@pcntl_unshare(CLONE_NEWPID) == false && pcntl_get_last_error() == PCNTL_EPERM) {
     die("skip Insufficient privileges for CLONE_NEWPID");
 }
+if (getenv("SKIP_REPEAT")) die("skip cannot be repeated");
 ?>
 --FILE--
 <?php
diff --git a/ext/pcntl/tests/pcntl_unshare_03.phpt b/ext/pcntl/tests/pcntl_unshare_03.phpt
index f82f27b719..8e9bd793e6 100644
--- a/ext/pcntl/tests/pcntl_unshare_03.phpt
+++ b/ext/pcntl/tests/pcntl_unshare_03.phpt
@@ -15,6 +15,7 @@ if (@pcntl_unshare(CLONE_NEWNET) == false && pcntl_get_last_error() == PCNTL_EPE
     die("skip Insufficient privileges for CLONE_NEWPID");
 }
 if (getenv("SKIP_ONLINE_TESTS")) die("skip online test");
+if (getenv("SKIP_REPEAT")) die("skip cannot be repeated");
 ?>
 --FILE--
 <?php
diff --git a/ext/pgsql/tests/skipif.inc b/ext/pgsql/tests/skipif.inc
index 38c2888afa..92712f3e05 100644
--- a/ext/pgsql/tests/skipif.inc
+++ b/ext/pgsql/tests/skipif.inc
@@ -12,6 +12,11 @@ include("lcmess.inc");
 if (!extension_loaded("pgsql")) {
     die("skip pgsql extension not loaded\n");
 }
+if (getenv("SKIP_REPEAT")) {
+    // pgsql tests are order-dependent.
+    // We should probably change that, but in the meantime do not allow repetition.
+    die("skip Cannot repeat pgsql tests");
+}
 $conn = @pg_connect($conn_str);
 if (!is_resource($conn)) {
     die("skip could not connect\n");
diff --git a/ext/readline/tests/libedit_info_001.phpt b/ext/readline/tests/libedit_info_001.phpt
index 46618120fb..8b6fb80671 100644
--- a/ext/readline/tests/libedit_info_001.phpt
+++ b/ext/readline/tests/libedit_info_001.phpt
@@ -6,6 +6,7 @@ if (READLINE_LIB != "libedit") die("skip libedit only");
 if(substr(PHP_OS, 0, 3) == 'WIN' ) {
     die('skip not for windows');
 }
+if (getenv('SKIP_REPEAT')) die("skip readline has global state");
 ?>
 --FILE--
 <?php
diff --git a/ext/readline/tests/libedit_write_history_001.phpt b/ext/readline/tests/libedit_write_history_001.phpt
index 96424e232f..69739c488e 100644
--- a/ext/readline/tests/libedit_write_history_001.phpt
+++ b/ext/readline/tests/libedit_write_history_001.phpt
@@ -6,6 +6,7 @@ if (READLINE_LIB != "libedit") die("skip libedit only");
 if(substr(PHP_OS, 0, 3) == 'WIN' ) {
     die('skip not for windows');
 }
+if (getenv('SKIP_REPEAT')) die("skip readline has global state");
 ?>
 --FILE--
 <?php
diff --git a/ext/readline/tests/readline_info_001.phpt b/ext/readline/tests/readline_info_001.phpt
index 8f5d641d94..f0d4e67ca5 100644
--- a/ext/readline/tests/readline_info_001.phpt
+++ b/ext/readline/tests/readline_info_001.phpt
@@ -3,6 +3,7 @@ readline_info(): Basic test
 --SKIPIF--
 <?php if (!extension_loaded("readline")) die("skip");
 if (READLINE_LIB == "libedit") die("skip readline only");
+if (getenv('SKIP_REPEAT')) die("skip readline has global state");
 ?>
 --FILE--
 <?php
diff --git a/ext/readline/tests/readline_read_history_001.phpt b/ext/readline/tests/readline_read_history_001.phpt
index 6cabaf369b..8fbcaf1a27 100644
--- a/ext/readline/tests/readline_read_history_001.phpt
+++ b/ext/readline/tests/readline_read_history_001.phpt
@@ -1,7 +1,10 @@
 --TEST--
 readline_read_history(): Basic test
 --SKIPIF--
-<?php if (!extension_loaded("readline") || !function_exists('readline_list_history')) die("skip"); ?>
+<?php
+if (!extension_loaded("readline") || !function_exists('readline_list_history')) die("skip");
+if (getenv('SKIP_REPEAT')) die("skip readline has global state");
+?>
 --FILE--
 <?php
 
diff --git a/ext/readline/tests/readline_write_history_001.phpt b/ext/readline/tests/readline_write_history_001.phpt
index 95c34e3e89..bcbeb5efa4 100644
--- a/ext/readline/tests/readline_write_history_001.phpt
+++ b/ext/readline/tests/readline_write_history_001.phpt
@@ -3,6 +3,7 @@ readline_write_history(): Basic test
 --SKIPIF--
 <?php if (!extension_loaded("readline") || !function_exists('readline_add_history')) die("skip");
 if (READLINE_LIB == "libedit") die("skip readline only");
+if (getenv('SKIP_REPEAT')) die("skip readline has global state");
 ?>
 --FILE--
 <?php
diff --git a/ext/standard/tests/file/realpath_bug77484.phpt b/ext/standard/tests/file/realpath_bug77484.phpt
index 85595d09e3..9aa3a335c2 100644
--- a/ext/standard/tests/file/realpath_bug77484.phpt
+++ b/ext/standard/tests/file/realpath_bug77484.phpt
@@ -9,6 +9,10 @@ if (PHP_ZTS) {
     /* TODO eliminate difference in TS build. */
     die("skip Not for ZTS");
 }
+if (getenv('SKIP_REPEAT')) {
+    /* The cwd is persistent across repeats */
+    die("skip Not repeatable");
+}
 ?>
 --FILE--
 <?php
diff --git a/ext/standard/tests/general_functions/http_response_code.phpt b/ext/standard/tests/general_functions/http_response_code.phpt
index bc2775f3d3..ab290c3cef 100644
--- a/ext/standard/tests/general_functions/http_response_code.phpt
+++ b/ext/standard/tests/general_functions/http_response_code.phpt
@@ -2,6 +2,15 @@
 Test http_response_code basic functionality
 --CREDITS--
 Gabriel Caruso (carusogabriel@php.net)
+--SKIPIF--
+<?php
+if (getenv('SKIP_REPEAT')) {
+    /* The http_response_code leaks across repeats. Technically that's a bug, but it's something
+     * that only affects the CLI repeat option, where the response code is not meaningful in the
+     * first place. Other SAPIs will properly reset the response code for each request. */
+    die('skip Not repeatable');
+}
+?>
 --FILE--
 <?php
 var_dump(
diff --git a/ext/standard/tests/general_functions/proc_nice_basic.phpt b/ext/standard/tests/general_functions/proc_nice_basic.phpt
index b8cc3cd128..17bca0f7b7 100644
--- a/ext/standard/tests/general_functions/proc_nice_basic.phpt
+++ b/ext/standard/tests/general_functions/proc_nice_basic.phpt
@@ -25,7 +25,7 @@ if ($exit_code !== 0) {
         else
             return -1;
     }
-    $delta = 10;
+    $delta = 5;
     $pid = getmypid();
     $niceBefore = getNice($pid);
     proc_nice($delta);
diff --git a/run-tests.php b/run-tests.php
index 00881cf779..4beafa90b2 100755
--- a/run-tests.php
+++ b/run-tests.php
@@ -147,7 +147,7 @@ function main(): void
            $repeat, $result_tests_file, $slow_min_ms, $start_time, $switch,
            $temp_source, $temp_target, $test_cnt, $test_dirs,
            $test_files, $test_idx, $test_list, $test_results, $testfile,
-           $user_tests, $valgrind, $sum_results, $shuffle, $file_cache;
+           $user_tests, $valgrind, $sum_results, $shuffle, $file_cache, $num_repeats;
     // Parallel testing
     global $workers, $workerID;
     global $context_line_count;
@@ -405,6 +405,7 @@ function main(): void
     $shuffle = false;
     $workers = null;
     $context_line_count = 3;
+    $num_repeats = 1;
 
     $cfgtypes = ['show', 'keep'];
     $cfgfiles = ['skip', 'php', 'clean', 'out', 'diff', 'exp', 'mem'];
@@ -623,6 +624,10 @@ function main(): void
                             . ':print_suppressions=0';
                     }
                     break;
+                case '--repeat':
+                    $num_repeats = (int) $argv[++$i];
+                    $environment['SKIP_REPEAT'] = 1;
+                    break;
                 //case 'w'
                 case '-':
                     // repeat check with full switch
@@ -1802,6 +1807,13 @@ function show_file_block(string $file, string $block, ?string $section = null):
     }
 }
 
+function skip_test(string $tested, string $tested_file, string $shortname, string $reason) {
+    show_result('SKIP', $tested, $tested_file, "reason: $reason");
+    junit_init_suite(junit_get_suitename_for($shortname));
+    junit_mark_test_as('SKIP', $shortname, $tested, 0, $reason);
+    return 'SKIPPED';
+}
+
 //
 //  Run an individual test case.
 //
@@ -1818,6 +1830,7 @@ function run_test(string $php, $file, array $env): string
     global $no_file_cache;
     global $slow_min_ms;
     global $preload, $file_cache;
+    global $num_repeats;
     // Parallel testing
     global $workerID;
     $temp_filenames = null;
@@ -1911,6 +1924,10 @@ TEST $file
         }
     }
 
+    $shortname = str_replace(TEST_PHP_SRCDIR . '/', '', $file);
+    $tested_file = $shortname;
+    $tested = trim($section_text['TEST']);
+
     // the redirect section allows a set of tests to be reused outside of
     // a given test dir
     if ($bork_info === null) {
@@ -1928,6 +1945,10 @@ TEST $file
                 unset($section_text['FILEEOF']);
             }
 
+            if ($num_repeats > 1 && isset($section_text['FILE_EXTERNAL'])) {
+                return skip_test($tested, $tested_file, $shortname, 'Test with FILE_EXTERNAL might not be repeatable');
+            }
+
             foreach (['FILE', 'EXPECT', 'EXPECTF', 'EXPECTREGEX'] as $prefix) {
                 $key = $prefix . '_EXTERNAL';
 
@@ -1951,9 +1972,6 @@ TEST $file
     }
     fclose($fp);
 
-    $shortname = str_replace(TEST_PHP_SRCDIR . '/', '', $file);
-    $tested_file = $shortname;
-
     if ($bork_info !== null) {
         show_result("BORK", $bork_info, $tested_file);
         $PHP_FAILED_TESTS['BORKED'][] = [
@@ -1983,8 +2001,6 @@ TEST $file
         $cmdRedirect = '';
     }
 
-    $tested = trim($section_text['TEST']);
-
     /* For GET/POST/PUT tests, check if cgi sapi is available and if it is, use it. */
     if (array_key_exists('CGI', $section_text) || !empty($section_text['GET']) || !empty($section_text['POST']) || !empty($section_text['GZIP_POST']) || !empty($section_text['DEFLATE_POST']) || !empty($section_text['POST_RAW']) || !empty($section_text['PUT']) || !empty($section_text['COOKIE']) || !empty($section_text['EXPECTHEADERS'])) {
         if (isset($php_cgi)) {
@@ -1999,13 +2015,12 @@ TEST $file
             } elseif (file_exists(dirname($php) . "/php-cgi")) {
                 $php = realpath(dirname($php) . "/php-cgi") . ' -C ';
             } else {
-                show_result('SKIP', $tested, $tested_file, "reason: CGI not available");
-
-                junit_init_suite(junit_get_suitename_for($shortname));
-                junit_mark_test_as('SKIP', $shortname, $tested, 0, 'CGI not available');
-                return 'SKIPPED';
+                return skip_test($tested, $tested_file, $shortname, 'CGI not available');
             }
         }
+        if ($num_repeats > 1) {
+            return skip_test($tested, $tested_file, $shortname, 'CGI does not support --repeat');
+        }
         $uses_cgi = true;
     }
 
@@ -2023,11 +2038,22 @@ TEST $file
             // be run straight away. For example, EXTENSIONS, SKIPIF, CLEAN.
             $extra_options = '-rr';
         } else {
-            show_result('SKIP', $tested, $tested_file, "reason: phpdbg not available");
+            return skip_test($tested, $tested_file, $shortname, 'phpdbg not available');
+        }
+        if ($num_repeats > 1) {
+            return skip_test($tested, $tested_file, $shortname, 'phpdbg does not support --repeat');
+        }
+    }
 
-            junit_init_suite(junit_get_suitename_for($shortname));
-            junit_mark_test_as('SKIP', $shortname, $tested, 0, 'phpdbg not available');
-            return 'SKIPPED';
+    if ($num_repeats > 1) {
+        if (array_key_exists('CLEAN', $section_text)) {
+            return skip_test($tested, $tested_file, $shortname, 'Test with CLEAN might not be repeatable');
+        }
+        if (array_key_exists('STDIN', $section_text)) {
+            return skip_test($tested, $tested_file, $shortname, 'Test with STDIN might not be repeatable');
+        }
+        if (array_key_exists('CAPTURE_STDIO', $section_text)) {
+            return skip_test($tested, $tested_file, $shortname, 'Test with CAPTURE_STDIO might not be repeatable');
         }
     }
 
@@ -2173,6 +2199,9 @@ TEST $file
             // even though all the files are re-created.
             $ini_settings['opcache.validate_timestamps'] = '0';
         }
+    } else if ($num_repeats > 1) {
+        // Make sure warnings still show up on the second run.
+        $ini_settings['opcache.record_warnings'] = '1';
     }
 
     // Any special ini settings
@@ -2183,6 +2212,10 @@ TEST $file
         $replacement = IS_WINDOWS ? '"' . PHP_BINARY . ' -r \"while ($in = fgets(STDIN)) echo $in;\" > $1"' : 'tee $1 >/dev/null';
         $section_text['INI'] = preg_replace('/{MAIL:(\S+)}/', $replacement, $section_text['INI']);
         settings2array(preg_split("/[\n\r]+/", $section_text['INI']), $ini_settings);
+
+        if ($num_repeats > 1 && isset($ini_settings['opcache.opt_debug_level'])) {
+            return skip_test($tested, $tested_file, $shortname, 'opt_debug_level tests are not repeatable');
+        }
     }
 
     $ini_settings = settings2params($ini_settings);
@@ -2484,7 +2517,8 @@ TEST $file
         $env['CONTENT_TYPE'] = '';
         $env['CONTENT_LENGTH'] = '';
 
-        $cmd = "$php $pass_options $ini_settings -f \"$test_file\" $args$cmdRedirect";
+        $repeat_option = $num_repeats > 1 ? "--repeat $num_repeats" : "";
+        $cmd = "$php $pass_options $repeat_option $ini_settings -f \"$test_file\" $args$cmdRedirect";
     }
 
     if ($valgrind) {
@@ -2557,6 +2591,25 @@ COMMAND $cmd
         }
     }
 
+    if ($num_repeats > 1) {
+        // In repeat mode, retain the output before the first execution,
+        // and of the last execution. Do this early, because the trimming below
+        // makes the newline handling complicated.
+        $separator1 = "Executing for the first time...\n";
+        $separator1_pos = strpos($out, $separator1);
+        if ($separator1_pos !== false) {
+            $separator2 = "Finished execution, repeating...\n";
+            $separator2_pos = strrpos($out, $separator2);
+            if ($separator2_pos !== false) {
+                $out = substr($out, 0, $separator1_pos)
+                     . substr($out, $separator2_pos + strlen($separator2));
+            } else {
+                $out = substr($out, 0, $separator1_pos)
+                     . substr($out, $separator1_pos + strlen($separator1));
+            }
+        }
+    }
+
     // Does the output match what is expected?
     $output = preg_replace("/\r\n/", "\n", trim($out));
 
diff --git a/sapi/cli/php_cli.c b/sapi/cli/php_cli.c
index d28f5a5378..5092fb0ffd 100644
--- a/sapi/cli/php_cli.c
+++ b/sapi/cli/php_cli.c
@@ -172,6 +172,9 @@ const opt_struct OPTIONS[] = {
 	{14,  1, "ri"},
 	{14,  1, "rextinfo"},
 	{15,  0, "ini"},
+	/* Internal testing option -- may be changed or removed without notice,
+	 * including in patch releases. */
+	{16,  1, "repeat"},
 	{'-', 0, NULL} /* end of args */
 };
 
@@ -529,7 +532,7 @@ static void php_cli_usage(char *argv0)
 
 static php_stream *s_in_process = NULL;
 
-static void cli_register_file_handles(void) /* {{{ */
+static void cli_register_file_handles(zend_bool no_close) /* {{{ */
 {
 	php_stream *s_in, *s_out, *s_err;
 	php_stream_context *sc_in=NULL, *sc_out=NULL, *sc_err=NULL;
@@ -546,11 +549,11 @@ static void cli_register_file_handles(void) /* {{{ */
 		return;
 	}
 
-#if PHP_DEBUG
-	/* do not close stdout and stderr */
-	s_out->flags |= PHP_STREAM_FLAG_NO_CLOSE;
-	s_err->flags |= PHP_STREAM_FLAG_NO_CLOSE;
-#endif
+	if (no_close) {
+		s_in->flags |= PHP_STREAM_FLAG_NO_CLOSE;
+		s_out->flags |= PHP_STREAM_FLAG_NO_CLOSE;
+		s_err->flags |= PHP_STREAM_FLAG_NO_CLOSE;
+	}
 
 	s_in_process = s_in;
 
@@ -614,6 +617,8 @@ static int do_cli(int argc, char **argv) /* {{{ */
 	int interactive=0;
 	const char *param_error=NULL;
 	int hide_argv = 0;
+	int num_repeats = 1;
+	pid_t pid = getpid();
 
 	zend_try {
 
@@ -839,6 +844,9 @@ static int do_cli(int argc, char **argv) /* {{{ */
 			case 15:
 				behavior = PHP_MODE_SHOW_INI_CONFIG;
 				break;
+			case 16:
+				num_repeats = atoi(php_optarg);
+				break;
 			default:
 				break;
 			}
@@ -873,6 +881,12 @@ static int do_cli(int argc, char **argv) /* {{{ */
 			fflush(stdout);
 		}
 
+		if (num_repeats > 1) {
+			fprintf(stdout, "Executing for the first time...\n");
+			fflush(stdout);
+		}
+
+do_repeat:
 		/* only set script_file if not set already and not in direct mode and not at end of parameter list */
 		if (argc > php_optind
 		  && !script_file
@@ -940,7 +954,7 @@ static int do_cli(int argc, char **argv) /* {{{ */
 		switch (behavior) {
 		case PHP_MODE_STANDARD:
 			if (strcmp(file_handle.filename, "Standard input code")) {
-				cli_register_file_handles();
+				cli_register_file_handles(/* no_close */ PHP_DEBUG || num_repeats > 1);
 			}
 
 			if (interactive && cli_shell_callbacks.cli_shell_run) {
@@ -975,7 +989,7 @@ static int do_cli(int argc, char **argv) /* {{{ */
 			}
 			break;
 		case PHP_MODE_CLI_DIRECT:
-			cli_register_file_handles();
+			cli_register_file_handles(/* no_close */ PHP_DEBUG || num_repeats > 1);
 			zend_eval_string_ex(exec_direct, NULL, "Command line code", 1);
 			break;
 
@@ -985,7 +999,7 @@ static int do_cli(int argc, char **argv) /* {{{ */
 				size_t len, index = 0;
 				zval argn, argi;
 
-				cli_register_file_handles();
+				cli_register_file_handles(/* no_close */ PHP_DEBUG || num_repeats > 1);
 
 				if (exec_begin) {
 					zend_eval_string_ex(exec_begin, NULL, "Command line begin code", 1);
@@ -1112,6 +1126,12 @@ out:
 	if (translated_path) {
 		free(translated_path);
 	}
+	/* Don't repeat fork()ed processes. */
+	if (--num_repeats && pid == getpid()) {
+		fprintf(stdout, "Finished execution, repeating...\n");
+		fflush(stdout);
+		goto do_repeat;
+	}
 	return EG(exit_status);
 err:
 	sapi_deactivate();