diff options
Diffstat (limited to 'Lib/test/regrtest.py')
| -rwxr-xr-x | Lib/test/regrtest.py | 282 |
1 files changed, 200 insertions, 82 deletions
diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index 84beb8dee5..ae62c6e7a0 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -20,6 +20,11 @@ python -E -Wd -m test [options] [test_name1 ...] Options: -h/--help -- print this text and exit +--timeout TIMEOUT + -- dump the traceback and exit if a test takes more + than TIMEOUT seconds; disabled if TIMEOUT is negative + or equals to zero +--wait -- wait for user input, e.g., allow a debugger to be attached Verbosity @@ -44,6 +49,9 @@ Selecting tests -- specify which special resource intensive tests to run -M/--memlimit LIMIT -- run very large memory-consuming tests + --testdir DIR + -- execute test files in the specified directory (instead + of the Python stdlib test suite) Special runs @@ -125,6 +133,8 @@ resources to test. Currently only the following are defined: all - Enable all special resources. + none - Disable all special resources (this is the default). + audio - Tests that use the audio device. (There are known cases of broken audio drivers that can crash Python or even the Linux kernel.) @@ -155,8 +165,11 @@ example, to run all the tests except for the gui tests, give the option '-uall,-gui'. """ +# We import importlib *ASAP* in order to test #15386 +import importlib + import builtins -import errno +import faulthandler import getopt import io import json @@ -166,6 +179,7 @@ import platform import random import re import shutil +import signal import sys import sysconfig import tempfile @@ -225,6 +239,7 @@ ENV_CHANGED = -1 SKIPPED = -2 RESOURCE_DENIED = -3 INTERRUPTED = -4 +CHILD_ERROR = -5 # error in a child process from test import support @@ -268,6 +283,18 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, on the command line. """ + # Display the Python traceback on fatal errors (e.g. segfault) + faulthandler.enable(all_threads=True) + + # Display the Python traceback on SIGALRM or SIGUSR1 signal + signals = [] + if hasattr(signal, 'SIGALRM'): + signals.append(signal.SIGALRM) + if hasattr(signal, 'SIGUSR1'): + signals.append(signal.SIGUSR1) + for signum in signals: + faulthandler.register(signum, chain=True) + replace_stdout() support.record_original_stdout(sys.stdout) @@ -278,7 +305,8 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, 'use=', 'threshold=', 'coverdir=', 'nocoverdir', 'runleaks', 'huntrleaks=', 'memlimit=', 'randseed=', 'multiprocess=', 'coverage', 'slaveargs=', 'forever', 'debug', - 'start=', 'nowindows', 'header', 'failfast', 'match=']) + 'start=', 'nowindows', 'header', 'testdir=', 'timeout=', 'wait', + 'failfast', 'match=']) except getopt.error as msg: usage(msg) @@ -289,6 +317,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, use_resources = [] debug = False start = None + timeout = None for o, a in opts: if o in ('-h', '--help'): print(__doc__) @@ -317,6 +346,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, elif o in ('-r', '--randomize'): randomize = True elif o == '--randseed': + randomize = True random_seed = int(a) elif o in ('-f', '--fromfile'): fromfile = a @@ -332,7 +362,9 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, elif o in ('-T', '--coverage'): trace = True elif o in ('-D', '--coverdir'): - coverdir = os.path.join(os.getcwd(), a) + # CWD is replaced with a temporary dir before calling main(), so we + # need join it with the saved CWD so it goes where the user expects. + coverdir = os.path.join(support.SAVEDCWD, a) elif o in ('-N', '--nocoverdir'): coverdir = None elif o in ('-R', '--huntrleaks'): @@ -350,9 +382,9 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, huntrleaks[1] = int(huntrleaks[1]) if len(huntrleaks) == 2 or not huntrleaks[2]: huntrleaks[2:] = ["reflog.txt"] - # Avoid false positives due to the character cache in - # stringobject.c filling slowly with random data - warm_char_cache() + # Avoid false positives due to various caches + # filling slowly with random data: + warm_caches() elif o in ('-M', '--memlimit'): support.set_memlimit(a) elif o in ('-u', '--use'): @@ -361,6 +393,9 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, if r == 'all': use_resources[:] = RESOURCE_NAMES continue + if r == 'none': + del use_resources[:] + continue remove = False if r[0] == '-': remove = True @@ -391,18 +426,45 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, forever = True elif o in ('-j', '--multiprocess'): use_mp = int(a) + if use_mp <= 0: + try: + import multiprocessing + # Use all cores + extras for tests that like to sleep + use_mp = 2 + multiprocessing.cpu_count() + except (ImportError, NotImplementedError): + use_mp = 3 + if use_mp == 1: + use_mp = None elif o == '--header': header = True elif o == '--slaveargs': args, kwargs = json.loads(a) try: result = runtest(*args, **kwargs) + except KeyboardInterrupt: + result = INTERRUPTED, '' except BaseException as e: - result = INTERRUPTED, e.__class__.__name__ + traceback.print_exc() + result = CHILD_ERROR, str(e) sys.stdout.flush() print() # Force a newline (just in case) print(json.dumps(result)) sys.exit(0) + elif o == '--testdir': + # CWD is replaced with a temporary dir before calling main(), so we + # join it with the saved CWD so it ends up where the user expects. + testdir = os.path.join(support.SAVEDCWD, a) + elif o == '--timeout': + if hasattr(faulthandler, 'dump_traceback_later'): + timeout = float(a) + if timeout <= 0: + timeout = None + else: + print("Warning: The timeout option requires " + "faulthandler.dump_traceback_later") + timeout = None + elif o == '--wait': + input("Press any key to continue...") else: print(("No handler for option {}. Please report this as a bug " "at http://bugs.python.org.").format(o), file=sys.stderr) @@ -481,7 +543,13 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, print("== ", os.getcwd()) print("Testing with flags:", sys.flags) - alltests = findtests(testdir, stdtests, nottests) + # if testdir is set, then we are not running the python tests suite, so + # don't add default tests to be executed or skipped (pass empty values) + if testdir: + alltests = findtests(testdir, list(), set()) + else: + alltests = findtests(testdir, stdtests, nottests) + selected = tests or args or alltests if single: selected = selected[:1] @@ -501,7 +569,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, random.shuffle(selected) if trace: import trace, tempfile - tracer = trace.Trace(ignoredirs=[sys.prefix, sys.exec_prefix, + tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix, tempfile.gettempdir()], trace=False, count=True) @@ -566,7 +634,8 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, (test, verbose, quiet), dict(huntrleaks=huntrleaks, use_resources=use_resources, debug=debug, output_on_failure=verbose3, - failfast=failfast, match_tests=match_tests) + timeout=timeout, failfast=failfast, + match_tests=match_tests) ) # -E is needed by some tests, e.g. test_import # Running the child from the same working directory ensures @@ -578,10 +647,15 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, close_fds=(os.name != 'nt'), cwd=support.SAVEDCWD) stdout, stderr = popen.communicate() + retcode = popen.wait() # Strip last refcount output line if it exists, since it # comes from the shutdown of the interpreter in the subcommand. stderr = debug_output_pat.sub("", stderr) stdout, _, result = stdout.strip().rpartition("\n") + if retcode != 0: + result = (CHILD_ERROR, "Exit code %s" % retcode) + output.put((test, stdout.rstrip(), stderr.rstrip(), result)) + return if not result: output.put((None, None, None, None)) return @@ -614,8 +688,9 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, sys.stdout.flush() sys.stderr.flush() if result[0] == INTERRUPTED: - assert result[1] == 'KeyboardInterrupt' - raise KeyboardInterrupt # What else? + raise KeyboardInterrupt + if result[0] == CHILD_ERROR: + raise Exception("Child error on {}: {}".format(test, result[1])) test_index += 1 except KeyboardInterrupt: interrupted = True @@ -632,13 +707,14 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, if trace: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. - tracer.runctx('runtest(test, verbose, quiet)', + tracer.runctx('runtest(test, verbose, quiet, timeout=timeout)', globals=globals(), locals=vars()) else: try: result = runtest(test, verbose, quiet, huntrleaks, debug, output_on_failure=verbose3, - failfast=failfast, match_tests=match_tests) + timeout=timeout, failfast=failfast, + match_tests=match_tests) accumulate_result(test, result) except KeyboardInterrupt: interrupted = True @@ -709,7 +785,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, sys.stdout.flush() try: verbose = True - ok = runtest(test, True, quiet, huntrleaks, debug) + ok = runtest(test, True, quiet, huntrleaks, debug, timeout=timeout) except KeyboardInterrupt: # print a newline separate from the ^C print() @@ -734,6 +810,8 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, sys.exit(len(bad) > 0 or interrupted) +# small set of tests to determine if we have a basically functioning interpreter +# (i.e. if any of these fail, then anything else is likely to follow) STDTESTS = [ 'test_grammar', 'test_opcodes', @@ -744,12 +822,11 @@ STDTESTS = [ 'test_unittest', 'test_doctest', 'test_doctest2', + 'test_support' ] -NOTTESTS = { - 'test_future1', - 'test_future2', -} +# set of tests that we don't want to be executed when using regrtest +NOTTESTS = set() def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): """Return a list of all applicable test modules.""" @@ -758,9 +835,9 @@ def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): tests = [] others = set(stdtests) | nottests for name in names: - modname, ext = os.path.splitext(name) - if modname[:5] == "test_" and ext == ".py" and modname not in others: - tests.append(modname) + mod, ext = os.path.splitext(name) + if mod[:5] == "test_" and ext in (".py", "") and mod not in others: + tests.append(mod) return stdtests + sorted(tests) # We do not use a generator so multiple threads can call next(). @@ -785,17 +862,14 @@ class MultiprocessTests(object): def replace_stdout(): """Set stdout encoder error handler to backslashreplace (as stderr error handler) to avoid UnicodeEncodeError when printing a traceback""" - if os.name == "nt": - # Replace sys.stdout breaks the stdout newlines on Windows: issue #8533 - return - import atexit stdout = sys.stdout sys.stdout = open(stdout.fileno(), 'w', encoding=stdout.encoding, errors="backslashreplace", - closefd=False) + closefd=False, + newline='\n') def restore_stdout(): sys.stdout.close() @@ -804,7 +878,8 @@ def replace_stdout(): def runtest(test, verbose, quiet, huntrleaks=False, debug=False, use_resources=None, - output_on_failure=False, failfast=False, match_tests=None): + output_on_failure=False, failfast=False, match_tests=None, + timeout=None): """Run a single test. test -- the name of the test @@ -814,6 +889,8 @@ def runtest(test, verbose, quiet, huntrleaks -- run multiple times to test for leaks; requires a debug build; a triple corresponding to -R's three arguments output_on_failure -- if true, display test output on failure + timeout -- dump the traceback and exit if a test takes more than + timeout seconds Returns one of the test result constants: INTERRUPTED KeyboardInterrupt when run under -j @@ -826,6 +903,9 @@ def runtest(test, verbose, quiet, if use_resources is not None: support.use_resources = use_resources + use_timeout = (timeout is not None) + if use_timeout: + faulthandler.dump_traceback_later(timeout, exit=True) try: support.match_tests = match_tests if failfast: @@ -864,6 +944,8 @@ def runtest(test, verbose, quiet, display_failure=not verbose) return result finally: + if use_timeout: + faulthandler.cancel_dump_traceback_later() cleanup_test_droppings(test, verbose) runtest.stringio = None @@ -909,10 +991,10 @@ class saved_test_environment: resources = ('sys.argv', 'cwd', 'sys.stdin', 'sys.stdout', 'sys.stderr', 'os.environ', 'sys.path', 'sys.path_hooks', '__import__', 'warnings.filters', 'asyncore.socket_map', - 'logging._handlers', 'logging._handlerList', - 'shutil.archive_formats', 'shutil.unpack_formats', + 'logging._handlers', 'logging._handlerList', 'sys.gettrace', 'sys.warnoptions', 'threading._dangling', 'multiprocessing.process._dangling', + 'sysconfig._CONFIG_VARS', 'sysconfig._INSTALL_SCHEMES', 'support.TESTFN', ) @@ -961,6 +1043,11 @@ class saved_test_environment: sys.path_hooks = saved_hooks[1] sys.path_hooks[:] = saved_hooks[2] + def get_sys_gettrace(self): + return sys.gettrace() + def restore_sys_gettrace(self, trace_fxn): + sys.settrace(trace_fxn) + def get___import__(self): return builtins.__import__ def restore___import__(self, import_): @@ -1044,6 +1131,24 @@ class saved_test_environment: multiprocessing.process._dangling.clear() multiprocessing.process._dangling.update(saved) + def get_sysconfig__CONFIG_VARS(self): + # make sure the dict is initialized + sysconfig.get_config_var('prefix') + return (id(sysconfig._CONFIG_VARS), sysconfig._CONFIG_VARS, + dict(sysconfig._CONFIG_VARS)) + def restore_sysconfig__CONFIG_VARS(self, saved): + sysconfig._CONFIG_VARS = saved[1] + sysconfig._CONFIG_VARS.clear() + sysconfig._CONFIG_VARS.update(saved[2]) + + def get_sysconfig__INSTALL_SCHEMES(self): + return (id(sysconfig._INSTALL_SCHEMES), sysconfig._INSTALL_SCHEMES, + sysconfig._INSTALL_SCHEMES.copy()) + def restore_sysconfig__INSTALL_SCHEMES(self, saved): + sysconfig._INSTALL_SCHEMES = saved[1] + sysconfig._INSTALL_SCHEMES.clear() + sysconfig._INSTALL_SCHEMES.update(saved[2]) + def get_support_TESTFN(self): if os.path.isfile(support.TESTFN): result = 'f' @@ -1108,14 +1213,15 @@ def runtest_inner(test, verbose, quiet, start_time = time.time() the_package = __import__(abstest, globals(), locals(), []) the_module = getattr(the_package, test) - # Old tests run to completion simply as a side-effect of - # being imported. For tests based on unittest or doctest, - # explicitly invoke their test_main() function (if it exists). - indirect_test = getattr(the_module, "test_main", None) - if indirect_test is not None: - indirect_test() + # If the test has a test_main, that will run the appropriate + # tests. If not, use normal unittest test loading. + test_runner = getattr(the_module, "test_main", None) + if test_runner is None: + tests = unittest.TestLoader().loadTestsFromModule(the_module) + test_runner = lambda: support.run_unittest(tests) + test_runner() if huntrleaks: - refleak = dash_R(the_module, test, indirect_test, + refleak = dash_R(the_module, test, test_runner, huntrleaks) test_time = time.time() - start_time except support.ResourceDenied as msg: @@ -1198,7 +1304,8 @@ def dash_R(the_module, test, indirect_test, huntrleaks): False if the test didn't leak references; True if we detected refleaks. """ # This code is hackish and inelegant, but it seems to do the job. - import copyreg, _abcoll + import copyreg + import collections.abc if not hasattr(sys, 'gettotalrefcount'): raise Exception("Tracking reference leaks requires a debug build " @@ -1215,7 +1322,7 @@ def dash_R(the_module, test, indirect_test, huntrleaks): else: zdc = zipimport._zip_directory_cache.copy() abcs = {} - for abc in [getattr(_abcoll, a) for a in _abcoll.__all__]: + for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: if not isabstract(abc): continue for obj in abc.__subclasses__() + [abc]: @@ -1261,7 +1368,7 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): import gc, copyreg import _strptime, linecache import urllib.parse, urllib.request, mimetypes, doctest - import struct, filecmp, _abcoll + import struct, filecmp, collections.abc from distutils.dir_util import _path_created from weakref import WeakSet @@ -1288,7 +1395,7 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): sys._clear_type_cache() # Clear ABC registries, restoring previously saved ABC registries. - for abc in [getattr(_abcoll, a) for a in _abcoll.__all__]: + for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: if not isabstract(abc): continue for obj in abc.__subclasses__() + [abc]: @@ -1324,10 +1431,15 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): # Collect cyclic trash. gc.collect() -def warm_char_cache(): +def warm_caches(): + # char cache s = bytes(range(256)) for i in range(256): s[i:i+1] + # unicode cache + x = [chr(i) for i in range(256)] + # int cache + x = list(range(-5, 257)) def findtestdir(path=None): return path or os.path.dirname(__file__) or os.curdir @@ -1374,13 +1486,14 @@ def printlist(x, width=70, indent=4): # Tests that are expected to be skipped everywhere except on one platform # are also handled separately. -_expectations = { - 'win32': +_expectations = ( + ('win32', """ test__locale test_crypt test_curses test_dbm + test_devpoll test_fcntl test_fork1 test_epoll @@ -1403,15 +1516,16 @@ _expectations = { test_threadsignals test_wait3 test_wait4 - """, - 'linux2': + """), + ('linux', """ test_curses + test_devpoll test_largefile test_kqueue test_ossaudiodev - """, - 'unixware7': + """), + ('unixware', """ test_epoll test_largefile @@ -1421,8 +1535,8 @@ _expectations = { test_pyexpat test_sax test_sundry - """, - 'openunix8': + """), + ('openunix', """ test_epoll test_largefile @@ -1432,8 +1546,8 @@ _expectations = { test_pyexpat test_sax test_sundry - """, - 'sco_sv3': + """), + ('sco_sv', """ test_asynchat test_fork1 @@ -1452,11 +1566,12 @@ _expectations = { test_threaded_import test_threadedtempfile test_threading - """, - 'darwin': + """), + ('darwin', """ test__locale test_curses + test_devpoll test_epoll test_dbm_gnu test_gdb @@ -1465,8 +1580,8 @@ _expectations = { test_minidom test_ossaudiodev test_poll - """, - 'sunos5': + """), + ('sunos', """ test_curses test_dbm @@ -1477,8 +1592,8 @@ _expectations = { test_openpty test_zipfile test_zlib - """, - 'hp-ux11': + """), + ('hp-ux', """ test_curses test_epoll @@ -1493,11 +1608,12 @@ _expectations = { test_sax test_zipfile test_zlib - """, - 'cygwin': + """), + ('cygwin', """ test_curses test_dbm + test_devpoll test_epoll test_ioctl test_kqueue @@ -1505,8 +1621,8 @@ _expectations = { test_locale test_ossaudiodev test_socketserver - """, - 'os2emx': + """), + ('os2emx', """ test_audioop test_curses @@ -1519,9 +1635,10 @@ _expectations = { test_pty test_resource test_signal - """, - 'freebsd4': + """), + ('freebsd', """ + test_devpoll test_epoll test_dbm_gnu test_locale @@ -1536,8 +1653,8 @@ _expectations = { test_timeout test_urllibnet test_multiprocessing - """, - 'aix5': + """), + ('aix', """ test_bz2 test_epoll @@ -1551,10 +1668,11 @@ _expectations = { test_ttk_textonly test_zipimport test_zlib - """, - 'openbsd3': + """), + ('openbsd', """ test_ctypes + test_devpoll test_epoll test_dbm_gnu test_locale @@ -1566,11 +1684,12 @@ _expectations = { test_ttk_guionly test_ttk_textonly test_multiprocessing - """, - 'netbsd3': + """), + ('netbsd', """ test_ctypes test_curses + test_devpoll test_epoll test_dbm_gnu test_locale @@ -1581,12 +1700,8 @@ _expectations = { test_ttk_guionly test_ttk_textonly test_multiprocessing - """, -} -_expectations['freebsd5'] = _expectations['freebsd4'] -_expectations['freebsd6'] = _expectations['freebsd4'] -_expectations['freebsd7'] = _expectations['freebsd4'] -_expectations['freebsd8'] = _expectations['freebsd4'] + """), +) class _ExpectedSkips: def __init__(self): @@ -1594,9 +1709,13 @@ class _ExpectedSkips: from test import test_timeout self.valid = False - if sys.platform in _expectations: - s = _expectations[sys.platform] - self.expected = set(s.split()) + expected = None + for item in _expectations: + if sys.platform.startswith(item[0]): + expected = item[1] + break + if expected is not None: + self.expected = set(expected.split()) # These are broken tests, for now skipped on every platform. # XXX Fix these! @@ -1656,9 +1775,8 @@ def _make_temp_dir_for_build(TEMPDIR): TEMPDIR = os.path.abspath(TEMPDIR) try: os.mkdir(TEMPDIR) - except OSError as e: - if e.errno != errno.EEXIST: - raise + except FileExistsError: + pass # Define a writable temp dir that will be used as cwd while running # the tests. The name of the dir includes the pid to allow parallel |