diff options
Diffstat (limited to 'testsuite/driver')
| -rw-r--r-- | testsuite/driver/runtests.py | 50 | ||||
| -rw-r--r-- | testsuite/driver/testglobals.py | 6 | ||||
| -rw-r--r-- | testsuite/driver/testlib.py | 48 |
3 files changed, 73 insertions, 31 deletions
diff --git a/testsuite/driver/runtests.py b/testsuite/driver/runtests.py index 39689c6255..33b432fd6d 100644 --- a/testsuite/driver/runtests.py +++ b/testsuite/driver/runtests.py @@ -276,10 +276,24 @@ else: # set stdout to unbuffered (is this the best way to do it?) sys.stdout = os.fdopen(sys.__stdout__.fileno(), "w", 0) -tempdir = tempfile.mkdtemp('', 'ghctest-') +if config.local: + tempdir = '' +else: + # See note [Running tests in /tmp] + tempdir = tempfile.mkdtemp('', 'ghctest-') + + # opts.testdir should be quoted when used, to make sure the testsuite + # keeps working when it contains backward slashes, for example from + # using os.path.join. Windows native and mingw* python + # (/mingw64/bin/python) set `os.path.sep = '\\'`, while msys2 python + # (/bin/python, /usr/bin/python or /usr/local/bin/python) sets + # `os.path.sep = '/'`. + # To catch usage of unquoted opts.testdir early, insert some spaces into + # tempdir. + tempdir = os.path.join(tempdir, 'test spaces') def cleanup_and_exit(exitcode): - if config.cleanup: + if config.cleanup and tempdir: shutil.rmtree(tempdir, ignore_errors=True) exit(exitcode) @@ -334,3 +348,35 @@ else: summary(t, open(config.summary_file, 'w')) cleanup_and_exit(0) + +# Note [Running tests in /tmp] +# +# Use LOCAL=0 to run tests in /tmp, to catch tests that use files from +# the source directory without copying them to the test directory first. +# +# As an example, take a run_command test with a Makefile containing +# `$(TEST_HC) ../Foo.hs`. GHC will now create the output files Foo.o and +# Foo.hi in the source directory. There are 2 problems with this: +# * Output files in the source directory won't get cleaned up automatically. +# * Two tests might (over)write the same output file. +# +# Tests that only fail when run concurrently with other tests are the +# worst, so we try to catch them early by enabling LOCAL=0 in validate. +# +# Adding -outputdir='.' to TEST_HC_OPTS would help a bit, but it requires +# making changes to quite a few tests. The problem is that +# `$(TEST_HC) ../Foo.hs -outputdir=.` with Foo.hs containing +# `module Main where` does not produce Foo.o, as it would without +# -outputdir, but Main.o. See [1]. +# +# Using -outputdir='.' is not foolproof anyway, since it does not change +# the destination of the final executable (Foo.exe). +# +# Another hardening method that could be tried is to `chmod -w` the +# source directory. +# +# By default we set LOCAL=1, because it makes it easier to inspect the +# test directory while working on a new test. +# +# [1] +# https://downloads.haskell.org/~ghc/8.0.1/docs/html/users_guide/separate_compilation.html#output-files diff --git a/testsuite/driver/testglobals.py b/testsuite/driver/testglobals.py index 17aa6d36b8..d08141f251 100644 --- a/testsuite/driver/testglobals.py +++ b/testsuite/driver/testglobals.py @@ -274,6 +274,12 @@ class TestOptions: self.compile_timeout_multiplier = 1.0 self.run_timeout_multiplier = 1.0 + self.cleanup = True + + # Sould we run tests in a local subdirectory (<testname>-run) or + # in temporary directory in /tmp? See Note [Running tests in /tmp]. + self.local = True + # The default set of options global default_testopts default_testopts = TestOptions() diff --git a/testsuite/driver/testlib.py b/testsuite/driver/testlib.py index f6db8288fa..32b6951fe9 100644 --- a/testsuite/driver/testlib.py +++ b/testsuite/driver/testlib.py @@ -546,25 +546,6 @@ def executeSetups(fs, name, opts): # The current directory of tests def newTestDir(tempdir, dir): - # opts.testdir should be quoted when used, to make sure the testsuite - # keeps working when it contains backward slashes, for example from - # using os.path.join. Windows native and mingw* python - # (/mingw64/bin/python) set `os.path.sep = '\\'`, while msys2 python - # (/bin/python, /usr/bin/python or /usr/local/bin/python) sets - # `os.path.sep = '/'`. - # To catch usage of unquoted opts.testdir early, insert some spaces into - # tempdir. - tempdir = os.path.join(tempdir, 'test spaces') - - # Hack. A few tests depend on files in ancestor directories - # (e.g. extra_files(['../../../../libraries/base/dist-install/haddock.t'])) - # Make sure tempdir is sufficiently "deep", such that copying/linking those - # files won't cause any problems. - # - # If you received a framework failure about adding an extra level: - # * add one extra '../' to the startswith('../../../../../') in do_test - # * add one more number here: - tempdir = os.path.join(tempdir, '1', '2', '3') global thisdir_settings # reset the options for this test directory @@ -572,10 +553,12 @@ def newTestDir(tempdir, dir): return _newTestDir(name, opts, tempdir, dir) thisdir_settings = settings +# Should be equal to entry in toplevel .gitignore. +testdir_suffix = '.run' def _newTestDir(name, opts, tempdir, dir): opts.srcdir = os.path.join(os.getcwd(), dir) - opts.testdir = os.path.join(tempdir, dir, name) + opts.testdir = os.path.join(tempdir, dir, name + testdir_suffix) opts.compiler_always_flags = config.compiler_always_flags # ----------------------------------------------------------------------------- @@ -718,13 +701,10 @@ def test_common_work (name, opts, func, args): # this seems to be necessary for only about 10% of all # tests). files = set((f for f in os.listdir(opts.srcdir) - if f.startswith(name))) + if f.startswith(name) and + not f.endswith(testdir_suffix))) for filename in (opts.extra_files + extra_src_files.get(name, [])): - if filename.startswith('../../../../../../'): - framework_fail(name, 'whole-test', - 'add extra level to testlib.py:newTestDir for: ' + filename) - - elif filename.startswith('/'): + if filename.startswith('/'): framework_fail(name, 'whole-test', 'no absolute paths in extra_files please: ' + filename) @@ -790,8 +770,18 @@ def do_test(name, way, func, args, files): # would otherwise (accidentally) write to the same output file. # It also makes it easier to keep the testsuite clean. - for filename in files: - src = in_srcdir(filename) + for extra_file in files: + src = in_srcdir(extra_file) + if extra_file.startswith('..'): + # In case the extra_file is a file in an ancestor + # directory (e.g. extra_files(['../shell.hs'])), make + # sure it is copied to the test directory + # (testdir/shell.hs), instead of ending up somewhere + # else in the tree (testdir/../shell.hs) + filename = os.path.basename(extra_file) + else: + filename = extra_file + assert not '..' in filename # no funny stuff (foo/../../bar) dst = in_testdir(filename) if os.path.isfile(src): @@ -821,7 +811,7 @@ def do_test(name, way, func, args, files): pass else: framework_fail(name, way, - 'extra_file does not exist: ' + filename) + 'extra_file does not exist: ' + extra_file) if not files: # Always create the testdir, even when no files were copied |