update to current master againwip/T13904

1 files changed, 83 insertions, 67 deletions

diff --git a/testsuite/driver/testlib.py b/testsuite/driver/testlib.py
index 26e3d17679..ff6a8c8e74 100644
--- a/testsuite/driver/testlib.py
+++ b/testsuite/driver/testlib.py
@@ -3,13 +3,9 @@
 # (c) Simon Marlow 2002
 #
 
-from __future__ import print_function
-
 import io
 import shutil
 import os
-import errno
-import string
 import re
 import traceback
 import time
@@ -18,33 +14,30 @@ import copy
 import glob
 import sys
 from math import ceil, trunc
+from pathlib import PurePath
 import collections
 import subprocess
 
-from testglobals import *
-from testutil import *
+from testglobals import config, ghc_env, default_testopts, brokens, t
+from testutil import strip_quotes, lndir, link_or_copy_file
 extra_src_files = {'T4198': ['exitminus1.c']} # TODO: See #12223
 
+global pool_sema
 if config.use_threads:
     import threading
-    try:
-        import thread
-    except ImportError: # Python 3
-        import _thread as thread
+    pool_sema = threading.BoundedSemaphore(value=config.threads)
 
 global wantToStop
 wantToStop = False
 
-global pool_sema
-if config.use_threads:
-    pool_sema = threading.BoundedSemaphore(value=config.threads)
-
 def stopNow():
     global wantToStop
     wantToStop = True
+
 def stopping():
     return wantToStop
 
+
 # Options valid for the current test only (these get reset to
 # testdir_testopts after each test).
 
@@ -91,7 +84,7 @@ def normal( name, opts ):
     return;
 
 def skip( name, opts ):
-    opts.skip = 1
+    opts.skip = True
 
 def expect_fail( name, opts ):
     # The compiler, testdriver, OS or platform is missing a certain
@@ -143,7 +136,8 @@ def _reqlib( name, opts, lib ):
         cmd = strip_quotes(config.ghc_pkg)
         p = subprocess.Popen([cmd, '--no-user-package-db', 'describe', lib],
                              stdout=subprocess.PIPE,
-                             stderr=subprocess.PIPE)
+                             stderr=subprocess.PIPE,
+                             env=ghc_env)
         # read from stdout and stderr to avoid blocking due to
         # buffers filling
         p.communicate()
@@ -209,7 +203,6 @@ def _expect_broken_for( name, opts, bug, ways ):
     opts.expect_fail_for = ways
 
 def record_broken(name, opts, bug):
-    global brokens
     me = (bug, opts.testdir, name)
     if not me in brokens:
         brokens.append(me)
@@ -412,6 +405,12 @@ def compiler_profiled( ):
 def compiler_debugged( ):
     return config.compiler_debugged
 
+def have_gdb( ):
+    return config.have_gdb
+
+def have_readelf( ):
+    return config.have_readelf
+
 # ---
 
 def high_memory_usage(name, opts):
@@ -424,19 +423,19 @@ def multi_cpu_race(name, opts):
 
 # ---
 def literate( name, opts ):
-    opts.literate = 1;
+    opts.literate = True
 
 def c_src( name, opts ):
-    opts.c_src = 1;
+    opts.c_src = True
 
 def objc_src( name, opts ):
-    opts.objc_src = 1;
+    opts.objc_src = True
 
 def objcpp_src( name, opts ):
-    opts.objcpp_src = 1;
+    opts.objcpp_src = True
 
 def cmm_src( name, opts ):
-    opts.cmm_src = 1;
+    opts.cmm_src = True
 
 def outputdir( odir ):
     return lambda name, opts, d=odir: _outputdir(name, opts, d)
@@ -454,12 +453,6 @@ def _pre_cmd( name, opts, cmd ):
 
 # ----
 
-def clean_cmd( cmd ):
-    # TODO. Remove all calls to clean_cmd.
-    return lambda _name, _opts: None
-
-# ----
-
 def cmd_prefix( prefix ):
     return lambda name, opts, p=prefix: _cmd_prefix(name, opts, prefix)
 
@@ -497,7 +490,6 @@ def no_check_hp(name, opts):
 
 def filter_stdout_lines( regex ):
     """ Filter lines of stdout with the given regular expression """
-    import re
     def f( name, opts ):
         _normalise_fun(name, opts, lambda s: '\n'.join(re.findall(regex, s)))
     return f
@@ -520,6 +512,19 @@ def normalise_errmsg_fun( *fs ):
 def _normalise_errmsg_fun( name, opts, *fs ):
     opts.extra_errmsg_normaliser =  join_normalisers(opts.extra_errmsg_normaliser, fs)
 
+def check_errmsg(needle):
+    def norm(str):
+        if needle in str:
+            return "%s contained in -ddump-simpl\n" % needle
+        else:
+            return "%s not contained in -ddump-simpl\n" % needle
+    return normalise_errmsg_fun(norm)
+
+def grep_errmsg(needle):
+    def norm(str):
+        return "".join(filter(lambda l: re.search(needle, l), str.splitlines(True)))
+    return normalise_errmsg_fun(norm)
+
 def normalise_whitespace_fun(f):
     return lambda name, opts: _normalise_whitespace_fun(name, opts, f)
 
@@ -607,8 +612,9 @@ def newTestDir(tempdir, dir):
 testdir_suffix = '.run'
 
 def _newTestDir(name, opts, tempdir, dir):
+    testdir = os.path.join('', *(p for p in PurePath(dir).parts if p != '..'))
     opts.srcdir = os.path.join(os.getcwd(), dir)
-    opts.testdir = os.path.join(tempdir, dir, name + testdir_suffix)
+    opts.testdir = os.path.join(tempdir, testdir, name + testdir_suffix)
     opts.compiler_always_flags = config.compiler_always_flags
 
 # -----------------------------------------------------------------------------
@@ -796,7 +802,7 @@ def do_test(name, way, func, args, files):
     full_name = name + '(' + way + ')'
 
     if_verbose(2, "=====> {0} {1} of {2} {3}".format(
-        full_name, t.total_tests, len(allTestNames), 
+        full_name, t.total_tests, len(allTestNames),
         [len(t.unexpected_passes),
          len(t.unexpected_failures),
          len(t.framework_failures)]))
@@ -848,6 +854,7 @@ def do_test(name, way, func, args, files):
 
         if exit_code != 0:
             framework_fail(name, way, 'pre_cmd failed: {0}'.format(exit_code))
+            if_verbose(1, '** pre_cmd was "{0}". Running trace'.format(override_options(opts.pre_cmd)))
 
     result = func(*[name,way] + args)
 
@@ -863,6 +870,7 @@ def do_test(name, way, func, args, files):
 
     if passFail == 'pass':
         if _expect_pass(way):
+            t.expected_passes.append((directory, name, way))
             t.n_expected_passes += 1
         else:
             if_verbose(1, '*** unexpected pass for %s' % full_name)
@@ -948,8 +956,9 @@ def ghci_script( name, way, script):
 
     # We pass HC and HC_OPTS as environment variables, so that the
     # script can invoke the correct compiler by using ':! $HC $HC_OPTS'
-    cmd = ('HC={{compiler}} HC_OPTS="{flags}" {{compiler}} {flags} {way_flags}'
+    cmd = ('HC={{compiler}} HC_OPTS="{flags}" {{compiler}} {way_flags} {flags}'
           ).format(flags=flags, way_flags=way_flags)
+      # NB: put way_flags before flags so that flags in all.T can overrie others
 
     getTestOpts().stdin = script
     return simple_run( name, way, cmd, getTestOpts().extra_run_opts )
@@ -964,19 +973,19 @@ def compile_fail( name, way, extra_hc_opts ):
     return do_compile( name, way, 1, '', [], extra_hc_opts )
 
 def backpack_typecheck( name, way, extra_hc_opts ):
-    return do_compile( name, way, 0, '', [], "-fno-code -fwrite-interface " + extra_hc_opts, backpack=1 )
+    return do_compile( name, way, 0, '', [], "-fno-code -fwrite-interface " + extra_hc_opts, backpack=True )
 
 def backpack_typecheck_fail( name, way, extra_hc_opts ):
-    return do_compile( name, way, 1, '', [], "-fno-code -fwrite-interface " + extra_hc_opts, backpack=1 )
+    return do_compile( name, way, 1, '', [], "-fno-code -fwrite-interface " + extra_hc_opts, backpack=True )
 
 def backpack_compile( name, way, extra_hc_opts ):
-    return do_compile( name, way, 0, '', [], extra_hc_opts, backpack=1 )
+    return do_compile( name, way, 0, '', [], extra_hc_opts, backpack=True )
 
 def backpack_compile_fail( name, way, extra_hc_opts ):
-    return do_compile( name, way, 1, '', [], extra_hc_opts, backpack=1 )
+    return do_compile( name, way, 1, '', [], extra_hc_opts, backpack=True )
 
 def backpack_run( name, way, extra_hc_opts ):
-    return compile_and_run__( name, way, '', [], extra_hc_opts, backpack=1 )
+    return compile_and_run__( name, way, '', [], extra_hc_opts, backpack=True )
 
 def multimod_compile( name, way, top_mod, extra_hc_opts ):
     return do_compile( name, way, 0, top_mod, [], extra_hc_opts )
@@ -1206,7 +1215,7 @@ def simple_build(name, way, extra_hc_opts, should_fail, top_mod, link, addsuf, b
         if config.verbose >= 1 and _expect_pass(way):
             print('Compile failed (exit code {0}) errors were:'.format(exit_code))
             actual_stderr_path = in_testdir(name, 'comp.stderr')
-            if_verbose_dump(1, actual_stderr_path)
+            dump_file(actual_stderr_path)
 
     # ToDo: if the sub-shell was killed by ^C, then exit
 
@@ -1415,7 +1424,7 @@ def stdout_ok(name, way):
                           expected_stdout_file, actual_stdout_file)
 
 def dump_stdout( name ):
-    with open(in_testdir(name, 'run.stdout')) as f:
+    with open(in_testdir(name, 'run.stdout'), encoding='utf8') as f:
         str = f.read().strip()
         if str:
             print("Stdout (", name, "):")
@@ -1431,7 +1440,7 @@ def stderr_ok(name, way):
                           whitespace_normaliser=normalise_whitespace)
 
 def dump_stderr( name ):
-    with open(in_testdir(name, 'run.stderr')) as f:
+    with open(in_testdir(name, 'run.stderr'), encoding='utf8') as f:
         str = f.read().strip()
         if str:
             print("Stderr (", name, "):")
@@ -1553,7 +1562,7 @@ def compare_outputs(way, kind, normaliser, expected_file, actual_file,
 
     # See Note [Output comparison].
     if whitespace_normaliser(expected_str) == whitespace_normaliser(actual_str):
-        return 1
+        return True
     else:
         if config.verbose >= 1 and _expect_pass(way):
             print('Actual ' + kind + ' output differs from expected:')
@@ -1568,29 +1577,39 @@ def compare_outputs(way, kind, normaliser, expected_file, actual_file,
             # See Note [Output comparison].
             r = runCmd('diff -uw "{0}" "{1}"'.format(expected_normalised_path,
                                                         actual_normalised_path),
-                        print_output = 1)
+                        print_output=True)
 
             # If for some reason there were no non-whitespace differences,
             # then do a full diff
             if r == 0:
                 r = runCmd('diff -u "{0}" "{1}"'.format(expected_normalised_path,
                                                            actual_normalised_path),
-                           print_output = 1)
+                           print_output=True)
 
         if config.accept and (getTestOpts().expect == 'fail' or
                               way in getTestOpts().expect_fail_for):
             if_verbose(1, 'Test is expected to fail. Not accepting new output.')
-            return 0
+            return False
         elif config.accept and actual_raw:
-            if_verbose(1, 'Accepting new output.')
+            if config.accept_platform:
+                if_verbose(1, 'Accepting new output for platform "'
+                              + config.platform + '".')
+                expected_path += '-' + config.platform
+            elif config.accept_os:
+                if_verbose(1, 'Accepting new output for os "'
+                              + config.os + '".')
+                expected_path += '-' + config.os
+            else:
+                if_verbose(1, 'Accepting new output.')
+
             write_file(expected_path, actual_raw)
-            return 1
+            return True
         elif config.accept:
             if_verbose(1, 'No output. Deleting "{0}".'.format(expected_path))
             os.remove(expected_path)
-            return 1
+            return True
         else:
-            return 0
+            return False
 
 # Note [Output comparison]
 #
@@ -1610,7 +1629,7 @@ def compare_outputs(way, kind, normaliser, expected_file, actual_file,
 
 def normalise_whitespace( str ):
     # Merge contiguous whitespace characters into a single space.
-    return ' '.join(w for w in str.split())
+    return ' '.join(str.split())
 
 callSite_re = re.compile(r', called at (.+):[\d]+:[\d]+ in [\w\-\.]+:')
 
@@ -1730,6 +1749,7 @@ def normalise_prof (str):
 
 def normalise_slashes_( str ):
     str = re.sub('\\\\', '/', str)
+    str = re.sub('//', '/', str)
     return str
 
 def normalise_exe_( str ):
@@ -1773,15 +1793,14 @@ def if_verbose( n, s ):
     if config.verbose >= n:
         print(s)
 
-def if_verbose_dump( n, f ):
-    if config.verbose >= n:
-        try:
-            with io.open(f) as file:
-                print(file.read())
-        except Exception:
-            print('')
+def dump_file(f):
+    try:
+        with io.open(f) as file:
+            print(file.read())
+    except Exception:
+        print('')
 
-def runCmd(cmd, stdin=None, stdout=None, stderr=None, timeout_multiplier=1.0, print_output=0):
+def runCmd(cmd, stdin=None, stdout=None, stderr=None, timeout_multiplier=1.0, print_output=False):
     timeout_prog = strip_quotes(config.timeout_prog)
     timeout = str(int(ceil(config.timeout * timeout_multiplier)))
 
@@ -1789,9 +1808,6 @@ def runCmd(cmd, stdin=None, stdout=None, stderr=None, timeout_multiplier=1.0, pr
     cmd = cmd.format(**config.__dict__)
     if_verbose(3, cmd + ('< ' + os.path.basename(stdin) if stdin else ''))
 
-    # declare the buffers to a default
-    stdin_buffer  = None
-
     stdin_file = io.open(stdin, 'rb') if stdin else None
     stdout_buffer = b''
     stderr_buffer = b''
@@ -1809,13 +1825,14 @@ def runCmd(cmd, stdin=None, stdout=None, stderr=None, timeout_multiplier=1.0, pr
         r = subprocess.Popen([timeout_prog, timeout, cmd],
                              stdin=stdin_file,
                              stdout=subprocess.PIPE,
-                             stderr=hStdErr)
+                             stderr=hStdErr,
+                             env=ghc_env)
 
         stdout_buffer, stderr_buffer = r.communicate()
     finally:
         if stdin_file:
             stdin_file.close()
-        if config.verbose >= 1 and print_output >= 1:
+        if config.verbose >= 1 and print_output:
             if stdout_buffer:
                 sys.stdout.buffer.write(stdout_buffer)
             if stderr_buffer:
@@ -1848,7 +1865,7 @@ def gsNotWorking():
     print("GhostScript not available for hp2ps tests")
 
 global gs_working
-gs_working = 0
+gs_working = False
 if config.have_profiling:
   if config.gs != '':
     resultGood = runCmd(genGSCmd(config.confdir + '/good.ps'));
@@ -1857,7 +1874,7 @@ if config.have_profiling:
                                    ' >/dev/null 2>&1')
         if resultBad != 0:
             print("GhostScript available for hp2ps tests")
-            gs_working = 1;
+            gs_working = True
         else:
             gsNotWorking();
     else:
@@ -1897,7 +1914,7 @@ def in_srcdir(name, suffix=''):
 
 # Finding the sample output.  The filename is of the form
 #
-#   <test>.stdout[-ws-<wordsize>][-<platform>]
+#   <test>.stdout[-ws-<wordsize>][-<platform>|-<os>]
 #
 def find_expected_file(name, suff):
     basename = add_suffix(name, suff)
@@ -1914,7 +1931,6 @@ def find_expected_file(name, suff):
 
 if config.msys:
     import stat
-    import time
     def cleanup():
         testdir = getTestOpts().testdir
         max_attempts = 5
@@ -1968,7 +1984,7 @@ def findTFiles(roots):
     for root in roots:
         for path, dirs, files in os.walk(root, topdown=True):
             # Never pick up .T files in uncleaned .run directories.
-            dirs[:] = [dir for dir in sorted(dirs)  
+            dirs[:] = [dir for dir in sorted(dirs)
                            if not dir.endswith(testdir_suffix)]
             for filename in files:
                 if filename.endswith('.T'):