Driver Rework Patch

This patch comprises of four different but closely related ideas. The
net result is fixing a large number of open issues with the driver
whilst making it simpler to understand.

1. Use the hash of the source file to determine whether the source file
has changed or not. This makes the recompilation checking more robust to
modern build systems which are liable to copy files around changing
their modification times.

2. Remove the concept of a "stable module", a stable module was one
where the object file was older than the source file, and all transitive
dependencies were also stable. Now we don't rely on the modification
time of the source file, the notion of stability is moot.

3. Fix TH/plugin recompilation after the removal of stable modules. The
TH recompilation check used to rely on stable modules. Now there is a
uniform and simple way, we directly track the linkables which were
loaded into the interpreter whilst compiling a module. This is an
over-approximation but more robust wrt package dependencies changing.

4. Fix recompilation checking for dynamic object files. Now we actually
check if the dynamic object file exists when compiling with -dynamic-too

Fixes #19774 #19771 #19758 #17434 #11556 #9121 #8211 #16495 #7277 #16093

4 files changed, 129 insertions, 109 deletions

diff --git a/testsuite/tests/perf/compiler/Makefile b/testsuite/tests/perf/compiler/Makefile
index 66597883b6..20f5704450 100644
--- a/testsuite/tests/perf/compiler/Makefile
+++ b/testsuite/tests/perf/compiler/Makefile
@@ -12,3 +12,7 @@ T11068:
 	'$(TEST_HC)' $(TEST_HC_OPTS) -c -O T11068a.hs
 	'$(TEST_HC)' $(TEST_HC_OPTS) -c -O T11068b.hs
 	-'$(TEST_HC)' $(TEST_HC_OPTS) -c -O T11068.hs -ddump-simpl | grep 'Generic'
+
+MultiModulesRecomp:
+	./genMultiLayerModules
+	'$(TEST_HC)' $(TEST_HC_OPTS) -v0 MultiLayerModules.hs
diff --git a/testsuite/tests/perf/compiler/all.T b/testsuite/tests/perf/compiler/all.T
index 04d03e0568..eadf2ff920 100644
--- a/testsuite/tests/perf/compiler/all.T
+++ b/testsuite/tests/perf/compiler/all.T
@@ -276,6 +276,22 @@ test('MultiLayerModules',
      multimod_compile,
      ['MultiLayerModules', '-v0'])
 
+# MultiLayerModules flip flops by 2.5%, depending on the
+# number of modules within GHC (#19293). Just widen the
+# acceptance window until we figured out how to fix it.
+test('MultiLayerModulesRecomp',
+     [ collect_compiler_stats('bytes allocated',3),
+       pre_cmd('$MAKE -s --no-print-directory MultiModulesRecomp'),
+       extra_files(['genMultiLayerModules']),
+       compile_timeout_multiplier(5)
+       # this is _a lot_
+       # but this test has been failing every now and then,
+       # especially on i386. Let's just give it some room
+       # to complete successfully reliably everywhere.
+     ],
+     multimod_compile,
+     ['MultiLayerModules', '-v0'])
+
 test('ManyConstructors',
      [ collect_compiler_stats('bytes allocated',2),
        pre_cmd('./genManyConstructors'),
diff --git a/testsuite/tests/perf/space_leaks/T4029.script b/testsuite/tests/perf/space_leaks/T4029.script
index 91135c9477..f2faa4fd1a 100644
--- a/testsuite/tests/perf/space_leaks/T4029.script
+++ b/testsuite/tests/perf/space_leaks/T4029.script
@@ -102,203 +102,203 @@
 
 -- Load a minimalist module and reload it 99 times
 :load T4029a
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
-:! touch T4029a.hs
+:! echo >> T4029a.hs
 :reload
 
 -- Load a more complex module 10 times
@@ -315,21 +315,21 @@
 
 -- Load a more complex module and reload it 9 times
 :load T4029b
-:! touch T4029b.hs
+:! echo >> T4029b.hs
 :reload
-:! touch T4029b.hs
+:! echo >> T4029b.hs
 :reload
-:! touch T4029b.hs
+:! echo >> T4029b.hs
 :reload
-:! touch T4029b.hs
+:! echo >> T4029b.hs
 :reload
-:! touch T4029b.hs
+:! echo >> T4029b.hs
 :reload
-:! touch T4029b.hs
+:! echo >> T4029b.hs
 :reload
-:! touch T4029b.hs
+:! echo >> T4029b.hs
 :reload
-:! touch T4029b.hs
+:! echo >> T4029b.hs
 :reload
-:! touch T4029b.hs
+:! echo >> T4029b.hs
 :reload
diff --git a/testsuite/tests/perf/space_leaks/all.T b/testsuite/tests/perf/space_leaks/all.T
index deb5ea38e4..f6638555c9 100644
--- a/testsuite/tests/perf/space_leaks/all.T
+++ b/testsuite/tests/perf/space_leaks/all.T
@@ -31,7 +31,7 @@ test('T4018',
      compile_and_run, ['-fno-state-hack'])
 
 test('T4029',
-     [collect_runtime_residency(10),
+     [copy_files, collect_runtime_residency(10),
       ],
      ghci_script,
      ['T4029.script'])