Refactor temp files cleanup

Remove filesToNotIntermediateClean from DynFlags, create a data type
FilesToClean, and change filesToClean in DynFlags to be a FilesToClean.

Modify SysTools.newTempName and the Temporary constructor of
PipelineMonad.PipelineOutput to take a TempFileLifetime, which specifies
whether a temp file should live until the end of GhcMonad.withSession,
or until the next time cleanIntermediateTempFiles is called.

These changes allow the cleaning of intermediate files in GhcMake to be
much more efficient.

HscTypes.hptObjs is removed as it is no longer used.

A new performance test T13701 is added, which passes both with and
without -keep-tmp-files.  The test fails by 25% without the patch, and
passes when -keep-tmp-files is added.

Note that there are still at two hotspots caused by
algorithms quadratic in the number of modules, however neither of them
allocate. They are:

* DriverPipeline.compileOne'.needsLinker
* GhcMake.getModLoop

DriverPipeline.compileOne'.needsLinker is changed slightly to improve
the situation.

I don't like adding these Types to DynFlags, but they need to be seen by
Dynflags, SysTools and PipelineMonad. The alternative seems to be to
create a new module.

Reviewers: austin, hvr, bgamari, dfeuer, niteria, simonmar, erikd

Reviewed By: simonmar

Subscribers: rwbarton, thomie

GHC Trac Issues: #13701

Differential Revision: https://phabricator.haskell.org/D3620

1 files changed, 37 insertions, 44 deletions

diff --git a/compiler/main/GhcMake.hs b/compiler/main/GhcMake.hs
index e11503b9d1..134a0607bc 100644
--- a/compiler/main/GhcMake.hs
+++ b/compiler/main/GhcMake.hs
@@ -1,4 +1,5 @@
 {-# LANGUAGE BangPatterns, CPP, NondecreasingIndentation, ScopedTypeVariables #-}
+{-# LANGUAGE NamedFieldPuns #-}
 {-# OPTIONS_GHC -fno-warn-warnings-deprecations #-}
 -- NB: we specifically ignore deprecations. GHC 7.6 marks the .QSem module as
 -- deprecated, although it became un-deprecated later. As a result, using 7.6
@@ -59,7 +60,6 @@ import Outputable
 import Panic
 import SrcLoc
 import StringBuffer
-import SysTools
 import UniqFM
 import UniqDSet
 import TcBackpack
@@ -68,6 +68,7 @@ import UniqSet
 import Util
 import qualified GHC.LanguageExtensions as LangExt
 import NameEnv
+import FileCleanup
 
 import Data.Either ( rights, partitionEithers )
 import qualified Data.Map as Map
@@ -373,10 +374,7 @@ load' how_much mHscMessage mod_graph = do
         mg = stable_mg ++ unstable_mg
 
     -- clean up between compilations
-    let cleanup hsc_env = intermediateCleanTempFiles (hsc_dflags hsc_env)
-                              (flattenSCCs mg2_with_srcimps)
-                              hsc_env
-
+    let cleanup = cleanCurrentModuleTempFiles . hsc_dflags
     liftIO $ debugTraceMsg dflags 2 (hang (text "Ready for upsweep")
                                2 (ppr mg))
 
@@ -407,7 +405,7 @@ load' how_much mHscMessage mod_graph = do
 
           -- Clean up after ourselves
           hsc_env1 <- getSession
-          liftIO $ intermediateCleanTempFiles dflags modsDone hsc_env1
+          liftIO $ cleanCurrentModuleTempFiles dflags
 
           -- Issue a warning for the confusing case where the user
           -- said '-o foo' but we're not going to do any linking.
@@ -448,29 +446,42 @@ load' how_much mHscMessage mod_graph = do
           let mods_to_zap_names
                  = findPartiallyCompletedCycles modsDone_names
                       mg2_with_srcimps
-          let mods_to_keep
-                 = filter ((`Set.notMember` mods_to_zap_names).ms_mod)
-                      modsDone
-
+          let (mods_to_clean, mods_to_keep) =
+                partition ((`Set.member` mods_to_zap_names).ms_mod) modsDone
           hsc_env1 <- getSession
-          let hpt4 = retainInTopLevelEnvs (map ms_mod_name mods_to_keep)
-                                          (hsc_HPT hsc_env1)
+          let hpt4 = hsc_HPT hsc_env1
+              -- We must change the lifetime to TFL_CurrentModule for any temp
+              -- file created for an element of mod_to_clean during the upsweep.
+              -- These include preprocessed files and object files for loaded
+              -- modules.
+              unneeded_temps = concat
+                [ms_hspp_file : object_files
+                | ModSummary{ms_mod, ms_hspp_file} <- mods_to_clean
+                , let object_files = maybe [] linkableObjs $
+                        lookupHpt hpt4 (moduleName ms_mod)
+                        >>= hm_linkable
+                ]
+          liftIO $
+            changeTempFilesLifetime dflags TFL_CurrentModule unneeded_temps
+          liftIO $ cleanCurrentModuleTempFiles dflags
+
+          let hpt5 = retainInTopLevelEnvs (map ms_mod_name mods_to_keep)
+                                          hpt4
 
           -- Clean up after ourselves
-          liftIO $ intermediateCleanTempFiles dflags mods_to_keep hsc_env1
 
           -- there should be no Nothings where linkables should be, now
           let just_linkables =
                     isNoLink (ghcLink dflags)
                  || allHpt (isJust.hm_linkable)
                         (filterHpt ((== HsSrcFile).mi_hsc_src.hm_iface)
-                                hpt4)
+                                hpt5)
           ASSERT( just_linkables ) do
 
           -- Link everything together
-          linkresult <- liftIO $ link (ghcLink dflags) dflags False hpt4
+          linkresult <- liftIO $ link (ghcLink dflags) dflags False hpt5
 
-          modifySession $ \hsc_env -> hsc_env{ hsc_HPT = hpt4 }
+          modifySession $ \hsc_env -> hsc_env{ hsc_HPT = hpt5 }
           loadFinish Failed linkresult
 
 
@@ -518,23 +529,6 @@ discardIC hsc_env
     this_pkg = thisPackage dflags
     old_name = ic_name old_ic
 
-intermediateCleanTempFiles :: DynFlags -> [ModSummary] -> HscEnv -> IO ()
-intermediateCleanTempFiles dflags summaries hsc_env
- = do notIntermediate <- readIORef (filesToNotIntermediateClean dflags)
-      cleanTempFilesExcept dflags (notIntermediate ++ except)
-  where
-    except =
-          -- Save preprocessed files. The preprocessed file *might* be
-          -- the same as the source file, but that doesn't do any
-          -- harm.
-          map ms_hspp_file summaries ++
-          -- Save object files for loaded modules.  The point of this
-          -- is that we might have generated and compiled a stub C
-          -- file, and in the case of GHCi the object file will be a
-          -- temporary file which we must not remove because we need
-          -- to load/link it later.
-          hptObjs (hsc_HPT hsc_env)
-
 -- | If there is no -o option, guess the name of target executable
 -- by using top-level source file name as a base.
 guessOutputFile :: GhcMonad m => m ()
@@ -927,7 +921,7 @@ parUpsweep n_jobs mHscMessage old_hpt stable_mods cleanup sccs = do
                 -- compilation for that module is finished) without having to
                 -- worry about accidentally deleting a simultaneous compile's
                 -- important files.
-                lcl_files_to_clean <- newIORef []
+                lcl_files_to_clean <- newIORef emptyFilesToClean
                 let lcl_dflags = dflags { log_action = parLogAction log_queue
                                         , filesToClean = lcl_files_to_clean }
 
@@ -960,9 +954,12 @@ parUpsweep n_jobs mHscMessage old_hpt stable_mods cleanup sccs = do
 
                 -- Add the remaining files that weren't cleaned up to the
                 -- global filesToClean ref, for cleanup later.
-                files_kept <- readIORef (filesToClean lcl_dflags)
-                addFilesToClean dflags files_kept
-
+                FilesToClean
+                  { ftcCurrentModule = cm_files
+                  , ftcGhcSession = gs_files
+                  } <- readIORef (filesToClean lcl_dflags)
+                addFilesToClean dflags TFL_CurrentModule $ Set.toList cm_files
+                addFilesToClean dflags TFL_GhcSession $ Set.toList gs_files
 
         -- Kill all the workers, masking interrupts (since killThread is
         -- interruptible). XXX: This is not ideal.
@@ -1971,14 +1968,10 @@ enableCodeGenForTH target nodemap =
         } <- ms
       , ms_mod `Set.member` needs_codegen_set
       = do
-        let add_intermediate_file f =
-              consIORef (filesToNotIntermediateClean dflags) f
-            new_temp_file suf dynsuf = do
-              tn <- newTempName dflags suf
+        let new_temp_file suf dynsuf = do
+              tn <- newTempName dflags TFL_CurrentModule suf
               let dyn_tn = tn -<.> dynsuf
-              add_intermediate_file tn
-              add_intermediate_file dyn_tn
-              addFilesToClean dflags [dyn_tn]
+              addFilesToClean dflags TFL_GhcSession [dyn_tn]
               return tn
           -- We don't want to create .o or .hi files unless we have been asked
           -- to by the user. But we need them, so we patch their locations in