Interface Files with Core Definitions

This commit adds three new flags

* -fwrite-if-simplified-core: Writes the whole core program into an interface
  file
* -fbyte-code-and-object-code: Generate both byte code and object code
  when compiling a file
* -fprefer-byte-code: Prefer to use byte-code if it's available when
  running TH splices.

The goal for including the core bindings in an interface file is to be able to restart the compiler pipeline
at the point just after simplification and before code generation. Once compilation is
restarted then code can be created for the byte code backend.
This can significantly speed up
start-times for projects in GHCi. HLS already implements its own version of these extended interface
files for this reason.

Preferring to use byte-code means that we can avoid some potentially
expensive code generation steps (see #21700)

* Producing object code is much slower than producing bytecode, and normally you
  need to compile with `-dynamic-too` to produce code in the static and dynamic way, the
  dynamic way just for Template Haskell execution when using a dynamically linked compiler.

* Linking many large object files, which happens once per splice, can be quite
  expensive compared to linking bytecode.

And you can get GHC to compile the necessary byte code so
`-fprefer-byte-code` has access to it by using
`-fbyte-code-and-object-code`.

Fixes #21067

2 files changed, 49 insertions, 5 deletions

diff --git a/compiler/GHC/Linker/Loader.hs b/compiler/GHC/Linker/Loader.hs
index 18ab333c08..3c9baf45cf 100644
--- a/compiler/GHC/Linker/Loader.hs
+++ b/compiler/GHC/Linker/Loader.hs
@@ -77,6 +77,7 @@ import GHC.Unit.Env
 import GHC.Unit.Finder
 import GHC.Unit.Module
 import GHC.Unit.Module.ModIface
+import GHC.Unit.Module.WholeCoreBindings
 import GHC.Unit.Module.Deps
 import GHC.Unit.Home.ModInfo
 import GHC.Unit.State as Packages
@@ -840,11 +841,17 @@ getLinkDeps hsc_env pls replace_osuf span mods
 
     while_linking_expr = text "while linking an interpreted expression"
 
-        -- This one is a build-system bug
+
+    -- See Note [Using Byte Code rather than Object Code for Template Haskell]
+    homeModLinkable :: DynFlags -> HomeModInfo -> Maybe Linkable
+    homeModLinkable dflags hmi =
+      if gopt Opt_UseBytecodeRatherThanObjects dflags
+        then homeModInfoByteCode hmi <|> homeModInfoObject hmi
+        else homeModInfoObject hmi   <|> homeModInfoByteCode hmi
 
     get_linkable osuf mod      -- A home-package module
         | Just mod_info <- lookupHugByModule mod (hsc_HUG hsc_env)
-        = adjust_linkable (Maybes.expectJust "getLinkDeps" (hm_linkable mod_info))
+        = adjust_linkable (Maybes.expectJust "getLinkDeps" (homeModLinkable dflags mod_info))
         | otherwise
         = do    -- It's not in the HPT because we are in one shot mode,
                 -- so use the Finder to get a ModLocation...
@@ -889,7 +896,34 @@ getLinkDeps hsc_env pls replace_osuf span mods
             adjust_ul _ (DotA fp) = panic ("adjust_ul DotA " ++ show fp)
             adjust_ul _ (DotDLL fp) = panic ("adjust_ul DotDLL " ++ show fp)
             adjust_ul _ l@(BCOs {}) = return l
+            adjust_ul _ l@LoadedBCOs{} = return l
+            adjust_ul _ (CoreBindings (WholeCoreBindings _ mod _))     = pprPanic "Unhydrated core bindings" (ppr mod)
+
+{-
+Note [Using Byte Code rather than Object Code for Template Haskell]
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The `-fprefer-byte-code` flag allows a user to specify that they want to use
+byte code (if availble) rather than object code for home module dependenices
+when executing Template Haskell splices.
+
+Why might you want to use byte code rather than object code?
 
+* Producing object code is much slower than producing byte code (for example if you're using -fno-code)
+* Linking many large object files, which happens once per splice, is quite expensive. (#21700)
+
+So we allow the user to choose to use byte code rather than object files if they want to avoid these
+two pitfalls.
+
+When using `-fprefer-byte-code` you have to arrange to have the byte code availble.
+In normal --make mode it will not be produced unless you enable `-fbyte-code-and-object-code`.
+See Note [Home module build products] for some more information about that.
+
+The only other place where the flag is consulted is when enabling code generation
+with `-fno-code`, which does so to anticipate what decision we will make at the
+splice point about what we would prefer.
+
+-}
 
 {- **********************************************************************
 
@@ -1133,7 +1167,7 @@ dynLinkBCOs bco_opts interp pls bcos = do
             unlinkeds                = concatMap linkableUnlinked new_bcos
 
             cbcs :: [CompiledByteCode]
-            cbcs      = map byteCodeOfObject unlinkeds
+            cbcs      = concatMap byteCodeOfObject unlinkeds
 
 
             ies        = map bc_itbls cbcs
diff --git a/compiler/GHC/Linker/Types.hs b/compiler/GHC/Linker/Types.hs
index 25df199b0f..605754b0ae 100644
--- a/compiler/GHC/Linker/Types.hs
+++ b/compiler/GHC/Linker/Types.hs
@@ -49,6 +49,7 @@ import Data.Maybe
 import GHC.Unit.Module.Env
 import GHC.Types.Unique.DSet
 import GHC.Types.Unique.DFM
+import GHC.Unit.Module.WholeCoreBindings
 
 
 {- **********************************************************************
@@ -156,6 +157,10 @@ data Unlinked
   = DotO ObjFile       -- ^ An object file (.o)
   | DotA FilePath      -- ^ Static archive file (.a)
   | DotDLL FilePath    -- ^ Dynamically linked library file (.so, .dll, .dylib)
+  | CoreBindings WholeCoreBindings -- ^ Serialised core which we can turn into BCOs (or object files), or used by some other backend
+                       -- See Note [Interface Files with Core Definitions]
+  | LoadedBCOs [Unlinked] -- ^ A list of BCOs, but hidden behind extra indirection to avoid
+                          -- being too strict.
   | BCOs CompiledByteCode
          [SptEntry]    -- ^ A byte-code object, lives only in memory. Also
                        -- carries some static pointer table entries which
@@ -168,6 +173,8 @@ instance Outputable Unlinked where
   ppr (DotA path)   = text "DotA" <+> text path
   ppr (DotDLL path) = text "DotDLL" <+> text path
   ppr (BCOs bcos spt) = text "BCOs" <+> ppr bcos <+> ppr spt
+  ppr (LoadedBCOs{})  = text "LoadedBCOs"
+  ppr (CoreBindings {})       = text "FI"
 
 -- | An entry to be inserted into a module's static pointer table.
 -- See Note [Grand plan for static forms] in "GHC.Iface.Tidy.StaticPtrTable".
@@ -205,6 +212,8 @@ nameOfObject_maybe :: Unlinked -> Maybe FilePath
 nameOfObject_maybe (DotO fn)   = Just fn
 nameOfObject_maybe (DotA fn)   = Just fn
 nameOfObject_maybe (DotDLL fn) = Just fn
+nameOfObject_maybe (CoreBindings {}) = Nothing
+nameOfObject_maybe (LoadedBCOs{}) = Nothing
 nameOfObject_maybe (BCOs {})   = Nothing
 
 -- | Retrieve the filename of the linkable if possible. Panic if it is a byte-code object
@@ -212,8 +221,9 @@ nameOfObject :: Unlinked -> FilePath
 nameOfObject o = fromMaybe (pprPanic "nameOfObject" (ppr o)) (nameOfObject_maybe o)
 
 -- | Retrieve the compiled byte-code if possible. Panic if it is a file-based linkable
-byteCodeOfObject :: Unlinked -> CompiledByteCode
-byteCodeOfObject (BCOs bc _) = bc
+byteCodeOfObject :: Unlinked -> [CompiledByteCode]
+byteCodeOfObject (BCOs bc _) = [bc]
+byteCodeOfObject (LoadedBCOs ul) = concatMap byteCodeOfObject ul
 byteCodeOfObject other       = pprPanic "byteCodeOfObject" (ppr other)
 
 {- **********************************************************************