Build system: cleanup BUILD_DIRS + add lots of Notes

Summary:
See Note [CrossCompiling vs Stage1Only] in mk/config.mk.in.
See Note [Stage1Only vs stage=1] in mk/config.mk.in.
See Note [No stage2 packages when CrossCompiling or Stage1Only].

Also:
  * use stage2 to build mkUserGuidePart, as was probably intended.
    Now the following represent the same set of packages:
    - packages that we build with ghc-stage2
    - packages that depend on the ghc library
    Those packages are: haddock, mkUserGuidePart and ghctags.
  * don't let utils that don't depend on the ghc library depend on its
    package-data.mk file. Instead, let those utils directly depend on
    the package-data.mk files of the stage1 packages. Not sure if it
    improves anything, but I found it easier to explain what's going on
    this way.

(partially) reviewed by: austin

Differential Revision: https://phabricator.haskell.org/D1218

1 files changed, 117 insertions, 57 deletions

diff --git a/ghc.mk b/ghc.mk
index 7337c27dfe..f933ce7b1b 100644
--- a/ghc.mk
+++ b/ghc.mk
@@ -423,7 +423,9 @@ endif
 endif
 PACKAGES_STAGE1 += haskeline
 
-ifneq "$(CrossCompiling)" "YES"
+# See Note [No stage2 packages when CrossCompiling or Stage1Only].
+# See Note [Stage1Only vs stage=1] in mk/config.mk.in.
+ifeq "$(CrossCompiling) $(Stage1Only)" "NO NO"
 define addExtraPackage
 ifeq "$2" "-"
 # Do nothing; this package is already handled above
@@ -445,6 +447,7 @@ endif
 
 # We install all packages that we build.
 INSTALL_PACKAGES := $(addprefix libraries/,$(PACKAGES_STAGE1))
+# See Note [Stage1Only vs stage=1] in mk/config.mk.in.
 ifneq "$(Stage1Only)" "YES"
 INSTALL_PACKAGES += compiler
 endif
@@ -453,7 +456,7 @@ INSTALL_PACKAGES += $(addprefix libraries/,$(PACKAGES_STAGE2))
 endif # CLEANING
 
 # -------------------------------------------
-# Dependencies between package-data.mk files
+# Note [Dependencies between package-data.mk files].
 
 # We cannot run ghc-cabal to configure a package until we have
 # configured and registered all of its dependencies.  So the following
@@ -473,29 +476,35 @@ ifneq "$(BINDIST)" "YES"
 fixed_pkg_prev=
 $(foreach pkg,$(PACKAGES_STAGE1),$(eval $(call fixed_pkg_dep,$(pkg),dist-install)))
 
+# Intermezzo: utils that we build with the stage1 compiler. They depend on
+# the stage1 packages, so we have to make sure those packages get configured
+# and registered before we can start with these. Note that they don't depend on
+# eachother, so we can configure them in parallel.
+utils/ghc-pwd/dist-install/package-data.mk: $(fixed_pkg_prev)
+utils/ghc-cabal/dist-install/package-data.mk: $(fixed_pkg_prev)
+utils/dll-split/dist-install/package-data.mk: $(fixed_pkg_prev)
+utils/hpc/dist-install/package-data.mk: $(fixed_pkg_prev)
+utils/ghc-pkg/dist-install/package-data.mk: $(fixed_pkg_prev)
+utils/hsc2hs/dist-install/package-data.mk: $(fixed_pkg_prev)
+utils/compare_sizes/dist-install/package-data.mk: $(fixed_pkg_prev)
+utils/runghc/dist-install/package-data.mk: $(fixed_pkg_prev)
+
 # the GHC package doesn't live in libraries/, so we add its dependency manually:
 compiler/stage2/package-data.mk: $(fixed_pkg_prev)
-fixed_pkg_prev:=compiler/stage2/package-data.mk
 
 # and continue with PACKAGES_STAGE2, which depend on GHC:
+fixed_pkg_prev:=compiler/stage2/package-data.mk
 $(foreach pkg,$(PACKAGES_STAGE2),$(eval $(call fixed_pkg_dep,$(pkg),dist-install)))
 
 ghc/stage1/package-data.mk: compiler/stage1/package-data.mk
 ghc/stage2/package-data.mk: compiler/stage2/package-data.mk
-# haddock depends on ghc and some libraries, but depending on GHC's
-# package-data.mk is sufficient, as that in turn depends on all the
-# libraries
-utils/haddock/dist/package-data.mk: compiler/stage2/package-data.mk
-utils/ghc-pwd/dist-install/package-data.mk: compiler/stage2/package-data.mk
-utils/ghc-cabal/dist-install/package-data.mk: compiler/stage2/package-data.mk
 
+# Utils that we build with the stage2 compiler.
+# They depend on the ghc library and some other libraries, but depending on
+# the ghc library's package-data.mk is sufficient, as that in turn depends on
+# all the other libraries' package-data.mk files.
+utils/haddock/dist/package-data.mk: compiler/stage2/package-data.mk
 utils/ghctags/dist-install/package-data.mk: compiler/stage2/package-data.mk
-utils/dll-split/dist-install/package-data.mk: compiler/stage2/package-data.mk
-utils/hpc/dist-install/package-data.mk: compiler/stage2/package-data.mk
-utils/ghc-pkg/dist-install/package-data.mk: compiler/stage2/package-data.mk
-utils/hsc2hs/dist-install/package-data.mk: compiler/stage2/package-data.mk
-utils/compare_sizes/dist-install/package-data.mk: compiler/stage2/package-data.mk
-utils/runghc/dist-install/package-data.mk: compiler/stage2/package-data.mk
 utils/mkUserGuidePart/dist/package-data.mk: compiler/stage2/package-data.mk
 
 # add the final package.conf dependency: ghc-prim depends on RTS
@@ -563,26 +572,12 @@ endif
 
 # -----------------------------------------------------------------------------
 # Include build instructions from all subdirs
-
-ifneq "$(BINDIST)" "YES"
 BUILD_DIRS += utils/mkdirhier
-endif
-
-ifeq "$(Windows_Host)" "YES"
 BUILD_DIRS += utils/touchy
-endif
-
 BUILD_DIRS += utils/unlit
 BUILD_DIRS += utils/hp2ps
-
-ifneq "$(GhcUnregisterised)" "YES"
 BUILD_DIRS += driver/split
-endif
-
-ifneq "$(BINDIST)" "YES"
 BUILD_DIRS += utils/genprimopcode
-endif
-
 BUILD_DIRS += driver
 BUILD_DIRS += driver/ghci
 BUILD_DIRS += driver/ghc
@@ -591,11 +586,8 @@ BUILD_DIRS += libffi
 BUILD_DIRS += utils/deriveConstants
 BUILD_DIRS += includes
 BUILD_DIRS += rts
-
-ifneq "$(BINDIST)" "YES"
 BUILD_DIRS += bindisttest
 BUILD_DIRS += utils/genapply
-endif
 
 # When cleaning, don't add any library packages to BUILD_DIRS. We include
 # ghc.mk files for all BUILD_DIRS, but they don't exist until after running
@@ -619,52 +611,71 @@ BUILD_DIRS += $(wildcard libraries/dph)
 endif
 endif
 
-
-ifeq "$(INTEGER_LIBRARY)" "integer-gmp"
-BUILD_DIRS += libraries/integer-gmp/gmp
-else ifeq "$(CLEANING)" "YES"
 BUILD_DIRS += libraries/integer-gmp/gmp
-endif
-
-ifeq "$(HADDOCK_DOCS)" "YES"
-BUILD_DIRS += utils/haddock
-BUILD_DIRS += utils/haddock/doc
-else ifeq "$(CLEANING)" "YES"
 BUILD_DIRS += utils/haddock
 BUILD_DIRS += utils/haddock/doc
-endif
-
 BUILD_DIRS += compiler
 BUILD_DIRS += utils/hsc2hs
 BUILD_DIRS += utils/ghc-pkg
 BUILD_DIRS += utils/testremove
-ifneq "$(Stage1Only)" "YES"
 BUILD_DIRS += utils/ghctags
-endif
 BUILD_DIRS += utils/dll-split
 BUILD_DIRS += utils/ghc-pwd
 BUILD_DIRS += utils/ghc-cabal
 BUILD_DIRS += utils/hpc
-ifeq "$(GhcWithInterpreter)" "YES"
-BUILD_DIRS += utils/runghc
-else ifeq "$(CLEANING)" "YES"
 BUILD_DIRS += utils/runghc
-endif
 BUILD_DIRS += ghc
-
-ifneq "$(BINDIST)" "YES"
-ifneq "$(CrossCompiling)-$(phase)" "YES-final"
 BUILD_DIRS += utils/mkUserGuidePart
-endif
-endif
-
 BUILD_DIRS += docs/users_guide
 BUILD_DIRS += docs/man
 BUILD_DIRS += utils/count_lines
 BUILD_DIRS += utils/compare_sizes
 
 # ----------------------------------------------
-# Actually include all the sub-ghc.mk's
+# Actually include the sub-ghc.mk's
+
+ifeq "$(CLEANING)" "YES"
+# Don't exclude any BUILD_DIRS when cleaning. When you for example build
+# haddock once, but later set HADDOCK_DOCS back to NO, then 'make clean'
+# should still clean the haddock directory.
+else # CLEANING
+ifeq "$(BINDIST)" "YES"
+BUILD_DIRS := $(filter-out utils/mkdirhier,$(BUILD_DIRS))
+BUILD_DIRS := $(filter-out utils/genprimopcode,$(BUILD_DIRS))
+BUILD_DIRS := $(filter-out bindisttest,$(BUILD_DIRS))
+BUILD_DIRS := $(filter-out utils/genapply,$(BUILD_DIRS))
+BUILD_DIRS := $(filter-out utils/mkUserGuidePart,$(BUILD_DIRS))
+endif
+ifeq "$(HADDOCK_DOCS)" "NO"
+BUILD_DIRS := $(filter-out utils/haddock,$(BUILD_DIRS))
+BUILD_DIRS := $(filter-out utils/haddock/doc,$(BUILD_DIRS))
+endif
+ifeq "$(BUILD_DOCBOOK_HTML) $(BUILD_DOCBOOK_PS) $(BUILD_DOCBOOK_PDF)" "NO NO NO"
+# Don't to build this little utility if we're not building the User's Guide.
+BUILD_DIRS := $(filter-out utils/mkUserGuidePart,$(BUILD_DIRS))
+endif
+ifeq "$(Windows_Host)" "NO"
+BUILD_DIRS := $(filter-out utils/touchy,$(BUILD_DIRS))
+endif
+ifeq "$(GhcUnregisterised)" "YES"
+BUILD_DIRS := $(filter-out driver/split,$(BUILD_DIRS))
+endif
+ifeq "$(GhcWithInterpreter)" "NO"
+# runghc is just GHCi in disguise
+BUILD_DIRS := $(filter-out utils/runghc,$(BUILD_DIRS))
+endif
+ifneq "$(INTEGER_LIBRARY)" "integer-gmp"
+BUILD_DIRS := $(filter-out libraries/integer-gmp/gmp,$(BUILD_DIRS))
+endif
+ifneq "$(CrossCompiling) $(Stage1Only)" "NO NO"
+# See Note [No stage2 packages when CrossCompiling or Stage1Only].
+# See Note [Stage1Only vs stage=1] in mk/config.mk.in.
+BUILD_DIRS := $(filter-out utils/haddock,$(BUILD_DIRS))
+BUILD_DIRS := $(filter-out utils/haddock/doc,$(BUILD_DIRS))
+BUILD_DIRS := $(filter-out utils/ghctags,$(BUILD_DIRS))
+BUILD_DIRS := $(filter-out utils/mkUserGuidePart,$(BUILD_DIRS))
+endif
+endif # CLEANING
 
 include $(patsubst %, %/ghc.mk, $(BUILD_DIRS))
 
@@ -707,6 +718,11 @@ $(eval $(call clean-target,root,bootstrapping_conf,$(BOOTSTRAPPING_CONF)))
 # lost).
 fixed_pkg_prev=
 $(foreach pkg,$(PACKAGES_STAGE0),$(eval $(call fixed_pkg_dep,$(pkg),dist-boot)))
+# ghc-pkg, unlike other utils that we build with the stage0 compiler (TODO: is
+# this really true?), depends on several boot packages (e.g. Cabal and
+# bin-package-db). They need to be configured before ghc-pkg, so we add a
+# dependency between their package-data.mk files. See also Note
+# [Dependencies between package-data.mk files].
 utils/ghc-pkg/dist/package-data.mk: $(fixed_pkg_prev)
 compiler/stage1/package-data.mk:    $(fixed_pkg_prev)
 endif
@@ -1376,6 +1392,50 @@ endif
 	cd libraries/xhtml && ./Setup clean   --builddir=dist-bindist
 	cd libraries/xhtml && rm -f Setup Setup.exe Setup.hi Setup.o
 
+# Note [No stage2 packages when CrossCompiling or Stage1Only]
+#
+# (first read Note [CrossCompiling vs Stage1Only] and
+#  Note [Stage1Only vs stage=1] in mk/config.mk.in)
+#
+# When either CrossCompiling=YES or Stage1Only=YES, we have to exclude the
+# following packages from the build:
+#   * packages that we build with ghc-stage2 [1]
+#   * packages that depend on the ghc library [2]
+#
+# Here's why:
+#  - first of all, ghc-stage1 can't use stage0's ghc library (it's too old)
+#  - neither do we register the ghc library (compiler/stage1) that we build
+#    with stage0. TODO Why not? We do build it...
+#  - as a result, we need to a) use ghc-stage2 to build packages that depend on
+#    the ghc library (e.g. ghctags [4] and mkUserGuidePart) and b) exclude
+#    those packages when ghc-stage2 is not available.
+#  - when Stage1Only=YES, it's clear that ghc-stage2 is not available (we just
+#    said we didn't want it), so we have to exclude the stage2 packages from
+#    the build. This includes the case where Stage1Only=YES is combined with
+#    CrossCompiling=YES (Building GHC as a cross-compiler [3]).
+#  - when CrossCompiling=YES, but Stage1Only=NO (Cross-compiling GHC itself
+#    [3]), we can not use ghc-stage2 either. The reason is that stage2 doesn't
+#    run on the host platform at all; it is built to run on $(TARGETPLATFORM)"
+#    [5]. Therefore in this case we also have to exclude the stage2 packages
+#    from the build.
+#
+#  [1] find utils -name ghc.mk | xargs grep -l 'build-prog.*,2'
+#
+#  [2]
+#  find utils -name package-data.mk | xargs grep -l 'DEP_NAMES =.* ghc\($\| \)'
+#
+#  [3] https://ghc.haskell.org/trac/ghc/wiki/Building/CrossCompiling
+#
+#  [4] 5fb72555f7b7ab67a33583f33ad9160761ca434f
+#      "ghctags needs the stage2 compiler, since it uses the GHC API."
+#
+#  [5] * bc31dbe8ee22819054df60f5ef219fed393a1c54
+#      "Disable any packages built with stage 2 when cross-compiling
+#       Since we can't run stage 2 on the host."
+#
+#      * 72995160b0b190577b5c0cb8d7bd0426cc455b05
+#      "We cannot use the stage 2 compiler, it runs on $(TARGETPLATFORM)"
+
 # -----------------------------------------------------------------------------
 # Numbered phase targets