Build: ./boot && ./configure && make sdist (#8723)

Make it possible to run `make sdist` right after configure, without completing
a complete build first.

Test Plan:
I compared the contents of the created `.tar.bz2` files in the `sdistprep`
directory, after running `make sdist` both before and after completing a full
build, using `diff -r`. There weren't any differences (after applying the
patches from D914).

Note that the `.tar.bz2` files were not exactly the same size, but they aren't
either when tarring and bzipping the same directory twice. It seems tarring
and bzipping is not deterministic (on my system).

Differential Revision: https://phabricator.haskell.org/D917

4 files changed, 102 insertions, 26 deletions

diff --git a/Makefile b/Makefile
index 9913803ebe..e6e6a90601 100644
--- a/Makefile
+++ b/Makefile
@@ -61,7 +61,29 @@ endif
 endif
 
 # No need to update makefiles for these targets:
-REALGOALS=$(filter-out binary-dist binary-dist-prep bootstrapping-files framework-pkg clean clean_% distclean maintainer-clean show echo help test fulltest fasttest,$(MAKECMDGOALS))
+# (the ones we're filtering out)
+REALGOALS=$(filter-out \
+    binary-dist \
+    binary-dist-prep \
+    sdist sdist-ghc \
+    sdist-ghc-prep \
+    sdist-windows-tarballs \
+    sdist-windows-tarballs-prep \
+    sdist-testsuite \
+    sdist-testsuite-prep \
+    bootstrapping-files \
+    framework-pkg \
+    clean \
+    clean_% \
+    distclean \
+    maintainer-clean \
+    show \
+    echo \
+    help \
+    test \
+    fulltest \
+    fasttest \
+  ,$(MAKECMDGOALS))
 
 # configure touches certain files even if they haven't changed.  This
 # can mean a lot of unnecessary recompilation after a re-configure, so
@@ -101,6 +123,12 @@ else
 	$(MAKE) --no-print-directory -f ghc.mk unix-binary-dist-prep
 endif
 
+.PHONY: sdist sdist-ghc sdist-ghc-prep sdist-windows-tarballs sdist-windows-tarballs-prep sdist-testsuite sdist-testsuite-prep
+# Just running `./boot && ./configure && make sdist` should work, so skip
+# phase 0 and 1 and don't build any dependency files.
+sdist sdist-ghc sdist-ghc-prep sdist-windows-tarballs sdist-windows-tarballs-prep sdist-testsuite sdist-testsuite-prep :
+	$(MAKE) --no-print-directory -f ghc.mk $@ NO_INCLUDE_DEPS=YES NO_INCLUDE_PKGDATA=YES
+
 .PHONY: clean distclean maintainer-clean
 clean distclean maintainer-clean:
 	$(MAKE) --no-print-directory -f ghc.mk $@ CLEANING=YES
diff --git a/ghc.mk b/ghc.mk
index 3b38372353..42b1784045 100644
--- a/ghc.mk
+++ b/ghc.mk
@@ -309,9 +309,10 @@ include rules/manual-package-config.mk
 include rules/docbook.mk
 
 # -----------------------------------------------------------------------------
-# Making bindists
+# Making bindists and sdists
 
 include rules/bindist.mk
+include rules/sdist-ghc-file.mk
 
 # -----------------------------------------------------------------------------
 # Directories
@@ -1052,19 +1053,20 @@ publish-docs:
 
 # Do it like this:
 #
-#	$ make
+#	$ ./boot
+#	$ ./configure
 #	$ make sdist
 #
 
-# A source dist is built from a complete build tree, because we
+# A source dist is built from a (partial) build tree, because we
 # require some extra files not contained in a git checkout: the
 # output from Happy and Alex, for example.
 #
-# The steps performed by 'make dist' are as follows:
+# The steps performed by 'make sdist' are as follows:
+#   - build those extra files
 #   - create a complete link-tree of the current build tree in /tmp
 #   - run 'make distclean' on that tree
 #   - remove a bunch of other files that we know shouldn't be in the dist
-#   - tar up first the extralibs package, then the main source package
 
 #
 # Directory in which we're going to build the src dist
@@ -1118,22 +1120,13 @@ GIT_COMMIT_ID:
 	then echo "$@ needs no update"; \
 	else echo "update $@ ($(ProjectGitCommitId))"; echo -n "$(ProjectGitCommitId)" > $@; fi
 
-sdist-ghc-prep : VERSION GIT_COMMIT_ID
-
-# Use:
-#     $(call sdist_ghc_file,compiler,stage2,cmm,Foo/Bar,CmmLex,x)
-# to copy the generated file that replaces compiler/cmm/Foo/Bar/CmmLex.x, where
-# "stage2" is the dist dir.
-define sdist_ghc_file
-	"$(CP)" $1/$2/build/$4/$5.hs $(SRC_DIST_GHC_DIR)/$1/$3/$4
-	mv $(SRC_DIST_GHC_DIR)/$1/$3/$4/$5.$6 $(SRC_DIST_GHC_DIR)/$1/$3/$4/$5.$6.source
-endef
+sdist-ghc-prep-tree : VERSION GIT_COMMIT_ID
 
 # Extra packages which shouldn't be in the source distribution: see #8801
 EXTRA_PACKAGES=parallel stm random primitive vector dph
 
-.PHONY: sdist-ghc-prep
-sdist-ghc-prep :
+.PHONY: sdist-ghc-prep-tree
+sdist-ghc-prep-tree :
 	$(call removeTrees,$(SRC_DIST_GHC_ROOT))
 	$(call removeFiles,$(SRC_DIST_GHC_TARBALL))
 	mkdir -p $(SRC_DIST_ROOT)
@@ -1147,15 +1140,21 @@ sdist-ghc-prep :
 	$(call removeTrees,$(SRC_DIST_GHC_DIR)/compiler/stage[123])
 	$(call removeFiles,$(SRC_DIST_GHC_DIR)/mk/build.mk)
 	for i in $(EXTRA_PACKAGES); do $(RM) $(RM_OPTS_REC) $(SRC_DIST_GHC_DIR)/libraries/$$i/; done
-	$(call sdist_ghc_file,compiler,stage2,cmm,,CmmLex,x)
-	$(call sdist_ghc_file,compiler,stage2,cmm,,CmmParse,y)
-	$(call sdist_ghc_file,compiler,stage2,parser,,Lexer,x)
-	$(call sdist_ghc_file,compiler,stage2,parser,,Parser,y)
-	$(call sdist_ghc_file,utils/hpc,dist-install,,,HpcParser,y)
-	$(call sdist_ghc_file,utils/genprimopcode,dist,,,Lexer,x)
-	$(call sdist_ghc_file,utils/genprimopcode,dist,,,Parser,y)
 	cd $(SRC_DIST_GHC_DIR) && "$(FIND)" $(SRC_DIST_GHC_DIRS) \( -name .git -o -name "autom4te*" -o -name "*~" -o -name "\#*" -o -name ".\#*" -o -name "log" -o -name "*-SAVE" -o -name "*.orig" -o -name "*.rej" \) -print | "$(XARGS)" $(XARGS_OPTS) "$(RM)" $(RM_OPTS_REC)
 
+# Add files generated by alex and happy.
+# These rules depend on sdist-ghc-prep-tree.
+$(eval $(call sdist-ghc-file,compiler,stage2,cmm,CmmLex,x))
+$(eval $(call sdist-ghc-file,compiler,stage2,cmm,CmmParse,y))
+$(eval $(call sdist-ghc-file,compiler,stage2,parser,Lexer,x))
+$(eval $(call sdist-ghc-file,compiler,stage2,parser,Parser,y))
+$(eval $(call sdist-ghc-file,utils/hpc,dist-install,,HpcParser,y))
+$(eval $(call sdist-ghc-file,utils/genprimopcode,dist,,Lexer,x))
+$(eval $(call sdist-ghc-file,utils/genprimopcode,dist,,Parser,y))
+
+.PHONY: sdist-ghc-prep
+sdist-ghc-prep : sdist-ghc-prep-tree
+
 .PHONY: sdist-windows-tarballs-prep
 sdist-windows-tarballs-prep :
 	$(call removeTrees,$(SRC_DIST_WINDOWS_TARBALLS_ROOT))
diff --git a/libraries/integer-gmp/gmp/ghc.mk b/libraries/integer-gmp/gmp/ghc.mk
index 2e81956f92..6a1ca2175a 100644
--- a/libraries/integer-gmp/gmp/ghc.mk
+++ b/libraries/integer-gmp/gmp/ghc.mk
@@ -47,7 +47,10 @@ endif
 ifeq "$(phase)" "final"
 
 ifneq "$(CLEANING)" "YES"
-include libraries/integer-gmp/gmp/config.mk
+# Hack. The file gmp/config.mk doesn't exist yet after running ./configure in
+# the toplevel (ghc) directory. To let some toplevel make commands such as
+# sdist go through, right after ./configure, don't consider this an error.
+-include libraries/integer-gmp/gmp/config.mk
 endif
 
 gmp_CC_OPTS += $(addprefix -I,$(GMP_INCLUDE_DIRS))
diff --git a/rules/sdist-ghc-file.mk b/rules/sdist-ghc-file.mk
new file mode 100644
index 0000000000..9ea0b6521a
--- /dev/null
+++ b/rules/sdist-ghc-file.mk
@@ -0,0 +1,46 @@
+# -----------------------------------------------------------------------------
+#
+# This file is part of the GHC build system.
+#
+# To understand how the build system works and how to modify it, see
+#      http://ghc.haskell.org/trac/ghc/wiki/Building/Architecture
+#      http://ghc.haskell.org/trac/ghc/wiki/Building/Modifying
+#
+# -----------------------------------------------------------------------------
+
+
+# Add files generated by alex and happy to a source distribution.
+#
+# Invoke like this:
+#
+# $(eval $(call sdist-ghc-file,compiler,stage2,parser,Parser,y))
+#
+# This adds the file 'compiler/stage2/build/Parser.hs' to the sdist, in the
+# same directory as 'compiler/parser/Parser.y' (which is renamed).
+
+define sdist-ghc-file
+# $1 = dir
+# $2 = distdir
+# $3 = srcdir (can be empty)
+# $4 = filename
+# $5 = extension
+
+.PHONY: sdist_$1_$2_$4
+
+# We should do this before creating the `sdist-ghc` tarball, or when just
+# running `make sdist-ghc-prep`.
+sdist-ghc-prep : sdist_$1_$2_$4
+
+# But first create SRC_DIST_GHC_DIR.
+sdist_$1_$2_$4 : sdist-ghc-prep-tree
+
+# Generate the .hs files if they don't exist yet, then do actual copying and
+# moving.
+sdist_$1_$2_$4 : $1/$2/build/$4.hs
+	"$(CP)" $1/$2/build/$4.hs $(SRC_DIST_GHC_DIR)/$1/$3
+	mv $(SRC_DIST_GHC_DIR)/$1/$3/$4.$5 $(SRC_DIST_GHC_DIR)/$1/$3/$4.$5.source
+
+# And make sure the rules for generating the .hs files exist, even when we
+# didn't generate all package-data.mk files.
+$$(eval $$(call hs-suffix-rules-srcdir,$1,$2,$3))
+endef