WIP: Run `makedepend` in parallel by using `make`

This moves the per-file loop body of `makedepend` into a separate
file named `makedepend_file` and then uses `make` to launch
the `makedepend_file` processes for each target potentially in parallel.

This reduces the time for

  time sh ./makedepend MAKE=make cflags

from 5 seconds to 2 seconds with MAKEFLAGS=-j8

5 files changed, 197 insertions, 109 deletions

diff --git a/.gitignore b/.gitignore
index 546c4b275b..30416d1afb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -27,6 +27,7 @@ bug*.pl
 /config.sh
 /makeaperl
 /makedepend
+/makedepend_file
 /makedir
 /makefile
 /myconfig
diff --git a/MANIFEST b/MANIFEST
index 72f8c69d61..e65e751181 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -5031,6 +5031,7 @@ make_ext.pl			Used by Makefile to execute extension Makefiles
 make_patchnum.pl		Script to generate git_version.h and lib/Config_git.pl files for all OS'es
 makedef.pl			Create symbol export lists for linking
 makedepend.SH			Precursor to makedepend
+makedepend_file.SH			Precursor to makedepend_file
 Makefile.micro			microperl Makefile
 Makefile.SH			A script that generates Makefile
 malloc.c			A version of malloc you might not want
diff --git a/Makefile.SH b/Makefile.SH
index 7e75d47365..ecbe8c136b 100755
--- a/Makefile.SH
+++ b/Makefile.SH
@@ -1376,6 +1376,7 @@ veryclean:	_verycleaner _mopup _clobber
 _mopup:
 	rm -f *$(OBJ_EXT) *$(LIB_EXT) all perlmain.c opmini.c perlmini.c generate_uudmap$(EXE_EXT) $(generated_headers)
 	-rmdir .depending
+	-rm *.depends
 	-@test -f extra.pods && rm -f `cat extra.pods`
 	-@test -f vms/README_vms.pod && rm -f vms/README_vms.pod
 	-rm -f perl.exp ext.libs $(generated_pods) uni.data opmini.o perlmini.o pod/roffitall
@@ -1499,7 +1500,7 @@ cscope.out cscope: $(c) $(h)
 # The README below ensures that the dependency list is never empty and
 # that when MAKEDEPEND is empty $(FIRSTMAKEFILE) doesn't need rebuilding.
 
-MAKEDEPEND = Makefile makedepend
+MAKEDEPEND = Makefile makedepend_file makedepend
 
 $(FIRSTMAKEFILE):	README $(MAKEDEPEND)
 	$(MAKE) depend MAKEDEPEND=
@@ -1520,6 +1521,9 @@ $spitshell >>$Makefile <<'!NO!SUBS!'
 depend: makedepend $(DTRACE_H) $(generated_headers)
 	sh ./makedepend MAKE="$(MAKE)" cflags
 
+%.c.depends: %.c
+	sh ./makedepend_file $< $@ cflags
+
 .PHONY: test check test_prep test_prep_nodll test_prep_pre \
 	test_prep_reonly test_tty test-tty test_notty test-notty \
 	test_harness test_harness_notty minitest test-reonly _test
diff --git a/makedepend.SH b/makedepend.SH
index 600288f3de..5cf5a38c77 100755
--- a/makedepend.SH
+++ b/makedepend.SH
@@ -36,9 +36,9 @@ fi
 
 mkdir .depending
 
-# This script should be called with 
+# This script should be called with
 #     sh ./makedepend MAKE=$(MAKE)
-case "$1" in 
+case "$1" in
 	MAKE=*) eval $1; shift ;;
 esac
 
@@ -112,114 +112,17 @@ $test -d UU || mkdir UU
 
 $MAKE clist || ($echo "Searching for .c files..."; \
 	$echo *.c | $tr ' ' $trnl | $egrep -v '\*' >.clist)
-for file in `$cat .clist`; do
-# for file in `cat /dev/null`; do
-    case "$osname" in
-    uwin)     uwinfix="-e s,\\\\\\\\,/,g -e s,\\([a-zA-Z]\\):/,/\\1/,g" ;;
-    os2)      uwinfix="-e s,\\\\\\\\,/,g" ;;
-    cygwin)   uwinfix="-e s,\\\\\\\\,/,g" ;;
-    posix-bc) uwinfix="-e s/\\*POSIX(\\(.*\\))/\\1/" ;;
-    vos)      uwinfix="-e s/\#/\\\#/" ;;
-    *)        uwinfix="" ;;
-    esac
-    case "$file" in
-    *.c) filebase=`basename $file .c` ;;
-    *.y) filebase=`basename $file .y` ;;
-    esac
-    case "$file" in
-    */*) finc="-I`echo $file | sed 's#/[^/]*$##'`" ;;
-    *)   finc= ;;
-    esac
-    $echo "Finding dependencies for $filebase$_o"
-    # Below, we strip out all but preprocessor directives.
-    # We have to take care of situations like
-    #       #if defined(FOO) BAR   /* comment line 1
-    #                                 more comment lines */
-    # If we just delete text starting from the '/*' to the end of line, we will
-    # screw up cases like
-    #      #if defined(FOO)    /* comment */ \
-    #          && defined(BAR) /* comment */ \
-    #          && defined(BAZ) /* comment */ \
-    #               etc.
-    # Also, in lines like
-    #      #defined FOO(a,b)    a/**/b
-    # the comment may be important and so needs to be retained.
-    # This code processes the single-line comments first; it assumes there is
-    # at most one straightforward comment per continued preprocessor line,
-    # replacing each non-empty comment (and its surrounding white space) by a
-    # single space.  (sed only has a greedy '*' quantifier, so this doesn't
-    # work right if there are multiple comments per line, and strings can look
-    # like comments to it; both are unlikely in a preprocessor statement.) Any
-    # continuation line is joined, and the process repeated on the enlarged
-    # line as long as there are continuations.  At the end, if there are any
-    # comments remaining, they are either completely empty or are like the
-    # first situation.  The latter are just deleted by first deleting to the
-    # end of line (including preceding white space) things that start with '/*'
-    # and the next char isn't a '*'; then things that start with '/**', but the
-    # next char isn't a '/'.  (Subsequent lines of the comment are irrelevant
-    # and get dropped.)  At the end, we unjoin very long lines to avoid
-    # preprocessor limitations
-    ( $echo "#line 2 \"$file\"";                               \
-      $sed -n <$file                                           \
-	-e "/^${filebase}_init(/q"                             \
-        -e ': tstcont'                                         \
-	-e '/^[ 	]*#/s|[ 	]*/\*..*\*/[ 	]*| |' \
-        -e '/\\$/{'                                            \
-            -e 'N'                                             \
-            -e 'b tstcont'                                     \
-        -e '}'                                                 \
-	-e 's/\\\n//g'                                         \
-	-e '/^#line/d'                                         \
-	-e '/^[ 	]*#/{'                                 \
-	    -e 's|[ 	]*/\*[^*].*$||'                        \
-	    -e 's|[ 	]*/\*\*[^/].*$||'                      \
-            -e 's/.\{255\}/&\\\n/g'                           \
-	    -e p                                               \
-	-e '}' ) >UU/$file.c
 
-    # We're not sure why this was there; the #endif is extraneous on modern z/OS
-    #if [ "$osname" = os390 -a "$file" = perly.c ]; then
-    #   $echo '#endif' >>UU/$file.c
-    #fi
+clist=`$cat .clist | $sed -e 's,$,.depends,'`
 
-    if [ "$osname" = os390 ]; then
-        $cppstdin $finc -I. $cppflags $cppminus <UU/$file.c |
-        $sed \
-    	    -e '/^#.*<stdin>/d' \
-	    -e '/^#.*"-"/d' \
-	    -e '/^#.*git_version\.h/d' \
-	    -e 's#\.[0-9][0-9]*\.c#'"$file.c#" \
-	    -e 's/^[	 ]*#[	 ]*line/#/' \
-	    -e '/^# *[0-9][0-9]* *[".\/]/!d' \
-	    -e 's/^.*"\(.*\)".*$/'$filebase'\$(OBJ_EXT): \1/' \
-	    -e 's/^# *[0-9][0-9]* \(.*\)$/'$filebase'\$(OBJ_EXT): \1/' \
-	    -e 's|: \./|: |' \
-	    -e 's|\.c\.c|.c|' $uwinfix | \
-        $uniq | $sort | $uniq >> .deptmp
-    else
-        $cppstdin $finc -I. $cppflags $cppminus <UU/$file.c >.cout 2>.cerr
-        $sed \
-	    -e '1d' \
-	    -e '/^#.*<stdin>/d' \
-            -e '/^#.*<builtin>/d' \
-            -e '/^#.*<built-in>/d' \
-            -e '/^#.*<command line>/d' \
-            -e '/^#.*<command-line>/d' \
-	    -e '/^#.*"-"/d' \
-	    -e '/^#.*"\/.*\/"/d' \
-	    -e '/: file path prefix .* never used$/d' \
-	    -e '/^#.*git_version\.h/d' \
-	    -e 's#\.[0-9][0-9]*\.c#'"$file.c#" \
-	    -e 's/^[	 ]*#[	 ]*line/#/' \
-	    -e '/^# *[0-9][0-9]* *[".\/]/!d' \
-	    -e 's/^.*"\(.*\)".*$/'$filebase'\$(OBJ_EXT): \1/' \
-	    -e 's/^# *[0-9][0-9]* \(.*\)$/'$filebase'\$(OBJ_EXT): \1/' \
-	    -e 's|: \./|: |' \
-           -e 's|\.c\.c|.c|' $uwinfix .cout .cerr| \
-        $uniq | $sort | $uniq >> .deptmp
-    fi
-    echo "$filebase\$(OBJ_EXT): $@" >> .deptmp
-done
+# Now, create a Makefile from .clist and run that in parallel
+# Makefiles creating more Makefiles
+# A Makefile exists to beget more Makefiles. Perl is
+# just a vehicle.
+rm -f $clist
+$MAKE $clist
+$cat $clist >.deptmp
+rm -f $clist
 
 $sed <$mf >$mf.new -e '1,/^# AUTOMATICALLY/!d'
 
@@ -235,6 +138,8 @@ if $test -s .deptmp; then
        -e 'h; s/mini\(perlmain\)/\1/p; g' \
        .deptmp >>$mf.new
 else
+    $echo "Should not get here"
+    exit 1
     $MAKE hlist || ($echo "Searching for .h files..."; \
 	$echo *.h | $tr ' ' $trnl | $egrep -v '\*' >.hlist)
     $echo "You don't seem to have a proper C preprocessor.  Using grep instead."
diff --git a/makedepend_file.SH b/makedepend_file.SH
new file mode 100644
index 0000000000..51f5191fe9
--- /dev/null
+++ b/makedepend_file.SH
@@ -0,0 +1,177 @@
+#! /bin/sh
+case $PERL_CONFIG_SH in
+'')
+	if test -f config.sh; then TOP=.;
+	elif test -f ../config.sh; then TOP=..;
+	elif test -f ../../config.sh; then TOP=../..;
+	elif test -f ../../../config.sh; then TOP=../../..;
+	elif test -f ../../../../config.sh; then TOP=../../../..;
+	else
+		echo "Can't find config.sh."; exit 1
+	fi
+	. $TOP/config.sh
+	;;
+esac
+: This forces SH files to create target in same directory as SH file.
+: This is so that make depend always knows where to find SH derivatives.
+case "$0" in
+*/*) cd `expr X$0 : 'X\(.*\)/'` ;;
+esac
+
+echo "Extracting makedepend_file (with variable substitutions)"
+rm -f makedepend_file
+$spitshell >makedepend_file <<!GROK!THIS!
+$startsh
+# makedepend_file.SH
+#
+MAKE=$make
+trnl='$trnl'
+!GROK!THIS!
+$spitshell >>makedepend_file <<'!NO!SUBS!'
+
+file=$1
+shift
+outfile=$1
+shift
+
+case $PERL_CONFIG_SH in
+'')
+	if test -f config.sh; then TOP=.;
+	elif test -f ../config.sh; then TOP=..;
+	elif test -f ../../config.sh; then TOP=../..;
+	elif test -f ../../../config.sh; then TOP=../../..;
+	elif test -f ../../../../config.sh; then TOP=../../../..;
+	else
+		echo "Can't find config.sh."; exit 1
+	fi
+	. $TOP/config.sh
+	;;
+esac
+
+# Avoid localized gcc messages
+case "$ccname" in
+    gcc) LC_ALL=C ; export LC_ALL ;;
+esac
+
+# We need .. when we are in the x2p directory if we are using the
+# cppstdin wrapper script.
+# Put .. and . first so that we pick up the present cppstdin, not
+# an older one lying about in /usr/local/bin.
+PATH=".$path_sep..$path_sep$PATH"
+export PATH
+
+case "$osname" in
+amigaos) cat=/bin/cat ;; # must be absolute
+esac
+
+    case "$osname" in
+    uwin)     uwinfix="-e s,\\\\\\\\,/,g -e s,\\([a-zA-Z]\\):/,/\\1/,g" ;;
+    os2)      uwinfix="-e s,\\\\\\\\,/,g" ;;
+    cygwin)   uwinfix="-e s,\\\\\\\\,/,g" ;;
+    posix-bc) uwinfix="-e s/\\*POSIX(\\(.*\\))/\\1/" ;;
+    vos)      uwinfix="-e s/\#/\\\#/" ;;
+    *)        uwinfix="" ;;
+    esac
+    case "$file" in
+    *.c) filebase=`basename $file .c` ;;
+    *.y) filebase=`basename $file .y` ;;
+    esac
+    case "$file" in
+    */*) finc="-I`echo $file | sed 's#/[^/]*$##'`" ;;
+    *)   finc= ;;
+    esac
+    $echo "Finding dependencies for $filebase$_o"
+    # Below, we strip out all but preprocessor directives.
+    # We have to take care of situations like
+    #       #if defined(FOO) BAR   /* comment line 1
+    #                                 more comment lines */
+    # If we just delete text starting from the '/*' to the end of line, we will
+    # screw up cases like
+    #      #if defined(FOO)    /* comment */ \
+    #          && defined(BAR) /* comment */ \
+    #          && defined(BAZ) /* comment */ \
+    #               etc.
+    # Also, in lines like
+    #      #defined FOO(a,b)    a/**/b
+    # the comment may be important and so needs to be retained.
+    # This code processes the single-line comments first; it assumes there is
+    # at most one straightforward comment per continued preprocessor line,
+    # replacing each non-empty comment (and its surrounding white space) by a
+    # single space.  (sed only has a greedy '*' quantifier, so this doesn't
+    # work right if there are multiple comments per line, and strings can look
+    # like comments to it; both are unlikely in a preprocessor statement.) Any
+    # continuation line is joined, and the process repeated on the enlarged
+    # line as long as there are continuations.  At the end, if there are any
+    # comments remaining, they are either completely empty or are like the
+    # first situation.  The latter are just deleted by first deleting to the
+    # end of line (including preceding white space) things that start with '/*'
+    # and the next char isn't a '*'; then things that start with '/**', but the
+    # next char isn't a '/'.  (Subsequent lines of the comment are irrelevant
+    # and get dropped.)  At the end, we unjoin very long lines to avoid
+    # preprocessor limitations
+    ( $echo "#line 2 \"$file\"";                               \
+      $sed -n <$file                                           \
+	-e "/^${filebase}_init(/q"                             \
+        -e ': tstcont'                                         \
+	-e '/^[ 	]*#/s|[ 	]*/\*..*\*/[ 	]*| |' \
+        -e '/\\$/{'                                            \
+            -e 'N'                                             \
+            -e 'b tstcont'                                     \
+        -e '}'                                                 \
+	-e 's/\\\n//g'                                         \
+	-e '/^#line/d'                                         \
+	-e '/^[ 	]*#/{'                                 \
+	    -e 's|[ 	]*/\*[^*].*$||'                        \
+	    -e 's|[ 	]*/\*\*[^/].*$||'                      \
+            -e 's/.\{255\}/&\\\n/g'                           \
+	    -e p                                               \
+	-e '}' | $grep -v ":" ) >UU/$file.c
+
+    # We're not sure why this was there; the #endif is extraneous on modern z/OS
+    #if [ "$osname" = os390 -a "$file" = perly.c ]; then
+    #   $echo '#endif' >>UU/$file.c
+    #fi
+
+    if [ "$osname" = os390 ]; then
+        $cppstdin $finc -I. $cppflags $cppminus <UU/$file.c |
+        $sed \
+    	    -e '/^#.*<stdin>/d' \
+	    -e '/^#.*"-"/d' \
+	    -e '/^#.*git_version\.h/d' \
+	    -e 's#\.[0-9][0-9]*\.c#'"$file.c#" \
+	    -e 's/^[	 ]*#[	 ]*line/#/' \
+	    -e '/^# *[0-9][0-9]* *[".\/]/!d' \
+	    -e 's/^.*"\(.*\)".*$/'$filebase'\$(OBJ_EXT): \1/' \
+	    -e 's/^# *[0-9][0-9]* \(.*\)$/'$filebase'\$(OBJ_EXT): \1/' \
+	    -e 's|: \./|: |' \
+	    -e 's|\.c\.c|.c|' $uwinfix | \
+        $uniq | $sort | $uniq >>$outfile
+    else
+        $cppstdin $finc -I. $cppflags $cppminus <UU/$file.c >$$.cout 2>$$.cerr
+        $sed \
+	    -e '1d' \
+	    -e '/^#.*<stdin>/d' \
+            -e '/^#.*<builtin>/d' \
+            -e '/^#.*<built-in>/d' \
+            -e '/^#.*<command line>/d' \
+            -e '/^#.*<command-line>/d' \
+	    -e '/^#.*"-"/d' \
+	    -e '/^#.*"\/.*\/"/d' \
+	    -e '/: file path prefix .* never used$/d' \
+	    -e '/^#.*git_version\.h/d' \
+	    -e 's#\.[0-9][0-9]*\.c#'"$file.c#" \
+	    -e 's/^[	 ]*#[	 ]*line/#/' \
+	    -e '/^# *[0-9][0-9]* *[".\/]/!d' \
+	    -e 's/^.*"\(.*\)".*$/'$filebase'\$(OBJ_EXT): \1/' \
+	    -e 's/^# *[0-9][0-9]* \(.*\)$/'$filebase'\$(OBJ_EXT): \1/' \
+	    -e 's|: \./|: |' \
+           -e 's|\.c\.c|.c|' $uwinfix $$.cout $$.cerr| \
+        $uniq | $sort | $uniq >>$outfile
+        rm $$.cout
+        rm $$.cerr
+    fi
+    $echo "$filebase\$(OBJ_EXT): $@" >>$outfile
+
+!NO!SUBS!
+$eunicefix makedepend_file
+chmod +x makedepend_file