Fix issues in the initial DEFAULTS file and version 7 migration script

First, embedding it in the Python script was stupid as \ characters
got mangled, it's now in a separate file.

The strip commands were totally broken, causing certain builds to fail,
which is fixed now.

The default commands for the 'cpan' build system are now in line with
those built into Morph.

The 'autotools' commands have been tweaked too, so that now they produce
identical cache keys to the defaults that are built into Morph (as of
morph.git commit 07df32fbd57477e5808cdbace965edcd0a81348f). This means
that moving to definitions format version 7 should not trigger any
rebuilds of anything.

Also, I added the 'module-build' build system (which was added to Morph
in commit f6613fe1ee6d879192fd4e503cb632b0dcab1fe7), so that it can be
used in the reference systems once they use definitions format version 7.

The migration script will warn if the definitions are version 7 already
but there is no DEFAULTS file, because that's unlikely to be deliberate.

Change-Id: I0a739ef38f521530e0d86d7330d1bcecf0a5bb73

2 files changed, 205 insertions, 184 deletions

diff --git a/migrations/007-defaults-in-definitions.py b/migrations/007-defaults-in-definitions.py
index fbc08dbd..489baf9b 100755
--- a/migrations/007-defaults-in-definitions.py
+++ b/migrations/007-defaults-in-definitions.py
@@ -24,6 +24,7 @@ repo.
 
 
 import os
+import shutil
 import sys
 import warnings
 
@@ -33,188 +34,6 @@ import migrations
 TO_VERSION = 7
 
 
-REFERENCE_DEFAULTS = '''# Baserock definitions defaults
-# =============================
-#
-# The DEFAULTS file is treated specially by Baserock build tools.
-#
-# For more information, see: <http://wiki.baserock.org/definitions/current>.
-
-
-# Predefined build commands
-# -------------------------
-#
-# Common patterns in build instructions can be defined here, which can save
-# users from having to write lots of similar-looking chunk .morph files.
-#
-# There are pre- and post- variants for each set of commands. These exist so
-# you can add more commands without having to copy the defaults. For example,
-# to create an extra symlink after running `make install`, you can use
-# post-install-commands. Since these exist as a way of extending the defaults,
-# you cannot set default values for the pre- and post- commands.
-#
-# The set of environment variables available when these commands are executed
-# is not formally specified right now, but you can assume PREFIX, DESTDIR and
-# MORPH_ARCH are all set.
-#
-build-systems:
-  manual:
-    # The special, default 'no-op' build system.
-    configure-commands: []
-    build-commands: []
-    install-commands: []
-    strip-commands: []
-
-  autotools:
-    # GNU Autoconf and GNU Automake, or anything which follow the same pattern.
-    #
-    # See also: https://github.com/cgwalters/build-api/blob/master/build-api.md
-    configure-commands:
-    - |
-      export NOCONFIGURE=1;
-      if [ -e autogen ]; then ./autogen;
-      elif [ -e autogen.sh ]; then ./autogen.sh;
-      elif [ ! -e ./configure ]; then autoreconf -ivf;
-      fi
-      ./configure --prefix="$PREFIX"
-    build-commands:
-    - make
-    install-commands:
-    - make DESTDIR="$DESTDIR" install
-    strip-commands:
-      # TODO: Make idempotent when files are hardlinks
-      # Strip all ELF binary files that are executable or named like a library.
-      # .so files for C, .cmxs for OCaml and .node for Node.
-      #
-      # The file name and permissions checks are done with the `find` command before
-      # the ELF header is checked with the shell command, because it is a lot cheaper
-      # to check the mode and file name first, because it is a metadata check, rather
-      # than a subprocess and a file read.
-      #
-      # `file` is not used, to keep the dependency requirements down.
-      - &generic-strip-command |
-        find "$DESTDIR" -type f
-          '(' -perm -111 -o -name '*.so*' -o -name '*.cmxs' -o -name '*.node' ')'
-          -exec sh -ec
-          read -n4 hdr <"$1"   # check for elf header
-          if [ "$hdr" != "$(printf \x7ELF)" ]; then
-            exit 0
-          fi
-          debugfile="$DESTDIR$PREFIX/lib/debug/$(basename "$1")"
-          mkdir -p "$(dirname "$debugfile")"
-          objcopy --only-keep-debug "$1" "$debugfile"
-          chmod 644 "$debugfile"
-          strip --remove-section=.comment --remove-section=.note --strip-unneeded "$1"
-          objcopy --add-gnu-debuglink "$debugfile" "$1"' - {} ';'
-
-  python-distutils:
-    # The Python distutils build systems.
-    configure-commands: []
-    build-commands:
-    - python setup.py build
-    install-commands:
-    - python setup.py install --prefix "$PREFIX" --root "$DESTDIR"
-    strip-commands:
-      - *generic-strip-command
-
-  cpan:
-    # The Perl ExtUtil::MakeMaker build system.
-    configure-commands:
-      # This is subject to change, see: https://gerrit.baserock.org/#/c/986/
-      - |
-        perl Makefile.PL INSTALLDIRS=perl
-            INSTALLARCHLIB="$PREFIX/lib/perl"
-            INSTALLPRIVLIB="$PREFIX/lib/perl"
-            INSTALLBIN="$PREFIX/bin"
-            INSTALLSCRIPT="$PREFIX/bin"
-            INSTALLMAN1DIR="$PREFIX/share/man/man1"
-            INSTALLMAN3DIR="$PREFIX/share/man/man3"
-    build-commands:
-    - make
-    install-commands:
-    - make DESTDIR="$DESTDIR" install
-    strip-commands:
-      - *generic-strip-command
-
-  cmake:
-    # The CMake build system.
-    configure-commands:
-    - cmake -DCMAKE_INSTALL_PREFIX="$PREFIX"'
-    build-commands:
-    - make
-    install-commands:
-    - make DESTDIR="$DESTDIR" install
-    strip-commands:
-      - *generic-strip-command
-
-  qmake:
-    # The Qt build system.
-    configure-commands:
-    - qmake -makefile
-    build-commands:
-    - make
-    install-commands:
-    - make INSTALL_ROOT="$DESTDIR" install
-    strip-commands:
-      - *generic-strip-command
-
-
-# Predefined artifact splitting rules
-# -----------------------------------
-#
-# Once a build has completed, you have some files that have been installed into
-# $DESTDIR. The splitting rules control how many 'artifact' tarballs are
-# generated as a result of the build, and which files from $DESTDIR end up in
-# which 'artifact'.
-#
-# The default split rules are defined here. These can be overriden in
-# individual chunk .morph files and stratum .morph files using the 'products'
-# field.
-#
-split-rules:
-  chunk:
-    - artifact: -bins
-      include:
-        - (usr/)?s?bin/.*
-    - artifact: -libs
-      include:
-        - (usr/)?lib(32|64)?/lib[^/]*\.so(\.\d+)*
-        - (usr/)libexec/.*
-    - artifact: -devel
-      include:
-        - (usr/)?include/.*
-        - (usr/)?lib(32|64)?/lib.*\.a
-        - (usr/)?lib(32|64)?/lib.*\.la
-        - (usr/)?(lib(32|64)?|share)/pkgconfig/.*\.pc
-    - artifact: -doc
-      include:
-        - (usr/)?share/doc/.*
-        - (usr/)?share/man/.*
-        - (usr/)?share/info/.*
-    - artifact: -locale
-      include:
-        - (usr/)?share/locale/.*
-        - (usr/)?share/i18n/.*
-        - (usr/)?share/zoneinfo/.*
-    - artifact: -misc
-      include:
-        - .*
-
-  stratum:
-    - artifact: -devel
-      include:
-        - .*-devel
-        - .*-debug
-        - .*-doc
-    - artifact: -runtime
-      include:
-        - .*-bins
-        - .*-libs
-        - .*-locale
-        - .*-misc
-        - .*
-'''
-
 
 try:
     if migrations.check_definitions_version(TO_VERSION - 1):
@@ -223,8 +42,9 @@ try:
                 "DEFAULTS file already exists in these definitions.")
             valid = False
         else:
-            with open('DEFAULTS', 'w') as f:
-                f.write(REFERENCE_DEFAULTS)
+            shutil.copy(
+                'migrations/007-initial-defaults',
+                'DEFAULTS')
             valid = True
 
         if valid:
@@ -236,6 +56,10 @@ try:
                 "Migration failed due to one or more warnings.\n")
             sys.exit(1)
     else:
+        if not os.path.exists('DEFAULTS'):
+            warnings.warn(
+                "These definitions are marked as version 7 but there is no "
+                "DEFAULTS file.")
         sys.stdout.write("Nothing to do.\n")
         sys.exit(0)
 except RuntimeError as e:
diff --git a/migrations/007-initial-defaults b/migrations/007-initial-defaults
new file mode 100644
index 00000000..826ec8d0
--- /dev/null
+++ b/migrations/007-initial-defaults
@@ -0,0 +1,197 @@
+# Baserock definitions defaults
+# =============================
+#
+# The DEFAULTS file is treated specially by Baserock build tools.
+#
+# For more information, see: <http://wiki.baserock.org/definitions/current>.
+
+
+# Predefined build commands
+# -------------------------
+#
+# Common patterns in build instructions can be defined here, which can save
+# users from having to write lots of similar-looking chunk .morph files.
+#
+# There are pre- and post- variants for each set of commands. These exist so
+# you can add more commands without having to copy the defaults. For example,
+# to create an extra symlink after running `make install`, you can use
+# post-install-commands. Since these exist as a way of extending the defaults,
+# you cannot set default values for the pre- and post- commands.
+#
+# The set of environment variables available when these commands are executed
+# is not formally specified right now, but you can assume PREFIX, DESTDIR and
+# MORPH_ARCH are all set.
+#
+build-systems:
+  manual:
+    # The special, default 'no-op' build system.
+    configure-commands: []
+    build-commands: []
+    install-commands: []
+    strip-commands: []
+
+  autotools:
+    # GNU Autoconf and GNU Automake, or anything which follow the same pattern.
+    #
+    # See also: https://github.com/cgwalters/build-api/blob/master/build-api.md
+    configure-commands:
+    - >-
+      export NOCONFIGURE=1;
+      if [ -e autogen ]; then ./autogen;
+      elif [ -e autogen.sh ]; then ./autogen.sh;
+      elif [ ! -e ./configure ]; then autoreconf -ivf;
+      fi
+    - ./configure --prefix="$PREFIX"
+    build-commands:
+    - make
+    install-commands:
+    - make DESTDIR="$DESTDIR" install
+    strip-commands:
+      # TODO: Make idempotent when files are hardlinks
+      # Strip all ELF binary files that are executable or named like a library.
+      # .so files for C, .cmxs for OCaml and .node for Node.
+      #
+      # The file name and permissions checks are done with the `find` command before
+      # the ELF header is checked with the shell command, because it is a lot cheaper
+      # to check the mode and file name first, because it is a metadata check, rather
+      # than a subprocess and a file read.
+      #
+      # `file` is not used, to keep the dependency requirements down.
+      - &generic-strip-command |
+        find "$DESTDIR" -type f \
+          '(' -perm -111 -o -name '*.so*' -o -name '*.cmxs' -o -name '*.node' ')' \
+          -exec sh -ec \
+          'read -n4 hdr <"$1" # check for elf header
+           if [ "$hdr" != "$(printf \\x7fELF)" ]; then
+               exit 0
+           fi
+           debugfile="$DESTDIR$PREFIX/lib/debug/$(basename "$1")"
+           mkdir -p "$(dirname "$debugfile")"
+           objcopy --only-keep-debug "$1" "$debugfile"
+           chmod 644 "$debugfile"
+           strip --remove-section=.comment --remove-section=.note --strip-unneeded "$1"
+           objcopy --add-gnu-debuglink "$debugfile" "$1"' - {} ';'
+
+  python-distutils:
+    # The Python distutils build systems.
+    configure-commands: []
+    build-commands:
+    - python setup.py build
+    install-commands:
+    - python setup.py install --prefix "$PREFIX" --root "$DESTDIR"
+    strip-commands:
+      - *generic-strip-command
+
+  cpan:
+    # The Perl ExtUtil::MakeMaker build system. This is called the 'cpan' build
+    # system for historical reasons.
+    #
+    # To install perl distributions into the correct location in our chroot
+    # we need to set PREFIX to <destdir>/<prefix> in the configure-commands.
+    #
+    # The mapping between PREFIX and the final installation
+    # directories is complex and depends upon the configuration of perl
+    # see,
+    # https://metacpan.org/pod/distribution/perl/INSTALL#Installation-Directories
+    # and ExtUtil::MakeMaker's documentation for more details.
+    configure-commands:
+      - perl Makefile.PL PREFIX=$DESTDIR$PREFIX
+    build-commands:
+    - make
+    install-commands:
+    - make install
+    strip-commands:
+      - *generic-strip-command
+
+  module-build:
+    # The Module::Build build system
+    #
+    # See the comment in ExtUtilsMakeMakerBuildSystem to see why --prefix is
+    # set to $DESTDIR$PREFIX here (--prefix in Module::Build has the same
+    # meaning as PREFIX in ExtUtils::MakeMaker).
+    configure-commands:
+      - perl Build.PL --prefix "$DESTDIR$PREFIX"
+    build-commands:
+      - ./Build
+    install-commands:
+      - ./Build install
+    strip-commands:
+      - *generic-strip-command
+
+  cmake:
+    # The CMake build system.
+    configure-commands:
+    - cmake -DCMAKE_INSTALL_PREFIX="$PREFIX"'
+    build-commands:
+    - make
+    install-commands:
+    - make DESTDIR="$DESTDIR" install
+    strip-commands:
+      - *generic-strip-command
+
+  qmake:
+    # The Qt build system.
+    configure-commands:
+    - qmake -makefile
+    build-commands:
+    - make
+    install-commands:
+    - make INSTALL_ROOT="$DESTDIR" install
+    strip-commands:
+      - *generic-strip-command
+
+
+# Predefined artifact splitting rules
+# -----------------------------------
+#
+# Once a build has completed, you have some files that have been installed into
+# $DESTDIR. The splitting rules control how many 'artifact' tarballs are
+# generated as a result of the build, and which files from $DESTDIR end up in
+# which 'artifact'.
+#
+# The default split rules are defined here. These can be overriden in
+# individual chunk .morph files and stratum .morph files using the 'products'
+# field.
+#
+split-rules:
+  chunk:
+    - artifact: -bins
+      include:
+        - (usr/)?s?bin/.*
+    - artifact: -libs
+      include:
+        - (usr/)?lib(32|64)?/lib[^/]*\.so(\.\d+)*
+        - (usr/)libexec/.*
+    - artifact: -devel
+      include:
+        - (usr/)?include/.*
+        - (usr/)?lib(32|64)?/lib.*\.a
+        - (usr/)?lib(32|64)?/lib.*\.la
+        - (usr/)?(lib(32|64)?|share)/pkgconfig/.*\.pc
+    - artifact: -doc
+      include:
+        - (usr/)?share/doc/.*
+        - (usr/)?share/man/.*
+        - (usr/)?share/info/.*
+    - artifact: -locale
+      include:
+        - (usr/)?share/locale/.*
+        - (usr/)?share/i18n/.*
+        - (usr/)?share/zoneinfo/.*
+    - artifact: -misc
+      include:
+        - .*
+
+  stratum:
+    - artifact: -devel
+      include:
+        - .*-devel
+        - .*-debug
+        - .*-doc
+    - artifact: -runtime
+      include:
+        - .*-bins
+        - .*-libs
+        - .*-locale
+        - .*-misc
+        - .*