Move content to migrations directory

13 files changed, 1422 insertions, 0 deletions

diff --git a/migrations/000-version-info.py b/migrations/000-version-info.py
new file mode 100755
index 0000000..2bff51f
--- /dev/null
+++ b/migrations/000-version-info.py
@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+# Copyright (C) 2015  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+'''Migration to Baserock Definitions format version 0.
+
+The format of version 0 is not formally specified, except by the Morph
+codebase. It marks the starting point of the work to formalise the Baserock
+Definitions format.
+
+'''
+
+
+import os
+import sys
+
+import migrations
+
+
+TO_VERSION = 0
+
+
+try:
+    if os.path.exists('./VERSION'):
+        # This will raise an exception if the VERSION file is invalid, which
+        # might be useful.
+        migrations.check_definitions_version(TO_VERSION)
+
+        sys.stdout.write("Nothing to do.\n")
+        sys.exit(0)
+    else:
+        sys.stdout.write("No VERSION file found, creating one.\n")
+        migrations.set_definitions_version(TO_VERSION)
+        sys.exit(0)
+except RuntimeError as e:
+    sys.stderr.write("Error: %s\n" % e.message)
+    sys.exit(1)
diff --git a/migrations/001-empty-build-depends.py b/migrations/001-empty-build-depends.py
new file mode 100755
index 0000000..5d4296d
--- /dev/null
+++ b/migrations/001-empty-build-depends.py
@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+# Copyright (C) 2015  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+'''Migration to Baserock Definitions format version 1.
+
+In version 1, the 'build-depends' parameter was made optional. It was
+previously mandatory to specify 'build-depends' for a chunk, even if it was an
+empty list.
+
+'''
+
+
+import sys
+import warnings
+
+import migrations
+
+
+TO_VERSION = 1
+
+
+def check_empty_build_depends(contents, filename):
+    assert contents['kind'] == 'stratum'
+
+    valid = True
+    for chunk_ref in contents.get('chunks', []):
+        if 'build-depends' not in chunk_ref:
+            chunk_ref_name = chunk_ref.get('name', chunk_ref.get('morph'))
+            warnings.warn(
+                "%s:%s has no build-depends field, which "
+                "is invalid in definitions version 0." %
+                (contents['name'], chunk_ref_name))
+            valid = False
+
+    return valid
+
+
+def remove_empty_build_depends(contents, filename):
+    assert contents['kind'] == 'stratum'
+
+    changed = False
+    for chunk_ref in contents.get('chunks', []):
+        if 'build-depends' in chunk_ref:
+            if len(chunk_ref['build-depends']) == 0:
+                del chunk_ref['build-depends']
+                changed = True
+
+    return changed
+
+
+try:
+    if migrations.check_definitions_version(TO_VERSION - 1):
+        success = migrations.process_definitions(
+            path='.', kinds=['stratum'],
+            validate_cb=check_empty_build_depends,
+            modify_cb=remove_empty_build_depends)
+        if success:
+            migrations.set_definitions_version(TO_VERSION)
+            sys.stdout.write("Migration completed successfully.\n")
+            sys.exit(0)
+        else:
+            sys.stderr.write("Migration failed due to warnings.\n")
+            sys.exit(1)
+    else:
+        sys.stdout.write("Nothing to do.\n")
+        sys.exit(0)
+except RuntimeError as e:
+    sys.stderr.write("Error: %s\n" % e.message)
+    sys.exit(1)
diff --git a/migrations/002-missing-chunk-morphs.py b/migrations/002-missing-chunk-morphs.py
new file mode 100755
index 0000000..2c93804
--- /dev/null
+++ b/migrations/002-missing-chunk-morphs.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+# Copyright (C) 2015  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+'''Migration to Baserock Definitions format version 2.
+
+In version 2, the processing of the 'morph:' field within stratum .morph files
+became more strict.  This migration checks whether definitions are valid
+according to version 2 of the format.
+
+'''
+
+
+import os
+import sys
+import warnings
+
+import migrations
+
+
+TO_VERSION = 2
+
+
+def check_missing_chunk_morphs(contents, filename):
+    assert contents['kind'] == 'stratum'
+
+    valid = True
+
+    for chunk_ref in contents.get('chunks', []):
+        if 'morph' in chunk_ref:
+            chunk_path = os.path.join('.', chunk_ref['morph'])
+            if not os.path.exists(chunk_path):
+                # There's no way we can really fix this, so
+                # just warn and say the migration failed.
+                warnings.warn(
+                    "%s points to non-existant file %s" %
+                    (contents['name'], chunk_ref['morph']))
+                valid = False
+
+    return valid
+
+
+try:
+    if migrations.check_definitions_version(TO_VERSION - 1):
+        safe_to_migrate = migrations.process_definitions(
+            kinds=['stratum'], validate_cb=check_missing_chunk_morphs)
+
+        if not safe_to_migrate:
+            sys.stderr.write(
+                "Migration failed due to one or more warnings.\n")
+            sys.exit(1)
+        else:
+            migrations.set_definitions_version(TO_VERSION)
+            sys.stdout.write("Migration completed successfully.\n")
+            sys.exit(0)
+    else:
+        sys.stdout.write("Nothing to do.\n")
+        sys.exit(0)
+except RuntimeError as e:
+    sys.stderr.write("Error: %s\n" % e.message)
+    sys.exit(1)
diff --git a/migrations/003-arch-armv5.py b/migrations/003-arch-armv5.py
new file mode 100755
index 0000000..58eb79d
--- /dev/null
+++ b/migrations/003-arch-armv5.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+# Copyright (C) 2015  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+'''Migration to Baserock Definitions format version 3.
+
+In version 3, there were two additions:
+
+    - the 'armv5' architecture
+    - the install-essential-files.configure configuration extension
+
+This migration checks that neither of these are in use in the input (version 2)
+definitions. Which isn't particularly useful.
+
+'''
+
+
+import sys
+import warnings
+
+import migrations
+
+
+TO_VERSION = 3
+
+
+def check_arch(contents, filename):
+    assert contents['kind'] == 'system'
+
+    valid = True
+
+    if contents['arch'] == 'armv5':
+        warnings.warn(
+            "%s uses armv5 architecture that is not understood until version "
+            "3." % filename)
+        valid = False
+
+    return valid
+
+
+def check_configuration_extensions(contents, filename):
+    assert contents['kind'] == 'system'
+
+    valid = True
+
+    for extension in contents.get('configuration-extensions', []):
+        if extension == 'install-essential-files':
+            warnings.warn(
+                "%s uses install-essential-files.configure extension, which "
+                "was not present in morph.git until commit 423dc974a61f1c0 "
+                "(tag baserock-definitions-v3)." % filename)
+            valid = False
+
+    return valid
+
+
+try:
+    if migrations.check_definitions_version(TO_VERSION - 1):
+        safe_to_migrate = migrations.process_definitions(
+            kinds=['system'], validate_cb=check_arch)
+        safe_to_migrate = migrations.process_definitions(
+            kinds=['system'], validate_cb=check_configuration_extensions)
+
+        if not safe_to_migrate:
+            sys.stderr.write(
+                "Migration failed due to one or more warnings.\n")
+            sys.exit(1)
+        else:
+            migrations.set_definitions_version(TO_VERSION)
+            sys.stdout.write("Migration completed successfully.\n")
+            sys.exit(0)
+    else:
+        sys.stdout.write("Nothing to do.\n")
+        sys.exit(0)
+except RuntimeError as e:
+    sys.stderr.write("Error: %s\n" % e.message)
+    sys.exit(1)
diff --git a/migrations/004-install-files-overwrite-symlink.py b/migrations/004-install-files-overwrite-symlink.py
new file mode 100755
index 0000000..6853dcb
--- /dev/null
+++ b/migrations/004-install-files-overwrite-symlink.py
@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+# Copyright (C) 2015  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+'''Migration to Baserock Definitions format version 4.
+
+This change to the format was made to work around a bug in a deployment
+extension present in morph.git.
+
+Automated migration is not really possible for this change, and unless you
+are experiencing the install-files.configure extension crashing, you can ignore
+it completely.
+
+We have now moved all .configure and .write extensions into the definitions.git
+repository. Changes like this no longer require a marking a new version of the
+Baserock definitions format in order to prevent build tools crashing.
+
+Morph commit c373f5a403b0ec introduces version 4 of the definitions format. In
+older versions of Morph the install-files.configure extension would crash if it
+tried to overwrite a symlink. This bug is fixed in the version of Morph that
+can build definitions version 4.
+
+If you need to overwrite a symlink at deploytime using install-files.configure,
+please use VERSION to 4 or above in your definitions.git repo so older versions
+of Morph gracefully refuse to deploy, instead of crashing.
+
+'''
+
+
+import sys
+
+import migrations
+
+
+TO_VERSION = 4
+
+
+try:
+    if migrations.check_definitions_version(TO_VERSION - 1):
+        migrations.set_definitions_version(TO_VERSION)
+        sys.stdout.write("Migration completed successfully.\n")
+        sys.exit(0)
+    else:
+        sys.stdout.write("Nothing to do.\n")
+        sys.exit(0)
+except RuntimeError as e:
+    sys.stderr.write("Error: %s\n" % e.message)
+    sys.exit(1)
diff --git a/migrations/005-strip-commands.py b/migrations/005-strip-commands.py
new file mode 100755
index 0000000..da3de94
--- /dev/null
+++ b/migrations/005-strip-commands.py
@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+# Copyright (C) 2015  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+'''Migration to Baserock Definitions format version 5.
+
+Version 5 of the definitions format adds a 'strip-commands' field that can
+be set in chunk definitions.
+
+Version 5 also allows deployment extensions to live in definitions.git instead
+of morph.git. This greatly reduces the interface surface of the Baserock
+definitions format specification, because we no longer have to mark a new
+version of the definitions format each time an extension in morph.git is added,
+removed, or changes its API in any way.
+
+In commit 6f4929946 of git://git.baserock.org/baserock/baserock/definitions.git
+the deployment extensions were moved into an extensions/ subdirectory, and the
+system and cluster .morph files that referred to them were all updated to
+prepend 'extension/' to the filenames. This migration doesn't (re)do that
+change.
+
+'''
+
+
+import sys
+import warnings
+
+import migrations
+
+
+TO_VERSION = 5
+
+
+def check_strip_commands(contents, filename):
+    assert contents['kind'] == 'chunk'
+
+    valid = True
+
+    if 'strip-commands' in contents:
+        warnings.warn(
+            "%s has strip-commands, which are not valid until version 5" %
+            filename)
+        valid = False
+
+    return valid
+
+
+try:
+    if migrations.check_definitions_version(TO_VERSION - 1):
+        safe_to_migrate = migrations.process_definitions(
+            kinds=['chunk'], validate_cb=check_strip_commands)
+
+        if not safe_to_migrate:
+            sys.stderr.write(
+                "Migration failed due to one or more warnings.\n")
+            sys.exit(1)
+        else:
+            migrations.set_definitions_version(TO_VERSION)
+            sys.stdout.write("Migration completed successfully.\n")
+            sys.exit(0)
+    else:
+        sys.stdout.write("Nothing to do.\n")
+        sys.exit(0)
+except RuntimeError as e:
+    sys.stderr.write("Error: %s\n" % e.message)
+    sys.exit(1)
diff --git a/migrations/006-specify-build-system.py b/migrations/006-specify-build-system.py
new file mode 100755
index 0000000..b66736c
--- /dev/null
+++ b/migrations/006-specify-build-system.py
@@ -0,0 +1,354 @@
+#!/usr/bin/env python
+# Copyright (C) 2015  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+# THIS MIGRATION REQUIRES NETWORK ACCESS TO A BASEROCK GIT CACHE SERVER! If
+# you do not have your own Trove, or don't know what a Trove is, it should
+# work as-is, provided you have internet access that allows access to
+# http://git.baserock.org:8080/.
+#
+# If you do have your own Trove, change the value of TROVE_HOST below to
+# point to it.
+#
+# This migration uses the same autodetection mechanism that Morph and YBD use
+# at build time, in order to fill in the 'build-system' field in strata where
+# it is now needed.
+
+
+'''Migration to Baserock Definitions format version 6.
+
+In definitions version 6, build system autodetection no longer happens. This
+means that any chunk that wants to use one of the predefined build systems
+(those built into Morph) must say so explicitly, using the 'build-system'
+field.
+
+The build-system field for a chunk can now be specified within a stratum that
+contains it. Previously you needed to add a .morph file for a chunk in order to
+specify its build-system, but we want to avoid needing a .morph file for
+components that follow standard patterns.
+
+Previously, if build-system wasn't given, Morph would scan the contents of the
+chunk's Git repo and try to autodetect which build system was used. This could
+be slow, could fail in confusing ways, and meant that to fully parse
+definitions you needed access to some or all of the repos they referenced.
+
+The chosen build-system affects which predefined command sequences are set for
+a chunk. It is valid to omit the field if a chunk has its own build commands
+defined in a .morph file. When listing the chunks included in a stratum, either
+'morph' or 'build-system' must be specified, but not both (to avoid the
+possibility of conflicting values).
+
+'''
+
+
+import requests
+import yaml
+
+import logging
+import os
+import sys
+import warnings
+
+import migrations
+
+
+TROVE_HOST = 'git.baserock.org'
+
+REPO_ALIASES = {
+    'baserock:': 'git://%s/baserock/' % TROVE_HOST,
+    'freedesktop:': 'git://anongit.freedesktop.org/',
+    'github:': 'git://github.com/',
+    'gnome:': 'git://git.gnome.org/',
+    'upstream:': 'git://%s/delta/' % TROVE_HOST,
+}
+
+GIT_CACHE_SERVER_URL = 'http://%s:8080/' % TROVE_HOST
+
+FAIL_ON_REMOTE_CACHE_ERRORS = False
+
+
+TO_VERSION = 6
+
+
+# From ybd.git file repos.py at commit eb3bf397ba729387f0d4145a8df8d3c1f9eb707f
+
+def get_repo_url(repo):
+    for alias, url in REPO_ALIASES.items():
+        repo = repo.replace(alias, url)
+    if repo.endswith('.git'):
+        repo = repo[:-4]
+    return repo
+
+
+# Based on morph.git file buildsystem.py at commit a7748f9cdaaf4112c30d7c1.
+#
+# I have copied and pasted this code here, as it should not be needed anywhere
+# once everyone has migrated to definitions version 6.
+
+class BuildSystem(object):
+    def used_by_project(self, file_list):
+        '''Does a project use this build system?
+
+        ``exists`` is a function that returns a boolean telling if a
+        filename, relative to the project source directory, exists or not.
+
+        '''
+        raise NotImplementedError()  # pragma: no cover
+
+
+class ManualBuildSystem(BuildSystem):
+
+    '''A manual build system where the morphology must specify all commands.'''
+
+    name = 'manual'
+
+    def used_by_project(self, file_list):
+        return False
+
+
+class DummyBuildSystem(BuildSystem):
+
+    '''A dummy build system, useful for debugging morphologies.'''
+
+    name = 'dummy'
+
+    def used_by_project(self, file_list):
+        return False
+
+
+class AutotoolsBuildSystem(BuildSystem):
+
+    '''The automake/autoconf/libtool holy trinity.'''
+
+    name = 'autotools'
+
+    def used_by_project(self, file_list):
+        indicators = [
+            'autogen',
+            'autogen.sh',
+            'configure',
+            'configure.ac',
+            'configure.in',
+            'configure.in.in',
+        ]
+
+        return any(x in file_list for x in indicators)
+
+
+class PythonDistutilsBuildSystem(BuildSystem):
+
+    '''The Python distutils build systems.'''
+
+    name = 'python-distutils'
+
+    def used_by_project(self, file_list):
+        indicators = [
+            'setup.py',
+        ]
+
+        return any(x in file_list for x in indicators)
+
+
+class CPANBuildSystem(BuildSystem):
+
+    '''The Perl cpan build system.'''
+
+    name = 'cpan'
+
+    def used_by_project(self, file_list):
+        indicators = [
+            'Makefile.PL',
+        ]
+
+        return any(x in file_list for x in indicators)
+
+
+class CMakeBuildSystem(BuildSystem):
+
+    '''The cmake build system.'''
+
+    name = 'cmake'
+
+    def used_by_project(self, file_list):
+        indicators = [
+            'CMakeLists.txt',
+        ]
+
+        return any(x in file_list for x in indicators)
+
+
+class QMakeBuildSystem(BuildSystem):
+
+    '''The Qt build system.'''
+
+    name = 'qmake'
+
+    def used_by_project(self, file_list):
+        indicator = '.pro'
+
+        for x in file_list:
+            if x.endswith(indicator):
+                return True
+
+        return False
+
+
+build_systems = [
+    ManualBuildSystem(),
+    AutotoolsBuildSystem(),
+    PythonDistutilsBuildSystem(),
+    CPANBuildSystem(),
+    CMakeBuildSystem(),
+    QMakeBuildSystem(),
+    DummyBuildSystem(),
+]
+
+
+def detect_build_system(file_list):
+    '''Automatically detect the build system, if possible.
+
+    If the build system cannot be detected automatically, return None.
+    For ``exists`` see the ``BuildSystem.exists`` method.
+
+    '''
+    for bs in build_systems:
+        if bs.used_by_project(file_list):
+            return bs
+    return None
+
+
+## End of code based on morph.git file buildsystem.py.
+
+def get_toplevel_file_list_from_repo(url, ref):
+    '''Try to list the set of files in the root directory of the repo at 'url'.
+
+    '''
+    try:
+        response = requests.get(
+            GIT_CACHE_SERVER_URL + '1.0/trees',
+            params={'repo': url, 'ref': ref},
+            headers={'Accept': 'application/json'},
+            timeout=9)
+        logging.debug("Got response: %s" % response)
+        try:
+            response.raise_for_status()
+            toplevel_tree = response.json()['tree']
+        except Exception as e:
+            raise RuntimeError(
+                "Unexpected response from server %s for repo %s: %s" %
+                (GIT_CACHE_SERVER_URL, url, e.message))
+        toplevel_filenames = toplevel_tree.keys()
+    except requests.exceptions.ConnectionError as e:
+        raise RuntimeError("Unable to connect to cache server %s while trying "
+                           "to query file list of repo %s. Error was: %s" %
+                           (GIT_CACHE_SERVER_URL, url, e.message))
+    return toplevel_filenames
+
+
+def validate_chunk_refs(contents, filename):
+    assert contents['kind'] == 'stratum'
+
+    valid = True
+    for chunk_ref in contents.get('chunks', []):
+        if chunk_ref.get('morph') is None:
+            # No chunk .morph file -- this stratum was relying on build-system
+            # autodetection here.
+
+            if 'repo' not in chunk_ref:
+                warnings.warn("%s: Chunk %s doesn't specify a source repo." %
+                              (filename, chunk_ref.get('name')))
+                valid = False
+
+            if 'ref' not in chunk_ref:
+                warnings.warn("%s: Chunk %s doesn't specify a source ref." %
+                              (filename, chunk_ref.get('name')))
+                valid = False
+    return valid
+
+
+def move_dict_entry_last(dict_object, key, error_if_missing=False):
+    '''Move an entry in a ordered dict to the end.'''
+
+    # This is a hack, I couldn't find a method on the 'CommentedMap' type dict
+    # that we receive from ruamel.yaml that would allow doing this neatly.
+    if key in dict_object:
+        value = dict_object[key]
+        del dict_object[key]
+        dict_object[key] = value
+    else:
+        if error_if_missing:
+            raise KeyError(key)
+
+
+def ensure_buildsystem_defined_where_needed(contents, filename):
+    assert contents['kind'] == 'stratum'
+
+    changed = False
+    for chunk_ref in contents.get('chunks', []):
+        if chunk_ref.get('morph') is None:
+            # No chunk .morph file -- this stratum was relying on build-system
+            # autodetection here.
+
+            chunk_git_url = get_repo_url(chunk_ref['repo'])
+            chunk_git_ref = chunk_ref['ref']
+
+            try:
+                toplevel_file_list = get_toplevel_file_list_from_repo(
+                    chunk_git_url, chunk_git_ref)
+            except Exception as e:
+                warnings.warn(str(e))
+                message = (
+                    "Unable to look up one or more repos on remote Git "
+                    "server %s. If you are using a Trove that is not %s, "
+                    "please edit the TROVE_HOST constant in this script "
+                    "and run it again." % (TROVE_HOST, TROVE_HOST))
+                if FAIL_ON_REMOTE_CACHE_ERRORS:
+                    raise RuntimeError(message)
+                else:
+                    warnings.warn(message)
+                    continue
+
+            logging.debug(
+                '%s: got file list %s', chunk_git_url, toplevel_file_list)
+            build_system = detect_build_system(toplevel_file_list)
+
+            chunk_ref['build-system'] = build_system.name
+            move_dict_entry_last(chunk_ref, 'build-depends')
+
+            changed = True
+
+    return changed
+
+
+try:
+    if migrations.check_definitions_version(TO_VERSION - 1):
+        success = migrations.process_definitions(
+            kinds=['stratum'],
+            validate_cb=validate_chunk_refs,
+            modify_cb=ensure_buildsystem_defined_where_needed)
+        if not success:
+            sys.stderr.write(
+                "Migration failed due to one or more warnings.\n")
+            sys.exit(1)
+        else:
+            migrations.set_definitions_version(TO_VERSION)
+            sys.stderr.write("Migration completed successfully.\n")
+            sys.exit(0)
+    else:
+        sys.stderr.write("Nothing to do.\n")
+        sys.exit(0)
+except RuntimeError as e:
+    sys.stderr.write("Error: %s\n" % e.message)
+    sys.exit(1)
diff --git a/migrations/007-defaults-in-definitions.py b/migrations/007-defaults-in-definitions.py
new file mode 100755
index 0000000..489baf9
--- /dev/null
+++ b/migrations/007-defaults-in-definitions.py
@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+# Copyright (C) 2015  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+'''Migration to Baserock Definitions format version 7.
+
+Definitions version 7 adds a file named DEFAULTS which sets the default
+build commands and default split rules for the set of definitions in that
+repo.
+
+'''
+
+
+import os
+import shutil
+import sys
+import warnings
+
+import migrations
+
+
+TO_VERSION = 7
+
+
+
+try:
+    if migrations.check_definitions_version(TO_VERSION - 1):
+        if os.path.exists('DEFAULTS'):
+            warnings.warn(
+                "DEFAULTS file already exists in these definitions.")
+            valid = False
+        else:
+            shutil.copy(
+                'migrations/007-initial-defaults',
+                'DEFAULTS')
+            valid = True
+
+        if valid:
+            migrations.set_definitions_version(TO_VERSION)
+            sys.stdout.write("Migration completed successfully.\n")
+            sys.exit(0)
+        else:
+            sys.stderr.write(
+                "Migration failed due to one or more warnings.\n")
+            sys.exit(1)
+    else:
+        if not os.path.exists('DEFAULTS'):
+            warnings.warn(
+                "These definitions are marked as version 7 but there is no "
+                "DEFAULTS file.")
+        sys.stdout.write("Nothing to do.\n")
+        sys.exit(0)
+except RuntimeError as e:
+    sys.stderr.write("Error: %s\n" % e.message)
+    sys.exit(1)
diff --git a/migrations/007-initial-defaults b/migrations/007-initial-defaults
new file mode 100644
index 0000000..ab034a0
--- /dev/null
+++ b/migrations/007-initial-defaults
@@ -0,0 +1,199 @@
+# Baserock definitions defaults
+# =============================
+#
+# The DEFAULTS file is treated specially by Baserock build tools.
+#
+# For more information, see: <http://wiki.baserock.org/definitions/current>.
+
+
+# Predefined build commands
+# -------------------------
+#
+# Common patterns in build instructions can be defined here, which can save
+# users from having to write lots of similar-looking chunk .morph files.
+#
+# There are pre- and post- variants for each set of commands. These exist so
+# you can add more commands without having to copy the defaults. For example,
+# to create an extra symlink after running `make install`, you can use
+# post-install-commands. Since these exist as a way of extending the defaults,
+# you cannot set default values for the pre- and post- commands.
+#
+# The set of environment variables available when these commands are executed
+# is not formally specified right now, but you can assume PREFIX, DESTDIR and
+# MORPH_ARCH are all set.
+#
+build-systems:
+  manual:
+    # The special, default 'no-op' build system.
+    configure-commands: []
+    build-commands: []
+    install-commands: []
+    strip-commands: []
+
+  autotools:
+    # GNU Autoconf and GNU Automake, or anything which follow the same pattern.
+    #
+    # See also: https://github.com/cgwalters/build-api/blob/master/build-api.md
+    configure-commands:
+    - >-
+      export NOCONFIGURE=1;
+      if [ -e autogen ]; then ./autogen;
+      elif [ -e autogen.sh ]; then ./autogen.sh;
+      elif [ -e bootstrap ]; then ./bootstrap;
+      elif [ -e bootstrap.sh ]; then ./bootstrap.sh;
+      elif [ ! -e ./configure ]; then autoreconf -ivf;
+      fi
+    - ./configure --prefix="$PREFIX"
+    build-commands:
+    - make
+    install-commands:
+    - make DESTDIR="$DESTDIR" install
+    strip-commands:
+      # TODO: Make idempotent when files are hardlinks
+      # Strip all ELF binary files that are executable or named like a library.
+      # .so files for C, .cmxs for OCaml and .node for Node.
+      #
+      # The file name and permissions checks are done with the `find` command before
+      # the ELF header is checked with the shell command, because it is a lot cheaper
+      # to check the mode and file name first, because it is a metadata check, rather
+      # than a subprocess and a file read.
+      #
+      # `file` is not used, to keep the dependency requirements down.
+      - &generic-strip-command |
+        find "$DESTDIR" -type f \
+          '(' -perm -111 -o -name '*.so*' -o -name '*.cmxs' -o -name '*.node' ')' \
+          -exec sh -ec \
+          'read -n4 hdr <"$1" # check for elf header
+           if [ "$hdr" != "$(printf \\x7fELF)" ]; then
+               exit 0
+           fi
+           debugfile="$DESTDIR$PREFIX/lib/debug/$(basename "$1")"
+           mkdir -p "$(dirname "$debugfile")"
+           objcopy --only-keep-debug "$1" "$debugfile"
+           chmod 644 "$debugfile"
+           strip --remove-section=.comment --remove-section=.note --strip-unneeded "$1"
+           objcopy --add-gnu-debuglink "$debugfile" "$1"' - {} ';'
+
+  python-distutils:
+    # The Python distutils build systems.
+    configure-commands: []
+    build-commands:
+    - python setup.py build
+    install-commands:
+    - python setup.py install --prefix "$PREFIX" --root "$DESTDIR"
+    strip-commands:
+      - *generic-strip-command
+
+  cpan:
+    # The Perl ExtUtil::MakeMaker build system. This is called the 'cpan' build
+    # system for historical reasons.
+    #
+    # To install perl distributions into the correct location in our chroot
+    # we need to set PREFIX to <destdir>/<prefix> in the configure-commands.
+    #
+    # The mapping between PREFIX and the final installation
+    # directories is complex and depends upon the configuration of perl
+    # see,
+    # https://metacpan.org/pod/distribution/perl/INSTALL#Installation-Directories
+    # and ExtUtil::MakeMaker's documentation for more details.
+    configure-commands:
+      - perl Makefile.PL PREFIX=$DESTDIR$PREFIX
+    build-commands:
+    - make
+    install-commands:
+    - make install
+    strip-commands:
+      - *generic-strip-command
+
+  module-build:
+    # The Module::Build build system
+    #
+    # See the comment in ExtUtilsMakeMakerBuildSystem to see why --prefix is
+    # set to $DESTDIR$PREFIX here (--prefix in Module::Build has the same
+    # meaning as PREFIX in ExtUtils::MakeMaker).
+    configure-commands:
+      - perl Build.PL --prefix "$DESTDIR$PREFIX"
+    build-commands:
+      - ./Build
+    install-commands:
+      - ./Build install
+    strip-commands:
+      - *generic-strip-command
+
+  cmake:
+    # The CMake build system.
+    configure-commands:
+    - cmake -DCMAKE_INSTALL_PREFIX="$PREFIX"
+    build-commands:
+    - make
+    install-commands:
+    - make DESTDIR="$DESTDIR" install
+    strip-commands:
+      - *generic-strip-command
+
+  qmake:
+    # The Qt build system.
+    configure-commands:
+    - qmake -makefile
+    build-commands:
+    - make
+    install-commands:
+    - make INSTALL_ROOT="$DESTDIR" install
+    strip-commands:
+      - *generic-strip-command
+
+
+# Predefined artifact splitting rules
+# -----------------------------------
+#
+# Once a build has completed, you have some files that have been installed into
+# $DESTDIR. The splitting rules control how many 'artifact' tarballs are
+# generated as a result of the build, and which files from $DESTDIR end up in
+# which 'artifact'.
+#
+# The default split rules are defined here. These can be overriden in
+# individual chunk .morph files and stratum .morph files using the 'products'
+# field.
+#
+split-rules:
+  chunk:
+    - artifact: -bins
+      include:
+        - (usr/)?s?bin/.*
+    - artifact: -libs
+      include:
+        - (usr/)?lib(32|64)?/lib[^/]*\.so(\.\d+)*
+        - (usr/)libexec/.*
+    - artifact: -devel
+      include:
+        - (usr/)?include/.*
+        - (usr/)?lib(32|64)?/lib.*\.a
+        - (usr/)?lib(32|64)?/lib.*\.la
+        - (usr/)?(lib(32|64)?|share)/pkgconfig/.*\.pc
+    - artifact: -doc
+      include:
+        - (usr/)?share/doc/.*
+        - (usr/)?share/man/.*
+        - (usr/)?share/info/.*
+    - artifact: -locale
+      include:
+        - (usr/)?share/locale/.*
+        - (usr/)?share/i18n/.*
+        - (usr/)?share/zoneinfo/.*
+    - artifact: -misc
+      include:
+        - .*
+
+  stratum:
+    - artifact: -devel
+      include:
+        - .*-devel
+        - .*-debug
+        - .*-doc
+    - artifact: -runtime
+      include:
+        - .*-bins
+        - .*-libs
+        - .*-locale
+        - .*-misc
+        - .*
diff --git a/migrations/GUIDELINES b/migrations/GUIDELINES
new file mode 100644
index 0000000..3694e2c
--- /dev/null
+++ b/migrations/GUIDELINES
@@ -0,0 +1,35 @@
+Guidelines for writing migrations
+---------------------------------
+
+All changes to the definitions format must have a migration, but it is valid
+for the migration to do nothing except update the version number (see
+004-install-files-overwrite-symlink.py for an example of that).
+
+This small set of rules exists to ensure that the migrations are consistent and
+easy to understand. If you are writing a migration and these rules don't make
+any sense, we should probably change them. Please sign up to the
+baserock-dev@baserock.org mailing list to suggest the change.
+
+- Write migrations in Python. They must be valid Python 3. For now, since
+  only Python 2 is available in Baserock 'build' and 'devel' reference systems
+  up to the 15.25 release of Baserock, they must also be valid Python 2.
+
+- Don't use any external libraries.
+
+- Follow the existing file naming pattern, and the existing code convention.
+
+- Keep the migration code as simple as possible.
+
+- Avoid crashing on malformed input data, where practical. For example, use
+  contents.get('field') instead of contents['field'] to avoid crashing when
+  'field' is not present. The idea of this is to avoid a "cascade of errors"
+  problem when running the migrations on bad inputs. It is confusing when
+  migrations break on problems that are unrelated to the actual area where
+  they operate, even if they are theoretically "within their rights" to do so.
+
+- Migrate the definitions in line with current best practices. For example,
+  migrations/001-empty-build-depends.py doesn't need to remove empty
+  build-depends fields: they are still valid in version 1 of the format. But
+  best practice is now to remove them. Users who don't agree with this practice
+  can choose to not run that migration, which can be done with `chmod -x
+  migrations/xxx.py`.
diff --git a/migrations/indent b/migrations/indent
new file mode 100755
index 0000000..8d6f034
--- /dev/null
+++ b/migrations/indent
@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+# Copyright (C) 2015  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+'''Automatically reformat a set of Baserock definition files.
+
+This tool expects to be able to use ruamel.yaml to load and write YAML.
+It will totally ruin things if used with PyYAML.
+
+It makes sense to run this script on your definitions, and check through
+and commit the result, before running any of the automated migrations. This
+way, you can be sure that the migrations will only change things that they need 
+to in the .morph files.
+
+'''
+
+
+import migrations
+
+
+def force_rewrite(contents, filename):
+    return True
+
+migrations.process_definitions(path='.', modify_cb=force_rewrite)
diff --git a/migrations/migrations.py b/migrations/migrations.py
new file mode 100644
index 0000000..22ed132
--- /dev/null
+++ b/migrations/migrations.py
@@ -0,0 +1,228 @@
+#!/usr/bin/env python
+# Copyright (C) 2015  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+'''Tools for migrating Baserock definitions from one format version to another.
+
+'''
+
+
+# ruamel.yaml is a fork of PyYAML which allows rewriting YAML files without
+# destroying all of the comments, ordering and formatting. The more
+# widely-used PyYAML library will produce output totally different to the
+# input file in most cases.
+#
+# See: <https://bitbucket.org/ruamel/yaml>
+import ruamel.yaml as yaml
+
+import logging
+import os
+import warnings
+
+
+# Uncomment this to cause all log messages to be written to stdout. By
+# default they are hidden, but if you are debugging something this might help!
+#
+# logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
+
+
+def pretty_warnings(message, category, filename, lineno,
+                    file=None, line=None):
+    '''Format warning messages from warnings.warn().'''
+    return 'WARNING: %s\n' % (message)
+
+# Override the default warning formatter (which is ugly), and add a filter to
+# ensure duplicate warnings only get displayed once.
+warnings.simplefilter("once", append=True)
+warnings.formatwarning = pretty_warnings
+
+
+
+def parse_yaml_with_roundtrip_info(text):
+    return yaml.load(text, yaml.RoundTripLoader)
+
+def write_yaml_with_roundtrip_info(contents, stream, **kwargs):
+    yaml.dump(contents, stream, Dumper=yaml.RoundTripDumper, **kwargs)
+
+
+
+class VersionFileError(RuntimeError):
+    '''Represents errors in the version marker file (./VERSION).'''
+    pass
+
+
+class MigrationOutOfOrderError(RuntimeError):
+    '''Raised if a migration is run on too old a version of definitions.
+
+    It's not an error to run a migration on a version that is already migrated.
+
+    '''
+    pass
+
+
+def check_definitions_version(from_version, version_file='./VERSION',
+                              to_version=None):
+    '''Check if migration between 'from_version' and 'to_version' is needed.
+
+    Both 'from_version' and 'to_version' should be whole numbers. The
+    'to_version' defaults to from_version + 1.
+
+    This function reads the version marker file specified by 'version_file'.
+    Returns True if the version is between 'from_version' and 'to_version',
+    indicating that migration needs to be done. Returns False if the version is
+    already at or beyond 'to_version'. Raises MigrationOutOfOrderError if the
+    version is below 'from_version'.
+
+    If 'version_file' is missing or invalid, it raises VersionFileError. The
+    version file is expected to follow the following format:
+
+        version: 1
+
+    '''
+    to_version = to_version or (from_version + 1)
+    need_to_migrate = False
+
+    if os.path.exists(version_file):
+        logging.info("Found version information file: %s" % version_file)
+
+        with open(version_file) as f:
+            version_text = f.read()
+
+        if len(version_text) == 0:
+            raise VersionFileError(
+                "File %s exists but is empty." % version_file)
+
+        try:
+            version_info = yaml.safe_load(version_text)
+            current_version = version_info['version']
+
+            if current_version >= to_version:
+                logging.info(
+                    "Already at version %i." % current_version)
+            elif current_version < from_version:
+                raise MigrationOutOfOrderError(
+                    "This tool expects to migrate from version %i to version "
+                    "%i of the Baserock Definitions syntax. These definitions "
+                    "claim to be version %i." % (
+                        from_version, to_version, current_version))
+            else:
+                logging.info("Need to migrate from %i to %i.",
+                             current_version, to_version)
+                need_to_migrate = True
+        except (KeyError, TypeError, ValueError) as e:
+            logging.exception(e)
+            raise VersionFileError(
+                "Invalid version info: '%s'" % version_text)
+    else:
+        raise VersionFileError(
+            "No file %s was found. Please run the migration scripts in order,"
+            "starting from 000-version-info.py." % version_file)
+
+    return need_to_migrate
+
+
+def set_definitions_version(new_version, version_file='./VERSION'):
+    '''Update the version information stored in 'version_file'.
+
+    The new version must be a whole number. If 'version_file' doesn't exist,
+    it will be created.
+
+    '''
+    version_info = {'version': new_version}
+    with open(version_file, 'w') as f:
+        # If 'default_flow_style' is True (the default) then the output here
+        # will look like "{version: 0}" instead of "version: 0".
+        yaml.safe_dump(version_info, f, default_flow_style=False)
+
+
+def walk_definition_files(path='.', extensions=['.morph']):
+    '''Recursively yield all files under 'path' with the given extension(s).
+
+    This is safe to run in the top level of a Git repository, as anything under
+    '.git' will be ignored.
+
+    '''
+    for dirname, dirnames, filenames in os.walk('.'):
+        filenames.sort()
+        dirnames.sort()
+        if '.git' in dirnames:
+            dirnames.remove('.git')
+        for filename in filenames:
+            for extension in extensions:
+                if filename.endswith(extension):
+                    yield os.path.join(dirname, filename)
+
+
+ALL_KINDS = ['cluster', 'system', 'stratum', 'chunk']
+
+
+def process_definitions(path='.', kinds=ALL_KINDS, validate_cb=None,
+                        modify_cb=None):
+    '''Run callbacks for all Baserock definitions found in 'path'.
+
+    If 'validate_cb' is set, it will be called for each definition and can
+    return True or False to indicate whether that definition is valid according
+    a new version of the format. The process_definitions() function will return
+    True if all definitions were valid according to validate_cb(), and False
+    otherwise.
+
+    If 'modify_cb' is set, it will be called for each definition and can
+    modify the 'content' dict. It should return True if the dict was modified,
+    and in this case the definition file will be overwritten with the new
+    contents. The 'ruamel.yaml' library is used if it is available, which will
+    try to preserve comments, ordering and some formatting in the YAML
+    definition files.
+
+    If 'validate_cb' is set and returns False for a definition, 'modify_cb'
+    will not be called.
+
+    Both callbacks are passed two parameters: a dict containing the contents of
+    the definition file, and its filename. The filename is passed so you can
+    use it when reporting errors.
+
+    The 'kinds' setting can be used to ignore some definitions according to the
+    'kind' field.
+
+    '''
+    all_valid = True
+
+    for filename in walk_definition_files(path=path):
+        with open(filename) as f:
+            text = f.read()
+
+        if modify_cb is None:
+            contents = yaml.load(text)
+        else:
+            contents = parse_yaml_with_roundtrip_info(text)
+
+        if 'kind' in contents:
+            if contents['kind'] in kinds:
+                valid = True
+                changed = False
+
+                if validate_cb is not None:
+                    valid = validate_cb(contents, filename)
+                    all_valid &= valid
+
+                if valid and modify_cb is not None:
+                    changed = modify_cb(contents, filename)
+
+                if changed:
+                    with open(filename, 'w') as f:
+                        write_yaml_with_roundtrip_info(contents, f, width=80)
+        else:
+            warnings.warn("%s is invalid: no 'kind' field set." % filename)
+
+    return all_valid
diff --git a/migrations/run-all b/migrations/run-all
new file mode 100755
index 0000000..5a817ee
--- /dev/null
+++ b/migrations/run-all
@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+# Copyright (C) 2015  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+'''Run a set of migration scripts.
+
+This script does exactly what `PYTHONPATH=. run-parts --exit-on-error` would
+do. I avoided using 'run-parts' purely because the implementation in Fedora 22
+doesn't have an '--exit-on-error' option. The Busybox and Debian
+implementations do have that option.
+
+Please fix run-parts in https://git.fedorahosted.org/cgit/crontabs.git/tree/ 
+so we can simplify this script :-)
+
+'''
+
+
+import os
+import subprocess
+import sys
+
+
+if len(sys.argv) == 2:
+    migration_dir = sys.argv[1]
+elif len(sys.argv) == 1:
+    migration_dir = os.path.dirname(__file__)
+else:
+    sys.stderr.write("Usage: %s [MIGRATION_DIR]\n" % sys.argv[0])
+    sys.exit(1)
+
+
+def is_executable(fpath):
+    return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
+
+env = os.environ
+if 'PYTHONPATH' in env:
+    env['PYTHONPATH'] = env['PYTHONPATH'] + ':' + migration_dir
+else:
+    env['PYTHONPATH'] = migration_dir
+
+try:
+    migrations_found = 0
+    for fname in sorted(os.listdir(migration_dir)):
+        migration_fpath = os.path.join(migration_dir, fname)
+        if is_executable(migration_fpath):
+            if not os.path.samefile(migration_fpath, __file__) and \
+                    fname != 'indent':
+                migrations_found += 1
+                sys.stdout.write(migration_fpath + ":\n")
+                subprocess.check_call(
+                    migration_fpath, env=env)
+
+    if migrations_found == 0:
+        sys.stderr.write("No migration files found in '%s'\n" % migration_dir)
+        sys.exit(1)
+    else:
+        sys.exit(0)
+
+except (subprocess.CalledProcessError, RuntimeError) as e:
+    sys.stderr.write(str(e) + '\n')
+    sys.exit(1)