diff options
author | Sam Thursfield <sam@afuera.me.uk> | 2016-02-20 11:38:51 +0000 |
---|---|---|
committer | Sam Thursfield <sam@afuera.me.uk> | 2016-02-20 12:31:53 +0000 |
commit | 6c4c284968b0d541ffca465f9116ec842d711f77 (patch) | |
tree | 5047da7f0338405b2a569ec50f5fa5e54d5ecc3d /migrations | |
download | spec-6c4c284968b0d541ffca465f9116ec842d711f77.tar.gz |
Creation of Baserock definitions format specification repo
This contains:
- textual description of the Baserock definitions format, and the list
of changes since version 0, taken from: git://baserock.branchable.com/
- migrations and schemas taken from
git://git.baserock.org/baserock/baserock/definitions.
Diffstat (limited to 'migrations')
-rwxr-xr-x | migrations/000-version-info.py | 49 | ||||
-rwxr-xr-x | migrations/001-empty-build-depends.py | 82 | ||||
-rwxr-xr-x | migrations/002-missing-chunk-morphs.py | 73 | ||||
-rwxr-xr-x | migrations/003-arch-armv5.py | 89 | ||||
-rwxr-xr-x | migrations/004-install-files-overwrite-symlink.py | 59 | ||||
-rwxr-xr-x | migrations/005-strip-commands.py | 78 | ||||
-rwxr-xr-x | migrations/006-specify-build-system.py | 354 | ||||
-rwxr-xr-x | migrations/007-defaults-in-definitions.py | 67 | ||||
-rw-r--r-- | migrations/007-initial-defaults | 199 | ||||
-rw-r--r-- | migrations/GUIDELINES | 35 | ||||
-rwxr-xr-x | migrations/indent | 36 | ||||
-rw-r--r-- | migrations/migrations.py | 228 | ||||
-rwxr-xr-x | migrations/run-all | 73 |
13 files changed, 1422 insertions, 0 deletions
diff --git a/migrations/000-version-info.py b/migrations/000-version-info.py new file mode 100755 index 0000000..2bff51f --- /dev/null +++ b/migrations/000-version-info.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +# Copyright (C) 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + + +'''Migration to Baserock Definitions format version 0. + +The format of version 0 is not formally specified, except by the Morph +codebase. It marks the starting point of the work to formalise the Baserock +Definitions format. + +''' + + +import os +import sys + +import migrations + + +TO_VERSION = 0 + + +try: + if os.path.exists('./VERSION'): + # This will raise an exception if the VERSION file is invalid, which + # might be useful. + migrations.check_definitions_version(TO_VERSION) + + sys.stdout.write("Nothing to do.\n") + sys.exit(0) + else: + sys.stdout.write("No VERSION file found, creating one.\n") + migrations.set_definitions_version(TO_VERSION) + sys.exit(0) +except RuntimeError as e: + sys.stderr.write("Error: %s\n" % e.message) + sys.exit(1) diff --git a/migrations/001-empty-build-depends.py b/migrations/001-empty-build-depends.py new file mode 100755 index 0000000..5d4296d --- /dev/null +++ b/migrations/001-empty-build-depends.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python +# Copyright (C) 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + + +'''Migration to Baserock Definitions format version 1. + +In version 1, the 'build-depends' parameter was made optional. It was +previously mandatory to specify 'build-depends' for a chunk, even if it was an +empty list. + +''' + + +import sys +import warnings + +import migrations + + +TO_VERSION = 1 + + +def check_empty_build_depends(contents, filename): + assert contents['kind'] == 'stratum' + + valid = True + for chunk_ref in contents.get('chunks', []): + if 'build-depends' not in chunk_ref: + chunk_ref_name = chunk_ref.get('name', chunk_ref.get('morph')) + warnings.warn( + "%s:%s has no build-depends field, which " + "is invalid in definitions version 0." % + (contents['name'], chunk_ref_name)) + valid = False + + return valid + + +def remove_empty_build_depends(contents, filename): + assert contents['kind'] == 'stratum' + + changed = False + for chunk_ref in contents.get('chunks', []): + if 'build-depends' in chunk_ref: + if len(chunk_ref['build-depends']) == 0: + del chunk_ref['build-depends'] + changed = True + + return changed + + +try: + if migrations.check_definitions_version(TO_VERSION - 1): + success = migrations.process_definitions( + path='.', kinds=['stratum'], + validate_cb=check_empty_build_depends, + modify_cb=remove_empty_build_depends) + if success: + migrations.set_definitions_version(TO_VERSION) + sys.stdout.write("Migration completed successfully.\n") + sys.exit(0) + else: + sys.stderr.write("Migration failed due to warnings.\n") + sys.exit(1) + else: + sys.stdout.write("Nothing to do.\n") + sys.exit(0) +except RuntimeError as e: + sys.stderr.write("Error: %s\n" % e.message) + sys.exit(1) diff --git a/migrations/002-missing-chunk-morphs.py b/migrations/002-missing-chunk-morphs.py new file mode 100755 index 0000000..2c93804 --- /dev/null +++ b/migrations/002-missing-chunk-morphs.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +# Copyright (C) 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + + +'''Migration to Baserock Definitions format version 2. + +In version 2, the processing of the 'morph:' field within stratum .morph files +became more strict. This migration checks whether definitions are valid +according to version 2 of the format. + +''' + + +import os +import sys +import warnings + +import migrations + + +TO_VERSION = 2 + + +def check_missing_chunk_morphs(contents, filename): + assert contents['kind'] == 'stratum' + + valid = True + + for chunk_ref in contents.get('chunks', []): + if 'morph' in chunk_ref: + chunk_path = os.path.join('.', chunk_ref['morph']) + if not os.path.exists(chunk_path): + # There's no way we can really fix this, so + # just warn and say the migration failed. + warnings.warn( + "%s points to non-existant file %s" % + (contents['name'], chunk_ref['morph'])) + valid = False + + return valid + + +try: + if migrations.check_definitions_version(TO_VERSION - 1): + safe_to_migrate = migrations.process_definitions( + kinds=['stratum'], validate_cb=check_missing_chunk_morphs) + + if not safe_to_migrate: + sys.stderr.write( + "Migration failed due to one or more warnings.\n") + sys.exit(1) + else: + migrations.set_definitions_version(TO_VERSION) + sys.stdout.write("Migration completed successfully.\n") + sys.exit(0) + else: + sys.stdout.write("Nothing to do.\n") + sys.exit(0) +except RuntimeError as e: + sys.stderr.write("Error: %s\n" % e.message) + sys.exit(1) diff --git a/migrations/003-arch-armv5.py b/migrations/003-arch-armv5.py new file mode 100755 index 0000000..58eb79d --- /dev/null +++ b/migrations/003-arch-armv5.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +# Copyright (C) 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + + +'''Migration to Baserock Definitions format version 3. + +In version 3, there were two additions: + + - the 'armv5' architecture + - the install-essential-files.configure configuration extension + +This migration checks that neither of these are in use in the input (version 2) +definitions. Which isn't particularly useful. + +''' + + +import sys +import warnings + +import migrations + + +TO_VERSION = 3 + + +def check_arch(contents, filename): + assert contents['kind'] == 'system' + + valid = True + + if contents['arch'] == 'armv5': + warnings.warn( + "%s uses armv5 architecture that is not understood until version " + "3." % filename) + valid = False + + return valid + + +def check_configuration_extensions(contents, filename): + assert contents['kind'] == 'system' + + valid = True + + for extension in contents.get('configuration-extensions', []): + if extension == 'install-essential-files': + warnings.warn( + "%s uses install-essential-files.configure extension, which " + "was not present in morph.git until commit 423dc974a61f1c0 " + "(tag baserock-definitions-v3)." % filename) + valid = False + + return valid + + +try: + if migrations.check_definitions_version(TO_VERSION - 1): + safe_to_migrate = migrations.process_definitions( + kinds=['system'], validate_cb=check_arch) + safe_to_migrate = migrations.process_definitions( + kinds=['system'], validate_cb=check_configuration_extensions) + + if not safe_to_migrate: + sys.stderr.write( + "Migration failed due to one or more warnings.\n") + sys.exit(1) + else: + migrations.set_definitions_version(TO_VERSION) + sys.stdout.write("Migration completed successfully.\n") + sys.exit(0) + else: + sys.stdout.write("Nothing to do.\n") + sys.exit(0) +except RuntimeError as e: + sys.stderr.write("Error: %s\n" % e.message) + sys.exit(1) diff --git a/migrations/004-install-files-overwrite-symlink.py b/migrations/004-install-files-overwrite-symlink.py new file mode 100755 index 0000000..6853dcb --- /dev/null +++ b/migrations/004-install-files-overwrite-symlink.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +# Copyright (C) 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + +'''Migration to Baserock Definitions format version 4. + +This change to the format was made to work around a bug in a deployment +extension present in morph.git. + +Automated migration is not really possible for this change, and unless you +are experiencing the install-files.configure extension crashing, you can ignore +it completely. + +We have now moved all .configure and .write extensions into the definitions.git +repository. Changes like this no longer require a marking a new version of the +Baserock definitions format in order to prevent build tools crashing. + +Morph commit c373f5a403b0ec introduces version 4 of the definitions format. In +older versions of Morph the install-files.configure extension would crash if it +tried to overwrite a symlink. This bug is fixed in the version of Morph that +can build definitions version 4. + +If you need to overwrite a symlink at deploytime using install-files.configure, +please use VERSION to 4 or above in your definitions.git repo so older versions +of Morph gracefully refuse to deploy, instead of crashing. + +''' + + +import sys + +import migrations + + +TO_VERSION = 4 + + +try: + if migrations.check_definitions_version(TO_VERSION - 1): + migrations.set_definitions_version(TO_VERSION) + sys.stdout.write("Migration completed successfully.\n") + sys.exit(0) + else: + sys.stdout.write("Nothing to do.\n") + sys.exit(0) +except RuntimeError as e: + sys.stderr.write("Error: %s\n" % e.message) + sys.exit(1) diff --git a/migrations/005-strip-commands.py b/migrations/005-strip-commands.py new file mode 100755 index 0000000..da3de94 --- /dev/null +++ b/migrations/005-strip-commands.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +# Copyright (C) 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + + +'''Migration to Baserock Definitions format version 5. + +Version 5 of the definitions format adds a 'strip-commands' field that can +be set in chunk definitions. + +Version 5 also allows deployment extensions to live in definitions.git instead +of morph.git. This greatly reduces the interface surface of the Baserock +definitions format specification, because we no longer have to mark a new +version of the definitions format each time an extension in morph.git is added, +removed, or changes its API in any way. + +In commit 6f4929946 of git://git.baserock.org/baserock/baserock/definitions.git +the deployment extensions were moved into an extensions/ subdirectory, and the +system and cluster .morph files that referred to them were all updated to +prepend 'extension/' to the filenames. This migration doesn't (re)do that +change. + +''' + + +import sys +import warnings + +import migrations + + +TO_VERSION = 5 + + +def check_strip_commands(contents, filename): + assert contents['kind'] == 'chunk' + + valid = True + + if 'strip-commands' in contents: + warnings.warn( + "%s has strip-commands, which are not valid until version 5" % + filename) + valid = False + + return valid + + +try: + if migrations.check_definitions_version(TO_VERSION - 1): + safe_to_migrate = migrations.process_definitions( + kinds=['chunk'], validate_cb=check_strip_commands) + + if not safe_to_migrate: + sys.stderr.write( + "Migration failed due to one or more warnings.\n") + sys.exit(1) + else: + migrations.set_definitions_version(TO_VERSION) + sys.stdout.write("Migration completed successfully.\n") + sys.exit(0) + else: + sys.stdout.write("Nothing to do.\n") + sys.exit(0) +except RuntimeError as e: + sys.stderr.write("Error: %s\n" % e.message) + sys.exit(1) diff --git a/migrations/006-specify-build-system.py b/migrations/006-specify-build-system.py new file mode 100755 index 0000000..b66736c --- /dev/null +++ b/migrations/006-specify-build-system.py @@ -0,0 +1,354 @@ +#!/usr/bin/env python +# Copyright (C) 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + + +# THIS MIGRATION REQUIRES NETWORK ACCESS TO A BASEROCK GIT CACHE SERVER! If +# you do not have your own Trove, or don't know what a Trove is, it should +# work as-is, provided you have internet access that allows access to +# http://git.baserock.org:8080/. +# +# If you do have your own Trove, change the value of TROVE_HOST below to +# point to it. +# +# This migration uses the same autodetection mechanism that Morph and YBD use +# at build time, in order to fill in the 'build-system' field in strata where +# it is now needed. + + +'''Migration to Baserock Definitions format version 6. + +In definitions version 6, build system autodetection no longer happens. This +means that any chunk that wants to use one of the predefined build systems +(those built into Morph) must say so explicitly, using the 'build-system' +field. + +The build-system field for a chunk can now be specified within a stratum that +contains it. Previously you needed to add a .morph file for a chunk in order to +specify its build-system, but we want to avoid needing a .morph file for +components that follow standard patterns. + +Previously, if build-system wasn't given, Morph would scan the contents of the +chunk's Git repo and try to autodetect which build system was used. This could +be slow, could fail in confusing ways, and meant that to fully parse +definitions you needed access to some or all of the repos they referenced. + +The chosen build-system affects which predefined command sequences are set for +a chunk. It is valid to omit the field if a chunk has its own build commands +defined in a .morph file. When listing the chunks included in a stratum, either +'morph' or 'build-system' must be specified, but not both (to avoid the +possibility of conflicting values). + +''' + + +import requests +import yaml + +import logging +import os +import sys +import warnings + +import migrations + + +TROVE_HOST = 'git.baserock.org' + +REPO_ALIASES = { + 'baserock:': 'git://%s/baserock/' % TROVE_HOST, + 'freedesktop:': 'git://anongit.freedesktop.org/', + 'github:': 'git://github.com/', + 'gnome:': 'git://git.gnome.org/', + 'upstream:': 'git://%s/delta/' % TROVE_HOST, +} + +GIT_CACHE_SERVER_URL = 'http://%s:8080/' % TROVE_HOST + +FAIL_ON_REMOTE_CACHE_ERRORS = False + + +TO_VERSION = 6 + + +# From ybd.git file repos.py at commit eb3bf397ba729387f0d4145a8df8d3c1f9eb707f + +def get_repo_url(repo): + for alias, url in REPO_ALIASES.items(): + repo = repo.replace(alias, url) + if repo.endswith('.git'): + repo = repo[:-4] + return repo + + +# Based on morph.git file buildsystem.py at commit a7748f9cdaaf4112c30d7c1. +# +# I have copied and pasted this code here, as it should not be needed anywhere +# once everyone has migrated to definitions version 6. + +class BuildSystem(object): + def used_by_project(self, file_list): + '''Does a project use this build system? + + ``exists`` is a function that returns a boolean telling if a + filename, relative to the project source directory, exists or not. + + ''' + raise NotImplementedError() # pragma: no cover + + +class ManualBuildSystem(BuildSystem): + + '''A manual build system where the morphology must specify all commands.''' + + name = 'manual' + + def used_by_project(self, file_list): + return False + + +class DummyBuildSystem(BuildSystem): + + '''A dummy build system, useful for debugging morphologies.''' + + name = 'dummy' + + def used_by_project(self, file_list): + return False + + +class AutotoolsBuildSystem(BuildSystem): + + '''The automake/autoconf/libtool holy trinity.''' + + name = 'autotools' + + def used_by_project(self, file_list): + indicators = [ + 'autogen', + 'autogen.sh', + 'configure', + 'configure.ac', + 'configure.in', + 'configure.in.in', + ] + + return any(x in file_list for x in indicators) + + +class PythonDistutilsBuildSystem(BuildSystem): + + '''The Python distutils build systems.''' + + name = 'python-distutils' + + def used_by_project(self, file_list): + indicators = [ + 'setup.py', + ] + + return any(x in file_list for x in indicators) + + +class CPANBuildSystem(BuildSystem): + + '''The Perl cpan build system.''' + + name = 'cpan' + + def used_by_project(self, file_list): + indicators = [ + 'Makefile.PL', + ] + + return any(x in file_list for x in indicators) + + +class CMakeBuildSystem(BuildSystem): + + '''The cmake build system.''' + + name = 'cmake' + + def used_by_project(self, file_list): + indicators = [ + 'CMakeLists.txt', + ] + + return any(x in file_list for x in indicators) + + +class QMakeBuildSystem(BuildSystem): + + '''The Qt build system.''' + + name = 'qmake' + + def used_by_project(self, file_list): + indicator = '.pro' + + for x in file_list: + if x.endswith(indicator): + return True + + return False + + +build_systems = [ + ManualBuildSystem(), + AutotoolsBuildSystem(), + PythonDistutilsBuildSystem(), + CPANBuildSystem(), + CMakeBuildSystem(), + QMakeBuildSystem(), + DummyBuildSystem(), +] + + +def detect_build_system(file_list): + '''Automatically detect the build system, if possible. + + If the build system cannot be detected automatically, return None. + For ``exists`` see the ``BuildSystem.exists`` method. + + ''' + for bs in build_systems: + if bs.used_by_project(file_list): + return bs + return None + + +## End of code based on morph.git file buildsystem.py. + +def get_toplevel_file_list_from_repo(url, ref): + '''Try to list the set of files in the root directory of the repo at 'url'. + + ''' + try: + response = requests.get( + GIT_CACHE_SERVER_URL + '1.0/trees', + params={'repo': url, 'ref': ref}, + headers={'Accept': 'application/json'}, + timeout=9) + logging.debug("Got response: %s" % response) + try: + response.raise_for_status() + toplevel_tree = response.json()['tree'] + except Exception as e: + raise RuntimeError( + "Unexpected response from server %s for repo %s: %s" % + (GIT_CACHE_SERVER_URL, url, e.message)) + toplevel_filenames = toplevel_tree.keys() + except requests.exceptions.ConnectionError as e: + raise RuntimeError("Unable to connect to cache server %s while trying " + "to query file list of repo %s. Error was: %s" % + (GIT_CACHE_SERVER_URL, url, e.message)) + return toplevel_filenames + + +def validate_chunk_refs(contents, filename): + assert contents['kind'] == 'stratum' + + valid = True + for chunk_ref in contents.get('chunks', []): + if chunk_ref.get('morph') is None: + # No chunk .morph file -- this stratum was relying on build-system + # autodetection here. + + if 'repo' not in chunk_ref: + warnings.warn("%s: Chunk %s doesn't specify a source repo." % + (filename, chunk_ref.get('name'))) + valid = False + + if 'ref' not in chunk_ref: + warnings.warn("%s: Chunk %s doesn't specify a source ref." % + (filename, chunk_ref.get('name'))) + valid = False + return valid + + +def move_dict_entry_last(dict_object, key, error_if_missing=False): + '''Move an entry in a ordered dict to the end.''' + + # This is a hack, I couldn't find a method on the 'CommentedMap' type dict + # that we receive from ruamel.yaml that would allow doing this neatly. + if key in dict_object: + value = dict_object[key] + del dict_object[key] + dict_object[key] = value + else: + if error_if_missing: + raise KeyError(key) + + +def ensure_buildsystem_defined_where_needed(contents, filename): + assert contents['kind'] == 'stratum' + + changed = False + for chunk_ref in contents.get('chunks', []): + if chunk_ref.get('morph') is None: + # No chunk .morph file -- this stratum was relying on build-system + # autodetection here. + + chunk_git_url = get_repo_url(chunk_ref['repo']) + chunk_git_ref = chunk_ref['ref'] + + try: + toplevel_file_list = get_toplevel_file_list_from_repo( + chunk_git_url, chunk_git_ref) + except Exception as e: + warnings.warn(str(e)) + message = ( + "Unable to look up one or more repos on remote Git " + "server %s. If you are using a Trove that is not %s, " + "please edit the TROVE_HOST constant in this script " + "and run it again." % (TROVE_HOST, TROVE_HOST)) + if FAIL_ON_REMOTE_CACHE_ERRORS: + raise RuntimeError(message) + else: + warnings.warn(message) + continue + + logging.debug( + '%s: got file list %s', chunk_git_url, toplevel_file_list) + build_system = detect_build_system(toplevel_file_list) + + chunk_ref['build-system'] = build_system.name + move_dict_entry_last(chunk_ref, 'build-depends') + + changed = True + + return changed + + +try: + if migrations.check_definitions_version(TO_VERSION - 1): + success = migrations.process_definitions( + kinds=['stratum'], + validate_cb=validate_chunk_refs, + modify_cb=ensure_buildsystem_defined_where_needed) + if not success: + sys.stderr.write( + "Migration failed due to one or more warnings.\n") + sys.exit(1) + else: + migrations.set_definitions_version(TO_VERSION) + sys.stderr.write("Migration completed successfully.\n") + sys.exit(0) + else: + sys.stderr.write("Nothing to do.\n") + sys.exit(0) +except RuntimeError as e: + sys.stderr.write("Error: %s\n" % e.message) + sys.exit(1) diff --git a/migrations/007-defaults-in-definitions.py b/migrations/007-defaults-in-definitions.py new file mode 100755 index 0000000..489baf9 --- /dev/null +++ b/migrations/007-defaults-in-definitions.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python +# Copyright (C) 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + + +'''Migration to Baserock Definitions format version 7. + +Definitions version 7 adds a file named DEFAULTS which sets the default +build commands and default split rules for the set of definitions in that +repo. + +''' + + +import os +import shutil +import sys +import warnings + +import migrations + + +TO_VERSION = 7 + + + +try: + if migrations.check_definitions_version(TO_VERSION - 1): + if os.path.exists('DEFAULTS'): + warnings.warn( + "DEFAULTS file already exists in these definitions.") + valid = False + else: + shutil.copy( + 'migrations/007-initial-defaults', + 'DEFAULTS') + valid = True + + if valid: + migrations.set_definitions_version(TO_VERSION) + sys.stdout.write("Migration completed successfully.\n") + sys.exit(0) + else: + sys.stderr.write( + "Migration failed due to one or more warnings.\n") + sys.exit(1) + else: + if not os.path.exists('DEFAULTS'): + warnings.warn( + "These definitions are marked as version 7 but there is no " + "DEFAULTS file.") + sys.stdout.write("Nothing to do.\n") + sys.exit(0) +except RuntimeError as e: + sys.stderr.write("Error: %s\n" % e.message) + sys.exit(1) diff --git a/migrations/007-initial-defaults b/migrations/007-initial-defaults new file mode 100644 index 0000000..ab034a0 --- /dev/null +++ b/migrations/007-initial-defaults @@ -0,0 +1,199 @@ +# Baserock definitions defaults +# ============================= +# +# The DEFAULTS file is treated specially by Baserock build tools. +# +# For more information, see: <http://wiki.baserock.org/definitions/current>. + + +# Predefined build commands +# ------------------------- +# +# Common patterns in build instructions can be defined here, which can save +# users from having to write lots of similar-looking chunk .morph files. +# +# There are pre- and post- variants for each set of commands. These exist so +# you can add more commands without having to copy the defaults. For example, +# to create an extra symlink after running `make install`, you can use +# post-install-commands. Since these exist as a way of extending the defaults, +# you cannot set default values for the pre- and post- commands. +# +# The set of environment variables available when these commands are executed +# is not formally specified right now, but you can assume PREFIX, DESTDIR and +# MORPH_ARCH are all set. +# +build-systems: + manual: + # The special, default 'no-op' build system. + configure-commands: [] + build-commands: [] + install-commands: [] + strip-commands: [] + + autotools: + # GNU Autoconf and GNU Automake, or anything which follow the same pattern. + # + # See also: https://github.com/cgwalters/build-api/blob/master/build-api.md + configure-commands: + - >- + export NOCONFIGURE=1; + if [ -e autogen ]; then ./autogen; + elif [ -e autogen.sh ]; then ./autogen.sh; + elif [ -e bootstrap ]; then ./bootstrap; + elif [ -e bootstrap.sh ]; then ./bootstrap.sh; + elif [ ! -e ./configure ]; then autoreconf -ivf; + fi + - ./configure --prefix="$PREFIX" + build-commands: + - make + install-commands: + - make DESTDIR="$DESTDIR" install + strip-commands: + # TODO: Make idempotent when files are hardlinks + # Strip all ELF binary files that are executable or named like a library. + # .so files for C, .cmxs for OCaml and .node for Node. + # + # The file name and permissions checks are done with the `find` command before + # the ELF header is checked with the shell command, because it is a lot cheaper + # to check the mode and file name first, because it is a metadata check, rather + # than a subprocess and a file read. + # + # `file` is not used, to keep the dependency requirements down. + - &generic-strip-command | + find "$DESTDIR" -type f \ + '(' -perm -111 -o -name '*.so*' -o -name '*.cmxs' -o -name '*.node' ')' \ + -exec sh -ec \ + 'read -n4 hdr <"$1" # check for elf header + if [ "$hdr" != "$(printf \\x7fELF)" ]; then + exit 0 + fi + debugfile="$DESTDIR$PREFIX/lib/debug/$(basename "$1")" + mkdir -p "$(dirname "$debugfile")" + objcopy --only-keep-debug "$1" "$debugfile" + chmod 644 "$debugfile" + strip --remove-section=.comment --remove-section=.note --strip-unneeded "$1" + objcopy --add-gnu-debuglink "$debugfile" "$1"' - {} ';' + + python-distutils: + # The Python distutils build systems. + configure-commands: [] + build-commands: + - python setup.py build + install-commands: + - python setup.py install --prefix "$PREFIX" --root "$DESTDIR" + strip-commands: + - *generic-strip-command + + cpan: + # The Perl ExtUtil::MakeMaker build system. This is called the 'cpan' build + # system for historical reasons. + # + # To install perl distributions into the correct location in our chroot + # we need to set PREFIX to <destdir>/<prefix> in the configure-commands. + # + # The mapping between PREFIX and the final installation + # directories is complex and depends upon the configuration of perl + # see, + # https://metacpan.org/pod/distribution/perl/INSTALL#Installation-Directories + # and ExtUtil::MakeMaker's documentation for more details. + configure-commands: + - perl Makefile.PL PREFIX=$DESTDIR$PREFIX + build-commands: + - make + install-commands: + - make install + strip-commands: + - *generic-strip-command + + module-build: + # The Module::Build build system + # + # See the comment in ExtUtilsMakeMakerBuildSystem to see why --prefix is + # set to $DESTDIR$PREFIX here (--prefix in Module::Build has the same + # meaning as PREFIX in ExtUtils::MakeMaker). + configure-commands: + - perl Build.PL --prefix "$DESTDIR$PREFIX" + build-commands: + - ./Build + install-commands: + - ./Build install + strip-commands: + - *generic-strip-command + + cmake: + # The CMake build system. + configure-commands: + - cmake -DCMAKE_INSTALL_PREFIX="$PREFIX" + build-commands: + - make + install-commands: + - make DESTDIR="$DESTDIR" install + strip-commands: + - *generic-strip-command + + qmake: + # The Qt build system. + configure-commands: + - qmake -makefile + build-commands: + - make + install-commands: + - make INSTALL_ROOT="$DESTDIR" install + strip-commands: + - *generic-strip-command + + +# Predefined artifact splitting rules +# ----------------------------------- +# +# Once a build has completed, you have some files that have been installed into +# $DESTDIR. The splitting rules control how many 'artifact' tarballs are +# generated as a result of the build, and which files from $DESTDIR end up in +# which 'artifact'. +# +# The default split rules are defined here. These can be overriden in +# individual chunk .morph files and stratum .morph files using the 'products' +# field. +# +split-rules: + chunk: + - artifact: -bins + include: + - (usr/)?s?bin/.* + - artifact: -libs + include: + - (usr/)?lib(32|64)?/lib[^/]*\.so(\.\d+)* + - (usr/)libexec/.* + - artifact: -devel + include: + - (usr/)?include/.* + - (usr/)?lib(32|64)?/lib.*\.a + - (usr/)?lib(32|64)?/lib.*\.la + - (usr/)?(lib(32|64)?|share)/pkgconfig/.*\.pc + - artifact: -doc + include: + - (usr/)?share/doc/.* + - (usr/)?share/man/.* + - (usr/)?share/info/.* + - artifact: -locale + include: + - (usr/)?share/locale/.* + - (usr/)?share/i18n/.* + - (usr/)?share/zoneinfo/.* + - artifact: -misc + include: + - .* + + stratum: + - artifact: -devel + include: + - .*-devel + - .*-debug + - .*-doc + - artifact: -runtime + include: + - .*-bins + - .*-libs + - .*-locale + - .*-misc + - .* diff --git a/migrations/GUIDELINES b/migrations/GUIDELINES new file mode 100644 index 0000000..3694e2c --- /dev/null +++ b/migrations/GUIDELINES @@ -0,0 +1,35 @@ +Guidelines for writing migrations +--------------------------------- + +All changes to the definitions format must have a migration, but it is valid +for the migration to do nothing except update the version number (see +004-install-files-overwrite-symlink.py for an example of that). + +This small set of rules exists to ensure that the migrations are consistent and +easy to understand. If you are writing a migration and these rules don't make +any sense, we should probably change them. Please sign up to the +baserock-dev@baserock.org mailing list to suggest the change. + +- Write migrations in Python. They must be valid Python 3. For now, since + only Python 2 is available in Baserock 'build' and 'devel' reference systems + up to the 15.25 release of Baserock, they must also be valid Python 2. + +- Don't use any external libraries. + +- Follow the existing file naming pattern, and the existing code convention. + +- Keep the migration code as simple as possible. + +- Avoid crashing on malformed input data, where practical. For example, use + contents.get('field') instead of contents['field'] to avoid crashing when + 'field' is not present. The idea of this is to avoid a "cascade of errors" + problem when running the migrations on bad inputs. It is confusing when + migrations break on problems that are unrelated to the actual area where + they operate, even if they are theoretically "within their rights" to do so. + +- Migrate the definitions in line with current best practices. For example, + migrations/001-empty-build-depends.py doesn't need to remove empty + build-depends fields: they are still valid in version 1 of the format. But + best practice is now to remove them. Users who don't agree with this practice + can choose to not run that migration, which can be done with `chmod -x + migrations/xxx.py`. diff --git a/migrations/indent b/migrations/indent new file mode 100755 index 0000000..8d6f034 --- /dev/null +++ b/migrations/indent @@ -0,0 +1,36 @@ +#!/usr/bin/env python +# Copyright (C) 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + + +'''Automatically reformat a set of Baserock definition files. + +This tool expects to be able to use ruamel.yaml to load and write YAML. +It will totally ruin things if used with PyYAML. + +It makes sense to run this script on your definitions, and check through +and commit the result, before running any of the automated migrations. This +way, you can be sure that the migrations will only change things that they need +to in the .morph files. + +''' + + +import migrations + + +def force_rewrite(contents, filename): + return True + +migrations.process_definitions(path='.', modify_cb=force_rewrite) diff --git a/migrations/migrations.py b/migrations/migrations.py new file mode 100644 index 0000000..22ed132 --- /dev/null +++ b/migrations/migrations.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python +# Copyright (C) 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + + +'''Tools for migrating Baserock definitions from one format version to another. + +''' + + +# ruamel.yaml is a fork of PyYAML which allows rewriting YAML files without +# destroying all of the comments, ordering and formatting. The more +# widely-used PyYAML library will produce output totally different to the +# input file in most cases. +# +# See: <https://bitbucket.org/ruamel/yaml> +import ruamel.yaml as yaml + +import logging +import os +import warnings + + +# Uncomment this to cause all log messages to be written to stdout. By +# default they are hidden, but if you are debugging something this might help! +# +# logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) + + +def pretty_warnings(message, category, filename, lineno, + file=None, line=None): + '''Format warning messages from warnings.warn().''' + return 'WARNING: %s\n' % (message) + +# Override the default warning formatter (which is ugly), and add a filter to +# ensure duplicate warnings only get displayed once. +warnings.simplefilter("once", append=True) +warnings.formatwarning = pretty_warnings + + + +def parse_yaml_with_roundtrip_info(text): + return yaml.load(text, yaml.RoundTripLoader) + +def write_yaml_with_roundtrip_info(contents, stream, **kwargs): + yaml.dump(contents, stream, Dumper=yaml.RoundTripDumper, **kwargs) + + + +class VersionFileError(RuntimeError): + '''Represents errors in the version marker file (./VERSION).''' + pass + + +class MigrationOutOfOrderError(RuntimeError): + '''Raised if a migration is run on too old a version of definitions. + + It's not an error to run a migration on a version that is already migrated. + + ''' + pass + + +def check_definitions_version(from_version, version_file='./VERSION', + to_version=None): + '''Check if migration between 'from_version' and 'to_version' is needed. + + Both 'from_version' and 'to_version' should be whole numbers. The + 'to_version' defaults to from_version + 1. + + This function reads the version marker file specified by 'version_file'. + Returns True if the version is between 'from_version' and 'to_version', + indicating that migration needs to be done. Returns False if the version is + already at or beyond 'to_version'. Raises MigrationOutOfOrderError if the + version is below 'from_version'. + + If 'version_file' is missing or invalid, it raises VersionFileError. The + version file is expected to follow the following format: + + version: 1 + + ''' + to_version = to_version or (from_version + 1) + need_to_migrate = False + + if os.path.exists(version_file): + logging.info("Found version information file: %s" % version_file) + + with open(version_file) as f: + version_text = f.read() + + if len(version_text) == 0: + raise VersionFileError( + "File %s exists but is empty." % version_file) + + try: + version_info = yaml.safe_load(version_text) + current_version = version_info['version'] + + if current_version >= to_version: + logging.info( + "Already at version %i." % current_version) + elif current_version < from_version: + raise MigrationOutOfOrderError( + "This tool expects to migrate from version %i to version " + "%i of the Baserock Definitions syntax. These definitions " + "claim to be version %i." % ( + from_version, to_version, current_version)) + else: + logging.info("Need to migrate from %i to %i.", + current_version, to_version) + need_to_migrate = True + except (KeyError, TypeError, ValueError) as e: + logging.exception(e) + raise VersionFileError( + "Invalid version info: '%s'" % version_text) + else: + raise VersionFileError( + "No file %s was found. Please run the migration scripts in order," + "starting from 000-version-info.py." % version_file) + + return need_to_migrate + + +def set_definitions_version(new_version, version_file='./VERSION'): + '''Update the version information stored in 'version_file'. + + The new version must be a whole number. If 'version_file' doesn't exist, + it will be created. + + ''' + version_info = {'version': new_version} + with open(version_file, 'w') as f: + # If 'default_flow_style' is True (the default) then the output here + # will look like "{version: 0}" instead of "version: 0". + yaml.safe_dump(version_info, f, default_flow_style=False) + + +def walk_definition_files(path='.', extensions=['.morph']): + '''Recursively yield all files under 'path' with the given extension(s). + + This is safe to run in the top level of a Git repository, as anything under + '.git' will be ignored. + + ''' + for dirname, dirnames, filenames in os.walk('.'): + filenames.sort() + dirnames.sort() + if '.git' in dirnames: + dirnames.remove('.git') + for filename in filenames: + for extension in extensions: + if filename.endswith(extension): + yield os.path.join(dirname, filename) + + +ALL_KINDS = ['cluster', 'system', 'stratum', 'chunk'] + + +def process_definitions(path='.', kinds=ALL_KINDS, validate_cb=None, + modify_cb=None): + '''Run callbacks for all Baserock definitions found in 'path'. + + If 'validate_cb' is set, it will be called for each definition and can + return True or False to indicate whether that definition is valid according + a new version of the format. The process_definitions() function will return + True if all definitions were valid according to validate_cb(), and False + otherwise. + + If 'modify_cb' is set, it will be called for each definition and can + modify the 'content' dict. It should return True if the dict was modified, + and in this case the definition file will be overwritten with the new + contents. The 'ruamel.yaml' library is used if it is available, which will + try to preserve comments, ordering and some formatting in the YAML + definition files. + + If 'validate_cb' is set and returns False for a definition, 'modify_cb' + will not be called. + + Both callbacks are passed two parameters: a dict containing the contents of + the definition file, and its filename. The filename is passed so you can + use it when reporting errors. + + The 'kinds' setting can be used to ignore some definitions according to the + 'kind' field. + + ''' + all_valid = True + + for filename in walk_definition_files(path=path): + with open(filename) as f: + text = f.read() + + if modify_cb is None: + contents = yaml.load(text) + else: + contents = parse_yaml_with_roundtrip_info(text) + + if 'kind' in contents: + if contents['kind'] in kinds: + valid = True + changed = False + + if validate_cb is not None: + valid = validate_cb(contents, filename) + all_valid &= valid + + if valid and modify_cb is not None: + changed = modify_cb(contents, filename) + + if changed: + with open(filename, 'w') as f: + write_yaml_with_roundtrip_info(contents, f, width=80) + else: + warnings.warn("%s is invalid: no 'kind' field set." % filename) + + return all_valid diff --git a/migrations/run-all b/migrations/run-all new file mode 100755 index 0000000..5a817ee --- /dev/null +++ b/migrations/run-all @@ -0,0 +1,73 @@ +#!/usr/bin/env python +# Copyright (C) 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + + +'''Run a set of migration scripts. + +This script does exactly what `PYTHONPATH=. run-parts --exit-on-error` would +do. I avoided using 'run-parts' purely because the implementation in Fedora 22 +doesn't have an '--exit-on-error' option. The Busybox and Debian +implementations do have that option. + +Please fix run-parts in https://git.fedorahosted.org/cgit/crontabs.git/tree/ +so we can simplify this script :-) + +''' + + +import os +import subprocess +import sys + + +if len(sys.argv) == 2: + migration_dir = sys.argv[1] +elif len(sys.argv) == 1: + migration_dir = os.path.dirname(__file__) +else: + sys.stderr.write("Usage: %s [MIGRATION_DIR]\n" % sys.argv[0]) + sys.exit(1) + + +def is_executable(fpath): + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + +env = os.environ +if 'PYTHONPATH' in env: + env['PYTHONPATH'] = env['PYTHONPATH'] + ':' + migration_dir +else: + env['PYTHONPATH'] = migration_dir + +try: + migrations_found = 0 + for fname in sorted(os.listdir(migration_dir)): + migration_fpath = os.path.join(migration_dir, fname) + if is_executable(migration_fpath): + if not os.path.samefile(migration_fpath, __file__) and \ + fname != 'indent': + migrations_found += 1 + sys.stdout.write(migration_fpath + ":\n") + subprocess.check_call( + migration_fpath, env=env) + + if migrations_found == 0: + sys.stderr.write("No migration files found in '%s'\n" % migration_dir) + sys.exit(1) + else: + sys.exit(0) + +except (subprocess.CalledProcessError, RuntimeError) as e: + sys.stderr.write(str(e) + '\n') + sys.exit(1) |