diff options
Diffstat (limited to 'morphlib')
118 files changed, 562 insertions, 421 deletions
diff --git a/morphlib/__init__.py b/morphlib/__init__.py index d54340df..7c462aad 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. '''Baserock library.''' diff --git a/morphlib/app.py b/morphlib/app.py index b8bae850..c8fe397d 100644 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp diff --git a/morphlib/artifact.py b/morphlib/artifact.py index 7a40a81a..47b16823 100644 --- a/morphlib/artifact.py +++ b/morphlib/artifact.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012, 2013, 2014 Codethink Limited +# Copyright (C) 2012, 2013, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. class Artifact(object): diff --git a/morphlib/artifact_tests.py b/morphlib/artifact_tests.py index abd8767e..106459c8 100644 --- a/morphlib/artifact_tests.py +++ b/morphlib/artifact_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import copy diff --git a/morphlib/artifactcachereference.py b/morphlib/artifactcachereference.py index 8211f6b5..55a26b6d 100644 --- a/morphlib/artifactcachereference.py +++ b/morphlib/artifactcachereference.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. class ArtifactCacheReference(object): diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py index 5062f854..b49c1905 100644 --- a/morphlib/artifactresolver.py +++ b/morphlib/artifactresolver.py @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp diff --git a/morphlib/artifactresolver_tests.py b/morphlib/artifactresolver_tests.py index b958da4f..141ff948 100644 --- a/morphlib/artifactresolver_tests.py +++ b/morphlib/artifactresolver_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import itertools diff --git a/morphlib/artifactsplitrule.py b/morphlib/artifactsplitrule.py index 1511d694..b5ebdf83 100644 --- a/morphlib/artifactsplitrule.py +++ b/morphlib/artifactsplitrule.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import collections diff --git a/morphlib/bins.py b/morphlib/bins.py index 560e68bb..2e8ba0b3 100644 --- a/morphlib/bins.py +++ b/morphlib/bins.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2014 Codethink Limited +# Copyright (C) 2011-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. '''Functions for dealing with Baserock binaries. @@ -75,7 +74,7 @@ if sys.version_info < (2, 7, 3): # pragma: no cover else: if sys.platform != "os2emx": os.chown(targetpath, u, g) - except EnvironmentError, e: + except EnvironmentError as e: raise ExtractError("could not change owner") tarfile.TarFile.chown = fixed_chown @@ -187,7 +186,7 @@ def unpack_binary_from_file(f, dirname): # pragma: no cover prepare_extract(tarinfo, targetpath) try: ret = real(tarinfo, targetpath) - except (IOError, OSError), e: + except (IOError, OSError) as e: if e.errno != errno.EEXIST: if e.filename is None: e.filename = targetpath diff --git a/morphlib/bins_tests.py b/morphlib/bins_tests.py index 60361ece..3895680f 100644 --- a/morphlib/bins_tests.py +++ b/morphlib/bins_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2014 Codethink Limited +# Copyright (C) 2011-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import gzip diff --git a/morphlib/branchmanager.py b/morphlib/branchmanager.py index a33b4ccb..92a1f4be 100644 --- a/morphlib/branchmanager.py +++ b/morphlib/branchmanager.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp @@ -92,7 +91,7 @@ class LocalRefManager(object): op, args = d.pop() try: op(*args) - except Exception, e: + except Exception as e: exceptions.append((op, args, e)) if exceptions: raise RefCleanupError(primary, exceptions) @@ -201,7 +200,7 @@ class RemoteRefManager(object): remote, refspecs = d.pop() try: remote.push(*refspecs) - except Exception, e: + except Exception as e: exceptions.append((remote, refspecs, e)) if exceptions: raise RefCleanupError(primary, exceptions) diff --git a/morphlib/branchmanager_tests.py b/morphlib/branchmanager_tests.py index cf3be73c..9f3740d7 100644 --- a/morphlib/branchmanager_tests.py +++ b/morphlib/branchmanager_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp diff --git a/morphlib/buildbranch.py b/morphlib/buildbranch.py index 2482bc9a..80cecd75 100644 --- a/morphlib/buildbranch.py +++ b/morphlib/buildbranch.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import collections @@ -115,7 +114,7 @@ class BuildBranch(object): for morphology in morphologies: loader.unset_defaults(morphology) sha1 = gd.store_blob(loader.save_to_string(morphology)) - yield 0100644, sha1, morphology.filename + yield 0o100644, sha1, morphology.filename def inject_build_refs(self, loader, use_local_repos, inject_cb=lambda **kwargs: None): @@ -293,7 +292,7 @@ class BuildBranch(object): func, args, kwargs = self._cleanup.pop() try: func(*args, **kwargs) - except Exception, e: + except Exception as e: exceptions.append(e) if exceptions: raise BuildBranchCleanupError(self, exceptions) diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 8572450d..be8a1507 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import itertools @@ -544,4 +543,12 @@ class InitiatorBuildCommand(BuildCommand): self.MAX_RETRIES) loop.add_state_machine(cm) - loop.run() + try: + loop.run() + except KeyboardInterrupt: + # This will run if the user presses Ctrl+C or sends SIGINT during + # the build. It won't trigger on SIGTERM, SIGKILL or unhandled + # Python exceptions. + logging.info('Received KeyboardInterrupt, aborting.') + for initiator in loop.state_machines_of_type(distbuild.Initiator): + initiator.handle_cancel() diff --git a/morphlib/buildenvironment.py b/morphlib/buildenvironment.py index ef770f52..6ec82d45 100644 --- a/morphlib/buildenvironment.py +++ b/morphlib/buildenvironment.py @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import copy import cliapp diff --git a/morphlib/buildenvironment_tests.py b/morphlib/buildenvironment_tests.py index 7ae7c2d5..59a56aa2 100644 --- a/morphlib/buildenvironment_tests.py +++ b/morphlib/buildenvironment_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import copy diff --git a/morphlib/builder.py b/morphlib/builder.py index 0bb21434..04ebd149 100644 --- a/morphlib/builder.py +++ b/morphlib/builder.py @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import json @@ -290,7 +289,7 @@ class ChunkBuilder(BuilderBase): self.create_devices(destdir) os.rename(temppath, logpath) - except BaseException, e: + except BaseException as e: logging.error('Caught exception: %s' % str(e)) logging.info('Cleaning up staging area') self.staging_area.chroot_close() @@ -376,7 +375,7 @@ class ChunkBuilder(BuilderBase): if stdout: stdout.flush() - except cliapp.AppException, e: + except cliapp.AppException as e: if not stdout: with open(logfilepath, 'r') as log: self.app.output.write("%s failed\n" % step) @@ -406,7 +405,7 @@ class ChunkBuilder(BuilderBase): with morphlib.savefile.SaveFile(script_path, 'w') as f: f.write("#!/bin/sh\nset -xeu\n") f.write(script) - os.chmod(script_path, 0555) + os.chmod(script_path, 0o555) rel_script_path = os.path.join(SYSTEM_INTEGRATION_PATH, script_name) @@ -624,7 +623,7 @@ class SystemBuilder(BuilderBase): # pragma: no cover f.write('SUPPORT_URL="http://wiki.baserock.org/mailinglist"\n') f.write('BUG_REPORT_URL="http://wiki.baserock.org/mailinglist"\n') - os.chmod(os_release_file, 0644) + os.chmod(os_release_file, 0o644) def run_system_integration_commands(self, rootdir): # pragma: no cover ''' Run the system integration commands ''' @@ -657,7 +656,7 @@ class SystemBuilder(BuilderBase): # pragma: no cover msg = error_message_for_containerised_commandline( argv, err, container_config) raise cliapp.AppException(msg) - except BaseException, e: + except BaseException as e: self.app.status( msg='Error while running system integration commands', error=True) diff --git a/morphlib/builder_tests.py b/morphlib/builder_tests.py index 0cc90819..a571e3d0 100644 --- a/morphlib/builder_tests.py +++ b/morphlib/builder_tests.py @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import json diff --git a/morphlib/buildsystem.py b/morphlib/buildsystem.py index fb99e70e..7ed50d11 100644 --- a/morphlib/buildsystem.py +++ b/morphlib/buildsystem.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import os diff --git a/morphlib/buildsystem_tests.py b/morphlib/buildsystem_tests.py index 56ba64d7..80898ebd 100644 --- a/morphlib/buildsystem_tests.py +++ b/morphlib/buildsystem_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import os diff --git a/morphlib/cachedrepo.py b/morphlib/cachedrepo.py index 8b38c5c9..23639043 100644 --- a/morphlib/cachedrepo.py +++ b/morphlib/cachedrepo.py @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp diff --git a/morphlib/cachedrepo_tests.py b/morphlib/cachedrepo_tests.py index 6fe69ef5..404a0778 100644 --- a/morphlib/cachedrepo_tests.py +++ b/morphlib/cachedrepo_tests.py @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import logging diff --git a/morphlib/cachekeycomputer.py b/morphlib/cachekeycomputer.py index c3a01b9e..33eef082 100644 --- a/morphlib/cachekeycomputer.py +++ b/morphlib/cachekeycomputer.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import hashlib diff --git a/morphlib/cachekeycomputer_tests.py b/morphlib/cachekeycomputer_tests.py index 57739983..d0a4657f 100644 --- a/morphlib/cachekeycomputer_tests.py +++ b/morphlib/cachekeycomputer_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import copy diff --git a/morphlib/extensions.py b/morphlib/extensions.py index 051a54a7..aa0622b8 100644 --- a/morphlib/extensions.py +++ b/morphlib/extensions.py @@ -1,4 +1,4 @@ -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import asyncore import asynchat @@ -149,7 +148,7 @@ class get_extension_filename(): fd, ext_filename = tempfile.mkstemp() os.write(fd, ext_contents) os.close(fd) - os.chmod(ext_filename, 0700) + os.chmod(ext_filename, 0o700) self.delete = True self.ext_filename = ext_filename diff --git a/morphlib/extractedtarball.py b/morphlib/extractedtarball.py index fd98cd92..95c0582d 100644 --- a/morphlib/extractedtarball.py +++ b/morphlib/extractedtarball.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp @@ -42,7 +41,7 @@ class ExtractedTarball(object): # pragma: no cover self.tempdir = tempfile.mkdtemp(dir=self.app.settings['tempdir']) try: morphlib.bins.unpack_binary(self.tarball, self.tempdir) - except BaseException, e: + except BaseException as e: logging.error('Caught exception: %s' % str(e)) logging.debug('Removing temporary directory %s' % self.tempdir) shutil.rmtree(self.tempdir) @@ -54,7 +53,7 @@ class ExtractedTarball(object): # pragma: no cover tarball=os.path.basename(self.tarball), chatty=True) try: shutil.rmtree(self.tempdir) - except BaseException, e: + except BaseException as e: logging.warning( 'Error when removing temporary directory %s: %s' % (self.tempdir, str(e))) diff --git a/morphlib/exts/add-config-files.configure b/morphlib/exts/add-config-files.configure index 0094cf6b..2cf96fd1 100755 --- a/morphlib/exts/add-config-files.configure +++ b/morphlib/exts/add-config-files.configure @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # Copy all files located in $SRC_CONFIG_DIR to the image /etc. diff --git a/morphlib/exts/fstab.configure b/morphlib/exts/fstab.configure index a1287ea4..b9154eee 100755 --- a/morphlib/exts/fstab.configure +++ b/morphlib/exts/fstab.configure @@ -1,5 +1,6 @@ -#!/usr/bin/python -# Copyright (C) 2013 Codethink Limited +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright © 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= @@ -20,21 +20,9 @@ import os import sys +import morphlib -def asciibetical(strings): +envvars = {k: v for (k, v) in os.environ.iteritems() if k.startswith('FSTAB_')} - def key(s): - return [ord(c) for c in s] - - return sorted(strings, key=key) - - -fstab_filename = os.path.join(sys.argv[1], 'etc', 'fstab') - -fstab_vars = asciibetical(x for x in os.environ if x.startswith('FSTAB_')) -with open(fstab_filename, 'a') as f: - for var in fstab_vars: - f.write('%s\n' % os.environ[var]) - -os.chown(fstab_filename, 0, 0) -os.chmod(fstab_filename, 0644) +conf_file = os.path.join(sys.argv[1], 'etc/fstab') +morphlib.util.write_from_dict(conf_file, envvars) diff --git a/morphlib/exts/hosts.configure b/morphlib/exts/hosts.configure new file mode 100755 index 00000000..6b068d04 --- /dev/null +++ b/morphlib/exts/hosts.configure @@ -0,0 +1,48 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright © 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# =*= License: GPL-2 =*= + + +import os +import sys +import socket + +import morphlib + +def validate(var, line): + xs = line.split() + if len(xs) == 0: + raise morphlib.Error("`%s: %s': line is empty" % (var, line)) + + ip = xs[0] + hostnames = xs[1:] + + if len(hostnames) == 0: + raise morphlib.Error("`%s: %s': missing hostname" % (var, line)) + + family = socket.AF_INET6 if ':' in ip else socket.AF_INET + + try: + socket.inet_pton(family, ip) + except socket.error: + raise morphlib.Error("`%s: %s' invalid ip" % (var, ip)) + +envvars = {k: v for (k, v) in os.environ.iteritems() if k.startswith('HOSTS_')} + +conf_file = os.path.join(sys.argv[1], 'etc/hosts') +morphlib.util.write_from_dict(conf_file, envvars, validate) diff --git a/morphlib/exts/initramfs.write b/morphlib/exts/initramfs.write index f8af6d84..1059defa 100755 --- a/morphlib/exts/initramfs.write +++ b/morphlib/exts/initramfs.write @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= diff --git a/morphlib/exts/install-files.configure b/morphlib/exts/install-files.configure index 04dc5f18..58cf373a 100755 --- a/morphlib/exts/install-files.configure +++ b/morphlib/exts/install-files.configure @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. ''' A Morph configuration extension for adding arbitrary files to a system diff --git a/morphlib/exts/kvm.check b/morphlib/exts/kvm.check index b8877a89..62d76453 100755 --- a/morphlib/exts/kvm.check +++ b/morphlib/exts/kvm.check @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. '''Preparatory checks for Morph 'kvm' write extension''' diff --git a/morphlib/exts/kvm.write b/morphlib/exts/kvm.write index 30b43d6c..0d0c095b 100755 --- a/morphlib/exts/kvm.write +++ b/morphlib/exts/kvm.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. '''A Morph deployment write extension for deploying to KVM+libvirt. diff --git a/morphlib/exts/nfsboot.check b/morphlib/exts/nfsboot.check index 806e560a..e273f61c 100755 --- a/morphlib/exts/nfsboot.check +++ b/morphlib/exts/nfsboot.check @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. '''Preparatory checks for Morph 'nfsboot' write extension''' diff --git a/morphlib/exts/nfsboot.configure b/morphlib/exts/nfsboot.configure index 660d9c39..6a68dc48 100755 --- a/morphlib/exts/nfsboot.configure +++ b/morphlib/exts/nfsboot.configure @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # Remove all networking interfaces. On nfsboot systems, eth0 is set up diff --git a/morphlib/exts/nfsboot.write b/morphlib/exts/nfsboot.write index 49d71174..d928775e 100755 --- a/morphlib/exts/nfsboot.write +++ b/morphlib/exts/nfsboot.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. '''A Morph deployment write extension for deploying to an nfsboot server diff --git a/morphlib/exts/openstack.check b/morphlib/exts/openstack.check index 3850d481..4c21b604 100755 --- a/morphlib/exts/openstack.check +++ b/morphlib/exts/openstack.check @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. '''Preparatory checks for Morph 'openstack' write extension''' diff --git a/morphlib/exts/openstack.write b/morphlib/exts/openstack.write index d29d2661..67e07c18 100755 --- a/morphlib/exts/openstack.write +++ b/morphlib/exts/openstack.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2013 - 2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. '''A Morph deployment write extension for deploying to OpenStack.''' diff --git a/morphlib/exts/rawdisk.check b/morphlib/exts/rawdisk.check index 094adb72..9be0ce91 100755 --- a/morphlib/exts/rawdisk.check +++ b/morphlib/exts/rawdisk.check @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. '''Preparatory checks for Morph 'rawdisk' write extension''' diff --git a/morphlib/exts/rawdisk.write b/morphlib/exts/rawdisk.write index d91a4d5f..6f2d45ba 100755 --- a/morphlib/exts/rawdisk.write +++ b/morphlib/exts/rawdisk.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. '''A Morph deployment write extension for raw disk images.''' diff --git a/morphlib/exts/set-hostname.configure b/morphlib/exts/set-hostname.configure index e44c5d56..4b2424d8 100755 --- a/morphlib/exts/set-hostname.configure +++ b/morphlib/exts/set-hostname.configure @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # Set hostname on system from HOSTNAME. diff --git a/morphlib/exts/simple-network.configure b/morphlib/exts/simple-network.configure index b98b202c..61113325 100755 --- a/morphlib/exts/simple-network.configure +++ b/morphlib/exts/simple-network.configure @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,16 +11,17 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. -'''A Morph deployment configuration extension to handle /etc/network/interfaces +'''A Morph deployment configuration extension to handle network configutation -This extension prepares /etc/network/interfaces with the interfaces specified -during deployment. +This extension prepares /etc/network/interfaces and networkd .network files +in /etc/systemd/network/ with the interfaces specified during deployment. If no network configuration is provided, eth0 will be configured for DHCP -with the hostname of the system. +with the hostname of the system in the case of /etc/network/interfaces. +In the case of networkd, any interface starting by e* will be configured +for DHCP ''' @@ -37,20 +38,74 @@ class SimpleNetworkError(morphlib.Error): class SimpleNetworkConfigurationExtension(cliapp.Application): - '''Configure /etc/network/interfaces + '''Configure /etc/network/interfaces and generate networkd .network files - Reading NETWORK_CONFIG, this extension sets up /etc/network/interfaces. + Reading NETWORK_CONFIG, this extension sets up /etc/network/interfaces + and .network files in /etc/systemd/network/. ''' def process_args(self, args): - network_config = os.environ.get( - "NETWORK_CONFIG", "lo:loopback;eth0:dhcp,hostname=$(hostname)") + network_config = os.environ.get("NETWORK_CONFIG") - self.status(msg="Processing NETWORK_CONFIG=%(nc)s", nc=network_config) + self.rename_networkd_chunk_file(args) - stanzas = self.parse_network_stanzas(network_config) - iface_file = self.generate_iface_file(stanzas) + if network_config is None: + self.generate_default_network_config(args) + else: + self.status(msg="Processing NETWORK_CONFIG=%(nc)s", + nc=network_config) + + stanzas = self.parse_network_stanzas(network_config) + + self.generate_interfaces_file(args, stanzas) + self.generate_networkd_files(args, stanzas) + + def rename_networkd_chunk_file(self, args): + """Rename the 10-dchp.network file generated in the systemd chunk + The systemd chunk will place something in 10-dhcp.network, which will + have higher precedence than anything added in this extension (we + start at 50-*). + + We should check for that file and rename it instead remove it in + case the file is being used by the user. + + Until both the following happen, we should continue to rename that + default config file: + + 1. simple-network.configure is always run when systemd is included + 2. We've been building systems without systemd including that default + networkd config for long enough that nobody should be including + that config file. + """ + file_path = os.path.join(args[0], "etc", "systemd", "network", + "10-dhcp.network") + try: + os.rename(file_path, file_path + ".morph") + self.status(msg="Renaming networkd file from systemd chunk: %(f)s \ + to %(f)s.morph", f=file_path) + except OSError: + pass + + def generate_default_network_config(self, args): + """Generate default network config: DHCP in all the interfaces""" + + default_network_config_interfaces = "lo:loopback;" \ + "eth0:dhcp,hostname=$(hostname)" + default_network_config_networkd = "e*:dhcp" + + stanzas_interfaces = self.parse_network_stanzas( + default_network_config_interfaces) + stanzas_networkd = self.parse_network_stanzas( + default_network_config_networkd) + + self.generate_interfaces_file(args, stanzas_interfaces) + self.generate_networkd_files(args, stanzas_networkd) + + def generate_interfaces_file(self, args, stanzas): + """Generate /etc/network/interfaces file""" + + iface_file = self.generate_iface_file(stanzas) with open(os.path.join(args[0], "etc/network/interfaces"), "w") as f: f.write(iface_file) @@ -83,6 +138,74 @@ class SimpleNetworkConfigurationExtension(cliapp.Application): lines += [""] return "\n".join(lines) + def generate_networkd_files(self, args, stanzas): + """Generate .network files""" + + for i, stanza in enumerate(stanzas, 50): + iface_file = self.generate_networkd_file(stanza) + + if iface_file is None: + continue + + path = os.path.join(args[0], "etc", "systemd", "network", + "%s-%s.network" % (i, stanza['name'])) + + with open(path, "w") as f: + f.write(iface_file) + + def generate_networkd_file(self, stanza): + """Generate an .network file from the provided data.""" + + name = stanza['name'] + itype = stanza['type'] + pairs = stanza['args'].items() + + if itype == "loopback": + return + + lines = ["[Match]"] + lines += ["Name=%s\n" % name] + lines += ["[Network]"] + if itype == "dhcp": + lines += ["DHCP=yes"] + else: + lines += self.generate_networkd_entries(pairs) + + return "\n".join(lines) + + def generate_networkd_entries(self, pairs): + """Generate networkd configuration entries with the other parameters""" + + address = None + netmask = None + gateway = None + lines = [] + for pair in pairs: + if pair[0] == 'address': + address = pair[1] + elif pair[0] == 'netmask': + netmask = pair[1] + elif pair[0] == 'gateway': + gateway = pair[1] + + if address and netmask: + network_suffix = self.convert_net_mask_to_cidr_suffix (netmask); + address_line = address + '/' + str(network_suffix) + lines += ["Address=%s" % address_line] + elif address or netmask: + raise Exception('address and netmask must be specified together') + + if gateway is not None: + lines += ["Gateway=%s" % gateway] + + return lines + + def convert_net_mask_to_cidr_suffix(self, mask): + """Convert dotted decimal form of a subnet mask to CIDR suffix notation + + For example: 255.255.255.0 -> 24 + """ + return sum(bin(int(x)).count('1') for x in mask.split('.')) def parse_network_stanzas(self, config): """Parse a network config environment variable into stanzas. diff --git a/morphlib/exts/ssh-rsync.check b/morphlib/exts/ssh-rsync.check index 11446c28..c3bdfd29 100755 --- a/morphlib/exts/ssh-rsync.check +++ b/morphlib/exts/ssh-rsync.check @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. '''Preparatory checks for Morph 'ssh-rsync' write extension''' diff --git a/morphlib/exts/ssh-rsync.write b/morphlib/exts/ssh-rsync.write index c4577026..6d596500 100755 --- a/morphlib/exts/ssh-rsync.write +++ b/morphlib/exts/ssh-rsync.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. '''A Morph deployment write extension for upgrading systems over ssh.''' diff --git a/morphlib/exts/sysroot.check b/morphlib/exts/sysroot.check index bfacd3fc..8ed965bd 100755 --- a/morphlib/exts/sysroot.check +++ b/morphlib/exts/sysroot.check @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # Preparatory checks for Morph 'sysroot' write extension diff --git a/morphlib/exts/sysroot.write b/morphlib/exts/sysroot.write index be315365..0ad8d630 100755 --- a/morphlib/exts/sysroot.write +++ b/morphlib/exts/sysroot.write @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # A Morph write extension to deploy to another directory diff --git a/morphlib/exts/tar.check b/morphlib/exts/tar.check index cbeaf163..f2304d46 100755 --- a/morphlib/exts/tar.check +++ b/morphlib/exts/tar.check @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # Preparatory checks for Morph 'tar' write extension diff --git a/morphlib/exts/tar.write b/morphlib/exts/tar.write index 333626b5..01b545b4 100755 --- a/morphlib/exts/tar.write +++ b/morphlib/exts/tar.write @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # A Morph write extension to deploy to a .tar file diff --git a/morphlib/exts/vdaboot.configure b/morphlib/exts/vdaboot.configure index b88eb3a8..60de925b 100755 --- a/morphlib/exts/vdaboot.configure +++ b/morphlib/exts/vdaboot.configure @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # Change the "/" mount point to /dev/vda to use virtio disks. diff --git a/morphlib/exts/virtualbox-ssh.check b/morphlib/exts/virtualbox-ssh.check index 57d54db1..a97f3294 100755 --- a/morphlib/exts/virtualbox-ssh.check +++ b/morphlib/exts/virtualbox-ssh.check @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. '''Preparatory checks for Morph 'virtualbox-ssh' write extension''' diff --git a/morphlib/exts/virtualbox-ssh.write b/morphlib/exts/virtualbox-ssh.write index 7eafcff3..774f2b4f 100755 --- a/morphlib/exts/virtualbox-ssh.write +++ b/morphlib/exts/virtualbox-ssh.write @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. '''A Morph deployment write extension for deploying to VirtualBox via ssh. diff --git a/morphlib/fsutils.py b/morphlib/fsutils.py index 8a4128d9..a3b73bf6 100644 --- a/morphlib/fsutils.py +++ b/morphlib/fsutils.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import os import re diff --git a/morphlib/fsutils_tests.py b/morphlib/fsutils_tests.py index 47a4488e..f2772fb7 100644 --- a/morphlib/fsutils_tests.py +++ b/morphlib/fsutils_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013, 2014 Codethink Limited +# Copyright (C) 2013, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import os diff --git a/morphlib/git.py b/morphlib/git.py index 456aa4c0..acda6137 100644 --- a/morphlib/git.py +++ b/morphlib/git.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2014 Codethink Limited +# Copyright (C) 2011-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp @@ -191,7 +190,7 @@ def get_user_name(runcmd): try: config = check_config_set(runcmd, keys={"user.name": "My Name"}) return config['user.name'] - except ConfigNotSetException, e: + except ConfigNotSetException as e: raise IdentityNotSetException(e.missing) @@ -202,7 +201,7 @@ def get_user_email(runcmd): try: cfg = check_config_set(runcmd, keys={"user.email": "me@example.com"}) return cfg['user.email'] - except ConfigNotSetException, e: + except ConfigNotSetException as e: raise IdentityNotSetException(e.missing) def check_config_set(runcmd, keys, cwd='.'): diff --git a/morphlib/gitdir.py b/morphlib/gitdir.py index 391db91f..03640a22 100644 --- a/morphlib/gitdir.py +++ b/morphlib/gitdir.py @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= @@ -707,7 +706,7 @@ class GitDirectory(object): # this ensures it will fail if the branch already exists try: return self._update_ref((ref, sha1, '0' * 40), message) - except Exception, e: + except Exception as e: raise RefAddError(self, ref, sha1, e) def update_ref(self, ref, sha1, old_sha1, message=None): @@ -727,7 +726,7 @@ class GitDirectory(object): self._check_is_sha1(old_sha1) try: return self._update_ref((ref, sha1, old_sha1), message) - except Exception, e: + except Exception as e: raise RefUpdateError(self, ref, old_sha1, sha1, e) def delete_ref(self, ref, old_sha1, message=None): @@ -745,7 +744,7 @@ class GitDirectory(object): self._check_is_sha1(old_sha1) try: return self._update_ref(('-d', ref, old_sha1), message) - except Exception, e: + except Exception as e: raise RefDeleteError(self, ref, old_sha1, e) def describe(self): diff --git a/morphlib/gitdir_tests.py b/morphlib/gitdir_tests.py index 50de18d7..a6e1921d 100644 --- a/morphlib/gitdir_tests.py +++ b/morphlib/gitdir_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= diff --git a/morphlib/gitindex.py b/morphlib/gitindex.py index c5c07bd6..00098fbf 100644 --- a/morphlib/gitindex.py +++ b/morphlib/gitindex.py @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= diff --git a/morphlib/gitindex_tests.py b/morphlib/gitindex_tests.py index 3f9ff303..a3196764 100644 --- a/morphlib/gitindex_tests.py +++ b/morphlib/gitindex_tests.py @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= diff --git a/morphlib/gitversion.py b/morphlib/gitversion.py index c593c330..7b4459a4 100644 --- a/morphlib/gitversion.py +++ b/morphlib/gitversion.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 - 2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. '''Version information retrieved either from the package data, or the @@ -34,7 +33,7 @@ try: commit = pkgutil.get_data('morphlib', 'commit') tree = pkgutil.get_data('morphlib', 'tree') ref = pkgutil.get_data('morphlib', 'ref') -except IOError, e: +except IOError as e: from os.path import dirname def run_git(*args): command = ['git'] + list(args) diff --git a/morphlib/localartifactcache.py b/morphlib/localartifactcache.py index 955ee97f..e6695c4e 100644 --- a/morphlib/localartifactcache.py +++ b/morphlib/localartifactcache.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012, 2013, 2014 Codethink Limited +# Copyright (C) 2012, 2013, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import collections diff --git a/morphlib/localartifactcache_tests.py b/morphlib/localartifactcache_tests.py index 4325cfbe..9483f9f6 100644 --- a/morphlib/localartifactcache_tests.py +++ b/morphlib/localartifactcache_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012,2014 Codethink Limited +# Copyright (C) 2012,2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import unittest diff --git a/morphlib/localrepocache.py b/morphlib/localrepocache.py index 1565b913..4fc3916a 100644 --- a/morphlib/localrepocache.py +++ b/morphlib/localrepocache.py @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import os @@ -173,7 +172,7 @@ class LocalRepoCache(object): self._git(['config', 'remote.origin.mirror', 'true'], cwd=path) self._git(['config', 'remote.origin.fetch', '+refs/*:refs/*'], cwd=path) - except BaseException, e: # pragma: no cover + except BaseException as e: # pragma: no cover if self.fs.exists(path): self.fs.removedir(path, force=True) return False, 'Unable to extract tarball %s: %s' % ( @@ -193,7 +192,7 @@ class LocalRepoCache(object): try: return self.get_repo(reponame) - except NotCached, e: + except NotCached as e: pass repourl = self._resolver.pull_url(reponame) @@ -214,7 +213,7 @@ class LocalRepoCache(object): try: self._git(['clone', '--mirror', '-n', repourl, target], echo_stderr=self._app.settings['verbose']) - except cliapp.AppException, e: + except cliapp.AppException as e: errors.append('Unable to clone from %s to %s: %s' % (repourl, target, e)) if self.fs.exists(target): diff --git a/morphlib/localrepocache_tests.py b/morphlib/localrepocache_tests.py index aeb32961..aeef18f1 100644 --- a/morphlib/localrepocache_tests.py +++ b/morphlib/localrepocache_tests.py @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import unittest diff --git a/morphlib/morphloader.py b/morphlib/morphloader.py index 7d51dc1e..47cb03d7 100644 --- a/morphlib/morphloader.py +++ b/morphlib/morphloader.py @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= diff --git a/morphlib/morphloader_tests.py b/morphlib/morphloader_tests.py index a1fe1674..f9fa2a34 100644 --- a/morphlib/morphloader_tests.py +++ b/morphlib/morphloader_tests.py @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= diff --git a/morphlib/morphology.py b/morphlib/morphology.py index 009ed044..53a8ff39 100644 --- a/morphlib/morphology.py +++ b/morphlib/morphology.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= diff --git a/morphlib/morphology_tests.py b/morphlib/morphology_tests.py index 385f62ee..d3d99f15 100644 --- a/morphlib/morphology_tests.py +++ b/morphlib/morphology_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= diff --git a/morphlib/morphologyfinder.py b/morphlib/morphologyfinder.py index 87c0de1a..708c86a9 100644 --- a/morphlib/morphologyfinder.py +++ b/morphlib/morphologyfinder.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= diff --git a/morphlib/morphologyfinder_tests.py b/morphlib/morphologyfinder_tests.py index 67161f9b..a83f89a5 100644 --- a/morphlib/morphologyfinder_tests.py +++ b/morphlib/morphologyfinder_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= diff --git a/morphlib/morphset.py b/morphlib/morphset.py index bf061f94..a4cb5dae 100644 --- a/morphlib/morphset.py +++ b/morphlib/morphset.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= diff --git a/morphlib/morphset_tests.py b/morphlib/morphset_tests.py index 81b5810f..bf726d4c 100644 --- a/morphlib/morphset_tests.py +++ b/morphlib/morphset_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013, 2014 Codethink Limited +# Copyright (C) 2013, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= diff --git a/morphlib/mountableimage.py b/morphlib/mountableimage.py index f767228a..dcc496e7 100644 --- a/morphlib/mountableimage.py +++ b/morphlib/mountableimage.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp @@ -46,7 +45,7 @@ class MountableImage(object): # pragma: no cover infh = gzip.open(path, "rb") morphlib.util.copyfileobj(infh, outfh) infh.close() - except BaseException, e: + except BaseException as e: logging.error('Caught exception: %s' % str(e)) logging.info('Removing temporary file %s' % self.temp_path) os.unlink(self.temp_path) @@ -65,17 +64,17 @@ class MountableImage(object): # pragma: no cover chatty=True) try: morphlib.fsutils.unmount(self.app.runcmd, mount_point) - except BaseException, e: + except BaseException as e: logging.info('Ignoring error when unmounting: %s' % str(e)) try: morphlib.fsutils.undo_device_mapping(self.app.runcmd, path) - except BaseException, e: + except BaseException as e: logging.info( 'Ignoring error when undoing device mapping: %s' % str(e)) try: os.rmdir(mount_point) os.unlink(path) - except BaseException, e: + except BaseException as e: logging.info( 'Ignoring error when removing temporary files: %s' % str(e)) diff --git a/morphlib/plugins/add_binary_plugin.py b/morphlib/plugins/add_binary_plugin.py index dee1d9c4..45edae4c 100644 --- a/morphlib/plugins/add_binary_plugin.py +++ b/morphlib/plugins/add_binary_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp diff --git a/morphlib/plugins/artifact_inspection_plugin.py b/morphlib/plugins/artifact_inspection_plugin.py index 2d1fe979..b16f393c 100644 --- a/morphlib/plugins/artifact_inspection_plugin.py +++ b/morphlib/plugins/artifact_inspection_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp @@ -163,7 +162,7 @@ class VersionGuesser(object): version = guesser.guess_version(repo, ref, tree) if version: break - except cliapp.AppException, err: + except cliapp.AppException as err: self.app.status(msg='%(repo)s: Failed to list files in %(ref)s', repo=repo, ref=ref, chatty=True) return version diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py index 76da9736..08589ea6 100644 --- a/morphlib/plugins/branch_and_merge_plugin.py +++ b/morphlib/plugins/branch_and_merge_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012,2013,2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp diff --git a/morphlib/plugins/build_plugin.py b/morphlib/plugins/build_plugin.py index e9290b8d..2cc395fc 100644 --- a/morphlib/plugins/build_plugin.py +++ b/morphlib/plugins/build_plugin.py @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp diff --git a/morphlib/plugins/cross-bootstrap_plugin.py b/morphlib/plugins/cross-bootstrap_plugin.py index 2b5c8911..79609cb5 100644 --- a/morphlib/plugins/cross-bootstrap_plugin.py +++ b/morphlib/plugins/cross-bootstrap_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp import logging @@ -66,7 +65,7 @@ class BootstrapSystemBuilder(morphlib.builder.BuilderBase): self.unpack_sources(fs_root) self.write_build_script(fs_root) self.create_tarball(handle, fs_root, system_name) - except BaseException, e: + except BaseException as e: logging.error(traceback.format_exc()) self.app.status(msg='Error while building bootstrap image', error=True) @@ -85,7 +84,7 @@ class BootstrapSystemBuilder(morphlib.builder.BuilderBase): with cache.get(chunk_artifact) as chunk_file: try: morphlib.bins.unpack_binary_from_file(chunk_file, dest) - except BaseException, e: + except BaseException as e: self.app.status( msg='Error unpacking binary chunk %(name)s', name=chunk_artifact.name, @@ -111,7 +110,7 @@ class BootstrapSystemBuilder(morphlib.builder.BuilderBase): chunk_script = os.path.join(path, 'src', 'build-%s' % name) with morphlib.savefile.SaveFile(chunk_script, 'w') as f: self.write_chunk_build_script(s, f) - os.chmod(chunk_script, 0777) + os.chmod(chunk_script, 0o777) def write_build_script(self, path): '''Output a script to run build on the bootstrap target''' @@ -136,7 +135,7 @@ class BootstrapSystemBuilder(morphlib.builder.BuilderBase): f.write('if [ -e /sbin/ldconfig ]; then /sbin/ldconfig; fi\n') f.write(driver_footer) - os.chmod(driver_script, 0777) + os.chmod(driver_script, 0o777) def write_chunk_build_script(self, source, f): m = source.morphology diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index 6d506a21..7635a7b4 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import json diff --git a/morphlib/plugins/distbuild_plugin.py b/morphlib/plugins/distbuild_plugin.py index a7d69472..ac3957f3 100644 --- a/morphlib/plugins/distbuild_plugin.py +++ b/morphlib/plugins/distbuild_plugin.py @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp @@ -180,7 +179,6 @@ class ControllerDaemon(cliapp.Plugin): self.app.settings.string( ['initiator-step-output-dir'], 'write build output to files in DIR', - default='.', group=group_distbuild) self.app.settings.string( diff --git a/morphlib/plugins/expand_repo_plugin.py b/morphlib/plugins/expand_repo_plugin.py index 721287ca..bbee891d 100644 --- a/morphlib/plugins/expand_repo_plugin.py +++ b/morphlib/plugins/expand_repo_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp diff --git a/morphlib/plugins/gc_plugin.py b/morphlib/plugins/gc_plugin.py index 68f386eb..71522b04 100644 --- a/morphlib/plugins/gc_plugin.py +++ b/morphlib/plugins/gc_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import logging diff --git a/morphlib/plugins/graphing_plugin.py b/morphlib/plugins/graphing_plugin.py index 57166e51..0db63ffc 100644 --- a/morphlib/plugins/graphing_plugin.py +++ b/morphlib/plugins/graphing_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012, 2013 Codethink Limited +# Copyright (C) 2012, 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp diff --git a/morphlib/plugins/list_artifacts_plugin.py b/morphlib/plugins/list_artifacts_plugin.py index 53056bad..b7a5e080 100644 --- a/morphlib/plugins/list_artifacts_plugin.py +++ b/morphlib/plugins/list_artifacts_plugin.py @@ -10,13 +10,13 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # This plugin is used as part of the Baserock automated release process. # # See: <http://wiki.baserock.org/guides/release-process> for more information. +from __future__ import print_function import cliapp import morphlib @@ -68,7 +68,7 @@ class ListArtifactsPlugin(cliapp.Plugin): artifact_files.update(system_artifact_files) for artifact_file in sorted(artifact_files): - print artifact_file + print(artifact_file) def list_artifacts_for_system(self, repo, ref, system_filename): '''List all artifact files in the build graph of a single system.''' diff --git a/morphlib/plugins/print_architecture_plugin.py b/morphlib/plugins/print_architecture_plugin.py index 08f500d0..264334d9 100644 --- a/morphlib/plugins/print_architecture_plugin.py +++ b/morphlib/plugins/print_architecture_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp diff --git a/morphlib/plugins/push_pull_plugin.py b/morphlib/plugins/push_pull_plugin.py index ddc9a8af..8546e2bf 100644 --- a/morphlib/plugins/push_pull_plugin.py +++ b/morphlib/plugins/push_pull_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp import logging diff --git a/morphlib/plugins/show_dependencies_plugin.py b/morphlib/plugins/show_dependencies_plugin.py index e70f6bfb..ee96cb40 100644 --- a/morphlib/plugins/show_dependencies_plugin.py +++ b/morphlib/plugins/show_dependencies_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp diff --git a/morphlib/plugins/trovectl_plugin.py b/morphlib/plugins/trovectl_plugin.py index 80f3b4cf..ea780b65 100644 --- a/morphlib/plugins/trovectl_plugin.py +++ b/morphlib/plugins/trovectl_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp diff --git a/morphlib/recv-hole b/morphlib/recv-hole index d6504bf6..fe69f304 100755 --- a/morphlib/recv-hole +++ b/morphlib/recv-hole @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= diff --git a/morphlib/remoteartifactcache.py b/morphlib/remoteartifactcache.py index 4e09ce34..427e4cbb 100644 --- a/morphlib/remoteartifactcache.py +++ b/morphlib/remoteartifactcache.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp @@ -72,14 +71,14 @@ class RemoteArtifactCache(object): def get(self, artifact, log=logging.error): try: return self._get_file(artifact.basename()) - except urllib2.URLError, e: + except urllib2.URLError as e: log(str(e)) raise GetError(self, artifact) def get_artifact_metadata(self, artifact, name, log=logging.error): try: return self._get_file(artifact.metadata_basename(name)) - except urllib2.URLError, e: + except urllib2.URLError as e: log(str(e)) raise GetArtifactMetadataError(self, artifact, name) diff --git a/morphlib/remoteartifactcache_tests.py b/morphlib/remoteartifactcache_tests.py index 788882c2..18bef13f 100644 --- a/morphlib/remoteartifactcache_tests.py +++ b/morphlib/remoteartifactcache_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import StringIO diff --git a/morphlib/remoterepocache.py b/morphlib/remoterepocache.py index 004ba86e..4a6d9fe9 100644 --- a/morphlib/remoterepocache.py +++ b/morphlib/remoterepocache.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp @@ -55,7 +54,7 @@ class RemoteRepoCache(object): repo_url = self._resolver.pull_url(repo_name) try: return self._resolve_ref_for_repo_url(repo_url, ref) - except BaseException, e: + except BaseException as e: logging.error('Caught exception: %s' % str(e)) raise ResolveRefError(repo_name, ref) @@ -74,7 +73,7 @@ class RemoteRepoCache(object): try: info = json.loads(self._ls_tree_for_repo_url(repo_url, ref)) return info['tree'].keys() - except BaseException, e: + except BaseException as e: logging.error('Caught exception: %s' % str(e)) raise LsTreeError(repo_name, ref) diff --git a/morphlib/remoterepocache_tests.py b/morphlib/remoterepocache_tests.py index ef81506f..966e74d5 100644 --- a/morphlib/remoterepocache_tests.py +++ b/morphlib/remoterepocache_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import json diff --git a/morphlib/repoaliasresolver.py b/morphlib/repoaliasresolver.py index bc759dd4..de57628c 100644 --- a/morphlib/repoaliasresolver.py +++ b/morphlib/repoaliasresolver.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import logging diff --git a/morphlib/repoaliasresolver_tests.py b/morphlib/repoaliasresolver_tests.py index c4ea16b0..a10c4017 100644 --- a/morphlib/repoaliasresolver_tests.py +++ b/morphlib/repoaliasresolver_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import morphlib diff --git a/morphlib/savefile.py b/morphlib/savefile.py index 2d87a54f..7deb2b89 100644 --- a/morphlib/savefile.py +++ b/morphlib/savefile.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import logging diff --git a/morphlib/savefile_tests.py b/morphlib/savefile_tests.py index 7ae2360d..dc11086c 100644 --- a/morphlib/savefile_tests.py +++ b/morphlib/savefile_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import os diff --git a/morphlib/source.py b/morphlib/source.py index 4ad54ed9..2c96bcbd 100644 --- a/morphlib/source.py +++ b/morphlib/source.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import morphlib diff --git a/morphlib/source_tests.py b/morphlib/source_tests.py index 695041d3..4f6006e9 100644 --- a/morphlib/source_tests.py +++ b/morphlib/source_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import unittest diff --git a/morphlib/sourcepool.py b/morphlib/sourcepool.py index 6dfcb2c3..b340a9de 100644 --- a/morphlib/sourcepool.py +++ b/morphlib/sourcepool.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import collections diff --git a/morphlib/sourcepool_tests.py b/morphlib/sourcepool_tests.py index f3740049..85833901 100644 --- a/morphlib/sourcepool_tests.py +++ b/morphlib/sourcepool_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import unittest diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py index 387d2e0d..1e64c23a 100644 --- a/morphlib/sourceresolver.py +++ b/morphlib/sourceresolver.py @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import collections @@ -32,7 +31,7 @@ tree_cache_filename = 'trees.cache.pickle' buildsystem_cache_size = 10000 buildsystem_cache_filename = 'detected-chunk-buildsystems.cache.pickle' -not_supported_versions = [] +supported_versions = [0, 1] class PickleCacheManager(object): # pragma: no cover '''Cache manager for PyLRU that reads and writes to Pickle files. @@ -185,6 +184,8 @@ class SourceResolver(object): reponame, ref) return ref, self._resolved_trees[(reponame, ref)] + logging.debug('tree (%s, %s) not in cache', reponame, ref) + absref = None if self.lrc.has_repo(reponame): repo = self.lrc.get_repo(reponame) @@ -205,7 +206,7 @@ class SourceResolver(object): reponame=reponame, ref=ref, chatty=True) - except BaseException, e: + except BaseException as e: logging.warning('Caught (and ignored) exception: %s' % str(e)) if absref is None: @@ -345,6 +346,29 @@ class SourceResolver(object): loader.set_defaults(morph) return morph + def _parse_version_file(self, version_file): # pragma : no cover + '''Parse VERSION file and return the version of the format if: + + VERSION is a YAML file + and it's a dict + and has the key 'version' + and the type stored in the 'version' key is an int + and that int is not in the supported format + + otherwise returns None + + ''' + version = None + + yaml_obj = yaml.safe_load(version_file) + if yaml_obj is not None: + if type(yaml_obj) is dict: + if 'version' in yaml_obj.keys(): + if type(yaml_obj['version']) is int: + version = yaml_obj['version'] + + return version + def _check_version_file(self,definitions_repo, definitions_absref): # pragma: no cover version_file = self._get_file_contents( @@ -353,13 +377,10 @@ class SourceResolver(object): if version_file is None: return - try: - version = yaml.safe_load(version_file)['version'] - except (yaml.error.YAMLError, KeyError, TypeError): - version = 0 - - if version in not_supported_versions: - raise UnknownVersionError(version) + version = self._parse_version_file(version_file) + if version is not None: + if version not in supported_versions: + raise UnknownVersionError(version) def _process_definitions_with_children(self, system_filenames, definitions_repo, @@ -415,42 +436,77 @@ class SourceResolver(object): return chunk_queue + def _generate_morph_and_cache_buildsystem(self, + definition_key, chunk_key, + buildsystem, + morph_name): # pragma: no cover + logging.debug('Caching build system for chunk with key %s', chunk_key) + + self._resolved_buildsystems[chunk_key] = buildsystem + + morphology = self._create_morphology_for_build_system(buildsystem, + morph_name) + self._resolved_morphologies[definition_key] = morphology + return morphology + def process_chunk(self, definition_repo, definition_ref, chunk_repo, chunk_ref, filename, visit): # pragma: no cover absref = None tree = None - - definition_key = (definition_repo, definition_ref, filename) chunk_key = None + buildsystem = None morph_name = os.path.splitext(os.path.basename(filename))[0] + # Get morphology from definitions repo + definition_key = (definition_repo, definition_ref, filename) morphology = self._get_morphology(*definition_key) - buildsystem = None + + if morphology: + absref, tree = self._resolve_ref(chunk_repo, chunk_ref) + visit(chunk_repo, chunk_ref, filename, absref, tree, morphology) + return + + absref, tree = self._resolve_ref(chunk_repo, chunk_ref) + chunk_key = (chunk_repo, absref, filename) if chunk_key in self._resolved_buildsystems: + logging.debug('Build system for %s is cached', str(chunk_key)) + self.status(msg='Build system for %(chunk)s is cached', + chunk=str(chunk_key), + chatty=True) buildsystem = self._resolved_buildsystems[chunk_key] - if morphology is None and buildsystem is None: - # This is a slow operation (looking for a file in Git repo may - # potentially require cloning the whole thing). - absref, tree = self._resolve_ref(chunk_repo, chunk_ref) - chunk_key = (chunk_repo, absref, filename) + # If the build system for this chunk is cached then: + # * the chunk does not have a chunk morph + # (so we don't need to look for one) + # + # * a suitable (generated) morphology may already be cached. + # + # If the morphology is not already cached we can generate it + # from the build-system and cache it. + if definition_key in self._resolved_morphologies: + morphology = self._resolved_morphologies[definition_key] + else: + morphology = self._generate_morph_and_cache_buildsystem( + definition_key, chunk_key, buildsystem, morph_name) + else: + logging.debug('Build system for %s is NOT cached', str(chunk_key)) + # build-system not cached, look for morphology in chunk repo + # this can be slow (we may need to clone the repo from a remote) morphology = self._get_morphology(*chunk_key) - if morphology is None: - if buildsystem is None: - buildsystem = self._detect_build_system(*chunk_key) - if buildsystem is None: - raise MorphologyNotFoundError(filename) - else: - self._resolved_buildsystems[chunk_key] = buildsystem - morphology = self._create_morphology_for_build_system( - buildsystem, morph_name) + if morphology != None: self._resolved_morphologies[definition_key] = morphology + else: + # This chunk doesn't have a chunk morph + buildsystem = self._detect_build_system(*chunk_key) - if not absref or not tree: - absref, tree = self._resolve_ref(chunk_repo, chunk_ref) + if buildsystem is None: + raise MorphologyNotFoundError(filename) + else: + morphology = self._generate_morph_and_cache_buildsystem( + definition_key, chunk_key, buildsystem, morph_name) visit(chunk_repo, chunk_ref, filename, absref, tree, morphology) diff --git a/morphlib/sourceresolver_tests.py b/morphlib/sourceresolver_tests.py index 1239b437..298dae5d 100644 --- a/morphlib/sourceresolver_tests.py +++ b/morphlib/sourceresolver_tests.py @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import os diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py index fd3f6881..8c2781aa 100644 --- a/morphlib/stagingarea.py +++ b/morphlib/stagingarea.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import logging @@ -166,7 +165,7 @@ class StagingArea(object): try: morphlib.bins.unpack_binary_from_file( handle, savedir + '/') - except BaseException, e: # pragma: no cover + except BaseException as e: # pragma: no cover shutil.rmtree(savedir) raise # TODO: This rename is not concurrency safe if two builds are diff --git a/morphlib/stagingarea_tests.py b/morphlib/stagingarea_tests.py index dc43e4f6..97d78236 100644 --- a/morphlib/stagingarea_tests.py +++ b/morphlib/stagingarea_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp diff --git a/morphlib/stopwatch.py b/morphlib/stopwatch.py index 29e584bd..bb01d06f 100644 --- a/morphlib/stopwatch.py +++ b/morphlib/stopwatch.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2012 Codethink Limited +# Copyright (C) 2011-2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import operator diff --git a/morphlib/stopwatch_tests.py b/morphlib/stopwatch_tests.py index deb528d5..3c8376b1 100644 --- a/morphlib/stopwatch_tests.py +++ b/morphlib/stopwatch_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2012 Codethink Limited +# Copyright (C) 2011-2012,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import datetime diff --git a/morphlib/sysbranchdir.py b/morphlib/sysbranchdir.py index 80e56b33..69119f5b 100644 --- a/morphlib/sysbranchdir.py +++ b/morphlib/sysbranchdir.py @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= diff --git a/morphlib/sysbranchdir_tests.py b/morphlib/sysbranchdir_tests.py index 1aca54e6..341fedca 100644 --- a/morphlib/sysbranchdir_tests.py +++ b/morphlib/sysbranchdir_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= diff --git a/morphlib/systemmetadatadir.py b/morphlib/systemmetadatadir.py index 7e89142c..573d8a20 100644 --- a/morphlib/systemmetadatadir.py +++ b/morphlib/systemmetadatadir.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= diff --git a/morphlib/systemmetadatadir_tests.py b/morphlib/systemmetadatadir_tests.py index 0126f862..03496108 100644 --- a/morphlib/systemmetadatadir_tests.py +++ b/morphlib/systemmetadatadir_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= diff --git a/morphlib/util.py b/morphlib/util.py index ad9d7cd9..e733af9d 100644 --- a/morphlib/util.py +++ b/morphlib/util.py @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import contextlib import itertools @@ -645,3 +644,35 @@ def error_message_for_containerised_commandline( 'Containerisation settings: %s\n' \ 'Error output:\n%s' \ % (argv_string, container_kwargs, err) + + +def write_from_dict(filepath, d, validate=lambda x, y: True): #pragma: no cover + '''Takes a dictionary and appends the contents to a file + + An optional validation callback can be passed to perform validation on + each value in the dictionary. + + e.g. + + def validation_callback(dictionary_key, dictionary_value): + if not dictionary_value.isdigit(): + raise Exception('value contains non-digit character(s)') + + Any callback supplied to this function should raise an exception + if validation fails. + ''' + + # Sort items asciibetically + # the output of the deployment should not depend + # on the locale of the machine running the deployment + items = sorted(d.iteritems(), key=lambda (k, v): [ord(c) for c in v]) + + for (k, v) in items: + validate(k, v) + + with open(filepath, 'a') as f: + for (_, v) in items: + f.write('%s\n' % v) + + os.fchown(f.fileno(), 0, 0) + os.fchmod(f.fileno(), 0644) diff --git a/morphlib/util_tests.py b/morphlib/util_tests.py index 715892b6..20972a58 100644 --- a/morphlib/util_tests.py +++ b/morphlib/util_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2014 Codethink Limited +# Copyright (C) 2011-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import os diff --git a/morphlib/workspace.py b/morphlib/workspace.py index 27ccbe65..26986655 100644 --- a/morphlib/workspace.py +++ b/morphlib/workspace.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= diff --git a/morphlib/workspace_tests.py b/morphlib/workspace_tests.py index 9eef1053..23d7ad5e 100644 --- a/morphlib/workspace_tests.py +++ b/morphlib/workspace_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= diff --git a/morphlib/writeexts.py b/morphlib/writeexts.py index 270cade9..aa185a2b 100644 --- a/morphlib/writeexts.py +++ b/morphlib/writeexts.py @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import cliapp @@ -178,7 +177,7 @@ class WriteExtension(cliapp.Application): self.create_btrfs_system_layout( temp_root, mp, version_label='factory', disk_uuid=self.get_uuid(raw_disk)) - except BaseException, e: + except BaseException as e: sys.stderr.write('Error creating Btrfs system layout') raise @@ -280,7 +279,7 @@ class WriteExtension(cliapp.Application): cliapp.runcmd(['mount', location, mount_point]) else: cliapp.runcmd(['mount', '-o', 'loop', location, mount_point]) - except BaseException, e: + except BaseException as e: sys.stderr.write('Error mounting filesystem') os.rmdir(mount_point) raise @@ -360,7 +359,7 @@ class WriteExtension(cliapp.Application): self.status(msg='Creating %s subvolume' % state_subdir) subvolume = os.path.join(mountpoint, 'state', state_subdir) cliapp.runcmd(['btrfs', 'subvolume', 'create', subvolume]) - os.chmod(subvolume, 0755) + os.chmod(subvolume, 0o755) existing_state_dir = os.path.join(system_dir, state_subdir) files = [] diff --git a/morphlib/xfer-hole b/morphlib/xfer-hole index 22ee06bf..91f1be01 100755 --- a/morphlib/xfer-hole +++ b/morphlib/xfer-hole @@ -7,7 +7,7 @@ # SEEK_DATA and SEEK_HOLE. # # -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,8 +19,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. # # =*= License: GPL-2 =*= diff --git a/morphlib/yamlparse.py b/morphlib/yamlparse.py index 6f139304..075fe3b1 100644 --- a/morphlib/yamlparse.py +++ b/morphlib/yamlparse.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import morphlib diff --git a/morphlib/yamlparse_tests.py b/morphlib/yamlparse_tests.py index 38815168..d5da0ca9 100644 --- a/morphlib/yamlparse_tests.py +++ b/morphlib/yamlparse_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import unittest |