diff options
-rw-r--r-- | morphlib/buildcommand.py | 85 | ||||
-rw-r--r-- | morphlib/plugins/cross-bootstrap_plugin.py | 14 | ||||
-rw-r--r-- | morphlib/remoteartifactcache.py | 11 |
3 files changed, 60 insertions, 50 deletions
diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 7ad7909d..5abef54c 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -274,31 +274,31 @@ class BuildCommand(object): 'name': a.name, }) - self.app.status(msg='Checking if %(kind)s needs ' - 'building %(sha1)s', - kind=a.source.morphology['kind'], - sha1=a.source.sha1[:7]) - - if self.is_built(a): - self.cache_artifacts_locally([a]) - self.app.status( - msg='The %(kind)s is cached at %(cache)s', - kind=a.source.morphology['kind'], - cache=os.path.basename(self.lac.artifact_filename(a))[:7]) - else: - self.app.status(msg='Building %(kind)s %(name)s', - name=a.name, kind=a.source.morphology['kind']) - self.build_artifact(a, build_env) + self.cache_or_build_artifact(a, build_env) self.app.status(msg='%(kind)s %(name)s is cached at %(cachepath)s', kind=a.source.morphology['kind'], name=a.name, cachepath=self.lac.artifact_filename(a), chatty=(a.source.morphology['kind'] != "system")) + self.app.status_prefix = old_prefix - def is_built(self, artifact): - '''Does either cache already have the artifact?''' - return self.lac.has(artifact) or (self.rac and self.rac.has(artifact)) + def cache_or_build_artifact(self, artifact, build_env): + '''Make the built artifact available in the local cache. + + This can be done by retrieving from a remote artifact cache, or if + that doesn't work for some reason, by building the artifact locally. + + ''' + if self.rac is not None: + try: + self.cache_artifacts_locally([artifact]) + except morphlib.remoteartifactcache.GetError: + # Error is logged by the RemoteArtifactCache object. + pass + + if not self.lac.has(artifact): + self.build_artifact(artifact, build_env) def build_artifact(self, artifact, build_env): '''Build one artifact. @@ -307,6 +307,10 @@ class BuildCommand(object): in either the local or remote cache already. ''' + self.app.status(msg='Building %(kind)s %(name)s', + name=artifact.name, + kind=artifact.source.morphology['kind']) + self.get_sources(artifact) deps = self.get_recursive_deps(artifact) self.cache_artifacts_locally(deps) @@ -389,27 +393,42 @@ class BuildCommand(object): def cache_artifacts_locally(self, artifacts): '''Get artifacts missing from local cache from remote cache.''' - def copy(remote, local): - shutil.copyfileobj(remote, local) - remote.close() - local.close() + def fetch_files(to_fetch): + '''Fetch a set of files atomically. + + If an error occurs during the transfer of any files, all downloaded + data is deleted, to ensure integrity of the local cache. + + ''' + try: + for remote, local in to_fetch: + shutil.copyfileobj(remote, local) + except BaseException: + for remote, local in to_fetch: + local.abort() + raise + else: + for remote, local in to_fetch: + remote.close() + local.close() for artifact in artifacts: + to_fetch = [] if not self.lac.has(artifact): - self.app.status(msg='Fetching to local cache: ' - 'artifact %(name)s', - name=artifact.name) - rac_file = self.rac.get(artifact) - lac_file = self.lac.put(artifact) - copy(rac_file, lac_file) + to_fetch.append((self.rac.get(artifact), + self.lac.put(artifact))) if artifact.source.morphology.needs_artifact_metadata_cached: if not self.lac.has_artifact_metadata(artifact, 'meta'): - self.app.status(msg='Fetching to local cache: ' - 'artifact metadata %(name)s', - name=artifact.name) - copy(self.rac.get_artifact_metadata(artifact, 'meta'), - self.lac.put_artifact_metadata(artifact, 'meta')) + to_fetch.append(( + self.rac.get_artifact_metadata(artifact, 'meta'), + self.lac.put_artifact_metadata(artifact, 'meta'))) + + if len(to_fetch) > 0: + self.app.status( + msg='Fetching to local cache: artifact %(name)s', + name=artifact.name) + fetch_files(to_fetch) def create_staging_area(self, build_env, use_chroot=True, extra_env={}, extra_path=[]): diff --git a/morphlib/plugins/cross-bootstrap_plugin.py b/morphlib/plugins/cross-bootstrap_plugin.py index ec0cfbcb..bfd0d047 100644 --- a/morphlib/plugins/cross-bootstrap_plugin.py +++ b/morphlib/plugins/cross-bootstrap_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -299,18 +299,8 @@ class CrossBootstrapPlugin(cliapp.Plugin): 'Nothing to cross-compile. Only chunks built in \'bootstrap\' ' 'mode can be cross-compiled.') - # FIXME: merge with build-command's code for i, a in enumerate(cross_chunks): - if build_command.is_built(a): - self.app.status(msg='The %(kind)s %(name)s is already built', - kind=a.source.morphology['kind'], - name=a.name) - build_command.cache_artifacts_locally([a]) - else: - self.app.status(msg='Cross-building %(kind)s %(name)s', - kind=a.source.morphology['kind'], - name=a.name) - build_command.build_artifact(a, build_env) + build_command.cache_or_build_artifact(a, build_env) for i, a in enumerate(native_chunks): build_command.get_sources(a) diff --git a/morphlib/remoteartifactcache.py b/morphlib/remoteartifactcache.py index 9f6bf69e..0f8edce8 100644 --- a/morphlib/remoteartifactcache.py +++ b/morphlib/remoteartifactcache.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -33,18 +33,19 @@ class GetError(cliapp.AppException): cliapp.AppException.__init__( self, 'Failed to get the artifact %s with cache key %s ' 'from the artifact cache %s' % - (artifact, artifact.cache_key, cache)) + (artifact.basename(), artifact.cache_key, cache)) -class GetArtifactMetadataError(cliapp.AppException): +class GetArtifactMetadataError(GetError): def __init__(self, cache, artifact, name): cliapp.AppException.__init__( self, 'Failed to get metadata %s for the artifact %s ' - 'from the artifact cache %s' % (name, artifact, cache)) + 'from the artifact cache %s' % + (name, artifact.basename(), cache)) -class GetSourceMetadataError(cliapp.AppException): +class GetSourceMetadataError(GetError): def __init__(self, cache, source, cache_key, name): cliapp.AppException.__init__( |