Merge branch 'sam/fix-fetch-errors' into sam/distbuild-build-logs

Conflicts:
	without-test-modules

19 files changed, 609 insertions, 533 deletions

diff --git a/check b/check
index dc4f96bc..9c9bb4cb 100755
--- a/check
+++ b/check
@@ -2,7 +2,7 @@
 #
 # Run test suite for morph.
 #
-# Copyright (C) 2011-2014  Codethink Limited
+# Copyright (C) 2011-2015  Codethink Limited
 # 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -102,9 +102,7 @@ then
 
     echo 'Checking source code for silliness'
     if ! (git ls-files --cached |
-          grep -v '\.gz$' |
-          grep -Ev 'tests[^/]*/.*\.std(out|err)' |
-          grep -vF 'tests.build/build-system-autotools.script' |
+          grep '\.py$' |
           xargs -r scripts/check-silliness); then
         exit 1
     fi
diff --git a/distbuild/worker_build_scheduler.py b/distbuild/worker_build_scheduler.py
index be732153..be63c642 100644
--- a/distbuild/worker_build_scheduler.py
+++ b/distbuild/worker_build_scheduler.py
@@ -544,25 +544,15 @@ class WorkerConnection(distbuild.StateMachine):
 
         logging.debug('Requesting shared artifact cache to get artifacts')
 
-        kind = self._job.artifact.source.morphology['kind']
-
-        if kind == 'chunk':
-            source_artifacts = self._job.artifact.source.artifacts
-
-            suffixes = ['%s.%s' % (kind, name) for name in source_artifacts]
-            suffixes.append('build-log')
-        else:
-            filename = '%s.%s' % (kind, self._job.artifact.name)
-            suffixes = [filename]
-
-            if kind == 'stratum':
-                suffixes.append(filename + '.meta')
-        
-        suffixes = [urllib.quote(x) for x in suffixes]
+        files = self._job.artifact.source.files()
+        suffixes = []
+        for basename in files:
+            suffix = basename.lstrip(self._job.artifact.source.cache_key + '.')
+            suffixes.append(urllib.quote(suffix))
         suffixes = ','.join(suffixes)
 
         worker_host = self._conn.getpeername()[0]
-        
+
         url = urlparse.urljoin(
             self._writeable_cache_server, 
             '/1.0/fetch?host=%s:%d&cacheid=%s&artifacts=%s' %
diff --git a/morphlib/app.py b/morphlib/app.py
index 0c87f814..b8bae850 100644
--- a/morphlib/app.py
+++ b/morphlib/app.py
@@ -297,26 +297,6 @@ class Morph(cliapp.Application):
                    morphlib.util.sanitise_morphology_path(args[2]))
             args = args[3:]
 
-    def cache_repo_and_submodules(self, cache, url, ref, done):
-        subs_to_process = set()
-        subs_to_process.add((url, ref))
-        while subs_to_process:
-            url, ref = subs_to_process.pop()
-            done.add((url, ref))
-            cached_repo = cache.cache_repo(url)
-            cached_repo.update()
-
-            try:
-                submodules = morphlib.git.Submodules(self, cached_repo.path,
-                                                     ref)
-                submodules.load()
-            except morphlib.git.NoModulesFileError:
-                pass
-            else:
-                for submod in submodules:
-                    if (submod.url, submod.commit) not in done:
-                        subs_to_process.add((submod.url, submod.commit))
-
     def _write_status(self, text):
         timestamp = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime())
         self.output.write('%s %s\n' % (timestamp, text))
diff --git a/morphlib/artifact.py b/morphlib/artifact.py
index 7a40a81a..4a385281 100644
--- a/morphlib/artifact.py
+++ b/morphlib/artifact.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012, 2013, 2014  Codethink Limited
+# Copyright (C) 2012-2015  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -43,7 +43,6 @@ class Artifact(object):
     def __repr__(self): # pragma: no cover
         return 'Artifact(%s)' % str(self)
 
-
     def walk(self): # pragma: no cover
         '''Return list of an artifact and its build dependencies.
         
@@ -63,3 +62,22 @@ class Artifact(object):
                 yield a
 
         return list(depth_first(self))
+
+
+def find_all_deps(artifacts):
+    '''Return all sources that are dependencies of a set of artifacts.
+
+    Deps are returned in order of 'nearest' to 'furthest' from the root
+    artifacts.
+
+    '''
+
+    deps = set()
+    ordered_deps = []
+
+    for artifact in artifacts:
+        for dep in artifact.walk():
+            if dep.source not in deps and dep not in artifacts:
+                deps.add(dep.source)
+                ordered_deps.append(dep.source)
+    return ordered_deps
diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py
index 0aa50a3b..55716331 100644
--- a/morphlib/buildcommand.py
+++ b/morphlib/buildcommand.py
@@ -16,7 +16,6 @@
 
 import itertools
 import os
-import shutil
 import logging
 import tempfile
 import datetime
@@ -288,6 +287,18 @@ class BuildCommand(object):
 
         self.app.status_prefix = old_prefix
 
+    def maybe_cache_artifacts_locally(self, source):
+        try:
+            self.rac.get_artifacts_for_source(
+                source, self.lac, status_cb=self.app.status)
+        except morphlib.remoteartifactcache.NotCachedError as e:
+            self.app.status(msg='Not found in remote cache.')
+        except morphlib.remoteartifactcache.GetError as e:
+            # It's important to not hide the error, as the problem may be
+            # something unexpected like a loose network cable.
+            self.app.status(
+                msg='Unable to fetch artifact from cache: %(error)s', error=e)
+
     def cache_or_build_source(self, source, build_env):
         '''Make artifacts of the built source available in the local cache.
 
@@ -295,18 +306,19 @@ class BuildCommand(object):
         that doesn't work for some reason, by building the source locally.
 
         '''
-        artifacts = source.artifacts.values()
-        if self.rac is not None:
-            try:
-                self.cache_artifacts_locally(artifacts)
-            except morphlib.remoteartifactcache.GetError:
-                # Error is logged by the RemoteArtifactCache object.
-                pass
+        def artifacts_available_locally(source):
+            # If any of the artifacts are missing, we consider the whole thing
+            # missing.
+            artifacts = source.artifacts.values()
+            return all(self.lac.has(artifact) for artifact in artifacts)
 
-        if any(not self.lac.has(artifact) for artifact in artifacts):
+        if not artifacts_available_locally(source) and self.rac:
+            self.maybe_cache_artifacts_locally(source)
+
+        if not artifacts_available_locally(source):
             self.build_source(source, build_env)
 
-        for a in artifacts:
+        for a in source.artifacts.values():
             self.app.status(msg='%(kind)s %(name)s is cached at %(cachepath)s',
                             kind=source.morphology['kind'], name=a.name,
                             cachepath=self.lac.artifact_filename(a),
@@ -325,12 +337,8 @@ class BuildCommand(object):
                         kind=source.morphology['kind'])
 
         self.fetch_sources(source)
-        # TODO: Make an artifact.walk() that takes multiple root artifacts.
-        # as this does a walk for every artifact. This was the status
-        # quo before build logic was made to work per-source, but we can
-        # now do better.
-        deps = self.get_recursive_deps(source.artifacts.values())
-        self.cache_artifacts_locally(deps)
+
+        deps = morphlib.artifact.find_all_deps(source.artifacts.values())
 
         use_chroot = False
         setup_mounts = False
@@ -338,8 +346,8 @@ class BuildCommand(object):
             build_mode = source.build_mode
             extra_env = {'PREFIX': source.prefix}
 
-            dep_prefix_set = set(a.source.prefix for a in deps
-                                 if a.source.morphology['kind'] == 'chunk')
+            dep_prefix_set = set(source.prefix for source in deps
+                                 if source.morphology['kind'] == 'chunk')
             extra_path = [os.path.join(d, 'bin') for d in dep_prefix_set]
 
             if build_mode not in ['bootstrap', 'staging', 'test']:
@@ -373,97 +381,12 @@ class BuildCommand(object):
         td_string = "%02d:%02d:%02d" % (hours, minutes, seconds)
         self.app.status(msg="Elapsed time %(duration)s", duration=td_string)
 
-    def get_recursive_deps(self, artifacts):
-        deps = set()
-        ordered_deps = []
-        for artifact in artifacts:
-            for dep in artifact.walk():
-                if dep not in deps and dep not in artifacts:
-                    deps.add(dep)
-                    ordered_deps.append(dep)
-        return ordered_deps
-
     def fetch_sources(self, source):
         '''Update the local git repository cache with the sources.'''
 
         repo_name = source.repo_name
-        if self.app.settings['no-git-update']:
-            self.app.status(msg='Not updating existing git repository '
-                                '%(repo_name)s '
-                                'because of no-git-update being set',
-                            chatty=True,
-                            repo_name=repo_name)
-            source.repo = self.lrc.get_repo(repo_name)
-            return
-
-        if self.lrc.has_repo(repo_name):
-            source.repo = self.lrc.get_repo(repo_name)
-            try:
-                sha1 = source.sha1
-                source.repo.resolve_ref_to_commit(sha1)
-                self.app.status(msg='Not updating git repository '
-                                    '%(repo_name)s because it '
-                                    'already contains sha1 %(sha1)s',
-                                chatty=True, repo_name=repo_name,
-                                sha1=sha1)
-            except morphlib.gitdir.InvalidRefError:
-                self.app.status(msg='Updating %(repo_name)s',
-                                repo_name=repo_name)
-                source.repo.update()
-        else:
-            self.app.status(msg='Cloning %(repo_name)s',
-                            repo_name=repo_name)
-            source.repo = self.lrc.cache_repo(repo_name)
-
-        # Update submodules.
-        done = set()
-        self.app.cache_repo_and_submodules(
-            self.lrc, source.repo.url,
-            source.sha1, done)
-
-    def cache_artifacts_locally(self, artifacts):
-        '''Get artifacts missing from local cache from remote cache.'''
-
-        def fetch_files(to_fetch):
-            '''Fetch a set of files atomically.
-
-            If an error occurs during the transfer of any files, all downloaded
-            data is deleted, to ensure integrity of the local cache.
-
-            '''
-            try:
-                for remote, local in to_fetch:
-                    shutil.copyfileobj(remote, local)
-            except BaseException:
-                for remote, local in to_fetch:
-                    local.abort()
-                raise
-            else:
-                for remote, local in to_fetch:
-                    remote.close()
-                    local.close()
-
-        for artifact in artifacts:
-            # This block should fetch all artifact files in one go, using the
-            # 1.0/artifacts method of morph-cache-server. The code to do that
-            # needs bringing in from the distbuild.worker_build_connection
-            # module into morphlib.remoteartififactcache first.
-            to_fetch = []
-            if not self.lac.has(artifact):
-                to_fetch.append((self.rac.get(artifact),
-                                 self.lac.put(artifact)))
-
-            if artifact.source.morphology.needs_artifact_metadata_cached:
-                if not self.lac.has_artifact_metadata(artifact, 'meta'):
-                    to_fetch.append((
-                        self.rac.get_artifact_metadata(artifact, 'meta'),
-                        self.lac.put_artifact_metadata(artifact, 'meta')))
-
-            if len(to_fetch) > 0:
-                self.app.status(
-                    msg='Fetching to local cache: artifact %(name)s',
-                    name=artifact.name)
-                fetch_files(to_fetch)
+        source.repo = self.lrc.get_updated_repo(repo_name, ref=source.sha1)
+        self.lrc.ensure_submodules(source.repo, source.sha1)
 
     def create_staging_area(self, build_env, use_chroot=True, extra_env={},
                             extra_path=[]):
@@ -503,7 +426,7 @@ class BuildCommand(object):
             return set(s.morphology for s in dependent_strata)
         return dependent_stratum_morphs(s1) == dependent_stratum_morphs(s2)
 
-    def install_dependencies(self, staging_area, artifacts, target_source):
+    def install_dependencies(self, staging_area, sources, target_source):
         '''Install chunk artifacts into staging area.
 
         We only ever care about chunk artifacts as build dependencies,
@@ -514,19 +437,20 @@ class BuildCommand(object):
 
         '''
 
-        for artifact in artifacts:
-            if artifact.source.morphology['kind'] != 'chunk':
+        for source in sources:
+            if source.morphology['kind'] != 'chunk':
                 continue
-            if artifact.source.build_mode == 'bootstrap':
-               if not self.in_same_stratum(artifact.source, target_source):
+            if source.build_mode == 'bootstrap':
+               if not self.in_same_stratum(source, target_source):
                     continue
             self.app.status(
                 msg='Installing chunk %(chunk_name)s from cache %(cache)s',
-                chunk_name=artifact.name,
-                cache=artifact.source.cache_key[:7],
+                chunk_name=source.name,
+                cache=source.cache_key[:7],
                 chatty=True)
-            handle = self.lac.get(artifact)
-            staging_area.install_artifact(handle)
+            for artifact in source.artifacts.values():
+                handle = self.lac.get(artifact)
+                staging_area.install_artifact(handle)
 
         if target_source.build_mode == 'staging':
             morphlib.builder.ldconfig(self.app.runcmd, staging_area.dirname)
diff --git a/morphlib/builder.py b/morphlib/builder.py
index 0bb21434..00955abe 100644
--- a/morphlib/builder.py
+++ b/morphlib/builder.py
@@ -123,25 +123,6 @@ def ldconfig(runcmd, rootdir):  # pragma: no cover
         logging.debug('No %s, not running ldconfig' % conf)
 
 
-def download_depends(constituents, lac, rac, metadatas=None):
-    for constituent in constituents:
-        if not lac.has(constituent):
-            source = rac.get(constituent)
-            target = lac.put(constituent)
-            shutil.copyfileobj(source, target)
-            target.close()
-            source.close()
-        if metadatas is not None:
-            for metadata in metadatas:
-                if not lac.has_artifact_metadata(constituent, metadata):
-                    if rac.has_artifact_metadata(constituent, metadata):
-                        src = rac.get_artifact_metadata(constituent, metadata)
-                        dst = lac.put_artifact_metadata(constituent, metadata)
-                        shutil.copyfileobj(src, dst)
-                        dst.close()
-                        src.close()
-
-
 class BuilderBase(object):
 
     '''Base class for building artifacts.'''
@@ -492,15 +473,17 @@ class StratumBuilder(BuilderBase):
         with self.build_watch('overall-build'):
             constituents = [d for d in self.source.dependencies
                             if self.is_constituent(d)]
+            constituent_sources = {d.source for d in constituents}
 
             # the only reason the StratumBuilder has to download chunks is to
-            # check for overlap now that strata are lists of chunks
+            # check for overlap now that strata are lists of chunks. If there
+            # is no remote artifact cache, we assume they're all cached
+            # locally.
             with self.build_watch('check-chunks'):
-                for a_name, a in self.source.artifacts.iteritems():
-                    # download the chunk artifact if necessary
-                    download_depends(constituents,
-                                     self.local_artifact_cache,
-                                     self.remote_artifact_cache)
+                if self.remote_artifact_cache:
+                    self.remote_artifact_cache.get_artifacts_for_sources(
+                        constituent_sources, self.local_artifact_cache,
+                        status_cb=self.app.status)
 
             with self.build_watch('create-chunk-list'):
                 lac = self.local_artifact_cache
@@ -588,25 +571,18 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
         self.app.status(msg='Unpacking strata to %(path)s',
                         path=path, chatty=True)
         with self.build_watch('unpack-strata'):
-            for a_name, a in self.source.artifacts.iteritems():
-                # download the stratum artifacts if necessary
-                download_depends(self.source.dependencies,
-                                 self.local_artifact_cache,
-                                 self.remote_artifact_cache,
-                                 ('meta',))
-
-                # download the chunk artifacts if necessary
-                for stratum_artifact in self.source.dependencies:
-                    f = self.local_artifact_cache.get(stratum_artifact)
-                    chunks = [ArtifactCacheReference(c) for c in json.load(f)]
-                    download_depends(chunks,
-                                     self.local_artifact_cache,
-                                     self.remote_artifact_cache)
-                    f.close()
-
-                # unpack it from the local artifact cache
-                for stratum_artifact in self.source.dependencies:
-                    self.unpack_one_stratum(stratum_artifact, path)
+            dep_sources = morphlib.artifact.find_all_deps(
+                self.source.dependencies)
+
+            # download the stratum and chunk artifacts if necessary
+            if self.remote_artifact_cache:
+                self.remote_artifact_cache.get_artifacts_for_sources(
+                    dep_sources, self.local_artifact_cache,
+                    status_cb=self.app.status)
+
+            # unpack it from the local artifact cache
+            for stratum_artifact in self.source.dependencies:
+                self.unpack_one_stratum(stratum_artifact, path)
 
             ldconfig(self.app.runcmd, path)
 
diff --git a/morphlib/builder_tests.py b/morphlib/builder_tests.py
index 0cc90819..0067022f 100644
--- a/morphlib/builder_tests.py
+++ b/morphlib/builder_tests.py
@@ -15,12 +15,12 @@
 
 
 import json
-import os
 import StringIO
 import unittest
 
 import morphlib
 import morphlib.gitdir_tests
+import morphlib.testutils
 
 
 class FakeBuildSystem(object):
@@ -78,64 +78,6 @@ class FakeBuildEnv(object):
         }
 
 
-class FakeFileHandle(object):
-
-    def __init__(self, cache, key):
-        self._string = ""
-        self._cache = cache
-        self._key = key
-
-    def __enter__(self):
-        return self
-
-    def _writeback(self):
-        self._cache._cached[self._key] = self._string
-
-    def __exit__(self, type, value, traceback):
-        self._writeback()
-
-    def close(self):
-        self._writeback()
-
-    def write(self, string):
-        self._string += string
-
-
-class FakeArtifactCache(object):
-
-    def __init__(self):
-        self._cached = {}
-
-    def put(self, artifact):
-        return FakeFileHandle(self, (artifact.cache_key, artifact.name))
-
-    def put_artifact_metadata(self, artifact, name):
-        return FakeFileHandle(self, (artifact.cache_key, artifact.name, name))
-
-    def put_source_metadata(self, source, cachekey, name):
-        return FakeFileHandle(self, (cachekey, name))
-
-    def get(self, artifact):
-        return StringIO.StringIO(
-            self._cached[(artifact.cache_key, artifact.name)])
-
-    def get_artifact_metadata(self, artifact, name):
-        return StringIO.StringIO(
-            self._cached[(artifact.cache_key, artifact.name, name)])
-
-    def get_source_metadata(self, source, cachekey, name):
-        return StringIO.StringIO(self._cached[(cachekey, name)])
-
-    def has(self, artifact):
-        return (artifact.cache_key, artifact.name) in self._cached
-
-    def has_artifact_metadata(self, artifact, name):
-        return (artifact.cache_key, artifact.name, name) in self._cached
-
-    def has_source_metadata(self, source, cachekey, name):
-        return (cachekey, name) in self._cached
-
-
 class BuilderBaseTests(unittest.TestCase):
 
     def fake_runcmd(self, argv, *args, **kwargs):
@@ -151,7 +93,7 @@ class BuilderBaseTests(unittest.TestCase):
         self.commands_run = []
         self.app = FakeApp(self.fake_runcmd)
         self.staging_area = FakeStagingArea(self.fake_runcmd, FakeBuildEnv())
-        self.artifact_cache = FakeArtifactCache()
+        self.artifact_cache = morphlib.testutils.FakeLocalArtifactCache()
         self.artifact = FakeArtifact('le-artifact')
         self.repo_cache = None
         self.build_env = FakeBuildEnv()
@@ -187,33 +129,6 @@ class BuilderBaseTests(unittest.TestCase):
         self.assertEqual(sorted(events),
                          sorted(meta['build-times'].keys()))
 
-    def test_downloads_depends(self):
-        lac = FakeArtifactCache()
-        rac = FakeArtifactCache()
-        afacts = [FakeArtifact(name) for name in ('a', 'b', 'c')]
-        for a in afacts:
-            fh = rac.put(a)
-            fh.write(a.name)
-            fh.close()
-        morphlib.builder.download_depends(afacts, lac, rac)
-        self.assertTrue(all(lac.has(a) for a in afacts))
-
-    def test_downloads_depends_metadata(self):
-        lac = FakeArtifactCache()
-        rac = FakeArtifactCache()
-        afacts = [FakeArtifact(name) for name in ('a', 'b', 'c')]
-        for a in afacts:
-            fh = rac.put(a)
-            fh.write(a.name)
-            fh.close()
-            fh = rac.put_artifact_metadata(a, 'meta')
-            fh.write('metadata')
-            fh.close()
-        morphlib.builder.download_depends(afacts, lac, rac, ('meta',))
-        self.assertTrue(all(lac.has(a) for a in afacts))
-        self.assertTrue(all(lac.has_artifact_metadata(a, 'meta')
-                            for a in afacts))
-
 
 class ChunkBuilderTests(unittest.TestCase):
 
diff --git a/morphlib/localartifactcache.py b/morphlib/localartifactcache.py
index 955ee97f..c7eccf4c 100644
--- a/morphlib/localartifactcache.py
+++ b/morphlib/localartifactcache.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012, 2013, 2014  Codethink Limited
+# Copyright (C) 2012-2015  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -49,34 +49,33 @@ class LocalArtifactCache(object):
         self.cachefs = cachefs
 
     def put(self, artifact):
-        filename = self.artifact_filename(artifact)
-        return morphlib.savefile.SaveFile(filename, mode='w')
+        return self.put_file(artifact.basename())
 
     def put_artifact_metadata(self, artifact, name):
-        filename = self._artifact_metadata_filename(artifact, name)
-        return morphlib.savefile.SaveFile(filename, mode='w')
+        return self.put_file(artifact.metadata_basename(name))
 
     def put_source_metadata(self, source, cachekey, name):
-        filename = self._source_metadata_filename(source, cachekey, name)
+        return self.put_file('%s.%s' % (cachekey, name))
+
+    def put_file(self, basename):
+        filename = self._join(basename)
         return morphlib.savefile.SaveFile(filename, mode='w')
 
-    def _has_file(self, filename):
+    def has_file(self, basename):
+        filename = self._join(basename)
         if os.path.exists(filename):
             os.utime(filename, None)
             return True
         return False
 
     def has(self, artifact):
-        filename = self.artifact_filename(artifact)
-        return self._has_file(filename)
+        return self.has_file(artifact.basename())
 
     def has_artifact_metadata(self, artifact, name):
-        filename = self._artifact_metadata_filename(artifact, name)
-        return self._has_file(filename)
+        return self.has_file(artifact.metadata_basename(name))
 
     def has_source_metadata(self, source, cachekey, name):
-        filename = self._source_metadata_filename(source, cachekey, name)
-        return self._has_file(filename)
+        return self.has_file('%s.%s' % (cachekey, name))
 
     def get(self, artifact):
         filename = self.artifact_filename(artifact)
diff --git a/morphlib/localrepocache.py b/morphlib/localrepocache.py
index 9bccb20b..ac59f5c8 100644
--- a/morphlib/localrepocache.py
+++ b/morphlib/localrepocache.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2014  Codethink Limited
+# Copyright (C) 2012-2015  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -14,10 +14,7 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 
-import logging
 import os
-import re
-import urllib2
 import urlparse
 import string
 import sys
@@ -244,15 +241,68 @@ class LocalRepoCache(object):
                 return repo
         raise NotCached(reponame)
 
-    def get_updated_repo(self, reponame): # pragma: no cover
-        '''Return object representing cached repository, which is updated.'''
+    def get_updated_repo(self, repo_name, ref=None):  # pragma: no cover
+        '''Return object representing cached repository.
 
-        if not self._app.settings['no-git-update']:
-            cached_repo = self.cache_repo(reponame)
-            self._app.status(
-                msg='Updating git repository %s in cache' % reponame)
-            cached_repo.update()
-        else:
-            cached_repo = self.get_repo(reponame)
-        return cached_repo
+        If 'ref' is None, the repo will be updated unless
+        app.settings['no-git-update'] is set.
+
+        If 'ref' is set to a SHA1, the repo will only be updated if 'ref' isn't
+        already available locally.
 
+        '''
+
+        if self._app.settings['no-git-update']:
+            self._app.status(msg='Not updating existing git repository '
+                                 '%(repo_name)s '
+                                 'because of no-git-update being set',
+                             chatty=True,
+                             repo_name=repo_name)
+            return self.get_repo(repo_name)
+
+        if self.has_repo(repo_name):
+            repo = self.get_repo(repo_name)
+            if ref and morphlib.git.is_valid_sha1(ref):
+                try:
+                    repo.resolve_ref_to_commit(ref)
+                    self._app.status(msg='Not updating git repository '
+                                         '%(repo_name)s because it '
+                                         'already contains sha1 %(sha1)s',
+                                     chatty=True, repo_name=repo_name,
+                                     sha1=ref)
+                    return repo
+                except morphlib.gitdir.InvalidRefError:
+                    pass
+
+            self._app.status(msg='Updating %(repo_name)s',
+                             repo_name=repo_name)
+            repo.update()
+            return repo
+        else:
+            self._app.status(msg='Cloning %(repo_name)s',
+                             repo_name=repo_name)
+            return self.cache_repo(repo_name)
+
+    def ensure_submodules(self, toplevel_repo,
+                          toplevel_ref):  # pragma: no cover
+        '''Ensure any submodules of a given repo are cached and up to date.'''
+
+        def submodules_for_repo(repo_path, ref):
+            try:
+                submodules = morphlib.git.Submodules(self._app, repo_path, ref)
+                submodules.load()
+                return [(submod.url, submod.commit) for submod in submodules]
+            except morphlib.git.NoModulesFileError:
+                return []
+
+        done = set()
+        subs_to_process = submodules_for_repo(toplevel_repo.path, toplevel_ref)
+        while subs_to_process:
+            url, ref = subs_to_process.pop()
+            done.add((url, ref))
+
+            cached_repo = self.get_updated_repo(url, ref=ref)
+
+            for submod in submodules_for_repo(cached_repo.path, ref):
+                if (submod.url, submod.commit) not in done:
+                    subs_to_process.add((submod.url, submod.commit))
diff --git a/morphlib/localrepocache_tests.py b/morphlib/localrepocache_tests.py
index ab6e71fd..961e06b4 100644
--- a/morphlib/localrepocache_tests.py
+++ b/morphlib/localrepocache_tests.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2014  Codethink Limited
+# Copyright (C) 2012-2015  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -15,7 +15,6 @@
 
 
 import unittest
-import urllib2
 import os
 
 import cliapp
diff --git a/morphlib/remoteartifactcache.py b/morphlib/remoteartifactcache.py
index 4e09ce34..b7aa7466 100644
--- a/morphlib/remoteartifactcache.py
+++ b/morphlib/remoteartifactcache.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2014  Codethink Limited
+# Copyright (C) 2012-2015  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -16,96 +16,54 @@
 
 import cliapp
 import logging
+import socket
 import urllib
-import urllib2
 import urlparse
 
+import requests
 
-class HeadRequest(urllib2.Request):  # pragma: no cover
 
-    def get_method(self):
-        return 'HEAD'
+TIMEOUT_SECONDS = 10
 
 
 class GetError(cliapp.AppException):
-
-    def __init__(self, cache, artifact):
-        cliapp.AppException.__init__(
-            self, 'Failed to get the artifact %s '
-                  'from the artifact cache %s' %
-                  (artifact.basename(), cache))
-
-
-class GetArtifactMetadataError(GetError):
-
-    def __init__(self, cache, artifact, name):
+    def __init__(self, cache, filename, exception):
+        self.exception = exception
         cliapp.AppException.__init__(
-            self, 'Failed to get metadata %s for the artifact %s '
-                  'from the artifact cache %s' %
-                  (name, artifact.basename(), cache))
+            self, 'Failed to get the file %s from the artifact cache %s: %s' %
+                  (filename, cache, exception))
 
 
-class GetSourceMetadataError(GetError):
-
-    def __init__(self, cache, source, cache_key, name):
-        cliapp.AppException.__init__(
-            self, 'Failed to get metadata %s for source %s '
-                  'and cache key %s from the artifact cache %s' %
-                  (name, source, cache_key, cache))
+class NotCachedError(GetError):
+    pass
 
 
 class RemoteArtifactCache(object):
 
     def __init__(self, server_url):
+        adapter = requests.adapters.HTTPAdapter(max_retries=3)
+        self.requests = requests.Session()
+        self.requests.mount('http://', adapter)
+
         self.server_url = server_url
 
     def has(self, artifact):
         return self._has_file(artifact.basename())
 
-    def has_artifact_metadata(self, artifact, name):
-        return self._has_file(artifact.metadata_basename(name))
-
-    def has_source_metadata(self, source, cachekey, name):
-        filename = '%s.%s' % (cachekey, name)
-        return self._has_file(filename)
-
-    def get(self, artifact, log=logging.error):
-        try:
-            return self._get_file(artifact.basename())
-        except urllib2.URLError, e:
-            log(str(e))
-            raise GetError(self, artifact)
-
-    def get_artifact_metadata(self, artifact, name, log=logging.error):
-        try:
-            return self._get_file(artifact.metadata_basename(name))
-        except urllib2.URLError, e:
-            log(str(e))
-            raise GetArtifactMetadataError(self, artifact, name)
-
-    def get_source_metadata(self, source, cachekey, name):
-        filename = '%s.%s' % (cachekey, name)
-        try:
-            return self._get_file(filename)
-        except urllib2.URLError:
-            raise GetSourceMetadataError(self, source, cachekey, name)
-
-    def _has_file(self, filename):  # pragma: no cover
+    def _has_file(self, filename):
         url = self._request_url(filename)
         logging.debug('RemoteArtifactCache._has_file: url=%s' % url)
-        request = HeadRequest(url)
-        try:
-            urllib2.urlopen(request)
-            return True
-        except (urllib2.HTTPError, urllib2.URLError):
-            return False
 
-    def _get_file(self, filename):  # pragma: no cover
-        url = self._request_url(filename)
-        logging.debug('RemoteArtifactCache._get_file: url=%s' % url)
-        return urllib2.urlopen(url)
+        response = self.requests.head(url, timeout=TIMEOUT_SECONDS)
 
-    def _request_url(self, filename):  # pragma: no cover
+        if response.status_code == 404:
+            return False
+        elif response.ok:
+            return True
+        else:
+            response.raise_for_status()
+
+    def _request_url(self, filename):
         server_url = self.server_url
         if not server_url.endswith('/'):
             server_url += '/'
@@ -115,3 +73,107 @@ class RemoteArtifactCache(object):
 
     def __str__(self):  # pragma: no cover
         return self.server_url
+
+    def _fetch_file(self, remote_filename, local_file, status_cb=None,
+                    error_if_missing=True):
+        chunk_size = 10 * 1024 * 1024
+
+        def show_status(downloaded, total):
+            if not status_cb:
+                return
+            downloaded = min(downloaded, total)
+            if total == 0:
+                status_cb(msg='%(file)s: Fetched %(d).02fMB',
+                          file=remote_filename,
+                          d=max(downloaded / (1024 * 1024), 0.01),
+                          chatty=True)
+            else:
+                status_cb(msg='%(file)s: Fetched %(d).02fMB of %(t).02fMB',
+                          file=remote_filename,
+                          d=max(downloaded / (1024 * 1024), 0.01),
+                          t=max(total / (1024 * 1024), 0.01),
+                          chatty=True)
+
+        remote_url = self._request_url(remote_filename)
+        logging.debug('RemoteArtifactCache._fetch_file: url=%s' % remote_url)
+
+        try:
+            response = self.requests.get(remote_url, stream=True,
+                                         timeout=TIMEOUT_SECONDS)
+            response.raise_for_status()
+            content_length = int(response.headers.get('content-length', 0))
+            for i, chunk in enumerate(response.iter_content(chunk_size)):
+                local_file.write(chunk)
+                show_status((i+1) * chunk_size, content_length)
+        except requests.exceptions.HTTPError as e:
+            logging.debug(str(e))
+            if e.response.status_code == 404:
+                if error_if_missing:
+                    raise NotCachedError(self, remote_filename, e)
+            else:
+                raise GetError(self, remote_filename, e)
+        except (IOError, requests.exceptions.RequestException) as e:
+            logging.debug(str(e))
+            raise GetError(self, remote_filename, e)
+
+    def _fetch_files(self, to_fetch, status_cb):
+        '''Fetch a set of files atomically.
+
+        If an error occurs during the transfer of any files, all downloaded
+        data is deleted, to reduce the chances of having artifacts in the local
+        cache that are missing their metadata, and so on.
+
+        This assumes that the morphlib.savefile module is used so the file
+        handles passed in to_fetch have a .abort() method.
+
+        '''
+        def is_required(basename):
+            # This is a workaround for a historical bugs. Distbuild used to
+            # fail to transfer the source .meta and the .build-log file, so
+            # we need to cope if they are missing.
+            if basename.endswith('.build-log') or basename.endswith('.meta'):
+                return False
+            return True
+
+        try:
+            for remote_filename, local_file in to_fetch:
+                self._fetch_file(
+                    remote_filename, local_file, status_cb=status_cb,
+                    error_if_missing=is_required(remote_filename))
+        except BaseException:
+            for _, local_file in to_fetch:
+                local_file.abort()
+            raise
+        else:
+            for _, local_file in to_fetch:
+                local_file.close()
+
+    def get_artifacts_for_source(self, source, lac, status_cb=None):
+        '''Ensure all built artifacts for 'source' are in the local cache.
+
+        This includes all build metadata such as the .build-log file.
+
+        '''
+
+        to_fetch = []
+
+        for basename in source.files():
+            if not lac.has_file(basename):
+                to_fetch.append((basename, lac.put_file(basename)))
+
+        if len(to_fetch) > 0:
+            if status_cb:
+                status_cb(
+                    msg='Fetching built artifacts of %(name)s',
+                    name=source.name)
+
+            self._fetch_files(to_fetch, status_cb=status_cb)
+
+    def get_artifacts_for_sources(self, sources, lac, status_cb=None):
+        '''Ensure artifacts for multiple sources are available locally.'''
+
+        # Running the downloads in parallel might give a speed boost, as many
+        # of these are small files.
+
+        for source in sources:
+            self.get_artifacts_for_source(source, lac, status_cb=status_cb)
diff --git a/morphlib/remoteartifactcache_tests.py b/morphlib/remoteartifactcache_tests.py
index 788882c2..820958da 100644
--- a/morphlib/remoteartifactcache_tests.py
+++ b/morphlib/remoteartifactcache_tests.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2014  Codethink Limited
+# Copyright (C) 2012-2015  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -14,151 +14,164 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 
-import StringIO
+import BaseHTTPServer
+import contextlib
+import logging
+import socket
+import SocketServer
+import threading
 import unittest
-import urllib2
+import urlparse
 
 import morphlib
+import morphlib.testutils
 
 
-class RemoteArtifactCacheTests(unittest.TestCase):
+class FragileTCPServer(SocketServer.TCPServer):
+    '''A TCP server that will optionally refuse some connections.
+
+    This is used for simulating temporary network glitches.
+    '''
+
+    def __init__(self, *args):
+        SocketServer.TCPServer.__init__(self, *args)
+        self._refuse_next_connection = False
 
-    def setUp(self):
-        loader = morphlib.morphloader.MorphologyLoader()
-        morph = loader.load_from_string(
-            '''
-                name: chunk
-                kind: chunk
-                products:
-                    - artifact: chunk-runtime
-                      include:
-                          - usr/bin
-                          - usr/sbin
-                          - usr/lib
-                          - usr/libexec
-                    - artifact: chunk-devel
-                      include:
-                          - usr/include
-                    - artifact: chunk-doc
-                      include:
-                          - usr/share/doc
-            ''')
-        sources = morphlib.source.make_sources('repo', 'original/ref',
-                                               'chunk.morph', 'sha1',
-                                               'tree', morph)
-        self.source, = sources
-        self.runtime_artifact = morphlib.artifact.Artifact(
-            self.source, 'chunk-runtime')
-        self.runtime_artifact.cache_key = 'CHUNK'
-        self.devel_artifact = morphlib.artifact.Artifact(
-            self.source, 'chunk-devel')
-        self.devel_artifact.cache_key = 'CHUNK'
-        self.doc_artifact = morphlib.artifact.Artifact(
-            self.source, 'chunk-doc')
-        self.doc_artifact.cache_key = 'CHUNK'
-
-        self.existing_files = set([
-            self.runtime_artifact.basename(),
-            self.devel_artifact.basename(),
-            self.runtime_artifact.metadata_basename('meta'),
-            '%s.%s' % (self.runtime_artifact.cache_key, 'meta'),
-        ])
-
-        self.server_url = 'http://foo.bar:8080'
-        self.cache = morphlib.remoteartifactcache.RemoteArtifactCache(
-            self.server_url)
-        self.cache._has_file = self._has_file
-        self.cache._get_file = self._get_file
-
-    def _has_file(self, filename):
-        return filename in self.existing_files
-
-    def _get_file(self, filename):
-        if filename in self.existing_files:
-            return StringIO.StringIO('%s' % filename)
+    def verify_request(self, request, client_address):
+        if self._refuse_next_connection:
+            self._refuse_next_connection = False
+            return False
         else:
-            raise urllib2.URLError('foo')
-
-    def test_sets_server_url(self):
-        self.assertEqual(self.cache.server_url, self.server_url)
-
-    def test_has_existing_artifact(self):
-        self.assertTrue(self.cache.has(self.runtime_artifact))
-
-    def test_has_a_different_existing_artifact(self):
-        self.assertTrue(self.cache.has(self.devel_artifact))
-
-    def test_does_not_have_a_non_existent_artifact(self):
-        self.assertFalse(self.cache.has(self.doc_artifact))
-
-    def test_has_existing_artifact_metadata(self):
-        self.assertTrue(self.cache.has_artifact_metadata(
-            self.runtime_artifact, 'meta'))
-
-    def test_does_not_have_non_existent_artifact_metadata(self):
-        self.assertFalse(self.cache.has_artifact_metadata(
-            self.runtime_artifact, 'non-existent-meta'))
-
-    def test_has_existing_source_metadata(self):
-        self.assertTrue(self.cache.has_source_metadata(
-            self.runtime_artifact.source,
-            self.runtime_artifact.cache_key,
-            'meta'))
-
-    def test_does_not_have_non_existent_source_metadata(self):
-        self.assertFalse(self.cache.has_source_metadata(
-            self.runtime_artifact.source,
-            self.runtime_artifact.cache_key,
-            'non-existent-meta'))
-
-    def test_get_existing_artifact(self):
-        handle = self.cache.get(self.runtime_artifact)
-        data = handle.read()
-        self.assertEqual(data, self.runtime_artifact.basename())
-
-    def test_get_a_different_existing_artifact(self):
-        handle = self.cache.get(self.devel_artifact)
-        data = handle.read()
-        self.assertEqual(data, self.devel_artifact.basename())
-
-    def test_fails_to_get_a_non_existent_artifact(self):
-        self.assertRaises(morphlib.remoteartifactcache.GetError,
-                          self.cache.get, self.doc_artifact,
-                          log=lambda *args: None)
-
-    def test_get_existing_artifact_metadata(self):
-        handle = self.cache.get_artifact_metadata(
-            self.runtime_artifact, 'meta')
-        data = handle.read()
-        self.assertEqual(
-            data, '%s.%s' % (self.runtime_artifact.basename(), 'meta'))
-
-    def test_fails_to_get_non_existent_artifact_metadata(self):
-        self.assertRaises(
-            morphlib.remoteartifactcache.GetArtifactMetadataError,
-            self.cache.get_artifact_metadata,
-            self.runtime_artifact,
-            'non-existent-meta',
-            log=lambda *args: None)
-
-    def test_get_existing_source_metadata(self):
-        handle = self.cache.get_source_metadata(
-            self.runtime_artifact.source,
-            self.runtime_artifact.cache_key,
-            'meta')
-        data = handle.read()
-        self.assertEqual(
-            data, '%s.%s' % (self.runtime_artifact.cache_key, 'meta'))
-
-    def test_fails_to_get_non_existent_source_metadata(self):
-        self.assertRaises(
-            morphlib.remoteartifactcache.GetSourceMetadataError,
-            self.cache.get_source_metadata,
-            self.runtime_artifact.source,
-            self.runtime_artifact.cache_key,
-            'non-existent-meta')
+            return True
+
+    def refuse_next_connection(self):
+        self._refuse_next_connection = True
+
+
+@contextlib.contextmanager
+def http_server(request_handler):
+    server = FragileTCPServer(('', 0), request_handler)
+    thread = threading.Thread(target=server.serve_forever)
+    thread.setDaemon(True)
+    thread.start()
+
+    url = 'http://%s:%s' % (socket.gethostname(), server.server_address[1])
+    yield url, server
+
+    server.shutdown()
+
+
+def artifact_cache_server(valid_filenames):
+    '''Run a basic artifact cache server.
+
+    It implements the 'artifacts' request only. Dummy content will be returned
+    for any file listed in 'valid_filenames'.  Anything else will get a 404
+    error.
+
+    '''
+
+    class Handler(BaseHTTPServer.BaseHTTPRequestHandler):
+        dummy_content = 'I am an artifact.\n'
+
+        def log_message(self, *args, **kwargs):
+            # This is overridden to avoid dumping logs on stdout.
+            pass
+
+        def artifacts(self, query, send_body=True):
+            '''Return a cached artifact, or a 404 (not found) error.'''
+            filename = urlparse.parse_qs(query)['filename'][0]
+
+            if filename in valid_filenames:
+                self.send_response(200)
+                self.end_headers()
+                if send_body:
+                    self.wfile.write(self.dummy_content)
+            else:
+                self.send_response(404)
+
+        def do_GET(self, send_body=True):
+            parts = urlparse.urlsplit(self.path)
+            if parts.path == '/1.0/artifacts':
+                self.artifacts(parts.query, send_body=send_body)
+            else:
+                self.send_response(404)
+
+        def do_HEAD(self):
+            return self.do_GET(send_body=False)
+
+    return http_server(Handler)
+
+
+class RemoteArtifactCacheTests(unittest.TestCase):
+    def fake_status_cb(*args, **kwargs):
+        pass
+
+    def assert_can_download_artifacts(self, rac, source):
+        '''Assert that downloading artifacts from the remote cache works.'''
+        lac = morphlib.testutils.FakeLocalArtifactCache()
+
+        for artifact in source.artifacts.values():
+            self.assertFalse(lac.has(artifact))
+
+        rac.get_artifacts_for_source(
+            source, lac, self.fake_status_cb)
+        for artifact in source.artifacts.values():
+            self.assertTrue(lac.has(artifact))
+
+    def test_artifacts_are_cached(self):
+        '''Exercise fetching of artifacts that are cached.'''
+        chunk = morphlib.testutils.example_chunk()
+        valid_files = chunk.files()
+
+        with artifact_cache_server(valid_files) as (url, server):
+            rac = morphlib.remoteartifactcache.RemoteArtifactCache(url)
+            an_artifact = chunk.artifacts.values()[0]
+
+            self.assertTrue(
+                rac.has(an_artifact))
+            self.assert_can_download_artifacts(rac, chunk)
+
+    def test_artifacts_are_not_cached(self):
+        '''Exercise trying to fetch artifacts that the cache doesn't have.'''
+
+        chunk = morphlib.testutils.example_chunk()
+        valid_files = []
+
+        with artifact_cache_server(valid_files) as (url, server):
+            rac = morphlib.remoteartifactcache.RemoteArtifactCache(url)
+            an_artifact = chunk.artifacts.values()[0]
+
+            self.assertFalse(
+                rac.has(an_artifact))
+
+            with self.assertRaises(morphlib.remoteartifactcache.GetError):
+                lac = morphlib.testutils.FakeLocalArtifactCache()
+                rac.get_artifacts_for_source(
+                    chunk, lac, self.fake_status_cb)
+
+    def test_retry_requests(self):
+        '''Prove that requests are retried on failure.'''
+
+        chunk = morphlib.testutils.example_chunk()
+        valid_files = chunk.files()
+
+        log = logging.getLogger('requests.packages.urllib3.connectionpool')
+        log.setLevel(logging.ERROR)
+
+        with artifact_cache_server(valid_files) as (url, server):
+            rac = morphlib.remoteartifactcache.RemoteArtifactCache(url)
+            an_artifact = chunk.artifacts.values()[0]
+
+            server.refuse_next_connection()
+            self.assertTrue(rac.has(an_artifact))
+
+            server.refuse_next_connection()
+            self.assert_can_download_artifacts(rac, chunk)
 
     def test_escapes_pluses_in_request_urls(self):
-        returned_url = self.cache._request_url('gtk+')
-        correct_url = '%s/1.0/artifacts?filename=gtk%%2B' % self.server_url
+        rac = morphlib.remoteartifactcache.RemoteArtifactCache(
+            'http://example.com')
+
+        returned_url = rac._request_url('gtk+')
+        correct_url = 'http://example.com/1.0/artifacts?filename=gtk%2B'
         self.assertEqual(returned_url, correct_url)
diff --git a/morphlib/remoterepocache.py b/morphlib/remoterepocache.py
index 004ba86e..71115f0d 100644
--- a/morphlib/remoterepocache.py
+++ b/morphlib/remoterepocache.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2014  Codethink Limited
+# Copyright (C) 2012-2015  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -17,10 +17,11 @@
 import cliapp
 import json
 import logging
-import urllib2
 import urlparse
 import urllib
 
+import requests
+
 
 class ResolveRefError(cliapp.AppException):
 
@@ -63,9 +64,9 @@ class RemoteRepoCache(object):
         repo_url = self._resolver.pull_url(repo_name)
         try:
             return self._cat_file_for_repo_url(repo_url, ref, filename)
-        except urllib2.HTTPError as e:
+        except requests.exceptions.HTTPError as e:
             logging.error('Caught exception: %s' % str(e))
-            if e.code == 404:
+            if e.response.status_code == 404:
                 raise CatFileError(repo_name, ref, filename)
             raise # pragma: no cover
 
@@ -102,5 +103,10 @@ class RemoteRepoCache(object):
         if not server_url.endswith('/'):
             server_url += '/'
         url = urlparse.urljoin(server_url, '/1.0/%s' % path)
-        handle = urllib2.urlopen(url)
-        return handle.read()
+
+        response = requests.get(url)
+
+        if response.ok:
+            return response.text
+        else:
+            response.raise_for_status()
diff --git a/morphlib/remoterepocache_tests.py b/morphlib/remoterepocache_tests.py
index ef81506f..d4f7d494 100644
--- a/morphlib/remoterepocache_tests.py
+++ b/morphlib/remoterepocache_tests.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2014  Codethink Limited
+# Copyright (C) 2012-2015  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -16,7 +16,8 @@
 
 import json
 import unittest
-import urllib2
+
+import requests
 
 import morphlib
 
@@ -30,8 +31,9 @@ class RemoteRepoCacheTests(unittest.TestCase):
         try:
             return self.files[repo_url][sha1][filename]
         except KeyError:
-            raise urllib2.HTTPError(url='', code=404, msg='Not found',
-                                    hdrs={}, fp=None)
+            response = requests.Response()
+            response.status_code = 404
+            raise requests.exceptions.HTTPError(response=response)
 
     def _ls_tree_for_repo_url(self, repo_url, sha1):
         return json.dumps({
diff --git a/morphlib/source.py b/morphlib/source.py
index 4ad54ed9..6b809aa3 100644
--- a/morphlib/source.py
+++ b/morphlib/source.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2014  Codethink Limited
+# Copyright (C) 2012-2015  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -64,9 +64,36 @@ class Source(object):
     def __repr__(self): # pragma: no cover
         return 'Source(%s)' % str(self)
 
-    def basename(self): # pragma: no cover
+    def basename(self):
         return '%s.%s' % (self.cache_key, str(self.morphology['kind']))
 
+    def metadata_basename(self):
+        return '%s.meta' % (self.cache_key)
+
+    def build_log_basename(self):
+        return '%s.build-log' % (self.cache_key)
+
+    def files(self):
+        '''Return the name of all built artifacts of this source.
+
+        This includes every artifact and all associated metadata.
+
+        It's usually a bad idea to have only some of the files for a given
+        source available. Transfer all of them if you transfer any of them.
+
+        '''
+        files = {self.metadata_basename()}
+
+        if self.morphology['kind'] == 'chunk':
+            files.add(self.build_log_basename())
+
+        for artifact in self.artifacts.values():
+            files.add(artifact.basename())
+            if self.morphology.needs_artifact_metadata_cached:
+                files.add(artifact.metadata_basename('meta'))
+
+        return files
+
     def add_dependency(self, artifact): # pragma: no cover
         if artifact not in self.dependencies:
             self.dependencies.append(artifact)
diff --git a/morphlib/test_utils.py b/morphlib/test_utils.py
new file mode 100644
index 00000000..a3c7a80f
--- /dev/null
+++ b/morphlib/test_utils.py
@@ -0,0 +1,57 @@
+# Copyright (C) 2012-2015  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+'''Collected utilities for use in automated tests of Morph.'''
+
+
+import fs
+
+import morphlib
+
+
+def example_chunk():
+    '''Returns a chunk source with some artifacts.'''
+
+    loader = morphlib.morphloader.MorphologyLoader()
+    morph = loader.load_from_string(
+        '''
+            name: test
+            kind: chunk
+            products:
+              - artifact: chunk-runtime
+                include:
+                  - usr/bin
+                  - usr/sbin
+                  - usr/lib
+                  - usr/libexec
+              - artifact: chunk-devel
+                include:
+                  - usr/include
+              - artifact: chunk-doc
+                include:
+                  - usr/share/doc
+        ''')
+
+    source = next(morphlib.source.make_sources(
+        'repo', 'original/ref', 'chunk.morph', 'sha1', 'tree', morph))
+    source.cache_key = 'CHUNK'
+    return source
+
+
+
+class FakeLocalArtifactCache(morphlib.localartifactcache.LocalArtifactCache):
+    def __init__(self):
+        super(FakeLocalArtifactCache, self).__init__(fs.tempfs.TempFS())
diff --git a/morphlib/testutils.py b/morphlib/testutils.py
new file mode 100644
index 00000000..3678fc50
--- /dev/null
+++ b/morphlib/testutils.py
@@ -0,0 +1,56 @@
+# Copyright (C) 2012-2015  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+'''Collected utilities for use in automated tests of Morph.'''
+
+
+import fs
+
+import morphlib
+
+
+def example_chunk():
+    '''Returns a chunk source with some artifacts.'''
+
+    loader = morphlib.morphloader.MorphologyLoader()
+    morph = loader.load_from_string(
+        '''
+            name: test
+            kind: chunk
+            products:
+              - artifact: chunk-runtime
+                include:
+                  - usr/bin
+                  - usr/sbin
+                  - usr/lib
+                  - usr/libexec
+              - artifact: chunk-devel
+                include:
+                  - usr/include
+              - artifact: chunk-doc
+                include:
+                  - usr/share/doc
+        ''')
+
+    source = next(morphlib.source.make_sources(
+        'repo', 'original/ref', 'chunk.morph', 'sha1', 'tree', morph))
+    source.cache_key = 'CHUNK'
+    return source
+
+
+class FakeLocalArtifactCache(morphlib.localartifactcache.LocalArtifactCache):
+    def __init__(self):
+        super(FakeLocalArtifactCache, self).__init__(fs.tempfs.TempFS())
diff --git a/without-test-modules b/without-test-modules
index 55e5291d..58f671dc 100644
--- a/without-test-modules
+++ b/without-test-modules
@@ -52,3 +52,6 @@ distbuild/timer_event_source.py
 distbuild/worker_build_scheduler.py
 # Not unit tested, since it needs a full system branch
 morphlib/buildbranch.py
+
+# These are part of the test suite
+morphlib/testutils.py
diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn
index c60f704d..90168517 100644
--- a/yarns/implementations.yarn
+++ b/yarns/implementations.yarn
@@ -382,6 +382,7 @@ another to hold a chunk.
     cachedir = $DATADIR/cache
     tempdir = $DATADIR/tmp
     trove-host= []
+    cache-server =
     EOF
 
     mkdir "$DATADIR/cache"