Use OSTree for hardlink and artifact cache and a CoW unionfs to make system artifacts fasterbaserock/richardmaw/ostree-squash

This replaces the artifact cache and the hardlink cache with an OSTree
repository, which is a great performance improvement when the cache
directory and temporary directory are on the same filesystem.

Additionally it can de-duplicate file contents.

When we construct system artifacts deploy them, the staging area needs
to be writable, so OSTree on its own is insufficient, as its hardlinks
require the root to be kept read-only.

To handle this we use either the in-kernel overlayfs or unionfs-fuse,
though there is no automatic fall-back and it needs to be specified
manually.

To support distributed building, the artifact cache is extended to
support an OSTree repository.

Unfortunately cross-bootstrap is not expected to work with these changes
at this point in time.

IMPORTANT NOTE: We are well aware that this patch is too large to be
comprehensible. We intend to revert and apply a cleaned up series when
it is ready.

Change-Id: I693bb752500dab3c6db3b97393689239ae7071a8

1 files changed, 9 insertions, 10 deletions

diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py
index fd5acdf5..a4670f0a 100644
--- a/morphlib/buildcommand.py
+++ b/morphlib/buildcommand.py
@@ -74,7 +74,8 @@ class BuildCommand(object):
         This includes creating the directories on disk if they are missing.
 
         '''
-        return morphlib.util.new_artifact_caches(self.app.settings)
+        return morphlib.util.new_artifact_caches(
+            self.app.settings, status_cb=self.app.status)
 
     def new_repo_caches(self):
         return morphlib.util.new_repo_caches(self.app)
@@ -310,9 +311,8 @@ class BuildCommand(object):
             self.build_source(source, build_env)
 
         for a in artifacts:
-            self.app.status(msg='%(kind)s %(name)s is cached at %(cachepath)s',
+            self.app.status(msg='%(kind)s %(name)s is cached.',
                             kind=source.morphology['kind'], name=a.name,
-                            cachepath=self.lac.artifact_filename(a),
                             chatty=(source.morphology['kind'] != "system"))
 
     def build_source(self, source, build_env):
@@ -422,8 +422,10 @@ class BuildCommand(object):
             # module into morphlib.remoteartififactcache first.
             to_fetch = []
             if not self.lac.has(artifact):
-                to_fetch.append((self.rac.get(artifact),
-                                 self.lac.put(artifact)))
+                self.app.status(
+                    msg='Fetching to local cache: artifact %(name)s',
+                    name=artifact.name)
+                self.lac.copy_from_remote(artifact, self.rac)
 
             if artifact.source.morphology.needs_artifact_metadata_cached:
                 if not self.lac.has_artifact_metadata(artifact, 'meta'):
@@ -432,9 +434,6 @@ class BuildCommand(object):
                         self.lac.put_artifact_metadata(artifact, 'meta')))
 
             if len(to_fetch) > 0:
-                self.app.status(
-                    msg='Fetching to local cache: artifact %(name)s',
-                    name=artifact.name)
                 fetch_files(to_fetch)
 
     def create_staging_area(self, build_env, use_chroot=True, extra_env={},
@@ -492,13 +491,13 @@ class BuildCommand(object):
             if artifact.source.build_mode == 'bootstrap':
                if not self.in_same_stratum(artifact.source, target_source):
                     continue
+
             self.app.status(
                 msg='Installing chunk %(chunk_name)s from cache %(cache)s',
                 chunk_name=artifact.name,
                 cache=artifact.source.cache_key[:7],
                 chatty=True)
-            handle = self.lac.get(artifact)
-            staging_area.install_artifact(handle)
+            staging_area.install_artifact(self.lac, artifact)
 
         if target_source.build_mode == 'staging':
             morphlib.builder.ldconfig(self.app.runcmd, staging_area.dirname)