Use OSTree for hardlink and artifact cache and a CoW unionfs to make system artifacts fasterbaserock/richardmaw/ostree-squash

This replaces the artifact cache and the hardlink cache with an OSTree
repository, which is a great performance improvement when the cache
directory and temporary directory are on the same filesystem.

Additionally it can de-duplicate file contents.

When we construct system artifacts deploy them, the staging area needs
to be writable, so OSTree on its own is insufficient, as its hardlinks
require the root to be kept read-only.

To handle this we use either the in-kernel overlayfs or unionfs-fuse,
though there is no automatic fall-back and it needs to be specified
manually.

To support distributed building, the artifact cache is extended to
support an OSTree repository.

Unfortunately cross-bootstrap is not expected to work with these changes
at this point in time.

IMPORTANT NOTE: We are well aware that this patch is too large to be
comprehensible. We intend to revert and apply a cleaned up series when
it is ready.

Change-Id: I693bb752500dab3c6db3b97393689239ae7071a8

1 files changed, 13 insertions, 74 deletions

diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py
index 8c2781aa..859e7481 100644
--- a/morphlib/stagingarea.py
+++ b/morphlib/stagingarea.py
@@ -87,6 +87,14 @@ class StagingArea(object):
 
         return self._dir_for_source(source, 'inst')
 
+    def overlay_upperdir(self, source):
+        '''Create a directory to be upperdir for overlayfs, and return it.'''
+        return self._dir_for_source(source, 'overlay_upper')
+
+    def overlaydir(self, source):
+        '''Create a directory to be a mount point for overlayfs, return it'''
+        return self._dir_for_source(source, 'overlay')
+
     def relative(self, filename):
         '''Return a filename relative to the staging area.'''
 
@@ -100,83 +108,14 @@ class StagingArea(object):
         assert filename.startswith(dirname)
         return filename[len(dirname) - 1:]  # include leading slash
 
-    def hardlink_all_files(self, srcpath, destpath): # pragma: no cover
-        '''Hardlink every file in the path to the staging-area
-
-        If an exception is raised, the staging-area is indeterminate.
-
-        '''
-
-        file_stat = os.lstat(srcpath)
-        mode = file_stat.st_mode
-
-        if stat.S_ISDIR(mode):
-            # Ensure directory exists in destination, then recurse.
-            if not os.path.lexists(destpath):
-                os.makedirs(destpath)
-            dest_stat = os.stat(os.path.realpath(destpath))
-            if not stat.S_ISDIR(dest_stat.st_mode):
-                raise IOError('Destination not a directory. source has %s'
-                              ' destination has %s' % (srcpath, destpath))
-
-            for entry in os.listdir(srcpath):
-                self.hardlink_all_files(os.path.join(srcpath, entry),
-                                        os.path.join(destpath, entry))
-        elif stat.S_ISLNK(mode):
-            # Copy the symlink.
-            if os.path.lexists(destpath):
-                os.remove(destpath)
-            os.symlink(os.readlink(srcpath), destpath)
-
-        elif stat.S_ISREG(mode):
-            # Hardlink the file.
-            if os.path.lexists(destpath):
-                os.remove(destpath)
-            os.link(srcpath, destpath)
-
-        elif stat.S_ISCHR(mode) or stat.S_ISBLK(mode):
-            # Block or character device. Put contents of st_dev in a mknod.
-            if os.path.lexists(destpath):
-                os.remove(destpath)
-            os.mknod(destpath, file_stat.st_mode, file_stat.st_rdev)
-            os.chmod(destpath, file_stat.st_mode)
-
-        else:
-            # Unsupported type.
-            raise IOError('Cannot extract %s into staging-area. Unsupported'
-                          ' type.' % srcpath)
-
-    def install_artifact(self, handle):
-        '''Install a build artifact into the staging area.
-
-        We access the artifact via an open file handle. For now, we assume
-        the artifact is a tarball.
-
-        '''
-
-        chunk_cache_dir = os.path.join(self._app.settings['tempdir'], 'chunks')
-        unpacked_artifact = os.path.join(
-            chunk_cache_dir, os.path.basename(handle.name) + '.d')
-        if not os.path.exists(unpacked_artifact):
-            self._app.status(
-                msg='Unpacking chunk from cache %(filename)s',
-                filename=os.path.basename(handle.name))
-            savedir = tempfile.mkdtemp(dir=chunk_cache_dir)
-            try:
-                morphlib.bins.unpack_binary_from_file(
-                    handle, savedir + '/')
-            except BaseException as e: # pragma: no cover
-                shutil.rmtree(savedir)
-                raise
-            # TODO: This rename is not concurrency safe if two builds are
-            #       extracting the same chunk, one build will fail because
-            #       the other renamed its tempdir here first.
-            os.rename(savedir, unpacked_artifact)
-
+    def install_artifact(self, artifact_cache, artifact):
+        '''Install a build artifact into the staging area.'''
         if not os.path.exists(self.dirname):
             self._mkdir(self.dirname)
 
-        self.hardlink_all_files(unpacked_artifact, self.dirname)
+        artifact_cache.get(artifact, directory=self.dirname)
+
+        morphlib.util.create_devices(artifact.source.morphology, self.dirname)
 
     def remove(self):
         '''Remove the entire staging area.