Merge branch 'jmac/virtual_directories' into jmac/googlecas_and_virtual_directories_3jmac/googlecas_and_virtual_directories_3

102 files changed, 1196 insertions, 366 deletions

diff --git a/buildstream/__init__.py b/buildstream/__init__.py
index 7c4b5e5e6..cf56ecfe1 100644
--- a/buildstream/__init__.py
+++ b/buildstream/__init__.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
diff --git a/buildstream/_artifactcache/__init__.py b/buildstream/_artifactcache/__init__.py
index 66373fd75..07ed52b4b 100644
--- a/buildstream/_artifactcache/__init__.py
+++ b/buildstream/_artifactcache/__init__.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017-2018 Codethink Limited
 #
diff --git a/buildstream/_artifactcache/artifactcache.py b/buildstream/_artifactcache/artifactcache.py
index 1a0d14f74..f33b112bc 100644
--- a/buildstream/_artifactcache/artifactcache.py
+++ b/buildstream/_artifactcache/artifactcache.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017-2018 Codethink Limited
 #
diff --git a/buildstream/_cachekey.py b/buildstream/_cachekey.py
index 3d0c19b44..fe407e96f 100644
--- a/buildstream/_cachekey.py
+++ b/buildstream/_cachekey.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2018 Codethink Limited
 #
diff --git a/buildstream/_context.py b/buildstream/_context.py
index bf7f49515..1a59af2b9 100644
--- a/buildstream/_context.py
+++ b/buildstream/_context.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016-2018 Codethink Limited
 #
@@ -30,6 +29,7 @@ from ._exceptions import LoadError, LoadErrorReason, BstError
 from ._message import Message, MessageType
 from ._profile import Topics, profile_start, profile_end
 from ._artifactcache import ArtifactCache
+from ._workspaces import Workspaces
 
 
 # Context()
@@ -113,6 +113,7 @@ class Context():
         self._message_depth = deque()
         self._projects = []
         self._project_overrides = {}
+        self._workspaces = None
 
     # load()
     #
@@ -161,6 +162,7 @@ class Context():
             path = _yaml.node_get(defaults, str, directory)
             path = os.path.expanduser(path)
             path = os.path.expandvars(path)
+            path = os.path.normpath(path)
             setattr(self, directory, path)
 
         # Load artifact share configuration
@@ -218,6 +220,8 @@ class Context():
     #    project (Project): The project to add
     #
     def add_project(self, project):
+        if not self._projects:
+            self._workspaces = Workspaces(project)
         self._projects.append(project)
 
     # get_projects():
@@ -241,6 +245,9 @@ class Context():
     def get_toplevel_project(self):
         return self._projects[0]
 
+    def get_workspaces(self):
+        return self._workspaces
+
     # get_overrides():
     #
     # Fetch the override dictionary for the active project. This returns
diff --git a/buildstream/_elementfactory.py b/buildstream/_elementfactory.py
index bc1d95082..9d5b258cb 100644
--- a/buildstream/_elementfactory.py
+++ b/buildstream/_elementfactory.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
diff --git a/buildstream/_exceptions.py b/buildstream/_exceptions.py
index bcea65a8d..34ce91081 100644
--- a/buildstream/_exceptions.py
+++ b/buildstream/_exceptions.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
@@ -89,6 +88,7 @@ class ErrorDomain(Enum):
     ELEMENT = 11
     APP = 12
     STREAM = 13
+    VIRTUAL_FS = 14
 
 
 # BstError is an internal base exception class for BuildSream
@@ -198,6 +198,9 @@ class LoadErrorReason(Enum):
     # A project.conf file was missing
     MISSING_PROJECT_CONF = 17
 
+    # Try to load a directory not a yaml file
+    LOADING_DIRECTORY = 18
+
 
 # LoadError
 #
diff --git a/buildstream/_frontend/__init__.py b/buildstream/_frontend/__init__.py
index b2e41301e..febd4979d 100644
--- a/buildstream/_frontend/__init__.py
+++ b/buildstream/_frontend/__init__.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Codethink Limited
 #
diff --git a/buildstream/_frontend/app.py b/buildstream/_frontend/app.py
index fa07a9a54..4675b0eb0 100644
--- a/buildstream/_frontend/app.py
+++ b/buildstream/_frontend/app.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016-2018 Codethink Limited
 #
@@ -40,7 +39,6 @@ from .._exceptions import BstError, StreamError, LoadError, LoadErrorReason, App
 from .._message import Message, MessageType, unconditional_messages
 from .._stream import Stream
 from .._versions import BST_FORMAT_VERSION
-from .. import __version__ as build_stream_version
 from .. import _yaml
 
 # Import frontend assets
diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py
index 41e97cb0e..e59b1baec 100644
--- a/buildstream/_frontend/cli.py
+++ b/buildstream/_frontend/cli.py
@@ -12,6 +12,45 @@ from .complete import main_bashcomplete, complete_path, CompleteUnhandled
 #            Override of click's main entry point                #
 ##################################################################
 
+# search_command()
+#
+# Helper function to get a command and context object
+# for a given command.
+#
+# Args:
+#    commands (list): A list of command words following `bst` invocation
+#    context (click.Context): An existing toplevel context, or None
+#
+# Returns:
+#    context (click.Context): The context of the associated command, or None
+#
+def search_command(args, *, context=None):
+    if context is None:
+        context = cli.make_context('bst', args, resilient_parsing=True)
+
+    # Loop into the deepest command
+    command = cli
+    command_ctx = context
+    for cmd in args:
+        command = command_ctx.command.get_command(command_ctx, cmd)
+        if command is None:
+            return None
+        command_ctx = command.make_context(command.name, [command.name],
+                                           parent=command_ctx,
+                                           resilient_parsing=True)
+
+    return command_ctx
+
+
+# Completion for completing command names as help arguments
+def complete_commands(cmd, args, incomplete):
+    command_ctx = search_command(args[1:])
+    if command_ctx and command_ctx.command and isinstance(command_ctx.command, click.MultiCommand):
+        return [subcommand + " " for subcommand in command_ctx.command.list_commands(command_ctx)]
+
+    return []
+
+
 # Special completion for completing the bst elements in a project dir
 def complete_target(args, incomplete):
     """
@@ -20,6 +59,18 @@ def complete_target(args, incomplete):
     :return: all the possible user-specified completions for the param
     """
 
+    project_conf = 'project.conf'
+
+    def ensure_project_dir(directory):
+        directory = os.path.abspath(directory)
+        while not os.path.isfile(os.path.join(directory, project_conf)):
+            parent_dir = os.path.dirname(directory)
+            if directory == parent_dir:
+                break
+            directory = parent_dir
+
+        return directory
+
     # First resolve the directory, in case there is an
     # active --directory/-C option
     #
@@ -35,10 +86,14 @@ def complete_target(args, incomplete):
 
     if idx >= 0 and len(args) > idx + 1:
         base_directory = args[idx + 1]
+    else:
+        # Check if this directory or any of its parent directories
+        # contain a project config file
+        base_directory = ensure_project_dir(base_directory)
 
     # Now parse the project.conf just to find the element path,
     # this is unfortunately a bit heavy.
-    project_file = os.path.join(base_directory, 'project.conf')
+    project_file = os.path.join(base_directory, project_conf)
     try:
         project = _yaml.load(project_file)
     except LoadError:
@@ -57,7 +112,7 @@ def complete_target(args, incomplete):
     return complete_path("File", incomplete, base_directory=base_directory)
 
 
-def override_completions(cmd_param, args, incomplete):
+def override_completions(cmd, cmd_param, args, incomplete):
     """
     :param cmd_param: command definition
     :param args: full list of args typed before the incomplete arg
@@ -65,6 +120,9 @@ def override_completions(cmd_param, args, incomplete):
     :return: all the possible user-specified completions for the param
     """
 
+    if cmd.name == 'help':
+        return complete_commands(cmd, args, incomplete)
+
     # We can't easily extend click's data structures without
     # modifying click itself, so just do some weak special casing
     # right here and select which parameters we want to handle specially.
@@ -175,6 +233,33 @@ def cli(context, **kwargs):
 
 
 ##################################################################
+#                           Help Command                         #
+##################################################################
+@cli.command(name="help", short_help="Print usage information",
+             context_settings={"help_option_names": []})
+@click.argument("command", nargs=-1, metavar='COMMAND')
+@click.pass_context
+def help_command(ctx, command):
+    """Print usage information about a given command
+    """
+    command_ctx = search_command(command, context=ctx.parent)
+    if not command_ctx:
+        click.echo("Not a valid command: '{} {}'"
+                   .format(ctx.parent.info_name, " ".join(command)), err=True)
+        sys.exit(-1)
+
+    click.echo(command_ctx.command.get_help(command_ctx), err=True)
+
+    # Hint about available sub commands
+    if isinstance(command_ctx.command, click.MultiCommand):
+        detail = " "
+        if command:
+            detail = " {} ".format(" ".join(command))
+        click.echo("\nFor usage on a specific command: {} help{}COMMAND"
+                   .format(ctx.parent.info_name, detail), err=True)
+
+
+##################################################################
 #                           Init Command                         #
 ##################################################################
 @cli.command(short_help="Initialize a new BuildStream project")
@@ -206,20 +291,20 @@ def init(app, project_name, format_version, element_path, force):
 @click.option('--all', 'all_', default=False, is_flag=True,
               help="Build elements that would not be needed for the current build plan")
 @click.option('--track', 'track_', multiple=True,
-              type=click.Path(dir_okay=False, readable=True),
+              type=click.Path(readable=False),
               help="Specify elements to track during the build. Can be used "
                    "repeatedly to specify multiple elements")
 @click.option('--track-all', default=False, is_flag=True,
               help="Track all elements in the pipeline")
 @click.option('--track-except', multiple=True,
-              type=click.Path(dir_okay=False, readable=True),
+              type=click.Path(readable=False),
               help="Except certain dependencies from tracking")
 @click.option('--track-cross-junctions', '-J', default=False, is_flag=True,
               help="Allow tracking to cross junction boundaries")
 @click.option('--track-save', default=False, is_flag=True,
               help="Deprecated: This is ignored")
 @click.argument('elements', nargs=-1,
-                type=click.Path(dir_okay=False, readable=True))
+                type=click.Path(readable=False))
 @click.pass_obj
 def build(app, elements, all_, track_, track_save, track_all, track_except, track_cross_junctions):
     """Build elements in a pipeline"""
@@ -248,7 +333,7 @@ def build(app, elements, all_, track_, track_save, track_all, track_except, trac
 ##################################################################
 @cli.command(short_help="Fetch sources in a pipeline")
 @click.option('--except', 'except_', multiple=True,
-              type=click.Path(dir_okay=False, readable=True),
+              type=click.Path(readable=False),
               help="Except certain dependencies from fetching")
 @click.option('--deps', '-d', default='plan',
               type=click.Choice(['none', 'plan', 'all']),
@@ -258,7 +343,7 @@ def build(app, elements, all_, track_, track_save, track_all, track_except, trac
 @click.option('--track-cross-junctions', '-J', default=False, is_flag=True,
               help="Allow tracking to cross junction boundaries")
 @click.argument('elements', nargs=-1,
-                type=click.Path(dir_okay=False, readable=True))
+                type=click.Path(readable=False))
 @click.pass_obj
 def fetch(app, elements, deps, track_, except_, track_cross_junctions):
     """Fetch sources required to build the pipeline
@@ -299,7 +384,7 @@ def fetch(app, elements, deps, track_, except_, track_cross_junctions):
 ##################################################################
 @cli.command(short_help="Track new source references")
 @click.option('--except', 'except_', multiple=True,
-              type=click.Path(dir_okay=False, readable=True),
+              type=click.Path(readable=False),
               help="Except certain dependencies from tracking")
 @click.option('--deps', '-d', default='none',
               type=click.Choice(['none', 'all']),
@@ -307,7 +392,7 @@ def fetch(app, elements, deps, track_, except_, track_cross_junctions):
 @click.option('--cross-junctions', '-J', default=False, is_flag=True,
               help="Allow crossing junction boundaries")
 @click.argument('elements', nargs=-1,
-                type=click.Path(dir_okay=False, readable=True))
+                type=click.Path(readable=False))
 @click.pass_obj
 def track(app, elements, deps, except_, cross_junctions):
     """Consults the specified tracking branches for new versions available
@@ -339,7 +424,7 @@ def track(app, elements, deps, except_, cross_junctions):
 @click.option('--remote', '-r',
               help="The URL of the remote cache (defaults to the first configured cache)")
 @click.argument('elements', nargs=-1,
-                type=click.Path(dir_okay=False, readable=True))
+                type=click.Path(readable=False))
 @click.pass_obj
 def pull(app, elements, deps, remote):
     """Pull a built artifact from the configured remote artifact cache.
@@ -368,7 +453,7 @@ def pull(app, elements, deps, remote):
 @click.option('--remote', '-r', default=None,
               help="The URL of the remote cache (defaults to the first configured cache)")
 @click.argument('elements', nargs=-1,
-                type=click.Path(dir_okay=False, readable=True))
+                type=click.Path(readable=False))
 @click.pass_obj
 def push(app, elements, deps, remote):
     """Push a built artifact to a remote artifact cache.
@@ -391,7 +476,7 @@ def push(app, elements, deps, remote):
 ##################################################################
 @cli.command(short_help="Show elements in the pipeline")
 @click.option('--except', 'except_', multiple=True,
-              type=click.Path(dir_okay=False, readable=True),
+              type=click.Path(readable=False),
               help="Except certain dependencies")
 @click.option('--deps', '-d', default='all',
               type=click.Choice(['none', 'plan', 'run', 'build', 'all']),
@@ -403,7 +488,7 @@ def push(app, elements, deps, remote):
               type=click.STRING,
               help='Format string for each element')
 @click.argument('elements', nargs=-1,
-                type=click.Path(dir_okay=False, readable=True))
+                type=click.Path(readable=False))
 @click.pass_obj
 def show(app, elements, deps, except_, order, format_):
     """Show elements in the pipeline
@@ -482,7 +567,7 @@ def show(app, elements, deps, except_, order, format_):
 @click.option('--isolate', is_flag=True, default=False,
               help='Create an isolated build sandbox')
 @click.argument('element',
-                type=click.Path(dir_okay=False, readable=True))
+                type=click.Path(readable=False))
 @click.argument('command', type=click.STRING, nargs=-1)
 @click.pass_obj
 def shell(app, element, sysroot, mount, isolate, build_, command):
@@ -543,7 +628,7 @@ def shell(app, element, sysroot, mount, isolate, build_, command):
 @click.option('--hardlinks', default=False, is_flag=True,
               help="Checkout hardlinks instead of copies (handle with care)")
 @click.argument('element',
-                type=click.Path(dir_okay=False, readable=True))
+                type=click.Path(readable=False))
 @click.argument('directory', type=click.Path(file_okay=False))
 @click.pass_obj
 def checkout(app, element, directory, force, integrate, hardlinks):
@@ -577,7 +662,7 @@ def workspace():
 @click.option('--track', 'track_', default=False, is_flag=True,
               help="Track and fetch new source references before checking out the workspace")
 @click.argument('element',
-                type=click.Path(dir_okay=False, readable=True))
+                type=click.Path(readable=False))
 @click.argument('directory', type=click.Path(file_okay=False))
 @click.pass_obj
 def workspace_open(app, no_checkout, force, track_, element, directory):
@@ -609,7 +694,7 @@ def workspace_open(app, no_checkout, force, track_, element, directory):
 @click.option('--all', '-a', 'all_', default=False, is_flag=True,
               help="Close all open workspaces")
 @click.argument('elements', nargs=-1,
-                type=click.Path(dir_okay=False, readable=True))
+                type=click.Path(readable=False))
 @click.pass_obj
 def workspace_close(app, remove_dir, all_, elements):
     """Close a workspace"""
@@ -626,7 +711,7 @@ def workspace_close(app, remove_dir, all_, elements):
             sys.exit(0)
 
         if all_:
-            elements = [element_name for element_name, _ in app.project.workspaces.list()]
+            elements = [element_name for element_name, _ in app.context.get_workspaces().list()]
 
         elements = app.stream.redirect_element_names(elements)
 
@@ -658,7 +743,7 @@ def workspace_close(app, remove_dir, all_, elements):
 @click.option('--all', '-a', 'all_', default=False, is_flag=True,
               help="Reset all open workspaces")
 @click.argument('elements', nargs=-1,
-                type=click.Path(dir_okay=False, readable=True))
+                type=click.Path(readable=False))
 @click.pass_obj
 def workspace_reset(app, soft, track_, all_, elements):
     """Reset a workspace to its original state"""
@@ -678,7 +763,7 @@ def workspace_reset(app, soft, track_, all_, elements):
                 sys.exit(-1)
 
         if all_:
-            elements = tuple(element_name for element_name, _ in app.project.workspaces.list())
+            elements = tuple(element_name for element_name, _ in app.context.get_workspaces().list())
 
         app.stream.workspace_reset(elements, soft=soft, track_first=track_)
 
@@ -700,7 +785,7 @@ def workspace_list(app):
 ##################################################################
 @cli.command(name="source-bundle", short_help="Produce a build bundle to be manually executed")
 @click.option('--except', 'except_', multiple=True,
-              type=click.Path(dir_okay=False, readable=True),
+              type=click.Path(readable=False),
               help="Elements to except from the tarball")
 @click.option('--compression', default='gz',
               type=click.Choice(['none', 'gz', 'bz2', 'xz']),
@@ -712,7 +797,7 @@ def workspace_list(app):
 @click.option('--directory', default=os.getcwd(),
               help="The directory to write the tarball to")
 @click.argument('element',
-                type=click.Path(dir_okay=False, readable=True))
+                type=click.Path(readable=False))
 @click.pass_obj
 def source_bundle(app, element, force, directory,
                   track_, compression, except_):
diff --git a/buildstream/_frontend/complete.py b/buildstream/_frontend/complete.py
index fa986ee6b..79bb92758 100644
--- a/buildstream/_frontend/complete.py
+++ b/buildstream/_frontend/complete.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
@@ -209,7 +208,7 @@ def is_incomplete_argument(current_params, cmd_param):
     return False
 
 
-def get_user_autocompletions(args, incomplete, cmd_param, override):
+def get_user_autocompletions(args, incomplete, cmd, cmd_param, override):
     """
     :param args: full list of args typed before the incomplete arg
     :param incomplete: the incomplete text of the arg to autocomplete
@@ -222,7 +221,8 @@ def get_user_autocompletions(args, incomplete, cmd_param, override):
 
     # Use the type specific default completions unless it was overridden
     try:
-        return override(cmd_param=cmd_param,
+        return override(cmd=cmd,
+                        cmd_param=cmd_param,
                         args=args,
                         incomplete=incomplete)
     except CompleteUnhandled:
@@ -268,14 +268,14 @@ def get_choices(cli, prog_name, args, incomplete, override):
         # completion for option values by choices
         for cmd_param in ctx.command.params:
             if isinstance(cmd_param, Option) and is_incomplete_option(all_args, cmd_param):
-                choices.extend(get_user_autocompletions(all_args, incomplete, cmd_param, override))
+                choices.extend(get_user_autocompletions(all_args, incomplete, ctx.command, cmd_param, override))
                 found_param = True
                 break
     if not found_param:
         # completion for argument values by choices
         for cmd_param in ctx.command.params:
             if isinstance(cmd_param, Argument) and is_incomplete_argument(ctx.params, cmd_param):
-                choices.extend(get_user_autocompletions(all_args, incomplete, cmd_param, override))
+                choices.extend(get_user_autocompletions(all_args, incomplete, ctx.command, cmd_param, override))
                 found_param = True
                 break
 
diff --git a/buildstream/_frontend/linuxapp.py b/buildstream/_frontend/linuxapp.py
index 92586bc40..176c5d052 100644
--- a/buildstream/_frontend/linuxapp.py
+++ b/buildstream/_frontend/linuxapp.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2018 Codethink Limited
 #
diff --git a/buildstream/_frontend/profile.py b/buildstream/_frontend/profile.py
index 00a5980d3..dda0f7ffe 100644
--- a/buildstream/_frontend/profile.py
+++ b/buildstream/_frontend/profile.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2018 Codethink Limited
 #
diff --git a/buildstream/_frontend/status.py b/buildstream/_frontend/status.py
index 0e5855181..3f66e009a 100644
--- a/buildstream/_frontend/status.py
+++ b/buildstream/_frontend/status.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2018 Codethink Limited
 #
diff --git a/buildstream/_frontend/widget.py b/buildstream/_frontend/widget.py
index fe7229e8a..dab8cab56 100644
--- a/buildstream/_frontend/widget.py
+++ b/buildstream/_frontend/widget.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Codethink Limited
 #
diff --git a/buildstream/_fuse/__init__.py b/buildstream/_fuse/__init__.py
index 3ef9d631f..a5e882634 100644
--- a/buildstream/_fuse/__init__.py
+++ b/buildstream/_fuse/__init__.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Codethink Limited
 #
diff --git a/buildstream/_fuse/hardlinks.py b/buildstream/_fuse/hardlinks.py
index d23f3fff6..4da51bb22 100644
--- a/buildstream/_fuse/hardlinks.py
+++ b/buildstream/_fuse/hardlinks.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Stavros Korokithakis
 #  Copyright (C) 2017 Codethink Limited
diff --git a/buildstream/_fuse/mount.py b/buildstream/_fuse/mount.py
index 3848ad305..0ab1ce715 100644
--- a/buildstream/_fuse/mount.py
+++ b/buildstream/_fuse/mount.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Codethink Limited
 #
diff --git a/buildstream/_loader/__init__.py b/buildstream/_loader/__init__.py
index dbc89ba83..a2c31796e 100644
--- a/buildstream/_loader/__init__.py
+++ b/buildstream/_loader/__init__.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2018 Codethink Limited
 #
diff --git a/buildstream/_loader/loadelement.py b/buildstream/_loader/loadelement.py
index b270fbef8..065364a87 100644
--- a/buildstream/_loader/loadelement.py
+++ b/buildstream/_loader/loadelement.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
diff --git a/buildstream/_loader/loader.py b/buildstream/_loader/loader.py
index e0ceb4fb9..07b0de996 100644
--- a/buildstream/_loader/loader.py
+++ b/buildstream/_loader/loader.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2018 Codethink Limited
 #
@@ -107,9 +106,13 @@ class Loader():
 
         # First pass, recursively load files and populate our table of LoadElements
         #
+        deps = []
+
         for target in self._targets:
             profile_start(Topics.LOAD_PROJECT, target)
-            self._load_file(target, rewritable, ticker)
+            junction, name, loader = self._parse_name(target, rewritable, ticker)
+            loader._load_file(name, rewritable, ticker)
+            deps.append(Dependency(name, junction=junction))
             profile_end(Topics.LOAD_PROJECT, target)
 
         #
@@ -119,7 +122,8 @@ class Loader():
         # Set up a dummy element that depends on all top-level targets
         # to resolve potential circular dependencies between them
         DummyTarget = namedtuple('DummyTarget', ['name', 'full_name', 'deps'])
-        dummy = DummyTarget(name='', full_name='', deps=[Dependency(e) for e in self._targets])
+
+        dummy = DummyTarget(name='', full_name='', deps=deps)
         self._elements[''] = dummy
 
         profile_key = "_".join(t for t in self._targets)
@@ -127,17 +131,20 @@ class Loader():
         self._check_circular_deps('')
         profile_end(Topics.CIRCULAR_CHECK, profile_key)
 
+        ret = []
         #
         # Sort direct dependencies of elements by their dependency ordering
         #
         for target in self._targets:
             profile_start(Topics.SORT_DEPENDENCIES, target)
-            self._sort_dependencies(target)
+            junction, name, loader = self._parse_name(target, rewritable, ticker)
+            loader._sort_dependencies(name)
             profile_end(Topics.SORT_DEPENDENCIES, target)
+            # Finally, wrap what we have into LoadElements and return the target
+            #
+            ret.append(loader._collect_element(name))
 
-        # Finally, wrap what we have into LoadElements and return the target
-        #
-        return [self._collect_element(target) for target in self._targets]
+        return ret
 
     # cleanup():
     #
@@ -207,7 +214,33 @@ class Loader():
 
         # Load the data and process any conditional statements therein
         fullpath = os.path.join(self._basedir, filename)
-        node = _yaml.load(fullpath, shortname=filename, copy_tree=rewritable)
+        try:
+            node = _yaml.load(fullpath, shortname=filename, copy_tree=rewritable)
+        except LoadError as e:
+            if e.reason == LoadErrorReason.MISSING_FILE:
+                # If we can't find the file, try to suggest plausible
+                # alternatives by stripping the element-path from the given
+                # filename, and verifying that it exists.
+                message = "Could not find element '{}' in elements directory '{}'".format(filename, self._basedir)
+                detail = None
+                elements_dir = os.path.relpath(self._basedir, self.project.directory)
+                element_relpath = os.path.relpath(filename, elements_dir)
+                if filename.startswith(elements_dir) and os.path.exists(os.path.join(self._basedir, element_relpath)):
+                    detail = "Did you mean '{}'?".format(element_relpath)
+                raise LoadError(LoadErrorReason.MISSING_FILE,
+                                message, detail=detail) from e
+            elif e.reason == LoadErrorReason.LOADING_DIRECTORY:
+                # If a <directory>.bst file exists in the element path,
+                # let's suggest this as a plausible alternative.
+                message = str(e)
+                detail = None
+                if os.path.exists(os.path.join(self._basedir, filename + '.bst')):
+                    element_name = filename + '.bst'
+                    detail = "Did you mean '{}'?\n".format(element_name)
+                raise LoadError(LoadErrorReason.LOADING_DIRECTORY,
+                                message, detail=detail) from e
+            else:
+                raise
         self._options.process_node(node)
 
         element = LoadElement(node, filename, self)
@@ -538,3 +571,30 @@ class Loader():
             return self._loaders[dep.junction]
         else:
             return self
+
+    # _parse_name():
+    #
+    # Get junction and base name of element along with loader for the sub-project
+    #
+    # Args:
+    #   name (str): Name of target
+    #   rewritable (bool): Whether the loaded files should be rewritable
+    #                      this is a bit more expensive due to deep copies
+    #   ticker (callable): An optional function for tracking load progress
+    #
+    # Returns:
+    #   (tuple): - (str): name of the junction element
+    #            - (str): name of the element
+    #            - (Loader): loader for sub-project
+    #
+    def _parse_name(self, name, rewritable, ticker):
+        # We allow to split only once since deep junctions names are forbidden.
+        # Users who want to refer to elements in sub-sub-projects are required
+        # to create junctions on the top level project.
+        junction_path = name.rsplit(':', 1)
+        if len(junction_path) == 1:
+            return None, junction_path[-1], self
+        else:
+            self._load_file(junction_path[-2], rewritable, ticker)
+            loader = self._get_loader(junction_path[-2], rewritable=rewritable, ticker=ticker)
+            return junction_path[-2], junction_path[-1], loader
diff --git a/buildstream/_loader/metaelement.py b/buildstream/_loader/metaelement.py
index 7ba6ed0ed..16788e92b 100644
--- a/buildstream/_loader/metaelement.py
+++ b/buildstream/_loader/metaelement.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
diff --git a/buildstream/_loader/metasource.py b/buildstream/_loader/metasource.py
index 75e191595..3bcc21ec6 100644
--- a/buildstream/_loader/metasource.py
+++ b/buildstream/_loader/metasource.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
diff --git a/buildstream/_loader/types.py b/buildstream/_loader/types.py
index 9d96894c0..000925a6e 100644
--- a/buildstream/_loader/types.py
+++ b/buildstream/_loader/types.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2018 Codethink Limited
 #
diff --git a/buildstream/_message.py b/buildstream/_message.py
index 9073e3803..32650450a 100644
--- a/buildstream/_message.py
+++ b/buildstream/_message.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Codethink Limited
 #
diff --git a/buildstream/_options/__init__.py b/buildstream/_options/__init__.py
index 7b8f36553..70bbe35aa 100644
--- a/buildstream/_options/__init__.py
+++ b/buildstream/_options/__init__.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Codethink Limited
 #
diff --git a/buildstream/_options/option.py b/buildstream/_options/option.py
index 9501a2bde..ffdb4d272 100644
--- a/buildstream/_options/option.py
+++ b/buildstream/_options/option.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Codethink Limited
 #
diff --git a/buildstream/_options/optionarch.py b/buildstream/_options/optionarch.py
index 2ced60935..13a691643 100644
--- a/buildstream/_options/optionarch.py
+++ b/buildstream/_options/optionarch.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Codethink Limited
 #
diff --git a/buildstream/_options/optionbool.py b/buildstream/_options/optionbool.py
index e0e1474d9..ffef55ca1 100644
--- a/buildstream/_options/optionbool.py
+++ b/buildstream/_options/optionbool.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Codethink Limited
 #
diff --git a/buildstream/_options/optioneltmask.py b/buildstream/_options/optioneltmask.py
index 46c7fcd62..09c2ce8c2 100644
--- a/buildstream/_options/optioneltmask.py
+++ b/buildstream/_options/optioneltmask.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Codethink Limited
 #
diff --git a/buildstream/_options/optionenum.py b/buildstream/_options/optionenum.py
index bc21bd81c..095b9c356 100644
--- a/buildstream/_options/optionenum.py
+++ b/buildstream/_options/optionenum.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Codethink Limited
 #
diff --git a/buildstream/_options/optionflags.py b/buildstream/_options/optionflags.py
index 84ecc1360..0271208d9 100644
--- a/buildstream/_options/optionflags.py
+++ b/buildstream/_options/optionflags.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Codethink Limited
 #
diff --git a/buildstream/_options/optionpool.py b/buildstream/_options/optionpool.py
index 70acd268e..f90fd820c 100644
--- a/buildstream/_options/optionpool.py
+++ b/buildstream/_options/optionpool.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Codethink Limited
 #
diff --git a/buildstream/_ostree.py b/buildstream/_ostree.py
index 6fee37dc0..217790d84 100644
--- a/buildstream/_ostree.py
+++ b/buildstream/_ostree.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Codethink Limited
 #
@@ -150,6 +149,42 @@ def exists(repo, ref):
     return has_object
 
 
+# remove():
+#
+# Removes the given commit or symbolic ref from the repo.
+#
+# Args:
+#    repo (OSTree.Repo): The repo
+#    ref (str): A commit checksum or symbolic ref
+#    defer_prune (bool): Whether to defer pruning to the caller. NOTE:
+#                        The space won't be freed until you manually
+#                        call repo.prune.
+#
+# Returns:
+#    (int|None) The amount of space pruned from the repository in
+#               Bytes, or None if defer_prune is True
+#
+def remove(repo, ref, *, defer_prune=False):
+
+    # Get the commit checksum, this will:
+    #
+    #  o Return a commit checksum if ref is a symbolic branch
+    #  o Return the same commit checksum if ref is a valid commit checksum
+    #  o Return None if the ostree repo doesnt know this ref.
+    #
+    check = checksum(repo, ref)
+    if check is None:
+        raise OSTreeError("Could not find artifact for ref '{}'".format(ref))
+
+    repo.set_ref_immediate(None, ref, None)
+
+    if not defer_prune:
+        _, _, _, pruned = repo.prune(OSTree.RepoPruneFlags.REFS_ONLY, -1)
+        return pruned
+
+    return None
+
+
 # checksum():
 #
 # Returns the commit checksum for a given symbolic ref,
@@ -275,3 +310,47 @@ def configure_remote(repo, remote, url, key_url=None):
             repo.remote_gpg_import(remote, stream, None, 0, None)
         except GLib.GError as e:
             raise OSTreeError("Failed to add gpg key from url '{}': {}".format(key_url, e.message)) from e
+
+
+# list_artifacts():
+#
+# List cached artifacts in Least Recently Modified (LRM) order.
+#
+# Returns:
+#     (list) - A list of refs in LRM order
+#
+def list_artifacts(repo):
+    # string of: /path/to/repo/refs/heads
+    ref_heads = os.path.join(repo.get_path().get_path(), 'refs', 'heads')
+
+    # obtain list of <project>/<element>/<key>
+    refs = _list_all_refs(repo).keys()
+
+    mtimes = []
+    for ref in refs:
+        ref_path = os.path.join(ref_heads, ref)
+        if os.path.exists(ref_path):
+            # Obtain the mtime (the time a file was last modified)
+            mtimes.append(os.path.getmtime(ref_path))
+
+    # NOTE: Sorted will sort from earliest to latest, thus the
+    # first element of this list will be the file modified earliest.
+    return [ref for _, ref in sorted(zip(mtimes, refs))]
+
+
+# _list_all_refs():
+#
+# Create a list of all refs.
+#
+# Args:
+#    repo (OSTree.Repo): The repo
+#
+# Returns:
+#    (dict): A dict of refs to checksums.
+#
+def _list_all_refs(repo):
+    try:
+        _, refs = repo.list_refs(None)
+        return refs
+    except GLib.GError as e:
+        raise OSTreeError(message=e.message) from e
diff --git a/buildstream/_pipeline.py b/buildstream/_pipeline.py
index ba27ca6b6..9f4504d3f 100644
--- a/buildstream/_pipeline.py
+++ b/buildstream/_pipeline.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016-2018 Codethink Limited
 #
@@ -346,6 +345,8 @@ class Pipeline():
     # lists targetted at tracking.
     #
     # Args:
+    #    project (Project): Project used for cross_junction filtering.
+    #                       All elements are expected to belong to that project.
     #    elements (list of Element): The list of elements to filter
     #    cross_junction_requested (bool): Whether the user requested
     #                                     cross junction tracking
@@ -353,12 +354,11 @@ class Pipeline():
     # Returns:
     #    (list of Element): The filtered or asserted result
     #
-    def track_cross_junction_filter(self, elements, cross_junction_requested):
+    def track_cross_junction_filter(self, project, elements, cross_junction_requested):
         # Filter out cross junctioned elements
-        if cross_junction_requested:
-            self._assert_junction_tracking(elements)
-        else:
-            elements = self._filter_cross_junctions(elements)
+        if not cross_junction_requested:
+            elements = self._filter_cross_junctions(project, elements)
+        self._assert_junction_tracking(elements)
 
         return elements
 
@@ -403,16 +403,17 @@ class Pipeline():
     # Filters out cross junction elements from the elements
     #
     # Args:
+    #    project (Project): The project on which elements are allowed
     #    elements (list of Element): The list of elements to be tracked
     #
     # Returns:
     #    (list): A filtered list of `elements` which does
     #            not contain any cross junction elements.
     #
-    def _filter_cross_junctions(self, elements):
+    def _filter_cross_junctions(self, project, elements):
         return [
             element for element in elements
-            if element._get_project() is self._project
+            if element._get_project() is project
         ]
 
     # _assert_junction_tracking()
diff --git a/buildstream/_platform/__init__.py b/buildstream/_platform/__init__.py
index 49400c3f2..29a29894b 100644
--- a/buildstream/_platform/__init__.py
+++ b/buildstream/_platform/__init__.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Codethink Limited
 #
diff --git a/buildstream/_platform/linux.py b/buildstream/_platform/linux.py
index f620f25be..56ebb410e 100644
--- a/buildstream/_platform/linux.py
+++ b/buildstream/_platform/linux.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Codethink Limited
 #
diff --git a/buildstream/_platform/platform.py b/buildstream/_platform/platform.py
index fc6a74bdd..29da33563 100644
--- a/buildstream/_platform/platform.py
+++ b/buildstream/_platform/platform.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Codethink Limited
 #
diff --git a/buildstream/_platform/unix.py b/buildstream/_platform/unix.py
index e9c62a45b..be4c129d3 100644
--- a/buildstream/_platform/unix.py
+++ b/buildstream/_platform/unix.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Codethink Limited
 #
diff --git a/buildstream/_plugincontext.py b/buildstream/_plugincontext.py
index 0be3de1b1..38d2231ba 100644
--- a/buildstream/_plugincontext.py
+++ b/buildstream/_plugincontext.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
diff --git a/buildstream/_profile.py b/buildstream/_profile.py
index 4d39cfc5e..40cd2ab7e 100644
--- a/buildstream/_profile.py
+++ b/buildstream/_profile.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Codethink Limited
 #
diff --git a/buildstream/_project.py b/buildstream/_project.py
index 87f14ee0d..54ec9ee34 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016-2018 Codethink Limited
 #
@@ -34,12 +33,14 @@ from ._elementfactory import ElementFactory
 from ._sourcefactory import SourceFactory
 from ._projectrefs import ProjectRefs, ProjectRefStorage
 from ._versions import BST_FORMAT_VERSION
-from ._workspaces import Workspaces
 
 
 # The separator we use for user specified aliases
 _ALIAS_SEPARATOR = ':'
 
+# Project Configuration file
+_PROJECT_CONF_FILE = 'project.conf'
+
 
 # HostMount()
 #
@@ -75,7 +76,7 @@ class Project():
         self.name = None
 
         # The project directory
-        self.directory = os.path.abspath(directory)
+        self.directory = self._ensure_project_dir(directory)
 
         # Absolute path to where elements are loaded from within the project
         self.element_path = None
@@ -84,7 +85,6 @@ class Project():
         self.refs = ProjectRefs(self.directory, 'project.refs')
         self.junction_refs = ProjectRefs(self.directory, 'junction.refs')
 
-        self.workspaces = None                   # Workspaces
         self.options = None                      # OptionPool
         self.junction = junction                 # The junction Element object, if this is a subproject
         self.fail_on_overlap = False             # Whether overlaps are treated as errors
@@ -211,7 +211,7 @@ class Project():
     def _load(self):
 
         # Load builtin default
-        projectfile = os.path.join(self.directory, "project.conf")
+        projectfile = os.path.join(self.directory, _PROJECT_CONF_FILE)
         config = _yaml.load(_site.default_project_config)
 
         # Load project local config and override the builtin
@@ -298,9 +298,6 @@ class Project():
         # Load artifacts pull/push configuration for this project
         self.artifact_cache_specs = ArtifactCache.specs_from_config_node(config, self.directory)
 
-        # Workspace configurations
-        self.workspaces = Workspaces(self)
-
         # Plugin origins and versions
         origins = _yaml.node_get(config, list, 'plugins', default_value=[])
         for origin in origins:
@@ -458,3 +455,27 @@ class Project():
                 # paths are passed in relative to the project, but must be absolute
                 origin_dict['path'] = os.path.join(self.directory, origin_dict['path'])
             destination.append(origin_dict)
+
+    # _ensure_project_dir()
+    #
+    # Returns path of the project directory, if a configuration file is found
+    # in given directory or any of its parent directories.
+    #
+    # Args:
+    #    directory (str) - directory from where the command was invoked
+    #
+    # Raises:
+    #    LoadError if project.conf is not found
+    #
+    def _ensure_project_dir(self, directory):
+        directory = os.path.abspath(directory)
+        while not os.path.isfile(os.path.join(directory, _PROJECT_CONF_FILE)):
+            parent_dir = os.path.dirname(directory)
+            if directory == parent_dir:
+                raise LoadError(
+                    LoadErrorReason.MISSING_PROJECT_CONF,
+                    '{} not found in current directory or any of its parent directories'
+                    .format(_PROJECT_CONF_FILE))
+            directory = parent_dir
+
+        return directory
diff --git a/buildstream/_projectrefs.py b/buildstream/_projectrefs.py
index 83dd2619b..4009d7449 100644
--- a/buildstream/_projectrefs.py
+++ b/buildstream/_projectrefs.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2018 Codethink Limited
 #
diff --git a/buildstream/_scheduler/__init__.py b/buildstream/_scheduler/__init__.py
index 14cdebf8e..80523db6f 100644
--- a/buildstream/_scheduler/__init__.py
+++ b/buildstream/_scheduler/__init__.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Codethink Limited
 #
diff --git a/buildstream/_scheduler/buildqueue.py b/buildstream/_scheduler/buildqueue.py
index 24a124b32..50ba312ff 100644
--- a/buildstream/_scheduler/buildqueue.py
+++ b/buildstream/_scheduler/buildqueue.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
diff --git a/buildstream/_scheduler/fetchqueue.py b/buildstream/_scheduler/fetchqueue.py
index 61055725d..24512bddb 100644
--- a/buildstream/_scheduler/fetchqueue.py
+++ b/buildstream/_scheduler/fetchqueue.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
diff --git a/buildstream/_scheduler/job.py b/buildstream/_scheduler/job.py
index b8b81f2a9..cc350649e 100644
--- a/buildstream/_scheduler/job.py
+++ b/buildstream/_scheduler/job.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
@@ -138,7 +137,28 @@ class Job():
         with _signals.blocked([signal.SIGINT, signal.SIGTSTP, signal.SIGTERM], ignore=False):
             self._process.start()
 
-        # Wait for it to complete
+        # Wait for the child task to complete.
+        #
+        # This is a tricky part of python which doesnt seem to
+        # make it to the online docs:
+        #
+        #  o asyncio.get_child_watcher() will return a SafeChildWatcher() instance
+        #    which is the default type of watcher, and the instance belongs to the
+        #    "event loop policy" in use (so there is only one in the main process).
+        #
+        #  o SafeChildWatcher() will register a SIGCHLD handler with the asyncio
+        #    loop, and will selectively reap any child pids which have been
+        #    terminated.
+        #
+        #  o At registration time, the process will immediately be checked with
+        #    `os.waitpid()` and will be reaped immediately, before add_child_handler()
+        #    returns.
+        #
+        # The self._parent_child_completed callback passed here will normally
+        # be called after the child task has been reaped with `os.waitpid()`, in
+        # an event loop callback. Otherwise, if the job completes too fast, then
+        # the callback is called immediately.
+        #
         self._watcher = asyncio.get_child_watcher()
         self._watcher.add_child_handler(self._process.pid, self._parent_child_completed)
 
diff --git a/buildstream/_scheduler/pullqueue.py b/buildstream/_scheduler/pullqueue.py
index f9928a342..b4f5b0d73 100644
--- a/buildstream/_scheduler/pullqueue.py
+++ b/buildstream/_scheduler/pullqueue.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
diff --git a/buildstream/_scheduler/pushqueue.py b/buildstream/_scheduler/pushqueue.py
index 8a68d5953..624eefd1d 100644
--- a/buildstream/_scheduler/pushqueue.py
+++ b/buildstream/_scheduler/pushqueue.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
diff --git a/buildstream/_scheduler/queue.py b/buildstream/_scheduler/queue.py
index 7c4ad6919..15caf8348 100644
--- a/buildstream/_scheduler/queue.py
+++ b/buildstream/_scheduler/queue.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
@@ -269,10 +268,11 @@ class Queue():
         # Handle any workspace modifications now
         #
         if job.workspace_dict:
-            project = element._get_project()
-            if project.workspaces.update_workspace(element.name, job.workspace_dict):
+            context = element._get_context()
+            workspaces = context.get_workspaces()
+            if workspaces.update_workspace(element._get_full_name(), job.workspace_dict):
                 try:
-                    project.workspaces.save_config()
+                    workspaces.save_config()
                 except BstError as e:
                     self._message(element, MessageType.ERROR, "Error saving workspaces", detail=str(e))
                 except Exception as e:   # pylint: disable=broad-except
diff --git a/buildstream/_scheduler/scheduler.py b/buildstream/_scheduler/scheduler.py
index 25e1e6790..f8a66ae92 100644
--- a/buildstream/_scheduler/scheduler.py
+++ b/buildstream/_scheduler/scheduler.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
diff --git a/buildstream/_scheduler/trackqueue.py b/buildstream/_scheduler/trackqueue.py
index 2e7bc8b97..e48e1ae28 100644
--- a/buildstream/_scheduler/trackqueue.py
+++ b/buildstream/_scheduler/trackqueue.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
diff --git a/buildstream/_signals.py b/buildstream/_signals.py
index 06849c9f9..0035485a5 100644
--- a/buildstream/_signals.py
+++ b/buildstream/_signals.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Codethink Limited
 #
diff --git a/buildstream/_site.py b/buildstream/_site.py
index f4780ef3d..ff169180f 100644
--- a/buildstream/_site.py
+++ b/buildstream/_site.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
diff --git a/buildstream/_sourcefactory.py b/buildstream/_sourcefactory.py
index dad3ddf68..88a130e10 100644
--- a/buildstream/_sourcefactory.py
+++ b/buildstream/_sourcefactory.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
diff --git a/buildstream/_stream.py b/buildstream/_stream.py
index f2806b4c8..5c0976bd7 100644
--- a/buildstream/_stream.py
+++ b/buildstream/_stream.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2018 Codethink Limited
 #
@@ -388,13 +387,13 @@ class Stream():
             with target._prepare_sandbox(Scope.RUN, None, integrate=integrate) as sandbox:
 
                 # Copy or move the sandbox to the target directory
-                sandbox_root = sandbox.get_directory()
+                sandbox_vroot = sandbox.get_virtual_directory()
                 with target.timed_activity("Checking out files in {}".format(directory)):
                     try:
                         if hardlinks:
-                            self._checkout_hardlinks(sandbox_root, directory)
+                            self._checkout_hardlinks(sandbox_vroot, directory)
                         else:
-                            utils.copy_files(sandbox_root, directory)
+                            sandbox_vroot.export_files(directory)
                     except OSError as e:
                         raise StreamError("Failed to checkout files: {}".format(e)) from e
         except BstError as e:
@@ -434,8 +433,10 @@ class Stream():
             detail += "  \n".join(build_depends)
             raise StreamError("The given element has no sources", detail=detail)
 
+        workspaces = self._context.get_workspaces()
+
         # Check for workspace config
-        workspace = self._project.workspaces.get_workspace(target.name)
+        workspace = workspaces.get_workspace(target._get_full_name())
         if workspace:
             raise StreamError("Workspace '{}' is already defined at: {}"
                               .format(target.name, workspace.path))
@@ -460,13 +461,13 @@ class Stream():
         except OSError as e:
             raise StreamError("Failed to create workspace directory: {}".format(e)) from e
 
-        self._project.workspaces.create_workspace(target.name, workdir)
+        workspaces.create_workspace(target._get_full_name(), workdir)
 
         if not no_checkout:
             with target.timed_activity("Staging sources to {}".format(directory)):
                 target._open_workspace()
 
-        self._project.workspaces.save_config()
+        workspaces.save_config()
         self._message(MessageType.INFO, "Saved workspace configuration")
 
     # workspace_close
@@ -478,7 +479,8 @@ class Stream():
     #    remove_dir (bool): Whether to remove the associated directory
     #
     def workspace_close(self, element_name, *, remove_dir):
-        workspace = self._project.workspaces.get_workspace(element_name)
+        workspaces = self._context.get_workspaces()
+        workspace = workspaces.get_workspace(element_name)
 
         # Remove workspace directory if prompted
         if remove_dir:
@@ -491,8 +493,8 @@ class Stream():
                                       .format(workspace.path, e)) from e
 
         # Delete the workspace and save the configuration
-        self._project.workspaces.delete_workspace(element_name)
-        self._project.workspaces.save_config()
+        workspaces.delete_workspace(element_name)
+        workspaces.save_config()
         self._message(MessageType.INFO, "Closed workspace for {}".format(element_name))
 
     # workspace_reset
@@ -525,8 +527,10 @@ class Stream():
         if track_first:
             self._fetch(elements, track_elements=track_elements)
 
+        workspaces = self._context.get_workspaces()
+
         for element in elements:
-            workspace = self._project.workspaces.get_workspace(element.name)
+            workspace = workspaces.get_workspace(element._get_full_name())
 
             if soft:
                 workspace.prepared = False
@@ -542,15 +546,15 @@ class Stream():
                     raise StreamError("Could not remove  '{}': {}"
                                       .format(workspace.path, e)) from e
 
-            self._project.workspaces.delete_workspace(element.name)
-            self._project.workspaces.create_workspace(element.name, workspace.path)
+            workspaces.delete_workspace(element._get_full_name())
+            workspaces.create_workspace(element._get_full_name(), workspace.path)
 
             with element.timed_activity("Staging sources to {}".format(workspace.path)):
                 element._open_workspace()
 
             self._message(MessageType.INFO, "Reset workspace for {} at: {}".format(element.name, workspace.path))
 
-        self._project.workspaces.save_config()
+        workspaces.save_config()
 
     # workspace_exists
     #
@@ -566,11 +570,12 @@ class Stream():
     # True if there are any existing workspaces.
     #
     def workspace_exists(self, element_name=None):
+        workspaces = self._context.get_workspaces()
         if element_name:
-            workspace = self._project.workspaces.get_workspace(element_name)
+            workspace = workspaces.get_workspace(element_name)
             if workspace:
                 return True
-        elif any(self._project.workspaces.list()):
+        elif any(workspaces.list()):
             return True
 
         return False
@@ -581,7 +586,7 @@ class Stream():
     #
     def workspace_list(self):
         workspaces = []
-        for element_name, workspace_ in self._project.workspaces.list():
+        for element_name, workspace_ in self._context.get_workspaces().list():
             workspace_detail = {
                 'element': element_name,
                 'directory': workspace_.path,
@@ -825,12 +830,30 @@ class Stream():
         # done before resolving element states.
         #
         assert track_selection != PipelineSelection.PLAN
-        track_selected = self._pipeline.get_selection(track_elements, track_selection)
+
+        # Tracked elements are split by owner projects in order to
+        # filter cross junctions tracking dependencies on their
+        # respective project.
+        track_projects = {}
+        for element in track_elements:
+            project = element._get_project()
+            if project not in track_projects:
+                track_projects[project] = [element]
+            else:
+                track_projects[project].append(element)
+
+        track_selected = []
+
+        for project, project_elements in track_projects.items():
+            selected = self._pipeline.get_selection(project_elements, track_selection)
+            selected = self._pipeline.track_cross_junction_filter(project,
+                                                                  selected,
+                                                                  track_cross_junctions)
+            track_selected.extend(selected)
+
         track_selected = self._pipeline.except_elements(track_elements,
                                                         track_selected,
                                                         track_except_elements)
-        track_selected = self._pipeline.track_cross_junction_filter(track_selected,
-                                                                    track_cross_junctions)
 
         for element in track_selected:
             element._schedule_tracking()
@@ -967,22 +990,17 @@ class Stream():
 
     # Helper function for checkout()
     #
-    def _checkout_hardlinks(self, sandbox_root, directory):
+    def _checkout_hardlinks(self, sandbox_vroot, directory):
         try:
             removed = utils.safe_remove(directory)
         except OSError as e:
             raise StreamError("Failed to remove checkout directory: {}".format(e)) from e
 
         if removed:
-            # Try a simple rename of the sandbox root; if that
-            # doesnt cut it, then do the regular link files code path
-            try:
-                os.rename(sandbox_root, directory)
-            except OSError:
-                os.makedirs(directory, exist_ok=True)
-                utils.link_files(sandbox_root, directory)
+            os.makedirs(directory, exist_ok=True)
+            sandbox_vroot.export_files(directory, can_link=True, can_destroy=True)
         else:
-            utils.link_files(sandbox_root, directory)
+            sandbox_vroot.export_files(directory, can_link=True, can_destroy=False)
 
     # Write the element build script to the given directory
     def _write_element_script(self, directory, element):
diff --git a/buildstream/_variables.py b/buildstream/_variables.py
index b4c920a71..8299f1c1e 100644
--- a/buildstream/_variables.py
+++ b/buildstream/_variables.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
diff --git a/buildstream/_versions.py b/buildstream/_versions.py
index 6b4e3bddb..4b1bc7ec2 100644
--- a/buildstream/_versions.py
+++ b/buildstream/_versions.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2018 Codethink Limited
 #
@@ -24,7 +23,7 @@
 # This version is bumped whenever enhancements are made
 # to the `project.conf` format or the core element format.
 #
-BST_FORMAT_VERSION = 8
+BST_FORMAT_VERSION = 9
 
 
 # The base BuildStream artifact version
diff --git a/buildstream/_workspaces.py b/buildstream/_workspaces.py
index f6cdeb88b..3f474b8ca 100644
--- a/buildstream/_workspaces.py
+++ b/buildstream/_workspaces.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2018 Codethink Limited
 #
@@ -46,7 +45,7 @@ _WORKSPACE_MEMBERS = [
 # methods.
 #
 # Args:
-#    project (Project): The project this workspace is part of
+#    toplevel_project (Project): Top project. Will be used for resolving relative workspace paths.
 #    path (str): The path that should host this workspace
 #    last_successful (str): The key of the last successful build of this workspace
 #    running_files (dict): A dict mapping dependency elements to files
@@ -54,13 +53,13 @@ _WORKSPACE_MEMBERS = [
 #                          made obsolete with failed build artifacts.
 #
 class Workspace():
-    def __init__(self, project, *, last_successful=None, path=None, prepared=False, running_files=None):
+    def __init__(self, toplevel_project, *, last_successful=None, path=None, prepared=False, running_files=None):
         self.prepared = prepared
         self.last_successful = last_successful
         self.path = path
         self.running_files = running_files if running_files is not None else {}
 
-        self._project = project
+        self._toplevel_project = toplevel_project
         self._key = None
 
     # to_dict()
@@ -81,17 +80,17 @@ class Workspace():
     # when loading from a YAML file.
     #
     # Args:
-    #    project (Project): The Project to load this for
+    #    toplevel_project (Project): Top project. Will be used for resolving relative workspace paths.
     #    dictionary: A simple dictionary object
     #
     # Returns:
     #    (Workspace): A newly instantiated Workspace
     #
     @classmethod
-    def from_dict(cls, project, dictionary):
+    def from_dict(cls, toplevel_project, dictionary):
 
         # Just pass the dictionary as kwargs
-        return cls(project, **dictionary)
+        return cls(toplevel_project, **dictionary)
 
     # differs()
     #
@@ -201,7 +200,7 @@ class Workspace():
     # Returns: The absolute path of the element's workspace.
     #
     def get_absolute_path(self):
-        return os.path.join(self._project.directory, self.path)
+        return os.path.join(self._toplevel_project.directory, self.path)
 
 
 # Workspaces()
@@ -209,11 +208,12 @@ class Workspace():
 # A class to manage Workspaces for multiple elements.
 #
 # Args:
-#    project (Project): The project the workspaces should be associated to
+#    toplevel_project (Project): Top project used to resolve paths.
 #
 class Workspaces():
-    def __init__(self, project):
-        self._project = project
+    def __init__(self, toplevel_project):
+        self._toplevel_project = toplevel_project
+        self._bst_directory = os.path.join(toplevel_project.directory, ".bst")
         self._workspaces = self._load_config()
 
     # list()
@@ -236,7 +236,7 @@ class Workspaces():
     #    path (str) - The path in which the workspace should be kept
     #
     def create_workspace(self, element_name, path):
-        self._workspaces[element_name] = Workspace(self._project, path=path)
+        self._workspaces[element_name] = Workspace(self._toplevel_project, path=path)
 
         return self._workspaces[element_name]
 
@@ -270,7 +270,7 @@ class Workspaces():
     def update_workspace(self, element_name, workspace_dict):
         assert element_name in self._workspaces
 
-        workspace = Workspace.from_dict(self._project, workspace_dict)
+        workspace = Workspace.from_dict(self._toplevel_project, workspace_dict)
         if self._workspaces[element_name].differs(workspace):
             self._workspaces[element_name] = workspace
             return True
@@ -305,9 +305,9 @@ class Workspaces():
                 for element, workspace in _yaml.node_items(self._workspaces)
             }
         }
-        os.makedirs(os.path.join(self._project.directory, ".bst"), exist_ok=True)
+        os.makedirs(self._bst_directory, exist_ok=True)
         _yaml.dump(_yaml.node_sanitize(config),
-                   os.path.join(self._project.directory, ".bst", "workspaces.yml"))
+                   self._get_filename())
 
     # _load_config()
     #
@@ -319,7 +319,7 @@ class Workspaces():
     # Raises: LoadError if there was a problem with the workspace config
     #
     def _load_config(self):
-        workspace_file = os.path.join(self._project.directory, ".bst", "workspaces.yml")
+        workspace_file = self._get_filename()
         try:
             node = _yaml.load(workspace_file)
         except LoadError as e:
@@ -361,8 +361,7 @@ class Workspaces():
                                  "This is not supported anymore.\n" + \
                                  "Please remove this element from '{}'."
                         raise LoadError(LoadErrorReason.INVALID_DATA,
-                                        detail.format(element,
-                                                      os.path.join(self._project.directory, ".bst", "workspaces.yml")))
+                                        detail.format(element, self._get_filename()))
 
                     workspaces[element] = sources[0][1]
 
@@ -371,13 +370,13 @@ class Workspaces():
                                     "Workspace config is in unexpected format.")
 
             res = {
-                element: Workspace(self._project, path=config)
+                element: Workspace(self._toplevel_project, path=config)
                 for element, config in _yaml.node_items(workspaces)
             }
 
         elif version >= 1 and version <= BST_WORKSPACE_FORMAT_VERSION:
             workspaces = _yaml.node_get(workspaces, dict, "workspaces", default_value={})
-            res = {element: self._load_workspace(self._project, node)
+            res = {element: self._load_workspace(node)
                    for element, node in _yaml.node_items(workspaces)}
 
         else:
@@ -394,16 +393,24 @@ class Workspaces():
     #
     # Args:
     #    node: A YAML Node
-    #    project (Project): The Project to load this for
     #
     # Returns:
     #    (Workspace): A newly instantiated Workspace
     #
-    def _load_workspace(self, project, node):
+    def _load_workspace(self, node):
         dictionary = {
             'prepared': _yaml.node_get(node, bool, 'prepared', default_value=False),
             'path': _yaml.node_get(node, str, 'path'),
             'last_successful': _yaml.node_get(node, str, 'last_successful', default_value=None),
             'running_files': _yaml.node_get(node, dict, 'running_files', default_value=None),
         }
-        return Workspace.from_dict(self._project, dictionary)
+        return Workspace.from_dict(self._toplevel_project, dictionary)
+
+    # _get_filename():
+    #
+    # Get the workspaces.yml file path.
+    #
+    # Returns:
+    #    (str): The path to workspaces.yml file.
+    def _get_filename(self):
+        return os.path.join(self._bst_directory, "workspaces.yml")
diff --git a/buildstream/_yaml.py b/buildstream/_yaml.py
index 8954c7da9..0e090e2e7 100644
--- a/buildstream/_yaml.py
+++ b/buildstream/_yaml.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
@@ -185,6 +184,10 @@ def load(filename, shortname=None, copy_tree=False):
     except FileNotFoundError as e:
         raise LoadError(LoadErrorReason.MISSING_FILE,
                         "Could not find file at {}".format(filename)) from e
+    except IsADirectoryError as e:
+        raise LoadError(LoadErrorReason.LOADING_DIRECTORY,
+                        "{} is a directory. bst command expects a .bst file."
+                        .format(filename)) from e
 
 
 # Like load(), but doesnt require the data to be in a file
diff --git a/buildstream/buildelement.py b/buildstream/buildelement.py
index 5f9a856d1..ec05acbeb 100644
--- a/buildstream/buildelement.py
+++ b/buildstream/buildelement.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
@@ -18,8 +17,8 @@
 #  Authors:
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 """
-BuildElement
-============
+BuildElement - Abstract class for build elements
+================================================
 The BuildElement class is a convenience element one can derive from for
 implementing the most common case of element.
 
diff --git a/buildstream/element.py b/buildstream/element.py
index 832f0dd93..6f06cbb1c 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016-2018 Codethink Limited
 #
@@ -19,8 +18,8 @@
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 
 """
-Element
-=======
+Element - Base element class
+============================
 
 
 .. _core_element_abstract_methods:
@@ -66,7 +65,7 @@ Miscellaneous abstract methods also exist:
 
 * :func:`Element.generate_script() <buildstream.element.Element.generate_script>`
 
-  For the purpose of ``bst source bundle``, an Element may optionally implmenent this.
+  For the purpose of ``bst source bundle``, an Element may optionally implement this.
 
 
 Class Reference
@@ -81,7 +80,6 @@ from collections import Mapping, OrderedDict
 from contextlib import contextmanager
 from enum import Enum
 import tempfile
-import time
 import shutil
 
 from . import _yaml
@@ -98,6 +96,9 @@ from . import _site
 from ._platform import Platform
 from .sandbox._config import SandboxConfig
 
+from .storage.directory import Directory
+from .storage._filebaseddirectory import FileBasedDirectory, VirtualDirectoryError
+
 
 # _KeyStrength():
 #
@@ -192,6 +193,13 @@ class Element(Plugin):
     *Since: 1.2*
     """
 
+    BST_VIRTUAL_DIRECTORY = False
+    """Whether to raise exceptions if an element uses Sandbox.get_directory
+    instead of Sandbox.get_virtual_directory.
+
+    *Since: 1.2*
+    """
+
     def __init__(self, context, project, artifacts, meta, plugin_conf):
 
         super().__init__(meta.name, context, project, meta.provenance, "element")
@@ -620,10 +628,10 @@ class Element(Plugin):
 
             # Hard link it into the staging area
             #
-            basedir = sandbox.get_directory()
-            stagedir = basedir \
+            vbasedir = sandbox.get_virtual_directory()
+            vstagedir = vbasedir \
                 if path is None \
-                else os.path.join(basedir, path.lstrip(os.sep))
+                else vbasedir.descend(path.lstrip(os.sep).split(os.sep))
 
             files = list(self.__compute_splits(include, exclude, orphans))
 
@@ -635,15 +643,8 @@ class Element(Plugin):
                 link_files = files
                 copy_files = []
 
-            link_result = utils.link_files(artifact, stagedir, files=link_files,
-                                           report_written=True)
-            copy_result = utils.copy_files(artifact, stagedir, files=copy_files,
-                                           report_written=True)
-
-            cur_time = time.time()
-
-            for f in copy_result.files_written:
-                os.utime(os.path.join(stagedir, f), times=(cur_time, cur_time))
+            link_result = vstagedir.import_files(artifact, files=link_files, report_written=True, can_link=True)
+            copy_result = vstagedir.import_files(artifact, files=copy_files, report_written=True, update_utimes=True)
 
         return link_result.combine(copy_result)
 
@@ -673,7 +674,6 @@ class Element(Plugin):
         overlaps = OrderedDict()
         files_written = {}
         old_dep_keys = {}
-        project = self._get_project()
         workspace = self._get_workspace()
 
         if self.__can_build_incrementally() and workspace.last_successful:
@@ -702,7 +702,7 @@ class Element(Plugin):
                     # In case we are running `bst shell`, this happens in the
                     # main process and we need to update the workspace config
                     if utils._is_main_process():
-                        project.workspaces.save_config()
+                        self._get_context().get_workspaces().save_config()
 
             result = dep.stage_artifact(sandbox,
                                         path=path,
@@ -1157,7 +1157,7 @@ class Element(Plugin):
     def _preflight(self):
 
         if self.BST_FORBID_RDEPENDS and self.BST_FORBID_BDEPENDS:
-            if any(self.dependencies(Scope.RUN, recurse=False)) or any(self.dependencies(Scope.RUN, recurse=False)):
+            if any(self.dependencies(Scope.RUN, recurse=False)) or any(self.dependencies(Scope.BUILD, recurse=False)):
                 raise ElementError("{}: Dependencies are forbidden for '{}' elements"
                                    .format(self, self.get_kind()), reason="element-forbidden-depends")
 
@@ -1288,8 +1288,8 @@ class Element(Plugin):
             sandbox._set_mount_source(directory, workspace.get_absolute_path())
 
         # Stage all sources that need to be copied
-        sandbox_root = sandbox.get_directory()
-        host_directory = os.path.join(sandbox_root, directory.lstrip(os.sep))
+        sandbox_vroot = sandbox.get_virtual_directory()
+        host_directory = sandbox_vroot.descend(directory.lstrip(os.sep).split(os.sep), create=True)
         self._stage_sources_at(host_directory, mount_workspaces=mount_workspaces)
 
     # _stage_sources_at():
@@ -1300,28 +1300,33 @@ class Element(Plugin):
     #     directory (str): An absolute path to stage the sources at
     #     mount_workspaces (bool): mount workspaces if True, copy otherwise
     #
-    def _stage_sources_at(self, directory, mount_workspaces=True):
+    def _stage_sources_at(self, vdirectory, mount_workspaces=True):
         with self.timed_activity("Staging sources", silent_nested=True):
 
-            if os.path.isdir(directory) and os.listdir(directory):
-                raise ElementError("Staging directory '{}' is not empty".format(directory))
-
-            workspace = self._get_workspace()
-            if workspace:
-                # If mount_workspaces is set and we're doing incremental builds,
-                # the workspace is already mounted into the sandbox.
-                if not (mount_workspaces and self.__can_build_incrementally()):
-                    with self.timed_activity("Staging local files at {}".format(workspace.path)):
-                        workspace.stage(directory)
-            else:
-                # No workspace, stage directly
-                for source in self.sources():
-                    source._stage(directory)
-
+            if not isinstance(vdirectory, Directory):
+                vdirectory = FileBasedDirectory(vdirectory)
+            if not vdirectory.is_empty():
+                raise ElementError("Staging directory '{}' is not empty".format(vdirectory))
+
+            with tempfile.TemporaryDirectory() as temp_staging_directory:
+
+                workspace = self._get_workspace()
+                if workspace:
+                    # If mount_workspaces is set and we're doing incremental builds,
+                    # the workspace is already mounted into the sandbox.
+                    if not (mount_workspaces and self.__can_build_incrementally()):
+                        with self.timed_activity("Staging local files at {}".format(workspace.path)):
+                            workspace.stage(temp_staging_directory)
+                else:
+                    # No workspace, stage directly
+                    for source in self.sources():
+                        source._stage(temp_staging_directory)
+
+                vdirectory.import_files(temp_staging_directory, None)
         # Ensure deterministic mtime of sources at build time
-        utils._set_deterministic_mtime(directory)
+        vdirectory.set_deterministic_mtime()
         # Ensure deterministic owners of sources at build time
-        utils._set_deterministic_user(directory)
+        vdirectory.set_deterministic_user()
 
     # _set_required():
     #
@@ -1393,12 +1398,11 @@ class Element(Plugin):
             # For this reason, it is safe to update and
             # save the workspaces configuration
             #
-            project = self._get_project()
             key = self._get_cache_key()
             workspace = self._get_workspace()
             workspace.last_successful = key
             workspace.clear_running_files()
-            project.workspaces.save_config()
+            self._get_context().get_workspaces().save_config()
 
     # _assemble():
     #
@@ -1428,7 +1432,7 @@ class Element(Plugin):
             with _signals.terminator(cleanup_rootdir), \
                 self.__sandbox(rootdir, output_file, output_file, self.__sandbox_config) as sandbox:  # nopep8
 
-                sandbox_root = sandbox.get_directory()
+                sandbox_vroot = sandbox.get_virtual_directory()
 
                 # By default, the dynamic public data is the same as the static public data.
                 # The plugin's assemble() method may modify this, though.
@@ -1458,23 +1462,24 @@ class Element(Plugin):
                     #
                     workspace = self._get_workspace()
                     if workspace and self.__staged_sources_directory:
-                        sandbox_root = sandbox.get_directory()
-                        sandbox_path = os.path.join(sandbox_root,
-                                                    self.__staged_sources_directory.lstrip(os.sep))
+                        sandbox_vroot = sandbox.get_virtual_directory()
+                        path_components = self.__staged_sources_directory.lstrip(os.sep).split(os.sep)
+                        sandbox_vpath = sandbox_vroot.descend(path_components)
                         try:
-                            utils.copy_files(workspace.path, sandbox_path)
+                            sandbox_vpath.import_files(workspace.path)
                         except UtilError as e:
                             self.warn("Failed to preserve workspace state for failed build sysroot: {}"
                                       .format(e))
 
                     raise
 
-                collectdir = os.path.join(sandbox_root, collect.lstrip(os.sep))
-                if not os.path.exists(collectdir):
+                try:
+                    collectvdir = sandbox_vroot.descend(collect.lstrip(os.sep).split(os.sep))
+                except VirtualDirectoryError:
                     raise ElementError(
-                        "Directory '{}' was not found inside the sandbox, "
+                        "Subdirectory '{}' of '{}' does not exist following assembly, "
                         "unable to collect artifact contents"
-                        .format(collect))
+                        .format(collect, sandbox_vroot))
 
                 # At this point, we expect an exception was raised leading to
                 # an error message, or we have good output to collect.
@@ -1490,7 +1495,7 @@ class Element(Plugin):
                 os.mkdir(metadir)
 
                 # Hard link files from collect dir to files directory
-                utils.link_files(collectdir, filesdir)
+                collectvdir.export_files(filesdir, can_link=True)
 
                 # Copy build log
                 if self.__log_path:
@@ -1763,8 +1768,8 @@ class Element(Plugin):
     #    (Workspace|None): A workspace associated with this element
     #
     def _get_workspace(self):
-        project = self._get_project()
-        return project.workspaces.get_workspace(self.name)
+        workspaces = self._get_context().get_workspaces()
+        return workspaces.get_workspace(self._get_full_name())
 
     # _write_script():
     #
@@ -1932,7 +1937,7 @@ class Element(Plugin):
                 'execution-environment': self.__sandbox_config.get_unique_key(),
                 'environment': cache_env,
                 'sources': [s._get_unique_key(workspace is None) for s in self.__sources],
-                'workspace': '' if workspace is None else workspace.get_key(),
+                'workspace': '' if workspace is None else workspace.get_key(self._get_project()),
                 'public': self.__public,
                 'cache': type(self.__artifacts).__name__
             }
@@ -2081,7 +2086,8 @@ class Element(Plugin):
                                               directory,
                                               stdout=stdout,
                                               stderr=stderr,
-                                              config=config)
+                                              config=config,
+                                              allow_real_directory=not self.BST_VIRTUAL_DIRECTORY)
             yield sandbox
 
         else:
diff --git a/buildstream/plugin.py b/buildstream/plugin.py
index b208e2d06..29fe2cb11 100644
--- a/buildstream/plugin.py
+++ b/buildstream/plugin.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Codethink Limited
 #
@@ -18,8 +17,8 @@
 #  Authors:
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 """
-Plugin
-======
+Plugin - Base plugin class
+==========================
 BuildStream supports third party plugins to define additional kinds of
 :mod:`Elements <buildstream.element>` and :mod:`Sources <buildstream.source>`.
 
diff --git a/buildstream/plugins/elements/autotools.py b/buildstream/plugins/elements/autotools.py
index 5f54c3953..14d04d9a3 100644
--- a/buildstream/plugins/elements/autotools.py
+++ b/buildstream/plugins/elements/autotools.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016, 2018 Codethink Limited
 #
@@ -18,8 +17,9 @@
 #  Authors:
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 
-"""Autotools build element
-
+"""
+autotools - Autotools build element
+===================================
 This is a :mod:`BuildElement <buildstream.buildelement>` implementation for
 using Autotools build scripts (also known as the `GNU Build System
 <https://en.wikipedia.org/wiki/GNU_Build_System>`_).
diff --git a/buildstream/plugins/elements/autotools.yaml b/buildstream/plugins/elements/autotools.yaml
index 97ab0664c..021d3815c 100644
--- a/buildstream/plugins/elements/autotools.yaml
+++ b/buildstream/plugins/elements/autotools.yaml
@@ -4,10 +4,12 @@ variables:
 
   autogen: |
     export NOCONFIGURE=1;
-    if [ -e autogen ]; then ./autogen;
-    elif [ -e autogen.sh ]; then ./autogen.sh;
-    elif [ -e bootstrap ]; then ./bootstrap;
-    elif [ -e bootstrap.sh ]; then ./bootstrap.sh;
+
+    if [ -x %{conf-cmd} ]; then true;
+    elif [ -x autogen ]; then ./autogen;
+    elif [ -x autogen.sh ]; then ./autogen.sh;
+    elif [ -x bootstrap ]; then ./bootstrap;
+    elif [ -x bootstrap.sh ]; then ./bootstrap.sh;
     else autoreconf -ivf;
     fi
 
diff --git a/buildstream/plugins/elements/cmake.py b/buildstream/plugins/elements/cmake.py
index 292785e81..8126a80ac 100644
--- a/buildstream/plugins/elements/cmake.py
+++ b/buildstream/plugins/elements/cmake.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016, 2018 Codethink Limited
 #
@@ -18,8 +17,9 @@
 #  Authors:
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 
-"""CMake build element
-
+"""
+cmake - CMake build element
+===========================
 This is a :mod:`BuildElement <buildstream.buildelement>` implementation for
 using the `CMake <https://cmake.org/>`_ build system.
 
diff --git a/buildstream/plugins/elements/compose.py b/buildstream/plugins/elements/compose.py
index 0e666c6e5..2a5979828 100644
--- a/buildstream/plugins/elements/compose.py
+++ b/buildstream/plugins/elements/compose.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Codethink Limited
 #
@@ -18,8 +17,9 @@
 #  Authors:
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 
-"""Compose element
-
+"""
+compose - Compose the output of multiple elements
+=================================================
 This element creates a selective composition of its dependencies.
 
 This is normally used at near the end of a pipeline to prepare
@@ -34,7 +34,6 @@ The default configuration and possible options are as such:
 """
 
 import os
-from buildstream import utils
 from buildstream import Element, Scope
 
 
@@ -56,6 +55,9 @@ class ComposeElement(Element):
     # added, to reduce the potential for confusion
     BST_FORBID_SOURCES = True
 
+    # This plugin has been modified to avoid the use of Sandbox.get_directory
+    BST_VIRTUAL_DIRECTORY = True
+
     def configure(self, node):
         self.node_validate(node, [
             'integrate', 'include', 'exclude', 'include-orphans'
@@ -104,7 +106,8 @@ class ComposeElement(Element):
                                                  orphans=self.include_orphans)
                     manifest.update(files)
 
-        basedir = sandbox.get_directory()
+        # Make a snapshot of all the files.
+        vbasedir = sandbox.get_virtual_directory()
         modified_files = set()
         removed_files = set()
         added_files = set()
@@ -116,38 +119,24 @@ class ComposeElement(Element):
                 if require_split:
 
                     # Make a snapshot of all the files before integration-commands are run.
-                    snapshot = {
-                        f: getmtime(os.path.join(basedir, f))
-                        for f in utils.list_relative_paths(basedir)
-                    }
+                    snapshot = set(vbasedir.list_relative_paths())
+                    vbasedir.mark_unmodified()
 
                 for dep in self.dependencies(Scope.BUILD):
                     dep.integrate(sandbox)
 
                 if require_split:
-
                     # Calculate added, modified and removed files
-                    basedir_contents = set(utils.list_relative_paths(basedir))
+                    post_integration_snapshot = vbasedir.list_relative_paths()
+                    modified_files = set(vbasedir.list_modified_paths())
+                    basedir_contents = set(post_integration_snapshot)
                     for path in manifest:
-                        if path in basedir_contents:
-                            if path in snapshot:
-                                preintegration_mtime = snapshot[path]
-                                if preintegration_mtime != getmtime(os.path.join(basedir, path)):
-                                    modified_files.add(path)
-                            else:
-                                # If the path appears in the manifest but not the initial snapshot,
-                                # it may be a file staged inside a directory symlink. In this case
-                                # the path we got from the manifest won't show up in the snapshot
-                                # because utils.list_relative_paths() doesn't recurse into symlink
-                                # directories.
-                                pass
-                        elif path in snapshot:
+                        if path in snapshot and path not in basedir_contents:
                             removed_files.add(path)
 
                     for path in basedir_contents:
                         if path not in snapshot:
                             added_files.add(path)
-
                     self.info("Integration modified {}, added {} and removed {} files"
                               .format(len(modified_files), len(added_files), len(removed_files)))
 
@@ -166,8 +155,7 @@ class ComposeElement(Element):
         # instead of into a subdir. The element assemble() method should
         # support this in some way.
         #
-        installdir = os.path.join(basedir, 'buildstream', 'install')
-        os.makedirs(installdir, exist_ok=True)
+        installdir = vbasedir.descend(['buildstream', 'install'], create=True)
 
         # We already saved the manifest for created files in the integration phase,
         # now collect the rest of the manifest.
@@ -191,7 +179,7 @@ class ComposeElement(Element):
 
         with self.timed_activity("Creating composition", detail=detail, silent_nested=True):
             self.info("Composing {} files".format(len(manifest)))
-            utils.link_files(basedir, installdir, files=manifest)
+            installdir.import_files(vbasedir, files=manifest, can_link=True)
 
         # And we're done
         return os.path.join(os.sep, 'buildstream', 'install')
diff --git a/buildstream/plugins/elements/distutils.py b/buildstream/plugins/elements/distutils.py
index 948e08b62..5201013c1 100644
--- a/buildstream/plugins/elements/distutils.py
+++ b/buildstream/plugins/elements/distutils.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
@@ -18,8 +17,9 @@
 #  Authors:
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 
-"""Python Distutils element
-
+"""
+distutils - Python distutils element
+====================================
 A :mod:`BuildElement <buildstream.buildelement>` implementation for using
 python distutils
 
diff --git a/buildstream/plugins/elements/filter.py b/buildstream/plugins/elements/filter.py
index 8ce16ff9f..22fddd14f 100644
--- a/buildstream/plugins/elements/filter.py
+++ b/buildstream/plugins/elements/filter.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2018 Codethink Limited
 #
@@ -18,8 +17,9 @@
 #  Authors:
 #        Jonathan Maw <jonathan.maw@codethink.co.uk>
 
-"""Filter element
-
+"""
+filter - Extract a subset of files from another element
+=======================================================
 This filters another element by producing an output that is a subset of
 the filtered element.
 
diff --git a/buildstream/plugins/elements/import.py b/buildstream/plugins/elements/import.py
index 747455d70..11c3f9a6a 100644
--- a/buildstream/plugins/elements/import.py
+++ b/buildstream/plugins/elements/import.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
@@ -18,8 +17,9 @@
 #  Authors:
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 
-"""Import element
-
+"""
+import - Import sources directly
+================================
 Import elements produce artifacts directly from its sources
 without any kind of processing. These are typically used to
 import an SDK to build on top of or to overlay your build with
@@ -31,7 +31,6 @@ The empty configuration is as such:
 """
 
 import os
-import shutil
 from buildstream import Element, BuildElement, ElementError
 
 
@@ -68,27 +67,22 @@ class ImportElement(BuildElement):
         # Do not mount workspaces as the files are copied from outside the sandbox
         self._stage_sources_in_sandbox(sandbox, 'input', mount_workspaces=False)
 
-        rootdir = sandbox.get_directory()
-        inputdir = os.path.join(rootdir, 'input')
-        outputdir = os.path.join(rootdir, 'output')
+        rootdir = sandbox.get_virtual_directory()
+        inputdir = rootdir.descend(['input'])
+        outputdir = rootdir.descend(['output'], create=True)
 
         # The directory to grab
-        inputdir = os.path.join(inputdir, self.source.lstrip(os.sep))
-        inputdir = inputdir.rstrip(os.sep)
+        inputdir = inputdir.descend(self.source.strip(os.sep).split(os.sep))
 
         # The output target directory
-        outputdir = os.path.join(outputdir, self.target.lstrip(os.sep))
-        outputdir = outputdir.rstrip(os.sep)
-
-        # Ensure target directory parent
-        os.makedirs(os.path.dirname(outputdir), exist_ok=True)
+        outputdir = outputdir.descend(self.target.strip(os.sep).split(os.sep), create=True)
 
-        if not os.path.exists(inputdir):
+        if inputdir.is_empty():
             raise ElementError("{}: No files were found inside directory '{}'"
                                .format(self, self.source))
 
         # Move it over
-        shutil.move(inputdir, outputdir)
+        outputdir.import_files(inputdir)
 
         # And we're done
         return '/output'
diff --git a/buildstream/plugins/elements/junction.py b/buildstream/plugins/elements/junction.py
index 81fd57445..ee5ed24d5 100644
--- a/buildstream/plugins/elements/junction.py
+++ b/buildstream/plugins/elements/junction.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Codethink Limited
 #
@@ -18,8 +17,9 @@
 #  Authors:
 #        Jürg Billeter <juerg.billeter@codethink.co.uk>
 
-"""Junction element
-
+"""
+junction - Integrate subprojects
+================================
 This element is a link to another BuildStream project. It allows integration
 of multiple projects into a single pipeline.
 
diff --git a/buildstream/plugins/elements/make.py b/buildstream/plugins/elements/make.py
new file mode 100644
index 000000000..1f37cb412
--- /dev/null
+++ b/buildstream/plugins/elements/make.py
@@ -0,0 +1,46 @@
+#
+#  Copyright Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Ed Baunton <ebaunton1@bloomberg.net>
+
+"""
+make - Make build element
+=========================
+This is a :mod:`BuildElement <buildstream.buildelement>` implementation for
+using GNU make based build.
+
+.. note::
+
+   The ``make`` element is available since :ref:`format version 9 <project_format_version>`
+
+Here is the default configuration for the ``make`` element in full:
+
+  .. literalinclude:: ../../../buildstream/plugins/elements/make.yaml
+     :language: yaml
+"""
+
+from buildstream import BuildElement
+
+
+# Element implementation for the 'make' kind.
+class MakeElement(BuildElement):
+    pass
+
+
+# Plugin entry point
+def setup():
+    return MakeElement
diff --git a/buildstream/plugins/elements/make.yaml b/buildstream/plugins/elements/make.yaml
new file mode 100644
index 000000000..1438bb52b
--- /dev/null
+++ b/buildstream/plugins/elements/make.yaml
@@ -0,0 +1,42 @@
+# make default configurations
+
+variables:
+  make: make PREFIX="%{prefix}"
+  make-install: make -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install
+
+  # Set this if the sources cannot handle parallelization.
+  #
+  # notparallel: True
+
+config:
+
+  # Commands for building the software
+  #
+  build-commands:
+  - |
+    %{make}
+
+  # Commands for installing the software into a
+  # destination folder
+  #
+  install-commands:
+  - |
+    %{make-install}
+
+  # Commands for stripping debugging information out of
+  # installed binaries
+  #
+  strip-commands:
+  - |
+    %{strip-binaries}
+
+# Use max-jobs CPUs for building and enable verbosity
+environment:
+  MAKEFLAGS: -j%{max-jobs}
+  V: 1
+
+# And dont consider MAKEFLAGS or V as something which may
+# effect build output.
+environment-nocache:
+- MAKEFLAGS
+- V
diff --git a/buildstream/plugins/elements/makemaker.py b/buildstream/plugins/elements/makemaker.py
index 94d459f93..fccfaadab 100644
--- a/buildstream/plugins/elements/makemaker.py
+++ b/buildstream/plugins/elements/makemaker.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
@@ -18,8 +17,9 @@
 #  Authors:
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 
-"""Perl MakeMaker build element
-
+"""
+makemaker - Perl MakeMaker build element
+========================================
 A :mod:`BuildElement <buildstream.buildelement>` implementation for using
 the Perl ExtUtil::MakeMaker build system
 
diff --git a/buildstream/plugins/elements/manual.py b/buildstream/plugins/elements/manual.py
index 998394b05..c7bdba95f 100644
--- a/buildstream/plugins/elements/manual.py
+++ b/buildstream/plugins/elements/manual.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
@@ -18,8 +17,9 @@
 #  Authors:
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 
-"""Manual build element
-
+"""
+manual - Manual build element
+=============================
 The most basic build element does nothing but allows users to
 add custom build commands to the array understood by the :mod:`BuildElement <buildstream.buildelement>`
 
diff --git a/buildstream/plugins/elements/meson.py b/buildstream/plugins/elements/meson.py
index 2b7b7831a..228e90ad1 100644
--- a/buildstream/plugins/elements/meson.py
+++ b/buildstream/plugins/elements/meson.py
@@ -14,8 +14,9 @@
 #  You should have received a copy of the GNU Lesser General Public
 #  License along with this library. If not, see <http://www.gnu.org/licenses/>.
 
-"""Meson build element
-
+"""
+meson - Meson build element
+===========================
 This is a :mod:`BuildElement <buildstream.buildelement>` implementation for
 using `Meson <http://mesonbuild.com/>`_ build scripts.
 
diff --git a/buildstream/plugins/elements/modulebuild.py b/buildstream/plugins/elements/modulebuild.py
index c790cafb3..5189af1a6 100644
--- a/buildstream/plugins/elements/modulebuild.py
+++ b/buildstream/plugins/elements/modulebuild.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
@@ -18,8 +17,9 @@
 #  Authors:
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 
-"""Perl Module::Build build element
-
+"""
+modulebuild - Perl Module::Build build element
+==============================================
 A :mod:`BuildElement <buildstream.buildelement>` implementation for using
 the Perl Module::Build build system
 
diff --git a/buildstream/plugins/elements/pip.py b/buildstream/plugins/elements/pip.py
index b979a6d21..e62f713a6 100644
--- a/buildstream/plugins/elements/pip.py
+++ b/buildstream/plugins/elements/pip.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Mathieu Bridon
 #
@@ -18,8 +17,9 @@
 #  Authors:
 #        Mathieu Bridon <bochecha@daitauha.fr>
 
-"""Pip build element
-
+"""
+pip - Pip build element
+=======================
 A :mod:`BuildElement <buildstream.buildelement>` implementation for installing
 Python modules with pip
 
diff --git a/buildstream/plugins/elements/qmake.py b/buildstream/plugins/elements/qmake.py
index ab5843d8b..7896692a6 100644
--- a/buildstream/plugins/elements/qmake.py
+++ b/buildstream/plugins/elements/qmake.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
@@ -18,8 +17,9 @@
 #  Authors:
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 
-"""CMake build element
-
+"""
+qmake - QMake build element
+===========================
 A :mod:`BuildElement <buildstream.buildelement>` implementation for using
 the qmake build system
 
diff --git a/buildstream/plugins/elements/script.py b/buildstream/plugins/elements/script.py
index 6778b3fac..4e422c5db 100644
--- a/buildstream/plugins/elements/script.py
+++ b/buildstream/plugins/elements/script.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Codethink Limited
 #
@@ -19,8 +18,9 @@
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 #        Jonathan Maw <jonathan.maw@codethink.co.uk>
 
-"""Script element
-
+"""
+script - Run scripts to create output
+=====================================
 This element allows one to run some commands to mutate the
 input and create some output.
 
diff --git a/buildstream/plugins/elements/stack.py b/buildstream/plugins/elements/stack.py
index 45c49c514..d062b23bf 100644
--- a/buildstream/plugins/elements/stack.py
+++ b/buildstream/plugins/elements/stack.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
@@ -18,13 +17,13 @@
 #  Authors:
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 
-"""Stack element
-
+"""
+stack - Symbolic Element for dependency grouping
+================================================
 Stack elements are simply a symbolic element used for representing
 a logical group of elements.
 """
 
-import os
 from buildstream import Element
 
 
@@ -52,7 +51,7 @@ class StackElement(Element):
 
         # Just create a dummy empty artifact, its existence is a statement
         # that all this stack's dependencies are built.
-        rootdir = sandbox.get_directory()
+        vrootdir = sandbox.get_virtual_directory()
 
         # XXX FIXME: This is currently needed because the artifact
         #            cache wont let us commit an empty artifact.
@@ -61,10 +60,7 @@ class StackElement(Element):
         # the actual artifact data in a subdirectory, then we
         # will be able to store some additional state in the
         # artifact cache, and we can also remove this hack.
-        outputdir = os.path.join(rootdir, 'output', 'bst')
-
-        # Ensure target directory parent
-        os.makedirs(os.path.dirname(outputdir), exist_ok=True)
+        vrootdir.descend(['output', 'bst'], create=True)
 
         # And we're done
         return '/output'
diff --git a/buildstream/plugins/sources/bzr.py b/buildstream/plugins/sources/bzr.py
index 3732304fb..21121428c 100644
--- a/buildstream/plugins/sources/bzr.py
+++ b/buildstream/plugins/sources/bzr.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #  Copyright (C) 2017 Codethink Limited
 #
 #  This program is free software; you can redistribute it and/or
@@ -17,7 +16,13 @@
 #  Authors:
 #        Jonathan Maw <jonathan.maw@codethink.co.uk>
 
-"""A source implementation for staging bazaar branches
+"""
+bzr - stage files from a bazaar repository
+==========================================
+
+**Host dependencies:**
+
+  * bzr
 
 **Usage:**
 
diff --git a/buildstream/plugins/sources/deb.py b/buildstream/plugins/sources/deb.py
index daf6f94c7..1cf8beb22 100644
--- a/buildstream/plugins/sources/deb.py
+++ b/buildstream/plugins/sources/deb.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #  Copyright (C) 2017 Codethink Limited
 #
 #  This program is free software; you can redistribute it and/or
@@ -19,7 +18,13 @@
 #        Jonathan Maw <jonathan.maw@codethink.co.uk>
 #        Richard Maw <richard.maw@codethink.co.uk>
 
-"""A source implementation for staging deb files
+"""
+deb - stage files from .deb packages
+====================================
+
+**Host dependencies:**
+
+  * arpy (python package)
 
 **Usage:**
 
diff --git a/buildstream/plugins/sources/git.py b/buildstream/plugins/sources/git.py
index f178656b0..d079d8747 100644
--- a/buildstream/plugins/sources/git.py
+++ b/buildstream/plugins/sources/git.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
@@ -18,7 +17,13 @@
 #  Authors:
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 
-"""A Source implementation for staging git checkouts
+"""
+git - stage files from a git repository
+=======================================
+
+**Host dependencies:**
+
+  * git
 
 **Usage:**
 
diff --git a/buildstream/plugins/sources/local.py b/buildstream/plugins/sources/local.py
index 3193d101d..e3b019f1a 100644
--- a/buildstream/plugins/sources/local.py
+++ b/buildstream/plugins/sources/local.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
@@ -18,7 +17,9 @@
 #  Authors:
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 
-"""A Source implementation for staging local project files
+"""
+local - stage local files and directories
+=========================================
 
 **Usage:**
 
diff --git a/buildstream/plugins/sources/ostree.py b/buildstream/plugins/sources/ostree.py
index b311e24bf..94fe5093f 100644
--- a/buildstream/plugins/sources/ostree.py
+++ b/buildstream/plugins/sources/ostree.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
@@ -18,7 +17,9 @@
 #  Authors:
 #        Andrew Leeming <andrew.leeming@codethink.co.uk>
 
-"""A Source implementation for importing/staging of OSTree checkouts.
+"""
+ostree - stage files from an OSTree repository
+==============================================
 
 **Usage:**
 
diff --git a/buildstream/plugins/sources/patch.py b/buildstream/plugins/sources/patch.py
index c9e40b1e6..11b66b3ea 100644
--- a/buildstream/plugins/sources/patch.py
+++ b/buildstream/plugins/sources/patch.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright Bloomberg Finance LP
 #
@@ -18,7 +17,13 @@
 #  Authors:
 #        Chandan Singh <csingh43@bloomberg.net>
 
-"""A Source implementation for applying local patches
+"""
+patch - apply locally stored patches
+====================================
+
+**Host dependencies:**
+
+  * patch
 
 **Usage:**
 
diff --git a/buildstream/plugins/sources/tar.py b/buildstream/plugins/sources/tar.py
index e41824505..e32cc3dc8 100644
--- a/buildstream/plugins/sources/tar.py
+++ b/buildstream/plugins/sources/tar.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#
 #  Copyright (C) 2017 Codethink Limited
 #
 #  This program is free software; you can redistribute it and/or
@@ -17,7 +17,13 @@
 #  Authors:
 #        Jonathan Maw <jonathan.maw@codethink.co.uk>
 
-"""A source implementation for staging tar files
+"""
+tar - stage files from tar archives
+===================================
+
+**Host dependencies:**
+
+  * lzip (for .tar.lz files)
 
 **Usage:**
 
diff --git a/buildstream/plugins/sources/zip.py b/buildstream/plugins/sources/zip.py
index fdf8947ec..9b47d7f78 100644
--- a/buildstream/plugins/sources/zip.py
+++ b/buildstream/plugins/sources/zip.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Mathieu Bridon
 #
@@ -18,7 +17,9 @@
 #  Authors:
 #        Mathieu Bridon <bochecha@daitauha.fr>
 
-"""A source implementation for staging zip files
+"""
+zip - stage files from zip archives
+===================================
 
 **Usage:**
 
diff --git a/buildstream/sandbox/__init__.py b/buildstream/sandbox/__init__.py
index 7ee871cab..53e170fbd 100644
--- a/buildstream/sandbox/__init__.py
+++ b/buildstream/sandbox/__init__.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Codethink Limited
 #
diff --git a/buildstream/sandbox/_config.py b/buildstream/sandbox/_config.py
index 8893e3faa..5debe24b2 100644
--- a/buildstream/sandbox/_config.py
+++ b/buildstream/sandbox/_config.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2018 Codethink Limited
 #
diff --git a/buildstream/sandbox/_mount.py b/buildstream/sandbox/_mount.py
index 84ab30ada..0f96a92b7 100644
--- a/buildstream/sandbox/_mount.py
+++ b/buildstream/sandbox/_mount.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Codethink Limited
 #
@@ -33,7 +32,8 @@ from .._fuse import SafeHardlinks
 class Mount():
     def __init__(self, sandbox, mount_point, safe_hardlinks):
         scratch_directory = sandbox._get_scratch_directory()
-        root_directory = sandbox.get_directory()
+        # Getting external_directory here is acceptable as we're part of the sandbox code.
+        root_directory = sandbox.get_virtual_directory().external_directory
 
         self.mount_point = mount_point
         self.safe_hardlinks = safe_hardlinks
diff --git a/buildstream/sandbox/_mounter.py b/buildstream/sandbox/_mounter.py
index c039b31df..921d06bb6 100644
--- a/buildstream/sandbox/_mounter.py
+++ b/buildstream/sandbox/_mounter.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Codethink Limited
 #
diff --git a/buildstream/sandbox/_sandboxbwrap.py b/buildstream/sandbox/_sandboxbwrap.py
index d18cb9ec0..010e4791d 100644
--- a/buildstream/sandbox/_sandboxbwrap.py
+++ b/buildstream/sandbox/_sandboxbwrap.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
@@ -29,6 +28,7 @@ from contextlib import ExitStack
 
 import psutil
 
+from .._exceptions import SandboxError
 from .. import utils, _signals
 from ._mount import MountMap
 from . import Sandbox, SandboxFlags
@@ -56,7 +56,9 @@ class SandboxBwrap(Sandbox):
 
     def run(self, command, flags, *, cwd=None, env=None):
         stdout, stderr = self._get_output()
-        root_directory = self.get_directory()
+
+        # Allowable access to underlying storage as we're part of the sandbox
+        root_directory = self.get_virtual_directory().external_directory
 
         # Fallback to the sandbox default settings for
         # the cwd and env.
@@ -67,6 +69,11 @@ class SandboxBwrap(Sandbox):
         if env is None:
             env = self._get_environment()
 
+        if not self._has_command(command[0], env):
+            raise SandboxError("Staged artifacts do not provide command "
+                               "'{}'".format(command[0]),
+                               reason='missing-command')
+
         # We want command args as a list of strings
         if isinstance(command, str):
             command = [command]
@@ -207,7 +214,7 @@ class SandboxBwrap(Sandbox):
 
                 # Skip removal of directories which already existed before
                 # launching bwrap
-                if not existing_basedirs[basedir]:
+                if existing_basedirs[basedir]:
                     continue
 
                 base_directory = os.path.join(root_mount_source, basedir)
diff --git a/buildstream/sandbox/_sandboxchroot.py b/buildstream/sandbox/_sandboxchroot.py
index 7f27f50d0..de4eb46e2 100644
--- a/buildstream/sandbox/_sandboxchroot.py
+++ b/buildstream/sandbox/_sandboxchroot.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Codethink Limited
 #
@@ -59,6 +58,11 @@ class SandboxChroot(Sandbox):
         if env is None:
             env = self._get_environment()
 
+        if not self._has_command(command[0], env):
+            raise SandboxError("Staged artifacts do not provide command "
+                               "'{}'".format(command[0]),
+                               reason='missing-command')
+
         # Command must be a list
         if isinstance(command, str):
             command = [command]
@@ -86,7 +90,7 @@ class SandboxChroot(Sandbox):
             # Nonetheless a better solution could perhaps be found.
 
             rootfs = stack.enter_context(utils._tempdir(dir='/var/run/buildstream'))
-            stack.enter_context(self.create_devices(self.get_directory(), flags))
+            stack.enter_context(self.create_devices(self._root, flags))
             stack.enter_context(self.mount_dirs(rootfs, flags, stdout, stderr))
 
             if flags & SandboxFlags.INTERACTIVE:
diff --git a/buildstream/sandbox/sandbox.py b/buildstream/sandbox/sandbox.py
index 3ab75f1a8..f3cab41ec 100644
--- a/buildstream/sandbox/sandbox.py
+++ b/buildstream/sandbox/sandbox.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Codethink Limited
 #
@@ -19,9 +18,8 @@
 #        Andrew Leeming <andrew.leeming@codethink.co.uk>
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 """
-Sandbox
-=======
-
+Sandbox - The build sandbox
+===========================
 :class:`.Element` plugins which want to interface with the sandbox
 need only understand this interface, while it may be given a different
 sandbox implementation, any sandbox implementation it is given will
@@ -31,7 +29,8 @@ See also: :ref:`sandboxing`.
 """
 
 import os
-from .._exceptions import ImplError
+from .._exceptions import ImplError, BstError
+from ..storage._filebaseddirectory import FileBasedDirectory
 
 
 class SandboxFlags():
@@ -92,28 +91,63 @@ class Sandbox():
         self.__cwd = None
         self.__env = None
         self.__mount_sources = {}
+        self.__allow_real_directory = kwargs['allow_real_directory']
+
         # Configuration from kwargs common to all subclasses
         self.__config = kwargs['config']
         self.__stdout = kwargs['stdout']
         self.__stderr = kwargs['stderr']
 
-        # Setup the directories
+        # Setup the directories. Root should be available to subclasses, hence
+        # being single-underscore. The others are private to this class.
+        self._root = os.path.join(directory, 'root')
         self.__directory = directory
-        self.__root = os.path.join(self.__directory, 'root')
         self.__scratch = os.path.join(self.__directory, 'scratch')
-        for directory_ in [self.__root, self.__scratch]:
+        for directory_ in [self._root, self.__scratch]:
             os.makedirs(directory_, exist_ok=True)
 
     def get_directory(self):
         """Fetches the sandbox root directory
 
         The root directory is where artifacts for the base
-        runtime environment should be staged.
+        runtime environment should be staged. Only works if
+        BST_VIRTUAL_DIRECTORY is not set.
 
         Returns:
            (str): The sandbox root directory
+
         """
-        return self.__root
+        if self.__allow_real_directory:
+            return self._root
+        else:
+            raise BstError("You can't use get_directory")
+
+    def get_virtual_directory(self):
+        """Fetches the sandbox root directory
+
+        The root directory is where artifacts for the base
+        runtime environment should be staged. Only works if
+        BST_VIRTUAL_DIRECTORY is not set.
+
+        Returns:
+           (str): The sandbox root directory
+
+        """
+        # For now, just create a new Directory every time we're asked
+        return FileBasedDirectory(self._root)
+
+    def get_virtual_toplevel_directory(self):
+        """Fetches the sandbox's toplevel directory
+
+        The toplevel directory contains 'root', 'scratch' and later
+        'artifact' where output is copied to.
+
+        Returns:
+           (str): The sandbox toplevel directory
+
+        """
+        # For now, just create a new Directory every time we're asked
+        return FileBasedDirectory(self.__directory)
 
     def set_environment(self, environment):
         """Sets the environment variables for the sandbox
@@ -281,3 +315,25 @@ class Sandbox():
     #              data passed in during construction.
     def _get_config(self):
         return self.__config
+
+    # _has_command()
+    #
+    #  Tests whether a command exists inside the sandbox
+    #
+    #     Args:
+    #         command (list): The command to test.
+    #         env (dict): A dictionary of string key, value pairs to set as environment
+    #                     variables inside the sandbox environment.
+    #     Returns:
+    #         (bool): Whether a command exists inside the sandbox.
+    def _has_command(self, command, env=None):
+        if os.path.isabs(command):
+            return os.path.exists(os.path.join(
+                self.get_directory(), command.lstrip(os.sep)))
+
+        for path in env.get('PATH').split(':'):
+            if os.path.exists(os.path.join(
+                    self.get_directory(), path.lstrip(os.sep), command)):
+                return True
+
+        return False
diff --git a/buildstream/scriptelement.py b/buildstream/scriptelement.py
index 95e6928ee..645381a40 100644
--- a/buildstream/scriptelement.py
+++ b/buildstream/scriptelement.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Codethink Limited
 #
@@ -19,9 +18,8 @@
 #        Jonathan Maw <jonathan.maw@codethink.co.uk>
 
 """
-ScriptElement
-=============
-
+ScriptElement - Abstract class for scripting elements
+=====================================================
 The ScriptElement class is a convenience class one can derive for
 implementing elements that stage elements and run command-lines on them.
 
@@ -245,9 +243,8 @@ class ScriptElement(Element):
                     with self.timed_activity("Staging {} at {}"
                                              .format(element.name, item['destination']),
                                              silent_nested=True):
-                        real_dstdir = os.path.join(sandbox.get_directory(),
-                                                   item['destination'].lstrip(os.sep))
-                        os.makedirs(os.path.dirname(real_dstdir), exist_ok=True)
+                        virtual_dstdir = sandbox.get_virtual_directory()
+                        virtual_dstdir.descend(item['destination'].lstrip(os.sep).split(os.sep), create=True)
                         element.stage_dependency_artifacts(sandbox, Scope.RUN, path=item['destination'])
 
             for item in self.__layout:
@@ -265,8 +262,8 @@ class ScriptElement(Element):
                         for dep in element.dependencies(Scope.RUN):
                             dep.integrate(sandbox)
 
-        os.makedirs(os.path.join(sandbox.get_directory(), self.__install_root.lstrip(os.sep)),
-                    exist_ok=True)
+        install_root_path_components = self.__install_root.lstrip(os.sep).split(os.sep)
+        sandbox.get_virtual_directory().descend(install_root_path_components, create=True)
 
     def assemble(self, sandbox):
 
diff --git a/buildstream/source.py b/buildstream/source.py
index fa547d641..ec38ae8f2 100644
--- a/buildstream/source.py
+++ b/buildstream/source.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
@@ -18,8 +17,8 @@
 #  Authors:
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 """
-Source
-======
+Source - Base source class
+==========================
 
 
 .. _core_source_abstract_methods:
diff --git a/buildstream/storage/__init__.py b/buildstream/storage/__init__.py
new file mode 100644
index 000000000..49364bb86
--- /dev/null
+++ b/buildstream/storage/__init__.py
@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+#
+#  Copyright (C) 2017 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Jim MacArthur <jim.macarthur@codethink.co.uk>
+
+from ._filebaseddirectory import FileBasedDirectory
+from .directory import Directory
diff --git a/buildstream/storage/_filebaseddirectory.py b/buildstream/storage/_filebaseddirectory.py
new file mode 100644
index 000000000..60379eaed
--- /dev/null
+++ b/buildstream/storage/_filebaseddirectory.py
@@ -0,0 +1,253 @@
+#!/usr/bin/env python3
+#
+#  Copyright (C) 2018 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Jim MacArthur <jim.macarthur@codethink.co.uk>
+
+"""
+FileBasedDirectory
+=========
+
+Implementation of the Directory class which backs onto a normal POSIX filing system.
+
+See also: :ref:`sandboxing`.
+"""
+
+from typing import List
+from collections import OrderedDict
+
+import calendar
+import os
+import time
+from .._exceptions import BstError, ErrorDomain
+from .directory import Directory
+from ..utils import link_files, copy_files, FileListResult, list_relative_paths
+from ..utils import _set_deterministic_user, _set_deterministic_mtime
+
+
+class VirtualDirectoryError(BstError):
+    """Raised by Directory functions when system calls fail.
+    This will be handled internally by the BuildStream core,
+    if you need to handle this error, then it should be reraised,
+    or either of the :class:`.ElementError` or :class:`.SourceError`
+    exceptions should be raised from this error.
+    """
+    def __init__(self, message, reason=None):
+        super().__init__(message, domain=ErrorDomain.VIRTUAL_FS, reason=reason)
+
+
+# Like os.path.getmtime(), but doesnt explode on symlinks
+# Copy/pasted from compose.py
+def getmtime(path):
+    stat = os.lstat(path)
+    return stat.st_mtime
+
+# FileBasedDirectory intentionally doesn't call its superclass constuctor,
+# which is mean to be unimplemented.
+# pylint: disable=super-init-not-called
+
+
+class _FileObject():
+    """A description of a file in a virtual directory. The contents of
+    this class are never used, but there needs to be something present
+    for files so is_empty() works correctly.
+
+    """
+    def __init__(self, virtual_directory: Directory, filename: str):
+        self.directory = virtual_directory
+        self.filename = filename
+
+
+class FileBasedDirectory(Directory):
+    def __init__(self, external_directory=None):
+        self.external_directory = external_directory
+        self.index = OrderedDict()
+        self._directory_read = False
+
+    def _populate_index(self) -> None:
+        if self._directory_read:
+            return
+        for entry in os.listdir(self.external_directory):
+            if os.path.isdir(os.path.join(self.external_directory, entry)):
+                self.index[entry] = FileBasedDirectory(os.path.join(self.external_directory, entry))
+            else:
+                self.index[entry] = _FileObject(self, entry)
+        self._directory_read = True
+
+    def descend(self, subdirectory_spec: List[str], create: bool = False) -> Directory:
+        """ Descend one or more levels of directory hierarchy and return a new
+        Directory object for that directory.
+
+        Arguments:
+        * subdirectory_spec (list of strings): A list of strings which are all directory
+          names.
+        * create (boolean): If this is true, the directories will be created if
+          they don't already exist.
+        """
+
+        # It's very common to send a directory name instead of a list and this causes
+        # bizarre errors, so check for it here
+        if not isinstance(subdirectory_spec, list):
+            subdirectory_spec = [subdirectory_spec]
+        if not subdirectory_spec:
+            return self
+
+        # Because of the way split works, it's common to get a list which begins with
+        # an empty string. Detect these and remove them, then start again.
+        if subdirectory_spec[0] == "":
+            return self.descend(subdirectory_spec[1:], create)
+
+        self._populate_index()
+        if subdirectory_spec[0] in self.index:
+            entry = self.index[subdirectory_spec[0]]
+            if isinstance(entry, FileBasedDirectory):
+                new_path = os.path.join(self.external_directory, subdirectory_spec[0])
+                return FileBasedDirectory(new_path).descend(subdirectory_spec[1:], create)
+            else:
+                error = "Cannot descend into {}, which is a '{}' in the directory {}"
+                raise VirtualDirectoryError(error.format(subdirectory_spec[0],
+                                                         type(entry).__name__,
+                                                         self.external_directory))
+        else:
+            if create:
+                new_path = os.path.join(self.external_directory, subdirectory_spec[0])
+                os.makedirs(new_path, exist_ok=True)
+                return FileBasedDirectory(new_path).descend(subdirectory_spec[1:], create)
+            else:
+                error = "No entry called '{}' found in the directory rooted at {}"
+                raise VirtualDirectoryError(error.format(subdirectory_spec[0], self.external_directory))
+        return None
+
+    def import_files(self, external_pathspec: any, files: List[str] = None,
+                     report_written: bool = True, update_utimes: bool = False,
+                     can_link: bool = False) -> FileListResult:
+        """Imports some or all files from external_path into this directory.
+
+        Keyword arguments: external_pathspec: Either a string
+        containing a pathname, or a Directory object, to use as the
+        source.
+
+        files (list of strings): A list of all the files relative to
+        the external_pathspec to copy. If 'None' is supplied, all
+        files are copied.
+
+        report_written (bool): Return the full list of files
+        written. Defaults to true. If false, only a list of
+        overwritten files is returned.
+
+        update_utimes (bool): Update the access and modification time
+        of each file copied to the current time.
+
+        can_link (bool): Whether it's OK to create a hard link to the
+        original content, meaning the stored copy will change when the
+        original files change. Setting this doesn't guarantee hard
+        links will be made. can_link will never be used if
+        update_utimes is set.
+        """
+
+        if isinstance(external_pathspec, Directory):
+            source_directory = external_pathspec.external_directory
+        else:
+            source_directory = external_pathspec
+
+        if can_link and not update_utimes:
+            import_result = link_files(source_directory, self.external_directory, files=files,
+                                       ignore_missing=False, report_written=report_written)
+        else:
+            import_result = copy_files(source_directory, self.external_directory, files=files,
+                                       ignore_missing=False, report_written=report_written)
+        if update_utimes:
+            cur_time = time.time()
+
+            for f in import_result.files_written:
+                os.utime(os.path.join(self.external_directory, f), times=(cur_time, cur_time))
+        return import_result
+
+    def set_deterministic_mtime(self) -> None:
+        """ Sets a static modification time for all regular files in this directory.
+        The magic number for timestamps: 2011-11-11 11:11:11
+        """
+        _set_deterministic_mtime(self.external_directory)
+
+    def set_deterministic_user(self) -> None:
+        """ Sets all files in this directory to the current user's euid/egid.
+        """
+        _set_deterministic_user(self.external_directory)
+
+    def export_files(self, to_directory: str, can_link: bool = False, can_destroy: bool = False) -> None:
+        """Copies everything from this into to_directory.
+
+        Arguments:
+
+        to_directory (string): a path outside this directory object
+        where the contents will be copied to.
+
+        can_link (bool): Whether we can create hard links in to_directory
+        instead of copying.
+
+        """
+
+        if can_destroy:
+            # Try a simple rename of the sandbox root; if that
+            # doesnt cut it, then do the regular link files code path
+            try:
+                os.rename(self.external_directory, to_directory)
+                return
+            except OSError:
+                # Proceed using normal link/copy
+                pass
+
+        if can_link:
+            link_files(self.external_directory, to_directory)
+        else:
+            copy_files(self.external_directory, to_directory)
+
+    def is_empty(self) -> bool:
+        """ Return true if this directory has no files, subdirectories or links in it.
+        """
+        self._populate_index()
+        return len(self.index) == 0
+
+    def mark_unmodified(self) -> None:
+        """ Marks all files in this directory (recursively) as unmodified.
+        """
+        _set_deterministic_mtime(self.external_directory)
+
+    def list_modified_paths(self) -> List[str]:
+        """Provide a list of relative paths which have been modified since the
+        last call to mark_unmodified.
+
+        Return value: List(str) - list of modified paths
+        """
+        magic_timestamp = calendar.timegm([2011, 11, 11, 11, 11, 11])
+
+        return [f for f in list_relative_paths(self.external_directory)
+                if getmtime(os.path.join(self.external_directory, f)) != magic_timestamp]
+
+    def list_relative_paths(self) -> List[str]:
+        """Provide a list of all relative paths.
+
+        Return value: List(str) - list of all paths
+        """
+
+        return list_relative_paths(self.external_directory)
+
+    def __str__(self) -> str:
+        # This returns the whole path (since we don't know where the directory started)
+        # which exposes the sandbox directory; we will have to assume for the time being
+        # that people will not abuse __str__.
+        return self.external_directory
diff --git a/buildstream/storage/directory.py b/buildstream/storage/directory.py
new file mode 100644
index 000000000..f37fb98ad
--- /dev/null
+++ b/buildstream/storage/directory.py
@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+#
+#  Copyright (C) 2018 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Jim MacArthur <jim.macarthur@codethink.co.uk>
+
+"""
+Directory
+=========
+
+Virtual Directory class to isolate the rest of BuildStream from the backing store implementation.
+Sandboxes are allowed to read from and write to the underlying storage, but all others must use this
+Directory class to access files and directories in the sandbox.
+
+See also: :ref:`sandboxing`.
+"""
+
+from typing import List
+from ..utils import FileListResult
+
+
+class Directory():
+    def __init__(self, external_directory=None):
+        raise NotImplementedError()
+
+    def descend(self, subdirectory_spec: List[str]) -> 'Directory':
+        """
+        Descend one or more levels of directory hierarchy and return a new
+        Directory object for that directory.
+
+        Arguments:
+        subdirectory_spec (list of strings): A list of strings which are all directory
+        names.
+        create (boolean): If this is true, the directories will be created if
+        they don't already exist.
+        """
+        raise NotImplementedError()
+
+    # Import and export of files and links
+    def import_files(self, external_pathspec: any, files: List[str] = None,
+                     report_written: bool = True, update_utimes: bool = False,
+                     can_link: bool = False) -> FileListResult:
+        """Imports some or all files from external_path into this directory.
+
+        Keyword arguments: external_pathspec: Either a string
+        containing a pathname, or a Directory object, to use as the
+        source.
+
+        files (list of strings): A list of all the files relative to
+        the external_pathspec to copy. If 'None' is supplied, all
+        files are copied.
+
+        report_written (bool): Return the full list of files
+        written. Defaults to true. If false, only a list of
+        overwritten files is returned.
+
+        update_utimes (bool): Update the access and modification time
+        of each file copied to the current time.
+
+        can_link (bool): Whether it's OK to create a hard link to the
+        original content, meaning the stored copy will change when the
+        original files change. Setting this doesn't guarantee hard
+        links will be made. can_link will never be used if
+        update_utimes is set.
+        """
+
+        raise NotImplementedError()
+
+    def export_files(self, to_directory: str, can_link: bool = False, can_destroy: bool = False) -> None:
+        """Copies everything from this into to_directory.
+
+        Arguments:
+
+        to_directory (string): a path outside this directory object
+        where the contents will be copied to.
+
+        can_link (bool): Whether we can create hard links in to_directory
+        instead of copying. Setting this does not guarantee hard links will be used.
+
+        can_destroy (bool): Can we destroy the data already in this
+        directory when exporting? If set, this may allow data to be
+        moved rather than copied which will be quicker.
+        """
+
+        raise NotImplementedError()
+
+    # Convenience functions
+    def is_empty(self) -> bool:
+        raise NotImplementedError()
+
+    def set_deterministic_mtime(self) -> None:
+        """ Sets a static modification time for all regular files in this directory.
+        The magic number for timestamps: 2011-11-11 11:11:11
+        """
+        raise NotImplementedError()
+
+    def set_deterministic_user(self) -> None:
+        """ Sets all files in this directory to the current user's euid/egid.
+        """
+        raise NotImplementedError()
+
+    def mark_unmodified(self) -> None:
+        """ Marks all files in this directory (recursively) as unmodified.
+        """
+        raise NotImplementedError()
+
+    def list_modified_paths(self) -> List[str]:
+        """Provide a list of relative paths which have been modified since the
+        last call to mark_unmodified.
+
+        Return value: List(str) - dictionary with all paths
+        """
+        raise NotImplementedError()
+
+    def list_relative_paths(self) -> List[str]:
+        """Provide a list of all relative paths in this directory.
+
+        Return value: List(str) - dictionary with all paths
+        """
+        raise NotImplementedError()
diff --git a/buildstream/utils.py b/buildstream/utils.py
index 8e7219631..b81a6c852 100644
--- a/buildstream/utils.py
+++ b/buildstream/utils.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016-2018 Codethink Limited
 #
@@ -307,7 +306,7 @@ def safe_remove(path):
     """Removes a file or directory
 
     This will remove a file if it exists, and will
-    remove a directory if the directory is not empty.
+    remove a directory if the directory is empty.
 
     Args:
        path (str): The path to remove
@@ -472,6 +471,10 @@ def get_bst_version():
     from . import __version__
     versions = __version__.split('.')[:2]
 
+    if versions[0] == '0+untagged':
+        raise UtilError("Your git repository has no tags - BuildStream can't "
+                        "determine its version. Please run `git fetch --tags`.")
+
     return (int(versions[0]), int(versions[1]))