Merge branch 'traveltissues/985' into 'master'

workspaces via sourcecache 1

Closes #1088

See merge request BuildStream/buildstream!1563

38 files changed, 271 insertions, 113 deletions

diff --git a/src/buildstream/_cas/cascache.py b/src/buildstream/_cas/cascache.py
index 2603b13c5..6002adc4a 100644
--- a/src/buildstream/_cas/cascache.py
+++ b/src/buildstream/_cas/cascache.py
@@ -163,6 +163,16 @@ class CASCache():
     def has_open_grpc_channels(self):
         return bool(self._casd_channel)
 
+    # close_channel():
+    #
+    # Close the casd channel if it exists
+    #
+    def close_channel(self):
+        if self._casd_channel:
+            self._local_cas = None
+            self._casd_channel.close()
+            self._casd_channel = None
+
     # release_resources():
     #
     # Release resources used by CASCache.
@@ -172,11 +182,7 @@ class CASCache():
             self._cache_usage_monitor.release_resources()
 
         if self._casd_process:
-            if self._casd_channel:
-                self._local_cas = None
-                self._casd_channel.close()
-                self._casd_channel = None
-
+            self.close_channel()
             self._casd_process.terminate()
             try:
                 # Don't print anything if buildbox-casd terminates quickly
diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py
index 5599c8b71..d578787a3 100644
--- a/src/buildstream/_loader/loader.py
+++ b/src/buildstream/_loader/loader.py
@@ -438,17 +438,38 @@ class Loader():
         sources = node.get_sequence(Symbol.SOURCES, default=[])
         element_kind = node.get_str(Symbol.KIND)
 
-        for index, source in enumerate(sources):
-            kind = source.get_str(Symbol.KIND)
-            del source[Symbol.KIND]
+        def make_metasource(_index, _source, skip_workspace=True):
+            kind = _source.get_str(Symbol.KIND)
+            # the workspace source plugin cannot be used unless the element is workspaced
+            if kind == 'workspace' and skip_workspace:
+                return None
+
+            del _source[Symbol.KIND]
 
             # Directory is optional
-            directory = source.get_str(Symbol.DIRECTORY, default=None)
+            directory = _source.get_str(Symbol.DIRECTORY, default=None)
             if directory:
-                del source[Symbol.DIRECTORY]
+                del _source[Symbol.DIRECTORY]
+
+            return MetaSource(element.name, _index, element_kind, kind, _source, directory)
 
-            meta_source = MetaSource(element.name, index, element_kind, kind, source, directory)
-            meta_sources.append(meta_source)
+        for index, source in enumerate(sources):
+            meta_source = make_metasource(index, source)
+            if meta_source:
+                meta_sources.append(meta_source)
+
+        # if there's a workspace for this element then just append a dummy workspace
+        # metasources. When sources are instantiated, the workspace will own the
+        # element sources. These can then be referred to when resetting or similar operations.
+        workspace = self._context.get_workspaces().get_workspace(element.name)
+        if workspace:
+            workspace_node = {'kind': 'workspace'}
+            workspace_node['path'] = workspace.get_absolute_path()
+            workspace_node['ref'] = str(workspace.to_dict().get('last_successful', 'ignored'))
+            sources.append(workspace_node)
+            meta_source = make_metasource(len(sources), sources.mapping_at(-1), False)
+            if meta_source:
+                meta_sources.append(meta_source)
 
         meta_element = MetaElement(self.project, element.name, element_kind,
                                    elt_provenance, meta_sources,
@@ -606,11 +627,7 @@ class Loader():
                             LoadErrorReason.SUBPROJECT_INCONSISTENT, detail=detail)
 
         sources = list(element.sources())
-        workspace = element._get_workspace()
-        if workspace:
-            # If a workspace is open, load it from there instead
-            basedir = workspace.get_absolute_path()
-        elif len(sources) == 1 and sources[0]._get_local_path():
+        if len(sources) == 1 and sources[0]._get_local_path():
             # Optimization for junctions with a single local source
             basedir = sources[0]._get_local_path()
         else:
@@ -620,7 +637,7 @@ class Loader():
                                    filename, element._get_cache_key())
             if not os.path.exists(basedir):
                 os.makedirs(basedir, exist_ok=True)
-                element._stage_sources_at(basedir, mount_workspaces=False)
+                element._stage_sources_at(basedir)
 
         # Load the project
         project_dir = os.path.join(basedir, element.path)
diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py
index 83db24d94..75b3dd84e 100644
--- a/src/buildstream/_stream.py
+++ b/src/buildstream/_stream.py
@@ -1528,7 +1528,7 @@ class Stream():
             element_source_dir = self._get_element_dirname(directory, element)
             if list(element.sources()):
                 os.makedirs(element_source_dir)
-                element._stage_sources_at(element_source_dir, mount_workspaces=False)
+                element._stage_sources_at(element_source_dir)
 
     # Create a tarball from the content of directory
     def _create_tarball(self, directory, tar_name, compression):
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 044b97458..4ebb17d09 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -92,7 +92,7 @@ from ._variables import Variables
 from ._versions import BST_CORE_ARTIFACT_VERSION
 from ._exceptions import BstError, LoadError, LoadErrorReason, ImplError, \
     ErrorDomain, SourceCacheError
-from .utils import FileListResult, UtilError
+from .utils import FileListResult
 from . import utils
 from . import _cachekey
 from . import _signals
@@ -1023,6 +1023,9 @@ class Element(Plugin):
         element = meta.project.create_element(meta, first_pass=meta.first_pass)
         cls.__instantiated_elements[meta] = element
 
+        # do the metasources include a workspace source?
+        _workspace_source = None
+
         # Instantiate sources and generate their keys
         for meta_source in meta.sources:
             meta_source.first_pass = meta.is_junction
@@ -1030,12 +1033,24 @@ class Element(Plugin):
                                                 first_pass=meta.first_pass)
 
             redundant_ref = source._load_ref()
+
+            if meta_source.kind == 'workspace':
+                _workspace_source = source
+                continue
+
             element.__sources.append(source)
 
             # Collect redundant refs which occurred at load time
             if redundant_ref is not None:
                 cls.__redundant_source_refs.append((source, redundant_ref))
 
+        # workspace handling: if the metasources included a workspace source, then
+        # this should replace the element.__sources and should in turn own those sources
+        # directly
+        if _workspace_source is not None:
+            _workspace_source.set_element_sources(element.__sources)
+            element.__sources = [_workspace_source]
+
         # Instantiate dependencies
         for meta_dep in meta.dependencies:
             dependency = Element._new_from_meta(meta_dep, task)
@@ -1264,10 +1279,6 @@ class Element(Plugin):
             # Tracking may still be pending
             return
 
-        if self._get_workspace() and self.__assemble_scheduled:
-            self.__reset_cache_data()
-            return
-
         self.__update_cache_keys()
         self.__update_artifact_state()
 
@@ -1438,21 +1449,18 @@ class Element(Plugin):
     # Args:
     #     sandbox (:class:`.Sandbox`): The build sandbox
     #     directory (str): An absolute path to stage the sources at
-    #     mount_workspaces (bool): mount workspaces if True, copy otherwise
     #
-    def _stage_sources_in_sandbox(self, sandbox, directory, mount_workspaces=True):
+    def _stage_sources_in_sandbox(self, sandbox, directory):
 
         # Only artifact caches that implement diff() are allowed to
         # perform incremental builds.
-        if mount_workspaces and self.__can_build_incrementally():
-            workspace = self._get_workspace()
+        if self.__can_build_incrementally():
             sandbox.mark_directory(directory)
-            sandbox._set_mount_source(directory, workspace.get_absolute_path())
 
         # Stage all sources that need to be copied
         sandbox_vroot = sandbox.get_virtual_directory()
         host_vdirectory = sandbox_vroot.descend(*directory.lstrip(os.sep).split(os.sep), create=True)
-        self._stage_sources_at(host_vdirectory, mount_workspaces=mount_workspaces, usebuildtree=sandbox._usebuildtree)
+        self._stage_sources_at(host_vdirectory, usebuildtree=sandbox._usebuildtree)
 
     # _stage_sources_at():
     #
@@ -1460,10 +1468,9 @@ class Element(Plugin):
     #
     # Args:
     #     vdirectory (:class:`.storage.Directory`): A virtual directory object to stage sources into.
-    #     mount_workspaces (bool): mount workspaces if True, copy otherwise
     #     usebuildtree (bool): use a the elements build tree as its source.
     #
-    def _stage_sources_at(self, vdirectory, mount_workspaces=True, usebuildtree=False):
+    def _stage_sources_at(self, vdirectory, usebuildtree=False):
 
         context = self._get_context()
 
@@ -1479,24 +1486,16 @@ class Element(Plugin):
             if not vdirectory.is_empty():
                 raise ElementError("Staging directory '{}' is not empty".format(vdirectory))
 
-            workspace = self._get_workspace()
-            if workspace:
-                # If mount_workspaces is set and we're doing incremental builds,
-                # the workspace is already mounted into the sandbox.
-                if not (mount_workspaces and self.__can_build_incrementally()):
-                    with self.timed_activity("Staging local files at {}"
-                                             .format(workspace.get_absolute_path())):
-                        workspace.stage(import_dir)
-
             # Check if we have a cached buildtree to use
-            elif usebuildtree:
+            if usebuildtree:
                 import_dir = self.__artifact.get_buildtree()
                 if import_dir.is_empty():
                     detail = "Element type either does not expect a buildtree or it was explictily cached without one."
                     self.warn("WARNING: {} Artifact contains an empty buildtree".format(self.name), detail=detail)
 
-            # No workspace or cached buildtree, stage source from source cache
+            # No cached buildtree, stage source from source cache
             else:
+
                 # Assert sources are cached
                 assert self._source_cached()
 
@@ -1511,6 +1510,7 @@ class Element(Plugin):
                         for source in self.__sources[last_required_previous_ix:]:
                             source_dir = sourcecache.export(source)
                             import_dir.import_files(source_dir)
+
                     except SourceCacheError as e:
                         raise ElementError("Error trying to export source for {}: {}"
                                            .format(self.name, e))
@@ -1722,24 +1722,6 @@ class Element(Plugin):
                     # Shelling into a sandbox is useful to debug this error
                     e.sandbox = True
 
-                    # If there is a workspace open on this element, it will have
-                    # been mounted for sandbox invocations instead of being staged.
-                    #
-                    # In order to preserve the correct failure state, we need to
-                    # copy over the workspace files into the appropriate directory
-                    # in the sandbox.
-                    #
-                    workspace = self._get_workspace()
-                    if workspace and self.__staged_sources_directory:
-                        sandbox_vroot = sandbox.get_virtual_directory()
-                        path_components = self.__staged_sources_directory.lstrip(os.sep).split(os.sep)
-                        sandbox_vpath = sandbox_vroot.descend(*path_components)
-                        try:
-                            sandbox_vpath.import_files(workspace.get_absolute_path())
-                        except UtilError as e2:
-                            self.warn("Failed to preserve workspace state for failed build sysroot: {}"
-                                      .format(e2))
-
                     self.__set_build_result(success=False, description=str(e), detail=e.detail)
                     self._cache_artifact(rootdir, sandbox, e.collect)
 
@@ -2221,7 +2203,6 @@ class Element(Plugin):
             }
 
             project = self._get_project()
-            workspace = self._get_workspace()
 
             self.__cache_key_dict = {
                 'core-artifact-version': BST_CORE_ARTIFACT_VERSION,
@@ -2237,15 +2218,9 @@ class Element(Plugin):
                 return {'key': _source._get_unique_key(True),
                         'name': _source._get_source_name()}
 
-            def __get_workspace_entry(workspace):
-                return {'key': workspace.get_key()}
-
-            if workspace is None:
-                self.__cache_key_dict['sources'] = \
-                    [__get_source_entry(s) for s in self.__sources]
-            else:
-                self.__cache_key_dict['sources'] = \
-                    [__get_workspace_entry(workspace)]
+            self._source_cached()
+            self.__cache_key_dict['sources'] = \
+                [__get_source_entry(s) for s in self.__sources]
 
             self.__cache_key_dict['fatal-warnings'] = sorted(project._fatal_warnings)
 
@@ -2256,7 +2231,7 @@ class Element(Plugin):
 
     # Check if sources are cached, generating the source key if it hasn't been
     def _source_cached(self):
-        if self.__sources and not self._get_workspace():
+        if self.__sources:
             sourcecache = self._get_context().sourcecache
 
             # Go through sources we'll cache generating keys
diff --git a/src/buildstream/plugins/elements/import.py b/src/buildstream/plugins/elements/import.py
index 6ae8cef46..9568bd08e 100644
--- a/src/buildstream/plugins/elements/import.py
+++ b/src/buildstream/plugins/elements/import.py
@@ -74,8 +74,7 @@ class ImportElement(Element):
     def assemble(self, sandbox):
 
         # Stage sources into the input directory
-        # Do not mount workspaces as the files are copied from outside the sandbox
-        self._stage_sources_in_sandbox(sandbox, 'input', mount_workspaces=False)
+        self.stage_sources(sandbox, 'input')
 
         rootdir = sandbox.get_virtual_directory()
         inputdir = rootdir.descend('input')
diff --git a/src/buildstream/plugins/sources/workspace.py b/src/buildstream/plugins/sources/workspace.py
new file mode 100644
index 000000000..ee145babb
--- /dev/null
+++ b/src/buildstream/plugins/sources/workspace.py
@@ -0,0 +1,150 @@
+#
+#  Copyright (C) 2019 Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+
+"""
+:orphan:
+
+workspace - stage an opened workspace directory
+===============================================
+
+**Usage:**
+
+The workspace plugin must not be directly used. This plugin is used as the
+kind for a synthetic node representing the sources of an element with an open
+workspace. The node constructed would be specified as follows:
+
+.. code:: yaml
+
+   # Specify the workspace source kind
+   kind: workspace
+
+   # Specify the absolute path to the directory
+   path: /path/to/workspace
+"""
+
+import os
+from buildstream.storage.directory import Directory
+from buildstream.storage._casbaseddirectory import CasBasedDirectory
+from buildstream import Source, SourceError, Consistency
+from buildstream import utils
+from buildstream.types import SourceRef
+from buildstream.node import MappingNode
+
+
+class WorkspaceSource(Source):
+    # pylint: disable=attribute-defined-outside-init
+
+    BST_STAGE_VIRTUAL_DIRECTORY = True
+
+    def __init__(self, context, project, meta) -> None:
+        super().__init__(context, project, meta)
+
+        # Cached unique key
+        self.__unique_key = None
+        # the element source objects from the specified metasources
+        self.__element_sources = []
+        # the digest of the Directory following the import of the workspace
+        self.__digest = None
+        # the CasBasedDirectory which the path is imported into
+        self.__cas_dir = None
+
+    def set_element_sources(self, _element_sources: [Source]) -> None:
+        self.__element_sources = _element_sources
+
+    def get_element_sources(self) -> [Source]:
+        return self.__element_sources
+
+    def track(self) -> SourceRef:
+        return None
+
+    def configure(self, node: MappingNode) -> None:
+        node.validate_keys(['path', 'ref', 'kind'])
+        self.path = node.get_str('path')
+        self.__digest = node.get_str('ref')
+
+    def preflight(self) -> None:
+        for source in self.get_element_sources():
+            source.preflight()
+
+    def get_ref(self) -> None:
+        return None
+
+    def load_ref(self, node: MappingNode) -> None:
+        pass  # pragma: nocover
+
+    def set_ref(self, ref: SourceRef, node: MappingNode) -> None:
+        pass  # pragma: nocover
+
+    def get_unique_key(self) -> (str, SourceRef):
+        sourcecache = self._get_context().sourcecache
+
+        if self.__cas_dir is None:
+            self.__cas_dir = CasBasedDirectory(sourcecache.cas)
+
+        if self.__digest is None:
+
+            with self.timed_activity("Staging local files into CAS"):
+                result = self.__cas_dir.import_files(self.path)
+                if result.overwritten or result.ignored:
+                    raise SourceError(
+                        "Failed to stage source: files clash with existing directory",
+                        reason='ensure-stage-dir-fail')
+                self.__digest = self.__cas_dir._get_digest().hash
+
+        # commit to cache if not cached
+        if not sourcecache.contains(self):
+            sourcecache.commit(self, [])
+
+        #  now close down grpc channels
+        sourcecache.cas.close_channel()
+        assert not sourcecache.cas.has_open_grpc_channels()
+        return (self.path, self.__digest)
+
+    def init_workspace(self, directory: Directory) -> None:
+        # for each source held by the workspace we must call init_workspace
+        # those sources may override `init_workspace` expecting str or Directory
+        # and this will need to be extracted from the directory passed to this method
+        assert isinstance(directory, Directory)
+        directory = directory.external_directory
+        for source in self.get_element_sources():
+            source._init_workspace(directory)
+
+    def get_consistency(self):
+        # always return cached state
+        return Consistency.CACHED
+
+    def fetch(self) -> None:
+        pass  # pragma: nocover
+
+    def stage(self, directory: Directory) -> None:
+        # directory should always be a Directory object
+        assert isinstance(directory, Directory)
+        assert isinstance(self.__cas_dir, CasBasedDirectory)
+        with self.timed_activity("Staging Workspace files"):
+            result = directory.import_files(self.__cas_dir)
+
+            if result.overwritten or result.ignored:
+                raise SourceError(
+                    "Failed to stage source: files clash with existing directory",
+                    reason='ensure-stage-dir-fail')
+
+    def _get_local_path(self) -> str:
+        return self.path
+
+
+# Plugin entry point
+def setup() -> WorkspaceSource:
+    return WorkspaceSource
diff --git a/tests/cachekey/project/elements/build1.expected b/tests/cachekey/project/elements/build1.expected
index 275abd2d0..d335bb846 100644
--- a/tests/cachekey/project/elements/build1.expected
+++ b/tests/cachekey/project/elements/build1.expected
@@ -1 +1 @@
-ba3cc6aeaef5d4c559acbd3a92e7a2512a72674c3e98aaf44256eb1c951a1a29
\ No newline at end of file
+f5e3487eb2802ee1a7ebcbca6c3c185af2709aba98f4de2004bb5e5ab587884c
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/build2.expected b/tests/cachekey/project/elements/build2.expected
index ded4d5ba8..d8183a9ed 100644
--- a/tests/cachekey/project/elements/build2.expected
+++ b/tests/cachekey/project/elements/build2.expected
@@ -1 +1 @@
-707ca16d9d0efd925cdf7e5f2586c7ca6446ad739a0ecce22d3cb967dc557edb
\ No newline at end of file
+98bd367885521bd42a60f926a93cb0d9d5795829fbf433703d331cad10cd3875
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose1.expected b/tests/cachekey/project/elements/compose1.expected
index e62634f37..cb99535cc 100644
--- a/tests/cachekey/project/elements/compose1.expected
+++ b/tests/cachekey/project/elements/compose1.expected
@@ -1 +1 @@
-67fe128c47c16737fbb0c675a866a07bea3921715953b2c191ac282b1100b747
\ No newline at end of file
+9c17f7ae6220dd70a099a2a20444a102c26eec76c09861791a0e9f6d67cc660f
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose2.expected b/tests/cachekey/project/elements/compose2.expected
index 9c1169326..25e199334 100644
--- a/tests/cachekey/project/elements/compose2.expected
+++ b/tests/cachekey/project/elements/compose2.expected
@@ -1 +1 @@
-436d4c69b1d120315c7b073f564ae07568fd7d3ead0be3df9e90a702793ad107
\ No newline at end of file
+94022d64108335210fec8604ea60c99471586d69c9cb3deef95f29d09e2cb3b7
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose3.expected b/tests/cachekey/project/elements/compose3.expected
index 74347beab..62053720e 100644
--- a/tests/cachekey/project/elements/compose3.expected
+++ b/tests/cachekey/project/elements/compose3.expected
@@ -1 +1 @@
-cfb87e685ad0569f92eb2ac7d0f121dd62e3eb4e0a82f559ccf402887461d7aa
\ No newline at end of file
+ee78f0fb0b3247259af41c5bbd4180feeed209da644a2ed373b8e2721da89ca5
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose4.expected b/tests/cachekey/project/elements/compose4.expected
index 4a809312d..b353dd4af 100644
--- a/tests/cachekey/project/elements/compose4.expected
+++ b/tests/cachekey/project/elements/compose4.expected
@@ -1 +1 @@
-dc962993b42725a3fbfcb0e38557a663b610e5033e6e2a31dcb8a2e87a5b9117
\ No newline at end of file
+627befddfd44d93b93e59dc8d10906f9cb3616325bdd64c607858e29e7c7622f
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose5.expected b/tests/cachekey/project/elements/compose5.expected
index 355a9fefc..be875921d 100644
--- a/tests/cachekey/project/elements/compose5.expected
+++ b/tests/cachekey/project/elements/compose5.expected
@@ -1 +1 @@
-ea2ca85b60271c877f45067de8c7786bbb5317447e127fbf011cc888e47c264e
\ No newline at end of file
+8acb2a21f97dc3622a983449e8e64d0e6b1d1ae7ad072dc164f9daea5241e4eb
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import1.expected b/tests/cachekey/project/elements/import1.expected
index 5cb6a1973..dfb798e77 100644
--- a/tests/cachekey/project/elements/import1.expected
+++ b/tests/cachekey/project/elements/import1.expected
@@ -1 +1 @@
-402b946498422abb82133fa5998b97f93b08d66ee1f332154413886687d8e7e9
\ No newline at end of file
+8920a31ae292e892d62692702bcd43bbb08af53187f1766dcba07c9eea9ea1f0
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import2.expected b/tests/cachekey/project/elements/import2.expected
index 5815a2075..6d37ae6fe 100644
--- a/tests/cachekey/project/elements/import2.expected
+++ b/tests/cachekey/project/elements/import2.expected
@@ -1 +1 @@
-030b97f6f854c270126aa3cc7ac5d541e77e42f984b61aef63b2bce183982310
\ No newline at end of file
+98f292215ab793c2607aa18d8a61ef026d4c99e39f70be5aaaf466659ba2f19c
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import3.expected b/tests/cachekey/project/elements/import3.expected
index ecd6f8f08..1d20a9c44 100644
--- a/tests/cachekey/project/elements/import3.expected
+++ b/tests/cachekey/project/elements/import3.expected
@@ -1 +1 @@
-6c0b0c42d438312a775084341f99547149b6a269ef4a24122b6f75b343802877
\ No newline at end of file
+f57a93e7711b973a6a541272540d21b47579b7b7ef389a336558244592da2d50
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/script1.expected b/tests/cachekey/project/elements/script1.expected
index 1b43ee996..bf7cf9209 100644
--- a/tests/cachekey/project/elements/script1.expected
+++ b/tests/cachekey/project/elements/script1.expected
@@ -1 +1 @@
-fb83946f47528e7df4d322ca1848616905d6e259e6db3cea26f906093ce519e0
\ No newline at end of file
+372af727f8f7c8ca235a140bd6c3f9baa8ca9da765b61ff7070f61a9b6a44c41
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/bzr1.expected b/tests/cachekey/project/sources/bzr1.expected
index a53c63adb..9ef16836f 100644
--- a/tests/cachekey/project/sources/bzr1.expected
+++ b/tests/cachekey/project/sources/bzr1.expected
@@ -1 +1 @@
-673bb938cc3fabe0be55e98c6b8b80853168becc86b4fa102fc0c538879bf83a
\ No newline at end of file
+88fee9ea1330cc7a7bc34ba8bac23a4c2dcf21126eae023042fb0eb48a4123f8
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/git1.expected b/tests/cachekey/project/sources/git1.expected
index 9fa018ccc..bcbcb78ac 100644
--- a/tests/cachekey/project/sources/git1.expected
+++ b/tests/cachekey/project/sources/git1.expected
@@ -1 +1 @@
-53a367133fb8f3ca86ba772801ea62681414271da9582800dd56a62e9c6d7e5d
\ No newline at end of file
+2811ce3bbe1dc77cda9da12563487183da660c06783b220d9b1bfc4b151f9869
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/git2.expected b/tests/cachekey/project/sources/git2.expected
index d9cad4dcd..aa2b09e64 100644
--- a/tests/cachekey/project/sources/git2.expected
+++ b/tests/cachekey/project/sources/git2.expected
@@ -1 +1 @@
-86e0cf4f3154fa006899acf64317930ebf08ca6d01113abfa35ccceed2961fcd
\ No newline at end of file
+84e90059df19d521073b69b512a40cf3e149d2ac8ee6d9ebd075145fd516d603
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/git3.expected b/tests/cachekey/project/sources/git3.expected
index df76f3108..42da1512f 100644
--- a/tests/cachekey/project/sources/git3.expected
+++ b/tests/cachekey/project/sources/git3.expected
@@ -1 +1 @@
-ad5dff8a422c9de7c3d02773aeed7b425d43501ee5c2d5d13064b6f1e1ed9dec
\ No newline at end of file
+45b5d40a80309002754478344228bc0347151b282173823c7919e1512fdea502
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/local1.expected b/tests/cachekey/project/sources/local1.expected
index 5cb6a1973..dfb798e77 100644
--- a/tests/cachekey/project/sources/local1.expected
+++ b/tests/cachekey/project/sources/local1.expected
@@ -1 +1 @@
-402b946498422abb82133fa5998b97f93b08d66ee1f332154413886687d8e7e9
\ No newline at end of file
+8920a31ae292e892d62692702bcd43bbb08af53187f1766dcba07c9eea9ea1f0
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/local2.expected b/tests/cachekey/project/sources/local2.expected
index e098d9b72..4df7d6598 100644
--- a/tests/cachekey/project/sources/local2.expected
+++ b/tests/cachekey/project/sources/local2.expected
@@ -1 +1 @@
-2eff179c8bab2441c2a6f115f5c313ece4a131c067b15e7fa23b58480341550d
\ No newline at end of file
+787bf27af094bce38eb749b20b75885a427f69cea66dd614912685d08e38745c
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch1.expected b/tests/cachekey/project/sources/patch1.expected
index c51922bf4..23c82305c 100644
--- a/tests/cachekey/project/sources/patch1.expected
+++ b/tests/cachekey/project/sources/patch1.expected
@@ -1 +1 @@
-4d2d8e8e92a20255a38d167abd93b5e6843f7b2738cdfe11ce64bc662fcaa886
\ No newline at end of file
+a821f7acacd978ba0fec07b8df3b06b5a275e20506a7b6518265214bb47bc4c5
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch2.expected b/tests/cachekey/project/sources/patch2.expected
index c9abff542..382bfb550 100644
--- a/tests/cachekey/project/sources/patch2.expected
+++ b/tests/cachekey/project/sources/patch2.expected
@@ -1 +1 @@
-21a53c232671f21cd717a4c94274e2decdba2c916dde56e030f944fe92ae785e
\ No newline at end of file
+81a1bf0b99d5888d9366fe3a69a508fa184bb884a16dd6dcc6b9c3276ec708d2
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch3.expected b/tests/cachekey/project/sources/patch3.expected
index 03c01c4ef..f3af6ae8c 100644
--- a/tests/cachekey/project/sources/patch3.expected
+++ b/tests/cachekey/project/sources/patch3.expected
@@ -1 +1 @@
-5ec3023b14bb2a44c94e205d4edc0e366d187357d6661bbc699f73e014b0630b
\ No newline at end of file
+f4c515a5ea11dd25215001c6bd3bf56f7cd431a6ea16f6d19273dc60c40d5439
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/pip1.expected b/tests/cachekey/project/sources/pip1.expected
index fc24de1b0..e950952c8 100644
--- a/tests/cachekey/project/sources/pip1.expected
+++ b/tests/cachekey/project/sources/pip1.expected
@@ -1 +1 @@
-60d6200ba331e3cff4b3255cb218569e387c571ee57761f6b3883b1283a937a2
\ No newline at end of file
+355a16ca00ac83589f97c5544102748930b61e8ca7210f888358cec28ecebebb
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/remote1.expected b/tests/cachekey/project/sources/remote1.expected
index 8d0c82516..c385682e8 100644
--- a/tests/cachekey/project/sources/remote1.expected
+++ b/tests/cachekey/project/sources/remote1.expected
@@ -1 +1 @@
-46f09da7ea078bf0d630ec2e14a668f8144df5175ee1c19c9af367873047b482
\ No newline at end of file
+2edc017887e7b64a83b768d3014e37dbec723dff8694a0e74a71ebb3b51c43cb
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/remote2.expected b/tests/cachekey/project/sources/remote2.expected
index 8e50da27a..743a4321e 100644
--- a/tests/cachekey/project/sources/remote2.expected
+++ b/tests/cachekey/project/sources/remote2.expected
@@ -1 +1 @@
-aaa2d0c22b40d2f9b87d40ff24c37769240edba4902c50fa948e8ab6c9848f6f
\ No newline at end of file
+f94a23cd2cdc858c90f47485f546c25f478ff44b8519d21e231cd1cf50395be2
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/tar1.expected b/tests/cachekey/project/sources/tar1.expected
index 5805323c3..e9ed6c005 100644
--- a/tests/cachekey/project/sources/tar1.expected
+++ b/tests/cachekey/project/sources/tar1.expected
@@ -1 +1 @@
-6b550e20ab7b8a11912ca14171e39c76badf7fa161a01c83d817c789b84e45c3
\ No newline at end of file
+5fadb949d4db843c325a07e75a5a9688a0a45a87f05e204f1703ab92eba3d30c
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/tar2.expected b/tests/cachekey/project/sources/tar2.expected
index bd4c41be3..bd13d194d 100644
--- a/tests/cachekey/project/sources/tar2.expected
+++ b/tests/cachekey/project/sources/tar2.expected
@@ -1 +1 @@
-f890b611cc83036b9c52dddf4eb2a02ccac5a73ae3ddcb34586406d7deba5a11
\ No newline at end of file
+4d1c56b1064d158c257c60aad45fdbf9c7fde404fea87d31c0736da629e66d77
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/zip1.expected b/tests/cachekey/project/sources/zip1.expected
index 892073dc2..6358b912e 100644
--- a/tests/cachekey/project/sources/zip1.expected
+++ b/tests/cachekey/project/sources/zip1.expected
@@ -1 +1 @@
-5393d513abcc88bd1cdbf03cff65d470285a906a43cf2e192ce0770fbceb933d
\ No newline at end of file
+7950c66d1ed51d1ee0a0980897a1addae574d14c8443c6c725800a64f607566d
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/zip2.expected b/tests/cachekey/project/sources/zip2.expected
index 3ca0b3542..e677e163c 100644
--- a/tests/cachekey/project/sources/zip2.expected
+++ b/tests/cachekey/project/sources/zip2.expected
@@ -1 +1 @@
-a03196c4878e0a585c54c0e75cabe069068d5e37b49f07ca95f5aeb6e3b1cf5b
\ No newline at end of file
+2bc0bb553d0e3de63eb93daf95f5fc202b340c85c3fd256fb2de88ec894dd63f
\ No newline at end of file
diff --git a/tests/cachekey/project/target.expected b/tests/cachekey/project/target.expected
index 276b5286b..0f142186f 100644
--- a/tests/cachekey/project/target.expected
+++ b/tests/cachekey/project/target.expected
@@ -1 +1 @@
-aeb288aa36ad3121822a5d5858a851670292828dedfa1840fd986bbbd59235a8
\ No newline at end of file
+27bbe79819a8fc3be20a33f606b0176caa7cb7d6845f37d053dd42800d6d0ef0
\ No newline at end of file
diff --git a/tests/examples/developing.py b/tests/examples/developing.py
index 2f53c8a33..53a554b86 100644
--- a/tests/examples/developing.py
+++ b/tests/examples/developing.py
@@ -76,7 +76,6 @@ def test_open_workspace(cli, tmpdir, datafiles):
 @pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
                     reason='Examples are written for x86-64')
 @pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with SANDBOX')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 @pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This is not meant to work with chroot')
 @pytest.mark.datafiles(DATA_DIR)
 def test_make_change_in_workspace(cli, tmpdir, datafiles):
diff --git a/tests/frontend/workspace.py b/tests/frontend/workspace.py
index a21538325..a49762cae 100644
--- a/tests/frontend/workspace.py
+++ b/tests/frontend/workspace.py
@@ -1142,6 +1142,9 @@ def test_external_push_pull(cli, datafiles, tmpdir_factory, guess_element):
         result.assert_success()
 
 
+# Attempting to track in an open workspace is not a sensible thing and it's not compatible with workspaces as plugin
+# sources: The new ref (if it differed from the old) would have been ignored regardless.
+# The user should be expected to simply close the workspace before tracking.
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.parametrize("guess_element", [True, False], ids=["guess", "no-guess"])
 def test_external_track(cli, datafiles, tmpdir_factory, guess_element):
@@ -1151,17 +1154,31 @@ def test_external_track(cli, datafiles, tmpdir_factory, guess_element):
     arg_elm = [element_name] if not guess_element else []
 
     # Delete the ref from the source so that we can detect if the
-    # element has been tracked
+    # element has been tracked after closing the workspace
     element_contents = _yaml.load(element_file)
+    ref1 = element_contents.get_sequence('sources').mapping_at(0).get_str('ref')
     del element_contents.get_sequence('sources').mapping_at(0)['ref']
     _yaml.roundtrip_dump(element_contents, element_file)
 
     result = cli.run(project=project, args=['-C', workspace, 'source', 'track', *arg_elm])
     result.assert_success()
 
-    # Element is tracked now
+    # Element is not tracked now
     element_contents = _yaml.load(element_file)
-    assert 'ref' in element_contents.get_sequence('sources').mapping_at(0)
+    assert 'ref' not in element_contents.get_sequence('sources').mapping_at(0)
+
+    # close the workspace
+    result = cli.run(project=project, args=['-C', workspace, 'workspace', 'close', *arg_elm])
+    result.assert_success()
+
+    # and retrack the element
+    result = cli.run(project=project, args=['source', 'track', element_name])
+    result.assert_success()
+
+    element_contents = _yaml.load(element_file)
+    ref2 = element_contents.get_sequence('sources').mapping_at(0).get_str('ref')
+    # these values should be equivalent
+    assert ref1 == ref2
 
 
 @pytest.mark.datafiles(DATA_DIR)
diff --git a/tests/integration/shell.py b/tests/integration/shell.py
index e4c521815..b0de76ef5 100644
--- a/tests/integration/shell.py
+++ b/tests/integration/shell.py
@@ -295,7 +295,6 @@ def test_cli_mount(cli, datafiles, path):
 # Test that we can see the workspace files in a shell
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_workspace_visible(cli, datafiles):
     project = str(datafiles)
     workspace = os.path.join(cli.directory, 'workspace')
@@ -374,7 +373,6 @@ def test_integration_devices(cli, datafiles):
 @pytest.mark.parametrize("build_shell", [("build"), ("nobuild")])
 @pytest.mark.parametrize("guess_element", [True, False], ids=["guess", "no-guess"])
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_integration_external_workspace(cli, tmpdir_factory, datafiles, build_shell, guess_element):
     tmpdir = tmpdir_factory.mktemp("")
     project = str(datafiles)
diff --git a/tests/integration/workspace.py b/tests/integration/workspace.py
index 9e74e574f..a4cb454d1 100644
--- a/tests/integration/workspace.py
+++ b/tests/integration/workspace.py
@@ -21,7 +21,6 @@ DATA_DIR = os.path.join(
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_workspace_mount(cli, datafiles):
     project = str(datafiles)
     workspace = os.path.join(cli.directory, 'workspace')
@@ -53,6 +52,7 @@ def test_workspace_mount_on_read_only_directory(cli, datafiles):
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 @pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
+@pytest.mark.xfail(reason="Incremental builds are currently incompatible with workspace source plugin.", strict=True)
 def test_workspace_commanddir(cli, datafiles):
     project = str(datafiles)
     workspace = os.path.join(cli.directory, 'workspace')
@@ -70,7 +70,6 @@ def test_workspace_commanddir(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_workspace_updated_dependency(cli, datafiles):
     project = str(datafiles)
     workspace = os.path.join(cli.directory, 'workspace')
@@ -125,7 +124,6 @@ def test_workspace_updated_dependency(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_workspace_update_dependency_failed(cli, datafiles):
     project = str(datafiles)
     workspace = os.path.join(cli.directory, 'workspace')
@@ -201,7 +199,6 @@ def test_workspace_update_dependency_failed(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_updated_dependency_nested(cli, datafiles):
     project = str(datafiles)
     workspace = os.path.join(cli.directory, 'workspace')
@@ -256,6 +253,7 @@ def test_updated_dependency_nested(cli, datafiles):
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 @pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
+@pytest.mark.xfail(reason="Incremental builds are currently incompatible with workspace source plugin.", strict=True)
 def test_incremental_configure_commands_run_only_once(cli, datafiles):
     project = str(datafiles)
     workspace = os.path.join(cli.directory, 'workspace')
@@ -334,7 +332,6 @@ def test_workspace_missing_last_successful(cli, datafiles):
 # Check that we can still read failed workspace logs
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_workspace_failed_logs(cli, datafiles):
     project = str(datafiles)
     workspace = os.path.join(cli.directory, 'failing_amhello')