Move source from 'buildstream' to 'src/buildstream'

This was discussed in #1008.

Fixes #1009.

1 files changed, 449 insertions, 0 deletions

diff --git a/src/buildstream/_artifact.py b/src/buildstream/_artifact.py
new file mode 100644
index 000000000..c353a5151
--- /dev/null
+++ b/src/buildstream/_artifact.py
@@ -0,0 +1,449 @@
+#
+#  Copyright (C) 2019 Codethink Limited
+#  Copyright (C) 2019 Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Tom Pollard <tom.pollard@codethink.co.uk>
+#        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+
+"""
+Artifact
+=========
+
+Implementation of the Artifact class which aims to 'abstract' direct
+artifact composite interaction away from Element class
+
+"""
+
+import os
+import tempfile
+
+from ._protos.buildstream.v2.artifact_pb2 import Artifact as ArtifactProto
+from . import _yaml
+from . import utils
+from .types import Scope
+from .storage._casbaseddirectory import CasBasedDirectory
+
+
+# An Artifact class to abtract artifact operations
+# from the Element class
+#
+# Args:
+#     element (Element): The Element object
+#     context (Context): The BuildStream context
+#     strong_key (str): The elements strong cache key, dependant on context
+#     weak_key (str): The elements weak cache key
+#
+class Artifact():
+
+    version = 0
+
+    def __init__(self, element, context, *, strong_key=None, weak_key=None):
+        self._element = element
+        self._context = context
+        self._artifacts = context.artifactcache
+        self._cache_key = strong_key
+        self._weak_cache_key = weak_key
+        self._artifactdir = context.artifactdir
+        self._cas = context.get_cascache()
+        self._tmpdir = context.tmpdir
+        self._proto = None
+
+        self._metadata_keys = None                    # Strong and weak key tuple extracted from the artifact
+        self._metadata_dependencies = None             # Dictionary of dependency strong keys from the artifact
+        self._metadata_workspaced = None              # Boolean of whether it's a workspaced artifact
+        self._metadata_workspaced_dependencies = None  # List of which dependencies are workspaced from the artifact
+        self._cached = None                          # Boolean of whether the artifact is cached
+
+    # get_files():
+    #
+    # Get a virtual directory for the artifact files content
+    #
+    # Returns:
+    #    (Directory): The virtual directory object
+    #
+    def get_files(self):
+        files_digest = self._get_field_digest("files")
+
+        return CasBasedDirectory(self._cas, digest=files_digest)
+
+    # get_buildtree():
+    #
+    # Get a virtual directory for the artifact buildtree content
+    #
+    # Returns:
+    #    (Directory): The virtual directory object
+    #
+    def get_buildtree(self):
+        buildtree_digest = self._get_field_digest("buildtree")
+
+        return CasBasedDirectory(self._cas, digest=buildtree_digest)
+
+    # get_extract_key():
+    #
+    # Get the key used to extract the artifact
+    #
+    # Returns:
+    #    (str): The key
+    #
+    def get_extract_key(self):
+        return self._cache_key or self._weak_cache_key
+
+    # cache():
+    #
+    # Create the artifact and commit to cache
+    #
+    # Args:
+    #    rootdir (str): An absolute path to the temp rootdir for artifact construct
+    #    sandbox_build_dir (Directory): Virtual Directory object for the sandbox build-root
+    #    collectvdir (Directory): Virtual Directoy object from within the sandbox for collection
+    #    buildresult (tuple): bool, short desc and detailed desc of result
+    #    publicdata (dict): dict of public data to commit to artifact metadata
+    #
+    # Returns:
+    #    (int): The size of the newly cached artifact
+    #
+    def cache(self, rootdir, sandbox_build_dir, collectvdir, buildresult, publicdata):
+
+        context = self._context
+        element = self._element
+        size = 0
+
+        filesvdir = None
+        buildtreevdir = None
+
+        artifact = ArtifactProto()
+
+        artifact.version = self.version
+
+        # Store result
+        artifact.build_success = buildresult[0]
+        artifact.build_error = buildresult[1]
+        artifact.build_error_details = "" if not buildresult[2] else buildresult[2]
+
+        # Store keys
+        artifact.strong_key = self._cache_key
+        artifact.weak_key = self._weak_cache_key
+
+        artifact.was_workspaced = bool(element._get_workspace())
+
+        # Store files
+        if collectvdir:
+            filesvdir = CasBasedDirectory(cas_cache=self._cas)
+            filesvdir.import_files(collectvdir)
+            artifact.files.CopyFrom(filesvdir._get_digest())
+            size += filesvdir.get_size()
+
+        # Store public data
+        with tempfile.NamedTemporaryFile(dir=self._tmpdir) as tmp:
+            _yaml.dump(_yaml.node_sanitize(publicdata), tmp.name)
+            public_data_digest = self._cas.add_object(path=tmp.name, link_directly=True)
+            artifact.public_data.CopyFrom(public_data_digest)
+            size += public_data_digest.size_bytes
+
+        # store build dependencies
+        for e in element.dependencies(Scope.BUILD):
+            new_build = artifact.build_deps.add()
+            new_build.element_name = e.name
+            new_build.cache_key = e._get_cache_key()
+            new_build.was_workspaced = bool(e._get_workspace())
+
+        # Store log file
+        log_filename = context.get_log_filename()
+        if log_filename:
+            digest = self._cas.add_object(path=log_filename)
+            element._build_log_path = self._cas.objpath(digest)
+            log = artifact.logs.add()
+            log.name = os.path.basename(log_filename)
+            log.digest.CopyFrom(digest)
+            size += log.digest.size_bytes
+
+        # Store build tree
+        if sandbox_build_dir:
+            buildtreevdir = CasBasedDirectory(cas_cache=self._cas)
+            buildtreevdir.import_files(sandbox_build_dir)
+            artifact.buildtree.CopyFrom(buildtreevdir._get_digest())
+            size += buildtreevdir.get_size()
+
+        os.makedirs(os.path.dirname(os.path.join(
+            self._artifactdir, element.get_artifact_name())), exist_ok=True)
+        keys = utils._deduplicate([self._cache_key, self._weak_cache_key])
+        for key in keys:
+            path = os.path.join(self._artifactdir, element.get_artifact_name(key=key))
+            with open(path, mode='w+b') as f:
+                f.write(artifact.SerializeToString())
+
+        return size
+
+    # cached_buildtree()
+    #
+    # Check if artifact is cached with expected buildtree. A
+    # buildtree will not be present if the rest of the partial artifact
+    # is not cached.
+    #
+    # Returns:
+    #     (bool): True if artifact cached with buildtree, False if
+    #             missing expected buildtree. Note this only confirms
+    #             if a buildtree is present, not its contents.
+    #
+    def cached_buildtree(self):
+
+        buildtree_digest = self._get_field_digest("buildtree")
+        if buildtree_digest:
+            return self._cas.contains_directory(buildtree_digest, with_files=True)
+        else:
+            return False
+
+    # buildtree_exists()
+    #
+    # Check if artifact was created with a buildtree. This does not check
+    # whether the buildtree is present in the local cache.
+    #
+    # Returns:
+    #     (bool): True if artifact was created with buildtree
+    #
+    def buildtree_exists(self):
+
+        artifact = self._get_proto()
+        return bool(str(artifact.buildtree))
+
+    # load_public_data():
+    #
+    # Loads the public data from the cached artifact
+    #
+    # Returns:
+    #    (dict): The artifacts cached public data
+    #
+    def load_public_data(self):
+
+        # Load the public data from the artifact
+        artifact = self._get_proto()
+        meta_file = self._cas.objpath(artifact.public_data)
+        data = _yaml.load(meta_file, shortname='public.yaml')
+
+        return data
+
+    # load_build_result():
+    #
+    # Load the build result from the cached artifact
+    #
+    # Returns:
+    #    (bool): Whether the artifact of this element present in the artifact cache is of a success
+    #    (str): Short description of the result
+    #    (str): Detailed description of the result
+    #
+    def load_build_result(self):
+
+        artifact = self._get_proto()
+        build_result = (artifact.build_success,
+                        artifact.build_error,
+                        artifact.build_error_details)
+
+        return build_result
+
+    # get_metadata_keys():
+    #
+    # Retrieve the strong and weak keys from the given artifact.
+    #
+    # Returns:
+    #    (str): The strong key
+    #    (str): The weak key
+    #
+    def get_metadata_keys(self):
+
+        if self._metadata_keys is not None:
+            return self._metadata_keys
+
+        # Extract proto
+        artifact = self._get_proto()
+
+        strong_key = artifact.strong_key
+        weak_key = artifact.weak_key
+
+        self._metadata_keys = (strong_key, weak_key)
+
+        return self._metadata_keys
+
+    # get_metadata_dependencies():
+    #
+    # Retrieve the hash of dependency keys from the given artifact.
+    #
+    # Returns:
+    #    (dict): A dictionary of element names and their keys
+    #
+    def get_metadata_dependencies(self):
+
+        if self._metadata_dependencies is not None:
+            return self._metadata_dependencies
+
+        # Extract proto
+        artifact = self._get_proto()
+
+        self._metadata_dependencies = {dep.element_name: dep.cache_key for dep in artifact.build_deps}
+
+        return self._metadata_dependencies
+
+    # get_metadata_workspaced():
+    #
+    # Retrieve the hash of dependency from the given artifact.
+    #
+    # Returns:
+    #    (bool): Whether the given artifact was workspaced
+    #
+    def get_metadata_workspaced(self):
+
+        if self._metadata_workspaced is not None:
+            return self._metadata_workspaced
+
+        # Extract proto
+        artifact = self._get_proto()
+
+        self._metadata_workspaced = artifact.was_workspaced
+
+        return self._metadata_workspaced
+
+    # get_metadata_workspaced_dependencies():
+    #
+    # Retrieve the hash of workspaced dependencies keys from the given artifact.
+    #
+    # Returns:
+    #    (list): List of which dependencies are workspaced
+    #
+    def get_metadata_workspaced_dependencies(self):
+
+        if self._metadata_workspaced_dependencies is not None:
+            return self._metadata_workspaced_dependencies
+
+        # Extract proto
+        artifact = self._get_proto()
+
+        self._metadata_workspaced_dependencies = [dep.element_name for dep in artifact.build_deps
+                                                  if dep.was_workspaced]
+
+        return self._metadata_workspaced_dependencies
+
+    # cached():
+    #
+    # Check whether the artifact corresponding to the stored cache key is
+    # available. This also checks whether all required parts of the artifact
+    # are available, which may depend on command and configuration. The cache
+    # key used for querying is dependant on the current context.
+    #
+    # Returns:
+    #     (bool): Whether artifact is in local cache
+    #
+    def cached(self):
+
+        if self._cached is not None:
+            return self._cached
+
+        context = self._context
+
+        artifact = self._get_proto()
+
+        if not artifact:
+            self._cached = False
+            return False
+
+        # Determine whether directories are required
+        require_directories = context.require_artifact_directories
+        # Determine whether file contents are required as well
+        require_files = (context.require_artifact_files or
+                         self._element._artifact_files_required())
+
+        # Check whether 'files' subdirectory is available, with or without file contents
+        if (require_directories and str(artifact.files) and
+                not self._cas.contains_directory(artifact.files, with_files=require_files)):
+            self._cached = False
+            return False
+
+        self._cached = True
+        return True
+
+    # cached_logs()
+    #
+    # Check if the artifact is cached with log files.
+    #
+    # Returns:
+    #     (bool): True if artifact is cached with logs, False if
+    #             element not cached or missing logs.
+    #
+    def cached_logs(self):
+        if not self._element._cached():
+            return False
+
+        artifact = self._get_proto()
+
+        for logfile in artifact.logs:
+            if not self._cas.contains(logfile.digest.hash):
+                return False
+
+        return True
+
+    # reset_cached()
+    #
+    # Allow the Artifact to query the filesystem to determine whether it
+    # is cached or not.
+    #
+    # NOTE: Due to the fact that a normal buildstream run does not make an
+    # artifact *not* cached (`bst artifact delete` can do so, but doesn't
+    # query the Artifact afterwards), it does not update_cached if the
+    # artifact is already cached. If a cached artifact ever has its key
+    # changed, this will need to be revisited.
+    #
+    def reset_cached(self):
+        if self._cached is False:
+            self._cached = None
+
+    # _get_proto()
+    #
+    # Returns:
+    #     (Artifact): Artifact proto
+    #
+    def _get_proto(self):
+        # Check if we've already cached the proto object
+        if self._proto is not None:
+            return self._proto
+
+        key = self.get_extract_key()
+
+        proto_path = os.path.join(self._artifactdir,
+                                  self._element.get_artifact_name(key=key))
+        artifact = ArtifactProto()
+        try:
+            with open(proto_path, mode='r+b') as f:
+                artifact.ParseFromString(f.read())
+        except FileNotFoundError:
+            return None
+
+        os.utime(proto_path)
+        # Cache the proto object
+        self._proto = artifact
+
+        return self._proto
+
+    # _get_artifact_field()
+    #
+    # Returns:
+    #     (Digest): Digest of field specified
+    #
+    def _get_field_digest(self, field):
+        artifact_proto = self._get_proto()
+        digest = getattr(artifact_proto, field)
+        if not str(digest):
+            return None
+
+        return digest