diff options
author | Jim MacArthur <jim.macarthur@codethink.co.uk> | 2018-05-08 16:19:22 +0100 |
---|---|---|
committer | Jim MacArthur <jim.macarthur@codethink.co.uk> | 2018-08-01 13:07:33 +0100 |
commit | 849724ce197850afb4094a5484a5aabbb8a192e4 (patch) | |
tree | 872c34516462f74bacb8179098571409345330ec | |
parent | fe1a342895a99d57d913325bd2b61bede908049d (diff) | |
download | buildstream-849724ce197850afb4094a5484a5aabbb8a192e4.tar.gz |
element.py: Conversion to use virtual directories
-rw-r--r-- | buildstream/element.py | 113 |
1 files changed, 62 insertions, 51 deletions
diff --git a/buildstream/element.py b/buildstream/element.py index 9b9d41319..2218ef94b 100644 --- a/buildstream/element.py +++ b/buildstream/element.py @@ -80,7 +80,6 @@ from collections import Mapping, OrderedDict from contextlib import contextmanager from enum import Enum import tempfile -import time import shutil from . import _yaml @@ -97,6 +96,10 @@ from . import _site from ._platform import Platform from .sandbox._config import SandboxConfig +from .storage.directory import Directory +from .storage._filebaseddirectory import FileBasedDirectory, VirtualDirectoryError + + # _KeyStrength(): # # Strength of cache key @@ -198,7 +201,7 @@ class Element(Plugin): """Whether to raise exceptions if an element uses Sandbox.get_directory instead of Sandbox.get_virtual_directory. - *Since: 1.2* + *Since: 1.4* """ def __init__(self, context, project, artifacts, meta, plugin_conf): @@ -633,10 +636,10 @@ class Element(Plugin): # Hard link it into the staging area # - basedir = sandbox.get_directory() - stagedir = basedir \ + vbasedir = sandbox.get_virtual_directory() + vstagedir = vbasedir \ if path is None \ - else os.path.join(basedir, path.lstrip(os.sep)) + else vbasedir.descend(path.lstrip(os.sep).split(os.sep)) files = list(self.__compute_splits(include, exclude, orphans)) @@ -648,15 +651,8 @@ class Element(Plugin): link_files = files copy_files = [] - link_result = utils.link_files(artifact, stagedir, files=link_files, - report_written=True) - copy_result = utils.copy_files(artifact, stagedir, files=copy_files, - report_written=True) - - cur_time = time.time() - - for f in copy_result.files_written: - os.utime(os.path.join(stagedir, f), times=(cur_time, cur_time)) + link_result = vstagedir.import_files(artifact, files=link_files, report_written=True, can_link=True) + copy_result = vstagedir.import_files(artifact, files=copy_files, report_written=True, update_utimes=True) return link_result.combine(copy_result) @@ -1365,40 +1361,45 @@ class Element(Plugin): sandbox._set_mount_source(directory, workspace.get_absolute_path()) # Stage all sources that need to be copied - sandbox_root = sandbox.get_directory() - host_directory = os.path.join(sandbox_root, directory.lstrip(os.sep)) - self._stage_sources_at(host_directory, mount_workspaces=mount_workspaces) + sandbox_vroot = sandbox.get_virtual_directory() + host_vdirectory = sandbox_vroot.descend(directory.lstrip(os.sep).split(os.sep), create=True) + self._stage_sources_at(host_vdirectory, mount_workspaces=mount_workspaces) # _stage_sources_at(): # # Stage this element's sources to a directory # # Args: - # directory (str): An absolute path to stage the sources at + # vdirectory (:class:`.storage.Directory`): A virtual directory object to stage sources into. # mount_workspaces (bool): mount workspaces if True, copy otherwise # - def _stage_sources_at(self, directory, mount_workspaces=True): + def _stage_sources_at(self, vdirectory, mount_workspaces=True): with self.timed_activity("Staging sources", silent_nested=True): - if os.path.isdir(directory) and os.listdir(directory): - raise ElementError("Staging directory '{}' is not empty".format(directory)) - - workspace = self._get_workspace() - if workspace: - # If mount_workspaces is set and we're doing incremental builds, - # the workspace is already mounted into the sandbox. - if not (mount_workspaces and self.__can_build_incrementally()): - with self.timed_activity("Staging local files at {}".format(workspace.path)): - workspace.stage(directory) - else: - # No workspace, stage directly - for source in self.sources(): - source._stage(directory) - + if not isinstance(vdirectory, Directory): + vdirectory = FileBasedDirectory(vdirectory) + if not vdirectory.is_empty(): + raise ElementError("Staging directory '{}' is not empty".format(vdirectory)) + + with tempfile.TemporaryDirectory() as temp_staging_directory: + + workspace = self._get_workspace() + if workspace: + # If mount_workspaces is set and we're doing incremental builds, + # the workspace is already mounted into the sandbox. + if not (mount_workspaces and self.__can_build_incrementally()): + with self.timed_activity("Staging local files at {}".format(workspace.path)): + workspace.stage(temp_staging_directory) + else: + # No workspace, stage directly + for source in self.sources(): + source._stage(temp_staging_directory) + + vdirectory.import_files(temp_staging_directory) # Ensure deterministic mtime of sources at build time - utils._set_deterministic_mtime(directory) + vdirectory.set_deterministic_mtime() # Ensure deterministic owners of sources at build time - utils._set_deterministic_user(directory) + vdirectory.set_deterministic_user() # _set_required(): # @@ -1514,7 +1515,7 @@ class Element(Plugin): with _signals.terminator(cleanup_rootdir), \ self.__sandbox(rootdir, output_file, output_file, self.__sandbox_config) as sandbox: # nopep8 - sandbox_root = sandbox.get_directory() + sandbox_vroot = sandbox.get_virtual_directory() # By default, the dynamic public data is the same as the static public data. # The plugin's assemble() method may modify this, though. @@ -1546,11 +1547,11 @@ class Element(Plugin): # workspace = self._get_workspace() if workspace and self.__staged_sources_directory: - sandbox_root = sandbox.get_directory() - sandbox_path = os.path.join(sandbox_root, - self.__staged_sources_directory.lstrip(os.sep)) + sandbox_vroot = sandbox.get_virtual_directory() + path_components = self.__staged_sources_directory.lstrip(os.sep).split(os.sep) + sandbox_vpath = sandbox_vroot.descend(path_components) try: - utils.copy_files(workspace.path, sandbox_path) + sandbox_vpath.import_files(workspace.path) except UtilError as e: self.warn("Failed to preserve workspace state for failed build sysroot: {}" .format(e)) @@ -1562,7 +1563,11 @@ class Element(Plugin): raise finally: if collect is not None: - collectdir = os.path.join(sandbox_root, collect.lstrip(os.sep)) + try: + collectvdir = sandbox_vroot.descend(collect.lstrip(os.sep).split(os.sep)) + except VirtualDirectoryError: + # No collect directory existed + collectvdir = None # Create artifact directory structure assembledir = os.path.join(rootdir, 'artifact') @@ -1571,20 +1576,26 @@ class Element(Plugin): metadir = os.path.join(assembledir, 'meta') buildtreedir = os.path.join(assembledir, 'buildtree') os.mkdir(assembledir) - if collect is not None and os.path.exists(collectdir): + if collect is not None and collectvdir is not None: os.mkdir(filesdir) os.mkdir(logsdir) os.mkdir(metadir) os.mkdir(buildtreedir) # Hard link files from collect dir to files directory - if collect is not None and os.path.exists(collectdir): - utils.link_files(collectdir, filesdir) - - sandbox_build_dir = os.path.join(sandbox_root, self.get_variable('build-root').lstrip(os.sep)) - # Hard link files from build-root dir to buildtreedir directory - if os.path.isdir(sandbox_build_dir): - utils.link_files(sandbox_build_dir, buildtreedir) + if collect is not None and collectvdir is not None: + collectvdir.export_files(filesdir, can_link=True) + + try: + sandbox_build_dir = sandbox_vroot.descend( + self.get_variable('build-root').lstrip(os.sep).split(os.sep)) + # Hard link files from build-root dir to buildtreedir directory + sandbox_build_dir.export_files(buildtreedir) + except VirtualDirectoryError: + # Directory could not be found. Pre-virtual + # directory behaviour was to continue silently + # if the directory could not be found. + pass # Copy build log log_filename = context.get_log_filename() @@ -1632,7 +1643,7 @@ class Element(Plugin): self.__artifact_size = utils._get_dir_size(assembledir) self.__artifacts.commit(self, assembledir, self.__get_cache_keys_for_commit()) - if collect is not None and not os.path.exists(collectdir): + if collect is not None and collectvdir is None: raise ElementError( "Directory '{}' was not found inside the sandbox, " "unable to collect artifact contents" |