From 5e10e2e81bee10bab56ed5b8190f8332170d7096 Mon Sep 17 00:00:00 2001 From: Raoul Hidalgo Charman Date: Fri, 25 Jan 2019 16:43:36 +0000 Subject: CASQuota: Move cache check methods into new Class This sits in Context allowing artifact cache to check the cas quota while not being used for CASServer. A lot of code that checks cache quota has been touched. Part of #870 --- buildstream/_artifactcache.py | 289 ++------------------------- buildstream/_cas/__init__.py | 2 +- buildstream/_cas/cascache.py | 290 ++++++++++++++++++++++++++-- buildstream/_context.py | 31 ++- buildstream/_frontend/status.py | 2 +- buildstream/_frontend/widget.py | 2 +- buildstream/_scheduler/jobs/cachesizejob.py | 6 +- buildstream/_scheduler/jobs/cleanupjob.py | 8 +- tests/artifactcache/cache_size.py | 4 +- tests/artifactcache/expiry.py | 8 +- tests/testutils/artifactshare.py | 1 - 11 files changed, 339 insertions(+), 304 deletions(-) diff --git a/buildstream/_artifactcache.py b/buildstream/_artifactcache.py index cab9bff3c..b73304fac 100644 --- a/buildstream/_artifactcache.py +++ b/buildstream/_artifactcache.py @@ -22,12 +22,12 @@ import os from collections.abc import Mapping from .types import _KeyStrength -from ._exceptions import ArtifactError, CASError, LoadError, LoadErrorReason +from ._exceptions import ArtifactError, CASError from ._message import Message, MessageType from . import utils from . import _yaml -from ._cas import CASRemote, CASRemoteSpec +from ._cas import CASRemote, CASRemoteSpec, CASCacheUsage from .storage._casbaseddirectory import CasBasedDirectory @@ -46,39 +46,6 @@ class ArtifactCacheSpec(CASRemoteSpec): pass -# ArtifactCacheUsage -# -# A simple object to report the current artifact cache -# usage details. -# -# Note that this uses the user configured cache quota -# rather than the internal quota with protective headroom -# removed, to provide a more sensible value to display to -# the user. -# -# Args: -# artifacts (ArtifactCache): The artifact cache to get the status of -# -class ArtifactCacheUsage(): - - def __init__(self, artifacts): - context = artifacts.context - self.quota_config = context.config_cache_quota # Configured quota - self.quota_size = artifacts._cache_quota_original # Resolved cache quota in bytes - self.used_size = artifacts.get_cache_size() # Size used by artifacts in bytes - self.used_percent = 0 # Percentage of the quota used - if self.quota_size is not None: - self.used_percent = int(self.used_size * 100 / self.quota_size) - - # Formattable into a human readable string - # - def __str__(self): - return "{} / {} ({}%)" \ - .format(utils._pretty_size(self.used_size, dec_places=1), - self.quota_config, - self.used_percent) - - # An ArtifactCache manages artifacts. # # Args: @@ -90,16 +57,14 @@ class ArtifactCache(): self.extractdir = context.extractdir self.cas = context.get_cascache() + self.casquota = context.get_casquota() + self.casquota._calculate_cache_quota() self.global_remote_specs = [] self.project_remote_specs = {} self._required_elements = set() # The elements required for this session - self._cache_size = None # The current cache size, sometimes it's an estimate - self._cache_quota = None # The cache quota - self._cache_quota_original = None # The cache quota as specified by the user, in bytes - self._cache_quota_headroom = None # The headroom in bytes before reaching the quota or full disk - self._cache_lower_threshold = None # The target cache size for a cleanup + self._remotes_setup = False # Check to prevent double-setup of remotes # Per-project list of _CASRemote instances. @@ -110,8 +75,6 @@ class ArtifactCache(): os.makedirs(self.extractdir, exist_ok=True) - self._calculate_cache_quota() - # setup_remotes(): # # Sets up which remotes to use @@ -235,7 +198,7 @@ class ArtifactCache(): space_saved = 0 # Start off with an announcement with as much info as possible - volume_size, volume_avail = self._get_cache_volume_size() + volume_size, volume_avail = self.casquota._get_cache_volume_size() self._message(MessageType.STATUS, "Starting cache cleanup", detail=("Elements required by the current build plan: {}\n" + "User specified quota: {} ({})\n" + @@ -243,8 +206,8 @@ class ArtifactCache(): "Cache volume: {} total, {} available") .format(len(self._required_elements), context.config_cache_quota, - utils._pretty_size(self._cache_quota_original, dec_places=2), - utils._pretty_size(self.get_cache_size(), dec_places=2), + utils._pretty_size(self.casquota._cache_quota, dec_places=2), + utils._pretty_size(self.casquota.get_cache_size(), dec_places=2), utils._pretty_size(volume_size, dec_places=2), utils._pretty_size(volume_avail, dec_places=2))) @@ -261,9 +224,11 @@ class ArtifactCache(): ]) # Do a real computation of the cache size once, just in case - self.compute_cache_size() + self.casquota.compute_cache_size() + usage = CASCacheUsage(self.casquota) + self._message(MessageType.STATUS, "Cache usage recomputed: {}".format(usage)) - while self.get_cache_size() >= self._cache_lower_threshold: + while self.casquota.get_cache_size() >= self.casquota._cache_lower_threshold: try: to_remove = artifacts.pop(0) except IndexError: @@ -280,7 +245,7 @@ class ArtifactCache(): "Please increase the cache-quota in {} and/or make more disk space." .format(removed_ref_count, utils._pretty_size(space_saved, dec_places=2), - utils._pretty_size(self.get_cache_size(), dec_places=2), + utils._pretty_size(self.casquota.get_cache_size(), dec_places=2), len(self._required_elements), (context.config_origin or default_conf))) @@ -306,7 +271,7 @@ class ArtifactCache(): to_remove)) # Remove the size from the removed size - self.set_cache_size(self._cache_size - size) + self.casquota.set_cache_size(self.casquota._cache_size - size) # User callback # @@ -322,29 +287,12 @@ class ArtifactCache(): "Cache usage is now: {}") .format(removed_ref_count, utils._pretty_size(space_saved, dec_places=2), - utils._pretty_size(self.get_cache_size(), dec_places=2))) - - return self.get_cache_size() - - # compute_cache_size() - # - # Computes the real artifact cache size by calling - # the abstract calculate_cache_size() method. - # - # Returns: - # (int): The size of the artifact cache. - # - def compute_cache_size(self): - old_cache_size = self._cache_size - new_cache_size = self.cas.calculate_cache_size() - - if old_cache_size != new_cache_size: - self._cache_size = new_cache_size + utils._pretty_size(self.casquota.get_cache_size(), dec_places=2))) - usage = ArtifactCacheUsage(self) - self._message(MessageType.STATUS, "Cache usage recomputed: {}".format(usage)) + return self.casquota.get_cache_size() - return self._cache_size + def full(self): + return self.casquota.full() # add_artifact_size() # @@ -355,71 +303,10 @@ class ArtifactCache(): # artifact_size (int): The size to add. # def add_artifact_size(self, artifact_size): - cache_size = self.get_cache_size() + cache_size = self.casquota.get_cache_size() cache_size += artifact_size - self.set_cache_size(cache_size) - - # get_cache_size() - # - # Fetches the cached size of the cache, this is sometimes - # an estimate and periodically adjusted to the real size - # when a cache size calculation job runs. - # - # When it is an estimate, the value is either correct, or - # it is greater than the actual cache size. - # - # Returns: - # (int) An approximation of the artifact cache size, in bytes. - # - def get_cache_size(self): - - # If we don't currently have an estimate, figure out the real cache size. - if self._cache_size is None: - stored_size = self._read_cache_size() - if stored_size is not None: - self._cache_size = stored_size - else: - self.compute_cache_size() - - return self._cache_size - - # set_cache_size() - # - # Forcefully set the overall cache size. - # - # This is used to update the size in the main process after - # having calculated in a cleanup or a cache size calculation job. - # - # Args: - # cache_size (int): The size to set. - # - def set_cache_size(self, cache_size): - - assert cache_size is not None - - self._cache_size = cache_size - self._write_cache_size(self._cache_size) - - # full() - # - # Checks if the artifact cache is full, either - # because the user configured quota has been exceeded - # or because the underlying disk is almost full. - # - # Returns: - # (bool): True if the artifact cache is full - # - def full(self): - - if self.get_cache_size() > self._cache_quota: - return True - - _, volume_avail = self._get_cache_volume_size() - if volume_avail < self._cache_quota_headroom: - return True - - return False + self.casquota.set_cache_size(cache_size) # preflight(): # @@ -885,142 +772,6 @@ class ArtifactCache(): with self.context.timed_activity("Initializing remote caches", silent_nested=True): self.initialize_remotes(on_failure=remote_failed) - # _write_cache_size() - # - # Writes the given size of the artifact to the cache's size file - # - # Args: - # size (int): The size of the artifact cache to record - # - def _write_cache_size(self, size): - assert isinstance(size, int) - size_file_path = os.path.join(self.context.casdir, CACHE_SIZE_FILE) - with utils.save_file_atomic(size_file_path, "w") as f: - f.write(str(size)) - - # _read_cache_size() - # - # Reads and returns the size of the artifact cache that's stored in the - # cache's size file - # - # Returns: - # (int): The size of the artifact cache, as recorded in the file - # - def _read_cache_size(self): - size_file_path = os.path.join(self.context.casdir, CACHE_SIZE_FILE) - - if not os.path.exists(size_file_path): - return None - - with open(size_file_path, "r") as f: - size = f.read() - - try: - num_size = int(size) - except ValueError as e: - raise ArtifactError("Size '{}' parsed from '{}' was not an integer".format( - size, size_file_path)) from e - - return num_size - - # _calculate_cache_quota() - # - # Calculates and sets the cache quota and lower threshold based on the - # quota set in Context. - # It checks that the quota is both a valid expression, and that there is - # enough disk space to satisfy that quota - # - def _calculate_cache_quota(self): - # Headroom intended to give BuildStream a bit of leeway. - # This acts as the minimum size of cache_quota and also - # is taken from the user requested cache_quota. - # - if 'BST_TEST_SUITE' in os.environ: - self._cache_quota_headroom = 0 - else: - self._cache_quota_headroom = 2e9 - - try: - cache_quota = utils._parse_size(self.context.config_cache_quota, - self.context.casdir) - except utils.UtilError as e: - raise LoadError(LoadErrorReason.INVALID_DATA, - "{}\nPlease specify the value in bytes or as a % of full disk space.\n" - "\nValid values are, for example: 800M 10G 1T 50%\n" - .format(str(e))) from e - - total_size, available_space = self._get_cache_volume_size() - cache_size = self.get_cache_size() - - # Ensure system has enough storage for the cache_quota - # - # If cache_quota is none, set it to the maximum it could possibly be. - # - # Also check that cache_quota is at least as large as our headroom. - # - if cache_quota is None: # Infinity, set to max system storage - cache_quota = cache_size + available_space - if cache_quota < self._cache_quota_headroom: # Check minimum - raise LoadError(LoadErrorReason.INVALID_DATA, - "Invalid cache quota ({}): ".format(utils._pretty_size(cache_quota)) + - "BuildStream requires a minimum cache quota of 2G.") - elif cache_quota > total_size: - # A quota greater than the total disk size is certianly an error - raise ArtifactError("Your system does not have enough available " + - "space to support the cache quota specified.", - detail=("You have specified a quota of {quota} total disk space.\n" + - "The filesystem containing {local_cache_path} only " + - "has {total_size} total disk space.") - .format( - quota=self.context.config_cache_quota, - local_cache_path=self.context.casdir, - total_size=utils._pretty_size(total_size)), - reason='insufficient-storage-for-quota') - elif cache_quota > cache_size + available_space: - # The quota does not fit in the available space, this is a warning - if '%' in self.context.config_cache_quota: - available = (available_space / total_size) * 100 - available = '{}% of total disk space'.format(round(available, 1)) - else: - available = utils._pretty_size(available_space) - - self._message(MessageType.WARN, - "Your system does not have enough available " + - "space to support the cache quota specified.", - detail=("You have specified a quota of {quota} total disk space.\n" + - "The filesystem containing {local_cache_path} only " + - "has {available_size} available.") - .format(quota=self.context.config_cache_quota, - local_cache_path=self.context.casdir, - available_size=available)) - - # Place a slight headroom (2e9 (2GB) on the cache_quota) into - # cache_quota to try and avoid exceptions. - # - # Of course, we might still end up running out during a build - # if we end up writing more than 2G, but hey, this stuff is - # already really fuzzy. - # - self._cache_quota_original = cache_quota - self._cache_quota = cache_quota - self._cache_quota_headroom - self._cache_lower_threshold = self._cache_quota / 2 - - # _get_cache_volume_size() - # - # Get the available space and total space for the volume on - # which the artifact cache is located. - # - # Returns: - # (int): The total number of bytes on the volume - # (int): The number of available bytes on the volume - # - # NOTE: We use this stub to allow the test cases - # to override what an artifact cache thinks - # about it's disk size and available bytes. - # - def _get_cache_volume_size(self): - return utils._get_volume_size(self.context.casdir) - # _configured_remote_artifact_cache_specs(): # diff --git a/buildstream/_cas/__init__.py b/buildstream/_cas/__init__.py index a88e41371..46bd9567f 100644 --- a/buildstream/_cas/__init__.py +++ b/buildstream/_cas/__init__.py @@ -17,5 +17,5 @@ # Authors: # Tristan Van Berkom -from .cascache import CASCache +from .cascache import CASCache, CASQuota, CASCacheUsage from .casremote import CASRemote, CASRemoteSpec diff --git a/buildstream/_cas/cascache.py b/buildstream/_cas/cascache.py index 792bf3eb9..fe25efce6 100644 --- a/buildstream/_cas/cascache.py +++ b/buildstream/_cas/cascache.py @@ -32,17 +32,53 @@ from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 from .._protos.buildstream.v2 import buildstream_pb2 from .. import utils -from .._exceptions import CASCacheError +from .._exceptions import CASCacheError, LoadError, LoadErrorReason +from .._message import Message, MessageType from .casremote import BlobNotFound, _CASBatchRead, _CASBatchUpdate _BUFFER_SIZE = 65536 +CACHE_SIZE_FILE = "cache_size" + + +# CASCacheUsage +# +# A simple object to report the current CAS cache usage details. +# +# Note that this uses the user configured cache quota +# rather than the internal quota with protective headroom +# removed, to provide a more sensible value to display to +# the user. +# +# Args: +# cas (CASQuota): The CAS cache to get the status of +# +class CASCacheUsage(): + + def __init__(self, casquota): + self.quota_config = casquota._config_cache_quota # Configured quota + self.quota_size = casquota._cache_quota_original # Resolved cache quota in bytes + self.used_size = casquota.get_cache_size() # Size used by artifacts in bytes + self.used_percent = 0 # Percentage of the quota used + if self.quota_size is not None: + self.used_percent = int(self.used_size * 100 / self.quota_size) + + # Formattable into a human readable string + # + def __str__(self): + return "{} / {} ({}%)" \ + .format(utils._pretty_size(self.used_size, dec_places=1), + self.quota_config, + self.used_percent) + + # A CASCache manages a CAS repository as specified in the Remote Execution API. # # Args: # path (str): The root directory for the CAS repository +# cache_quota (int): User configured cache quota # class CASCache(): @@ -459,16 +495,6 @@ class CASCache(): except FileNotFoundError as e: raise CASCacheError("Attempt to access unavailable ref: {}".format(e)) from e - # calculate_cache_size() - # - # Return the real disk usage of the CAS cache. - # - # Returns: - # (int): The size of the cache. - # - def calculate_cache_size(self): - return utils._get_dir_size(self.casdir) - # list_refs(): # # List refs in Least Recently Modified (LRM) order. @@ -1043,6 +1069,248 @@ class CASCache(): batch.send() +class CASQuota: + def __init__(self, context): + self.cas = context.get_cascache() + self.casdir = self.cas.casdir + self._config_cache_quota = context.config_cache_quota + self._config_cache_quota_string = context.config_cache_quota_string + self._cache_size = None # The current cache size, sometimes it's an estimate + self._cache_quota = None # The cache quota + self._cache_quota_original = None # The cache quota as specified by the user, in bytes + self._cache_quota_headroom = None # The headroom in bytes before reaching the quota or full disk + self._cache_lower_threshold = None # The target cache size for a cleanup + self.available_space = None + + self._message = context.message + + self._calculate_cache_quota() + + # compute_cache_size() + # + # Computes the real artifact cache size by calling + # the abstract calculate_cache_size() method. + # + # Returns: + # (int): The size of the artifact cache. + # + def compute_cache_size(self): + old_cache_size = self._cache_size + new_cache_size = self.calculate_cache_size() + + if old_cache_size != new_cache_size: + self._cache_size = new_cache_size + + return self._cache_size + + # calculate_cache_size() + # + # Return the real disk usage of the CAS cache. + # + # Returns: + # (int): The size of the cache. + # + def calculate_cache_size(self): + return utils._get_dir_size(self.casdir) + + # get_cache_size() + # + # Fetches the cached size of the cache, this is sometimes + # an estimate and periodically adjusted to the real size + # when a cache size calculation job runs. + # + # When it is an estimate, the value is either correct, or + # it is greater than the actual cache size. + # + # Returns: + # (int) An approximation of the artifact cache size, in bytes. + # + def get_cache_size(self): + + # If we don't currently have an estimate, figure out the real cache size. + if self._cache_size is None: + stored_size = self._read_cache_size() + if stored_size is not None: + self._cache_size = stored_size + else: + self._cache_size = self.compute_cache_size() + + return self._cache_size + + # set_cache_size() + # + # Forcefully set the overall cache size. + # + # This is used to update the size in the main process after + # having calculated in a cleanup or a cache size calculation job. + # + # Args: + # cache_size (int): The size to set. + # + def set_cache_size(self, cache_size): + + assert cache_size is not None + + self._cache_size = cache_size + self._write_cache_size(self._cache_size) + + # full() + # + # Checks if the artifact cache is full, either + # because the user configured quota has been exceeded + # or because the underlying disk is almost full. + # + # Returns: + # (bool): True if the artifact cache is full + # + def full(self): + + if self.get_cache_size() > self._cache_quota: + return True + + _, volume_avail = self._get_cache_volume_size() + if volume_avail < self._cache_quota_headroom: + return True + + return False + + ################################################ + # Local Private Methods # + ################################################ + + # _read_cache_size() + # + # Reads and returns the size of the artifact cache that's stored in the + # cache's size file + # + # Returns: + # (int): The size of the artifact cache, as recorded in the file + # + def _read_cache_size(self): + size_file_path = os.path.join(self.casdir, CACHE_SIZE_FILE) + + if not os.path.exists(size_file_path): + return None + + with open(size_file_path, "r") as f: + size = f.read() + + try: + num_size = int(size) + except ValueError as e: + raise CASCacheError("Size '{}' parsed from '{}' was not an integer".format( + size, size_file_path)) from e + + return num_size + + # _write_cache_size() + # + # Writes the given size of the artifact to the cache's size file + # + # Args: + # size (int): The size of the artifact cache to record + # + def _write_cache_size(self, size): + assert isinstance(size, int) + size_file_path = os.path.join(self.casdir, CACHE_SIZE_FILE) + with utils.save_file_atomic(size_file_path, "w") as f: + f.write(str(size)) + + # _get_cache_volume_size() + # + # Get the available space and total space for the volume on + # which the artifact cache is located. + # + # Returns: + # (int): The total number of bytes on the volume + # (int): The number of available bytes on the volume + # + # NOTE: We use this stub to allow the test cases + # to override what an artifact cache thinks + # about it's disk size and available bytes. + # + def _get_cache_volume_size(self): + return utils._get_volume_size(self.casdir) + + # _calculate_cache_quota() + # + # Calculates and sets the cache quota and lower threshold based on the + # quota set in Context. + # It checks that the quota is both a valid expression, and that there is + # enough disk space to satisfy that quota + # + def _calculate_cache_quota(self): + # Headroom intended to give BuildStream a bit of leeway. + # This acts as the minimum size of cache_quota and also + # is taken from the user requested cache_quota. + # + if 'BST_TEST_SUITE' in os.environ: + self._cache_quota_headroom = 0 + else: + self._cache_quota_headroom = 2e9 + + total_size, available_space = self._get_cache_volume_size() + cache_size = self.get_cache_size() + self.available_space = available_space + + # Ensure system has enough storage for the cache_quota + # + # If cache_quota is none, set it to the maximum it could possibly be. + # + # Also check that cache_quota is at least as large as our headroom. + # + cache_quota = self._config_cache_quota + if cache_quota is None: # Infinity, set to max system storage + cache_quota = cache_size + available_space + if cache_quota < self._cache_quota_headroom: # Check minimum + raise LoadError(LoadErrorReason.INVALID_DATA, + "Invalid cache quota ({}): ".format(utils._pretty_size(cache_quota)) + + "BuildStream requires a minimum cache quota of 2G.") + elif cache_quota > total_size: + # A quota greater than the total disk size is certianly an error + raise CASCacheError("Your system does not have enough available " + + "space to support the cache quota specified.", + detail=("You have specified a quota of {quota} total disk space.\n" + + "The filesystem containing {local_cache_path} only " + + "has {total_size} total disk space.") + .format( + quota=self._config_cache_quota, + local_cache_path=self.casdir, + total_size=utils._pretty_size(total_size)), + reason='insufficient-storage-for-quota') + + elif cache_quota > cache_size + available_space: + # The quota does not fit in the available space, this is a warning + if '%' in self._config_cache_quota_string: + available = (available_space / total_size) * 100 + available = '{}% of total disk space'.format(round(available, 1)) + else: + available = utils._pretty_size(available_space) + + self._message(Message( + None, + MessageType.WARN, + "Your system does not have enough available " + + "space to support the cache quota specified.", + detail=("You have specified a quota of {quota} total disk space.\n" + + "The filesystem containing {local_cache_path} only " + + "has {available_size} available.") + .format(quota=self._config_cache_quota, + local_cache_path=self.casdir, + available_size=available))) + + # Place a slight headroom (2e9 (2GB) on the cache_quota) into + # cache_quota to try and avoid exceptions. + # + # Of course, we might still end up running out during a build + # if we end up writing more than 2G, but hey, this stuff is + # already really fuzzy. + # + self._cache_quota_original = cache_quota + self._cache_quota = cache_quota - self._cache_quota_headroom + self._cache_lower_threshold = self._cache_quota / 2 + + def _grouper(iterable, n): while True: try: diff --git a/buildstream/_context.py b/buildstream/_context.py index f7f298f3b..75edac39d 100644 --- a/buildstream/_context.py +++ b/buildstream/_context.py @@ -30,8 +30,8 @@ from . import _yaml from ._exceptions import LoadError, LoadErrorReason, BstError from ._message import Message, MessageType from ._profile import Topics, profile_start, profile_end -from ._artifactcache import ArtifactCache, ArtifactCacheUsage -from ._cas import CASCache +from ._artifactcache import ArtifactCache +from ._cas import CASCache, CASQuota, CASCacheUsage from ._workspaces import Workspaces, WorkspaceProjectCache from .plugin import _plugin_lookup from .sandbox import SandboxRemote @@ -127,6 +127,9 @@ class Context(): # Size of the artifact cache in bytes self.config_cache_quota = None + # User specified cache quota, used for display messages + self.config_cache_quota_string = None + # Whether or not to attempt to pull build trees globally self.pull_buildtrees = None @@ -151,6 +154,7 @@ class Context(): self._log_handle = None self._log_filename = None self._cascache = None + self._casquota = None self._directory = directory # load() @@ -232,7 +236,15 @@ class Context(): cache = _yaml.node_get(defaults, Mapping, 'cache') _yaml.node_validate(cache, ['quota', 'pull-buildtrees', 'cache-buildtrees']) - self.config_cache_quota = _yaml.node_get(cache, str, 'quota') + self.config_cache_quota_string = _yaml.node_get(cache, str, 'quota') + try: + self.config_cache_quota = utils._parse_size(self.config_cache_quota_string, + self.casdir) + except utils.UtilError as e: + raise LoadError(LoadErrorReason.INVALID_DATA, + "{}\nPlease specify the value in bytes or as a % of full disk space.\n" + "\nValid values are, for example: 800M 10G 1T 50%\n" + .format(str(e))) from e # Load artifact share configuration self.artifact_cache_specs = ArtifactCache.specs_from_config_node(defaults) @@ -292,15 +304,15 @@ class Context(): return self._artifactcache - # get_artifact_cache_usage() + # get_cache_usage() # # Fetches the current usage of the artifact cache # # Returns: - # (ArtifactCacheUsage): The current status + # (CASCacheUsage): The current status # - def get_artifact_cache_usage(self): - return ArtifactCacheUsage(self.artifactcache) + def get_cache_usage(self): + return CASCacheUsage(self.get_casquota()) # add_project(): # @@ -673,6 +685,11 @@ class Context(): self._cascache = CASCache(self.cachedir) return self._cascache + def get_casquota(self): + if self._casquota is None: + self._casquota = CASQuota(self) + return self._casquota + # _node_get_option_str() # diff --git a/buildstream/_frontend/status.py b/buildstream/_frontend/status.py index 70f233357..91f47221a 100644 --- a/buildstream/_frontend/status.py +++ b/buildstream/_frontend/status.py @@ -404,7 +404,7 @@ class _StatusHeader(): # # ~~~~~~ cache: 69% ~~~~~~ # - usage = self._context.get_artifact_cache_usage() + usage = self._context.get_cache_usage() usage_percent = '{}%'.format(usage.used_percent) size = 21 diff --git a/buildstream/_frontend/widget.py b/buildstream/_frontend/widget.py index 9d1467178..d1df06284 100644 --- a/buildstream/_frontend/widget.py +++ b/buildstream/_frontend/widget.py @@ -486,7 +486,7 @@ class LogLine(Widget): values["Session Start"] = starttime.strftime('%A, %d-%m-%Y at %H:%M:%S') values["Project"] = "{} ({})".format(project.name, project.directory) values["Targets"] = ", ".join([t.name for t in stream.targets]) - values["Cache Usage"] = "{}".format(context.get_artifact_cache_usage()) + values["Cache Usage"] = "{}".format(context.get_cache_usage()) text += self._format_values(values) # User configurations diff --git a/buildstream/_scheduler/jobs/cachesizejob.py b/buildstream/_scheduler/jobs/cachesizejob.py index a96b92353..5f27b7fc1 100644 --- a/buildstream/_scheduler/jobs/cachesizejob.py +++ b/buildstream/_scheduler/jobs/cachesizejob.py @@ -25,14 +25,14 @@ class CacheSizeJob(Job): self._complete_cb = complete_cb context = self._scheduler.context - self._artifacts = context.artifactcache + self._casquota = context.get_casquota() def child_process(self): - return self._artifacts.compute_cache_size() + return self._casquota.compute_cache_size() def parent_complete(self, status, result): if status == JobStatus.OK: - self._artifacts.set_cache_size(result) + self._casquota.set_cache_size(result) if self._complete_cb: self._complete_cb(status, result) diff --git a/buildstream/_scheduler/jobs/cleanupjob.py b/buildstream/_scheduler/jobs/cleanupjob.py index a1d49f339..e016d4cd7 100644 --- a/buildstream/_scheduler/jobs/cleanupjob.py +++ b/buildstream/_scheduler/jobs/cleanupjob.py @@ -25,27 +25,27 @@ class CleanupJob(Job): self._complete_cb = complete_cb context = self._scheduler.context + self._casquota = context.get_casquota() self._artifacts = context.artifactcache def child_process(self): def progress(): self.send_message('update-cache-size', - self._artifacts.get_cache_size()) + self._casquota.get_cache_size()) return self._artifacts.clean(progress) def handle_message(self, message_type, message): - # Update the cache size in the main process as we go, # this provides better feedback in the UI. if message_type == 'update-cache-size': - self._artifacts.set_cache_size(message) + self._casquota.set_cache_size(message) return True return False def parent_complete(self, status, result): if status == JobStatus.OK: - self._artifacts.set_cache_size(result) + self._casquota.set_cache_size(result) if self._complete_cb: self._complete_cb(status, result) diff --git a/tests/artifactcache/cache_size.py b/tests/artifactcache/cache_size.py index be0f1989b..dcfc13424 100644 --- a/tests/artifactcache/cache_size.py +++ b/tests/artifactcache/cache_size.py @@ -81,11 +81,11 @@ def test_quota_over_1024T(cli, tmpdir): _yaml.dump({'name': 'main'}, str(project.join("project.conf"))) volume_space_patch = mock.patch( - "buildstream._artifactcache.ArtifactCache._get_cache_volume_size", + "buildstream._cas.CASQuota._get_cache_volume_size", autospec=True, return_value=(1025 * TiB, 1025 * TiB) ) with volume_space_patch: result = cli.run(project, args=["build", "file.bst"]) - result.assert_main_error(ErrorDomain.ARTIFACT, 'insufficient-storage-for-quota') + result.assert_main_error(ErrorDomain.CAS, 'insufficient-storage-for-quota') diff --git a/tests/artifactcache/expiry.py b/tests/artifactcache/expiry.py index d67362dff..8ece6295c 100644 --- a/tests/artifactcache/expiry.py +++ b/tests/artifactcache/expiry.py @@ -341,7 +341,7 @@ def test_never_delete_required_track(cli, datafiles, tmpdir): ("200%", ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA), # Not enough space on disk even if you cleaned up - ("11K", ErrorDomain.ARTIFACT, 'insufficient-storage-for-quota'), + ("11K", ErrorDomain.CAS, 'insufficient-storage-for-quota'), # Not enough space for these caches ("7K", 'warning', 'Your system does not have enough available'), @@ -355,7 +355,7 @@ def test_invalid_cache_quota(cli, datafiles, tmpdir, quota, err_domain, err_reas cli.configure({ 'cache': { 'quota': quota, - } + }, }) # We patch how we get space information @@ -373,13 +373,13 @@ def test_invalid_cache_quota(cli, datafiles, tmpdir, quota, err_domain, err_reas total_space = 10000 volume_space_patch = mock.patch( - "buildstream._artifactcache.ArtifactCache._get_cache_volume_size", + "buildstream.utils._get_volume_size", autospec=True, return_value=(total_space, free_space), ) cache_size_patch = mock.patch( - "buildstream._artifactcache.ArtifactCache.get_cache_size", + "buildstream._cas.CASQuota.get_cache_size", autospec=True, return_value=0, ) diff --git a/tests/testutils/artifactshare.py b/tests/testutils/artifactshare.py index 6b03d8d36..8abc0fa2c 100644 --- a/tests/testutils/artifactshare.py +++ b/tests/testutils/artifactshare.py @@ -46,7 +46,6 @@ class ArtifactShare(): # in tests as a remote artifact push/pull configuration # self.repodir = os.path.join(self.directory, 'repo') - os.makedirs(self.repodir) self.cas = CASCache(self.repodir) -- cgit v1.2.1