From d03bf31619f739cb85ece06eb3e94a314441e097 Mon Sep 17 00:00:00 2001 From: Benjamin Schubert Date: Fri, 10 Aug 2018 18:10:13 +0100 Subject: Mock storage space checks for tests. Fix #530 - Extract free space computation in a function for easier mocking - Mock space computation during cache quota tests - Mock cache size during cachque quota tests - Add two more tests when the configuration would require to much storage space --- buildstream/_artifactcache/artifactcache.py | 20 ++++++++++--- tests/artifactcache/expiry.py | 33 +++++++++++++++++++++- tests/testutils/mock_os.py | 44 ----------------------------- tests/utils/misc.py | 19 +++++++------ 4 files changed, 59 insertions(+), 57 deletions(-) delete mode 100644 tests/testutils/mock_os.py diff --git a/buildstream/_artifactcache/artifactcache.py b/buildstream/_artifactcache/artifactcache.py index 7771851ae..b4b8df320 100644 --- a/buildstream/_artifactcache/artifactcache.py +++ b/buildstream/_artifactcache/artifactcache.py @@ -874,9 +874,7 @@ class ArtifactCache(): "\nValid values are, for example: 800M 10G 1T 50%\n" .format(str(e))) from e - stat = os.statvfs(artifactdir_volume) - available_space = (stat.f_bsize * stat.f_bavail) - + available_space, total_size = self._get_volume_space_info_for(artifactdir_volume) cache_size = self.get_cache_size() # Ensure system has enough storage for the cache_quota @@ -893,7 +891,7 @@ class ArtifactCache(): "BuildStream requires a minimum cache quota of 2G.") elif cache_quota > cache_size + available_space: # Check maximum if '%' in self.context.config_cache_quota: - available = (available_space / (stat.f_blocks * stat.f_bsize)) * 100 + available = (available_space / total_size) * 100 available = '{}% of total disk space'.format(round(available, 1)) else: available = utils._pretty_size(available_space) @@ -919,6 +917,20 @@ class ArtifactCache(): self._cache_quota = cache_quota - headroom self._cache_lower_threshold = self._cache_quota / 2 + # _get_volume_space_info_for + # + # Get the available space and total space for the given volume + # + # Args: + # volume: volume for which to get the size + # + # Returns: + # A tuple containing first the availabe number of bytes on the requested + # volume, then the total number of bytes of the volume. + def _get_volume_space_info_for(self, volume): + stat = os.statvfs(volume) + return stat.f_bsize * stat.f_bavail, stat.f_bsize * stat.f_blocks + # _configured_remote_artifact_cache_specs(): # diff --git a/tests/artifactcache/expiry.py b/tests/artifactcache/expiry.py index 0ac86b91f..05cbe3209 100644 --- a/tests/artifactcache/expiry.py +++ b/tests/artifactcache/expiry.py @@ -18,6 +18,7 @@ # import os +from unittest import mock import pytest @@ -311,6 +312,8 @@ def test_never_delete_required_track(cli, datafiles, tmpdir): ("0", True), ("-1", False), ("pony", False), + ("7K", False), + ("70%", False), ("200%", False) ]) @pytest.mark.datafiles(DATA_DIR) @@ -324,7 +327,35 @@ def test_invalid_cache_quota(cli, datafiles, tmpdir, quota, success): } }) - res = cli.run(project=project, args=['workspace', 'list']) + # We patch how we get space information + # Ideally we would instead create a FUSE device on which we control + # everything. + # If the value is a percentage, we fix the current values to take into + # account the block size, since this is important in how we compute the size + + if quota.endswith("%"): # We set the used space at 60% of total space + stats = os.statvfs(".") + free_space = 0.6 * stats.f_bsize * stats.f_blocks + total_space = stats.f_bsize * stats.f_blocks + else: + free_space = 6000 + total_space = 10000 + + volume_space_patch = mock.patch( + "buildstream._artifactcache.artifactcache.ArtifactCache._get_volume_space_info_for", + autospec=True, + return_value=(free_space, total_space), + ) + + cache_size_patch = mock.patch( + "buildstream._artifactcache.artifactcache.ArtifactCache.get_cache_size", + autospec=True, + return_value=0, + ) + + with volume_space_patch, cache_size_patch: + res = cli.run(project=project, args=['workspace', 'list']) + if success: res.assert_success() else: diff --git a/tests/testutils/mock_os.py b/tests/testutils/mock_os.py deleted file mode 100644 index 109593b16..000000000 --- a/tests/testutils/mock_os.py +++ /dev/null @@ -1,44 +0,0 @@ -from contextlib import contextmanager -import os - - -# MockAttributeResult -# -# A class to take a dictionary of kwargs and make them accessible via -# attributes of the object. -# -class MockAttributeResult(dict): - __getattr__ = dict.get - - -# mock_statvfs(): -# -# Gets a function which mocks statvfs and returns a statvfs result with the kwargs accessible. -# -# Returns: -# func(path) -> object: object will have all the kwargs accessible via object.kwarg -# -# Example: -# statvfs = mock_statvfs(f_blocks=10) -# result = statvfs("regardless/of/path") -# assert result.f_blocks == 10 # True -def mock_statvfs(**kwargs): - def statvfs(path): - return MockAttributeResult(kwargs) - return statvfs - - -# monkey_patch() -# -# with monkey_patch("statvfs", custom_statvfs): -# assert os.statvfs == custom_statvfs # True -# assert os.statvfs == custom_statvfs # False -# -@contextmanager -def monkey_patch(to_patch, patched_func): - orig = getattr(os, to_patch) - setattr(os, to_patch, patched_func) - try: - yield - finally: - setattr(os, to_patch, orig) diff --git a/tests/utils/misc.py b/tests/utils/misc.py index 7df08aec5..4ab29ad38 100644 --- a/tests/utils/misc.py +++ b/tests/utils/misc.py @@ -1,9 +1,9 @@ +import os +from unittest import mock + from buildstream import _yaml -from ..testutils import mock_os -from ..testutils.runcli import cli -import os -import pytest +from ..testutils.runcli import cli KiB = 1024 @@ -13,7 +13,6 @@ TiB = (GiB * 1024) def test_parse_size_over_1024T(cli, tmpdir): - BLOCK_SIZE = 4096 cli.configure({ 'cache': { 'quota': 2048 * TiB @@ -23,9 +22,13 @@ def test_parse_size_over_1024T(cli, tmpdir): os.makedirs(str(project)) _yaml.dump({'name': 'main'}, str(project.join("project.conf"))) - bavail = (1025 * TiB) / BLOCK_SIZE - patched_statvfs = mock_os.mock_statvfs(f_bavail=bavail, f_bsize=BLOCK_SIZE) - with mock_os.monkey_patch("statvfs", patched_statvfs): + volume_space_patch = mock.patch( + "buildstream._artifactcache.artifactcache.ArtifactCache._get_volume_space_info_for", + autospec=True, + return_value=(1025 * TiB, 1025 * TiB) + ) + + with volume_space_patch: result = cli.run(project, args=["build", "file.bst"]) failure_msg = 'Your system does not have enough available space to support the cache quota specified.' assert failure_msg in result.stderr -- cgit v1.2.1