summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJürg Billeter <j@bitron.ch>2019-03-06 15:34:02 +0100
committerJürg Billeter <j@bitron.ch>2019-03-16 07:24:58 +0100
commit6a5940148078ea08991bf15736283fdd978258ea (patch)
tree7aa20f2e91fd79092ec5be77b14f77c79b684d46
parent3951eb56ceb810bc1279927efd836c833759823a (diff)
downloadbuildstream-6a5940148078ea08991bf15736283fdd978258ea.tar.gz
Change default of cache-buildtrees option to 'auto'
Caching buildtrees may significantly increase CAS disk usage. Change the default to cache buildtrees only when required for core functionality.
-rw-r--r--NEWS2
-rw-r--r--buildstream/_artifact.py7
-rw-r--r--buildstream/data/userconfig.yaml2
-rw-r--r--buildstream/element.py6
-rw-r--r--tests/integration/artifact.py17
-rw-r--r--tests/integration/pullbuildtrees.py1
-rw-r--r--tests/integration/shellbuildtrees.py20
7 files changed, 28 insertions, 27 deletions
diff --git a/NEWS b/NEWS
index 5da4a1322..a19182c09 100644
--- a/NEWS
+++ b/NEWS
@@ -138,7 +138,7 @@ buildstream 1.3.1
directory, this can now be extended to all or successful artifacts to save on cache
overheads. The cli main option '--cache-buildtrees' or the user configuration cache
group option 'cache-buildtrees' can be set as 'always', 'auto' or 'never', with
- the default being always. Note, as the cache-key for the artifact is independant of
+ the default being 'auto'. Note, as the cache-key for the artifact is independent of
the cached build tree input it will remain unaltered, however the availbility of the
build tree content may differ.
diff --git a/buildstream/_artifact.py b/buildstream/_artifact.py
index a2600b995..07c02f4f7 100644
--- a/buildstream/_artifact.py
+++ b/buildstream/_artifact.py
@@ -141,13 +141,6 @@ class Artifact():
filesvdir = assemblevdir.descend("files", create=True)
filesvdir.import_files(collectvdir)
- # cache_buildtrees defaults to 'always', as such the
- # default behaviour is to attempt to cache them. If only
- # caching failed artifact buildtrees, then query the build
- # result. Element types without a build-root dir will be cached
- # with an empty buildtreedir regardless of this configuration as
- # there will not be an applicable sandbox_build_dir.
-
if sandbox_build_dir:
buildtreevdir = assemblevdir.descend("buildtree", create=True)
buildtreevdir.import_files(sandbox_build_dir)
diff --git a/buildstream/data/userconfig.yaml b/buildstream/data/userconfig.yaml
index e70c41312..34fd300d1 100644
--- a/buildstream/data/userconfig.yaml
+++ b/buildstream/data/userconfig.yaml
@@ -47,7 +47,7 @@ cache:
# debugging failed builds and may break additional functionality
# in future versions.
#
- cache-buildtrees: always
+ cache-buildtrees: auto
#
diff --git a/buildstream/element.py b/buildstream/element.py
index 45bb983bb..e5bc0792e 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -1724,6 +1724,12 @@ class Element(Plugin):
cache_buildtrees = context.cache_buildtrees
build_success = buildresult[0]
+ # cache_buildtrees defaults to 'auto', only caching buildtrees
+ # when necessary, which includes failed builds.
+ # If only caching failed artifact buildtrees, then query the build
+ # result. Element types without a build-root dir will be cached
+ # with an empty buildtreedir regardless of this configuration.
+
if cache_buildtrees == 'always' or (cache_buildtrees == 'auto' and not build_success):
try:
sandbox_build_dir = sandbox_vroot.descend(
diff --git a/tests/integration/artifact.py b/tests/integration/artifact.py
index 98c6ce13f..1f6028e4b 100644
--- a/tests/integration/artifact.py
+++ b/tests/integration/artifact.py
@@ -68,11 +68,11 @@ def test_cache_buildtrees(cli, tmpdir, datafiles):
finally:
utils._force_rmtree(extractdir)
- # Build autotools element with cache-buildtrees set via the
- # cli. The artifact should be successfully pushed to the share1 remote
+ # Build autotools element with the default behavior of caching buildtrees
+ # only when necessary. The artifact should be successfully pushed to the share1 remote
# and cached locally with an 'empty' buildtree digest, as it's not a
# dangling ref
- result = cli.run(project=project, args=['--cache-buildtrees', 'never', 'build', element_name])
+ result = cli.run(project=project, args=['build', element_name])
assert result.exit_code == 0
assert cli.get_element_state(project, element_name) == 'cached'
assert share1.has_artifact('test', element_name, cli.get_element_key(project, element_name))
@@ -103,13 +103,13 @@ def test_cache_buildtrees(cli, tmpdir, datafiles):
assert not os.path.isdir(buildtreedir)
shutil.rmtree(os.path.join(str(tmpdir), 'cas'))
- # Repeat building the artifacts, this time with the default behaviour of caching buildtrees,
- # as such the buildtree dir should not be empty
+ # Repeat building the artifacts, this time with cache-buildtrees set to
+ # 'always' via the cli, as such the buildtree dir should not be empty
cli.configure({
'artifacts': {'url': share2.repo, 'push': True},
'cachedir': str(tmpdir)
})
- result = cli.run(project=project, args=['build', element_name])
+ result = cli.run(project=project, args=['--cache-buildtrees', 'always', 'build', element_name])
assert result.exit_code == 0
assert cli.get_element_state(project, element_name) == 'cached'
assert share2.has_artifact('test', element_name, cli.get_element_key(project, element_name))
@@ -137,7 +137,7 @@ def test_cache_buildtrees(cli, tmpdir, datafiles):
cli.configure({
'artifacts': {'url': share3.repo, 'push': True},
'cachedir': str(tmpdir),
- 'cache': {'cache-buildtrees': 'never'}
+ 'cache': {'cache-buildtrees': 'always'}
})
result = cli.run(project=project, args=['build', element_name])
assert result.exit_code == 0
@@ -145,4 +145,5 @@ def test_cache_buildtrees(cli, tmpdir, datafiles):
cache_key = cli.get_element_key(project, element_name)
elementdigest = share3.has_artifact('test', element_name, cache_key)
with cas_extract_buildtree(elementdigest) as buildtreedir:
- assert not os.path.isdir(buildtreedir)
+ assert os.path.isdir(buildtreedir)
+ assert os.listdir(buildtreedir)
diff --git a/tests/integration/pullbuildtrees.py b/tests/integration/pullbuildtrees.py
index efcf9cf87..6a3b92723 100644
--- a/tests/integration/pullbuildtrees.py
+++ b/tests/integration/pullbuildtrees.py
@@ -48,6 +48,7 @@ def test_pullbuildtrees(cli2, tmpdir, datafiles):
cli2.configure({
'artifacts': {'url': share1.repo, 'push': True},
'cachedir': str(tmpdir),
+ 'cache': {'cache-buildtrees': 'always'},
})
@contextmanager
diff --git a/tests/integration/shellbuildtrees.py b/tests/integration/shellbuildtrees.py
index 8c5411526..fed3c5167 100644
--- a/tests/integration/shellbuildtrees.py
+++ b/tests/integration/shellbuildtrees.py
@@ -26,7 +26,7 @@ def test_buildtree_staged(cli_integration, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
element_name = 'build-shell/buildtree.bst'
- res = cli_integration.run(project=project, args=['build', element_name])
+ res = cli_integration.run(project=project, args=['--cache-buildtrees', 'always', 'build', element_name])
res.assert_success()
res = cli_integration.run(project=project, args=[
@@ -42,7 +42,7 @@ def test_buildtree_staged_forced_true(cli_integration, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
element_name = 'build-shell/buildtree.bst'
- res = cli_integration.run(project=project, args=['build', element_name])
+ res = cli_integration.run(project=project, args=['--cache-buildtrees', 'always', 'build', element_name])
res.assert_success()
res = cli_integration.run(project=project, args=[
@@ -65,7 +65,7 @@ def test_buildtree_staged_warn_empty_cached(cli_integration, tmpdir, datafiles):
'cachedir': str(tmpdir)
})
- res = cli_integration.run(project=project, args=['--cache-buildtrees', 'never', 'build', element_name])
+ res = cli_integration.run(project=project, args=['build', element_name])
res.assert_success()
res = cli_integration.run(project=project, args=[
@@ -82,7 +82,7 @@ def test_buildtree_staged_if_available(cli_integration, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
element_name = 'build-shell/buildtree.bst'
- res = cli_integration.run(project=project, args=['build', element_name])
+ res = cli_integration.run(project=project, args=['--cache-buildtrees', 'always', 'build', element_name])
res.assert_success()
res = cli_integration.run(project=project, args=[
@@ -99,7 +99,7 @@ def test_buildtree_staged_forced_false(cli_integration, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
element_name = 'build-shell/buildtree.bst'
- res = cli_integration.run(project=project, args=['build', element_name])
+ res = cli_integration.run(project=project, args=['--cache-buildtrees', 'always', 'build', element_name])
res.assert_success()
res = cli_integration.run(project=project, args=[
@@ -154,19 +154,19 @@ def test_buildtree_from_failure_option_never(cli_integration, tmpdir, datafiles)
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-def test_buildtree_from_failure_option_auto(cli_integration, tmpdir, datafiles):
+def test_buildtree_from_failure_option_always(cli_integration, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
element_name = 'build-shell/buildtree-fail.bst'
- # build with --cache-buildtrees set to 'auto', behaviour should match
+ # build with --cache-buildtrees set to 'always', behaviour should match
# default behaviour (which is always) as the buildtree will explicitly have been
# cached with content.
cli_integration.configure({
'cachedir': str(tmpdir)
})
- res = cli_integration.run(project=project, args=['--cache-buildtrees', 'auto', 'build', element_name])
+ res = cli_integration.run(project=project, args=['--cache-buildtrees', 'always', 'build', element_name])
res.assert_main_error(ErrorDomain.STREAM, None)
res = cli_integration.run(project=project, args=[
@@ -190,7 +190,7 @@ def test_buildtree_pulled(cli, tmpdir, datafiles):
cli.configure({
'artifacts': {'url': share.repo, 'push': True}
})
- result = cli.run(project=project, args=['build', element_name])
+ result = cli.run(project=project, args=['--cache-buildtrees', 'always', 'build', element_name])
result.assert_success()
assert cli.get_element_state(project, element_name) == 'cached'
@@ -222,7 +222,7 @@ def test_buildtree_options(cli, tmpdir, datafiles):
cli.configure({
'artifacts': {'url': share.repo, 'push': True}
})
- result = cli.run(project=project, args=['build', element_name])
+ result = cli.run(project=project, args=['--cache-buildtrees', 'always', 'build', element_name])
result.assert_success()
assert cli.get_element_state(project, element_name) == 'cached'
assert share.has_artifact('test', element_name, cli.get_element_key(project, element_name))