1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
|
#
# Copyright (C) 2018 Codethink Limited
# Copyright (C) 2018 Bloomberg Finance LP
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see <http://www.gnu.org/licenses/>.
#
# Authors: Richard Maw <richard.maw@codethink.co.uk>
#
from contextlib import contextmanager
import os
import pytest
import shutil
import tempfile
from buildstream import utils
from buildstream.plugintestutils import cli_integration as cli
from tests.testutils import create_artifact_share
from tests.testutils.site import HAVE_SANDBOX
from buildstream._cas import CASCache
from buildstream._exceptions import ErrorDomain
pytestmark = pytest.mark.integration
# Project directory
DATA_DIR = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
"project",
)
# A test to capture the integration of the cachebuildtrees
# behaviour, which by default is to include the buildtree
# content of an element on caching.
@pytest.mark.integration
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
def test_cache_buildtrees(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
element_name = 'autotools/amhello.bst'
# Create artifact shares for pull & push testing
with create_artifact_share(os.path.join(str(tmpdir), 'share1')) as share1,\
create_artifact_share(os.path.join(str(tmpdir), 'share2')) as share2,\
create_artifact_share(os.path.join(str(tmpdir), 'share3')) as share3:
cli.configure({
'artifacts': {'url': share1.repo, 'push': True},
'cachedir': str(tmpdir)
})
@contextmanager
def cas_extract_buildtree(digest):
extractdir = tempfile.mkdtemp(prefix="tmp", dir=str(tmpdir))
try:
cas = CASCache(str(tmpdir))
cas.checkout(extractdir, digest)
yield os.path.join(extractdir, 'buildtree')
finally:
utils._force_rmtree(extractdir)
# Build autotools element with cache-buildtrees set via the
# cli. The artifact should be successfully pushed to the share1 remote
# and cached locally with an 'empty' buildtree digest, as it's not a
# dangling ref
result = cli.run(project=project, args=['--cache-buildtrees', 'never', 'build', element_name])
assert result.exit_code == 0
assert cli.get_element_state(project, element_name) == 'cached'
assert share1.has_artifact('test', element_name, cli.get_element_key(project, element_name))
# The extracted buildtree dir should be empty, as we set the config
# to not cache buildtrees
cache_key = cli.get_element_key(project, element_name)
elementdigest = share1.has_artifact('test', element_name, cache_key)
with cas_extract_buildtree(elementdigest) as buildtreedir:
assert os.path.isdir(buildtreedir)
assert not os.listdir(buildtreedir)
# Delete the local cached artifacts, and assert the when pulled with --pull-buildtrees
# that is was cached in share1 as expected with an empty buildtree dir
shutil.rmtree(os.path.join(str(tmpdir), 'cas'))
assert cli.get_element_state(project, element_name) != 'cached'
result = cli.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name])
assert element_name in result.get_pulled_elements()
with cas_extract_buildtree(elementdigest) as buildtreedir:
assert os.path.isdir(buildtreedir)
assert not os.listdir(buildtreedir)
shutil.rmtree(os.path.join(str(tmpdir), 'cas'))
# Assert that the default behaviour of pull to not include buildtrees on the artifact
# in share1 which was purposely cached with an empty one behaves as expected. As such the
# pulled artifact will have a dangling ref for the buildtree dir, regardless of content,
# leading to no buildtreedir being extracted
result = cli.run(project=project, args=['artifact', 'pull', element_name])
assert element_name in result.get_pulled_elements()
with cas_extract_buildtree(elementdigest) as buildtreedir:
assert not os.path.isdir(buildtreedir)
shutil.rmtree(os.path.join(str(tmpdir), 'cas'))
# Repeat building the artifacts, this time with the default behaviour of caching buildtrees,
# as such the buildtree dir should not be empty
cli.configure({
'artifacts': {'url': share2.repo, 'push': True},
'cachedir': str(tmpdir)
})
result = cli.run(project=project, args=['build', element_name])
assert result.exit_code == 0
assert cli.get_element_state(project, element_name) == 'cached'
assert share2.has_artifact('test', element_name, cli.get_element_key(project, element_name))
# Cache key will be the same however the digest hash will have changed as expected, so reconstruct paths
elementdigest = share2.has_artifact('test', element_name, cache_key)
with cas_extract_buildtree(elementdigest) as buildtreedir:
assert os.path.isdir(buildtreedir)
assert os.listdir(buildtreedir) is not None
# Delete the local cached artifacts, and assert that when pulled with --pull-buildtrees
# that it was cached in share2 as expected with a populated buildtree dir
shutil.rmtree(os.path.join(str(tmpdir), 'cas'))
assert cli.get_element_state(project, element_name) != 'cached'
result = cli.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name])
assert element_name in result.get_pulled_elements()
with cas_extract_buildtree(elementdigest) as buildtreedir:
assert os.path.isdir(buildtreedir)
assert os.listdir(buildtreedir) is not None
shutil.rmtree(os.path.join(str(tmpdir), 'cas'))
# Clarify that the user config option for cache-buildtrees works as the cli
# main option does. Point to share3 which does not have the artifacts cached to force
# a build
cli.configure({
'artifacts': {'url': share3.repo, 'push': True},
'cachedir': str(tmpdir),
'cache': {'cache-buildtrees': 'never'}
})
result = cli.run(project=project, args=['build', element_name])
assert result.exit_code == 0
assert cli.get_element_state(project, element_name) == 'cached'
cache_key = cli.get_element_key(project, element_name)
elementdigest = share3.has_artifact('test', element_name, cache_key)
with cas_extract_buildtree(elementdigest) as buildtreedir:
assert os.path.isdir(buildtreedir)
assert not os.listdir(buildtreedir)
|
