1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
|
import os
import pytest
from buildstream import _yaml
from buildstream._exceptions import ErrorDomain, LoadErrorReason
from tests.testutils import cli, create_element_size
DATA_DIR = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
"expiry"
)
# Ensure that the cache successfully removes an old artifact if we do
# not have enough space left.
@pytest.mark.datafiles(DATA_DIR)
def test_artifact_expires(cli, datafiles, tmpdir):
project = os.path.join(datafiles.dirname, datafiles.basename)
element_path = 'elements'
cache_location = os.path.join(project, 'cache', 'artifacts', 'ostree')
checkout = os.path.join(project, 'checkout')
cli.configure({
'cache': {
'quota': 10000000,
}
})
# Create an element that uses almost the entire cache (an empty
# ostree cache starts at about ~10KiB, so we need a bit of a
# buffer)
create_element_size('target.bst', project, element_path, [], 6000000)
res = cli.run(project=project, args=['build', 'target.bst'])
res.assert_success()
assert cli.get_element_state(project, 'target.bst') == 'cached'
# Our cache should now be almost full. Let's create another
# artifact and see if we can cause buildstream to delete the old
# one.
create_element_size('target2.bst', project, element_path, [], 6000000)
res = cli.run(project=project, args=['build', 'target2.bst'])
res.assert_success()
# Check that the correct element remains in the cache
assert cli.get_element_state(project, 'target.bst') != 'cached'
assert cli.get_element_state(project, 'target2.bst') == 'cached'
# Ensure that we don't end up deleting the whole cache (or worse) if
# we try to store an artifact that is too large to fit in the quota.
@pytest.mark.parametrize('size', [
# Test an artifact that is obviously too large
(500000),
# Test an artifact that might be too large due to slight overhead
# of storing stuff in ostree
(399999)
])
@pytest.mark.datafiles(DATA_DIR)
def test_artifact_too_large(cli, datafiles, tmpdir, size):
project = os.path.join(datafiles.dirname, datafiles.basename)
element_path = 'elements'
cli.configure({
'cache': {
'quota': 400000
}
})
# Create an element whose artifact is too large
create_element_size('target.bst', project, element_path, [], size)
res = cli.run(project=project, args=['build', 'target.bst'])
res.assert_main_error(ErrorDomain.STREAM, None)
@pytest.mark.datafiles(DATA_DIR)
def test_expiry_order(cli, datafiles, tmpdir):
project = os.path.join(datafiles.dirname, datafiles.basename)
element_path = 'elements'
cache_location = os.path.join(project, 'cache', 'artifacts', 'ostree')
checkout = os.path.join(project, 'workspace')
cli.configure({
'cache': {
'quota': 9000000
}
})
# Create an artifact
create_element_size('dep.bst', project, element_path, [], 2000000)
res = cli.run(project=project, args=['build', 'dep.bst'])
res.assert_success()
# Create another artifact
create_element_size('unrelated.bst', project, element_path, [], 2000000)
res = cli.run(project=project, args=['build', 'unrelated.bst'])
res.assert_success()
# And build something else
create_element_size('target.bst', project, element_path, [], 2000000)
res = cli.run(project=project, args=['build', 'target.bst'])
res.assert_success()
create_element_size('target2.bst', project, element_path, [], 2000000)
res = cli.run(project=project, args=['build', 'target2.bst'])
res.assert_success()
# Now extract dep.bst
res = cli.run(project=project, args=['checkout', 'dep.bst', checkout])
res.assert_success()
# Finally, build something that will cause the cache to overflow
create_element_size('expire.bst', project, element_path, [], 2000000)
res = cli.run(project=project, args=['build', 'expire.bst'])
res.assert_success()
# While dep.bst was the first element to be created, it should not
# have been removed.
# Note that buildstream will reduce the cache to 50% of the
# original size - we therefore remove multiple elements.
assert (tuple(cli.get_element_state(project, element) for element in
('unrelated.bst', 'target.bst', 'target2.bst', 'dep.bst', 'expire.bst')) ==
('buildable', 'buildable', 'buildable', 'cached', 'cached', ))
# Ensure that we don't accidentally remove an artifact from something
# in the current build pipeline, because that would be embarassing,
# wouldn't it?
@pytest.mark.datafiles(DATA_DIR)
def test_keep_dependencies(cli, datafiles, tmpdir):
project = os.path.join(datafiles.dirname, datafiles.basename)
element_path = 'elements'
cache_location = os.path.join(project, 'cache', 'artifacts', 'ostree')
cli.configure({
'cache': {
'quota': 10000000
}
})
# Create a pretty big dependency
create_element_size('dependency.bst', project, element_path, [], 5000000)
res = cli.run(project=project, args=['build', 'dependency.bst'])
res.assert_success()
# Now create some other unrelated artifact
create_element_size('unrelated.bst', project, element_path, [], 4000000)
res = cli.run(project=project, args=['build', 'unrelated.bst'])
res.assert_success()
# Check that the correct element remains in the cache
assert cli.get_element_state(project, 'dependency.bst') == 'cached'
assert cli.get_element_state(project, 'unrelated.bst') == 'cached'
# We try to build an element which depends on the LRU artifact,
# and could therefore fail if we didn't make sure dependencies
# aren't removed.
#
# Since some artifact caches may implement weak cache keys by
# duplicating artifacts (bad!) we need to make this equal in size
# or smaller than half the size of its dependencies.
#
create_element_size('target.bst', project,
element_path, ['dependency.bst'], 2000000)
res = cli.run(project=project, args=['build', 'target.bst'])
res.assert_success()
assert cli.get_element_state(project, 'unrelated.bst') != 'cached'
assert cli.get_element_state(project, 'dependency.bst') == 'cached'
assert cli.get_element_state(project, 'target.bst') == 'cached'
# Assert that we never delete a dependency required for a build tree
@pytest.mark.datafiles(DATA_DIR)
def test_never_delete_dependencies(cli, datafiles, tmpdir):
project = os.path.join(datafiles.dirname, datafiles.basename)
element_path = 'elements'
cli.configure({
'cache': {
'quota': 10000000
}
})
# Create a build tree
create_element_size('dependency.bst', project,
element_path, [], 8000000)
create_element_size('related.bst', project,
element_path, ['dependency.bst'], 8000000)
create_element_size('target.bst', project,
element_path, ['related.bst'], 8000000)
create_element_size('target2.bst', project,
element_path, ['target.bst'], 8000000)
# We try to build this pipeline, but it's too big for the
# cache. Since all elements are required, the build should fail.
res = cli.run(project=project, args=['build', 'target2.bst'])
res.assert_main_error(ErrorDomain.STREAM, None)
assert cli.get_element_state(project, 'dependency.bst') == 'cached'
# This is *technically* above the cache limit. BuildStream accepts
# some fuzziness, since it's hard to assert that we don't create
# an artifact larger than the cache quota. We would have to remove
# the artifact after-the-fact, but since it is required for the
# current build and nothing broke yet, it's nicer to keep it
# around.
#
# This scenario is quite unlikely, and the cache overflow will be
# resolved if the user does something about it anyway.
#
assert cli.get_element_state(project, 'related.bst') == 'cached'
assert cli.get_element_state(project, 'target.bst') != 'cached'
assert cli.get_element_state(project, 'target2.bst') != 'cached'
# Ensure that only valid cache quotas make it through the loading
# process.
@pytest.mark.parametrize("quota,success", [
("1", True),
("1K", True),
("50%", True),
("infinity", True),
("0", True),
("-1", False),
("pony", False),
("200%", False)
])
@pytest.mark.datafiles(DATA_DIR)
def test_invalid_cache_quota(cli, datafiles, tmpdir, quota, success):
project = os.path.join(datafiles.dirname, datafiles.basename)
element_path = os.path.join(project, 'elements')
cli.configure({
'cache': {
'quota': quota,
}
})
res = cli.run(project=project, args=['workspace', 'list'])
if success:
res.assert_success()
else:
res.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
|
