summaryrefslogtreecommitdiff
path: root/tests/cachekey/cachekey.py
blob: 671bbb3a1a93116b69396fedd87afc3e6bc670d8 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
# Cache Key Test Instructions
#
# Adding Tests
# ~~~~~~~~~~~~
# Cache key tests are bst element files created created in such a way
# to exercise a feature which would cause the cache key for an element
# or source to be calculated differently.
#
# Adding tests is a matter to adding files to the project found in the
# 'project' subdirectory of this test case. Any files should be depended
# on by the main `target.bst` in the toplevel of the project.
#
# One test is comprised of one `<element-name>.bst` file and one
# '<element-name>.expected' file in the same directory, containing the
# expected cache key.
#
# Running the cache key test once will reveal what the new element's
# cache key should be and will also cause the depending elements to
# change cache keys.
#
#
# Updating tests
# ~~~~~~~~~~~~~~
# When a test fails it will come with a summary of which cache keys
# in the test project have mismatched.
#
# Also, in the case that the tests have changed or the artifact
# versions have changed in some way and the test needs to be
# updated; the expected cache keys for the given run are dumped to
# '<element-name>.actual' files beside the corresponding
# '<element-name>.expected' files they mismatched with, all inside
# a temporary test directory.
#
# One can now easily copy over the .actual files from a failed
# run over to the corresponding .expected source files and commit
# the result.
#

# Pylint doesn't play well with fixtures and dependency injection from pytest
# pylint: disable=redefined-outer-name

from collections import OrderedDict
import os

import pytest

from buildstream.testing.runcli import cli  # pylint: disable=unused-import
from buildstream.plugin import CoreWarnings
from buildstream import _yaml
from tests.testutils.site import HAVE_BZR, HAVE_GIT, HAVE_OSTREE, IS_LINUX, MACHINE_ARCH


##############################################
#                Some Helpers                #
##############################################

# Get whole filename in the temp project with
# the option of changing the .bst suffix to something else
#
def element_filename(project_dir, element_name, alt_suffix=None):

    if alt_suffix:

        # Just in case...
        assert element_name.endswith('.bst')

        # Chop off the 'bst' in '.bst' and add the new suffix
        element_name = element_name[:-3]
        element_name = element_name + alt_suffix

    return os.path.join(project_dir, element_name)


# Returns an OrderedDict of element names
# and their cache keys
#
def parse_output_keys(output):
    actual_keys = OrderedDict()
    lines = output.splitlines()
    for line in lines:
        split = line.split("::")
        name = split[0]
        key = split[1]
        actual_keys[name] = key

    return actual_keys


# Returns an OrderedDict of element names
# and their cache keys
#
def load_expected_keys(project_dir, actual_keys, raise_error=True):

    expected_keys = OrderedDict()
    for element_name in actual_keys:
        expected = element_filename(project_dir, element_name, 'expected')
        try:
            with open(expected, 'r') as f:
                expected_key = f.read()
                expected_key = expected_key.strip()
        except FileNotFoundError:
            expected_key = None
            if raise_error:
                raise Exception("Cache key test needs update, " +
                                "expected file {} not found.\n\n".format(expected) +
                                "Use tests/cachekey/update.py to automatically " +
                                "update this test case")

        expected_keys[element_name] = expected_key

    return expected_keys


def assert_cache_keys(project_dir, output):

    # Read in the expected keys from the cache key test directory
    # and parse the actual keys from the `bst show` output
    #
    actual_keys = parse_output_keys(output)
    expected_keys = load_expected_keys(project_dir, actual_keys)
    mismatches = []

    for element_name in actual_keys:
        if actual_keys[element_name] != expected_keys[element_name]:
            mismatches.append(element_name)

    if mismatches:
        info = ""
        for element_name in mismatches:
            info += "  Element: {}\n".format(element_name) + \
                    "    Expected: {}\n".format(expected_keys[element_name]) + \
                    "    Actual: {}\n".format(actual_keys[element_name])

        raise AssertionError("Cache key mismatches occurred:\n{}\n".format(info) +
                             "Use tests/cachekey/update.py to automatically " +
                             "update this test case")


##############################################
#             Test Entry Point               #
##############################################

# Project directory
DATA_DIR = os.path.join(
    os.path.dirname(os.path.realpath(__file__)),
    "project",
)


# The cache key test uses a project which exercises all plugins,
# so we cant run it at all if we dont have them installed.
#
@pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
                    reason='Cache keys depend on architecture')
@pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
@pytest.mark.skipif(HAVE_BZR is False, reason="bzr is not available")
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
@pytest.mark.skipif(HAVE_OSTREE is False, reason="ostree is not available")
@pytest.mark.datafiles(DATA_DIR)
def test_cache_key(datafiles, cli):
    project = str(datafiles)

    # Workaround bug in recent versions of setuptools: newer
    # versions of setuptools fail to preserve symbolic links
    # when creating a source distribution, causing this test
    # to fail from a dist tarball.
    goodbye_link = os.path.join(project, 'files', 'local',
                                'usr', 'bin', 'goodbye')
    os.unlink(goodbye_link)
    os.symlink('hello', goodbye_link)

    result = cli.run(project=project, silent=True, args=[
        'show',
        '--format', '%{name}::%{full-key}',
        'target.bst'
    ])
    result.assert_success()
    assert_cache_keys(project, result.output)


@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.parametrize("first_warnings, second_warnings, identical_keys", [
    [[], [], True],
    [[], [CoreWarnings.REF_NOT_IN_TRACK], False],
    [[CoreWarnings.REF_NOT_IN_TRACK], [], False],
    [[CoreWarnings.REF_NOT_IN_TRACK], [CoreWarnings.REF_NOT_IN_TRACK], True],
    [[CoreWarnings.REF_NOT_IN_TRACK, CoreWarnings.OVERLAPS],
     [CoreWarnings.OVERLAPS, CoreWarnings.REF_NOT_IN_TRACK], True],
])
def test_cache_key_fatal_warnings(cli, tmpdir, first_warnings, second_warnings, identical_keys):

    # Builds project, Runs bst show, gathers cache keys
    def run_get_cache_key(project_name, warnings):
        config = {
            'name': project_name,
            'element-path': 'elements',
            'fatal-warnings': warnings
        }

        project_dir = tmpdir.mkdir(project_name)
        project_config_file = str(project_dir.join('project.conf'))
        _yaml.dump(_yaml.node_sanitize(config), filename=project_config_file)

        elem_dir = project_dir.mkdir('elements')
        element_file = str(elem_dir.join('stack.bst'))
        _yaml.dump({'kind': 'stack'}, filename=element_file)

        result = cli.run(project=str(project_dir), args=[
            'show',
            '--format', '%{name}::%{full-key}',
            'stack.bst'
        ])
        return result.output

    # Returns true if all keys are identical
    def compare_cache_keys(first_keys, second_keys):
        return not any((x != y for x, y in zip(first_keys, second_keys)))

    first_keys = run_get_cache_key("first", first_warnings)
    second_keys = run_get_cache_key("second", second_warnings)

    assert compare_cache_keys(first_keys, second_keys) == identical_keys


@pytest.mark.datafiles(DATA_DIR)
def test_keys_stable_over_targets(cli, datafiles):
    root_element = 'elements/key-stability/top-level.bst'
    target1 = 'elements/key-stability/t1.bst'
    target2 = 'elements/key-stability/t2.bst'

    project = str(datafiles)
    full_graph_result = cli.run(project=project, args=[
        'show',
        '--format', '%{name}::%{full-key}',
        root_element
    ])
    full_graph_result.assert_success()
    all_cache_keys = parse_output_keys(full_graph_result.output)

    ordering1_result = cli.run(project=project, args=[
        'show',
        '--format', '%{name}::%{full-key}',
        target1,
        target2
    ])
    ordering1_result.assert_success()
    ordering1_cache_keys = parse_output_keys(ordering1_result.output)

    ordering2_result = cli.run(project=project, args=[
        'show',
        '--format', '%{name}::%{full-key}',
        target2,
        target1
    ])
    ordering2_result.assert_success()
    ordering2_cache_keys = parse_output_keys(ordering2_result.output)

    elements = ordering1_cache_keys.keys()

    assert {key: ordering2_cache_keys[key] for key in elements} == ordering1_cache_keys
    assert {key: all_cache_keys[key] for key in elements} == ordering1_cache_keys