path: root/tests/internals/yaml.py

import os
import pytest
import tempfile
from collections.abc import Mapping

from buildstream import _yaml
from buildstream._exceptions import LoadError, LoadErrorReason
from buildstream._context import Context
from buildstream._yamlcache import YamlCache

DATA_DIR = os.path.join(
    os.path.dirname(os.path.realpath(__file__)),
    'yaml',
)


@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_load_yaml(datafiles):

    filename = os.path.join(datafiles.dirname,
                            datafiles.basename,
                            'basics.yaml')

    loaded = _yaml.load(filename)
    assert(loaded.get('kind') == 'pony')


def assert_provenance(filename, line, col, node, key=None, indices=[]):
    provenance = _yaml.node_get_provenance(node, key=key, indices=indices)

    if key:
        if indices:
            assert(isinstance(provenance, _yaml.ElementProvenance))
        else:
            assert(isinstance(provenance, _yaml.MemberProvenance))
    else:
        assert(isinstance(provenance, _yaml.DictProvenance))

    assert(provenance.filename.shortname == filename)
    assert(provenance.line == line)
    assert(provenance.col == col)


@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_basic_provenance(datafiles):

    filename = os.path.join(datafiles.dirname,
                            datafiles.basename,
                            'basics.yaml')

    loaded = _yaml.load(filename)
    assert(loaded.get('kind') == 'pony')

    assert_provenance(filename, 1, 0, loaded)


@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_member_provenance(datafiles):

    filename = os.path.join(datafiles.dirname,
                            datafiles.basename,
                            'basics.yaml')

    loaded = _yaml.load(filename)
    assert(loaded.get('kind') == 'pony')
    assert_provenance(filename, 2, 13, loaded, 'description')


@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_element_provenance(datafiles):

    filename = os.path.join(datafiles.dirname,
                            datafiles.basename,
                            'basics.yaml')

    loaded = _yaml.load(filename)
    assert(loaded.get('kind') == 'pony')
    assert_provenance(filename, 5, 2, loaded, 'moods', [1])


@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_node_validate(datafiles):

    valid = os.path.join(datafiles.dirname,
                         datafiles.basename,
                         'basics.yaml')
    invalid = os.path.join(datafiles.dirname,
                           datafiles.basename,
                           'invalid.yaml')

    base = _yaml.load(valid)

    _yaml.node_validate(base, ['kind', 'description', 'moods', 'children', 'extra'])

    base = _yaml.load(invalid)

    with pytest.raises(LoadError) as exc:
        _yaml.node_validate(base, ['kind', 'description', 'moods', 'children', 'extra'])

    assert (exc.value.reason == LoadErrorReason.INVALID_DATA)


@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_node_get(datafiles):

    filename = os.path.join(datafiles.dirname,
                            datafiles.basename,
                            'basics.yaml')

    base = _yaml.load(filename)
    assert(base.get('kind') == 'pony')

    children = _yaml.node_get(base, list, 'children')
    assert(isinstance(children, list))
    assert(len(children) == 7)

    child = _yaml.node_get(base, Mapping, 'children', indices=[6])
    assert_provenance(filename, 20, 8, child, 'mood')

    extra = _yaml.node_get(base, Mapping, 'extra')
    with pytest.raises(LoadError) as exc:
        _yaml.node_get(extra, Mapping, 'old')

    assert (exc.value.reason == LoadErrorReason.INVALID_DATA)


# Really this is testing _yaml.node_copy(), we want to
# be sure that compositing values still preserves the original
# values in the copied dict.
#
@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_composite_preserve_originals(datafiles):

    filename = os.path.join(datafiles.dirname,
                            datafiles.basename,
                            'basics.yaml')
    overlayfile = os.path.join(datafiles.dirname,
                               datafiles.basename,
                               'composite.yaml')

    base = _yaml.load(filename)
    overlay = _yaml.load(overlayfile)
    base_copy = _yaml.node_copy(base)
    _yaml.composite_dict(base_copy, overlay)

    copy_extra = _yaml.node_get(base_copy, Mapping, 'extra')
    orig_extra = _yaml.node_get(base, Mapping, 'extra')

    # Test that the node copy has the overridden value...
    assert(_yaml.node_get(copy_extra, str, 'old') == 'override')

    # But the original node is not effected by the override.
    assert(_yaml.node_get(orig_extra, str, 'old') == 'new')


def load_yaml_file(filename, *, cache_path, shortname=None, from_cache='raw'):

    _, temppath = tempfile.mkstemp(dir=os.path.join(cache_path.dirname, cache_path.basename), text=True)
    context = Context()

    with YamlCache.open(context, temppath) as yc:
        if from_cache == 'raw':
            return _yaml.load(filename, shortname)
        elif from_cache == 'cached':
            _yaml.load(filename, shortname, yaml_cache=yc)
            return _yaml.load(filename, shortname, yaml_cache=yc)
        else:
            assert False


# Tests for list composition
#
# Each test composits a filename on top of basics.yaml, and tests
# the toplevel children list at the specified index
#
# Parameters:
#    filename: The file to composite on top of basics.yaml
#    index: The index in the children list
#    length: The expected length of the children list
#    mood: The expected value of the mood attribute of the dictionary found at index in children
#    prov_file: The expected provenance filename of "mood"
#    prov_line: The expected provenance line of "mood"
#    prov_col: The expected provenance column of "mood"
#
@pytest.mark.datafiles(os.path.join(DATA_DIR))
@pytest.mark.parametrize('caching', [('raw'), ('cached')])
@pytest.mark.parametrize("filename,index,length,mood,prov_file,prov_line,prov_col", [

    # Test results of compositing with the (<) prepend directive
    ('listprepend.yaml', 0, 9, 'prepended1', 'listprepend.yaml', 5, 10),
    ('listprepend.yaml', 1, 9, 'prepended2', 'listprepend.yaml', 7, 10),
    ('listprepend.yaml', 2, 9, 'silly', 'basics.yaml', 8, 8),
    ('listprepend.yaml', 8, 9, 'sleepy', 'basics.yaml', 20, 8),

    # Test results of compositing with the (>) append directive
    ('listappend.yaml', 7, 9, 'appended1', 'listappend.yaml', 5, 10),
    ('listappend.yaml', 8, 9, 'appended2', 'listappend.yaml', 7, 10),
    ('listappend.yaml', 0, 9, 'silly', 'basics.yaml', 8, 8),
    ('listappend.yaml', 6, 9, 'sleepy', 'basics.yaml', 20, 8),

    # Test results of compositing with both (<) and (>) directives
    ('listappendprepend.yaml', 0, 11, 'prepended1', 'listappendprepend.yaml', 5, 10),
    ('listappendprepend.yaml', 1, 11, 'prepended2', 'listappendprepend.yaml', 7, 10),
    ('listappendprepend.yaml', 2, 11, 'silly', 'basics.yaml', 8, 8),
    ('listappendprepend.yaml', 8, 11, 'sleepy', 'basics.yaml', 20, 8),
    ('listappendprepend.yaml', 9, 11, 'appended1', 'listappendprepend.yaml', 10, 10),
    ('listappendprepend.yaml', 10, 11, 'appended2', 'listappendprepend.yaml', 12, 10),

    # Test results of compositing with the (=) overwrite directive
    ('listoverwrite.yaml', 0, 2, 'overwrite1', 'listoverwrite.yaml', 5, 10),
    ('listoverwrite.yaml', 1, 2, 'overwrite2', 'listoverwrite.yaml', 7, 10),

    # Test results of compositing without any directive, implicitly overwriting
    ('implicitoverwrite.yaml', 0, 2, 'overwrite1', 'implicitoverwrite.yaml', 4, 8),
    ('implicitoverwrite.yaml', 1, 2, 'overwrite2', 'implicitoverwrite.yaml', 6, 8),
])
def test_list_composition(datafiles, filename, tmpdir,
                          index, length, mood,
                          prov_file, prov_line, prov_col, caching):
    base_file = os.path.join(datafiles.dirname, datafiles.basename, 'basics.yaml')
    overlay_file = os.path.join(datafiles.dirname, datafiles.basename, filename)

    base = load_yaml_file(base_file, cache_path=tmpdir, shortname='basics.yaml', from_cache=caching)
    overlay = load_yaml_file(overlay_file, cache_path=tmpdir, shortname=filename, from_cache=caching)

    _yaml.composite_dict(base, overlay)

    children = _yaml.node_get(base, list, 'children')
    assert len(children) == length
    child = children[index]

    assert child['mood'] == mood
    assert_provenance(prov_file, prov_line, prov_col, child, 'mood')


# Test that overwriting a list with an empty list works as expected.
@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_list_deletion(datafiles):
    base = os.path.join(datafiles.dirname, datafiles.basename, 'basics.yaml')
    overlay = os.path.join(datafiles.dirname, datafiles.basename, 'listoverwriteempty.yaml')

    base = _yaml.load(base, shortname='basics.yaml')
    overlay = _yaml.load(overlay, shortname='listoverwriteempty.yaml')
    _yaml.composite_dict(base, overlay)

    children = _yaml.node_get(base, list, 'children')
    assert len(children) == 0


# Tests for deep list composition
#
# Same as test_list_composition(), but adds an additional file
# in between so that lists are composited twice.
#
# This test will to two iterations for each parameter
# specification, expecting the same results
#
#   First iteration:
#      composited = basics.yaml & filename1
#      composited = composited & filename2
#
#   Second iteration:
#      composited = filename1 & filename2
#      composited = basics.yaml & composited
#
# Parameters:
#    filename1: The file to composite on top of basics.yaml
#    filename2: The file to composite on top of filename1
#    index: The index in the children list
#    length: The expected length of the children list
#    mood: The expected value of the mood attribute of the dictionary found at index in children
#    prov_file: The expected provenance filename of "mood"
#    prov_line: The expected provenance line of "mood"
#    prov_col: The expected provenance column of "mood"
#
@pytest.mark.datafiles(os.path.join(DATA_DIR))
@pytest.mark.parametrize('caching', [('raw'), ('cached')])
@pytest.mark.parametrize("filename1,filename2,index,length,mood,prov_file,prov_line,prov_col", [

    # Test results of compositing literal list with (>) and then (<)
    ('listprepend.yaml', 'listappend.yaml', 0, 11, 'prepended1', 'listprepend.yaml', 5, 10),
    ('listprepend.yaml', 'listappend.yaml', 1, 11, 'prepended2', 'listprepend.yaml', 7, 10),
    ('listprepend.yaml', 'listappend.yaml', 2, 11, 'silly', 'basics.yaml', 8, 8),
    ('listprepend.yaml', 'listappend.yaml', 8, 11, 'sleepy', 'basics.yaml', 20, 8),
    ('listprepend.yaml', 'listappend.yaml', 9, 11, 'appended1', 'listappend.yaml', 5, 10),
    ('listprepend.yaml', 'listappend.yaml', 10, 11, 'appended2', 'listappend.yaml', 7, 10),

    # Test results of compositing literal list with (<) and then (>)
    ('listappend.yaml', 'listprepend.yaml', 0, 11, 'prepended1', 'listprepend.yaml', 5, 10),
    ('listappend.yaml', 'listprepend.yaml', 1, 11, 'prepended2', 'listprepend.yaml', 7, 10),
    ('listappend.yaml', 'listprepend.yaml', 2, 11, 'silly', 'basics.yaml', 8, 8),
    ('listappend.yaml', 'listprepend.yaml', 8, 11, 'sleepy', 'basics.yaml', 20, 8),
    ('listappend.yaml', 'listprepend.yaml', 9, 11, 'appended1', 'listappend.yaml', 5, 10),
    ('listappend.yaml', 'listprepend.yaml', 10, 11, 'appended2', 'listappend.yaml', 7, 10),

    # Test results of compositing literal list with (>) and then (>)
    ('listappend.yaml', 'secondappend.yaml', 0, 11, 'silly', 'basics.yaml', 8, 8),
    ('listappend.yaml', 'secondappend.yaml', 6, 11, 'sleepy', 'basics.yaml', 20, 8),
    ('listappend.yaml', 'secondappend.yaml', 7, 11, 'appended1', 'listappend.yaml', 5, 10),
    ('listappend.yaml', 'secondappend.yaml', 8, 11, 'appended2', 'listappend.yaml', 7, 10),
    ('listappend.yaml', 'secondappend.yaml', 9, 11, 'secondappend1', 'secondappend.yaml', 5, 10),
    ('listappend.yaml', 'secondappend.yaml', 10, 11, 'secondappend2', 'secondappend.yaml', 7, 10),

    # Test results of compositing literal list with (>) and then (>)
    ('listprepend.yaml', 'secondprepend.yaml', 0, 11, 'secondprepend1', 'secondprepend.yaml', 5, 10),
    ('listprepend.yaml', 'secondprepend.yaml', 1, 11, 'secondprepend2', 'secondprepend.yaml', 7, 10),
    ('listprepend.yaml', 'secondprepend.yaml', 2, 11, 'prepended1', 'listprepend.yaml', 5, 10),
    ('listprepend.yaml', 'secondprepend.yaml', 3, 11, 'prepended2', 'listprepend.yaml', 7, 10),
    ('listprepend.yaml', 'secondprepend.yaml', 4, 11, 'silly', 'basics.yaml', 8, 8),
    ('listprepend.yaml', 'secondprepend.yaml', 10, 11, 'sleepy', 'basics.yaml', 20, 8),

    # Test results of compositing literal list with (>) or (<) and then another literal list
    ('listappend.yaml', 'implicitoverwrite.yaml', 0, 2, 'overwrite1', 'implicitoverwrite.yaml', 4, 8),
    ('listappend.yaml', 'implicitoverwrite.yaml', 1, 2, 'overwrite2', 'implicitoverwrite.yaml', 6, 8),
    ('listprepend.yaml', 'implicitoverwrite.yaml', 0, 2, 'overwrite1', 'implicitoverwrite.yaml', 4, 8),
    ('listprepend.yaml', 'implicitoverwrite.yaml', 1, 2, 'overwrite2', 'implicitoverwrite.yaml', 6, 8),

    # Test results of compositing literal list with (>) or (<) and then an explicit (=) overwrite
    ('listappend.yaml', 'listoverwrite.yaml', 0, 2, 'overwrite1', 'listoverwrite.yaml', 5, 10),
    ('listappend.yaml', 'listoverwrite.yaml', 1, 2, 'overwrite2', 'listoverwrite.yaml', 7, 10),
    ('listprepend.yaml', 'listoverwrite.yaml', 0, 2, 'overwrite1', 'listoverwrite.yaml', 5, 10),
    ('listprepend.yaml', 'listoverwrite.yaml', 1, 2, 'overwrite2', 'listoverwrite.yaml', 7, 10),

    # Test results of compositing literal list an explicit overwrite (=) and then with (>) or (<)
    ('listoverwrite.yaml', 'listappend.yaml', 0, 4, 'overwrite1', 'listoverwrite.yaml', 5, 10),
    ('listoverwrite.yaml', 'listappend.yaml', 1, 4, 'overwrite2', 'listoverwrite.yaml', 7, 10),
    ('listoverwrite.yaml', 'listappend.yaml', 2, 4, 'appended1', 'listappend.yaml', 5, 10),
    ('listoverwrite.yaml', 'listappend.yaml', 3, 4, 'appended2', 'listappend.yaml', 7, 10),
    ('listoverwrite.yaml', 'listprepend.yaml', 0, 4, 'prepended1', 'listprepend.yaml', 5, 10),
    ('listoverwrite.yaml', 'listprepend.yaml', 1, 4, 'prepended2', 'listprepend.yaml', 7, 10),
    ('listoverwrite.yaml', 'listprepend.yaml', 2, 4, 'overwrite1', 'listoverwrite.yaml', 5, 10),
    ('listoverwrite.yaml', 'listprepend.yaml', 3, 4, 'overwrite2', 'listoverwrite.yaml', 7, 10),
])
def test_list_composition_twice(datafiles, tmpdir, filename1, filename2,
                                index, length, mood,
                                prov_file, prov_line, prov_col, caching):
    file_base = os.path.join(datafiles.dirname, datafiles.basename, 'basics.yaml')
    file1 = os.path.join(datafiles.dirname, datafiles.basename, filename1)
    file2 = os.path.join(datafiles.dirname, datafiles.basename, filename2)

    #####################
    # Round 1 - Fight !
    #####################
    base = load_yaml_file(file_base, cache_path=tmpdir, shortname='basics.yaml', from_cache=caching)
    overlay1 = load_yaml_file(file1, cache_path=tmpdir, shortname=filename1, from_cache=caching)
    overlay2 = load_yaml_file(file2, cache_path=tmpdir, shortname=filename2, from_cache=caching)

    _yaml.composite_dict(base, overlay1)
    _yaml.composite_dict(base, overlay2)

    children = _yaml.node_get(base, list, 'children')
    assert len(children) == length
    child = children[index]

    assert child['mood'] == mood
    assert_provenance(prov_file, prov_line, prov_col, child, 'mood')

    #####################
    # Round 2 - Fight !
    #####################
    base = load_yaml_file(file_base, cache_path=tmpdir, shortname='basics.yaml', from_cache=caching)
    overlay1 = load_yaml_file(file1, cache_path=tmpdir, shortname=filename1, from_cache=caching)
    overlay2 = load_yaml_file(file2, cache_path=tmpdir, shortname=filename2, from_cache=caching)

    _yaml.composite_dict(overlay1, overlay2)
    _yaml.composite_dict(base, overlay1)

    children = _yaml.node_get(base, list, 'children')
    assert len(children) == length
    child = children[index]

    assert child['mood'] == mood
    assert_provenance(prov_file, prov_line, prov_col, child, 'mood')


@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_convert_value_to_string(datafiles):
    conf_file = os.path.join(datafiles.dirname,
                             datafiles.basename,
                             'convert_value_to_str.yaml')

    # Run file through yaml to convert it
    test_dict = _yaml.load(conf_file)

    user_config = _yaml.node_get(test_dict, str, "Test1")
    assert isinstance(user_config, str)
    assert user_config == "1_23_4"

    user_config = _yaml.node_get(test_dict, str, "Test2")
    assert isinstance(user_config, str)
    assert user_config == "1.23.4"

    user_config = _yaml.node_get(test_dict, str, "Test3")
    assert isinstance(user_config, str)
    assert user_config == "1.20"

    user_config = _yaml.node_get(test_dict, str, "Test4")
    assert isinstance(user_config, str)
    assert user_config == "OneTwoThree"


@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_value_doesnt_match_expected(datafiles):
    conf_file = os.path.join(datafiles.dirname,
                             datafiles.basename,
                             'convert_value_to_str.yaml')

    # Run file through yaml to convert it
    test_dict = _yaml.load(conf_file)

    with pytest.raises(LoadError) as exc:
        _yaml.node_get(test_dict, int, "Test4")
    assert exc.value.reason == LoadErrorReason.INVALID_DATA